EOX GitLab Instance
Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Menu
Open sidebar
View Server 2
scheduler
Commits
2921c26c
Commit
2921c26c
authored
Nov 10, 2021
by
Bernhard Mallinger
Browse files
Reformat code with black
parent
788fb0ed
Changes
3
Hide whitespace changes
Inline
Side-by-side
scheduler/cli.py
View file @
2921c26c
...
...
@@ -8,31 +8,30 @@ import jsonschema
from
.daemon
import
run_daemon
from
.config
import
load_config
def
setup_logging
(
debug
=
False
):
logging
.
config
.
dictConfig
({
'version'
:
1
,
'disable_existing_loggers'
:
False
,
'formatters'
:
{
'brief'
:
{
'format'
:
'%(levelname)s %(name)s: %(message)s'
}
},
'handlers'
:
{
'console'
:
{
'class'
:
'logging.StreamHandler'
,
'level'
:
'DEBUG'
if
debug
else
'INFO'
,
'formatter'
:
'brief'
,
}
},
'root'
:
{
'handlers'
:
[
'console'
],
'level'
:
'DEBUG'
if
debug
else
'INFO'
,
logging
.
config
.
dictConfig
(
{
"version"
:
1
,
"disable_existing_loggers"
:
False
,
"formatters"
:
{
"brief"
:
{
"format"
:
"%(levelname)s %(name)s: %(message)s"
}},
"handlers"
:
{
"console"
:
{
"class"
:
"logging.StreamHandler"
,
"level"
:
"DEBUG"
if
debug
else
"INFO"
,
"formatter"
:
"brief"
,
}
},
"root"
:
{
"handlers"
:
[
"console"
],
"level"
:
"DEBUG"
if
debug
else
"INFO"
,
},
}
}
)
)
def
validate_config
(
config
):
with
open
(
join
(
dirname
(
__file__
),
'
config-schema.yaml
'
))
as
f
:
with
open
(
join
(
dirname
(
__file__
),
"
config-schema.yaml
"
))
as
f
:
schema
=
yaml
.
load
(
f
)
jsonschema
.
validate
(
config
,
schema
)
...
...
@@ -43,12 +42,12 @@ def cli():
pass
@
cli
.
command
(
help
=
'
Run the scheduler daemon, attaching to a Redis db
'
)
@
click
.
option
(
'
--config-file
'
,
type
=
click
.
File
(
'r'
))
@
click
.
option
(
'
--validate/--no-validate
'
,
default
=
False
)
@
click
.
option
(
'
--host
'
,
type
=
str
)
@
click
.
option
(
'
--port
'
,
type
=
int
)
@
click
.
option
(
'
--debug/--no-debug
'
,
default
=
False
)
@
cli
.
command
(
help
=
"
Run the scheduler daemon, attaching to a Redis db
"
)
@
click
.
option
(
"
--config-file
"
,
type
=
click
.
File
(
"r"
))
@
click
.
option
(
"
--validate/--no-validate
"
,
default
=
False
)
@
click
.
option
(
"
--host
"
,
type
=
str
)
@
click
.
option
(
"
--port
"
,
type
=
int
)
@
click
.
option
(
"
--debug/--no-debug
"
,
default
=
False
)
def
daemon
(
config_file
=
None
,
validate
=
False
,
host
=
None
,
port
=
None
,
debug
=
False
):
setup_logging
(
debug
)
config
=
load_config
(
config_file
)
...
...
@@ -57,5 +56,5 @@ def daemon(config_file=None, validate=False, host=None, port=None, debug=False):
run_daemon
(
config
,
host
,
port
)
if
__name__
==
'
__main__
'
:
if
__name__
==
"
__main__
"
:
cli
()
scheduler/config.py
View file @
2921c26c
...
...
@@ -5,34 +5,35 @@ import re
import
yaml
ENV_PATTERN
=
re
.
compile
(
r
'.*?\${(\w+)}.*?'
)
ENV_PATTERN
=
re
.
compile
(
r
".*?\${(\w+)}.*?"
)
def
constructor_env_variables
(
loader
,
node
):
"""
Extracts the environment variable from the node's value
:param yaml.Loader loader: the yaml loader
:param node: the current node in the yaml
:return: the parsed string that contains the value of the environment
variable
"""
value
=
loader
.
construct_scalar
(
node
)
match
=
ENV_PATTERN
.
findall
(
value
)
# to find all env variables in line
if
match
:
full_value
=
value
for
g
in
match
:
env_variable
=
os
.
environ
.
get
(
g
,
)
if
env_variable
is
not
None
:
full_value
=
full_value
.
replace
(
f
'${{
{
g
}
}}'
,
env_variable
)
else
:
return
None
return
full_value
return
value
"""
Extracts the environment variable from the node's value
:param yaml.Loader loader: the yaml loader
:param node: the current node in the yaml
:return: the parsed string that contains the value of the environment
variable
"""
value
=
loader
.
construct_scalar
(
node
)
match
=
ENV_PATTERN
.
findall
(
value
)
# to find all env variables in line
if
match
:
full_value
=
value
for
g
in
match
:
env_variable
=
os
.
environ
.
get
(
g
,
)
if
env_variable
is
not
None
:
full_value
=
full_value
.
replace
(
f
"${{
{
g
}
}}"
,
env_variable
)
else
:
return
None
return
full_value
return
value
def
load_config
(
input_file
:
TextIO
)
->
dict
:
tag
=
'
!env
'
tag
=
"
!env
"
loader
=
yaml
.
SafeLoader
# the tag will be used to mark where to start searching for the pattern
...
...
scheduler/daemon.py
View file @
2921c26c
...
...
@@ -6,26 +6,26 @@ import schedule
logger
=
logging
.
getLogger
(
__name__
)
def
trigger_queue
(
name
:
str
,
client
:
redis
.
Redis
,
queue
:
str
):
logger
.
info
(
f
'
pushing
{
name
}
to
{
queue
}
'
)
logger
.
info
(
f
"
pushing
{
name
}
to
{
queue
}
"
)
client
.
lpush
(
queue
,
name
)
def
run_daemon
(
config
:
dict
,
host
:
str
,
port
:
str
):
""" Run the scheduler daemon, scheduling signals to send to other services
"""
"""Run the scheduler daemon, scheduling signals to send to other services"""
# initialize the queue client
client
=
redis
.
Redis
(
host
=
host
,
port
=
port
,
charset
=
"utf-8"
,
decode_responses
=
True
)
client
=
redis
.
Redis
(
host
=
host
,
port
=
port
,
charset
=
"utf-8"
,
decode_responses
=
True
)
schedules
=
config
[
'
schedules
'
]
schedules
=
config
[
"
schedules
"
]
for
s
in
schedules
:
number
,
time_unit
=
s
[
'
schedule
'
].
split
(
' '
)
number
,
time_unit
=
s
[
"
schedule
"
].
split
(
" "
)
job
=
schedule
.
every
(
int
(
number
))
job
=
getattr
(
job
,
time_unit
).
do
(
trigger_queue
,
name
=
s
[
'name'
],
client
=
client
,
queue
=
s
[
'queue'
])
job
=
getattr
(
job
,
time_unit
).
do
(
trigger_queue
,
name
=
s
[
"name"
],
client
=
client
,
queue
=
s
[
"queue"
]
)
while
True
:
schedule
.
run_pending
()
sleep
(
1
)
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment