EOX GitLab Instance

Commit 5ea7cbe4 authored by Nikola Jankovic's avatar Nikola Jankovic 💻
Browse files

Merge branch 'introduce-dev-tools' into 'master'

Introduce dev tools

See merge request !1
parents 788fb0ed 3ff6ffce
Pipeline #18580 passed with stage
in 23 seconds
stages: stages:
- test - test
- deploy - publish
- chart - chart
variables: variables:
...@@ -12,13 +12,16 @@ test: ...@@ -12,13 +12,16 @@ test:
script: script:
- python3 setup.py install - python3 setup.py install
- pip3 install -r requirements-test.txt - pip3 install -r requirements-test.txt
- pytest - pip3 install -r requirements-dev.txt
- pytest --cov scheduler --cov-report term-missing
- flake8
- mypy .
deploy_latest: publish_latest:
image: docker:20.10.8 image: docker:20.10.8
services: services:
- docker:20.10.8-dind - docker:20.10.8-dind
stage: deploy stage: publish
script: script:
- docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" "$CI_REGISTRY" - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" "$CI_REGISTRY"
- docker build -t $CI_REGISTRY_IMAGE . - docker build -t $CI_REGISTRY_IMAGE .
...@@ -26,11 +29,11 @@ deploy_latest: ...@@ -26,11 +29,11 @@ deploy_latest:
only: only:
- master - master
deploy: publish:
image: docker:20.10.8 image: docker:20.10.8
services: services:
- docker:20.10.8-dind - docker:20.10.8-dind
stage: deploy stage: publish
script: script:
- docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" "$CI_REGISTRY" - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" "$CI_REGISTRY"
- docker build --cache-from $CI_REGISTRY_IMAGE:latest -t $CI_REGISTRY_IMAGE:$CI_COMMIT_TAG . - docker build --cache-from $CI_REGISTRY_IMAGE:latest -t $CI_REGISTRY_IMAGE:$CI_COMMIT_TAG .
...@@ -47,4 +50,3 @@ helm: ...@@ -47,4 +50,3 @@ helm:
- curl -u $HELM_CHART_REPOSITORY_CREDENTIALS -v -X POST https://charts-public.hub.eox.at/api/charts --data-binary "@${upload_filename}" - curl -u $HELM_CHART_REPOSITORY_CREDENTIALS -v -X POST https://charts-public.hub.eox.at/api/charts --data-binary "@${upload_filename}"
only: only:
- tags - tags
- master
black==21.10b0
flake8==4.0.1
mypy==0.910
types-PyYAML
types-redis
...@@ -8,31 +8,30 @@ import jsonschema ...@@ -8,31 +8,30 @@ import jsonschema
from .daemon import run_daemon from .daemon import run_daemon
from .config import load_config from .config import load_config
def setup_logging(debug=False): def setup_logging(debug=False):
logging.config.dictConfig({ logging.config.dictConfig(
'version': 1, {
'disable_existing_loggers': False, "version": 1,
'formatters': { "disable_existing_loggers": False,
'brief': { "formatters": {"brief": {"format": "%(levelname)s %(name)s: %(message)s"}},
'format': '%(levelname)s %(name)s: %(message)s' "handlers": {
} "console": {
}, "class": "logging.StreamHandler",
'handlers': { "level": "DEBUG" if debug else "INFO",
'console': { "formatter": "brief",
'class': 'logging.StreamHandler', }
'level': 'DEBUG' if debug else 'INFO', },
'formatter': 'brief', "root": {
} "handlers": ["console"],
}, "level": "DEBUG" if debug else "INFO",
'root': { },
'handlers': ['console'],
'level': 'DEBUG' if debug else 'INFO',
} }
}) )
def validate_config(config): def validate_config(config):
with open(join(dirname(__file__), 'config-schema.yaml')) as f: with open(join(dirname(__file__), "config-schema.yaml")) as f:
schema = yaml.load(f) schema = yaml.load(f)
jsonschema.validate(config, schema) jsonschema.validate(config, schema)
...@@ -43,12 +42,12 @@ def cli(): ...@@ -43,12 +42,12 @@ def cli():
pass pass
@cli.command(help='Run the scheduler daemon, attaching to a Redis db') @cli.command(help="Run the scheduler daemon, attaching to a Redis db")
@click.option('--config-file', type=click.File('r')) @click.option("--config-file", type=click.File("r"))
@click.option('--validate/--no-validate', default=False) @click.option("--validate/--no-validate", default=False)
@click.option('--host', type=str) @click.option("--host", type=str)
@click.option('--port', type=int) @click.option("--port", type=int)
@click.option('--debug/--no-debug', default=False) @click.option("--debug/--no-debug", default=False)
def daemon(config_file=None, validate=False, host=None, port=None, debug=False): def daemon(config_file=None, validate=False, host=None, port=None, debug=False):
setup_logging(debug) setup_logging(debug)
config = load_config(config_file) config = load_config(config_file)
...@@ -57,5 +56,5 @@ def daemon(config_file=None, validate=False, host=None, port=None, debug=False): ...@@ -57,5 +56,5 @@ def daemon(config_file=None, validate=False, host=None, port=None, debug=False):
run_daemon(config, host, port) run_daemon(config, host, port)
if __name__ == '__main__': if __name__ == "__main__":
cli() cli()
...@@ -5,34 +5,35 @@ import re ...@@ -5,34 +5,35 @@ import re
import yaml import yaml
ENV_PATTERN = re.compile(r'.*?\${(\w+)}.*?') ENV_PATTERN = re.compile(r".*?\${(\w+)}.*?")
def constructor_env_variables(loader, node): def constructor_env_variables(loader, node):
""" """
Extracts the environment variable from the node's value Extracts the environment variable from the node's value
:param yaml.Loader loader: the yaml loader :param yaml.Loader loader: the yaml loader
:param node: the current node in the yaml :param node: the current node in the yaml
:return: the parsed string that contains the value of the environment :return: the parsed string that contains the value of the environment
variable variable
""" """
value = loader.construct_scalar(node) value = loader.construct_scalar(node)
match = ENV_PATTERN.findall(value) # to find all env variables in line match = ENV_PATTERN.findall(value) # to find all env variables in line
if match: if match:
full_value = value full_value = value
for g in match: for g in match:
env_variable = os.environ.get(g, ) env_variable = os.environ.get(
if env_variable is not None: g,
full_value = full_value.replace( )
f'${{{g}}}', env_variable if env_variable is not None:
) full_value = full_value.replace(f"${{{g}}}", env_variable)
else: else:
return None return None
return full_value return full_value
return value return value
def load_config(input_file: TextIO) -> dict: def load_config(input_file: TextIO) -> dict:
tag = '!env' tag = "!env"
loader = yaml.SafeLoader loader = yaml.SafeLoader
# the tag will be used to mark where to start searching for the pattern # the tag will be used to mark where to start searching for the pattern
......
...@@ -6,26 +6,26 @@ import schedule ...@@ -6,26 +6,26 @@ import schedule
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
def trigger_queue(name: str, client: redis.Redis, queue: str): def trigger_queue(name: str, client: redis.Redis, queue: str):
logger.info(f'pushing {name} to {queue}') logger.info(f"pushing {name} to {queue}")
client.lpush(queue, name) client.lpush(queue, name)
def run_daemon(config: dict, host: str, port: str):
""" Run the scheduler daemon, scheduling signals to send to other services def run_daemon(config: dict, host: str, port: int):
""" """Run the scheduler daemon, scheduling signals to send to other services"""
# initialize the queue client # initialize the queue client
client = redis.Redis( client = redis.Redis(host=host, port=port, charset="utf-8", decode_responses=True)
host=host, port=port, charset="utf-8", decode_responses=True
)
schedules = config['schedules'] schedules = config["schedules"]
for s in schedules: for s in schedules:
number, time_unit = s['schedule'].split(' ') number, time_unit = s["schedule"].split(" ")
job = schedule.every(int(number)) job = schedule.every(int(number))
job = getattr(job, time_unit).do(trigger_queue, name=s['name'], client=client, queue=s['queue']) job = getattr(job, time_unit).do(
trigger_queue, name=s["name"], client=client, queue=s["queue"]
)
while True: while True:
schedule.run_pending() schedule.run_pending()
sleep(1) sleep(1)
[flake8]
max-line-length = 90
exclude = .venv, build
ignore = W503,E203
[mypy]
exclude = (.venv|build)
[mypy-jsonschema.*]
ignore_missing_imports = True
[mypy-setuptools.*]
ignore_missing_imports = True
[mypy-schedule.*]
ignore_missing_imports = True
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment