EOX GitLab Instance

Commit 5ea7cbe4 authored by Nikola Jankovic's avatar Nikola Jankovic 💻
Browse files

Merge branch 'introduce-dev-tools' into 'master'

Introduce dev tools

See merge request !1
parents 788fb0ed 3ff6ffce
Pipeline #18580 passed with stage
in 23 seconds
stages:
- test
- deploy
- publish
- chart
variables:
......@@ -12,13 +12,16 @@ test:
script:
- python3 setup.py install
- pip3 install -r requirements-test.txt
- pytest
- pip3 install -r requirements-dev.txt
- pytest --cov scheduler --cov-report term-missing
- flake8
- mypy .
deploy_latest:
publish_latest:
image: docker:20.10.8
services:
- docker:20.10.8-dind
stage: deploy
stage: publish
script:
- docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" "$CI_REGISTRY"
- docker build -t $CI_REGISTRY_IMAGE .
......@@ -26,11 +29,11 @@ deploy_latest:
only:
- master
deploy:
publish:
image: docker:20.10.8
services:
- docker:20.10.8-dind
stage: deploy
stage: publish
script:
- docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" "$CI_REGISTRY"
- docker build --cache-from $CI_REGISTRY_IMAGE:latest -t $CI_REGISTRY_IMAGE:$CI_COMMIT_TAG .
......@@ -47,4 +50,3 @@ helm:
- curl -u $HELM_CHART_REPOSITORY_CREDENTIALS -v -X POST https://charts-public.hub.eox.at/api/charts --data-binary "@${upload_filename}"
only:
- tags
- master
black==21.10b0
flake8==4.0.1
mypy==0.910
types-PyYAML
types-redis
......@@ -8,31 +8,30 @@ import jsonschema
from .daemon import run_daemon
from .config import load_config
def setup_logging(debug=False):
logging.config.dictConfig({
'version': 1,
'disable_existing_loggers': False,
'formatters': {
'brief': {
'format': '%(levelname)s %(name)s: %(message)s'
}
},
'handlers': {
'console': {
'class': 'logging.StreamHandler',
'level': 'DEBUG' if debug else 'INFO',
'formatter': 'brief',
}
},
'root': {
'handlers': ['console'],
'level': 'DEBUG' if debug else 'INFO',
logging.config.dictConfig(
{
"version": 1,
"disable_existing_loggers": False,
"formatters": {"brief": {"format": "%(levelname)s %(name)s: %(message)s"}},
"handlers": {
"console": {
"class": "logging.StreamHandler",
"level": "DEBUG" if debug else "INFO",
"formatter": "brief",
}
},
"root": {
"handlers": ["console"],
"level": "DEBUG" if debug else "INFO",
},
}
})
)
def validate_config(config):
with open(join(dirname(__file__), 'config-schema.yaml')) as f:
with open(join(dirname(__file__), "config-schema.yaml")) as f:
schema = yaml.load(f)
jsonschema.validate(config, schema)
......@@ -43,12 +42,12 @@ def cli():
pass
@cli.command(help='Run the scheduler daemon, attaching to a Redis db')
@click.option('--config-file', type=click.File('r'))
@click.option('--validate/--no-validate', default=False)
@click.option('--host', type=str)
@click.option('--port', type=int)
@click.option('--debug/--no-debug', default=False)
@cli.command(help="Run the scheduler daemon, attaching to a Redis db")
@click.option("--config-file", type=click.File("r"))
@click.option("--validate/--no-validate", default=False)
@click.option("--host", type=str)
@click.option("--port", type=int)
@click.option("--debug/--no-debug", default=False)
def daemon(config_file=None, validate=False, host=None, port=None, debug=False):
setup_logging(debug)
config = load_config(config_file)
......@@ -57,5 +56,5 @@ def daemon(config_file=None, validate=False, host=None, port=None, debug=False):
run_daemon(config, host, port)
if __name__ == '__main__':
if __name__ == "__main__":
cli()
......@@ -5,34 +5,35 @@ import re
import yaml
ENV_PATTERN = re.compile(r'.*?\${(\w+)}.*?')
ENV_PATTERN = re.compile(r".*?\${(\w+)}.*?")
def constructor_env_variables(loader, node):
"""
Extracts the environment variable from the node's value
:param yaml.Loader loader: the yaml loader
:param node: the current node in the yaml
:return: the parsed string that contains the value of the environment
variable
"""
value = loader.construct_scalar(node)
match = ENV_PATTERN.findall(value) # to find all env variables in line
if match:
full_value = value
for g in match:
env_variable = os.environ.get(g, )
if env_variable is not None:
full_value = full_value.replace(
f'${{{g}}}', env_variable
)
else:
return None
return full_value
return value
"""
Extracts the environment variable from the node's value
:param yaml.Loader loader: the yaml loader
:param node: the current node in the yaml
:return: the parsed string that contains the value of the environment
variable
"""
value = loader.construct_scalar(node)
match = ENV_PATTERN.findall(value) # to find all env variables in line
if match:
full_value = value
for g in match:
env_variable = os.environ.get(
g,
)
if env_variable is not None:
full_value = full_value.replace(f"${{{g}}}", env_variable)
else:
return None
return full_value
return value
def load_config(input_file: TextIO) -> dict:
tag = '!env'
tag = "!env"
loader = yaml.SafeLoader
# the tag will be used to mark where to start searching for the pattern
......
......@@ -6,26 +6,26 @@ import schedule
logger = logging.getLogger(__name__)
def trigger_queue(name: str, client: redis.Redis, queue: str):
logger.info(f'pushing {name} to {queue}')
logger.info(f"pushing {name} to {queue}")
client.lpush(queue, name)
def run_daemon(config: dict, host: str, port: str):
""" Run the scheduler daemon, scheduling signals to send to other services
"""
def run_daemon(config: dict, host: str, port: int):
"""Run the scheduler daemon, scheduling signals to send to other services"""
# initialize the queue client
client = redis.Redis(
host=host, port=port, charset="utf-8", decode_responses=True
)
client = redis.Redis(host=host, port=port, charset="utf-8", decode_responses=True)
schedules = config['schedules']
schedules = config["schedules"]
for s in schedules:
number, time_unit = s['schedule'].split(' ')
number, time_unit = s["schedule"].split(" ")
job = schedule.every(int(number))
job = getattr(job, time_unit).do(trigger_queue, name=s['name'], client=client, queue=s['queue'])
job = getattr(job, time_unit).do(
trigger_queue, name=s["name"], client=client, queue=s["queue"]
)
while True:
schedule.run_pending()
sleep(1)
[flake8]
max-line-length = 90
exclude = .venv, build
ignore = W503,E203
[mypy]
exclude = (.venv|build)
[mypy-jsonschema.*]
ignore_missing_imports = True
[mypy-setuptools.*]
ignore_missing_imports = True
[mypy-schedule.*]
ignore_missing_imports = True
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment