Merge branch 'introduce-dev-tools' into 'master'

Introduce dev tools

See merge request !1

.gitlab-ci.yml

 stages:
   - test
-  - deploy
+  - publish
   - chart
 
 variables:
@@ -12,13 +12,16 @@ test:
   script:
     - python3 setup.py install
     - pip3 install -r requirements-test.txt
-    - pytest
+    - pip3 install -r requirements-dev.txt
+    - pytest --cov scheduler --cov-report term-missing
+    - flake8
+    - mypy .
 
-deploy_latest:
+publish_latest:
   image: docker:20.10.8
   services:
     - docker:20.10.8-dind
-  stage: deploy
+  stage: publish
   script:
     - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" "$CI_REGISTRY"
     - docker build -t $CI_REGISTRY_IMAGE .
@@ -26,11 +29,11 @@ deploy_latest:
   only:
     - master
 
-deploy:
+publish:
   image: docker:20.10.8
   services:
     - docker:20.10.8-dind
-  stage: deploy
+  stage: publish
   script:
     - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" "$CI_REGISTRY"
     - docker build --cache-from $CI_REGISTRY_IMAGE:latest -t $CI_REGISTRY_IMAGE:$CI_COMMIT_TAG .
@@ -47,4 +50,3 @@ helm:
     - curl -u $HELM_CHART_REPOSITORY_CREDENTIALS -v -X POST https://charts-public.hub.eox.at/api/charts --data-binary "@${upload_filename}"
   only:
     - tags
-    - master

requirements-dev.txt

+black==21.10b0
+flake8==4.0.1
+mypy==0.910
+
+types-PyYAML
+types-redis

scheduler/cli.py

@@ -8,31 +8,30 @@ import jsonschema
 from .daemon import run_daemon
 from .config import load_config
 
+
 def setup_logging(debug=False):
-    logging.config.dictConfig({
-        'version': 1,
-        'disable_existing_loggers': False,
-        'formatters': {
-            'brief': {
-                'format': '%(levelname)s %(name)s: %(message)s'
-            }
-        },
-        'handlers': {
-            'console': {
-                'class': 'logging.StreamHandler',
-                'level': 'DEBUG' if debug else 'INFO',
-                'formatter': 'brief',
-            }
-        },
-        'root': {
-            'handlers': ['console'],
-            'level': 'DEBUG' if debug else 'INFO',
+    logging.config.dictConfig(
+        {
+            "version": 1,
+            "disable_existing_loggers": False,
+            "formatters": {"brief": {"format": "%(levelname)s %(name)s: %(message)s"}},
+            "handlers": {
+                "console": {
+                    "class": "logging.StreamHandler",
+                    "level": "DEBUG" if debug else "INFO",
+                    "formatter": "brief",
+                }
+            },
+            "root": {
+                "handlers": ["console"],
+                "level": "DEBUG" if debug else "INFO",
+            },
         }
-    })
+    )
 
 
 def validate_config(config):
-    with open(join(dirname(__file__), 'config-schema.yaml')) as f:
+    with open(join(dirname(__file__), "config-schema.yaml")) as f:
         schema = yaml.load(f)
 
     jsonschema.validate(config, schema)
@@ -43,12 +42,12 @@ def cli():
     pass
 
 
-@cli.command(help='Run the scheduler daemon, attaching to a Redis db')
-@click.option('--config-file', type=click.File('r'))
-@click.option('--validate/--no-validate', default=False)
-@click.option('--host', type=str)
-@click.option('--port', type=int)
-@click.option('--debug/--no-debug', default=False)
+@cli.command(help="Run the scheduler daemon, attaching to a Redis db")
+@click.option("--config-file", type=click.File("r"))
+@click.option("--validate/--no-validate", default=False)
+@click.option("--host", type=str)
+@click.option("--port", type=int)
+@click.option("--debug/--no-debug", default=False)
 def daemon(config_file=None, validate=False, host=None, port=None, debug=False):
     setup_logging(debug)
     config = load_config(config_file)
@@ -57,5 +56,5 @@ def daemon(config_file=None, validate=False, host=None, port=None, debug=False):
     run_daemon(config, host, port)
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     cli()

scheduler/config.py

@@ -5,34 +5,35 @@ import re
 import yaml
 
 
-ENV_PATTERN = re.compile(r'.*?\${(\w+)}.*?')
+ENV_PATTERN = re.compile(r".*?\${(\w+)}.*?")
+
 
 def constructor_env_variables(loader, node):
-        """
-        Extracts the environment variable from the node's value
-        :param yaml.Loader loader: the yaml loader
-        :param node: the current node in the yaml
-        :return: the parsed string that contains the value of the environment
-        variable
-        """
-        value = loader.construct_scalar(node)
-        match = ENV_PATTERN.findall(value)  # to find all env variables in line
-        if match:
-            full_value = value
-            for g in match:
-                env_variable = os.environ.get(g, )
-                if env_variable is not None:
-                    full_value = full_value.replace(
-                        f'${{{g}}}', env_variable
-                    )
-                else:
-                    return None
-            return full_value
-        return value
+    """
+    Extracts the environment variable from the node's value
+    :param yaml.Loader loader: the yaml loader
+    :param node: the current node in the yaml
+    :return: the parsed string that contains the value of the environment
+    variable
+    """
+    value = loader.construct_scalar(node)
+    match = ENV_PATTERN.findall(value)  # to find all env variables in line
+    if match:
+        full_value = value
+        for g in match:
+            env_variable = os.environ.get(
+                g,
+            )
+            if env_variable is not None:
+                full_value = full_value.replace(f"${{{g}}}", env_variable)
+            else:
+                return None
+        return full_value
+    return value
 
 
 def load_config(input_file: TextIO) -> dict:
-    tag = '!env'
+    tag = "!env"
     loader = yaml.SafeLoader
 
     # the tag will be used to mark where to start searching for the pattern

scheduler/daemon.py

@@ -6,26 +6,26 @@ import schedule
 
 logger = logging.getLogger(__name__)
 
+
 def trigger_queue(name: str, client: redis.Redis, queue: str):
-    logger.info(f'pushing {name} to {queue}')
+    logger.info(f"pushing {name} to {queue}")
     client.lpush(queue, name)
 
-def run_daemon(config: dict, host: str, port: str):
-    """ Run the scheduler daemon, scheduling signals to send to other services
-    """
+
+def run_daemon(config: dict, host: str, port: int):
+    """Run the scheduler daemon, scheduling signals to send to other services"""
     # initialize the queue client
-    client = redis.Redis(
-        host=host, port=port, charset="utf-8", decode_responses=True
-    )
+    client = redis.Redis(host=host, port=port, charset="utf-8", decode_responses=True)
 
-    schedules = config['schedules']
+    schedules = config["schedules"]
 
     for s in schedules:
-        number, time_unit = s['schedule'].split(' ')
+        number, time_unit = s["schedule"].split(" ")
         job = schedule.every(int(number))
-        job = getattr(job, time_unit).do(trigger_queue, name=s['name'], client=client, queue=s['queue'])
+        job = getattr(job, time_unit).do(
+            trigger_queue, name=s["name"], client=client, queue=s["queue"]
+        )
 
     while True:
         schedule.run_pending()
         sleep(1)
-

setup.cfg

+[flake8]
+max-line-length = 90
+exclude = .venv, build
+ignore = W503,E203
+
+[mypy]
+exclude = (.venv|build)
+
+[mypy-jsonschema.*]
+ignore_missing_imports = True
+
+[mypy-setuptools.*]
+ignore_missing_imports = True
+
+[mypy-schedule.*]
+ignore_missing_imports = True
+