Initial commit

Dockerfile

+#------------------------------------------------------------------------------
+#
+# Project: prism view server
+# Authors: Stephan Meissl <stephan.meissl@eox.at>
+#
+#------------------------------------------------------------------------------
+# Copyright (C) 2019 EOX IT Services GmbH <https://eox.at>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to
+# deal in the Software without restriction, including without limitation the
+# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+# sell copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies of this Software or works derived from this Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+#-----------------------------------------------------------------------
+
+FROM python:3.8
+
+LABEL name="prism view server scheduler" \
+    vendor="EOX IT Services GmbH <https://eox.at>" \
+    license="MIT Copyright (C) 2021 EOX IT Services GmbH <https://eox.at>" \
+    type="prism view server scheduler" 
+
+WORKDIR /opt/scheduler
+
+ADD . .
+
+RUN python3 setup.py install
+RUN pip3 install schedule
+
+CMD run-scheduler.sh
+
+LABEL version="2.0.0-alpha.2"

run-scheduler.sh

+#!/bin/sh
+
+echo "Running scheduler" >&2
+
+
+scheduler daemon \
+    --config-file /config.yaml \
+    --host ${REDIS_HOST} \
+    --port ${REDIS_PORT}

scheduler/__main__.py

+from .cli import cli
+
+cli(prog_name="scheduler")

scheduler/cli.py

+from os.path import join, dirname
+import logging.config
+
+import click
+import yaml
+import jsonschema
+
+from .daemon import run_daemon
+from .config import load_config
+
+def setup_logging(debug=False):
+    logging.config.dictConfig({
+        'version': 1,
+        'disable_existing_loggers': False,
+        'formatters': {
+            'brief': {
+                'format': '%(levelname)s %(name)s: %(message)s'
+            }
+        },
+        'handlers': {
+            'console': {
+                'class': 'logging.StreamHandler',
+                'level': 'DEBUG' if debug else 'INFO',
+                'formatter': 'brief',
+            }
+        },
+        'root': {
+            'handlers': ['console'],
+            'level': 'DEBUG' if debug else 'INFO',
+        }
+    })
+
+
+def validate_config(config):
+    with open(join(dirname(__file__), 'config-schema.yaml')) as f:
+        schema = yaml.load(f)
+
+    jsonschema.validate(config, schema)
+
+
+@click.group()
+def cli():
+    pass
+
+
+@cli.command(help='Run the scheduler daemon, attaching to a Redis db')
+@click.option('--config-file', type=click.File('r'))
+@click.option('--validate/--no-validate', default=False)
+@click.option('--host', type=str)
+@click.option('--port', type=int)
+@click.option('--debug/--no-debug', default=False)
+def daemon(config_file=None, validate=False, host=None, port=None, debug=False):
+    setup_logging(debug)
+    config = load_config(config_file)
+    if validate:
+        validate_config(config)
+    run_daemon(config, host, port)
+
+
+if __name__ == '__main__':
+    cli()

scheduler/config.py

+import os
+from typing import TextIO
+import re
+
+import yaml
+
+
+ENV_PATTERN = re.compile(r'.*?\${(\w+)}.*?')
+
+def constructor_env_variables(loader, node):
+        """
+        Extracts the environment variable from the node's value
+        :param yaml.Loader loader: the yaml loader
+        :param node: the current node in the yaml
+        :return: the parsed string that contains the value of the environment
+        variable
+        """
+        value = loader.construct_scalar(node)
+        match = ENV_PATTERN.findall(value)  # to find all env variables in line
+        if match:
+            full_value = value
+            for g in match:
+                env_variable = os.environ.get(g, )
+                if env_variable is not None:
+                    full_value = full_value.replace(
+                        f'${{{g}}}', env_variable
+                    )
+                else:
+                    return None
+            return full_value
+        return value
+
+
+def load_config(input_file: TextIO) -> dict:
+    tag = '!env'
+    loader = yaml.SafeLoader
+
+    # the tag will be used to mark where to start searching for the pattern
+    # e.g. somekey: !env somestring${MYENVVAR}blah blah blah
+    loader.add_implicit_resolver(tag, ENV_PATTERN, None)
+    loader.add_constructor(tag, constructor_env_variables)
+
+    return yaml.load(input_file, Loader=loader)

scheduler/daemon.py

+import logging
+import redis
+from time import sleep
+
+import schedule
+
+logger = logging.getLogger(__name__)
+
+def trigger_queue(name: str, client: redis.Redis, queue: str):
+    logger.info(f'pushing {name} to {queue}')
+    client.lpush(queue, name)
+
+def run_daemon(config: dict, host: str, port: str):
+    """ Run the scheduler daemon, scheduling signals to send to other services
+    """
+    # initialize the queue client
+    client = redis.Redis(
+        host=host, port=port, charset="utf-8", decode_responses=True
+    )
+
+    schedules = config['schedules']
+
+    for s in schedules:
+        number, time_unit = s['schedule'].split(' ')
+        job = schedule.every(int(number))
+        job = getattr(job, time_unit).do(trigger_queue, name=s['name'], client=client, queue=s['queue'])
+
+    while True:
+        schedule.run_pending()
+        sleep(1)
+

setup.py

+from setuptools import setup, find_packages
+
+
+long_description = ""
+
+setup(
+    name="scheduler", 
+    version="2.0.0-alpha.2",
+    author="",
+    author_email="",
+    description="scheduler for PVS",
+    long_description=long_description,
+    long_description_content_type="text/markdown",
+    url="https://gitlab.eox.at/esa/prism/vs/-/tree/master/scheduler",
+    packages=find_packages(),
+    classifiers=[
+        "Programming Language :: Python :: 3",
+        "License :: OSI Approved :: MIT License",
+        "Operating System :: OS Independent",
+    ],
+    python_requires='>=3.6',
+    entry_points={
+        "console_scripts": [
+            "scheduler = scheduler.cli:cli",
+        ],
+    }
+)