EOX GitLab Instance

Commit 17aa97e3 authored by Nikola Jankovic's avatar Nikola Jankovic 💻
Browse files

Initial commit

parents
Pipeline #17429 canceled with stages
#------------------------------------------------------------------------------
#
# Project: prism view server
# Authors: Stephan Meissl <stephan.meissl@eox.at>
#
#------------------------------------------------------------------------------
# Copyright (C) 2019 EOX IT Services GmbH <https://eox.at>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to
# deal in the Software without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
# sell copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies of this Software or works derived from this Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
#-----------------------------------------------------------------------
FROM python:3.8
LABEL name="prism view server scheduler" \
vendor="EOX IT Services GmbH <https://eox.at>" \
license="MIT Copyright (C) 2021 EOX IT Services GmbH <https://eox.at>" \
type="prism view server scheduler"
WORKDIR /opt/scheduler
ADD . .
RUN python3 setup.py install
RUN pip3 install schedule
CMD run-scheduler.sh
LABEL version="2.0.0-alpha.2"
#!/bin/sh
echo "Running scheduler" >&2
scheduler daemon \
--config-file /config.yaml \
--host ${REDIS_HOST} \
--port ${REDIS_PORT}
from .cli import cli
cli(prog_name="scheduler")
from os.path import join, dirname
import logging.config
import click
import yaml
import jsonschema
from .daemon import run_daemon
from .config import load_config
def setup_logging(debug=False):
logging.config.dictConfig({
'version': 1,
'disable_existing_loggers': False,
'formatters': {
'brief': {
'format': '%(levelname)s %(name)s: %(message)s'
}
},
'handlers': {
'console': {
'class': 'logging.StreamHandler',
'level': 'DEBUG' if debug else 'INFO',
'formatter': 'brief',
}
},
'root': {
'handlers': ['console'],
'level': 'DEBUG' if debug else 'INFO',
}
})
def validate_config(config):
with open(join(dirname(__file__), 'config-schema.yaml')) as f:
schema = yaml.load(f)
jsonschema.validate(config, schema)
@click.group()
def cli():
pass
@cli.command(help='Run the scheduler daemon, attaching to a Redis db')
@click.option('--config-file', type=click.File('r'))
@click.option('--validate/--no-validate', default=False)
@click.option('--host', type=str)
@click.option('--port', type=int)
@click.option('--debug/--no-debug', default=False)
def daemon(config_file=None, validate=False, host=None, port=None, debug=False):
setup_logging(debug)
config = load_config(config_file)
if validate:
validate_config(config)
run_daemon(config, host, port)
if __name__ == '__main__':
cli()
import os
from typing import TextIO
import re
import yaml
ENV_PATTERN = re.compile(r'.*?\${(\w+)}.*?')
def constructor_env_variables(loader, node):
"""
Extracts the environment variable from the node's value
:param yaml.Loader loader: the yaml loader
:param node: the current node in the yaml
:return: the parsed string that contains the value of the environment
variable
"""
value = loader.construct_scalar(node)
match = ENV_PATTERN.findall(value) # to find all env variables in line
if match:
full_value = value
for g in match:
env_variable = os.environ.get(g, )
if env_variable is not None:
full_value = full_value.replace(
f'${{{g}}}', env_variable
)
else:
return None
return full_value
return value
def load_config(input_file: TextIO) -> dict:
tag = '!env'
loader = yaml.SafeLoader
# the tag will be used to mark where to start searching for the pattern
# e.g. somekey: !env somestring${MYENVVAR}blah blah blah
loader.add_implicit_resolver(tag, ENV_PATTERN, None)
loader.add_constructor(tag, constructor_env_variables)
return yaml.load(input_file, Loader=loader)
import logging
import redis
from time import sleep
import schedule
logger = logging.getLogger(__name__)
def trigger_queue(name: str, client: redis.Redis, queue: str):
logger.info(f'pushing {name} to {queue}')
client.lpush(queue, name)
def run_daemon(config: dict, host: str, port: str):
""" Run the scheduler daemon, scheduling signals to send to other services
"""
# initialize the queue client
client = redis.Redis(
host=host, port=port, charset="utf-8", decode_responses=True
)
schedules = config['schedules']
for s in schedules:
number, time_unit = s['schedule'].split(' ')
job = schedule.every(int(number))
job = getattr(job, time_unit).do(trigger_queue, name=s['name'], client=client, queue=s['queue'])
while True:
schedule.run_pending()
sleep(1)
from setuptools import setup, find_packages
long_description = ""
setup(
name="scheduler",
version="2.0.0-alpha.2",
author="",
author_email="",
description="scheduler for PVS",
long_description=long_description,
long_description_content_type="text/markdown",
url="https://gitlab.eox.at/esa/prism/vs/-/tree/master/scheduler",
packages=find_packages(),
classifiers=[
"Programming Language :: Python :: 3",
"License :: OSI Approved :: MIT License",
"Operating System :: OS Independent",
],
python_requires='>=3.6',
entry_points={
"console_scripts": [
"scheduler = scheduler.cli:cli",
],
}
)
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment