From 62a66b334575c6e509a378a227330059b4576abb Mon Sep 17 00:00:00 2001 From: Fabian Schindler Date: Mon, 31 May 2021 14:45:46 +0200 Subject: [PATCH 1/3] Adding deregistration capabilities to registrar --- core/registrar/backend.py | 23 ++++++++++++- core/registrar/cli.py | 37 +++++++++++++++++--- core/registrar/daemon.py | 52 +++++++++++++++++++--------- core/registrar/registrar.py | 67 ++++++++++++++++++++++++++++++++++--- 4 files changed, 154 insertions(+), 25 deletions(-) diff --git a/core/registrar/backend.py b/core/registrar/backend.py index 43475c61..1ff626b4 100644 --- a/core/registrar/backend.py +++ b/core/registrar/backend.py @@ -7,7 +7,7 @@ import json import django from django.db import transaction -from django.db.models import Count +from django.db.models import Count, Q from django.contrib.gis.geos import GEOSGeometry from osgeo import gdal @@ -346,6 +346,27 @@ class EOxServerBackend(Backend): collection = models.Collection.objects.get(identifier=collection_id) models.collection_insert_eo_object(collection, product) + @transaction.atomic + def deregister(self, item: Context): + # ugly, ugly hack + from eoxserver.resources.coverages import models + + product = models.Product.objects.get(identifier=item.identifier) + grids = list(models.Grid.objects.filter( + coverage__parent_product=product) + ) + product.delete() + + # clean up grids + for grid in grids: + grid_used = models.EOObject.objects.filter( + Q(coverage__grid=grid) | Q(mosaic__grid=grid), + ).exists() + # clean up grid as well, if it is not referenced + # anymore but saving named (user defined) grids + if grid and not grid.name and not grid_used: + grid.delete() + BACKENDS = { 'eoxserver': EOxServerBackend, diff --git a/core/registrar/cli.py b/core/registrar/cli.py index a9bf677c..bd9474e1 100644 --- a/core/registrar/cli.py +++ b/core/registrar/cli.py @@ -1,12 +1,11 @@ from os.path import join, dirname import logging.config -import json import click import yaml import jsonschema -from .registrar import register_file +from .registrar import register_file, deregister_file, deregister_identifier from .daemon import run_daemon from .config import load_config @@ -53,16 +52,23 @@ def cli(): @click.option('--host', type=str) @click.option('--port', type=int) @click.option('--listen-queue', type=str) +@click.option('--deregister-queue', type=str) @click.option('--progress-set', type=str) @click.option('--failure-set', type=str) @click.option('--success-set', type=str) @click.option('--debug/--no-debug', default=False) -def daemon(config_file=None, validate=False, replace=False, host=None, port=None, listen_queue=None, progress_set=None, failure_set=None, success_set=None, debug=False): +def daemon(config_file=None, validate=False, replace=False, host=None, + port=None, listen_queue=None, progress_set=None, + failure_set=None, success_set=None, + deregister_queue=None, debug=False): setup_logging(debug) config = load_config(config_file) if validate: validate_config(config) - run_daemon(config, replace, host, port, listen_queue, progress_set, failure_set, success_set) + run_daemon( + config, replace, host, port, listen_queue, progress_set, + failure_set, success_set, deregister_queue + ) @cli.command(help='Run a single, one-off registration') @@ -71,7 +77,8 @@ def daemon(config_file=None, validate=False, replace=False, host=None, port=None @click.option('--validate/--no-validate', default=False) @click.option('--replace/--no-replace', default=False) @click.option('--debug/--no-debug', default=False) -def register(file_path, config_file=None, validate=False, replace=False, debug=False): +def register(file_path, config_file=None, validate=False, replace=False, + debug=False): setup_logging(debug) config = load_config(config_file) if validate: @@ -79,5 +86,25 @@ def register(file_path, config_file=None, validate=False, replace=False, debug=F register_file(config, file_path, replace) + +@cli.command(help='Run a single, one-off de-registration') +@click.argument('--path', type=str) +@click.argument('--identifier', type=str) +@click.option('--config-file', type=click.File('r')) +@click.option('--validate/--no-validate', default=False) +@click.option('--debug/--no-debug', default=False) +def deregister(file_path=None, identifier=None, config_file=None, + validate=False, debug=False): + setup_logging(debug) + config = load_config(config_file) + if validate: + validate_config(config) + + if file_path: + deregister_file(config, file_path) + elif identifier: + deregister_identifier(config, identifier) + + if __name__ == '__main__': cli() diff --git a/core/registrar/daemon.py b/core/registrar/daemon.py index 8654695b..3c603df1 100644 --- a/core/registrar/daemon.py +++ b/core/registrar/daemon.py @@ -3,13 +3,14 @@ import json import redis -from .registrar import register_file +from .registrar import deregister_identifier, deregister_file, register_file logger = logging.getLogger(__name__) -def run_daemon(config, replace, host, port, listen_queue, progress_set, failure_set, success_set): +def run_daemon(config, replace, host, port, listen_queue, progress_set, + failure_set, success_set, deregister_queue=None): """ Run the registrar daemon, listening on a redis queue for files to be registered. """ @@ -18,18 +19,39 @@ def run_daemon(config, replace, host, port, listen_queue, progress_set, failure_ host=host, port=port, charset="utf-8", decode_responses=True ) logger.debug("waiting for redis queue '%s'..." % listen_queue) + queues = [listen_queue] + if deregister_queue: + queues.append(deregister_queue) + while True: # fetch an item from the queue to be registered - _, value = client.brpop(listen_queue) - client.sadd(progress_set, value) - # start the registration on that file - try: - register_file(config, value, replace) - client.sadd(success_set, value) - client.srem(progress_set, value) - except Exception as e: - if 'is already registered' not in "%s" % e: - # do not add to failure if skipped due to already registered - client.sadd(failure_set, value) - client.srem(progress_set, value) - logger.exception(e) + queue, value = client.brpop(queues) + + if queue == listen_queue: + client.sadd(progress_set, value) + # start the registration on that file + try: + register_file(config, value, replace) + client.sadd(success_set, value) + client.srem(progress_set, value) + except Exception as e: + if 'is already registered' not in "%s" % e: + # do not add to failure if skipped due to already + # registered + client.sadd(failure_set, value) + client.srem(progress_set, value) + logger.exception(e) + elif queue == deregister_queue: + parsed = json.loads(value) + try: + if 'identifier' in parsed: + deregister_identifier(config, parsed['identifier']) + elif 'url' in parsed: + deregister_file(config, parsed['url']) + else: + raise Exception( + "Neither 'identifer' nor 'url' passed to " + "deregistration" + ) + except Exception as e: + logger.exception(e) diff --git a/core/registrar/registrar.py b/core/registrar/registrar.py index 0b622ac4..0c24984b 100644 --- a/core/registrar/registrar.py +++ b/core/registrar/registrar.py @@ -1,5 +1,5 @@ -import re import logging +from typing import List from .context import Context from .source import get_source @@ -12,7 +12,7 @@ from .utils import import_by_path logger = logging.getLogger(__name__) -def register_file(config: dict, path: str, replace: bool=False): +def register_file(config: dict, path: str, replace: bool = False): """ Handle the registration of a single path. """ logger.info(f"Handling '{path}'.") @@ -33,7 +33,9 @@ def register_file(config: dict, path: str, replace: bool=False): logger.info(f"Replacing '{path}'.") backend.register(source, context, replace=True) else: - raise RegistrationError(f'Object {context} is already registered') + raise RegistrationError( + f'Object {context} is already registered' + ) else: logger.info(f"Registering '{path}'.") backend.register(source, context, replace=False) @@ -45,7 +47,52 @@ def register_file(config: dict, path: str, replace: bool=False): for post_handler in get_post_handlers(config): post_handler(config, path, context) - logger.info(f"Successfully {'replaced' if replace else 'registered'} '{path}'") + logger.info( + f"Successfully {'replaced' if replace else 'registered'} " + f"'{path}'" + ) + + return contexts + + +def deregister_identifier(config, identifier: str): + logger.info(f"Handling deregistration of identifier '{identifier}'.") + contexts = [Context( + identifier=identifier, + path=None, + scheme=None + )] + return _deregister_contexts(config, contexts) + + +def deregister_file(config: dict, path: str): + logger.info(f"Handling deregistration of path '{path}'.") + source = get_source(config, path) + scheme = get_scheme(config, path) + contexts = scheme.get_context(source, path) + return _deregister_contexts(config, contexts) + + +def _deregister_contexts(config, contexts: List[Context]): + if isinstance(contexts, Context): + contexts = [contexts] + + for context in contexts: + for pre_handler in get_deregister_pre_handlers(config): + pre_handler(config, context, context) + + try: + for backend in get_backends(config, context): + backend.deregister(context) + except Exception as e: + for error_handler in get_deregister_error_handlers(config): + error_handler(config, context, context, e) + raise + else: + for post_handler in get_deregister_post_handlers(config): + post_handler(config, context, context) + + logger.info(f"Successfully deregistered '{context.identifier}'") return contexts @@ -74,3 +121,15 @@ def get_post_handlers(config): def get_error_handlers(config): return _get_handlers(config, 'error_handlers') + + +def get_deregister_pre_handlers(config): + return _get_handlers(config, 'deregister_pre_handlers') + + +def get_deregister_post_handlers(config): + return _get_handlers(config, 'deregister_post_handlers') + + +def get_deregister_error_handlers(config): + return _get_handlers(config, 'deregister_error_handlers') -- GitLab From 1e690b225db8b7d5c33431507740ab8071bacaf5 Mon Sep 17 00:00:00 2001 From: Fabian Schindler Date: Mon, 31 May 2021 14:56:44 +0200 Subject: [PATCH 2/3] Fixing EOxServer backend deregistration --- core/registrar/backend.py | 44 ++++++++++++++++++++++++--------------- 1 file changed, 27 insertions(+), 17 deletions(-) diff --git a/core/registrar/backend.py b/core/registrar/backend.py index 1ff626b4..893f43b5 100644 --- a/core/registrar/backend.py +++ b/core/registrar/backend.py @@ -26,9 +26,12 @@ class RegistrationResult: class Backend: - def register_item(self, item: Context) -> RegistrationResult: + def register(self, item: Context) -> RegistrationResult: raise NotImplementedError + def deregister(self, item: Context) -> List[str]: + return [] + class EOxServerBackend(Backend): def __init__(self, instance_base_path: str, instance_name: str, @@ -350,22 +353,29 @@ class EOxServerBackend(Backend): def deregister(self, item: Context): # ugly, ugly hack from eoxserver.resources.coverages import models - - product = models.Product.objects.get(identifier=item.identifier) - grids = list(models.Grid.objects.filter( - coverage__parent_product=product) - ) - product.delete() - - # clean up grids - for grid in grids: - grid_used = models.EOObject.objects.filter( - Q(coverage__grid=grid) | Q(mosaic__grid=grid), - ).exists() - # clean up grid as well, if it is not referenced - # anymore but saving named (user defined) grids - if grid and not grid.name and not grid_used: - grid.delete() + try: + product = models.Product.objects.get(identifier=item.identifier) + grids = list(models.Grid.objects.filter( + coverage__parent_product=product) + ) + product.delete() + + # clean up grids + for grid in grids: + grid_used = models.EOObject.objects.filter( + Q(coverage__grid=grid) | Q(mosaic__grid=grid), + ).exists() + # clean up grid as well, if it is not referenced + # anymore but saving named (user defined) grids + if grid and not grid.name and not grid_used: + grid.delete() + except models.Product.DoesNotExist: + # no product found with that id + # return empty list + return [] + + # return the deleted identifier + return [item.identifier] BACKENDS = { -- GitLab From 966d3e133222e19434791f6cf053d0f5ed01402b Mon Sep 17 00:00:00 2001 From: Fabian Schindler Date: Mon, 31 May 2021 15:06:36 +0200 Subject: [PATCH 3/3] Adding missing deregister-queue CLI parameter and env var --- core/Dockerfile | 1 + core/run-registrar.sh | 1 + 2 files changed, 2 insertions(+) diff --git a/core/Dockerfile b/core/Dockerfile index 5fe58b39..9d320af0 100644 --- a/core/Dockerfile +++ b/core/Dockerfile @@ -74,6 +74,7 @@ ENV INSTANCE_ID="prism-view-server_core" \ REDIS_REGISTER_FAILURE_KEY="register-failure_set" \ REDIS_REGISTER_PROGRESS_KEY="registering_set" \ REDIS_REGISTER_SUCCESS_KEY="register-success_set" \ + REDIS_DEREGISTER_QUEUE_KEY="deregister_queue" \ INIT_SCRIPTS="/configure.sh" \ COLLECT_STATIC="false" \ REGISTRAR_REPLACE= \ diff --git a/core/run-registrar.sh b/core/run-registrar.sh index 5c40132c..8e2e83af 100644 --- a/core/run-registrar.sh +++ b/core/run-registrar.sh @@ -14,4 +14,5 @@ registrar daemon \ --progress-set ${REDIS_REGISTER_PROGRESS_KEY} \ --failure-set ${REDIS_REGISTER_FAILURE_KEY} \ --success-set ${REDIS_REGISTER_SUCCESS_KEY} \ + --deregister-queue ${REDIS_DEREGISTER_QUEUE_KEY} \ ${replace} >&2 -- GitLab