diff --git a/registrar/backend/abc.py b/registrar/backend/abc.py
index 631c032282ef874ee05a88ef76c098c5c06675de..f1a42f700df8a3871bb3d51550e4d75c6ff65bad 100644
--- a/registrar/backend/abc.py
+++ b/registrar/backend/abc.py
@@ -1,18 +1,39 @@
from abc import ABC, abstractmethod
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Optional
if TYPE_CHECKING:
from pystac import Item
-class Backend(ABC):
+class ItemBackend(ABC):
def __repr__(self) -> str:
return f"<{self.__class__.__name__}>"
@abstractmethod
- def register(self, item: 'Item'):
+ def item_exists(self, item: 'Item') -> bool:
...
@abstractmethod
- def deregister(self, item: 'Item') -> str:
+ def register_item(self, item: 'Item'):
+ ...
+
+ @abstractmethod
+ def deregister_identifier(self, identifier: str) -> Optional[str]:
+ ...
+
+
+class PathBackend(ABC):
+ def __repr__(self) -> str:
+ return f"<{self.__class__.__name__}>"
+
+ @abstractmethod
+ def path_exists(self, path: str) -> bool:
+ ...
+
+ @abstractmethod
+ def register_path(self, path: str):
+ ...
+
+ @abstractmethod
+ def deregister_path(self, path: str) -> Optional[str]:
...
diff --git a/registrar/backend/eoxserver.py b/registrar/backend/eoxserver.py
index 573e3d842c115e860a99647c5fb4ca3d7840f95a..b757aef03ac9e173c0a4b6162b66ded249068a59 100644
--- a/registrar/backend/eoxserver.py
+++ b/registrar/backend/eoxserver.py
@@ -8,7 +8,7 @@ Contains implementations for different backends where data may be registered
import os
import sys
import logging
-from typing import List, TYPE_CHECKING, TypedDict
+from typing import List, TYPE_CHECKING, Optional, TypedDict
import json
from urllib.parse import urlparse
@@ -86,12 +86,12 @@ class EOxServerBackend(Backend):
f"<{self.__class__.__name__} instance_name ={self.instance_name}>"
)
- def exists(self, source: Source, item: 'Item'):
+ def item_exists(self, source: Source, item: 'Item') -> bool:
"""Checks whether the item exists in the given source
Args:
source (Source): source of the data
- item (Context): item to be checked
+ item (Item): item to be checked
Returns:
(bool): true if exists, false otherwise
@@ -253,7 +253,7 @@ class EOxServerBackend(Backend):
return product
@transaction.atomic
- def register(self, source: Source, item: 'Item', replace: bool):
+ def register_item(self, source: Source, item: 'Item', replace: bool):
"""Registers the item to the endpoint
Args:
@@ -308,17 +308,17 @@ class EOxServerBackend(Backend):
models.collection_insert_eo_object(collection, product)
@transaction.atomic
- def deregister(self, item: 'Item') -> str:
+ def deregister_identifier(self, identifier: str) -> Optional[str]:
"""Attempts to deregister item
Args:
- item (Item): item to be deregistered
+ identifier (str): identifier to be deregistered
"""
# ugly, ugly hack
from eoxserver.resources.coverages import models
try:
- logger.info(f"Deregistering product '{item.id}'")
- product = models.Product.objects.get(identifier=item.ideidntifier)
+ logger.info(f"Deregistering product '{identifier}'")
+ product = models.Product.objects.get(identifier=identifier)
grids = list(models.Grid.objects.filter(
coverage__parent_product=product
))
@@ -335,14 +335,14 @@ class EOxServerBackend(Backend):
grid.delete()
except models.Product.DoesNotExist:
logger.info(
- f"No product with identifier '{item.id}' found"
+ f"No product with identifier '{identifier}' found"
)
# no product found with that id
# return empty list
- return []
+ return None
logger.info(
- f"Deregistered product with identifier '{item.id}'"
+ f"Deregistered product with identifier '{identifier}'"
)
# return the deleted identifier
- return item.id
+ return identifier
diff --git a/registrar/cli.py b/registrar/cli.py
index 9fba4bbdf40b3096bd74631b4c1ad3b345bb8c43..456fe357530fbacd2daec5a526a3ba25057bf40a 100644
--- a/registrar/cli.py
+++ b/registrar/cli.py
@@ -5,9 +5,13 @@ import click
import yaml
import jsonschema
-from .registrar import register_file, deregister_file, deregister_identifier
+from .registrar import (
+ deregister_path, register_item, register_path, deregister_item,
+ deregister_identifier
+)
from .daemon import run_daemon
from .config import load_config
+from .utils import import_by_path
def setup_logging(debug=False):
@@ -47,8 +51,8 @@ def cli():
@cli.command(help='Run the registrar daemon, attaching to a Redis queue')
@click.option('--config-file', type=click.File('r'))
-@click.option('--validate/--no-validate', default=False)
-@click.option('--replace/--no-replace', default=False)
+@click.option('--validate', is_flag=True)
+@click.option('--replace', is_flag=True)
@click.option('--host', type=str)
@click.option('--port', type=int)
@click.option('--listen-queue', type=str)
@@ -56,17 +60,49 @@ def cli():
@click.option('--progress-set', type=str)
@click.option('--failure-set', type=str)
@click.option('--success-set', type=str)
-@click.option('--debug/--no-debug', default=False)
+@click.option('--extra/-e', type=str, multiple=True, default=[])
+@click.option('--debug', is_flag=True)
def daemon(config_file=None, validate=False, replace=False, host=None,
port=None, listen_queue=None, progress_set=None,
failure_set=None, success_set=None,
- deregister_queue=None, debug=False):
+ deregister_queue=None, extra=None, debug=False):
+ """ Run the registrar daemon to listen on the given queues
+ and execute the (de-)registrations commands.
+
+ Examples:
+
+ \b
+ registrar daemon --config-file config.yaml \
+ --validate --replace \
+ --host redis --port 6379 \
+ --listen-queue register \
+ --deregister-queue deregister \
+ --progress-set register_progress \
+ --failure-set register_failure \
+ --success-set register_success \
+ --extra some_queue=path.to.handler \
+ --extra some_other_queue=path.to.another.handler \
+ --debug
+ """
setup_logging(debug)
config = load_config(config_file)
if validate:
validate_config(config)
+
+ handlers = {}
+ if listen_queue:
+ handlers[listen_queue] = register_item
+
+ if deregister_queue:
+ handlers[deregister_queue] = deregister_item
+
+ for extra_handler in extra:
+ queue, _, path = extra_handler.partition('=')
+ handler = import_by_path(path.strip())
+ handlers[queue.strip()] = handler
+
run_daemon(
- config, replace, host, port, listen_queue, progress_set,
+ config, replace, host, port, handlers, progress_set,
failure_set, success_set, deregister_queue
)
@@ -74,25 +110,29 @@ def daemon(config_file=None, validate=False, replace=False, host=None,
@cli.command(help='Run a single, one-off registration')
@click.argument('item', type=str)
@click.option('--config-file', type=click.File('r'))
-@click.option('--validate/--no-validate', default=False)
-@click.option('--replace/--no-replace', default=False)
-@click.option('--debug/--no-debug', default=False)
-def register(item, config_file=None, validate=False, replace=False,
- debug=False):
+@click.option('--is-path', is_flag=True)
+@click.option('--validate', is_flag=True)
+@click.option('--replace', is_flag=True)
+@click.option('--debug', is_flag=True)
+def register(item, is_path=False, config_file=None, validate=False,
+ replace=False, debug=False):
setup_logging(debug)
config = load_config(config_file)
if validate:
validate_config(config)
- register_file(config, item, replace)
+ if is_path:
+ register_item(config, item, replace)
+ else:
+ register_path(config, item, replace)
@cli.command(help='Run a single, one-off de-registration')
@click.argument('--path', type=str)
@click.argument('--identifier', type=str)
@click.option('--config-file', type=click.File('r'))
-@click.option('--validate/--no-validate', default=False)
-@click.option('--debug/--no-debug', default=False)
+@click.option('--validate', is_flag=True)
+@click.option('--debug', is_flag=True)
def deregister(file_path=None, identifier=None, config_file=None,
validate=False, debug=False):
setup_logging(debug)
@@ -101,7 +141,7 @@ def deregister(file_path=None, identifier=None, config_file=None,
validate_config(config)
if file_path:
- deregister_file(config, file_path)
+ deregister_path(config, file_path)
elif identifier:
deregister_identifier(config, identifier)
diff --git a/registrar/registrar.py b/registrar/registrar.py
index ad27fe3b03c5a231aa544ed5f8477995dcbf2e5e..cf466aa78330521a66e6b6627ecc89b288d33dbd 100644
--- a/registrar/registrar.py
+++ b/registrar/registrar.py
@@ -6,35 +6,33 @@ Contains all functions relevant for registration
"""
import logging
-from typing import List
+from typing import Optional
import json
-from .source import get_source
-from .backend import get_backends
+from pystac import Item
+
+from .source import get_source, get_source_from_path
+from .backend import get_backends, get_path_backends
from .exceptions import RegistrationError
from .utils import import_by_path
-from pystac import Item
logger = logging.getLogger(__name__)
-def register_file(config: dict, item: str, replace: bool = False):
+def register_item(config: dict, value: str, replace: bool = False):
"""Handle registration of a stac item
Args:
config (dict): configuration of the registrar
- item (str): STAC item to register
+ value (str): encoded STAC item to register
replace (bool, optional): replace the file in the endpoint or not.
Defaults to False.
Raises:
- RegistrationError: Raised if file is already registered and replace is
+ RegistrationError: Raised if item is already registered and replace is
not turned on
-
- Returns:
- list: List of contexts
"""
- item = Item.from_dict(json.loads(item))
+ item = Item.from_dict(json.loads(value))
source = get_source(config, item)
logger.info(f"Handling '{item!r}'.")
@@ -43,7 +41,7 @@ def register_file(config: dict, item: str, replace: bool = False):
try:
for backend in get_backends(config):
- if backend.exists(source, item):
+ if backend.item_exists(source, item):
if replace:
logger.info(f"Replacing '{item!r}'.")
backend.register(source, item, replace=True)
@@ -67,49 +65,102 @@ def register_file(config: dict, item: str, replace: bool = False):
f"'{item!r}'"
)
- return item
+def register_path(config: dict, path: str, replace: bool = False):
+ """Handle registration of a path
+
+ Args:
+ config (dict): configuration of the registrar
+ path (str): path to register
+ replace (bool, optional): replace the file in the endpoint or not.
+ Defaults to False.
+
+ Raises:
+ RegistrationError: Raised if file is already registered and replace is
+ not turned on
+ """
-def deregister_identifier(config, identifier: str):
+ source = get_source_from_path(config, path)
+ logger.info(f"Handling '{path!r}'.")
+
+ for pre_handler in get_path_pre_handlers(config):
+ pre_handler(config, path)
+
+ try:
+ for backend in get_path_backends(config):
+ if backend.path_exists(source, path):
+ if replace:
+ logger.info(f"Replacing '{path!r}'.")
+ backend.register_path(source, path, replace=True)
+ else:
+ raise RegistrationError(
+ f'Path {path!r} is already registered'
+ )
+ else:
+ logger.info(f"Registering '{path!r}'.")
+ backend.register_path(source, path, replace=False)
+ except Exception as e:
+ for error_handler in get_path_error_handlers(config):
+ error_handler(config, path, e)
+ raise
+ else:
+ for post_handler in get_path_post_handlers(config):
+ post_handler(config, path)
+
+ logger.info(
+ f"Successfully {'replaced' if replace else 'registered'} "
+ f"'{path!r}'"
+ )
+
+
+def deregister_item(config: dict, value: str, _: bool = False):
+ item = Item.from_dict(json.loads(value))
+ logger.info(f"Handling deregistration for '{item!r}'.")
+ return _deregister_identifier(config, item.id, item)
+
+
+def deregister_identifier(config, identifier: str, _: bool = False):
logger.info(f"Handling deregistration of identifier '{identifier}'.")
- contexts = [Context(
- identifier=identifier,
- path=None,
- scheme=None
- )]
- return _deregister_contexts(config, contexts)
+ return _deregister_identifier(config, identifier)
+
+
+def _deregister_identifier(config, identifier: str,
+ item: Optional[dict] = None):
+ for pre_handler in get_deregister_pre_handlers(config):
+ pre_handler(config, identifier, item)
-def deregister_file(config: dict, path: str):
- logger.info(f"Handling deregistration of path '{path}'.")
- source = get_source(config, path)
- scheme = get_scheme(config, path)
- contexts = scheme.get_context(source, path)
- return _deregister_contexts(config, contexts)
+ try:
+ for backend in get_backends(config):
+ backend.deregister_identifier(identifier)
+ except Exception as e:
+ for error_handler in get_deregister_error_handlers(config):
+ error_handler(config, identifier, item, e)
+ raise
+ else:
+ for post_handler in get_deregister_post_handlers(config):
+ post_handler(config, identifier, item)
+ logger.info(f"Successfully deregistered '{identifier}'")
-def _deregister_contexts(config, items: List[Item]):
- if isinstance(items, Item):
- items = [items]
- for item in items:
- for pre_handler in get_deregister_pre_handlers(config):
- pre_handler(config, context, context)
+def deregister_path(config, path: str, _: bool = False):
- try:
- for backend in get_backends(config, context):
- backend.deregister(context)
- except Exception as e:
- for error_handler in get_deregister_error_handlers(config):
- error_handler(config, context, context, e)
- raise
- else:
- for post_handler in get_deregister_post_handlers(config):
- post_handler(config, context, context)
+ for pre_handler in get_deregister_path_pre_handlers(config):
+ pre_handler(config, path)
- logger.info(f"Successfully deregistered '{context.identifier}'")
+ try:
+ for backend in get_path_backends(config):
+ backend.deregister_path(path)
+ except Exception as e:
+ for error_handler in get_deregister_path_error_handlers(config):
+ error_handler(config, path, e)
+ raise
+ else:
+ for post_handler in get_deregister_path_post_handlers(config):
+ post_handler(config, path)
- return items
+ logger.info(f"Successfully deregistered '{path}'")
def _get_handlers(config, name):
@@ -138,6 +189,18 @@ def get_error_handlers(config):
return _get_handlers(config, 'error_handlers')
+def get_path_pre_handlers(config):
+ return _get_handlers(config, 'path_pre_handlers')
+
+
+def get_path_post_handlers(config):
+ return _get_handlers(config, 'path_post_handlers')
+
+
+def get_path_error_handlers(config):
+ return _get_handlers(config, 'path_error_handlers')
+
+
def get_deregister_pre_handlers(config):
return _get_handlers(config, 'deregister_pre_handlers')
@@ -148,3 +211,15 @@ def get_deregister_post_handlers(config):
def get_deregister_error_handlers(config):
return _get_handlers(config, 'deregister_error_handlers')
+
+
+def get_deregister_path_pre_handlers(config):
+ return _get_handlers(config, 'deregister_path_pre_handlers')
+
+
+def get_deregister_path_post_handlers(config):
+ return _get_handlers(config, 'deregister_path_post_handlers')
+
+
+def get_deregister_path_error_handlers(config):
+ return _get_handlers(config, 'deregister_path_error_handlers')
diff --git a/registrar/source.py b/registrar/source.py
index 7cf274f7e5f555a3e2dfcf31824c63e82276993f..74bf74f38eefee5c8339e1aa49022a49006b69f2 100644
--- a/registrar/source.py
+++ b/registrar/source.py
@@ -335,7 +335,7 @@ def get_source(config: dict, item: 'Item') -> Source:
break
else:
# no source found
- raise RegistrationError(f'Could not find a suitable source for {item!r}')
+ cfg_source['name'],
return SOURCE_TYPES[cfg_source['type']](
cfg_source['name'],