From 1d06e931a53d3050df95f04eb2fdee35393a0570 Mon Sep 17 00:00:00 2001
From: Fabian Schindler <fabian.schindler.strauss@gmail.com>
Date: Fri, 26 Nov 2021 18:06:21 +0100
Subject: [PATCH] Fixing linting and flake8 issues
---
registrar/backend/__init__.py | 10 ++--
registrar/backend/abc.py | 6 +-
registrar/backend/eoxserver.py | 15 ++---
registrar/cli.py | 103 ++++++++++++++++++++-------------
registrar/daemon.py | 19 +++---
registrar/post_handlers.py | 27 ++-------
registrar/registrar.py | 24 ++++----
registrar/source.py | 21 +++----
registrar/xml.py | 5 +-
9 files changed, 109 insertions(+), 121 deletions(-)
diff --git a/registrar/backend/__init__.py b/registrar/backend/__init__.py
index f01ae69..dfb8615 100644
--- a/registrar/backend/__init__.py
+++ b/registrar/backend/__init__.py
@@ -17,12 +17,12 @@ def get_backends(config: dict) -> List[ItemBackend]:
backends: List[ItemBackend] = []
for cfg_backend in config["backends"]:
# construct backend
- backend_cls = import_by_path(cfg_backend['path'])
+ backend_cls = import_by_path(cfg_backend["path"])
backends.append(
backend_cls(
- *cfg_backend.get('args', []),
- **cfg_backend.get('kwargs', {}),
+ *cfg_backend.get("args", []),
+ **cfg_backend.get("kwargs", {}),
)
)
@@ -40,9 +40,9 @@ def get_path_backends(config: dict) -> List[PathBackend]:
carried out
"""
backends = []
- for cfg_backend in config['path_backends']:
+ for cfg_backend in config["path_backends"]:
# construct backend
- backend_cls = import_by_path(cfg_backend['path'])
+ backend_cls = import_by_path(cfg_backend["path"])
backends.append(
backend_cls(
diff --git a/registrar/backend/abc.py b/registrar/backend/abc.py
index b0e3d94..98497ca 100644
--- a/registrar/backend/abc.py
+++ b/registrar/backend/abc.py
@@ -4,19 +4,17 @@ from typing import TYPE_CHECKING, Optional
if TYPE_CHECKING:
from pystac import Item
-from ..source import Source
-
class ItemBackend(ABC):
def __repr__(self) -> str:
return f"<{self.__class__.__name__}>"
@abstractmethod
- def item_exists(self, item: 'Item') -> bool:
+ def item_exists(self, item: "Item") -> bool:
...
@abstractmethod
- def register_item(self, item: 'Item'):
+ def register_item(self, item: "Item"):
...
@abstractmethod
diff --git a/registrar/backend/eoxserver.py b/registrar/backend/eoxserver.py
index 7caf137..afcda97 100644
--- a/registrar/backend/eoxserver.py
+++ b/registrar/backend/eoxserver.py
@@ -15,7 +15,7 @@ from urllib.parse import urlparse
import django
from django.db import transaction
-# from django.db.models import Q
+from django.db.models import Q
if TYPE_CHECKING:
from pystac import Item
@@ -313,12 +313,11 @@ class EOxServerBackend(Backend):
"""
# ugly, ugly hack
from eoxserver.resources.coverages import models
+
try:
logger.info(f"Deregistering product '{identifier}'")
product = models.Product.objects.get(identifier=identifier)
- grids = list(models.Grid.objects.filter(
- coverage__parent_product=product
- ))
+ grids = list(models.Grid.objects.filter(coverage__parent_product=product))
product.delete()
# clean up grids
@@ -331,15 +330,11 @@ class EOxServerBackend(Backend):
if grid and not grid.name and not grid_used:
grid.delete()
except models.Product.DoesNotExist:
- logger.info(
- f"No product with identifier '{identifier}' found"
- )
+ logger.info(f"No product with identifier '{identifier}' found")
# no product found with that id
# return empty list
return None
- logger.info(
- f"Deregistered product with identifier '{identifier}'"
- )
+ logger.info(f"Deregistered product with identifier '{identifier}'")
# return the deleted identifier
return identifier
diff --git a/registrar/cli.py b/registrar/cli.py
index afe659b..be90037 100644
--- a/registrar/cli.py
+++ b/registrar/cli.py
@@ -1,14 +1,16 @@
from os.path import join, dirname
import logging.config
-from typing import TextIO
import click
import yaml
import jsonschema
from .registrar import (
- deregister_path, register_item, register_path, deregister_item,
- deregister_identifier
+ deregister_path,
+ register_item,
+ register_path,
+ deregister_item,
+ deregister_identifier,
)
from .daemon import run_daemon
from .config import load_config
@@ -48,23 +50,33 @@ def cli():
pass
-@cli.command(help='Run the registrar daemon, attaching to a Redis queue')
-@click.option('--config-file', type=click.File('r'))
-@click.option('--validate', is_flag=True)
-@click.option('--replace', is_flag=True)
-@click.option('--host', type=str)
-@click.option('--port', type=int)
-@click.option('--listen-queue', type=str)
-@click.option('--deregister-queue', type=str)
-@click.option('--progress-set', type=str)
-@click.option('--failure-set', type=str)
-@click.option('--success-set', type=str)
-@click.option('--extra/-e', type=str, multiple=True, default=[])
-@click.option('--debug', is_flag=True)
-def daemon(config_file=None, validate=False, replace=False, host=None,
- port=None, listen_queue=None, progress_set=None,
- failure_set=None, success_set=None,
- deregister_queue=None, extra=None, debug=False):
+@cli.command(help="Run the registrar daemon, attaching to a Redis queue")
+@click.option("--config-file", type=click.File("r"))
+@click.option("--validate", is_flag=True)
+@click.option("--replace", is_flag=True)
+@click.option("--host", type=str)
+@click.option("--port", type=int)
+@click.option("--listen-queue", type=str)
+@click.option("--deregister-queue", type=str)
+@click.option("--progress-set", type=str)
+@click.option("--failure-set", type=str)
+@click.option("--success-set", type=str)
+@click.option("--extra/-e", type=str, multiple=True, default=[])
+@click.option("--debug", is_flag=True)
+def daemon(
+ config_file=None,
+ validate=False,
+ replace=False,
+ host=None,
+ port=None,
+ listen_queue=None,
+ progress_set=None,
+ failure_set=None,
+ success_set=None,
+ deregister_queue=None,
+ extra=None,
+ debug=False,
+):
""" Run the registrar daemon to listen on the given queues
and execute the (de-)registrations commands.
@@ -96,25 +108,33 @@ def daemon(config_file=None, validate=False, replace=False, host=None,
handlers[deregister_queue] = deregister_item
for extra_handler in extra:
- queue, _, path = extra_handler.partition('=')
+ queue, _, path = extra_handler.partition("=")
handler = import_by_path(path.strip())
handlers[queue.strip()] = handler
run_daemon(
- config, replace, host, port, handlers, progress_set,
- failure_set, success_set, deregister_queue
+ config,
+ replace,
+ host,
+ port,
+ handlers,
+ progress_set,
+ failure_set,
+ success_set,
+ deregister_queue,
)
-@cli.command(help='Run a single, one-off registration')
-@click.argument('item', type=str)
-@click.option('--config-file', type=click.File('r'))
-@click.option('--is-path', is_flag=True)
-@click.option('--validate', is_flag=True)
-@click.option('--replace', is_flag=True)
-@click.option('--debug', is_flag=True)
-def register(item, is_path=False, config_file=None, validate=False,
- replace=False, debug=False):
+@cli.command(help="Run a single, one-off registration")
+@click.argument("item", type=str)
+@click.option("--config-file", type=click.File("r"))
+@click.option("--is-path", is_flag=True)
+@click.option("--validate", is_flag=True)
+@click.option("--replace", is_flag=True)
+@click.option("--debug", is_flag=True)
+def register(
+ item, is_path=False, config_file=None, validate=False, replace=False, debug=False
+):
setup_logging(debug)
config = load_config(config_file)
if validate:
@@ -126,14 +146,15 @@ def register(item, is_path=False, config_file=None, validate=False,
register_path(config, item, replace)
-@cli.command(help='Run a single, one-off de-registration')
-@click.argument('--path', type=str)
-@click.argument('--identifier', type=str)
-@click.option('--config-file', type=click.File('r'))
-@click.option('--validate', is_flag=True)
-@click.option('--debug', is_flag=True)
-def deregister(file_path=None, identifier=None, config_file=None,
- validate=False, debug=False):
+@cli.command(help="Run a single, one-off de-registration")
+@click.argument("--path", type=str)
+@click.argument("--identifier", type=str)
+@click.option("--config-file", type=click.File("r"))
+@click.option("--validate", is_flag=True)
+@click.option("--debug", is_flag=True)
+def deregister(
+ file_path=None, identifier=None, config_file=None, validate=False, debug=False
+):
setup_logging(debug)
config = load_config(config_file)
if validate:
@@ -145,5 +166,5 @@ def deregister(file_path=None, identifier=None, config_file=None,
deregister_identifier(config, identifier)
-if __name__ == '__main__':
+if __name__ == "__main__":
cli()
diff --git a/registrar/daemon.py b/registrar/daemon.py
index 878bb32..1ee95e6 100644
--- a/registrar/daemon.py
+++ b/registrar/daemon.py
@@ -19,10 +19,17 @@ logger = logging.getLogger(__name__)
QueueItemHandler = Callable[[dict, str, bool], None]
-def run_daemon(config: dict, replace: bool, host: str, port: int,
- handlers: Dict[str, QueueItemHandler],
- progress_set: str, failure_set: str, success_set: str):
- """ Run the registrar daemon, listening on a redis queue
+def run_daemon(
+ config: dict,
+ replace: bool,
+ host: str,
+ port: int,
+ handlers: Dict[str, QueueItemHandler],
+ progress_set: str,
+ failure_set: str,
+ success_set: str,
+):
+ """Run the registrar daemon, listening on a redis queue
for items to be (de-)registered.
Args:
@@ -41,9 +48,7 @@ def run_daemon(config: dict, replace: bool, host: str, port: int,
"""
# initialize the queue client
- client = redis.Redis(
- host=host, port=port, charset="utf-8", decode_responses=True
- )
+ client = redis.Redis(host=host, port=port, charset="utf-8", decode_responses=True)
queue_names = list(handlers.keys())
logger.debug(
f"waiting on items on redis "
diff --git a/registrar/post_handlers.py b/registrar/post_handlers.py
index 23ee9d3..9b10758 100644
--- a/registrar/post_handlers.py
+++ b/registrar/post_handlers.py
@@ -34,9 +34,7 @@ class ReportingPostHandler:
def __call__(self, config: dict, item: "Item"):
inserted = datetime.now()
timestamp = inserted.strftime("%Y%m%dT%H%M%S")
- sanitized_filename = self.sanitize_path(
- f'item_{timestamp}_{item.id}.xml'
- )
+ sanitized_filename = self.sanitize_path(f"item_{timestamp}_{item.id}.xml")
filename = os.path.join(self.reporting_dir, sanitized_filename)
logger.info(f"Generating report for product {item.id} at {filename}")
@@ -76,23 +74,6 @@ class ReportingPostHandler:
<Service>WMS</Service>
<URL>{wms_capabilities_url}</URL>
</URL>
-</DataAccessItem>\
-""".format(
- identifier=escape(item.id),
- availability_time=escape(isoformat(inserted)),
- wcs_capabilities_url=escape(
- '%s/ows?service=wcs&request=GetCapabilities'
- '&cql=identifier="%s_%s"'
- % (
- self.service_url, item.id, [
- i
- for i in item.assets.keys() if i != "gsc_metadata"
- ][0]
- )
- ),
- wms_capabilities_url=escape(
- '%s/ows?service=wms&request=GetCapabilities'
- '&cql=identifier="%s"'
- % (self.service_url, item.id)
- ),
- )))
+</DataAccessItem>"""
+ )
+ )
diff --git a/registrar/registrar.py b/registrar/registrar.py
index f4de201..6608887 100644
--- a/registrar/registrar.py
+++ b/registrar/registrar.py
@@ -90,9 +90,7 @@ def register_path(config: dict, path: str, replace: bool = False):
logger.info(f"Replacing '{path!r}'.")
backend.register_path(source, path, replace=True)
else:
- raise RegistrationError(
- f'Path {path!r} is already registered'
- )
+ raise RegistrationError(f"Path {path!r} is already registered")
else:
logger.info(f"Registering '{path!r}'.")
backend.register_path(source, path, replace=False)
@@ -105,8 +103,7 @@ def register_path(config: dict, path: str, replace: bool = False):
post_handler(config, path)
logger.info(
- f"Successfully {'replaced' if replace else 'registered'} "
- f"'{path!r}'"
+ f"Successfully {'replaced' if replace else 'registered'} " f"'{path!r}'"
)
@@ -121,8 +118,7 @@ def deregister_identifier(config, identifier: str, _: bool = False):
return _deregister_identifier(config, identifier)
-def _deregister_identifier(config, identifier: str,
- item: Optional[dict] = None):
+def _deregister_identifier(config, identifier: str, item: Optional[dict] = None):
for pre_handler in get_deregister_pre_handlers(config):
pre_handler(config, identifier, item)
@@ -187,15 +183,15 @@ def get_error_handlers(config):
def get_path_pre_handlers(config):
- return _get_handlers(config, 'path_pre_handlers')
+ return _get_handlers(config, "path_pre_handlers")
def get_path_post_handlers(config):
- return _get_handlers(config, 'path_post_handlers')
+ return _get_handlers(config, "path_post_handlers")
def get_path_error_handlers(config):
- return _get_handlers(config, 'path_error_handlers')
+ return _get_handlers(config, "path_error_handlers")
def get_deregister_pre_handlers(config):
@@ -207,16 +203,16 @@ def get_deregister_post_handlers(config):
def get_deregister_error_handlers(config):
- return _get_handlers(config, 'deregister_error_handlers')
+ return _get_handlers(config, "deregister_error_handlers")
def get_deregister_path_pre_handlers(config):
- return _get_handlers(config, 'deregister_path_pre_handlers')
+ return _get_handlers(config, "deregister_path_pre_handlers")
def get_deregister_path_post_handlers(config):
- return _get_handlers(config, 'deregister_path_post_handlers')
+ return _get_handlers(config, "deregister_path_post_handlers")
def get_deregister_path_error_handlers(config):
- return _get_handlers(config, 'deregister_path_error_handlers')
+ return _get_handlers(config, "deregister_path_error_handlers")
diff --git a/registrar/source.py b/registrar/source.py
index 2373b7f..5c6ea54 100644
--- a/registrar/source.py
+++ b/registrar/source.py
@@ -12,7 +12,6 @@ import shutil
from glob import glob
from fnmatch import fnmatch
import logging
-from urllib.parse import urlparse
from typing import TYPE_CHECKING, Optional
from abc import ABC, abstractmethod
@@ -36,13 +35,13 @@ class RegistrationError(Exception):
pass
-class Source:
+class Source(ABC):
def __init__(self, name: str = None, endpoint_url: str = ""):
self.name = name
self.endpoint_url = endpoint_url
def __repr__(self) -> str:
- return f'<{self.__class__.__name__} name={self.name}>'
+ return f"<{self.__class__.__name__} name={self.name}>"
@abstractmethod
def get_container_and_path(self, path: str):
@@ -382,10 +381,8 @@ def get_source(config: dict, item: "Item") -> Source:
# no source found
raise RegistrationError(f"Could not find a suitable source for {item!r}")
- return SOURCE_TYPES[cfg_source['type']](
- cfg_source['name'],
- *cfg_source.get('args', []),
- **cfg_source.get('kwargs', {})
+ return SOURCE_TYPES[cfg_source["type"]](
+ cfg_source["name"], *cfg_source.get("args", []), **cfg_source.get("kwargs", {})
)
@@ -402,20 +399,18 @@ def get_source_from_path(config, path: str) -> Optional[Source]:
Returns:
Source: source of the data
"""
- cfg_sources = config['sources']
+ cfg_sources = config["sources"]
for cfg_source in cfg_sources:
- if cfg_source.get('filter'):
- f = cfg_source.get('filter')
+ if cfg_source.get("filter"):
+ f = cfg_source.get("filter")
if any(filter(lambda item: re.match(f, item), path)):
break
else:
break
else:
# no source found
- raise RegistrationError(
- f'Could not find a suitable source for {path!r}'
- )
+ raise RegistrationError(f"Could not find a suitable source for {path!r}")
return SOURCE_TYPES[cfg_source["type"]](
cfg_source["name"], *cfg_source.get("args", []), **cfg_source.get("kwargs", {})
diff --git a/registrar/xml.py b/registrar/xml.py
index 571d2ae..e22fc2d 100644
--- a/registrar/xml.py
+++ b/registrar/xml.py
@@ -41,10 +41,7 @@ def parse_metadata_schema(
) -> dict:
out = {}
for key, param in schema.items():
- values = tree.xpath(
- param.xpath,
- namespaces=param.namespaces or namespaces
- )
+ values = tree.xpath(param.xpath, namespaces=param.namespaces or namespaces)
if param.multi:
value = [param.parser(v) if param.parser else v for v in values]
else:
--
GitLab