EOX GitLab Instance

Commit e65be197 authored by Fabian Schindler's avatar Fabian Schindler
Browse files

Adding postprocessing capabilities

parent 26b364eb
Pipeline #19505 failed with stage
in 45 seconds
...@@ -10,7 +10,7 @@ from .endpoint import get_endpoint ...@@ -10,7 +10,7 @@ from .endpoint import get_endpoint
from .source import get_source from .source import get_source
from .exceptions import HarvestError from .exceptions import HarvestError
from .utils import cql_filter from .utils import cql_filter
from .postprocess import get_postprocessor
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
...@@ -52,6 +52,13 @@ def main(config: dict, value: str, client: Redis): ...@@ -52,6 +52,13 @@ def main(config: dict, value: str, client: Redis):
# Perform harvest # Perform harvest
result = resource.harvest() result = resource.harvest()
if "postprocess" in harvest_config:
postprocessor = get_postprocessor(harvest_config["postprocess"])
result = (
postprocessor.postprocess(item)
for item in result
)
# Filter data # Filter data
result = cql_filter(harvest_config["filter"], result) result = cql_filter(harvest_config["filter"], result)
......
from abc import ABC, abstractmethod
class Postprocessor(ABC):
@abstractmethod
def postprocess(self, item: dict) -> dict:
pass
POSTPROCESSORS = {
}
def get_postprocessor(config: dict) -> Postprocessor:
cls = POSTPROCESSORS[config.pop("type")]
return cls(**config)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment