EOX GitLab Instance

Commit e65be197 authored by Fabian Schindler's avatar Fabian Schindler
Browse files

Adding postprocessing capabilities

parent 26b364eb
Pipeline #19505 failed with stage
in 45 seconds
......@@ -10,7 +10,7 @@ from .endpoint import get_endpoint
from .source import get_source
from .exceptions import HarvestError
from .utils import cql_filter
from .postprocess import get_postprocessor
logger = logging.getLogger(__name__)
......@@ -52,6 +52,13 @@ def main(config: dict, value: str, client: Redis):
# Perform harvest
result = resource.harvest()
if "postprocess" in harvest_config:
postprocessor = get_postprocessor(harvest_config["postprocess"])
result = (
for item in result
# Filter data
result = cql_filter(harvest_config["filter"], result)
from abc import ABC, abstractmethod
class Postprocessor(ABC):
def postprocess(self, item: dict) -> dict:
def get_postprocessor(config: dict) -> Postprocessor:
cls = POSTPROCESSORS[config.pop("type")]
return cls(**config)
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment