EOX GitLab Instance

Commit 6ea2d918 authored by Nikola Jankovic's avatar Nikola Jankovic 💻
Browse files

Merge branch 'postprocess' into 'main'

Postprocessing

See merge request !6
parents 26b364eb 1dee370c
Pipeline #19507 passed with stages
in 1 minute and 58 seconds
......@@ -10,7 +10,7 @@ from .endpoint import get_endpoint
from .source import get_source
from .exceptions import HarvestError
from .utils import cql_filter
from .postprocess import get_postprocessor
logger = logging.getLogger(__name__)
......@@ -52,6 +52,13 @@ def main(config: dict, value: str, client: Redis):
# Perform harvest
result = resource.harvest()
if "postprocess" in harvest_config:
postprocessor = get_postprocessor(harvest_config["postprocess"])
result = (
postprocessor.postprocess(item)
for item in result
)
# Filter data
result = cql_filter(harvest_config["filter"], result)
......
from abc import ABC, abstractmethod
from typing import Dict, Type
class Postprocessor(ABC):
def __init__(self, **kwargs):
...
@abstractmethod
def postprocess(self, item: dict) -> dict:
pass
POSTPROCESSORS: Dict[str, Type[Postprocessor]] = {
}
def get_postprocessor(config: dict) -> Postprocessor:
cls = POSTPROCESSORS[config.pop("type")]
return cls(**config)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment