EOX GitLab Instance

Commit ae50ad02 authored by Fabian Schindler's avatar Fabian Schindler
Browse files

Merge branch 'main' into restructure-resources

parents dfa3cdd8 6ea2d918
......@@ -9,7 +9,7 @@ from .endpoint import get_endpoint
from .filescheme import get_filescheme
from .exceptions import HarvestError
from .utils import cql_filter
from .postprocess import get_postprocessor
logger = logging.getLogger(__name__)
......@@ -51,6 +51,13 @@ def main(config: dict, value: str, client: Redis):
# Perform harvest
result = resource.harvest()
if "postprocess" in harvest_config:
postprocessor = get_postprocessor(harvest_config["postprocess"])
result = (
postprocessor.postprocess(item)
for item in result
)
# Filter data
result = cql_filter(harvest_config["filter"], result)
......
from abc import ABC, abstractmethod
from typing import Dict, Type
class Postprocessor(ABC):
def __init__(self, **kwargs):
...
@abstractmethod
def postprocess(self, item: dict) -> dict:
pass
POSTPROCESSORS: Dict[str, Type[Postprocessor]] = {
}
def get_postprocessor(config: dict) -> Postprocessor:
cls = POSTPROCESSORS[config.pop("type")]
return cls(**config)
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment