EOX GitLab Instance

Commit a9d70139 authored by Fabian Schindler's avatar Fabian Schindler
Browse files

Moving Postprocessor ABC to abc

Adding docs to ABCs
parent f77fd158
Pipeline #19542 passed with stages
in 2 minutes
......@@ -12,30 +12,76 @@ class Resource(ABC):
@abstractmethod
def harvest(self) -> Iterator[dict]:
"""
Starts the harvesting of the resource, returning an iterator of the
harvested items.
"""
...
@dataclass
class Stat:
"""
Rudimentary file metadata
"""
mtime: datetime
size: int
class Source(ABC):
"""
Remote file-system abstraction to list, open and
get metadata for files.
"""
@abstractmethod
def open(self, path: str) -> IO[AnyStr]:
"""
Open a file on the source identified by a path.
Returns a Python file-like object to directly read from.
"""
...
@abstractmethod
def listdir(self, path) -> List[str]:
def listdir(self, path: str) -> List[str]:
"""
Lists the filenames of the storage for the given base path (or prefix).
The returned paths include the given base path and are not relative to it.
"""
...
@abstractmethod
def stat(self, path: str) -> Stat:
"""
Returns the file metadata for the specified file
"""
...
class Endpoint(Resource):
"""
Endpoints are resources that use a search protocol (or something similar)
to harvest items. Thus, they are always associated with a specific URL.
"""
def __init__(self, url: str):
self.url = url
class FileScheme(Resource):
"""
FileSchemes are resources that operate on a file basis on a given file source.
"""
def __init__(self, source: Source):
self.source = source
class Postprocessor(ABC):
"""
Postprocessors can alter all harvested items.
"""
def __init__(self, **kwargs):
...
@abstractmethod
def postprocess(self, item: dict) -> dict:
...
from abc import ABC, abstractmethod
from typing import Dict, Type
from .abc import Postprocessor
from .utils import import_by_path
class Postprocessor(ABC):
def __init__(self, **kwargs):
...
@abstractmethod
def postprocess(self, item: dict) -> dict:
pass
POSTPROCESSORS: Dict[str, Type[Postprocessor]] = {
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment