EOX GitLab Instance

Commit 94d2f47c authored by Fabian Schindler's avatar Fabian Schindler
Browse files

Fixing wrong use of Generator with more generic Iterator

parent 6df05bb6
Pipeline #19408 failed with stages
in 39 seconds
from typing import Generator
from typing import Iterator
from ._endpoint import Endpoint
class OADSEndpoint(Endpoint):
type = "OADS"
def harvest(self) -> Generator[dict, None, None]:
def harvest(self) -> Iterator[dict]:
raise NotImplementedError()
from typing import Generator
from typing import Iterator
from ._endpoint import Endpoint
class OGCAPIEndpoint(Endpoint):
type = "OGCAPI"
def harvest(self) -> Generator[dict, None, None]:
def harvest(self) -> Iterator[dict]:
raise NotImplementedError()
from abc import abstractmethod, ABC
import logging
from typing import Any, Dict, Generator, List
from typing import Any, Dict, Iterator, List
from dataclasses import dataclass, field
import requests
......@@ -127,7 +127,7 @@ class OpenSearchEndpoint(Endpoint):
self.format = get_format(format_config)
self.query = OpenSearchQuery(**query)
def harvest(self) -> Generator[dict, None, None]:
def harvest(self) -> Iterator[dict]:
logger.info("Starting OpenSearch harvesting")
parser = ET.XMLParser(recover=True)
......
import logging
from typing import Generator
from typing import Iterator
import requests
......@@ -16,7 +16,7 @@ class STACAPIEndpoint(Endpoint):
super().__init__(*args, **kwargs)
self.query = StacAPIQuery(**query)
def harvest(self) -> Generator[dict, None, None]:
def harvest(self) -> Iterator[dict]:
logger.info("Starting STACAPI harvesting")
main = requests.get(self.url).json()
search_url = next(
......
import json
import logging
from typing import Generator, Optional
from typing import Iterator, Optional
from redis import Redis
......@@ -15,10 +15,10 @@ logger = logging.getLogger(__name__)
def stringify(
result: Generator[dict, None, None],
result: Iterator[dict],
mode: str = "item",
extract_property: Optional[str] = None,
) -> Generator[str, None, None]:
) -> Iterator[dict]:
if mode == "item":
yield from (json.dumps(item, default=str) for item in result)
elif mode == "property":
......
from abc import abstractmethod, ABC
from typing import Generator
from typing import Iterator
class Resource(ABC):
......@@ -15,5 +15,5 @@ class Resource(ABC):
# 4. return list of stac items as dictionaries
@abstractmethod
def harvest(self) -> Generator[dict, None, None]:
def harvest(self) -> Iterator[dict]:
pass
from typing import Generator
from typing import Iterator
from ._source import Source
class FTPSource(Source):
type = "FTP"
def harvest(self) -> Generator[dict, None, None]:
def harvest(self) -> Iterator[dict]:
raise NotImplementedError()
import re
import logging
from dateutil.parser import isoparse
from typing import TYPE_CHECKING, Generator, Iterator, Optional, Tuple
from typing import TYPE_CHECKING, Iterator, Tuple
from functools import cached_property
from urllib.parse import urlparse
import json
......@@ -87,7 +87,7 @@ class S3Source(Source):
bucket = self.parameters["url"].strip("https://").split(".")[0]
return bucket
def harvest(self) -> Generator[dict, None, None]:
def harvest(self) -> Iterator[dict]:
logger.info("Starting S3 harvesting")
paginator = self.client.get_paginator("list_objects_v2")
......
from typing import Generator
from typing import Iterator
from ._source import Source
class STACCatalogSource(Source):
type = "STACCatalog"
def harvest(self) -> Generator[dict, None, None]:
def harvest(self) -> Iterator[dict]:
raise NotImplementedError()
from typing import Generator
from typing import Iterator
from ._source import Source
class SwiftSource(Source):
type = "Swift"
def harvest(self) -> Generator[dict, None, None]:
def harvest(self) -> Iterator[dict]:
raise NotImplementedError()
from typing import Generator
from typing import Iterator
from pygeofilter.backends.native.evaluate import NativeEvaluator
from pygeofilter.parsers.cql_json import parse as json_parse
def cql_filter(
_filter: dict, data: Generator[dict, None, None]
) -> Generator[dict, None, None]:
def cql_filter(_filter: dict, data: Iterator[dict]) -> Iterator[dict]:
if not _filter:
yield from data
return
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment