improved tests

improved s3 harvesting

harvester/harvester.py

@@ -19,7 +19,7 @@ def stringify(
 ):
     encoded: List[str] = []
     if mode == "item":
-        encoded.extend((json.dumps(item) for item in result))
+        encoded.extend((json.dumps(item, default=str) for item in result))
     elif mode == "property":
         encoded.extend((item["properties"][extract_property] for item in result))
 

harvester/source/s3.py

@@ -6,8 +6,8 @@ from typing import TYPE_CHECKING
 if TYPE_CHECKING:
     from mypy_boto3_s3.client import S3Client
 
-import boto3
-import botocore
+import boto3.session
+import botocore.session
 import pystac
 
 from ._source import Source
@@ -59,8 +59,8 @@ class S3Source(Source):
     def _create_item(self, data, dt, url):
         identifier = dt.strftime("%Y%m%d_%H%M%S")
         properties = {
-            "datetime": dt.isoformat(),
-            "updated": data["LastModified"].isoformat(),
+            "datetime": dt,
+            "updated": data["LastModified"],
         }
         item = pystac.Item(
             id=identifier, geometry=None, bbox=None, datetime=dt, properties=properties

requirements-test.txt

 pytest
 pytest-cov
 pytest-mock
+requests-mock

tests/conftest.py

+import json
+
 import pytest
 import pathlib
 import redis
 import pytest_mock
+import requests_mock
 
 from harvester.config import load_config
 
-CFG_PATH = pathlib.Path(__file__).parent / "data" / "config.yml"
+DATA_PATH = pathlib.Path(__file__).parent / "data"
+
+
+def json_load(path: pathlib.Path) -> dict:
+    with open(path, "r") as f:
+        return json.load(f)
+
+
+def xml_load(path: pathlib.Path) -> str:
+    with open(path, "r") as f:
+        return f.read()
 
 
 @pytest.fixture
 def config():
-    with open(CFG_PATH) as conf:
+    with open(DATA_PATH / "config.yml") as conf:
         config = load_config(conf)
     return config
 
@@ -18,3 +31,46 @@ def config():
 @pytest.fixture
 def client(mocker: pytest_mock.MockerFixture):
     return mocker.patch.object(redis, "Redis")
+
+
+@pytest.fixture()
+def requests_mocker():
+    with requests_mock.Mocker() as requests_mocker:
+        yield requests_mocker
+
+
+@pytest.fixture()
+def data_map():
+    data = {
+        "S2L2A_Element84": [
+            (
+                "https://earth-search.aws.element84.com/v0/",
+                json_load(DATA_PATH / "stac_e84_root.json"),
+            ),
+            (
+                """https://earth-search.aws.element84.com/v0/search?datetime=2019-09-10T00%3A00%3A00%2B00%3A00%2F2019-09-11T00%3A00%3A00%2B00%3A00&limit=100&page=1&bbox=%5B14.9%2C47.7%2C16.4%2C48.7%5D&collections=%5B%22sentinel-s2-l2a-cogs%22%5D""",
+                json_load(DATA_PATH / "stac_e84_data.json"),
+            ),
+            (
+                "https://earth-search.aws.element84.com/v0/search?datetime=2019-09-10T00%3A00%3A00%2B00%3A00%2F2019-09-11T00%3A00%3A00%2B00%3A00&limit=100&page=2&bbox=%5B14.9%2C47.7%2C16.4%2C48.7%5D&collections=%5B%22sentinel-s2-l2a-cogs%22%5D",
+                json_load(DATA_PATH / "stac_e84_empty.json"),
+            ),
+        ],
+        "Creodias-Opensearch": {
+            "describe": (
+                "https://finder.creodias.eu/resto/api/collections/Sentinel2/describe.xml",
+                xml_load(DATA_PATH / "opensearch_creodias_describe.xml"),
+            ),
+            "data": (
+                "https://finder.creodias.eu/resto/api/collections/Sentinel2/search.json?box=14.9%2C47.7%2C16.4%2C48.7&startDate=2019-09-10T00%3A00%3A00%2B00%3A00&completionDate=2019-09-11T00%3A00%3A00%2B00%3A00&maxRecords=2000&index=1",
+                json_load(DATA_PATH / "opensearch_creodias_data.json"),
+            ),
+            "empty": (
+                "https://finder.creodias.eu/resto/api/collections/Sentinel2/search.json?box=14.9%2C47.7%2C16.4%2C48.7&startDate=2019-09-10T00%3A00%3A00%2B00%3A00&completionDate=2019-09-11T00%3A00%3A00%2B00%3A00&maxRecords=2000&index=2001",
+                json_load(DATA_PATH / "opensearch_creodias_empty.json"),
+            ),
+        },
+        "Fusion-data": json_load(DATA_PATH / "s3_fusion_data.json"),
+    }
+
+    return data

tests/data/config.yml

@@ -31,16 +31,7 @@ harvesters:
         file_regex: \d{4}-\d{2}-\d{2}.vrt
         time_regex: \d{4}-\d{2}-\d{2}
         time_format: "%Y-%m-%d"
-    filter:
-      and:
-        - during:
-            - property: datetime
-            - - "2020-06-01T00:00:00Z"
-              - "2020-06-30T00:00:00Z"
-        - during:
-            - property: updated
-            - - P5D
-              - !now
+    filter: null
     mode: item
     queue: register
   - name: Opensearch-composites-eox

tests/data/opensearch_creodias_empty.json

+{
+    "type": "FeatureCollection",
+    "properties": {
+        "id": "39685cf5-b9b0-5337-ac41-fc7817a814c7",
+        "totalResults": 0,
+        "exactCount": true,
+        "startIndex": 1,
+        "itemsPerPage": 0,
+        "query": {
+            "originalFilters": {
+                "box": "14.9,47.7,16.4,48.7",
+                "startDate": "2010-09-10T00:00:00",
+                "completionDate": "2010-09-11T00:00:00",
+                "maxRecords": "20",
+                "index": "1",
+                "status": 0,
+                "collection": "Sentinel2"
+            },
+            "appliedFilters": {
+                "box": "14.9,47.7,16.4,48.7",
+                "startDate": "2010-09-10T00:00:00",
+                "completionDate": "2010-09-11T00:00:00",
+                "maxRecords": "20",
+                "index": "1",
+                "status": 0,
+                "collection": "Sentinel2"
+            },
+            "analysis": {
+                "query": null,
+                "language": "en",
+                "analyze": {
+                    "What": [],
+                    "When": [],
+                    "Where": [],
+                    "Errors": [],
+                    "Explained": []
+                },
+                "processingTime": 0.0000021457672119141
+            },
+            "processingTime": 0.018907070159912
+        },
+        "links": [{
+                "rel": "self",
+                "type": "application/json",
+                "title": "self",
+                "href": "https://finder.creodias.eu/resto/api/collections/Sentinel2/search.json?&box=14.9%2C47.7%2C16.4%2C48.7&startDate=2010-09-10T00%3A00%3A00&completionDate=2010-09-11T00%3A00%3A00&maxRecords=20&index=1"
+            },
+            {
+                "rel": "search",
+                "type": "application/opensearchdescription+xml",
+                "title": "OpenSearch Description Document",
+                "href": "https://finder.creodias.eu/resto/api/collections/Sentinel2/describe.xml"
+            }
+        ]
+    },
+    "features": []
+}

tests/data/s3_fusion_data.json

+[{
+    "Contents": [{
+            "Key": "'vrt/SR/2019-03-15.vrt'",
+            "LastModified": "2019-03-15"
+        },
+        {
+            "Key": "'vrt/SR/2019-03-16.vrt'",
+            "LastModified": "2019-03-16"
+        },
+        {
+            "Key": "'vrt/SR/2019-03-17.vrt'",
+            "LastModified": "2019-03-17"
+        },
+        {
+            "Key": "'vrt/SR/2019-03-18.vrt'",
+            "LastModified": "2019-03-18"
+        }
+    ]
+}]

tests/data/stac_e84_empty.json

+{
+    "type": "FeatureCollection",
+    "stac_version": "mock",
+    "stac_extensions": [],
+    "context": {
+        "page": 1,
+        "limit": 10,
+        "matched": 0,
+        "returned": 0
+    },
+    "numberMatched": 0,
+    "numberReturned": 0,
+    "features": [{}]
+}

tests/data/stac_e84_root.json

+{
+    "stac_version": "mock",
+    "stac_api_version": "mock",
+    "id": "earth-search",
+    "title": "Earth Search",
+    "description": "A STAC API of AWS Public Datasets powered by stac-server",
+    "links": [{
+            "rel": "child",
+            "href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a"
+        },
+        {
+            "rel": "child",
+            "href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l1c"
+        },
+        {
+            "rel": "child",
+            "href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs"
+        },
+        {
+            "rel": "child",
+            "href": "https://earth-search.aws.element84.com/v0/collections/landsat-8-l1-c1"
+        },
+        {
+            "rel": "service-desc",
+            "type": "application/vnd.oai.openapi+json;version=3.0",
+            "href": "https://earth-search.aws.element84.com/v0/api"
+        },
+        {
+            "rel": "conformance",
+            "type": "application/json",
+            "href": "https://earth-search.aws.element84.com/v0/conformance"
+        },
+        {
+            "rel": "children",
+            "type": "application/json",
+            "href": "https://earth-search.aws.element84.com/v0/collections"
+        },
+        {
+            "rel": "self",
+            "type": "application/json",
+            "href": "https://earth-search.aws.element84.com/v0/"
+        },
+        {
+            "rel": "search",
+            "type": "application/json",
+            "href": "https://earth-search.aws.element84.com/v0/search"
+        },
+        {
+            "rel": "docs",
+            "href": "https://stac-utils.github.io/stac-server/"
+        }
+    ]
+}

tests/test_main.py

 import pytest
+import requests_mock
+import pytest_mock
+import unittest.mock
 
 from harvester.harvester import main
 
 
 @pytest.mark.parametrize("value", [("S2L2A_Element84")])
-def test_stacapi(config: dict, client, value: str):
+def test_stacapi(
+    value: str,
+    requests_mocker: requests_mock.Mocker,
+    config: dict,
+    data_map: dict,
+    client: unittest.mock.MagicMock,
+):
+    mock_data = data_map[value]
+    for url, mock_json in mock_data:
+        requests_mocker.get(url=url, json=mock_json)
+
     main(config, value, client)
+    client.lpush.assert_called_once()
 
 
 @pytest.mark.parametrize("value", [("Creodias-Opensearch")])
-def test_opensearch(config: dict, client, value: str):
+def test_opensearch(
+    value: str,
+    requests_mocker: requests_mock.Mocker,
+    config: dict,
+    data_map: dict,
+    client: unittest.mock.MagicMock,
+):
+    mock_data = data_map[value]
+    requests_mocker.get(url=mock_data["describe"][0], text=mock_data["describe"][1])
+    requests_mocker.get(url=mock_data["data"][0], json=mock_data["data"][1])
+    requests_mocker.get(url=mock_data["empty"][0], json=mock_data["empty"][1])
+
     main(config, value, client)
+    client.lpush.assert_called_once()
 
 
 @pytest.mark.parametrize("value", [("Fusion-data")])
-def test_s3(config: dict, client, value: str):
+def test_s3(
+    value: str,
+    mocker: pytest_mock.MockFixture,
+    config: dict,
+    data_map: dict,
+    client: unittest.mock.MagicMock,
+):
+    mock_data = data_map[value]
+    session_mock = mocker.patch("harvester.source.s3.boto3.session.Session")
+    session_mock().client().get_paginator().paginate.return_value = mock_data
+
     main(config, value, client)
+    client.lpush.assert_called_once()