EOX GitLab Instance

Commit 0cf5787a authored by Nikola Jankovic's avatar Nikola Jankovic 💻
Browse files

improved tests

improved s3 harvesting
parent 27f96bd0
Pipeline #18821 passed with stage
in 44 seconds
......@@ -19,7 +19,7 @@ def stringify(
):
encoded: List[str] = []
if mode == "item":
encoded.extend((json.dumps(item) for item in result))
encoded.extend((json.dumps(item, default=str) for item in result))
elif mode == "property":
encoded.extend((item["properties"][extract_property] for item in result))
......
......@@ -6,8 +6,8 @@ from typing import TYPE_CHECKING
if TYPE_CHECKING:
from mypy_boto3_s3.client import S3Client
import boto3
import botocore
import boto3.session
import botocore.session
import pystac
from ._source import Source
......@@ -59,8 +59,8 @@ class S3Source(Source):
def _create_item(self, data, dt, url):
identifier = dt.strftime("%Y%m%d_%H%M%S")
properties = {
"datetime": dt.isoformat(),
"updated": data["LastModified"].isoformat(),
"datetime": dt,
"updated": data["LastModified"],
}
item = pystac.Item(
id=identifier, geometry=None, bbox=None, datetime=dt, properties=properties
......
pytest
pytest-cov
pytest-mock
requests-mock
import json
import pytest
import pathlib
import redis
import pytest_mock
import requests_mock
from harvester.config import load_config
CFG_PATH = pathlib.Path(__file__).parent / "data" / "config.yml"
DATA_PATH = pathlib.Path(__file__).parent / "data"
def json_load(path: pathlib.Path) -> dict:
with open(path, "r") as f:
return json.load(f)
def xml_load(path: pathlib.Path) -> str:
with open(path, "r") as f:
return f.read()
@pytest.fixture
def config():
with open(CFG_PATH) as conf:
with open(DATA_PATH / "config.yml") as conf:
config = load_config(conf)
return config
......@@ -18,3 +31,46 @@ def config():
@pytest.fixture
def client(mocker: pytest_mock.MockerFixture):
return mocker.patch.object(redis, "Redis")
@pytest.fixture()
def requests_mocker():
with requests_mock.Mocker() as requests_mocker:
yield requests_mocker
@pytest.fixture()
def data_map():
data = {
"S2L2A_Element84": [
(
"https://earth-search.aws.element84.com/v0/",
json_load(DATA_PATH / "stac_e84_root.json"),
),
(
"""https://earth-search.aws.element84.com/v0/search?datetime=2019-09-10T00%3A00%3A00%2B00%3A00%2F2019-09-11T00%3A00%3A00%2B00%3A00&limit=100&page=1&bbox=%5B14.9%2C47.7%2C16.4%2C48.7%5D&collections=%5B%22sentinel-s2-l2a-cogs%22%5D""",
json_load(DATA_PATH / "stac_e84_data.json"),
),
(
"https://earth-search.aws.element84.com/v0/search?datetime=2019-09-10T00%3A00%3A00%2B00%3A00%2F2019-09-11T00%3A00%3A00%2B00%3A00&limit=100&page=2&bbox=%5B14.9%2C47.7%2C16.4%2C48.7%5D&collections=%5B%22sentinel-s2-l2a-cogs%22%5D",
json_load(DATA_PATH / "stac_e84_empty.json"),
),
],
"Creodias-Opensearch": {
"describe": (
"https://finder.creodias.eu/resto/api/collections/Sentinel2/describe.xml",
xml_load(DATA_PATH / "opensearch_creodias_describe.xml"),
),
"data": (
"https://finder.creodias.eu/resto/api/collections/Sentinel2/search.json?box=14.9%2C47.7%2C16.4%2C48.7&startDate=2019-09-10T00%3A00%3A00%2B00%3A00&completionDate=2019-09-11T00%3A00%3A00%2B00%3A00&maxRecords=2000&index=1",
json_load(DATA_PATH / "opensearch_creodias_data.json"),
),
"empty": (
"https://finder.creodias.eu/resto/api/collections/Sentinel2/search.json?box=14.9%2C47.7%2C16.4%2C48.7&startDate=2019-09-10T00%3A00%3A00%2B00%3A00&completionDate=2019-09-11T00%3A00%3A00%2B00%3A00&maxRecords=2000&index=2001",
json_load(DATA_PATH / "opensearch_creodias_empty.json"),
),
},
"Fusion-data": json_load(DATA_PATH / "s3_fusion_data.json"),
}
return data
......@@ -31,16 +31,7 @@ harvesters:
file_regex: \d{4}-\d{2}-\d{2}.vrt
time_regex: \d{4}-\d{2}-\d{2}
time_format: "%Y-%m-%d"
filter:
and:
- during:
- property: datetime
- - "2020-06-01T00:00:00Z"
- "2020-06-30T00:00:00Z"
- during:
- property: updated
- - P5D
- !now
filter: null
mode: item
queue: register
- name: Opensearch-composites-eox
......
This diff is collapsed.
This diff is collapsed.
{
"type": "FeatureCollection",
"properties": {
"id": "39685cf5-b9b0-5337-ac41-fc7817a814c7",
"totalResults": 0,
"exactCount": true,
"startIndex": 1,
"itemsPerPage": 0,
"query": {
"originalFilters": {
"box": "14.9,47.7,16.4,48.7",
"startDate": "2010-09-10T00:00:00",
"completionDate": "2010-09-11T00:00:00",
"maxRecords": "20",
"index": "1",
"status": 0,
"collection": "Sentinel2"
},
"appliedFilters": {
"box": "14.9,47.7,16.4,48.7",
"startDate": "2010-09-10T00:00:00",
"completionDate": "2010-09-11T00:00:00",
"maxRecords": "20",
"index": "1",
"status": 0,
"collection": "Sentinel2"
},
"analysis": {
"query": null,
"language": "en",
"analyze": {
"What": [],
"When": [],
"Where": [],
"Errors": [],
"Explained": []
},
"processingTime": 0.0000021457672119141
},
"processingTime": 0.018907070159912
},
"links": [{
"rel": "self",
"type": "application/json",
"title": "self",
"href": "https://finder.creodias.eu/resto/api/collections/Sentinel2/search.json?&box=14.9%2C47.7%2C16.4%2C48.7&startDate=2010-09-10T00%3A00%3A00&completionDate=2010-09-11T00%3A00%3A00&maxRecords=20&index=1"
},
{
"rel": "search",
"type": "application/opensearchdescription+xml",
"title": "OpenSearch Description Document",
"href": "https://finder.creodias.eu/resto/api/collections/Sentinel2/describe.xml"
}
]
},
"features": []
}
[{
"Contents": [{
"Key": "'vrt/SR/2019-03-15.vrt'",
"LastModified": "2019-03-15"
},
{
"Key": "'vrt/SR/2019-03-16.vrt'",
"LastModified": "2019-03-16"
},
{
"Key": "'vrt/SR/2019-03-17.vrt'",
"LastModified": "2019-03-17"
},
{
"Key": "'vrt/SR/2019-03-18.vrt'",
"LastModified": "2019-03-18"
}
]
}]
This diff is collapsed.
{
"type": "FeatureCollection",
"stac_version": "mock",
"stac_extensions": [],
"context": {
"page": 1,
"limit": 10,
"matched": 0,
"returned": 0
},
"numberMatched": 0,
"numberReturned": 0,
"features": [{}]
}
{
"stac_version": "mock",
"stac_api_version": "mock",
"id": "earth-search",
"title": "Earth Search",
"description": "A STAC API of AWS Public Datasets powered by stac-server",
"links": [{
"rel": "child",
"href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a"
},
{
"rel": "child",
"href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l1c"
},
{
"rel": "child",
"href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs"
},
{
"rel": "child",
"href": "https://earth-search.aws.element84.com/v0/collections/landsat-8-l1-c1"
},
{
"rel": "service-desc",
"type": "application/vnd.oai.openapi+json;version=3.0",
"href": "https://earth-search.aws.element84.com/v0/api"
},
{
"rel": "conformance",
"type": "application/json",
"href": "https://earth-search.aws.element84.com/v0/conformance"
},
{
"rel": "children",
"type": "application/json",
"href": "https://earth-search.aws.element84.com/v0/collections"
},
{
"rel": "self",
"type": "application/json",
"href": "https://earth-search.aws.element84.com/v0/"
},
{
"rel": "search",
"type": "application/json",
"href": "https://earth-search.aws.element84.com/v0/search"
},
{
"rel": "docs",
"href": "https://stac-utils.github.io/stac-server/"
}
]
}
import pytest
import requests_mock
import pytest_mock
import unittest.mock
from harvester.harvester import main
@pytest.mark.parametrize("value", [("S2L2A_Element84")])
def test_stacapi(config: dict, client, value: str):
def test_stacapi(
value: str,
requests_mocker: requests_mock.Mocker,
config: dict,
data_map: dict,
client: unittest.mock.MagicMock,
):
mock_data = data_map[value]
for url, mock_json in mock_data:
requests_mocker.get(url=url, json=mock_json)
main(config, value, client)
client.lpush.assert_called_once()
@pytest.mark.parametrize("value", [("Creodias-Opensearch")])
def test_opensearch(config: dict, client, value: str):
def test_opensearch(
value: str,
requests_mocker: requests_mock.Mocker,
config: dict,
data_map: dict,
client: unittest.mock.MagicMock,
):
mock_data = data_map[value]
requests_mocker.get(url=mock_data["describe"][0], text=mock_data["describe"][1])
requests_mocker.get(url=mock_data["data"][0], json=mock_data["data"][1])
requests_mocker.get(url=mock_data["empty"][0], json=mock_data["empty"][1])
main(config, value, client)
client.lpush.assert_called_once()
@pytest.mark.parametrize("value", [("Fusion-data")])
def test_s3(config: dict, client, value: str):
def test_s3(
value: str,
mocker: pytest_mock.MockFixture,
config: dict,
data_map: dict,
client: unittest.mock.MagicMock,
):
mock_data = data_map[value]
session_mock = mocker.patch("harvester.source.s3.boto3.session.Session")
session_mock().client().get_paginator().paginate.return_value = mock_data
main(config, value, client)
client.lpush.assert_called_once()
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment