EOX GitLab Instance

Commit 0cf5787a authored by Nikola Jankovic's avatar Nikola Jankovic 💻
Browse files

improved tests

improved s3 harvesting
parent 27f96bd0
Pipeline #18821 passed with stage
in 44 seconds
......@@ -19,7 +19,7 @@ def stringify(
):
encoded: List[str] = []
if mode == "item":
encoded.extend((json.dumps(item) for item in result))
encoded.extend((json.dumps(item, default=str) for item in result))
elif mode == "property":
encoded.extend((item["properties"][extract_property] for item in result))
......
......@@ -6,8 +6,8 @@ from typing import TYPE_CHECKING
if TYPE_CHECKING:
from mypy_boto3_s3.client import S3Client
import boto3
import botocore
import boto3.session
import botocore.session
import pystac
from ._source import Source
......@@ -59,8 +59,8 @@ class S3Source(Source):
def _create_item(self, data, dt, url):
identifier = dt.strftime("%Y%m%d_%H%M%S")
properties = {
"datetime": dt.isoformat(),
"updated": data["LastModified"].isoformat(),
"datetime": dt,
"updated": data["LastModified"],
}
item = pystac.Item(
id=identifier, geometry=None, bbox=None, datetime=dt, properties=properties
......
pytest
pytest-cov
pytest-mock
requests-mock
import json
import pytest
import pathlib
import redis
import pytest_mock
import requests_mock
from harvester.config import load_config
CFG_PATH = pathlib.Path(__file__).parent / "data" / "config.yml"
DATA_PATH = pathlib.Path(__file__).parent / "data"
def json_load(path: pathlib.Path) -> dict:
with open(path, "r") as f:
return json.load(f)
def xml_load(path: pathlib.Path) -> str:
with open(path, "r") as f:
return f.read()
@pytest.fixture
def config():
with open(CFG_PATH) as conf:
with open(DATA_PATH / "config.yml") as conf:
config = load_config(conf)
return config
......@@ -18,3 +31,46 @@ def config():
@pytest.fixture
def client(mocker: pytest_mock.MockerFixture):
return mocker.patch.object(redis, "Redis")
@pytest.fixture()
def requests_mocker():
with requests_mock.Mocker() as requests_mocker:
yield requests_mocker
@pytest.fixture()
def data_map():
data = {
"S2L2A_Element84": [
(
"https://earth-search.aws.element84.com/v0/",
json_load(DATA_PATH / "stac_e84_root.json"),
),
(
"""https://earth-search.aws.element84.com/v0/search?datetime=2019-09-10T00%3A00%3A00%2B00%3A00%2F2019-09-11T00%3A00%3A00%2B00%3A00&limit=100&page=1&bbox=%5B14.9%2C47.7%2C16.4%2C48.7%5D&collections=%5B%22sentinel-s2-l2a-cogs%22%5D""",
json_load(DATA_PATH / "stac_e84_data.json"),
),
(
"https://earth-search.aws.element84.com/v0/search?datetime=2019-09-10T00%3A00%3A00%2B00%3A00%2F2019-09-11T00%3A00%3A00%2B00%3A00&limit=100&page=2&bbox=%5B14.9%2C47.7%2C16.4%2C48.7%5D&collections=%5B%22sentinel-s2-l2a-cogs%22%5D",
json_load(DATA_PATH / "stac_e84_empty.json"),
),
],
"Creodias-Opensearch": {
"describe": (
"https://finder.creodias.eu/resto/api/collections/Sentinel2/describe.xml",
xml_load(DATA_PATH / "opensearch_creodias_describe.xml"),
),
"data": (
"https://finder.creodias.eu/resto/api/collections/Sentinel2/search.json?box=14.9%2C47.7%2C16.4%2C48.7&startDate=2019-09-10T00%3A00%3A00%2B00%3A00&completionDate=2019-09-11T00%3A00%3A00%2B00%3A00&maxRecords=2000&index=1",
json_load(DATA_PATH / "opensearch_creodias_data.json"),
),
"empty": (
"https://finder.creodias.eu/resto/api/collections/Sentinel2/search.json?box=14.9%2C47.7%2C16.4%2C48.7&startDate=2019-09-10T00%3A00%3A00%2B00%3A00&completionDate=2019-09-11T00%3A00%3A00%2B00%3A00&maxRecords=2000&index=2001",
json_load(DATA_PATH / "opensearch_creodias_empty.json"),
),
},
"Fusion-data": json_load(DATA_PATH / "s3_fusion_data.json"),
}
return data
......@@ -31,16 +31,7 @@ harvesters:
file_regex: \d{4}-\d{2}-\d{2}.vrt
time_regex: \d{4}-\d{2}-\d{2}
time_format: "%Y-%m-%d"
filter:
and:
- during:
- property: datetime
- - "2020-06-01T00:00:00Z"
- "2020-06-30T00:00:00Z"
- during:
- property: updated
- - P5D
- !now
filter: null
mode: item
queue: register
- name: Opensearch-composites-eox
......
{
"type": "FeatureCollection",
"properties": {
"id": "1812e55b-32ad-517b-8311-accc1250d3d8",
"totalResults": 12,
"exactCount": true,
"startIndex": 1,
"itemsPerPage": 12,
"query": {
"originalFilters": {
"box": "14.9,47.7,16.4,48.7",
"startDate": "2019-09-10T00:00:00",
"completionDate": "2019-09-11T00:00:00",
"maxRecords": "20",
"index": "1",
"status": 0,
"collection": "Sentinel2"
},
"appliedFilters": {
"box": "14.9,47.7,16.4,48.7",
"startDate": "2019-09-10T00:00:00",
"completionDate": "2019-09-11T00:00:00",
"maxRecords": "20",
"index": "1",
"status": 0,
"collection": "Sentinel2"
},
"analysis": {
"query": null,
"language": "en",
"analyze": {
"What": [],
"When": [],
"Where": [],
"Errors": [],
"Explained": []
},
"processingTime": 9.5367431640625e-7
},
"processingTime": 0.070756196975708
},
"links": [{
"rel": "self",
"type": "application/json",
"title": "self",
"href": "https://finder.creodias.eu/resto/api/collections/Sentinel2/search.json?&box=14.9%2C47.7%2C16.4%2C48.7&startDate=2019-09-10T00%3A00%3A00&completionDate=2019-09-11T00%3A00%3A00&maxRecords=20&index=1"
},
{
"rel": "search",
"type": "application/opensearchdescription+xml",
"title": "OpenSearch Description Document",
"href": "https://finder.creodias.eu/resto/api/collections/Sentinel2/describe.xml"
}
]
},
"features": [{
"type": "Feature",
"id": "1930e430-7833-58d2-8f2f-24d6d81188b9",
"geometry": {
"type": "Polygon",
"coordinates": [
[
[
16.33660022,
47.845921052
],
[
17.802846683,
47.819474403
],
[
17.751084999,
46.832628319
],
[
16.311886886,
46.858181972
],
[
16.33660022,
47.845921052
]
]
]
},
"properties": {
"collection": "Sentinel2",
"status": 0,
"license": {
"licenseId": "unlicensed",
"hasToBeSigned": "never",
"grantedCountries": null,
"grantedOrganizationCountries": null,
"grantedFlags": null,
"viewService": "public",
"signatureQuota": -1,
"description": {
"shortName": "No license"
}
},
"productIdentifier": "/eodata/Sentinel-2/MSI/L1C/2019/09/10/S2B_MSIL1C_20190910T095029_N0208_R079_T33TXN_20190910T120910.SAFE",
"parentIdentifier": null,
"title": "S2B_MSIL1C_20190910T095029_N0208_R079_T33TXN_20190910T120910.SAFE",
"description": null,
"organisationName": "ESA",
"startDate": "2019-09-10T09:50:29.024Z",
"completionDate": "2019-09-10T09:50:29.024Z",
"productType": "L1C",
"processingLevel": "LEVEL1C",
"platform": "S2B",
"instrument": "MSI",
"resolution": 60,
"sensorMode": "INS-NOBS",
"orbitNumber": 13118,
"quicklook": null,
"thumbnail": "https://finder.creodias.eu/files/Sentinel-2/MSI/L1C/2019/09/10/S2B_MSIL1C_20190910T095029_N0208_R079_T33TXN_20190910T120910.SAFE/S2B_MSIL1C_20190910T095029_N0208_R079_T33TXN_20190910T120910-ql.jpg",
"updated": "2019-09-10T16:02:32.33989Z",
"published": "2019-09-10T16:02:32.33989Z",
"snowCover": 0,
"cloudCover": 85.054,
"keywords": [{
"id": "bce097de57d1ab7",
"name": "Europe",
"normalized": "europe",
"type": "continent",
"href": "https://finder.creodias.eu/resto/api/collections/Sentinel2/search.json?&lang=en&q=Europe"
},
{
"id": "ecf800912d6773f",
"name": "Hungary",
"normalized": "hungary",
"type": "country",
"parentHash": "bce097de57d1ab7",
"value": 83.52,
"gcover": 10.8,
"href": "https://finder.creodias.eu/resto/api/collections/Sentinel2/search.json?&lang=en&q=Hungary"
},
{
"id": "941010134878349",
"name": "Western Transdanubia",
"normalized": "western-transdanubia",
"type": "region",
"parentHash": "ecf800912d6773f",
"value": 59.14,
"gcover": 64.66,
"href": "https://finder.creodias.eu/resto/api/collections/Sentinel2/search.json?&lang=en&q=%22Western%20Transdanubia%22"
},
{
"id": "e082c0be8c5509f",
"name": "Gyor-Moson-Sopron",
"normalized": "gyormosonsopron",
"type": "state",
"parentHash": "941010134878349",
"value": 24.79,
"gcover": 78.66,
"href": "https://finder.creodias.eu/resto/api/collections/Sentinel2/search.json?&lang=en&q=Gyor-Moson-Sopron"
},
{
"id": "7a5164aca4ad2fa",
"name": "Central Transdanubia",
"normalized": "central-transdanubia",
"type": "region",
"parentHash": "ecf800912d6773f",
"value": 24.53,
"gcover": 25.15,
"href": "https://finder.creodias.eu/resto/api/collections/Sentinel2/search.json?&lang=en&q=%22Central%20Transdanubia%22"
},
{
"id": "74889256c663daa",
"name": "Veszprém",
"normalized": "veszprem",
"type": "state",
"parentHash": "7a5164aca4ad2fa",
"value": 24.53,
"gcover": 61.73,
"href": "https://finder.creodias.eu/resto/api/collections/Sentinel2/search.json?&lang=en&q=Veszpr%C3%A9m"
},
{
"id": "ba93adf79eec421",
"name": "Southern Transdanubia",
"normalized": "southern-transdanubia",
"type": "region",
"parentHash": "ecf800912d6773f",
"value": 0.01,
"gcover": 0,
"href": "https://finder.creodias.eu/resto/api/collections/Sentinel2/search.json?&lang=en&q=%22Southern%20Transdanubia%22"
},
{
"id": "1cccb6c54cb3f84",
"name": "Somogy",
"normalized": "somogy",
"type": "state",
"parentHash": "ba93adf79eec421",
"value": 0.01,
"gcover": 0.02,
"href": "https://finder.creodias.eu/resto/api/collections/Sentinel2/search.json?&lang=en&q=Somogy"
},
{
"id": "de3aba58d677a1c",
"name": "Austria",
"normalized": "austria",
"type": "country",
"parentHash": "bce097de57d1ab7",
"value": 15.86,
"gcover": 2.27,
"href": "https://finder.creodias.eu/resto/api/collections/Sentinel2/search.json?&lang=en&q=Austria"
},
{
"id": "7d04d5a0bbb212f",
"name": "_all",
"normalized": "_all",
"type": "region",
"parentHash": "de3aba58d677a1c",
"href": "https://finder.creodias.eu/resto/api/collections/Sentinel2/search.json?&lang=en&q=_all"
},
{
"id": "5ea0175de888816",
"name": "Burgenland",
"normalized": "burgenland",
"type": "state",
"parentHash": "7d04d5a0bbb212f",
"value": 15.65,
"gcover": 47.38,
"href": "https://finder.creodias.eu/resto/api/collections/Sentinel2/search.json?&lang=en&q=Burgenland"
},
{
"id": "655a4c04b29aa55",
"name": "Niederösterreich",
"normalized": "niederosterreich",
"type": "state",
"parentHash": "7d04d5a0bbb212f",
"value": 0.2,
"gcover": 0.13,
"href": "https://finder.creodias.eu/resto/api/collections/Sentinel2/search.json?&lang=en&q=Nieder%C3%B6sterreich"
},
{
"id": "6ebff9cb55e5c93",
"name": "Slovakia",
"normalized": "slovakia",
"type": "country",
"parentHash": "bce097de57d1ab7",
"value": 0.48,
"gcover": 0.12,
"href": "https://finder.creodias.eu/resto/api/collections/Sentinel2/search.json?&lang=en&q=Slovakia"
},
{
"id": "4f249e05357402e",
"name": "_all",
"normalized": "_all",
"type": "region",
"parentHash": "6ebff9cb55e5c93",
"href": "https://finder.creodias.eu/resto/api/collections/Sentinel2/search.json?&lang=en&q=_all"
},
{
"id": "4a0047ac3fa6aed",
"name": "Nitriansky",
"normalized": "nitriansky",
"type": "state",
"parentHash": "4f249e05357402e",
"value": 0.26,
"gcover": 0.51,
"href": "https://finder.creodias.eu/resto/api/collections/Sentinel2/search.json?&lang=en&q=Nitriansky"
},
{
"id": "d19269f9a5cf277",
"name": "Trnavský",
"normalized": "trnavsky",
"type": "state",
"parentHash": "4f249e05357402e",
"value": 0.22,
"gcover": 0.65,
"href": "https://finder.creodias.eu/resto/api/collections/Sentinel2/search.json?&lang=en&q=Trnavsk%C3%BD"
},
{
"id": "a306c45408e6b6f",
"name": "Cultivated Area",
"normalized": "cultivated",
"type": "landuse",
"area": 9158.6277950569,
"value": 76.16,
"href": "https://finder.creodias.eu/resto/api/collections/Sentinel2/search.json?&lang=en&q=%22Cultivated%20Area%22"
},
{
"id": "5b8cdeb6f99ce56",
"name": "Forest",
"normalized": "forest",
"type": "landuse",
"area": 2597.1392010786,
"value": 21.59,
"href": "https://finder.creodias.eu/resto/api/collections/Sentinel2/search.json?&lang=en&q=Forest"
},
{
"id": "30ab8d81fb40d6c",
"name": "Water",
"normalized": "water",
"type": "landuse",
"area": 91.774744343155,
"value": 0.76,
"href": "https://finder.creodias.eu/resto/api/collections/Sentinel2/search.json?&lang=en&q=Water"
},
{
"id": "154109666ea230b",
"name": "Urban",
"normalized": "urban",
"type": "landuse",
"area": 91.133808300338,
"value": 0.75,
"href": "https://finder.creodias.eu/resto/api/collections/Sentinel2/search.json?&lang=en&q=Urban"
},
{
"id": "870b2149f9a08a2",
"name": "Herbaceous Area",
"normalized": "herbaceous",
"type": "landuse",
"area": 89.086705733273,
"value": 0.74,
"href": "https://finder.creodias.eu/resto/api/collections/Sentinel2/search.json?&lang=en&q=%22Herbaceous%20Area%22"
},
{
"id": "a0e4a10aa754ab4",
"name": "Northern",
"type": "location",
"href": "https://finder.creodias.eu/resto/api/collections/Sentinel2/search.json?&lang=en&q=Northern"
},
{
"id": "6278eea013ff0d3",
"name": "Summer",
"type": "season",
"href": "https://finder.creodias.eu/resto/api/collections/Sentinel2/search.json?&lang=en&q=Summer"
},
{
"id": "7dd5ce6fe9bcea1",
"name": "Sentinel2",
"type": "collection",
"href": "https://finder.creodias.eu/resto/api/collections/Sentinel2/search.json?&lang=en&q=Sentinel2"
},
{
"id": "b53413be0d10e41",
"name": "L1C",
"type": "productType",
"href": "https://finder.creodias.eu/resto/api/collections/Sentinel2/search.json?&lang=en&q=L1C"
},
{
"id": "44b15c0a6e62129",
"name": "LEVEL1C",
"type": "processingLevel",
"href": "https://finder.creodias.eu/resto/api/collections/Sentinel2/search.json?&lang=en&q=LEVEL1C"
},
{
"id": "56bf819406cae6b",
"name": "s2B",
"type": "platform",
"href": "https://finder.creodias.eu/resto/api/collections/Sentinel2/search.json?&lang=en&q=s2B"
},
{
"id": "f841767ba5c92d4",
"name": "msi",
"type": "instrument",
"parentHash": "56bf819406cae6b",
"href": "https://finder.creodias.eu/resto/api/collections/Sentinel2/search.json?&lang=en&q=msi"
},
{
"id": "aca78422271631a",
"name": "INS-NOBS",
"type": "sensorMode",
"parentHash": "f841767ba5c92d4",
"href": "https://finder.creodias.eu/resto/api/collections/Sentinel2/search.json?&lang=en&q=INS-NOBS"
},
{
"id": "bd69cb806e20886",
"name": "2019",
"type": "year",
"href": "https://finder.creodias.eu/resto/api/collections/Sentinel2/search.json?&lang=en&q=2019"
},
{
"id": "1fef767f5972fb0",
"name": "September",
"type": "month",
"parentHash": "bd69cb806e20886",
"href": "https://finder.creodias.eu/resto/api/collections/Sentinel2/search.json?&lang=en&q=September"
},
{
"id": "5c22da195cb3cfb",
"name": "10",
"type": "day",
"parentHash": "1fef767f5972fb0",
"href": "https://finder.creodias.eu/resto/api/collections/Sentinel2/search.json?&lang=en&q=10"
}
],
"gmlgeometry": "<gml:Polygon srsName=\"EPSG:4326\"><gml:outerBoundaryIs><gml:LinearRing><gml:coordinates>16.33660021997,47.845921052089 17.802846682925,47.819474402959 17.75108499862,46.832628319251 16.311886885512,46.858181972465 16.33660021997,47.845921052089</gml:coordinates></gml:LinearRing></gml:outerBoundaryIs></gml:Polygon>",
"centroid": {
"type": "Point",
"coordinates": [
17.050613957,
47.340585103
]
},
"orbitDirection": "descending",
"timeliness": null,
"relativeOrbitNumber": 79,
"processingBaseline": 2.08,
"missionTakeId": "GS2B_20190910T095029_013118_N02.08",
"services": {
"download": {
"url": "https://zipper.creodias.eu/download/1930e430-7833-58d2-8f2f-24d6d81188b9",
"mimeType": "application/unknown",
"size": 786191528
}
},
"links": [{
"rel": "self",
"type": "application/json",
"title": "GeoJSON link for 1930e430-7833-58d2-8f2f-24d6d81188b9",
"href": "https://finder.creodias.eu/resto/collections/Sentinel2/1930e430-7833-58d2-8f2f-24d6d81188b9.json?&lang=en"
}]
}
},
{
"type": "Feature",
"id": "4bea82ea-6fc4-5d87-a116-87e731e08b65",
"geometry": {
"type": "Polygon",
"coordinates": [
[
[
16.360279246,
48.744984112
],
[
17.852438708,
48.717694751
],
[
17.798094715,
47.731044478
],
[
16.334331349,
47.757409737
],
[
16.360279246,
48.744984112
]
]
]
},
"properties": {
"collection": "Sentinel2",
"status": 0,
"license": {
"licenseId": "unlicensed",
"hasToBeSigned": "never",
"grantedCountries": null,
"grantedOrganizationCountries": null,
"grantedFlags": null,
"viewService": "public",
"signatureQuota": -1,
"description": {
"shortName": "No license"
}
},
"productIdentifier": "/eodata/Sentinel-2/MSI/L1C/2019/09/10/S2B_MSIL1C_20190910T095029_N0208_R079_T33UXP_20190910T120910.SAFE",
"parentIdentifier": null,
"title": "S2B_MSIL1C_20190910T095029_N0208_R079_T33UXP_20190910T120910.SAFE",
"description": null,
"organisationName":