From 916805448aeba87f7b29ad5bc3c9221d0eafdd75 Mon Sep 17 00:00:00 2001 From: Angelos Tzotsos Date: Mon, 16 Nov 2020 17:04:11 +0200 Subject: [PATCH 01/26] Adding pycsw backend for registrar --- core/registrar/backend.py | 65 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 64 insertions(+), 1 deletion(-) diff --git a/core/registrar/backend.py b/core/registrar/backend.py index 17144f47..4b347e7b 100644 --- a/core/registrar/backend.py +++ b/core/registrar/backend.py @@ -4,10 +4,15 @@ import sys import logging from typing import List import json +import uuid import django from django.db import transaction from django.contrib.gis.geos import GEOSGeometry, Polygon +from lxml import etree +from pycsw.core import metadata, repository, util +import pycsw.core.admin +import pycsw.core.config from .exceptions import RegistrationError from .context import Context @@ -25,6 +30,63 @@ class Backend: def register_item(self, item: Context) -> RegistrationResult: raise NotImplementedError +class PycswBackend(Backend): + def __init__(self, instance_base_path: str, instance_name: str, mapping: dict, simplify_footprint_tolerance: int=None): + logger.debug('Setting up static context') + self.context = pycsw.core.config.StaticContext() + + logger.debug('Initializing pycsw repository') + logger.info(f"CONFIG: {os.environ.get('PYCSW_REPOSITORY_DATABASE_URI')}") + self.repo = repository.Repository('postgresql://postgres:mypass@metadata_store/pycsw', self.context, table='records') + + def exists(self, source: Source, item: Context): + # TODO: sort out identifier problem in ISO XML + if self.repo.query_ids([item.identifier]): + return True + return False + + def register(self, source: Source, item: Context, replace: bool) -> RegistrationResult: + + tmpfile = '/tmp/backend-pycsw-metadata.xml' + _, metadata_file = source.get_container_and_path(item.metadata_files[0]) + + inspire_xml = os.path.dirname(item.metadata_files[0]) + "/INSPIRE.xml" + logger.info(f"INSPIRE XML metadata file: {inspire_xml}") + + try: + source.get_file(inspire_xml, tmpfile) + except Exception as err: + logger.error(err) + return False + + logger.debug('Parsing XML') + try: + xml = etree.parse(tmpfile) + except Exception as err: + logger.error('XML parsing failed: {}'.format(err)) + return False + + logger.debug('Processing metadata') + try: + record = metadata.parse_record(self.context, xml, self.repo)[0] + # TODO: sort out identifier problem in ISO XML + logger.info(f"identifier: {record.identifier}") + if record.identifier is None: + logger.warning(f"identifier is empty; auto-generating") + record.identifier = str(uuid.uuid4()) + except Exception as err: + logger.error('Metadata parsing failed: {}'.format(err)) + return False + + logger.debug('Inserting record') + try: + self.repo.insert(record, 'local', util.get_today_and_now()) + except Exception as err: + logger.error('record insertion failed: {}'.format(err)) + return False + + return True + class EOxServerBackend(Backend): def __init__(self, instance_base_path: str, instance_name: str, mapping: dict, simplify_footprint_tolerance: int=None): @@ -234,7 +296,8 @@ class EOxServerBackend(Backend): BACKENDS = { - 'eoxserver': EOxServerBackend + 'eoxserver': EOxServerBackend, + 'pycsw': PycswBackend } def get_backends(config: dict, path: str) -> List[Backend]: -- GitLab From c0ae016406a57bcec2a6afb0c6e7544a36dd5fc6 Mon Sep 17 00:00:00 2001 From: Angelos Tzotsos Date: Mon, 16 Nov 2020 17:04:36 +0200 Subject: [PATCH 02/26] Update registrar dependencies for pycsw backend --- core/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/Dockerfile b/core/Dockerfile index 70fb15f1..8237df6b 100644 --- a/core/Dockerfile +++ b/core/Dockerfile @@ -43,7 +43,7 @@ RUN apt update && \ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* RUN pip3 install . && \ - pip3 install python-keystoneclient python-swiftclient redis click setuptools jsonschema boto3 + pip3 install python-keystoneclient python-swiftclient redis click setuptools jsonschema boto3 pyproj==2.0.2 pycsw SQLAlchemy ENV INSTANCE_ID="prism-view-server_core" \ INSTANCE_NAME="pvs_instance"\ -- GitLab From 2f1be5432d81ea7928d2f7f84c69bf279c225e67 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Mon, 16 Nov 2020 11:21:45 -0500 Subject: [PATCH 03/26] add resource catalogue publishing to ingest workflow --- core/Dockerfile | 2 +- core/registrar/backend.py | 40 ++++++++++++----- core/registrar/metadata.py | 89 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 118 insertions(+), 13 deletions(-) create mode 100644 core/registrar/metadata.py diff --git a/core/Dockerfile b/core/Dockerfile index 8237df6b..b1ffc78e 100644 --- a/core/Dockerfile +++ b/core/Dockerfile @@ -43,7 +43,7 @@ RUN apt update && \ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* RUN pip3 install . && \ - pip3 install python-keystoneclient python-swiftclient redis click setuptools jsonschema boto3 pyproj==2.0.2 pycsw SQLAlchemy + pip3 install python-keystoneclient python-swiftclient redis click setuptools jsonschema boto3 pygeometa pyproj==2.0.2 pycsw SQLAlchemy ENV INSTANCE_ID="prism-view-server_core" \ INSTANCE_NAME="pvs_instance"\ diff --git a/core/registrar/backend.py b/core/registrar/backend.py index 4b347e7b..97a2b900 100644 --- a/core/registrar/backend.py +++ b/core/registrar/backend.py @@ -4,7 +4,6 @@ import sys import logging from typing import List import json -import uuid import django from django.db import transaction @@ -17,6 +16,7 @@ import pycsw.core.config from .exceptions import RegistrationError from .context import Context from .source import Source, LocalSource, S3Source, SwiftSource +from .metadata import gen_iso_metadata logger = logging.getLogger(__name__) @@ -36,32 +36,42 @@ class PycswBackend(Backend): self.context = pycsw.core.config.StaticContext() logger.debug('Initializing pycsw repository') - logger.info(f"CONFIG: {os.environ.get('PYCSW_REPOSITORY_DATABASE_URI')}") - self.repo = repository.Repository('postgresql://postgres:mypass@metadata_store/pycsw', self.context, table='records') + self.repo = repository.Repository(os.environ.get('PYCSW_REPOSITORY_DATABASE_URI'), self.context, table='records') - def exists(self, source: Source, item: Context): + def exists(self, source: Source, item: Context) -> bool: # TODO: sort out identifier problem in ISO XML + logger.info(self.repo.query(constraint={})) if self.repo.query_ids([item.identifier]): + logger.info('identifier EXISTS') return True return False def register(self, source: Source, item: Context, replace: bool) -> RegistrationResult: - tmpfile = '/tmp/backend-pycsw-metadata.xml' - _, metadata_file = source.get_container_and_path(item.metadata_files[0]) + ingest_fail = False + esa_xml_local = '/tmp/esa-metadata.xml' + inspire_xml_local = '/tmp/inspire-metadata.xml' + + esa_xml = item.metadata_files[0] + logger.info(f"ESA XML metadata file: {esa_xml}") inspire_xml = os.path.dirname(item.metadata_files[0]) + "/INSPIRE.xml" logger.info(f"INSPIRE XML metadata file: {inspire_xml}") try: - source.get_file(inspire_xml, tmpfile) + source.get_file(inspire_xml, inspire_xml_local) + source.get_file(esa_xml, esa_xml_local) except Exception as err: logger.error(err) return False + logger.info('Generating ISO XML based on ESA and INSPIRE XML') + with open(esa_xml_local, 'rb') as a, open(inspire_xml_local, 'rb') as b: + iso_metadata = gen_iso_metadata(a.read(), b.read()) + logger.debug('Parsing XML') try: - xml = etree.parse(tmpfile) + xml = etree.fromstring(iso_metadata) except Exception as err: logger.error('XML parsing failed: {}'.format(err)) return False @@ -71,9 +81,6 @@ class PycswBackend(Backend): record = metadata.parse_record(self.context, xml, self.repo)[0] # TODO: sort out identifier problem in ISO XML logger.info(f"identifier: {record.identifier}") - if record.identifier is None: - logger.warning(f"identifier is empty; auto-generating") - record.identifier = str(uuid.uuid4()) except Exception as err: logger.error('Metadata parsing failed: {}'.format(err)) return False @@ -81,9 +88,18 @@ class PycswBackend(Backend): logger.debug('Inserting record') try: self.repo.insert(record, 'local', util.get_today_and_now()) + logger.info('record inserted') except Exception as err: + ingest_fail = True logger.error('record insertion failed: {}'.format(err)) - return False + + if ingest_fail and replace: + logger.error('Updating record') + try: + self.repo.update(record) + logger.info('record updated') + except Exception as err: + logger.error('record update failed: {}'.format(err)) return True diff --git a/core/registrar/metadata.py b/core/registrar/metadata.py new file mode 100644 index 00000000..f63d4833 --- /dev/null +++ b/core/registrar/metadata.py @@ -0,0 +1,89 @@ +import logging + +from lxml import etree +from owslib.iso import MD_Metadata +from pygeometa.core import read_mcf, render_template + +LANGUAGE = 'eng; CAN' + +logger = logging.getLogger(__name__) + +mcf = { + 'mcf': { + 'version': '1.0' + }, + 'metadata': { + 'language': LANGUAGE, + 'charset': 'utf8' + }, + 'spatial': { + }, + 'identification': { + 'charset': 'utf8', + 'keywords': {} + }, + 'contact': { + 'main': {}, + 'distribution': {} + }, + 'distribution': { + } +} + +def gen_iso_metadata(esa_xml: bytes, inspire_xml: bytes) -> str: + exml = etree.fromstring(esa_xml) + ixml = etree.fromstring(inspire_xml) + + m = MD_Metadata(ixml) + + mcf['metadata']['identifier'] = exml.xpath('//PRODUCT_URI/text()')[0] + mcf['metadata']['hierarchy'] = m.hierarchy + mcf['metadata']['datestamp'] = exml.xpath('//Product_Info/GENERATION_TIME/text()')[0] + + mcf['spatial']['datatype'] = 'grid' + mcf['spatial']['geomtype'] = 'solid' + + gfp = exml.xpath('//Global_Footprint/EXT_POS_LIST/text()')[0].split() + + minx = gfp[4] + miny = gfp[1] + maxx = gfp[0] + maxy = gfp[5] + + mcf['spatial']['bbox'] = ','.join([minx, miny, maxx, maxy]) + mcf['spatial']['crs'] = m.referencesystem.code + + + mcf['identification']['language'] = LANGUAGE + mcf['identification']['title'] = exml.xpath('//PRODUCT_URI/text()')[0] + mcf['identification']['abstract'] = exml.xpath('//PRODUCT_URI/text()')[0] + + mcf['identification']['dates'] = { + 'creation': mcf['metadata']['datestamp'], + 'publication': mcf['metadata']['datestamp'] + } + + for i, kws in enumerate(m.identification.keywords): + mcf['identification']['keywords'][i] = { + 'keywords': kws['keywords'] + } + mcf['identification']['keywords'][i]['keywords_type'] = kws['type'] or 'theme' + + mcf['identification']['temporal_begin'] = exml.xpath('//Product_Info/PRODUCT_START_TIME/text()')[0] + mcf['identification']['temporal_end'] = exml.xpath('//Product_Info/PRODUCT_STOP_TIME/text()')[0] + mcf['identification']['topic_category'] = [m.identification.topiccategory] + mcf['identification']['status'] = 'onGoing' + mcf['identification']['maintenancefrequency'] = 'continual' + mcf['identification']['accessconstraints'] = m.identification.accessconstraints[0] + + for image_file in exml.xpath('//Product_Organisation//IMAGE_FILE/text()'): + dist = { + 'url': image_file, + 'type': 'WWW:LINK', + 'name': image_file, + 'description': image_file, + 'function': 'download' + } + mcf['distribution'][image_file] = dist + + return render_template(mcf, 'iso19139') -- GitLab From e1a44d74e53d3a982d273ccf9547fd49e9c83a3e Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Mon, 16 Nov 2020 18:29:12 -0500 Subject: [PATCH 04/26] update to latest pygeometa MCF format --- core/Dockerfile | 3 +- core/registrar/backend.py | 6 ++-- core/registrar/metadata.py | 70 ++++++++++++++++++++++---------------- 3 files changed, 45 insertions(+), 34 deletions(-) diff --git a/core/Dockerfile b/core/Dockerfile index b1ffc78e..f0365207 100644 --- a/core/Dockerfile +++ b/core/Dockerfile @@ -43,7 +43,8 @@ RUN apt update && \ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* RUN pip3 install . && \ - pip3 install python-keystoneclient python-swiftclient redis click setuptools jsonschema boto3 pygeometa pyproj==2.0.2 pycsw SQLAlchemy + pip3 install python-keystoneclient python-swiftclient redis click setuptools jsonschema boto3 pyproj==2.0.2 pycsw SQLAlchemy && \ + pip3 install https://github.com/geopython/pygeometa/archive/master.zip ENV INSTANCE_ID="prism-view-server_core" \ INSTANCE_NAME="pvs_instance"\ diff --git a/core/registrar/backend.py b/core/registrar/backend.py index 97a2b900..bc507038 100644 --- a/core/registrar/backend.py +++ b/core/registrar/backend.py @@ -42,7 +42,7 @@ class PycswBackend(Backend): # TODO: sort out identifier problem in ISO XML logger.info(self.repo.query(constraint={})) if self.repo.query_ids([item.identifier]): - logger.info('identifier EXISTS') + logger.info('identifier exists') return True return False @@ -79,7 +79,7 @@ class PycswBackend(Backend): logger.debug('Processing metadata') try: record = metadata.parse_record(self.context, xml, self.repo)[0] - # TODO: sort out identifier problem in ISO XML + record.xml = record.xml.decode() logger.info(f"identifier: {record.identifier}") except Exception as err: logger.error('Metadata parsing failed: {}'.format(err)) @@ -94,7 +94,7 @@ class PycswBackend(Backend): logger.error('record insertion failed: {}'.format(err)) if ingest_fail and replace: - logger.error('Updating record') + logger.info('Updating record') try: self.repo.update(record) logger.info('record updated') diff --git a/core/registrar/metadata.py b/core/registrar/metadata.py index f63d4833..f55450b3 100644 --- a/core/registrar/metadata.py +++ b/core/registrar/metadata.py @@ -2,9 +2,9 @@ import logging from lxml import etree from owslib.iso import MD_Metadata -from pygeometa.core import read_mcf, render_template +from pygeometa.schemas.iso19139 import ISO19139OutputSchema -LANGUAGE = 'eng; CAN' +LANGUAGE = 'eng' logger = logging.getLogger(__name__) @@ -17,65 +17,71 @@ mcf = { 'charset': 'utf8' }, 'spatial': { + 'datatype': 'grid', + 'geomtype': 'solid' }, 'identification': { 'charset': 'utf8', + 'language': 'missing', 'keywords': {} }, 'contact': { 'main': {}, 'distribution': {} }, - 'distribution': { - } + 'distribution': {} } + def gen_iso_metadata(esa_xml: bytes, inspire_xml: bytes) -> str: exml = etree.fromstring(esa_xml) ixml = etree.fromstring(inspire_xml) m = MD_Metadata(ixml) - + mcf['metadata']['identifier'] = exml.xpath('//PRODUCT_URI/text()')[0] - mcf['metadata']['hierarchy'] = m.hierarchy + mcf['metadata']['hierarchylevel'] = m.hierarchy mcf['metadata']['datestamp'] = exml.xpath('//Product_Info/GENERATION_TIME/text()')[0] - - mcf['spatial']['datatype'] = 'grid' - mcf['spatial']['geomtype'] = 'solid' - + gfp = exml.xpath('//Global_Footprint/EXT_POS_LIST/text()')[0].split() - + minx = gfp[4] - miny = gfp[1] + miny = gfp[5] maxx = gfp[0] - maxy = gfp[5] - - mcf['spatial']['bbox'] = ','.join([minx, miny, maxx, maxy]) - mcf['spatial']['crs'] = m.referencesystem.code - - - mcf['identification']['language'] = LANGUAGE + maxy = gfp[1] + + mcf['identification']['extents'] = { + 'spatial': [{ + 'bbox': [minx, miny, maxx, maxy], + 'crs': 4326 + }], + 'temporal': [{ + 'begin': exml.xpath('//Product_Info/PRODUCT_START_TIME/text()')[0], + 'end': exml.xpath('//Product_Info/PRODUCT_STOP_TIME/text()')[0] + }] + } + mcf['identification']['title'] = exml.xpath('//PRODUCT_URI/text()')[0] mcf['identification']['abstract'] = exml.xpath('//PRODUCT_URI/text()')[0] - + mcf['identification']['dates'] = { 'creation': mcf['metadata']['datestamp'], 'publication': mcf['metadata']['datestamp'] } - + for i, kws in enumerate(m.identification.keywords): - mcf['identification']['keywords'][i] = { + kw_set = 'kw{}'.format(i) + + mcf['identification']['keywords'][kw_set] = { 'keywords': kws['keywords'] } - mcf['identification']['keywords'][i]['keywords_type'] = kws['type'] or 'theme' - - mcf['identification']['temporal_begin'] = exml.xpath('//Product_Info/PRODUCT_START_TIME/text()')[0] - mcf['identification']['temporal_end'] = exml.xpath('//Product_Info/PRODUCT_STOP_TIME/text()')[0] - mcf['identification']['topic_category'] = [m.identification.topiccategory] + mcf['identification']['keywords'][kw_set]['keywords_type'] = kws['type'] or 'theme' + + mcf['identification']['topiccategory'] = [m.identification.topiccategory[0]] mcf['identification']['status'] = 'onGoing' mcf['identification']['maintenancefrequency'] = 'continual' mcf['identification']['accessconstraints'] = m.identification.accessconstraints[0] - + for image_file in exml.xpath('//Product_Organisation//IMAGE_FILE/text()'): dist = { 'url': image_file, @@ -85,5 +91,9 @@ def gen_iso_metadata(esa_xml: bytes, inspire_xml: bytes) -> str: 'function': 'download' } mcf['distribution'][image_file] = dist - - return render_template(mcf, 'iso19139') + + logger.info("MCF: {}".format(mcf)) + + iso_os = ISO19139OutputSchema() + + return iso_os.write(mcf) -- GitLab From c3a0b8b0bc341aae2c6d3c7faf983acce5e781b3 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Wed, 18 Nov 2020 09:20:25 -0500 Subject: [PATCH 05/26] add fully qualified URL, mime type and link for enclosure --- core/registrar/backend.py | 4 +++- core/registrar/metadata.py | 36 +++++++++++++++++++++++++++++------- 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/core/registrar/backend.py b/core/registrar/backend.py index bc507038..cb3eeab8 100644 --- a/core/registrar/backend.py +++ b/core/registrar/backend.py @@ -58,6 +58,8 @@ class PycswBackend(Backend): inspire_xml = os.path.dirname(item.metadata_files[0]) + "/INSPIRE.xml" logger.info(f"INSPIRE XML metadata file: {inspire_xml}") + base_url = 's3://{}'.format(os.path.split(os.path.dirname(esa_xml))[0]) + try: source.get_file(inspire_xml, inspire_xml_local) source.get_file(esa_xml, esa_xml_local) @@ -67,7 +69,7 @@ class PycswBackend(Backend): logger.info('Generating ISO XML based on ESA and INSPIRE XML') with open(esa_xml_local, 'rb') as a, open(inspire_xml_local, 'rb') as b: - iso_metadata = gen_iso_metadata(a.read(), b.read()) + iso_metadata = gen_iso_metadata(base_url, a.read(), b.read()) logger.debug('Parsing XML') try: diff --git a/core/registrar/metadata.py b/core/registrar/metadata.py index f55450b3..f5f14533 100644 --- a/core/registrar/metadata.py +++ b/core/registrar/metadata.py @@ -33,13 +33,17 @@ mcf = { } -def gen_iso_metadata(esa_xml: bytes, inspire_xml: bytes) -> str: +def gen_iso_metadata(base_url: str, esa_xml: bytes, inspire_xml: bytes) -> str: exml = etree.fromstring(esa_xml) ixml = etree.fromstring(inspire_xml) m = MD_Metadata(ixml) - mcf['metadata']['identifier'] = exml.xpath('//PRODUCT_URI/text()')[0] + product_manifest = exml.xpath('//PRODUCT_URI/text()')[0] + product_identifier = product_manifest.replace('.SAFE', '') + product_manifest_link = f"{base_url}/{product_manifest}" + + mcf['metadata']['identifier'] = product_identifier mcf['metadata']['hierarchylevel'] = m.hierarchy mcf['metadata']['datestamp'] = exml.xpath('//Product_Info/GENERATION_TIME/text()')[0] @@ -61,8 +65,8 @@ def gen_iso_metadata(esa_xml: bytes, inspire_xml: bytes) -> str: }] } - mcf['identification']['title'] = exml.xpath('//PRODUCT_URI/text()')[0] - mcf['identification']['abstract'] = exml.xpath('//PRODUCT_URI/text()')[0] + mcf['identification']['title'] = product_identifier + mcf['identification']['abstract'] = product_identifier mcf['identification']['dates'] = { 'creation': mcf['metadata']['datestamp'], @@ -82,17 +86,35 @@ def gen_iso_metadata(esa_xml: bytes, inspire_xml: bytes) -> str: mcf['identification']['maintenancefrequency'] = 'continual' mcf['identification']['accessconstraints'] = m.identification.accessconstraints[0] + mcf['distribution'][product_manifest] = { + 'url': product_manifest_link, + 'type': 'WWW:LINK', + 'name': product_manifest, + 'description': product_manifest, + 'function': 'download' + } + + product_format = exml.xpath('//Granule_List/Granule/@imageFormat')[0] + + if product_format == 'JPEG2000': + mime_type = 'image/x.geotiff' + file_extension = 'jp2' + else: + logger.warning('unknown product format: {}'.format(product_format)) + mime_type = 'NA' + file_extension = 'NA' + for image_file in exml.xpath('//Product_Organisation//IMAGE_FILE/text()'): dist = { - 'url': image_file, - 'type': 'WWW:LINK', + 'url': f"{product_manifest_link}/{image_file}.{file_extension}", + 'type': mime_type, 'name': image_file, 'description': image_file, 'function': 'download' } mcf['distribution'][image_file] = dist - logger.info("MCF: {}".format(mcf)) + logger.debug("MCF: {}".format(mcf)) iso_os = ISO19139OutputSchema() -- GitLab From 4368ae6576b4f19b27a3dc61c78131763a136563 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Wed, 18 Nov 2020 10:56:26 -0500 Subject: [PATCH 06/26] fix ref --- core/registrar/metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/registrar/metadata.py b/core/registrar/metadata.py index f5f14533..951c3c07 100644 --- a/core/registrar/metadata.py +++ b/core/registrar/metadata.py @@ -97,7 +97,7 @@ def gen_iso_metadata(base_url: str, esa_xml: bytes, inspire_xml: bytes) -> str: product_format = exml.xpath('//Granule_List/Granule/@imageFormat')[0] if product_format == 'JPEG2000': - mime_type = 'image/x.geotiff' + mime_type = 'image/jp2' file_extension = 'jp2' else: logger.warning('unknown product format: {}'.format(product_format)) -- GitLab From 9bcd5076372d62b608f13993a05f84813971c7c4 Mon Sep 17 00:00:00 2001 From: Fabian Schindler Date: Wed, 18 Nov 2020 18:14:44 +0100 Subject: [PATCH 07/26] Honoring the '.Values.image.reposotory' setting if present --- chart/templates/client-deployment.yaml | 2 +- chart/templates/registrar-deployment.yaml | 4 +++- chart/templates/renderer-deployment.yaml | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/chart/templates/client-deployment.yaml b/chart/templates/client-deployment.yaml index ae66fa84..c0ac1423 100644 --- a/chart/templates/client-deployment.yaml +++ b/chart/templates/client-deployment.yaml @@ -29,7 +29,7 @@ spec: {{- end }} containers: - name: {{ .Chart.Name }}-client - image: 'registry.gitlab.eox.at/esa/prism/vs/pvs_client:{{ .Values.image.tag | default .Chart.AppVersion }}' + image: '{{ .Values.image.repository | default "registry.gitlab.eox.at/esa/prism/vs" }}/pvs_client:{{ .Values.image.tag | default .Chart.AppVersion }}' imagePullPolicy: {{ .Values.image.pullPolicy }} ports: - name: http diff --git a/chart/templates/registrar-deployment.yaml b/chart/templates/registrar-deployment.yaml index 1e4b4374..b0d39284 100644 --- a/chart/templates/registrar-deployment.yaml +++ b/chart/templates/registrar-deployment.yaml @@ -29,7 +29,7 @@ spec: {{- end }} containers: - name: {{ .Chart.Name }}-registrar - image: "registry.gitlab.eox.at/esa/prism/vs/pvs_core:{{ .Values.image.tag | default .Chart.AppVersion }}" + image: "{{ .Values.image.repository | default "registry.gitlab.eox.at/esa/prism/vs" }}/pvs_core:{{ .Values.image.tag | default .Chart.AppVersion }}" imagePullPolicy: {{ .Values.image.pullPolicy }} resources: {{- toYaml .Values.registrar.resources | nindent 12 }} @@ -70,6 +70,8 @@ spec: value: /wait-initialized.sh - name: WAIT_SERVICES value: {{ .Release.Name }}-database:{{ .Values.config.database.DB_PORT }} {{ .Release.Name }}-redis-master:{{ .Values.config.redis.REDIS_PORT }} + - name: PYCSW_REPOSITORY_DATABASE_URI + value: "postgresql://postgres:mypass@resource-catalogue-db/pycsw" volumeMounts: - mountPath: /init-db.sh name: init-db diff --git a/chart/templates/renderer-deployment.yaml b/chart/templates/renderer-deployment.yaml index 0bcb7cd0..de0f1beb 100644 --- a/chart/templates/renderer-deployment.yaml +++ b/chart/templates/renderer-deployment.yaml @@ -29,7 +29,7 @@ spec: {{- end }} containers: - name: {{ .Chart.Name }}-renderer - image: "registry.gitlab.eox.at/esa/prism/vs/pvs_core:{{ .Values.image.tag | default .Chart.AppVersion }}" + image: "{{ .Values.image.repository | default "registry.gitlab.eox.at/esa/prism/vs" }}/pvs_core:{{ .Values.image.tag | default .Chart.AppVersion }}" imagePullPolicy: {{ .Values.image.pullPolicy }} ports: - name: http -- GitLab From 754611d971a520f3dc3b89cee286123831f15f3d Mon Sep 17 00:00:00 2001 From: Fabian Schindler Date: Wed, 18 Nov 2020 18:15:02 +0100 Subject: [PATCH 08/26] Adding the pycsw backend --- chart/files/registrar-config.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/chart/files/registrar-config.yaml b/chart/files/registrar-config.yaml index 66a887ea..6451270e 100644 --- a/chart/files/registrar-config.yaml +++ b/chart/files/registrar-config.yaml @@ -54,3 +54,9 @@ backends: {{- end }} {{- end }} {{- end }} + - type: pycsw + filter: + kwargs: + instance_base_path: /var/www/pvs/dev + instance_name: pvs_instance + mapping: -- GitLab From 0e46f9e1b80a1a429a5cdc80550a1dee2f5d4934 Mon Sep 17 00:00:00 2001 From: Fabian Schindler Date: Wed, 18 Nov 2020 18:15:21 +0100 Subject: [PATCH 09/26] Adjusting time limits in the client. --- chart/files/index.html | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/chart/files/index.html b/chart/files/index.html index b74ad39f..0e986eef 100644 --- a/chart/files/index.html +++ b/chart/files/index.html @@ -37,18 +37,18 @@ var config = { "searchDebounceTime": 500, "language": "en", "timeDomain": [ - "2017-01-01T00:00:00Z", - "2019-12-31T23:59:59Z", + "2019-01-01T00:00:00Z", + "2020-12-31T23:59:59Z", ], "constrainTimeDomain": true, "displayTimeDomain": [ - "2017-01-01T00:00:00Z", - "2019-12-31T23:59:59Z", + "2019-01-01T00:00:00Z", + "2020-12-31T23:59:59Z", ], "displayInterval": "P1096D", "selectedTimeDomain": [ - "2018-08-01T00:00:00Z", - "2018-08-31T23:59:59Z", + "2020-08-01T00:00:00Z", + "2020-08-31T23:59:59Z", ], "selectableInterval": "P1096D", "timeSliderControls": true, -- GitLab From 5ac0741b20ec339bb9d95502cd3aafe887aa4753 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Wed, 18 Nov 2020 14:23:10 -0500 Subject: [PATCH 10/26] fix metadata axis order --- core/registrar/metadata.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/core/registrar/metadata.py b/core/registrar/metadata.py index 951c3c07..065dbf8f 100644 --- a/core/registrar/metadata.py +++ b/core/registrar/metadata.py @@ -49,10 +49,10 @@ def gen_iso_metadata(base_url: str, esa_xml: bytes, inspire_xml: bytes) -> str: gfp = exml.xpath('//Global_Footprint/EXT_POS_LIST/text()')[0].split() - minx = gfp[4] - miny = gfp[5] - maxx = gfp[0] - maxy = gfp[1] + minx = gfp[1] + miny = gfp[0] + maxx = gfp[5] + maxy = gfp[4] mcf['identification']['extents'] = { 'spatial': [{ @@ -99,6 +99,9 @@ def gen_iso_metadata(base_url: str, esa_xml: bytes, inspire_xml: bytes) -> str: if product_format == 'JPEG2000': mime_type = 'image/jp2' file_extension = 'jp2' + elif product_format == 'TIFF': + mime_type = 'image/x.geotiff' + file_extension = 'tif' else: logger.warning('unknown product format: {}'.format(product_format)) mime_type = 'NA' -- GitLab From be1147d65fba9cc846caebe3cce17511714f6557 Mon Sep 17 00:00:00 2001 From: Fabian Schindler Date: Thu, 19 Nov 2020 10:27:23 +0100 Subject: [PATCH 11/26] Adding default and other browse configuration for registration --- chart/files/registrar-config.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/chart/files/registrar-config.yaml b/chart/files/registrar-config.yaml index 6451270e..480e478c 100644 --- a/chart/files/registrar-config.yaml +++ b/chart/files/registrar-config.yaml @@ -48,6 +48,15 @@ backends: {{- end }} coverages: {{- toYaml $product_type.coverages | nindent 16 }} + browses: + {{- if hasKey $product_type "default_browse_locator" }} + {{ $product_type.default_browse_locator }}: null + {{- end }} + {{- range $browse_name, $browse_config := $product_type.browses }} + {{- if hasKey $browse_config "locator" }} + {{ $browse_config.locator }}: {{ $browse_name }} + {{- end }} + {{- end }} masks: {{- range $mask_name, $_ := $product_type.masks }} {{ $mask_name }}: {{ $mask_name }} -- GitLab From f30c53b4133e7f62d10cf9ed793af32b066bcb39 Mon Sep 17 00:00:00 2001 From: Fabian Schindler Date: Thu, 19 Nov 2020 12:02:26 +0100 Subject: [PATCH 12/26] Fixing product levels in registrar --- chart/files/registrar-config.yaml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/chart/files/registrar-config.yaml b/chart/files/registrar-config.yaml index 480e478c..fe813061 100644 --- a/chart/files/registrar-config.yaml +++ b/chart/files/registrar-config.yaml @@ -33,11 +33,16 @@ backends: kwargs: instance_base_path: /var/www/pvs/dev instance_name: pvs_instance - mapping: + {{- $product_levels := list }} {{- range $product_type_name, $product_type := .Values.config.products.types }} + {{- range $_, $collection_def := $.Values.config.collections }} + {{- if has $product_type_name $collection_def.product_types }} + {{ $product_levels = concat $product_levels ( get $collection_def "product_levels" | default list ) }} + {{- end }} + {{- end }} {{ $product_type_name }}: - {{- range $level := list "Level_1" "Level_3" "Level-2A" }} + {{- range $level := $product_levels | uniq }} {{ $level }}: product_type_name: {{ $product_type_name | quote }} collections: -- GitLab From 17b3897de7e3de46c56779d0ad44a44dec8e281c Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Thu, 19 Nov 2020 09:05:41 -0500 Subject: [PATCH 13/26] add atom enclosure link relation --- core/registrar/metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/registrar/metadata.py b/core/registrar/metadata.py index 065dbf8f..710ea8b6 100644 --- a/core/registrar/metadata.py +++ b/core/registrar/metadata.py @@ -88,7 +88,7 @@ def gen_iso_metadata(base_url: str, esa_xml: bytes, inspire_xml: bytes) -> str: mcf['distribution'][product_manifest] = { 'url': product_manifest_link, - 'type': 'WWW:LINK', + 'type': 'enclosure', 'name': product_manifest, 'description': product_manifest, 'function': 'download' -- GitLab From e770a6b46f183b0854d43774824c128563bdc37e Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Fri, 20 Nov 2020 09:32:55 -0500 Subject: [PATCH 14/26] fix ref --- core/registrar/metadata.py | 63 +++++++++++++++++++------------------- 1 file changed, 32 insertions(+), 31 deletions(-) diff --git a/core/registrar/metadata.py b/core/registrar/metadata.py index 710ea8b6..42a85b89 100644 --- a/core/registrar/metadata.py +++ b/core/registrar/metadata.py @@ -8,32 +8,33 @@ LANGUAGE = 'eng' logger = logging.getLogger(__name__) -mcf = { - 'mcf': { - 'version': '1.0' - }, - 'metadata': { - 'language': LANGUAGE, - 'charset': 'utf8' - }, - 'spatial': { - 'datatype': 'grid', - 'geomtype': 'solid' - }, - 'identification': { - 'charset': 'utf8', - 'language': 'missing', - 'keywords': {} - }, - 'contact': { - 'main': {}, - 'distribution': {} - }, - 'distribution': {} -} - def gen_iso_metadata(base_url: str, esa_xml: bytes, inspire_xml: bytes) -> str: + + mcf = { + 'mcf': { + 'version': '1.0' + }, + 'metadata': { + 'language': LANGUAGE, + 'charset': 'utf8' + }, + 'spatial': { + 'datatype': 'grid', + 'geomtype': 'solid' + }, + 'identification': { + 'charset': 'utf8', + 'language': 'missing', + 'keywords': {} + }, + 'contact': { + 'main': {}, + 'distribution': {} + }, + 'distribution': {} + } + exml = etree.fromstring(esa_xml) ixml = etree.fromstring(inspire_xml) @@ -41,7 +42,7 @@ def gen_iso_metadata(base_url: str, esa_xml: bytes, inspire_xml: bytes) -> str: product_manifest = exml.xpath('//PRODUCT_URI/text()')[0] product_identifier = product_manifest.replace('.SAFE', '') - product_manifest_link = f"{base_url}/{product_manifest}" + product_manifest_link = f'{base_url}/{product_manifest}' mcf['metadata']['identifier'] = product_identifier mcf['metadata']['hierarchylevel'] = m.hierarchy @@ -89,8 +90,8 @@ def gen_iso_metadata(base_url: str, esa_xml: bytes, inspire_xml: bytes) -> str: mcf['distribution'][product_manifest] = { 'url': product_manifest_link, 'type': 'enclosure', - 'name': product_manifest, - 'description': product_manifest, + 'name': 'product', + 'description': 'product', 'function': 'download' } @@ -109,15 +110,15 @@ def gen_iso_metadata(base_url: str, esa_xml: bytes, inspire_xml: bytes) -> str: for image_file in exml.xpath('//Product_Organisation//IMAGE_FILE/text()'): dist = { - 'url': f"{product_manifest_link}/{image_file}.{file_extension}", + 'url': f'{product_manifest_link}/{image_file}.{file_extension}', 'type': mime_type, - 'name': image_file, - 'description': image_file, + 'name': 'granule', + 'description': 'granule', 'function': 'download' } mcf['distribution'][image_file] = dist - logger.debug("MCF: {}".format(mcf)) + logger.debug('MCF: {}'.format(mcf)) iso_os = ISO19139OutputSchema() -- GitLab From 0ff7192b2d522f451a630e018f1bb98a8027c30d Mon Sep 17 00:00:00 2001 From: Fabian Schindler Date: Fri, 20 Nov 2020 17:06:19 +0100 Subject: [PATCH 15/26] Allowing backend configuration via Python path --- core/registrar/backend.py | 26 +++++++++++++++----------- core/registrar/registrar.py | 3 +-- core/registrar/utils.py | 12 ++++++++++++ 3 files changed, 28 insertions(+), 13 deletions(-) diff --git a/core/registrar/backend.py b/core/registrar/backend.py index 800c8461..189bc218 100644 --- a/core/registrar/backend.py +++ b/core/registrar/backend.py @@ -17,6 +17,7 @@ from .exceptions import RegistrationError from .context import Context from .source import Source, LocalSource, S3Source, SwiftSource from .metadata import gen_iso_metadata +from .utils import import_by_path logger = logging.getLogger(__name__) @@ -316,20 +317,23 @@ class EOxServerBackend(Backend): BACKENDS = { 'eoxserver': EOxServerBackend, - 'pycsw': PycswBackend } def get_backends(config: dict, path: str) -> List[Backend]: - cfg_backends = config['backends'] - - backends = [ - BACKENDS[cfg_backend['type']]( - *cfg_backend.get('args', []), - **cfg_backend.get('kwargs', {}), - ) - for cfg_backend in cfg_backends - if not cfg_backend.get('filter') or re.match(cfg_backend['filter'], path) - ] + backends = [] + for cfg_backend in config['backends']: + if not cfg_backend.get('filter') or re.match(cfg_backend['filter'], path): + if 'type' in cfg_backend: + backend_cls = BACKENDS[cfg_backend['type']] + else: + handler_cls = import_by_path(cfg_backend['path']) + + backends.append( + backend_cls( + *cfg_backend.get('args', []), + **cfg_backend.get('kwargs', {}), + ) + ) if not backends: raise RegistrationError(f'Could not find a suitable backend for the path {path}') diff --git a/core/registrar/registrar.py b/core/registrar/registrar.py index 968ff054..1c65006b 100644 --- a/core/registrar/registrar.py +++ b/core/registrar/registrar.py @@ -43,8 +43,7 @@ def register_file(config: dict, path: str, replace: bool=False): def _get_handlers(config, name): handlers = [] for handler_def in config.get(name, []): - module_path, _, handler_name = handler_def['path'].rpartition('.') - handler_cls = getattr(importlib.import_module(module_path), handler_name) + handler_cls = import_by_path(handler_def['path']) handlers.append( handler_cls( *handler_def.get('args', []), diff --git a/core/registrar/utils.py b/core/registrar/utils.py index 8a522c91..a5fb0255 100644 --- a/core/registrar/utils.py +++ b/core/registrar/utils.py @@ -1,3 +1,5 @@ +import importlib + def isoformat(dt): """ Formats a datetime object to an ISO string. Timezone naive datetimes are are treated as UTC Zulu. UTC Zulu is expressed with the proper "Z" @@ -10,3 +12,13 @@ def isoformat(dt): dt = dt.replace(tzinfo=None) return dt.isoformat("T") + "Z" return dt.isoformat("T") + + +def import_by_path(path: str): + """ Imports the object from the referenced module. + + :param path: the dotted Python path, where the last element is the + object in the referenced module. + """ + module_path, _, object_name = path.rpartition('.') + return getattr(importlib.import_module(module_path), object_name) -- GitLab From 7ebf5f54a2b4855757b0baf5a1c1ec7fe5c54991 Mon Sep 17 00:00:00 2001 From: Fabian Schindler Date: Tue, 24 Nov 2020 09:35:15 +0100 Subject: [PATCH 16/26] Fixing typo --- core/registrar/backend.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/registrar/backend.py b/core/registrar/backend.py index 189bc218..01566795 100644 --- a/core/registrar/backend.py +++ b/core/registrar/backend.py @@ -326,7 +326,7 @@ def get_backends(config: dict, path: str) -> List[Backend]: if 'type' in cfg_backend: backend_cls = BACKENDS[cfg_backend['type']] else: - handler_cls = import_by_path(cfg_backend['path']) + backend_cls = import_by_path(cfg_backend['path']) backends.append( backend_cls( -- GitLab From f0d5e2aef327441eb6955b4ad4d59cb194caad39 Mon Sep 17 00:00:00 2001 From: Fabian Schindler Date: Tue, 24 Nov 2020 15:43:59 +0100 Subject: [PATCH 17/26] Cleanup of pycsw related stuff --- core/registrar/backend.py | 79 ----------------------- core/registrar/metadata.py | 125 ------------------------------------- 2 files changed, 204 deletions(-) delete mode 100644 core/registrar/metadata.py diff --git a/core/registrar/backend.py b/core/registrar/backend.py index 01566795..2076e4db 100644 --- a/core/registrar/backend.py +++ b/core/registrar/backend.py @@ -9,14 +9,10 @@ import django from django.db import transaction from django.contrib.gis.geos import GEOSGeometry, Polygon from lxml import etree -from pycsw.core import metadata, repository, util -import pycsw.core.admin -import pycsw.core.config from .exceptions import RegistrationError from .context import Context from .source import Source, LocalSource, S3Source, SwiftSource -from .metadata import gen_iso_metadata from .utils import import_by_path @@ -31,81 +27,6 @@ class Backend: def register_item(self, item: Context) -> RegistrationResult: raise NotImplementedError -class PycswBackend(Backend): - def __init__(self, instance_base_path: str, instance_name: str, mapping: dict, simplify_footprint_tolerance: int=None): - logger.debug('Setting up static context') - self.context = pycsw.core.config.StaticContext() - - logger.debug('Initializing pycsw repository') - self.repo = repository.Repository(os.environ.get('PYCSW_REPOSITORY_DATABASE_URI'), self.context, table='records') - - def exists(self, source: Source, item: Context) -> bool: - # TODO: sort out identifier problem in ISO XML - logger.info(self.repo.query(constraint={})) - if self.repo.query_ids([item.identifier]): - logger.info('identifier exists') - return True - return False - - def register(self, source: Source, item: Context, replace: bool) -> RegistrationResult: - - ingest_fail = False - esa_xml_local = '/tmp/esa-metadata.xml' - inspire_xml_local = '/tmp/inspire-metadata.xml' - - esa_xml = item.metadata_files[0] - logger.info(f"ESA XML metadata file: {esa_xml}") - - inspire_xml = os.path.dirname(item.metadata_files[0]) + "/INSPIRE.xml" - logger.info(f"INSPIRE XML metadata file: {inspire_xml}") - - base_url = 's3://{}'.format(os.path.split(os.path.dirname(esa_xml))[0]) - - try: - source.get_file(inspire_xml, inspire_xml_local) - source.get_file(esa_xml, esa_xml_local) - except Exception as err: - logger.error(err) - return False - - logger.info('Generating ISO XML based on ESA and INSPIRE XML') - with open(esa_xml_local, 'rb') as a, open(inspire_xml_local, 'rb') as b: - iso_metadata = gen_iso_metadata(base_url, a.read(), b.read()) - - logger.debug('Parsing XML') - try: - xml = etree.fromstring(iso_metadata) - except Exception as err: - logger.error('XML parsing failed: {}'.format(err)) - return False - - logger.debug('Processing metadata') - try: - record = metadata.parse_record(self.context, xml, self.repo)[0] - record.xml = record.xml.decode() - logger.info(f"identifier: {record.identifier}") - except Exception as err: - logger.error('Metadata parsing failed: {}'.format(err)) - return False - - logger.debug('Inserting record') - try: - self.repo.insert(record, 'local', util.get_today_and_now()) - logger.info('record inserted') - except Exception as err: - ingest_fail = True - logger.error('record insertion failed: {}'.format(err)) - - if ingest_fail and replace: - logger.info('Updating record') - try: - self.repo.update(record) - logger.info('record updated') - except Exception as err: - logger.error('record update failed: {}'.format(err)) - - return True - class EOxServerBackend(Backend): def __init__(self, instance_base_path: str, instance_name: str, mapping: dict, simplify_footprint_tolerance: int=None): diff --git a/core/registrar/metadata.py b/core/registrar/metadata.py deleted file mode 100644 index 42a85b89..00000000 --- a/core/registrar/metadata.py +++ /dev/null @@ -1,125 +0,0 @@ -import logging - -from lxml import etree -from owslib.iso import MD_Metadata -from pygeometa.schemas.iso19139 import ISO19139OutputSchema - -LANGUAGE = 'eng' - -logger = logging.getLogger(__name__) - - -def gen_iso_metadata(base_url: str, esa_xml: bytes, inspire_xml: bytes) -> str: - - mcf = { - 'mcf': { - 'version': '1.0' - }, - 'metadata': { - 'language': LANGUAGE, - 'charset': 'utf8' - }, - 'spatial': { - 'datatype': 'grid', - 'geomtype': 'solid' - }, - 'identification': { - 'charset': 'utf8', - 'language': 'missing', - 'keywords': {} - }, - 'contact': { - 'main': {}, - 'distribution': {} - }, - 'distribution': {} - } - - exml = etree.fromstring(esa_xml) - ixml = etree.fromstring(inspire_xml) - - m = MD_Metadata(ixml) - - product_manifest = exml.xpath('//PRODUCT_URI/text()')[0] - product_identifier = product_manifest.replace('.SAFE', '') - product_manifest_link = f'{base_url}/{product_manifest}' - - mcf['metadata']['identifier'] = product_identifier - mcf['metadata']['hierarchylevel'] = m.hierarchy - mcf['metadata']['datestamp'] = exml.xpath('//Product_Info/GENERATION_TIME/text()')[0] - - gfp = exml.xpath('//Global_Footprint/EXT_POS_LIST/text()')[0].split() - - minx = gfp[1] - miny = gfp[0] - maxx = gfp[5] - maxy = gfp[4] - - mcf['identification']['extents'] = { - 'spatial': [{ - 'bbox': [minx, miny, maxx, maxy], - 'crs': 4326 - }], - 'temporal': [{ - 'begin': exml.xpath('//Product_Info/PRODUCT_START_TIME/text()')[0], - 'end': exml.xpath('//Product_Info/PRODUCT_STOP_TIME/text()')[0] - }] - } - - mcf['identification']['title'] = product_identifier - mcf['identification']['abstract'] = product_identifier - - mcf['identification']['dates'] = { - 'creation': mcf['metadata']['datestamp'], - 'publication': mcf['metadata']['datestamp'] - } - - for i, kws in enumerate(m.identification.keywords): - kw_set = 'kw{}'.format(i) - - mcf['identification']['keywords'][kw_set] = { - 'keywords': kws['keywords'] - } - mcf['identification']['keywords'][kw_set]['keywords_type'] = kws['type'] or 'theme' - - mcf['identification']['topiccategory'] = [m.identification.topiccategory[0]] - mcf['identification']['status'] = 'onGoing' - mcf['identification']['maintenancefrequency'] = 'continual' - mcf['identification']['accessconstraints'] = m.identification.accessconstraints[0] - - mcf['distribution'][product_manifest] = { - 'url': product_manifest_link, - 'type': 'enclosure', - 'name': 'product', - 'description': 'product', - 'function': 'download' - } - - product_format = exml.xpath('//Granule_List/Granule/@imageFormat')[0] - - if product_format == 'JPEG2000': - mime_type = 'image/jp2' - file_extension = 'jp2' - elif product_format == 'TIFF': - mime_type = 'image/x.geotiff' - file_extension = 'tif' - else: - logger.warning('unknown product format: {}'.format(product_format)) - mime_type = 'NA' - file_extension = 'NA' - - for image_file in exml.xpath('//Product_Organisation//IMAGE_FILE/text()'): - dist = { - 'url': f'{product_manifest_link}/{image_file}.{file_extension}', - 'type': mime_type, - 'name': 'granule', - 'description': 'granule', - 'function': 'download' - } - mcf['distribution'][image_file] = dist - - logger.debug('MCF: {}'.format(mcf)) - - iso_os = ISO19139OutputSchema() - - return iso_os.write(mcf) -- GitLab From 3de016629aad81b1ca2ce7ecfc569fc3e2e0ac08 Mon Sep 17 00:00:00 2001 From: Fabian Schindler Date: Tue, 24 Nov 2020 15:44:43 +0100 Subject: [PATCH 18/26] Making image of registrar/renderer configurable --- chart/templates/registrar-deployment.yaml | 2 +- chart/templates/renderer-deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/chart/templates/registrar-deployment.yaml b/chart/templates/registrar-deployment.yaml index b0d39284..e916a428 100644 --- a/chart/templates/registrar-deployment.yaml +++ b/chart/templates/registrar-deployment.yaml @@ -29,7 +29,7 @@ spec: {{- end }} containers: - name: {{ .Chart.Name }}-registrar - image: "{{ .Values.image.repository | default "registry.gitlab.eox.at/esa/prism/vs" }}/pvs_core:{{ .Values.image.tag | default .Chart.AppVersion }}" + image: "{{ .Values.image.repository | default "registry.gitlab.eox.at/esa/prism/vs" }}/{{ .Values.renderer.image | default "pvs_core" }}:{{ .Values.image.tag | default .Chart.AppVersion }}" imagePullPolicy: {{ .Values.image.pullPolicy }} resources: {{- toYaml .Values.registrar.resources | nindent 12 }} diff --git a/chart/templates/renderer-deployment.yaml b/chart/templates/renderer-deployment.yaml index de0f1beb..7e39fd7d 100644 --- a/chart/templates/renderer-deployment.yaml +++ b/chart/templates/renderer-deployment.yaml @@ -29,7 +29,7 @@ spec: {{- end }} containers: - name: {{ .Chart.Name }}-renderer - image: "{{ .Values.image.repository | default "registry.gitlab.eox.at/esa/prism/vs" }}/pvs_core:{{ .Values.image.tag | default .Chart.AppVersion }}" + image: "{{ .Values.image.repository | default "registry.gitlab.eox.at/esa/prism/vs" }}/{{ .Values.renderer.image | default "pvs_core" }}:{{ .Values.image.tag | default .Chart.AppVersion }}" imagePullPolicy: {{ .Values.image.pullPolicy }} ports: - name: http -- GitLab From dde9cf810f2cf8d94ffc731c884f2403aa746076 Mon Sep 17 00:00:00 2001 From: Fabian Schindler Date: Wed, 25 Nov 2020 19:28:08 +0100 Subject: [PATCH 19/26] Reverting pycsw dependencies --- core/Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core/Dockerfile b/core/Dockerfile index 274345db..eba367f4 100644 --- a/core/Dockerfile +++ b/core/Dockerfile @@ -43,8 +43,7 @@ RUN apt update && \ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* RUN pip3 install . && \ - pip3 install python-keystoneclient python-swiftclient redis click setuptools jsonschema boto3 pyproj==2.0.2 pycsw SQLAlchemy && \ - pip3 install https://github.com/geopython/pygeometa/archive/master.zip + pip3 install python-keystoneclient python-swiftclient redis click setuptools jsonschema boto3 ENV INSTANCE_ID="prism-view-server_core" \ INSTANCE_NAME="pvs_instance"\ -- GitLab From b809413cad92bd6b2a4b5f329f138184528bb04a Mon Sep 17 00:00:00 2001 From: Fabian Schindler Date: Wed, 25 Nov 2020 19:43:53 +0100 Subject: [PATCH 20/26] Removing pycsw configuration --- chart/templates/registrar-deployment.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/chart/templates/registrar-deployment.yaml b/chart/templates/registrar-deployment.yaml index e916a428..b2599209 100644 --- a/chart/templates/registrar-deployment.yaml +++ b/chart/templates/registrar-deployment.yaml @@ -70,8 +70,6 @@ spec: value: /wait-initialized.sh - name: WAIT_SERVICES value: {{ .Release.Name }}-database:{{ .Values.config.database.DB_PORT }} {{ .Release.Name }}-redis-master:{{ .Values.config.redis.REDIS_PORT }} - - name: PYCSW_REPOSITORY_DATABASE_URI - value: "postgresql://postgres:mypass@resource-catalogue-db/pycsw" volumeMounts: - mountPath: /init-db.sh name: init-db -- GitLab From 2b19b8169df18ce469b6ffe54f2eec4df6708c0f Mon Sep 17 00:00:00 2001 From: Fabian Schindler Date: Wed, 25 Nov 2020 19:46:10 +0100 Subject: [PATCH 21/26] Improvind Helm readme --- chart/README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/chart/README.md b/chart/README.md index acc5e453..6df4091a 100644 --- a/chart/README.md +++ b/chart/README.md @@ -8,3 +8,16 @@ helm dependency update helm template testing . --output-dir ../tmp/ -f values.yaml ``` + + +## Publishing chart + +Currently this approach does not work 100% such repos cannot be declared [in dependencies of other charts](https://github.com/helm/helm/issues/6593) + + +```bash + +helm registry login registry.gitlab.eox.at +helm chart save . registry.gitlab.eox.at/esa/prism/vs/chart +helm chart push registry.gitlab.eox.at/esa/prism/vs/chart:0.1.0-beta.1 +``` -- GitLab From 99726d500e8ad9ec0a1387ddab5bff68bad3b412 Mon Sep 17 00:00:00 2001 From: Fabian Schindler Date: Thu, 26 Nov 2020 15:12:27 +0100 Subject: [PATCH 22/26] Automatic chart deployment --- .gitlab-ci.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 74dd9a1c..255d3b1a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -72,13 +72,20 @@ build-master-staging: - docker build --cache-from "$IMAGE_6":"$TAG_USED" -t "$IMAGE_6":dev -t "$IMAGE_6":"$TAG_USED" ingestor/ - cd ./testing && ./gitlab_test.sh - if [ $? -ne 0 ]; then exit 1; fi # actually fail build + - cd - - docker push "$IMAGE_1":"$TAG_USED" - docker push "$IMAGE_2":"$TAG_USED" - docker push "$IMAGE_3":"$TAG_USED" - docker push "$IMAGE_4":"$TAG_USED" - docker push "$IMAGE_5":"$TAG_USED" - docker push "$IMAGE_6":"$TAG_USED" + # deploy helm charts + - cd chart + - helm package . + # TODO: figure out filename + - curl -u $CHART_DEPLOY_USERNAME:$CHART_DEPLOY_PASSWORD -v -X POST https://charts-public.hub.eox.at/api/charts --data-binary @vs-0.1.0-beta.1.tgz only: + - eoepca-staging - staging - master except: @@ -115,6 +122,7 @@ build: - cd ./testing && ./gitlab_test.sh - if [ $? -ne 0 ]; then exit 1; fi # actually fail build except: + - eoepca-staging - tags - staging - master -- GitLab From 71df715677892e20f2d1d010d7bed5759ae5dd79 Mon Sep 17 00:00:00 2001 From: Fabian Schindler Date: Mon, 30 Nov 2020 12:00:38 +0100 Subject: [PATCH 23/26] Adding missing import --- core/registrar/registrar.py | 1 + 1 file changed, 1 insertion(+) diff --git a/core/registrar/registrar.py b/core/registrar/registrar.py index 1c65006b..f2b6e205 100644 --- a/core/registrar/registrar.py +++ b/core/registrar/registrar.py @@ -6,6 +6,7 @@ from .source import get_source from .scheme import get_scheme from .backend import get_backends from .exceptions import RegistrationError +from .utils import import_by_path logger = logging.getLogger(__name__) -- GitLab From 4f548501d1f471fe93a0a896e7b9962877c2f230 Mon Sep 17 00:00:00 2001 From: Lubomir Bucek Date: Mon, 30 Nov 2020 14:09:36 +0100 Subject: [PATCH 24/26] try changing ingestor filedaemon to `process_IN_CLOSE_WRITE` --- ingestor/filedaemon.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ingestor/filedaemon.py b/ingestor/filedaemon.py index 8ea7e459..dd820171 100644 --- a/ingestor/filedaemon.py +++ b/ingestor/filedaemon.py @@ -81,7 +81,7 @@ client = redis.Redis( watchmanager = pyinotify.WatchManager() class EventHandler(pyinotify.ProcessEvent): - def process_IN_CREATE(self, event): + def process_IN_CLOSE_WRITE(self, event): logger.info(f'Parsing browse file: {event.pathname}') try: with open(event.pathname) as f: @@ -100,6 +100,6 @@ class EventHandler(pyinotify.ProcessEvent): handler = EventHandler() notifier = pyinotify.Notifier(watchmanager, handler) -wdd = watchmanager.add_watch(watch_dir, pyinotify.IN_CREATE, rec=True) +wdd = watchmanager.add_watch(watch_dir, pyinotify.IN_CLOSE_WRITE, rec=True) notifier.loop() -- GitLab From 419490b9cb05d01f713d2efed3a29b6527291523 Mon Sep 17 00:00:00 2001 From: Fabian Schindler Date: Mon, 30 Nov 2020 14:10:19 +0100 Subject: [PATCH 25/26] Using different event on file polling --- ingestor/filedaemon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ingestor/filedaemon.py b/ingestor/filedaemon.py index 8ea7e459..0ebfcdf4 100644 --- a/ingestor/filedaemon.py +++ b/ingestor/filedaemon.py @@ -81,7 +81,7 @@ client = redis.Redis( watchmanager = pyinotify.WatchManager() class EventHandler(pyinotify.ProcessEvent): - def process_IN_CREATE(self, event): + def process_IN_CLOSE_WRITE(self, event): logger.info(f'Parsing browse file: {event.pathname}') try: with open(event.pathname) as f: -- GitLab From bf510d05bd3888adc56c9d060c5c195a63e1fc74 Mon Sep 17 00:00:00 2001 From: Fabian Schindler Date: Mon, 30 Nov 2020 14:37:00 +0100 Subject: [PATCH 26/26] Removing Helm related CI commands, as this will be done manually --- .gitlab-ci.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1bd42ab9..5000cb0c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -84,13 +84,7 @@ build-master-staging: - docker push "$IMAGE_5":"$TAG_USED" - docker push "$IMAGE_6":"$TAG_USED" - docker push "$IMAGE_7":"$TAG_USED" - # deploy helm charts - - cd chart - - helm package . - # TODO: figure out filename - - curl -u $CHART_DEPLOY_USERNAME:$CHART_DEPLOY_PASSWORD -v -X POST https://charts-public.hub.eox.at/api/charts --data-binary @vs-0.1.0-beta.1.tgz only: - - eoepca-staging - staging - master except: @@ -135,7 +129,6 @@ build: - cd ./testing && ./gitlab_test.sh - if [ $? -ne 0 ]; then exit 1; fi # actually fail build except: - - eoepca-staging - tags - staging - master -- GitLab