diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c133a786bb6739ee1679902dc78858a9e2fb050a..5000cb0c89701a94cae52cb6514eb8673ac7ab55 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -76,6 +76,7 @@ build-master-staging:
- docker build --cache-from "$IMAGE_7":latest -t "$IMAGE_7":dev -t "$IMAGE_7":"$TAG_USED" sftp/
- cd ./testing && ./gitlab_test.sh
- if [ $? -ne 0 ]; then exit 1; fi # actually fail build
+ - cd -
- docker push "$IMAGE_1":"$TAG_USED"
- docker push "$IMAGE_2":"$TAG_USED"
- docker push "$IMAGE_3":"$TAG_USED"
diff --git a/chart/README.md b/chart/README.md
index acc5e4533912b96530d015174f4b1566a5815fbe..6df4091a929304de4e97a95a4435f86eb72101b5 100644
--- a/chart/README.md
+++ b/chart/README.md
@@ -8,3 +8,16 @@ helm dependency update
helm template testing . --output-dir ../tmp/ -f values.yaml
```
+
+
+## Publishing chart
+
+Currently this approach does not work 100% such repos cannot be declared [in dependencies of other charts](https://github.com/helm/helm/issues/6593)
+
+
+```bash
+
+helm registry login registry.gitlab.eox.at
+helm chart save . registry.gitlab.eox.at/esa/prism/vs/chart
+helm chart push registry.gitlab.eox.at/esa/prism/vs/chart:0.1.0-beta.1
+```
diff --git a/chart/files/index.html b/chart/files/index.html
index b74ad39fceb75c740a2aa076b90b5afaf62a7c37..0e986eeffad3e2be05776efeacf9469dacd71b32 100644
--- a/chart/files/index.html
+++ b/chart/files/index.html
@@ -37,18 +37,18 @@ var config = {
"searchDebounceTime": 500,
"language": "en",
"timeDomain": [
- "2017-01-01T00:00:00Z",
- "2019-12-31T23:59:59Z",
+ "2019-01-01T00:00:00Z",
+ "2020-12-31T23:59:59Z",
],
"constrainTimeDomain": true,
"displayTimeDomain": [
- "2017-01-01T00:00:00Z",
- "2019-12-31T23:59:59Z",
+ "2019-01-01T00:00:00Z",
+ "2020-12-31T23:59:59Z",
],
"displayInterval": "P1096D",
"selectedTimeDomain": [
- "2018-08-01T00:00:00Z",
- "2018-08-31T23:59:59Z",
+ "2020-08-01T00:00:00Z",
+ "2020-08-31T23:59:59Z",
],
"selectableInterval": "P1096D",
"timeSliderControls": true,
diff --git a/chart/files/registrar-config.yaml b/chart/files/registrar-config.yaml
index 66a887eabf12d3a32125764029cb7670fb619804..fe813061994345adf2d71c26f5742e6995ab5c77 100644
--- a/chart/files/registrar-config.yaml
+++ b/chart/files/registrar-config.yaml
@@ -33,11 +33,16 @@ backends:
kwargs:
instance_base_path: /var/www/pvs/dev
instance_name: pvs_instance
-
mapping:
+ {{- $product_levels := list }}
{{- range $product_type_name, $product_type := .Values.config.products.types }}
+ {{- range $_, $collection_def := $.Values.config.collections }}
+ {{- if has $product_type_name $collection_def.product_types }}
+ {{ $product_levels = concat $product_levels ( get $collection_def "product_levels" | default list ) }}
+ {{- end }}
+ {{- end }}
{{ $product_type_name }}:
- {{- range $level := list "Level_1" "Level_3" "Level-2A" }}
+ {{- range $level := $product_levels | uniq }}
{{ $level }}:
product_type_name: {{ $product_type_name | quote }}
collections:
@@ -48,9 +53,24 @@ backends:
{{- end }}
coverages:
{{- toYaml $product_type.coverages | nindent 16 }}
+ browses:
+ {{- if hasKey $product_type "default_browse_locator" }}
+ {{ $product_type.default_browse_locator }}: null
+ {{- end }}
+ {{- range $browse_name, $browse_config := $product_type.browses }}
+ {{- if hasKey $browse_config "locator" }}
+ {{ $browse_config.locator }}: {{ $browse_name }}
+ {{- end }}
+ {{- end }}
masks:
{{- range $mask_name, $_ := $product_type.masks }}
{{ $mask_name }}: {{ $mask_name }}
{{- end }}
{{- end }}
{{- end }}
+ - type: pycsw
+ filter:
+ kwargs:
+ instance_base_path: /var/www/pvs/dev
+ instance_name: pvs_instance
+ mapping:
diff --git a/chart/templates/client-deployment.yaml b/chart/templates/client-deployment.yaml
index ae66fa84a8155d70d9b7b7f53bfc2cce92b3b9da..c0ac1423a401176eec0284b952512ec28cadaf2c 100644
--- a/chart/templates/client-deployment.yaml
+++ b/chart/templates/client-deployment.yaml
@@ -29,7 +29,7 @@ spec:
{{- end }}
containers:
- name: {{ .Chart.Name }}-client
- image: 'registry.gitlab.eox.at/esa/prism/vs/pvs_client:{{ .Values.image.tag | default .Chart.AppVersion }}'
+ image: '{{ .Values.image.repository | default "registry.gitlab.eox.at/esa/prism/vs" }}/pvs_client:{{ .Values.image.tag | default .Chart.AppVersion }}'
imagePullPolicy: {{ .Values.image.pullPolicy }}
ports:
- name: http
diff --git a/chart/templates/registrar-deployment.yaml b/chart/templates/registrar-deployment.yaml
index 1e4b43743d8fac7f6a9dcd1135798a2e183d0719..b25992095b17ebf995f3425df4579782615cbcc7 100644
--- a/chart/templates/registrar-deployment.yaml
+++ b/chart/templates/registrar-deployment.yaml
@@ -29,7 +29,7 @@ spec:
{{- end }}
containers:
- name: {{ .Chart.Name }}-registrar
- image: "registry.gitlab.eox.at/esa/prism/vs/pvs_core:{{ .Values.image.tag | default .Chart.AppVersion }}"
+ image: "{{ .Values.image.repository | default "registry.gitlab.eox.at/esa/prism/vs" }}/{{ .Values.renderer.image | default "pvs_core" }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
imagePullPolicy: {{ .Values.image.pullPolicy }}
resources:
{{- toYaml .Values.registrar.resources | nindent 12 }}
diff --git a/chart/templates/renderer-deployment.yaml b/chart/templates/renderer-deployment.yaml
index 0bcb7cd0ca2b45f04514af432408f22b6c3439a5..7e39fd7d507e9af8098981526263062277d3ec66 100644
--- a/chart/templates/renderer-deployment.yaml
+++ b/chart/templates/renderer-deployment.yaml
@@ -29,7 +29,7 @@ spec:
{{- end }}
containers:
- name: {{ .Chart.Name }}-renderer
- image: "registry.gitlab.eox.at/esa/prism/vs/pvs_core:{{ .Values.image.tag | default .Chart.AppVersion }}"
+ image: "{{ .Values.image.repository | default "registry.gitlab.eox.at/esa/prism/vs" }}/{{ .Values.renderer.image | default "pvs_core" }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
imagePullPolicy: {{ .Values.image.pullPolicy }}
ports:
- name: http
diff --git a/core/registrar/backend.py b/core/registrar/backend.py
index b070498f519e08890f212b3e0270c4388ef2c93a..d37edf9a189d659356d039775d474c592a46ee38 100644
--- a/core/registrar/backend.py
+++ b/core/registrar/backend.py
@@ -9,11 +9,13 @@ import django
from django.db import transaction
from django.db.models import Count
from django.contrib.gis.geos import GEOSGeometry, Polygon
+from lxml import etree
from osgeo import gdal
from .exceptions import RegistrationError
from .context import Context
from .source import Source, LocalSource, S3Source, SwiftSource
+from .utils import import_by_path
logger = logging.getLogger(__name__)
@@ -251,20 +253,24 @@ class EOxServerBackend(Backend):
BACKENDS = {
- 'eoxserver': EOxServerBackend
+ 'eoxserver': EOxServerBackend,
}
def get_backends(config: dict, path: str) -> List[Backend]:
- cfg_backends = config['backends']
-
- backends = [
- BACKENDS[cfg_backend['type']](
- *cfg_backend.get('args', []),
- **cfg_backend.get('kwargs', {}),
- )
- for cfg_backend in cfg_backends
- if not cfg_backend.get('filter') or re.match(cfg_backend['filter'], path)
- ]
+ backends = []
+ for cfg_backend in config['backends']:
+ if not cfg_backend.get('filter') or re.match(cfg_backend['filter'], path):
+ if 'type' in cfg_backend:
+ backend_cls = BACKENDS[cfg_backend['type']]
+ else:
+ backend_cls = import_by_path(cfg_backend['path'])
+
+ backends.append(
+ backend_cls(
+ *cfg_backend.get('args', []),
+ **cfg_backend.get('kwargs', {}),
+ )
+ )
if not backends:
raise RegistrationError(f'Could not find a suitable backend for the path {path}')
diff --git a/core/registrar/registrar.py b/core/registrar/registrar.py
index 968ff05422939d68ec30bd83c3ad62f963a37091..f2b6e2054909e63f1acaf9f11dcdc6c544253371 100644
--- a/core/registrar/registrar.py
+++ b/core/registrar/registrar.py
@@ -6,6 +6,7 @@ from .source import get_source
from .scheme import get_scheme
from .backend import get_backends
from .exceptions import RegistrationError
+from .utils import import_by_path
logger = logging.getLogger(__name__)
@@ -43,8 +44,7 @@ def register_file(config: dict, path: str, replace: bool=False):
def _get_handlers(config, name):
handlers = []
for handler_def in config.get(name, []):
- module_path, _, handler_name = handler_def['path'].rpartition('.')
- handler_cls = getattr(importlib.import_module(module_path), handler_name)
+ handler_cls = import_by_path(handler_def['path'])
handlers.append(
handler_cls(
*handler_def.get('args', []),
diff --git a/core/registrar/utils.py b/core/registrar/utils.py
index 8a522c91ae78540930fb6edce786ba131e40b3fe..a5fb0255bf8f86398706f738917c5708d04602f3 100644
--- a/core/registrar/utils.py
+++ b/core/registrar/utils.py
@@ -1,3 +1,5 @@
+import importlib
+
def isoformat(dt):
""" Formats a datetime object to an ISO string. Timezone naive datetimes are
are treated as UTC Zulu. UTC Zulu is expressed with the proper "Z"
@@ -10,3 +12,13 @@ def isoformat(dt):
dt = dt.replace(tzinfo=None)
return dt.isoformat("T") + "Z"
return dt.isoformat("T")
+
+
+def import_by_path(path: str):
+ """ Imports the object from the referenced module.
+
+ :param path: the dotted Python path, where the last element is the
+ object in the referenced module.
+ """
+ module_path, _, object_name = path.rpartition('.')
+ return getattr(importlib.import_module(module_path), object_name)
diff --git a/ingestor/filedaemon.py b/ingestor/filedaemon.py
index 8ea7e459cc6d2d0f7adaf9d0e7830933e4f89059..dd820171b1b6d5f962ed92625e803b0c2c4f54f8 100644
--- a/ingestor/filedaemon.py
+++ b/ingestor/filedaemon.py
@@ -81,7 +81,7 @@ client = redis.Redis(
watchmanager = pyinotify.WatchManager()
class EventHandler(pyinotify.ProcessEvent):
- def process_IN_CREATE(self, event):
+ def process_IN_CLOSE_WRITE(self, event):
logger.info(f'Parsing browse file: {event.pathname}')
try:
with open(event.pathname) as f:
@@ -100,6 +100,6 @@ class EventHandler(pyinotify.ProcessEvent):
handler = EventHandler()
notifier = pyinotify.Notifier(watchmanager, handler)
-wdd = watchmanager.add_watch(watch_dir, pyinotify.IN_CREATE, rec=True)
+wdd = watchmanager.add_watch(watch_dir, pyinotify.IN_CLOSE_WRITE, rec=True)
notifier.loop()