EOEPCA staging

.gitlab-ci.yml

@@ -76,6 +76,7 @@ build-master-staging:
     - docker build --cache-from "$IMAGE_7":latest -t "$IMAGE_7":dev -t "$IMAGE_7":"$TAG_USED" sftp/
     - cd ./testing && ./gitlab_test.sh
     - if [ $? -ne 0 ]; then exit 1; fi  # actually fail build
+    - cd -
     - docker push "$IMAGE_1":"$TAG_USED"
     - docker push "$IMAGE_2":"$TAG_USED"
     - docker push "$IMAGE_3":"$TAG_USED"

chart/README.md

@@ -8,3 +8,16 @@ helm dependency update
 helm template testing . --output-dir ../tmp/ -f values.yaml
 
 ```
+
+
+## Publishing chart
+
+Currently this approach does not work 100% such repos cannot be declared [in dependencies of other charts](https://github.com/helm/helm/issues/6593)
+
+
+```bash
+
+helm registry login registry.gitlab.eox.at
+helm chart save . registry.gitlab.eox.at/esa/prism/vs/chart
+helm chart push registry.gitlab.eox.at/esa/prism/vs/chart:0.1.0-beta.1
+```

chart/files/index.html

@@ -37,18 +37,18 @@ var config = {
         "searchDebounceTime": 500,
         "language": "en",
         "timeDomain": [
-            "2017-01-01T00:00:00Z",
-            "2019-12-31T23:59:59Z",
+            "2019-01-01T00:00:00Z",
+            "2020-12-31T23:59:59Z",
         ],
         "constrainTimeDomain": true,
         "displayTimeDomain": [
-            "2017-01-01T00:00:00Z",
-            "2019-12-31T23:59:59Z",
+            "2019-01-01T00:00:00Z",
+            "2020-12-31T23:59:59Z",
         ],
         "displayInterval": "P1096D",
         "selectedTimeDomain": [
-            "2018-08-01T00:00:00Z",
-            "2018-08-31T23:59:59Z",
+            "2020-08-01T00:00:00Z",
+            "2020-08-31T23:59:59Z",
         ],
         "selectableInterval": "P1096D",
         "timeSliderControls": true,

chart/files/registrar-config.yaml

@@ -33,11 +33,16 @@ backends:
     kwargs:
       instance_base_path: /var/www/pvs/dev
       instance_name: pvs_instance
-
       mapping:
+        {{- $product_levels := list }}
         {{- range $product_type_name, $product_type := .Values.config.products.types }}
+        {{- range $_, $collection_def := $.Values.config.collections }}
+        {{- if has $product_type_name $collection_def.product_types }}
+        {{ $product_levels = concat $product_levels ( get $collection_def "product_levels" | default list ) }}
+        {{- end }}
+        {{- end }}
         {{ $product_type_name }}:
-          {{- range $level := list "Level_1" "Level_3" "Level-2A" }}
+          {{- range $level := $product_levels | uniq }}
           {{ $level }}:
             product_type_name: {{ $product_type_name | quote }}
             collections:
@@ -48,9 +53,24 @@ backends:
               {{- end }}
             coverages:
               {{- toYaml $product_type.coverages | nindent 16 }}
+            browses:
+              {{- if hasKey $product_type "default_browse_locator" }}
+              {{ $product_type.default_browse_locator }}: null
+              {{- end }}
+              {{- range $browse_name, $browse_config := $product_type.browses }}
+              {{- if hasKey $browse_config "locator" }}
+              {{ $browse_config.locator }}: {{ $browse_name }}
+              {{- end }}
+              {{- end }}
             masks:
               {{- range $mask_name, $_ := $product_type.masks }}
               {{ $mask_name }}: {{ $mask_name }}
               {{- end }}
           {{- end }}
         {{- end }}
+  - type: pycsw
+    filter:
+    kwargs:
+      instance_base_path: /var/www/pvs/dev
+      instance_name: pvs_instance
+      mapping:

chart/templates/client-deployment.yaml

@@ -29,7 +29,7 @@ spec:
       {{- end }}
       containers:
         - name: {{ .Chart.Name }}-client
-          image: 'registry.gitlab.eox.at/esa/prism/vs/pvs_client:{{ .Values.image.tag | default .Chart.AppVersion }}'
+          image: '{{ .Values.image.repository | default "registry.gitlab.eox.at/esa/prism/vs" }}/pvs_client:{{ .Values.image.tag | default .Chart.AppVersion }}'
           imagePullPolicy: {{ .Values.image.pullPolicy }}
           ports:
             - name: http

chart/templates/registrar-deployment.yaml

@@ -29,7 +29,7 @@ spec:
       {{- end }}
       containers:
         - name: {{ .Chart.Name }}-registrar
-          image: "registry.gitlab.eox.at/esa/prism/vs/pvs_core:{{ .Values.image.tag | default .Chart.AppVersion }}"
+          image: "{{ .Values.image.repository | default "registry.gitlab.eox.at/esa/prism/vs" }}/{{ .Values.renderer.image | default "pvs_core" }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
           imagePullPolicy: {{ .Values.image.pullPolicy }}
           resources:
             {{- toYaml .Values.registrar.resources | nindent 12 }}

chart/templates/renderer-deployment.yaml

@@ -29,7 +29,7 @@ spec:
       {{- end }}
       containers:
         - name: {{ .Chart.Name }}-renderer
-          image: "registry.gitlab.eox.at/esa/prism/vs/pvs_core:{{ .Values.image.tag | default .Chart.AppVersion }}"
+          image: "{{ .Values.image.repository | default "registry.gitlab.eox.at/esa/prism/vs" }}/{{ .Values.renderer.image | default "pvs_core" }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
           imagePullPolicy: {{ .Values.image.pullPolicy }}
           ports:
             - name: http

core/registrar/backend.py

@@ -9,11 +9,13 @@ import django
 from django.db import transaction
 from django.db.models import Count
 from django.contrib.gis.geos import GEOSGeometry, Polygon
+from lxml import etree
 from osgeo import gdal
 
 from .exceptions import RegistrationError
 from .context import Context
 from .source import Source, LocalSource, S3Source, SwiftSource
+from .utils import import_by_path
 
 
 logger = logging.getLogger(__name__)
@@ -251,20 +253,24 @@ class EOxServerBackend(Backend):
 
 
 BACKENDS = {
-    'eoxserver': EOxServerBackend
+    'eoxserver': EOxServerBackend,
 }
 
 def get_backends(config: dict, path: str) -> List[Backend]:
-    cfg_backends = config['backends']
-
-    backends = [
-        BACKENDS[cfg_backend['type']](
-            *cfg_backend.get('args', []),
-            **cfg_backend.get('kwargs', {}),
-        )
-        for cfg_backend in cfg_backends
-        if not cfg_backend.get('filter') or re.match(cfg_backend['filter'], path)
-    ]
+    backends = []
+    for cfg_backend in config['backends']:
+        if not cfg_backend.get('filter') or re.match(cfg_backend['filter'], path):
+            if 'type' in cfg_backend:
+                backend_cls = BACKENDS[cfg_backend['type']]
+            else:
+                backend_cls = import_by_path(cfg_backend['path'])
+
+            backends.append(
+                backend_cls(
+                    *cfg_backend.get('args', []),
+                    **cfg_backend.get('kwargs', {}),
+                )
+            )
 
     if not backends:
         raise RegistrationError(f'Could not find a suitable backend for the path {path}')

core/registrar/registrar.py

@@ -6,6 +6,7 @@ from .source import get_source
 from .scheme import get_scheme
 from .backend import get_backends
 from .exceptions import RegistrationError
+from .utils import import_by_path
 
 
 logger = logging.getLogger(__name__)
@@ -43,8 +44,7 @@ def register_file(config: dict, path: str, replace: bool=False):
 def _get_handlers(config, name):
     handlers = []
     for handler_def in config.get(name, []):
-        module_path, _, handler_name = handler_def['path'].rpartition('.')
-        handler_cls = getattr(importlib.import_module(module_path), handler_name)
+        handler_cls = import_by_path(handler_def['path'])
         handlers.append(
             handler_cls(
                 *handler_def.get('args', []),

core/registrar/utils.py

+import importlib
+
 def isoformat(dt):
     """ Formats a datetime object to an ISO string. Timezone naive datetimes are
         are treated as UTC Zulu. UTC Zulu is expressed with the proper "Z"
@@ -10,3 +12,13 @@ def isoformat(dt):
         dt = dt.replace(tzinfo=None)
         return dt.isoformat("T") + "Z"
     return dt.isoformat("T")
+
+
+def import_by_path(path: str):
+    """ Imports the object from the referenced module.
+
+        :param path: the dotted Python path, where the last element is the
+                     object in the referenced module.
+    """
+    module_path, _, object_name = path.rpartition('.')
+    return getattr(importlib.import_module(module_path), object_name)

ingestor/filedaemon.py

@@ -81,7 +81,7 @@ client = redis.Redis(
 watchmanager = pyinotify.WatchManager()
 
 class EventHandler(pyinotify.ProcessEvent):
-    def process_IN_CREATE(self, event):
+    def process_IN_CLOSE_WRITE(self, event):
         logger.info(f'Parsing browse file: {event.pathname}')
         try:
             with open(event.pathname) as f:
@@ -100,6 +100,6 @@ class EventHandler(pyinotify.ProcessEvent):
 handler = EventHandler()
 notifier = pyinotify.Notifier(watchmanager, handler)
 
-wdd = watchmanager.add_watch(watch_dir, pyinotify.IN_CREATE, rec=True)
+wdd = watchmanager.add_watch(watch_dir, pyinotify.IN_CLOSE_WRITE, rec=True)
 
 notifier.loop()