Merge branch 'main' into 'auto-update-vs-harvester-3.0.1'

# Conflicts:
#   Chart.lock

.bumpversion.cfg

 [bumpversion]
-current_version = 2.2.11
+current_version = 2.3.1
 commit = True
 tag = True
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-(?P<release>[a-z]+)\.(?P<build>\d+))?

.gitlab-ci.yml

@@ -34,8 +34,7 @@ integration-test:
     - k3d cluster create --volume $(pwd)/data:/k3d-host/ testgitlabci-${CI_JOB_ID} --agents 1 --wait
     - kubectl cluster-info
     - kubectl apply -f testing/pvc-redis.yaml
-    # default timeout is 5 min, move to 8m for some leeway
-    - helm install my-helm-release . --wait --timeout 20m --values ./testing/values-testing.yaml --values local-secrets.yaml
+    - helm install my-helm-release . --wait --timeout 10m --values ./testing/values-testing.yaml --values local-secrets.yaml
     - kubectl get po -A
   script:
     - cd testing && ./run_integration_test.sh
@@ -45,6 +44,8 @@ integration-test:
     # fetch all pod logs
     - mkdir "${CI_PROJECT_DIR}/logs"
     - kubectl get po -o name | while read POD ; do kubectl logs "$POD" >> "${CI_PROJECT_DIR}/logs/$(echo $POD | cut -d'/' -f2).txt" ; done || true
+    # fetch all pod events - to troubleshoot not-starting pods
+    - kubectl get po -o name | while read POD ; do kubectl get events --field-selector involvedObject.name=$(echo $POD | cut -d'/' -f2) >> "${CI_PROJECT_DIR}/logs/$(echo $POD | cut -d'/' -f2).txt" ; done || true
     - k3d cluster delete testgitlabci-${CI_JOB_ID}
   artifacts:
     when: always

Chart.lock

@@ -7,19 +7,19 @@ dependencies:
   version: 10.9.0
 - name: vs-cache
   repository: https://charts-public.hub.eox.at
-  version: 2.0.26
+  version: 2.0.27
 - name: vs-seeder
   repository: https://charts-public.hub.eox.at
-  version: 2.0.26
+  version: 2.0.27
 - name: vs-client
   repository: https://charts-public.hub.eox.at
   version: 2.1.2
 - name: vs-renderer
   repository: https://charts-public.hub.eox.at
-  version: 3.2.5-alpha.14
+  version: 3.3.1
 - name: vs-registrar
   repository: https://charts-public.hub.eox.at
-  version: 3.2.5-alpha.14
+  version: 3.3.1
 - name: vs-scheduler
   repository: https://charts-public.hub.eox.at
   version: 2.0.3
@@ -31,9 +31,9 @@ dependencies:
   version: 2.0.12
 - name: vs-preprocessor
   repository: https://charts-public.hub.eox.at
-  version: 2.1.15
+  version: 2.1.17
 - name: vs-preprocessor-v3
   repository: https://charts-public.hub.eox.at
   version: 3.3.0
-digest: sha256:12e6f7e49e575e55e00bc9a726f497ee9d404a96b473c37f153a8df373c62073
-generated: "2022-12-07T20:57:43.116833179Z"
+digest: sha256:5a665bee14f86854163c7337065f36cf6d5eeb01fda8c8803a9e6810851355d7
+generated: "2023-02-16T13:25:45.781564457Z"

Chart.yaml

@@ -5,10 +5,10 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: "2.2.11"
+version: "2.3.1"
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.
-appVersion: "2.2.11"
+appVersion: "2.3.1"
 maintainers:
   - name: EOX IT Services GmbH
     url: https://eox.at
@@ -22,11 +22,11 @@ dependencies:
     repository: "https://raw.githubusercontent.com/bitnami/charts/eb5f9a9513d987b519f0ecd732e7031241c50328/bitnami"
     alias: redis
   - name: vs-cache
-    version: "2.0.26"
+    version: "2.0.27"
     repository: "https://charts-public.hub.eox.at"
     alias: cache
   - name: vs-seeder
-    version: "2.0.26"
+    version: "2.0.27"
     repository: "https://charts-public.hub.eox.at"
     alias: seeder
   - name: vs-client
@@ -34,11 +34,11 @@ dependencies:
     repository: "https://charts-public.hub.eox.at"
     alias: client
   - name: vs-renderer
-    version: "3.2.5-alpha.14"
+    version: "3.3.1"
     repository: "https://charts-public.hub.eox.at"
     alias: renderer
   - name: vs-registrar
-    version: "3.2.5-alpha.14"
+    version: "3.3.1"
     repository: "https://charts-public.hub.eox.at"
     alias: registrar
   - name: vs-scheduler
@@ -54,7 +54,7 @@ dependencies:
     repository: "https://charts-public.hub.eox.at"
     alias: ingestor
   - name: vs-preprocessor
-    version: "2.1.15"
+    version: "2.1.17"
     repository: "https://charts-public.hub.eox.at"
     alias: preprocessor
   - name: vs-preprocessor-v3

local-secrets.yaml

@@ -2,11 +2,11 @@ global:
   storage:
     source:
       type: "local"
-      storage_path: "/mnt/data/"
+      root_directory: "/mnt/data/"
     data:
       directory-data:
         type: "directory"
         root_directory: "/mnt/data/"
     target:
       type: "local"
-      storage_path: "/mnt/data/preprocessor_results/"
\ No newline at end of file
+      root_directory: "/mnt/data/preprocessor_results/"

testing/demo_product_list.json

@@ -1114,7 +1114,36 @@
 		25.4434753140000005
     ],
 
+	"stac_extensions": []
+},
+{
+	"type": "Feature",
+	"stac_version": "1.0.0",
+	"id": "urn:eop:PHR:MULTISPECTRAL_0.5m:DS_PHR1B_201707271114024_FR1_PX_W006N35_0918_03164_b7e7",
+	"properties": {
+		"product_type": "PHR_FUS__3",
+		"start_datetime": "2017-07-27T11:14:02Z",
+		"end_datetime": "2017-07-27T11:14:07Z",
+		"datetime": "2017-07-27T12:25:55Z"
+	},
+	"geometry": null,
+	"links": [],
+	"assets": {
+		"gsc_metadata": {
+			"href": "OA/PH1B/0.2/e8/urn:eop:PHR:MULTISPECTRAL_0.5m:DS_PHR1B_201707271114024_FR1_PX_W006N35_0918_03164_b7e7/0000/PH1B_PHR_FUS__3_20170727T111402_20170727T111407_TOU_1234_b7e7.DIMA.tar/GSC#CR#ESA#D2_MG2b_FRTX_004a#20170727#122555.xml",
+			"type": "application/xml",
+			"title": "GSC Metadata file",
+			"description": "Accompanying GSC Metadata file copied over from source archive",
+			"roles": ["metadata"]
+		},
+		"ms": {
+			"href": "OA/PH1B/0.2/e8/urn:eop:PHR:MULTISPECTRAL_0.5m:DS_PHR1B_201707271114024_FR1_PX_W006N35_0918_03164_b7e7/0000/PH1B_PHR_FUS__3_20170727T111402_20170727T111407_TOU_1234_b7e7.DIMA.tar/IMG_PHR1B_PMS_201707271114024_ORT_2406862101-001_R1C1.tif",
+			"type": "image/tiff; application=geotiff; profile=cloud-optimized",
+			"title": "Preprocessed image",
+			"description": "Product image converted into a COG",
+			"roles": ["data"]
+		}
+	},
 	"stac_extensions": []
 }
-
 ]

testing/values-testing.yaml

 global:
+  # ingress: # docker swarm specific
+    # tls: false # docker swarm specific
   env:
     INGESTOR_SUCCESS_DIR: /var/ingestor/success
     INGESTOR_FAIL_DIR: /var/ingestor/fail
+    # DB_HOST: "database" # docker swarm specific
 
   collections:
     DEM:
@@ -20,6 +23,7 @@ global:
         - Emergency_Product_PH1B
         - HRG_THX_1A
         - HRS_PAN_1B
+        - PHR_FUS__3
     URBAN_ATLAS_2012:
       product_types:
         - WV1_PM8_SO
@@ -136,69 +140,6 @@ global:
       coverage_types:
         - int16_grayscale
 
-
-  disableDefaultCoverageTypes: false
-  coverageTypes:
-    - data_type: "Uint16"
-      name: "RGBNir"
-      bands:
-        - definition: "http://www.opengis.net/def/property/OGC/0/Radiance"
-          description: "Red Channel"
-          gdal_interpretation: "RedBand"
-          identifier: "red"
-          name: "red"
-          nil_values:
-            - reason: "http://www.opengis.net/def/nil/OGC/0/unknown"
-              value: 0
-          uom: "W.m-2.Sr-1"
-          significant_figures: 5
-          allowed_value_ranges:
-            -
-              - 0
-              - 65535
-        - definition: "http://www.opengis.net/def/property/OGC/0/Radiance"
-          description: "Green Channel"
-          gdal_interpretation: "GreenBand"
-          identifier: "green"
-          name: "green"
-          nil_values:
-            - reason: "http://www.opengis.net/def/nil/OGC/0/unknown"
-              value: 0
-          uom: "W.m-2.Sr-1"
-          significant_figures: 5
-          allowed_value_ranges:
-            -
-              - 0
-              - 65535
-        - definition: "http://www.opengis.net/def/property/OGC/0/Radiance"
-          description: "Blue Channel"
-          gdal_interpretation: "BlueBand"
-          identifier: "blue"
-          name: "blue"
-          nil_values:
-            - reason: "http://www.opengis.net/def/nil/OGC/0/unknown"
-              value: 0
-          uom: "W.m-2.Sr-1"
-          significant_figures: 5
-          allowed_value_ranges:
-            -
-              - 0
-              - 65535
-        - definition: "http://www.opengis.net/def/property/OGC/0/Radiance"
-          description: "Nir Channel"
-          gdal_interpretation: "NirBand"
-          identifier: "nir"
-          name: "nir"
-          nil_values:
-            - reason: "http://www.opengis.net/def/nil/OGC/0/unknown"
-              value: 0
-          uom: "W.m-2.Sr-1"
-          significant_figures: 5
-          allowed_value_ranges:
-            -
-              - 0
-              - 65535
-
   productTypes:
     - name: Emergency_Product_PH1B
       defaultBrowse: TRUE_COLOR
@@ -241,6 +182,47 @@ global:
           grey:
             expression: (nir-red)/(nir+red)
             range: [-1, 1]
+    - name: PHR_FUS__3
+      defaultBrowse: TRUE_COLOR
+      filter:
+        product_type: PHR_FUS__3
+      collections:
+        - Emergency
+      coverages:
+        RGBNir:
+          assets:
+            - ms
+      browses:
+        TRUE_COLOR:
+          red:
+            expression: red
+            range: [0, 1350]
+            nodata: 0
+          green:
+            expression: green
+            range: [0, 1350]
+            nodata: 0
+          blue:
+            expression: blue
+            range: [0, 1350]
+            nodata: 0
+        FALSE_COLOR:
+          red:
+            expression: nir
+            range: [0, 2200]
+            nodata: 0
+          green:
+            expression: red
+            range: [0, 1350]
+            nodata: 0
+          blue:
+            expression: green
+            range: [0, 1350]
+            nodata: 0
+        NDVI:
+          grey:
+            expression: (nir-red)/(nir+red)
+            range: [-1, 1]
     - name: DEM_Product_COP-DEM_GLO-30-DTED
       defaultBrowse: elevation
       collections:

values.yaml

@@ -238,10 +238,13 @@ database:
     enabled: false
     existingClaim: data-access-db # rename to persistence volume claim
   #### SHOULD BE CHANGED IN PROD ####
+  # NOTE: these value keys have to be changed when upgrading the chart version
+  #       https://docs.bitnami.com/kubernetes/infrastructure/postgresql/administration/upgrade/
   postgresqlUsername: dbuser
   postgresqlPassword: dbpw
   postgresqlDatabase: dbname
   postgresqlPostgresPassword: dbpgpw
+  postgresqlMaxConnections: "300"
   initdbScripts:
     enablePostgis.sh: |
       echo "Enabling postgis"