Merge branch 'staging' into shib-configs-update

chart/Chart.yaml

@@ -7,7 +7,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.3.14
+version: 0.3.15
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.

chart/files/registrar-config.yaml

@@ -18,7 +18,7 @@ sources:
       endpoint_url: {{ .endpoint_url }}
       access_key_id: {{ .access_key_id }}
       secret_access_key: {{ .secret_access_key }}
-      validate_bucket_name: {{ .validate_bucket_name | default true }}
+      validate_bucket_name: {{ if hasKey . "validate_bucket_name" }}{{ .validate_bucket_name }}{{ else }}true{{ end }}
       # TODO: 'client_kwargs'
       # region: {{ .region }}
     {{- end }}

core/registrar/daemon.py

@@ -24,10 +24,9 @@ def run_daemon(config, replace, host, port, listen_queue, progress_set, failure_
         client.sadd(progress_set, value)
         # start the registration on that file
         try:
-            items = register_file(config, value, replace)
-            for item in items:
-                client.sadd(success_set, item.identifier)
-                client.srem(progress_set, value)
+            register_file(config, value, replace)
+            client.sadd(success_set, value)
+            client.srem(progress_set, value)
         except Exception as e:
             if 'is already registered' not in "%s" % e:
                 # do not add to failure if skipped due to already registered

testing/gitlab_test.sh

@@ -9,8 +9,11 @@ cat $emg_db > ../env/emg_db.env
 cat $emg_django > ../env/emg_django.env
 cat $emg_obs > ../env/emg_obs.env
 
-# use `pvs_testing` bucket instead
+# randomize the bucket name to avoid clash of two CI pipelines (linux)
+PREPROCESSOR_RESULTS_BUCKET=$(cat /proc/sys/kernel/random/uuid)
+echo "PREPROCESSOR_RESULTS_BUCKET=$PREPROCESSOR_RESULTS_BUCKET" >> ../env/emg.env
 
+# use `pvs_testing` bucket instead
 sed -i -e 's/emg-data/pvs_testing/g' ../env/emg.env
 sed -i -e 's/vhr18-data/pvs_testing/g' ../env/vhr18.env
 
@@ -43,14 +46,13 @@ docker stack deploy -c ../docker-compose.emg.yml -c ../docker-compose.emg.dev.ym
 ./docker-stack-wait.sh -n renderer -n registrar -n preprocessor -n database -n sftp -n ingestor emg-pvs
 
 # perform the testing
-swift delete preprocessor_results
 bash ./preprocessor_prep.sh preprocessed_list.csv
 bash ./registrar_prep.sh product_list.csv
 
 pytest registrar_test.py preprocessor_test.py renderer_test.py
 if [ $? -ne 0 ]; then any_test_failed=0;fi
 
-swift delete preprocessor_results
+swift delete $PREPROCESSOR_RESULTS_BUCKET
 # ingest an item
 python3 ingestor_prep.py
 
@@ -60,7 +62,8 @@ sleep 60
 # test the existance of ingested item
 pytest ingestor_test.py
 if [ $? -ne 0 ]; then any_test_failed=0;fi
-
+# cleanup
+swift delete $PREPROCESSOR_RESULTS_BUCKET
 for service in $(docker service ls --format "{{.Name}}"); do
   docker service ps $service --no-trunc >> "$CI_PROJECT_DIR/$service.txt"
   docker service logs $service --timestamps >> "$CI_PROJECT_DIR/$service.txt"

testing/ingestor_test.py

@@ -19,15 +19,17 @@ def set_gdal_swift_auth():
     gdal.SetConfigOption("SWIFT_STORAGE_URL", storage_url)
     gdal.SetConfigOption("SWIFT_AUTH_TOKEN", auth_token)
 
+
 @pytest.fixture
 def list_tifs(product='KS03_AIS_PSH_1G_20160101T113020_20160101T113022_KGS_019339_75CE.tar'):
 
     preprocessed_list = []
+    container = os.getenv('PREPROCESSOR_RESULTS_BUCKET')
 
     with SwiftService() as swift:
         # auth_options["prefix"] = product[0]
         list_parts_gen = swift.list(
-            container='preprocessor_results', options= {"prefix": product[0]},
+            container=container, options={"prefix": product[0]},
         )
         for page in list_parts_gen:
             if page["success"]:
@@ -36,14 +38,14 @@ def list_tifs(product='KS03_AIS_PSH_1G_20160101T113020_20160101T113022_KGS_01933
                         preprocessed_list.append(item["name"])
     return preprocessed_list
 
+
 def test_preprocessor(list_tifs):
-    # check if there are preprocessed results in the buckets 
+    # check if there are preprocessed results in the buckets
     assert len(list_tifs) > 0
+    container = os.getenv('PREPROCESSOR_RESULTS_BUCKET')
     for item in list_tifs:
         set_gdal_swift_auth()
-        image = gdal.Open('/vsiswift/preprocessor_results/%s' % item, gdal.GA_ReadOnly)
+        image = gdal.Open('/vsiswift/%s/%s' % (container, item), gdal.GA_ReadOnly)
         srcband = image.GetRasterBand(1)
-        assert srcband.Checksum() != None
-        assert srcband.Checksum() > 0 
-
-
+        assert srcband.Checksum() is not None
+        assert srcband.Checksum() > 0

testing/preprocessor_test.py

 import os
 import sys
-import re
 import csv
 import pytest
 import subprocess
@@ -24,16 +23,17 @@ def set_gdal_swift_auth():
     gdal.SetConfigOption("SWIFT_STORAGE_URL", storage_url)
     gdal.SetConfigOption("SWIFT_AUTH_TOKEN", auth_token)
 
+
 @pytest.fixture
 def list_tifs(products):
 
     preprocessed_list = []
+    container = os.getenv('PREPROCESSOR_RESULTS_BUCKET')
     for product in products:
 
         with SwiftService() as swift:
-            # auth_options["prefix"] = product[0]
             list_parts_gen = swift.list(
-                container='preprocessor_results', options= {"prefix": product[0]},
+                container=container, options={"prefix": product[0]},
             )
             for page in list_parts_gen:
                 if page["success"]:
@@ -42,14 +42,14 @@ def list_tifs(products):
                             preprocessed_list.append(item["name"])
     return preprocessed_list
 
+
 def test_preprocessor(list_tifs):
-    # check if there are preprocessed results in the buckets 
+    # check if there are preprocessed results in the buckets
     assert len(list_tifs) > 0
+    container = os.getenv('PREPROCESSOR_RESULTS_BUCKET')
     for item in list_tifs:
         set_gdal_swift_auth()
-        image = gdal.Open('/vsiswift/preprocessor_results/%s' % item, gdal.GA_ReadOnly)
+        image = gdal.Open('/vsiswift/%s/%s' % (container, item), gdal.GA_ReadOnly)
         srcband = image.GetRasterBand(1)
-        assert srcband.Checksum() != None
-        assert srcband.Checksum() > 0 
-
-
+        assert srcband.Checksum() is not None
+        assert srcband.Checksum() > 0

testing/testing_preprocessor_config.yml

@@ -22,7 +22,7 @@ target:
     auth_version: !env '${ST_AUTH_VERSION}'
     auth_url: !env '${OS_AUTH_URL}'
     user_domain_name: !env '${OS_USER_DOMAIN_NAME}'
-    container: preprocessor_results # change this?
+    container: !env '${PREPROCESSOR_RESULTS_BUCKET}'
 workdir: /tmp
 keep_temp: false
 metadata_glob: "*GSC*.xml"