diff --git a/core/registrar/daemon.py b/core/registrar/daemon.py
index aa811dfed7501dbb4321d53ce06c338240ceac9f..8654695b88286d46eb14c253a529926aa24bfad3 100644
--- a/core/registrar/daemon.py
+++ b/core/registrar/daemon.py
@@ -24,10 +24,9 @@ def run_daemon(config, replace, host, port, listen_queue, progress_set, failure_
client.sadd(progress_set, value)
# start the registration on that file
try:
- items = register_file(config, value, replace)
- for item in items:
- client.sadd(success_set, item.identifier)
- client.srem(progress_set, value)
+ register_file(config, value, replace)
+ client.sadd(success_set, value)
+ client.srem(progress_set, value)
except Exception as e:
if 'is already registered' not in "%s" % e:
# do not add to failure if skipped due to already registered
diff --git a/testing/gitlab_test.sh b/testing/gitlab_test.sh
index 5637e626c0701fed5dfcca00223fcc0092d4b99f..a2084a3f53b7dabd53b159f7cfb9e6708a727a0b 100755
--- a/testing/gitlab_test.sh
+++ b/testing/gitlab_test.sh
@@ -9,8 +9,11 @@ cat $emg_db > ../env/emg_db.env
cat $emg_django > ../env/emg_django.env
cat $emg_obs > ../env/emg_obs.env
-# use `pvs_testing` bucket instead
+# randomize the bucket name to avoid clash of two CI pipelines (linux)
+PREPROCESSOR_RESULTS_BUCKET=$(cat /proc/sys/kernel/random/uuid)
+echo "PREPROCESSOR_RESULTS_BUCKET=$PREPROCESSOR_RESULTS_BUCKET" >> ../env/emg.env
+# use `pvs_testing` bucket instead
sed -i -e 's/emg-data/pvs_testing/g' ../env/emg.env
sed -i -e 's/vhr18-data/pvs_testing/g' ../env/vhr18.env
@@ -43,14 +46,13 @@ docker stack deploy -c ../docker-compose.emg.yml -c ../docker-compose.emg.dev.ym
./docker-stack-wait.sh -n renderer -n registrar -n preprocessor -n database -n sftp -n ingestor emg-pvs
# perform the testing
-swift delete preprocessor_results
bash ./preprocessor_prep.sh preprocessed_list.csv
bash ./registrar_prep.sh product_list.csv
pytest registrar_test.py preprocessor_test.py renderer_test.py
if [ $? -ne 0 ]; then any_test_failed=0;fi
-swift delete preprocessor_results
+swift delete $PREPROCESSOR_RESULTS_BUCKET
# ingest an item
python3 ingestor_prep.py
@@ -60,7 +62,8 @@ sleep 60
# test the existance of ingested item
pytest ingestor_test.py
if [ $? -ne 0 ]; then any_test_failed=0;fi
-
+# cleanup
+swift delete $PREPROCESSOR_RESULTS_BUCKET
for service in $(docker service ls --format "{{.Name}}"); do
docker service ps $service --no-trunc >> "$CI_PROJECT_DIR/$service.txt"
docker service logs $service --timestamps >> "$CI_PROJECT_DIR/$service.txt"
diff --git a/testing/ingestor_test.py b/testing/ingestor_test.py
index ac949138e37fabe53ec1b47e5ca265c3e8d0bafa..f00127c66347e8b10e754ddf66a2b574f84b35d7 100644
--- a/testing/ingestor_test.py
+++ b/testing/ingestor_test.py
@@ -19,15 +19,17 @@ def set_gdal_swift_auth():
gdal.SetConfigOption("SWIFT_STORAGE_URL", storage_url)
gdal.SetConfigOption("SWIFT_AUTH_TOKEN", auth_token)
+
@pytest.fixture
def list_tifs(product='KS03_AIS_PSH_1G_20160101T113020_20160101T113022_KGS_019339_75CE.tar'):
preprocessed_list = []
+ container = os.getenv('PREPROCESSOR_RESULTS_BUCKET')
with SwiftService() as swift:
# auth_options["prefix"] = product[0]
list_parts_gen = swift.list(
- container='preprocessor_results', options= {"prefix": product[0]},
+ container=container, options={"prefix": product[0]},
)
for page in list_parts_gen:
if page["success"]:
@@ -36,14 +38,14 @@ def list_tifs(product='KS03_AIS_PSH_1G_20160101T113020_20160101T113022_KGS_01933
preprocessed_list.append(item["name"])
return preprocessed_list
+
def test_preprocessor(list_tifs):
- # check if there are preprocessed results in the buckets
+ # check if there are preprocessed results in the buckets
assert len(list_tifs) > 0
+ container = os.getenv('PREPROCESSOR_RESULTS_BUCKET')
for item in list_tifs:
set_gdal_swift_auth()
- image = gdal.Open('/vsiswift/preprocessor_results/%s' % item, gdal.GA_ReadOnly)
+ image = gdal.Open('/vsiswift/%s/%s' % (container, item), gdal.GA_ReadOnly)
srcband = image.GetRasterBand(1)
- assert srcband.Checksum() != None
- assert srcband.Checksum() > 0
-
-
+ assert srcband.Checksum() is not None
+ assert srcband.Checksum() > 0
diff --git a/testing/preprocessor_test.py b/testing/preprocessor_test.py
index c0916efd49f774fb5af32be79ae402a045c59bc6..19826a02e9a9cc2aba65a6877d2c6349ee52dc5d 100644
--- a/testing/preprocessor_test.py
+++ b/testing/preprocessor_test.py
@@ -1,6 +1,5 @@
import os
import sys
-import re
import csv
import pytest
import subprocess
@@ -24,16 +23,17 @@ def set_gdal_swift_auth():
gdal.SetConfigOption("SWIFT_STORAGE_URL", storage_url)
gdal.SetConfigOption("SWIFT_AUTH_TOKEN", auth_token)
+
@pytest.fixture
def list_tifs(products):
preprocessed_list = []
+ container = os.getenv('PREPROCESSOR_RESULTS_BUCKET')
for product in products:
with SwiftService() as swift:
- # auth_options["prefix"] = product[0]
list_parts_gen = swift.list(
- container='preprocessor_results', options= {"prefix": product[0]},
+ container=container, options={"prefix": product[0]},
)
for page in list_parts_gen:
if page["success"]:
@@ -42,14 +42,14 @@ def list_tifs(products):
preprocessed_list.append(item["name"])
return preprocessed_list
+
def test_preprocessor(list_tifs):
- # check if there are preprocessed results in the buckets
+ # check if there are preprocessed results in the buckets
assert len(list_tifs) > 0
+ container = os.getenv('PREPROCESSOR_RESULTS_BUCKET')
for item in list_tifs:
set_gdal_swift_auth()
- image = gdal.Open('/vsiswift/preprocessor_results/%s' % item, gdal.GA_ReadOnly)
+ image = gdal.Open('/vsiswift/%s/%s' % (container, item), gdal.GA_ReadOnly)
srcband = image.GetRasterBand(1)
- assert srcband.Checksum() != None
- assert srcband.Checksum() > 0
-
-
+ assert srcband.Checksum() is not None
+ assert srcband.Checksum() > 0
diff --git a/testing/testing_preprocessor_config.yml b/testing/testing_preprocessor_config.yml
index 827c7b064f3eb441ad8934a6ae14dab84900681d..ee1445c66efaa4c333d95b53629a0acbe873f99d 100644
--- a/testing/testing_preprocessor_config.yml
+++ b/testing/testing_preprocessor_config.yml
@@ -22,7 +22,7 @@ target:
auth_version: !env '${ST_AUTH_VERSION}'
auth_url: !env '${OS_AUTH_URL}'
user_domain_name: !env '${OS_USER_DOMAIN_NAME}'
- container: preprocessor_results # change this?
+ container: !env '${PREPROCESSOR_RESULTS_BUCKET}'
workdir: /tmp
keep_temp: false
metadata_glob: "*GSC*.xml"