setting authentication in gitlab variables

testing/preprocessor_test.py

 import os
 import sys
 import re
+import csv
+import pytest
+import subprocess
 
 from osgeo import gdal
+from swiftclient.service import SwiftService
 
 
-
+@pytest.fixture
+def products():
+    with open('./preprocessed_list.csv') as f:
+        yield csv.reader(f)
 
 
 def set_gdal_swift_auth():
-    with open('preprocessor_test.txt','r') as auth_values:
-        values_list = []
-        for aline in auth_values:
-
-            x = re.search("(?:=)(.*)", aline)
-            values_list.append(x.group(1))
-
-    gdal.SetConfigOption("SWIFT_STORAGE_URL", values_list[0])
-    gdal.SetConfigOption("SWIFT_AUTH_TOKEN", values_list[1])
-
-
+    # parsing command line output of swift auth
+    auth_keys = subprocess.check_output(["swift", "auth"]).decode(sys.stdout.encoding).split("\n")
+    storage_url = auth_keys[0].split("OS_STORAGE_URL=")[1]
+    auth_token = auth_keys[1].split("OS_AUTH_TOKEN=")[1]
+    # setting gdal config
+    gdal.SetConfigOption("SWIFT_STORAGE_URL", storage_url)
+    gdal.SetConfigOption("SWIFT_AUTH_TOKEN", auth_token)
+
+@pytest.fixture
+def list_tifs(products):
+
+    preprocessed_list = []
+    for product in products:
+
+        with SwiftService() as swift:
+            # auth_options["prefix"] = product[0]
+            list_parts_gen = swift.list(
+                container='preprocessor_results', options= {"prefix": product[0]},
+            )
+            for page in list_parts_gen:
+                if page["success"]:
+                    for item in page["listing"]:
+                        if item["name"].endswith(".tif"):
+                            preprocessed_list.append(item["name"])
+    return preprocessed_list
+
+def test_preprocessor(list_tifs):
+    
+    for item in list_tifs:
+        set_gdal_swift_auth()
+        image = gdal.Open('/vsiswift/preprocessor_results/%s' % item, gdal.GA_ReadOnly)
+        srcband = image.GetRasterBand(1)
+        assert srcband.Checksum() != None
+        assert srcband.Checksum() > 0 
 
-def test_preprocessor():
-    set_gdal_swift_auth()
-    image = gdal.Open('/vsiswift/preprocessor_results/KS03_AIS_PSH_1G_20160101T113020_20160101T113022_KGS_019339_75CE.tar/K3_20160101113014_19339_02281264_L1G.tif', gdal.GA_ReadOnly)
-    srcband = image.GetRasterBand(1)
-    assert srcband.Checksum() != None
-    assert srcband.Checksum() > 0 
 
-test_preprocessor()
\ No newline at end of file

testing/preprocessor_test.sh

 #!/bin/bash
 product_list_file=$1
 
-OS_PASSWORD=$(docker exec -i $(docker ps -qf "name=emg-pvs_preprocessor") cat /run/secrets/OS_PASSWORD)
-
-ST_AUTH_VERSION=3
-OS_AUTH_URL_SHORT=https://auth.cloud.ovh.net/
-OS_AUTH_URL=https://auth.cloud.ovh.net/v3/
-OS_USERNAME=xqNChf3Rz5vs
-OS_TENANT_NAME=7398560954290261
-OS_TENANT_ID=1b418c4359064774af5d55da3f4bcac0
-OS_REGION_NAME=SERCO-DIAS1
-
-# gaining the necessary credentials and mapping them as env variables
-swift auth --auth-version $ST_AUTH_VERSION --os-identity-api-version 3 --os-auth-url $OS_AUTH_URL \
---os-username $OS_USERNAME --os-password $OS_PASSWORD --os-tenant-name $OS_TENANT_NAME --os-tenant-id $OS_TENANT_ID \
---os-region-name $OS_REGION_NAME preprocessor_results > preprocessor_test.txt
-IFS="="
-while read  key value ; do
-
-    if [ "$key" = "export OS_STORAGE_URL" ]; then
-
-        export SWIFT_STORAGE_URL=$value
-    else 
-        export SWIFT_AUTH_TOKEN=$value
-    fi
-done < preprocessor_test.txt
-
 while read product; do
     echo $product
     # preprocessing the images and save the results in the result bucket