Revert "try to split test pipeline to two independent phases"

This reverts commit e64e93c6.

.gitlab-ci.yml

@@ -54,33 +54,3 @@ integration-test:
       - logs/*
   except:
     - main
-
-integration-test-fullpipeline:
-  image: registry.gitlab.eox.at/vs/vs-deployment/docker-base-testing:latest
-  services:
-    - docker:20.10.8-dind
-  stage: test
-  before_script:
-    - k3d cluster create testgitlabci-${CI_JOB_ID} --agents 1 --wait
-    - kubectl cluster-info
-    - export PREPROCESSOR_RESULTS_BUCKET=$(cat /proc/sys/kernel/random/uuid)
-    - kubectl apply -f testing/pvc-redis.yaml
-    # default timeout is 5 min, move to 8m for some leeway
-    - helm install my-helm-release . --wait --timeout 10m --values "${SECRET_HELM_VALUES_TESTING_FULLPIPELINE}" --set global.storage.data.swift-data-fullpipeline.container="${PREPROCESSOR_RESULTS_BUCKET}" --set global.storage.target.container=${PREPROCESSOR_RESULTS_BUCKET}
-    - kubectl get po -A
-    - mkdir "${CI_PROJECT_DIR}/logs"
-  script:
-    - cd testing && ./run_integration_test_fullpipeline.sh
-  after_script:
-    # print info in case anything has gone wrong
-    - kubectl get po -A || true
-    # fetch all pod logs
-    - kubectl get po -o name | while read POD ; do kubectl logs "$POD" >> "${CI_PROJECT_DIR}/logs/$(echo $POD | cut -d'/' -f2).txt" ; done || true
-    - k3d cluster delete testgitlabci-${CI_JOB_ID}
-  artifacts:
-    when: always
-    expire_in: 4 week
-    paths:
-      - logs/*
-  except:
-    - main

testing/run_integration_test.sh

@@ -45,12 +45,36 @@ KUBECTL_PORT_FORWARD_PID=$!
 pytest test_preprocessor.py test_registrar.py test_renderer.py
 let "TEST_RESULT+=$?"
 
+# save logs of pods that are to be restarted due to helm upgrade
+kubectl get po -o name | grep 'preprocessor\|registrar' | while read POD ; do kubectl logs "$POD" >> "${CI_PROJECT_DIR}/logs/$(echo $POD | cut -d'/' -f2).txt" ; done || true
+# update used values and bucket used in registrar/preprocessor for full pipeline test
+helm upgrade my-helm-release ../ --values values-testing.yaml --values "${SECRET_HELM_VALUES_TESTING_FULLPIPELINE}" --set global.storage.data.swift-data-fullpipeline.container="${PREPROCESSOR_RESULTS_BUCKET}" --set global.storage.target.container=${PREPROCESSOR_RESULTS_BUCKET}
+
 swift delete "${PREPROCESSOR_RESULTS_BUCKET}"
 swift delete "${PREPROCESSOR_RESULTS_BUCKET}_segments"
 
+sleep 30 # to allow new pods to replace the old one after helm upgrade
+
+# ingestor tests
+kubectl cp ./tpzonlineitem.xml default/$(kubectl get po -l app.kubernetes.io/name=ingestor -o name | cut -d'/' -f2):/mnt/data
+kubectl cp ./defected_report.xml default/$(kubectl get po -l app.kubernetes.io/name=ingestor -o name | cut -d'/' -f2):/mnt/data
+
+sleep 15  # run ingestor
+kubectl exec deployment/my-helm-release-ingestor -- ls /var/ingestor/success > success_list.csv
+kubectl exec deployment/my-helm-release-ingestor -- ls /var/ingestor/fail > failure_list.csv
+
+
 #client
 #TODO
 
+# fetch logs for ingestor tests
+kubectl logs $(kubectl get po -l app.kubernetes.io/name=ingestor -o name | cut -d'/' -f2) > ingestor_log.txt
+pytest test_ingestor.py
+let "TEST_RESULT+=$?"
+
+pytest --fullpipeline test_renderer.py test_registrar.py
+let "TEST_RESULT+=$?"
+
 kill $KUBECTL_PORT_FORWARD_PID
 
 exit $TEST_RESULT

testing/run_integration_test_fullpipeline.sh

-#!/usr/bin/env bash
-
-set -o allexport
-source "${PYTEST_ENVIRONMENT}"
-set +o allexport
-
-# NOTE: only set -x after sourcing secret values, otherwise they are leaked
-set -u -x
-
-echo "Using bucket:"
-echo $PREPROCESSOR_RESULTS_BUCKET
-
-TEST_RESULT=0
-
-# expose database, renderer and client ports externally
-kubectl port-forward svc/my-helm-release-database 5432:5432 &
-# kubectl port-forward svc/my-helm-release-client 80:80 &
-kubectl port-forward svc/my-helm-release-renderer 81:80 &
-
-KUBECTL_PORT_FORWARD_PID=$!
-
-# ingestor tests
-kubectl cp ./tpzonlineitem.xml default/$(kubectl get po -l app.kubernetes.io/name=ingestor -o name | cut -d'/' -f2):/mnt/data
-kubectl cp ./defected_report.xml default/$(kubectl get po -l app.kubernetes.io/name=ingestor -o name | cut -d'/' -f2):/mnt/data
-
-sleep 10  # run ingestor
-kubectl exec deployment/my-helm-release-ingestor -- ls /var/ingestor/success > success_list.csv
-kubectl exec deployment/my-helm-release-ingestor -- ls /var/ingestor/fail > failure_list.csv
-
-# fetch logs for ingestor tests
-kubectl logs $(kubectl get po -l app.kubernetes.io/name=ingestor -o name | cut -d'/' -f2) > ingestor_log.txt
-pytest test_ingestor.py
-let "TEST_RESULT+=$?"
-
-pytest --fullpipeline test_renderer.py test_registrar.py
-let "TEST_RESULT+=$?"
-
-swift delete "${PREPROCESSOR_RESULTS_BUCKET}"
-swift delete "${PREPROCESSOR_RESULTS_BUCKET}_segments"
-
-kill $KUBECTL_PORT_FORWARD_PID
-
-exit $TEST_RESULT