EOX GitLab Instance

Skip to content
Snippets Groups Projects
Commit 5bb66838 authored by Lubomir Dolezal's avatar Lubomir Dolezal
Browse files

Revert "try to split test pipeline to two independent phases"

This reverts commit e64e93c6.
parent 0b5ad448
No related branches found
Tags release-2.1.5
1 merge request!96add full ingestor integration test
......@@ -54,33 +54,3 @@ integration-test:
- logs/*
except:
- main
integration-test-fullpipeline:
image: registry.gitlab.eox.at/vs/vs-deployment/docker-base-testing:latest
services:
- docker:20.10.8-dind
stage: test
before_script:
- k3d cluster create testgitlabci-${CI_JOB_ID} --agents 1 --wait
- kubectl cluster-info
- export PREPROCESSOR_RESULTS_BUCKET=$(cat /proc/sys/kernel/random/uuid)
- kubectl apply -f testing/pvc-redis.yaml
# default timeout is 5 min, move to 8m for some leeway
- helm install my-helm-release . --wait --timeout 10m --values "${SECRET_HELM_VALUES_TESTING_FULLPIPELINE}" --set global.storage.data.swift-data-fullpipeline.container="${PREPROCESSOR_RESULTS_BUCKET}" --set global.storage.target.container=${PREPROCESSOR_RESULTS_BUCKET}
- kubectl get po -A
- mkdir "${CI_PROJECT_DIR}/logs"
script:
- cd testing && ./run_integration_test_fullpipeline.sh
after_script:
# print info in case anything has gone wrong
- kubectl get po -A || true
# fetch all pod logs
- kubectl get po -o name | while read POD ; do kubectl logs "$POD" >> "${CI_PROJECT_DIR}/logs/$(echo $POD | cut -d'/' -f2).txt" ; done || true
- k3d cluster delete testgitlabci-${CI_JOB_ID}
artifacts:
when: always
expire_in: 4 week
paths:
- logs/*
except:
- main
......@@ -45,12 +45,36 @@ KUBECTL_PORT_FORWARD_PID=$!
pytest test_preprocessor.py test_registrar.py test_renderer.py
let "TEST_RESULT+=$?"
# save logs of pods that are to be restarted due to helm upgrade
kubectl get po -o name | grep 'preprocessor\|registrar' | while read POD ; do kubectl logs "$POD" >> "${CI_PROJECT_DIR}/logs/$(echo $POD | cut -d'/' -f2).txt" ; done || true
# update used values and bucket used in registrar/preprocessor for full pipeline test
helm upgrade my-helm-release ../ --values values-testing.yaml --values "${SECRET_HELM_VALUES_TESTING_FULLPIPELINE}" --set global.storage.data.swift-data-fullpipeline.container="${PREPROCESSOR_RESULTS_BUCKET}" --set global.storage.target.container=${PREPROCESSOR_RESULTS_BUCKET}
swift delete "${PREPROCESSOR_RESULTS_BUCKET}"
swift delete "${PREPROCESSOR_RESULTS_BUCKET}_segments"
sleep 30 # to allow new pods to replace the old one after helm upgrade
# ingestor tests
kubectl cp ./tpzonlineitem.xml default/$(kubectl get po -l app.kubernetes.io/name=ingestor -o name | cut -d'/' -f2):/mnt/data
kubectl cp ./defected_report.xml default/$(kubectl get po -l app.kubernetes.io/name=ingestor -o name | cut -d'/' -f2):/mnt/data
sleep 15 # run ingestor
kubectl exec deployment/my-helm-release-ingestor -- ls /var/ingestor/success > success_list.csv
kubectl exec deployment/my-helm-release-ingestor -- ls /var/ingestor/fail > failure_list.csv
#client
#TODO
# fetch logs for ingestor tests
kubectl logs $(kubectl get po -l app.kubernetes.io/name=ingestor -o name | cut -d'/' -f2) > ingestor_log.txt
pytest test_ingestor.py
let "TEST_RESULT+=$?"
pytest --fullpipeline test_renderer.py test_registrar.py
let "TEST_RESULT+=$?"
kill $KUBECTL_PORT_FORWARD_PID
exit $TEST_RESULT
#!/usr/bin/env bash
set -o allexport
source "${PYTEST_ENVIRONMENT}"
set +o allexport
# NOTE: only set -x after sourcing secret values, otherwise they are leaked
set -u -x
echo "Using bucket:"
echo $PREPROCESSOR_RESULTS_BUCKET
TEST_RESULT=0
# expose database, renderer and client ports externally
kubectl port-forward svc/my-helm-release-database 5432:5432 &
# kubectl port-forward svc/my-helm-release-client 80:80 &
kubectl port-forward svc/my-helm-release-renderer 81:80 &
KUBECTL_PORT_FORWARD_PID=$!
# ingestor tests
kubectl cp ./tpzonlineitem.xml default/$(kubectl get po -l app.kubernetes.io/name=ingestor -o name | cut -d'/' -f2):/mnt/data
kubectl cp ./defected_report.xml default/$(kubectl get po -l app.kubernetes.io/name=ingestor -o name | cut -d'/' -f2):/mnt/data
sleep 10 # run ingestor
kubectl exec deployment/my-helm-release-ingestor -- ls /var/ingestor/success > success_list.csv
kubectl exec deployment/my-helm-release-ingestor -- ls /var/ingestor/fail > failure_list.csv
# fetch logs for ingestor tests
kubectl logs $(kubectl get po -l app.kubernetes.io/name=ingestor -o name | cut -d'/' -f2) > ingestor_log.txt
pytest test_ingestor.py
let "TEST_RESULT+=$?"
pytest --fullpipeline test_renderer.py test_registrar.py
let "TEST_RESULT+=$?"
swift delete "${PREPROCESSOR_RESULTS_BUCKET}"
swift delete "${PREPROCESSOR_RESULTS_BUCKET}_segments"
kill $KUBECTL_PORT_FORWARD_PID
exit $TEST_RESULT
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment