From 70e3f3d33b480918a7750498d96540c743e98abe Mon Sep 17 00:00:00 2001
From: Lubomir Bucek <lubomir.bucek@eox.at>
Date: Mon, 16 Nov 2020 14:17:19 +0100
Subject: [PATCH] remove the need for separate <stack>_name.env file
---
cache/Dockerfile | 4 ++--
cache/run-seeder.sh | 2 +-
chart/values.yaml | 11 +++++++----
core/Dockerfile | 12 ++++++------
docker-compose.dem.yml | 5 -----
docker-compose.emg.yml | 5 -----
docker-compose.vhr18.yml | 5 -----
documentation/operator-guide/configuration.rst | 3 ---
env/dem_redis.env | 11 -----------
env/emg_redis.env | 11 -----------
env/vhr18_redis.env | 11 -----------
ingestor/Dockerfile | 2 +-
preprocessor/Dockerfile | 14 +++++++-------
13 files changed, 24 insertions(+), 72 deletions(-)
delete mode 100644 env/dem_redis.env
delete mode 100644 env/emg_redis.env
delete mode 100644 env/vhr18_redis.env
diff --git a/cache/Dockerfile b/cache/Dockerfile
index 2a4a3648..6bbd951a 100644
--- a/cache/Dockerfile
+++ b/cache/Dockerfile
@@ -47,9 +47,9 @@ ENV COLLECTION_ID= \
APACHE_ServerName="pvs_cache" \
APACHE_ServerAdmin="office@eox.at" \
APACHE_NGEO_CACHE_ALIAS="/ows" \
- REDIS_HOST= \
+ REDIS_HOST="redis" \
REDIS_PORT="6379" \
- REDIS_QUEUE_KEY="seed_queue" \
+ REDIS_SEED_QUEUE_KEY="seed_queue" \
ST_AUTH_VERSION=3 \
OS_AUTH_URL= \
OS_USERNAME= \
diff --git a/cache/run-seeder.sh b/cache/run-seeder.sh
index 53f789d7..9549c163 100644
--- a/cache/run-seeder.sh
+++ b/cache/run-seeder.sh
@@ -4,4 +4,4 @@
echo "Running seeder"
-python3 /seeder.py --mode redis --redis-host ${REDIS_HOST} --redis-port ${REDIS_PORT} --redis-queue-key ${REDIS_QUEUE_KEY}
+python3 /seeder.py --mode redis --redis-host ${REDIS_HOST} --redis-port ${REDIS_PORT} --redis-queue-key ${REDIS_SEED_QUEUE_KEY}
diff --git a/chart/values.yaml b/chart/values.yaml
index a501173a..fdb20631 100644
--- a/chart/values.yaml
+++ b/chart/values.yaml
@@ -49,12 +49,15 @@ config:
redis:
REDIS_PORT: "6379"
REDIS_PREPROCESS_QUEUE_KEY: preprocess_queue
- REDIS_QUEUE_KEY: seed_queue
+ REDIS_PREPROCESS_MD_QUEUE_KEY: preprocess-md_queue
+ REDIS_PREPROCESS_FAILURE_KEY: preprocess-failure_set
+ REDIS_PREPROCESS_PROGRESS_KEY: preprocessing_set
+ REDIS_PREPROCESS_SUCCESS_KEY: preprocess-success_set
REDIS_REGISTER_QUEUE_KEY: register_queue
- REDIS_REGISTERED_SET_KEY: registered_set
+ REDIS_REGISTER_FAILURE_KEY: register-failure_set
+ REDIS_REGISTER_PROGRESS_KEY: registering_set
+ REDIS_REGISTER_SUCCESS_KEY: register-success_set
REDIS_SEED_QUEUE_KEY: seed_queue
- REDIS_SET_KEY: registered_set
-
client:
layers: {}
# VHR_IMAGE_2018_Level_1:
diff --git a/core/Dockerfile b/core/Dockerfile
index 1d228e18..0fd5168b 100644
--- a/core/Dockerfile
+++ b/core/Dockerfile
@@ -65,12 +65,12 @@ ENV INSTANCE_ID="prism-view-server_core" \
APACHE_ServerName="pvs_instance" \
APACHE_ServerAdmin="office@eox.at" \
APACHE_ALIAS="pvs" \
- REDIS_HOST= redis \
- REDIS_PORT= 6379 \
- REDIS_REGISTER_QUEUE_KEY= register_queue \
- REDIS_REGISTER_FAILURE_KEY= register-failure_set \
- REDIS_REGISTER_PROGRESS_KEY= registering_set \
- REDIS_REGISTER_SUCCESS_KEY= register-success_set \
+ REDIS_HOST="redis" \
+ REDIS_PORT="6379" \
+ REDIS_REGISTER_QUEUE_KEY="register_queue" \
+ REDIS_REGISTER_FAILURE_KEY="register-failure_set" \
+ REDIS_REGISTER_PROGRESS_KEY="registering_set" \
+ REDIS_REGISTER_SUCCESS_KEY="register-success_set" \
INIT_SCRIPTS="/configure.sh" \
COLLECT_STATIC="false" \
REGISTRAR_REPLACE=
diff --git a/docker-compose.dem.yml b/docker-compose.dem.yml
index fc4b355f..f8c66d52 100644
--- a/docker-compose.dem.yml
+++ b/docker-compose.dem.yml
@@ -92,7 +92,6 @@ services:
env_file:
- env/dem.env
- env/dem_obs.env
- - env/dem_redis.env
secrets:
- OS_PASSWORD
environment:
@@ -111,7 +110,6 @@ services:
env_file:
- env/dem.env
- env/dem_obs.env
- - env/dem_redis.env
secrets:
- OS_PASSWORD
- OS_PASSWORD_DOWNLOAD
@@ -144,7 +142,6 @@ services:
- env/dem.env
- env/dem_db.env
- env/dem_obs.env
- - env/dem_redis.env
secrets:
- DJANGO_PASSWORD
- OS_PASSWORD
@@ -190,8 +187,6 @@ services:
replicas: 1
ingestor:
image: registry.gitlab.eox.at/esa/prism/vs/pvs_ingestor:latest
- env_file:
- - env/dem_redis.env
environment:
INSTANCE_ID: "prism-view-server_ingestor"
deploy:
diff --git a/docker-compose.emg.yml b/docker-compose.emg.yml
index ecbc6442..22c7dfb4 100644
--- a/docker-compose.emg.yml
+++ b/docker-compose.emg.yml
@@ -91,7 +91,6 @@ services:
env_file:
- env/emg.env
- env/emg_obs.env
- - env/emg_redis.env
secrets:
- OS_PASSWORD
environment:
@@ -107,8 +106,6 @@ services:
["/run-seeder.sh"]
ingestor:
image: registry.gitlab.eox.at/esa/prism/vs/pvs_ingestor:latest
- env_file:
- - env/emg_redis.env
environment:
INSTANCE_ID: "prism-view-server_ingestor"
deploy:
@@ -120,7 +117,6 @@ services:
env_file:
- env/emg.env
- env/emg_obs.env
- - env/emg_redis.env
secrets:
- OS_PASSWORD
- OS_PASSWORD_DOWNLOAD
@@ -153,7 +149,6 @@ services:
- env/emg.env
- env/emg_db.env
- env/emg_obs.env
- - env/emg_redis.env
secrets:
- DJANGO_PASSWORD
- OS_PASSWORD
diff --git a/docker-compose.vhr18.yml b/docker-compose.vhr18.yml
index fe74cc7e..b9aed16e 100644
--- a/docker-compose.vhr18.yml
+++ b/docker-compose.vhr18.yml
@@ -95,7 +95,6 @@ services:
env_file:
- env/vhr18.env
- env/vhr18_obs.env
- - env/vhr18_redis.env
secrets:
- OS_PASSWORD
environment:
@@ -111,8 +110,6 @@ services:
["/run-seeder.sh"]
ingestor:
image: registry.gitlab.eox.at/esa/prism/vs/pvs_ingestor:latest
- env_file:
- - env/vhr18_redis.env
environment:
INSTANCE_ID: "prism-view-server_ingestor"
deploy:
@@ -124,7 +121,6 @@ services:
env_file:
- env/vhr18.env
- env/vhr18_obs.env
- - env/vhr18_redis.env
secrets:
- OS_PASSWORD
- OS_PASSWORD_DOWNLOAD
@@ -157,7 +153,6 @@ services:
- env/vhr18.env
- env/vhr18_db.env
- env/vhr18_obs.env
- - env/vhr18_redis.env
secrets:
- DJANGO_PASSWORD
- OS_PASSWORD
diff --git a/documentation/operator-guide/configuration.rst b/documentation/operator-guide/configuration.rst
index dce3a837..be219154 100644
--- a/documentation/operator-guide/configuration.rst
+++ b/documentation/operator-guide/configuration.rst
@@ -96,7 +96,6 @@ Environment variables and ``.env`` files are passed to the services via the
- env/stack.env
- env/stack_db.env
- env/stack_obs.env
- - env/stack_redis.env
environment:
INSTANCE_ID: "prism-view-server_registrar"
INSTALL_DIR: "/var/www/pvs/dev/"
@@ -118,8 +117,6 @@ The following ``.env`` files are typically used:
django admin user to be used with the admin GUI.
* ``<stack-name>_obs.env``: This contains access parameters for the object
storage(s).
-* ``<stack-name>_redis.env``: Redis access credentials and queue names
-
Groups of Environment Variables
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
diff --git a/env/dem_redis.env b/env/dem_redis.env
deleted file mode 100644
index 3eff4afd..00000000
--- a/env/dem_redis.env
+++ /dev/null
@@ -1,11 +0,0 @@
-REDIS_HOST=redis
-REDIS_PORT=6379
-
-REDIS_QUEUE_KEY=seed_queue
-
-REDIS_PREPROCESS_QUEUE_KEY=preprocess_queue
-REDIS_PREPROCESS_MD_QUEUE_KEY=preprocess-md_queue
-REDIS_REGISTER_QUEUE_KEY=register_queue
-REDIS_REGISTERED_SET_KEY=registered_set
-REDIS_SET_KEY=registered_set
-REDIS_SEED_QUEUE_KEY=seed_queue
diff --git a/env/emg_redis.env b/env/emg_redis.env
deleted file mode 100644
index 3eff4afd..00000000
--- a/env/emg_redis.env
+++ /dev/null
@@ -1,11 +0,0 @@
-REDIS_HOST=redis
-REDIS_PORT=6379
-
-REDIS_QUEUE_KEY=seed_queue
-
-REDIS_PREPROCESS_QUEUE_KEY=preprocess_queue
-REDIS_PREPROCESS_MD_QUEUE_KEY=preprocess-md_queue
-REDIS_REGISTER_QUEUE_KEY=register_queue
-REDIS_REGISTERED_SET_KEY=registered_set
-REDIS_SET_KEY=registered_set
-REDIS_SEED_QUEUE_KEY=seed_queue
diff --git a/env/vhr18_redis.env b/env/vhr18_redis.env
deleted file mode 100644
index 3eff4afd..00000000
--- a/env/vhr18_redis.env
+++ /dev/null
@@ -1,11 +0,0 @@
-REDIS_HOST=redis
-REDIS_PORT=6379
-
-REDIS_QUEUE_KEY=seed_queue
-
-REDIS_PREPROCESS_QUEUE_KEY=preprocess_queue
-REDIS_PREPROCESS_MD_QUEUE_KEY=preprocess-md_queue
-REDIS_REGISTER_QUEUE_KEY=register_queue
-REDIS_REGISTERED_SET_KEY=registered_set
-REDIS_SET_KEY=registered_set
-REDIS_SEED_QUEUE_KEY=seed_queue
diff --git a/ingestor/Dockerfile b/ingestor/Dockerfile
index 53f44dcc..311f3d7a 100644
--- a/ingestor/Dockerfile
+++ b/ingestor/Dockerfile
@@ -43,7 +43,7 @@ COPY app.py config.py filedaemon.py /
COPY ingestor/ /ingestor
ENV INSTANCE_ID="prism-view-server_ingestor" \
- REDIS_HOST= \
+ REDIS_HOST="redis" \
REDIS_PORT="6379" \
REDIS_PREPROCESS_MD_QUEUE_KEY="preprocess-md_queue" \
INOTIFY_WATCH_DIR="/mnt/data"
diff --git a/preprocessor/Dockerfile b/preprocessor/Dockerfile
index 6be169ec..abc9d3e3 100644
--- a/preprocessor/Dockerfile
+++ b/preprocessor/Dockerfile
@@ -64,13 +64,13 @@ ENV INSTANCE_ID="prism-data-access-server_preprocessor" \
OS_TENANT_NAME_DOWNLOAD= \
OS_REGION_NAME_DOWNLOAD= \
OS_USER_DOMAIN_NAME_DOWNLOAD= \
- REDIS_HOST= redis \
- REDIS_PORT= 6379 \
- REDIS_PREPROCESS_QUEUE_KEY= preprocess_queue \
- REDIS_PREPROCESS_MD_QUEUE_KEY= preprocess-md_queue \
- REDIS_PREPROCESS_FAILURE_KEY= preprocess-failure_set \
- REDIS_PREPROCESS_PROGRESS_KEY= preprocessing_set \
- REDIS_PREPROCESS_SUCCESS_KEY= preprocess-success_set \
+ REDIS_HOST="redis" \
+ REDIS_PORT="6379" \
+ REDIS_PREPROCESS_QUEUE_KEY="preprocess_queue" \
+ REDIS_PREPROCESS_MD_QUEUE_KEY="preprocess-md_queue" \
+ REDIS_PREPROCESS_FAILURE_KEY="preprocess-failure_set" \
+ REDIS_PREPROCESS_PROGRESS_KEY="preprocessing_set" \
+ REDIS_PREPROCESS_SUCCESS_KEY="preprocess-success_set" \
PREPROCESSOR_DEBUG=
ADD run-preprocessor.sh \
--
GitLab