Further developing chart and templates

chart/Chart.lock

@@ -2,5 +2,8 @@ dependencies:
 - name: postgresql
   repository: https://charts.bitnami.com/bitnami
   version: 9.7.2
-digest: sha256:a59e13b07f7f8f203ada856c39d69e9a4d2f4810e1265d9b80d8fe661331979e
-generated: "2020-10-07T14:27:02.464388398+02:00"
+- name: redis
+  repository: https://charts.bitnami.com/bitnami
+  version: 10.9.0
+digest: sha256:19a4b34c5ee40a44b18979a052b53a54569b69bd8b37e6f805826482dc6432ea
+generated: "2020-11-09T16:57:52.383912+01:00"

chart/Chart.yaml

@@ -22,3 +22,7 @@ dependencies:
     version: "9.7.2"
     repository: "https://charts.bitnami.com/bitnami"
     alias: database
+  - name: "redis"
+    version: "10.9.0"
+    repository: "https://charts.bitnami.com/bitnami"
+    alias: redis

chart/files/init-db.sh

@@ -85,13 +85,36 @@
 {{- end -}}
 
 # Check if collection exits in database and initialize database only if not
-if python3 manage.py id check "${COLLECTION}"; then
+if python3 manage.py id check {{ index (keys .Values.config.collections) 0 | quote }}; then
     echo "Initialize database"
 
     python3 manage.py coveragetype import /rgbnir_definition.json \
         --traceback
 
-    echo "Initializing collection '${COLLECTION}'."
+
+# TODO: deleteme
+
+    python3 manage.py coveragetype create S2L2A_B01 --field-type B01 B01 "Solar irradiance" "W/m2/um" 1913.57
+    python3 manage.py coveragetype create S2L2A_B02 --field-type B02 B02 "Solar irradiance" "W/m2/um" 1941.63
+    python3 manage.py coveragetype create S2L2A_B03 --field-type B03 B03 "Solar irradiance" "W/m2/um" 1822.61
+    python3 manage.py coveragetype create S2L2A_B04 --field-type B04 B04 "Solar irradiance" "W/m2/um" 1512.79
+    python3 manage.py coveragetype create S2L2A_B05 --field-type B05 B05 "Solar irradiance" "W/m2/um" 1425.56
+    python3 manage.py coveragetype create S2L2A_B06 --field-type B06 B06 "Solar irradiance" "W/m2/um" 1288.32
+    python3 manage.py coveragetype create S2L2A_B07 --field-type B07 B07 "Solar irradiance" "W/m2/um" 1163.19
+    python3 manage.py coveragetype create S2L2A_B08 --field-type B08 B08 "Solar irradiance" "W/m2/um" 1036.39
+    python3 manage.py coveragetype create S2L2A_B8A --field-type B8A B8A "Solar irradiance" "W/m2/um" 955.19
+    python3 manage.py coveragetype create S2L2A_B09 --field-type B09 B09 "Solar irradiance" "W/m2/um" 813.04
+    python3 manage.py coveragetype create S2L2A_B11 --field-type B11 B11 "Solar irradiance" "W/m2/um" 245.59
+    python3 manage.py coveragetype create S2L2A_B12 --field-type B12 B12 "Solar irradiance" "W/m2/um" 85.25
+
+
+
+
+
+
+
+
+    echo "Initializing collection {{ index (keys .Values.config.collections) 0 | squote }}."
 
     {{- range $product_type_name, $product_type := .Values.config.products.types | default dict }}
 
@@ -101,9 +124,8 @@ if python3 manage.py id check "${COLLECTION}"; then
 
     # create the product type
     python3 manage.py producttype create {{ $product_type_name | quote }} \
-        {{ range $_, $coverage_type := $product_type.coverages }}--coverage-type { $coverage_type | quote }} \
-        {{- end }}
-        --traceback
+        {{ range $_, $coverage_type := $product_type.coverages }}--coverage-type {{ $coverage_type | quote }} \
+        {{ end }} --traceback
 
     {{- if hasKey $product_type "default_browse" }}
     {{- template "browsetype.cli" dict "product_type_name" $product_type_name "browse_type_name" nil "browse_type" (get $product_type.browses $product_type.default_browse) -}}
@@ -116,8 +138,7 @@ if python3 manage.py id check "${COLLECTION}"; then
     # create mask type
     {{- range $mask_type_name, $mask_type := $product_type.masks }}
     python3 manage.py masktype create {{ $product_type_name | quote }} {{ $mask_type_name | quote }} \
-        {{ if $mask_type.validity -}} --validity \ {{- end }}
-        --traceback
+        {{ if $mask_type.validity -}} --validity \ {{- end }} --traceback
     {{- end }}
     {{- end }} {{/* range .Values.config.products.types */}}
 

chart/files/registrar-config.yaml

 sources:
-  - kwargs:
-    {{- with .Values.config.objectStorage.download }}
-    type: {{ .type }}
+  {{- with .Values.config.objectStorage.data }}
+  {{- $type := ( .type | lower ) }}
+  - type: {{ $type }}
     name: name # TODO
     kwargs:
-    {{- if eq .type "swift" }}
+    {{- if eq $type "swift" }}
       username: {{ .username }}
       password: {{ .password }}
       tenant_name: {{ .tenant_name }}
@@ -13,12 +13,12 @@ sources:
       auth_url: {{ .auth_url }}
       auth_version: {{ .auth_version }}
       user_domain_name: {{ .user_domain_name }}
-    {{- else if eq .type "s3" }}
-      bucket: {{ .bucket }}
+    {{- else if eq $type "s3" }}
+      bucket_name: {{ .bucket | default "null" }}
       endpoint_url: {{ .endpoint_url }}
       access_key_id: {{ .access_key_id }}
       secret_access_key: {{ .secret_access_key }}
-      region: {{ .region }}
+      # region: {{ .region }}
     {{- end }}
     {{- end }}
 
@@ -37,7 +37,7 @@ backends:
       mapping:
         {{- range $product_type_name, $product_type := .Values.config.products.types }}
         {{ $product_type_name }}:
-          {{- range $level := list "Level_1" "Level_3" }}
+          {{- range $level := list "Level_1" "Level_3" "Level-2A" }}
           {{ $level }}:
             product_type_name: {{ $product_type_name | quote }}
             collections:
@@ -47,7 +47,7 @@ backends:
               {{- end }}
               {{- end }}
             coverages:
-              {{ toYaml $product_type.coverages }}
+              {{- toYaml $product_type.coverages | nindent 16 }}
             masks:
               {{- range $mask_name, $_ := $product_type.masks }}
               {{ $mask_name }}: {{ $mask_name }}

chart/templates/registrar-deployment.yaml

@@ -31,22 +31,10 @@ spec:
         - name: {{ .Chart.Name }}-registrar
           image: "registry.gitlab.eox.at/esa/prism/vs/pvs_core:{{ .Values.image.tag | default .Chart.AppVersion }}"
           imagePullPolicy: {{ .Values.image.pullPolicy }}
-          ports:
-            - name: http
-              containerPort: 80
-              protocol: TCP
-          livenessProbe:
-            httpGet:
-              path: /
-              port: http
-          readinessProbe:
-            httpGet:
-              path: /
-              port: http
           resources:
             {{- toYaml .Values.registrar.resources | nindent 12 }}
           args:
-            - /run-httpd.sh
+            - /run-registrar.sh
           env:
             {{- range $key, $value := .Values.config.general }}
             - name: {{ $key }}
@@ -66,6 +54,12 @@ spec:
             - name: {{ $key }}
               value: {{ $value | quote }}
             {{- end }}
+            {{- range $key, $value := .Values.config.redis }}
+            - name: {{ $key }}
+              value: {{ $value | quote }}
+            {{- end }}
+            - name: REDIS_HOST
+              value: {{ .Release.Name }}-redis-master
             - name: INIT_SCRIPTS
               value: /configure.sh /init-db.sh /initialized.sh
             - name: INSTALL_DIR
@@ -75,11 +69,14 @@ spec:
             - name: STARTUP_SCRIPTS
               value: /wait-initialized.sh
             - name: WAIT_SERVICES
-              value: {{ .Release.Name }}-database:{{ .Values.config.database.DB_PORT }}
+              value: {{ .Release.Name }}-database:{{ .Values.config.database.DB_PORT }} {{ .Release.Name }}-redis-master:{{ .Values.config.redis.REDIS_PORT }}
           volumeMounts:
             - mountPath: /init-db.sh
               name: init-db
               subPath: init-db.sh
+            - mountPath: /config.yaml
+              name: registrar-config
+              subPath: registrar-config.yaml
       {{- with .Values.registrar.affinity | default .Values.affinity }}
       affinity:
         {{- toYaml . | nindent 8 }}
@@ -91,3 +88,9 @@ spec:
                 path: init-db.sh
             name: {{ include "vs.fullname" . }}-init-db
           name: init-db
+        - configMap:
+            items:
+              - key: registrar-config.yaml
+                path: registrar-config.yaml
+            name: {{ include "vs.fullname" . }}-registrar-config
+          name: registrar-config

chart/templates/renderer-deployment.yaml

@@ -43,6 +43,12 @@ spec:
             httpGet:
               path: /
               port: http
+          startupProbe:
+            httpGet:
+              path: /
+              port: http
+            failureThreshold: 30
+            periodSeconds: 10
           resources:
             {{- toYaml .Values.renderer.resources | nindent 12 }}
           args:

chart/values.yaml

@@ -47,7 +47,6 @@ config:
       secret_access_key: "secret_access_key"
       region: "region"
   redis:
-    REDIS_HOST: redis
     REDIS_PORT: "6379"
     REDIS_PREPROCESS_QUEUE_KEY: preprocess_queue
     REDIS_QUEUE_KEY: seed_queue
@@ -287,6 +286,16 @@ database:
       echo "Enabling postgis"
       PGPASSWORD="$POSTGRES_POSTGRES_PASSWORD" psql -U postgres -d "${POSTGRES_DB}" -c "CREATE EXTENSION postgis;"
 
+redis:
+  usePassword: false
+  # persistence:
+  #   existingClaim: redis
+  # master:
+  #   persistence:
+  #     enabled: true
+  cluster:
+    enabled: false
+
 preprocessor:
   replicaCount: 1
   resources:
@@ -295,7 +304,7 @@ preprocessor:
       memory: 6Gi
     requests:
       cpu: 0.5
-      memory: 2Gi
+      memory: 0.5Gi
   affinity: {}
 
 registrar:
@@ -306,7 +315,7 @@ registrar:
       memory: 6Gi
     requests:
       cpu: 0.5
-      memory: 2Gi
+      memory: 0.5Gi
   affinity: {}
 
 renderer:
@@ -317,7 +326,7 @@ renderer:
       memory: 6Gi
     requests:
       cpu: 0.5
-      memory: 2Gi
+      memory: 0.5Gi
   affinity: {}
 
 client:
@@ -336,7 +345,7 @@ replicaCount: 1
 image:
   repository: registry.gitlab.eox.at/esa/prism/vs
   pullPolicy: IfNotPresent
-  tag: ""
+  tag: "registrar-modularization"
 
 imagePullSecrets: []
 nameOverride: ""