From 6fbb8e8ec04c3375c3e9e1bb3d82365e7e95dbe5 Mon Sep 17 00:00:00 2001
From: Fabian Schindler <fabian.schindler.strauss@gmail.com>
Date: Wed, 4 Nov 2020 18:20:08 +0100
Subject: [PATCH] Improving Helm templates Improvin on init-db template Adding
initial templates for preprocessor- and registar-configs Improved 'values'
with new config structs
---
chart/files/init-db.sh | 199 ++++++++-----
chart/files/preprocessor-config.yaml | 26 ++
chart/files/registrar-config.yaml | 0
.../preprocessor-config-configmap.yaml | 6 +
.../templates/registrar-config-configmap.yaml | 6 +
chart/values.yaml | 261 ++++++++++++++++--
6 files changed, 406 insertions(+), 92 deletions(-)
create mode 100644 chart/files/preprocessor-config.yaml
create mode 100644 chart/files/registrar-config.yaml
create mode 100644 chart/templates/preprocessor-config-configmap.yaml
create mode 100644 chart/templates/registrar-config-configmap.yaml
diff --git a/chart/files/init-db.sh b/chart/files/init-db.sh
index 58ff7402..80121bbe 100644
--- a/chart/files/init-db.sh
+++ b/chart/files/init-db.sh
@@ -1,85 +1,148 @@
-# Check if collection exits in database and initialize database only if not
-if python3 manage.py id check "${COLLECTION}"; then
- echo "Initialize database"
+{{/*
+ Template to create the invocation of a browse type from a given
+ product type name, browse type name and browse type definition
- python3 manage.py coveragetype import /rgbnir_definition.json --traceback
+ Expects '.' to be a dictionary in the following shape:
-{{- if .Values.initDb }}
+ {
+ product_type_name: "<product_type_name>",
+ browse_type_name: "<browse_type_name>", // or nil
+ browse_type: {
+ // either
+ grey: {
+ expression: "",
+ range: [low, high], // optional
+ nodata: nodatavalue, // optional
+ }
+ // or
+ red: {
+ expression: "",
+ range: [low, high], // optional
+ nodata: nodatavalue, // optional
+ },
+ green: {
+ expression: "",
+ range: [low, high], // optional
+ nodata: nodatavalue, // optional
+ },
+ blue: {
+ expression: "",
+ range: [low, high], // optional
+ nodata: nodatavalue, // optional
+ },
+ // optionally:
+ alpha: {
+ expression: "",
+ range: [low, high], // optional
+ nodata: nodatavalue, // optional
+ }
+ }
+ }
+*/}}
+{{- define "browsetype.cli" }}
+ python3 manage.py browsetype create "{{ .product_type_name }}" {{ if .browse_type_name }} "{{ .browse_type_name }}" {{- end }} \
+ {{- if hasKey .browse_type "grey" }}
+ --grey {{ .browse_type.grey.expression | quote }} \
+ {{- if hasKey .browse_type.grey "range" }}
+ --grey-range {{ range .browse_type.grey.range }}{{ . }} {{ end }}\
+ {{- end }}
+ {{- if hasKey .browse_type.grey "nodata" }}
+ --grey-nodata {{ .browse_type.grey.nodata }} \
+ {{- end }}
+ {{- else if and (.browse_type.red) (and .browse_type.green .browse_type.blue) }}
+ --red {{ .browse_type.red.expression | quote }} \
+ --green {{ .browse_type.green.expression | quote }} \
+ --blue {{ .browse_type.blue.expression | quote }} \
+ {{- if hasKey .browse_type.red "range" }}
+ --red-range {{ range .browse_type.red.range }}{{ . }} {{ end }}\
+ {{- end }}
+ {{- if hasKey .browse_type.green "range" }}
+ --green-range {{ range .browse_type.green.range }}{{ . }} {{ end }}\
+ {{- end }}
+ {{- if hasKey .browse_type.blue "range" }}
+ --blue-range {{ range .browse_type.blue.range }}{{ . }} {{ end }}\
+ {{- end }}
+ {{- if hasKey .browse_type.red "nodata" }}
+ --red-nodata {{ .browse_type.red.nodata }} \
+ {{- end }}
+ {{- if hasKey .browse_type.green "nodata" }}
+ --green-nodata {{ .browse_type.green.nodata }} \
+ {{- end }}
+ {{- if hasKey .browse_type.blue "nodata" }}
+ --blue-nodata {{ .browse_type.blue.nodata }} \
+ {{- end }}
+ {{- if hasKey .browse_type "alpha" }}
+ --grey {{ .browse_type.alpha.expression | quote }} \
+ {{- if hasKey .browse_type.alpha "range" }}
+ --alpha-range {{ range .browse_type.alpha.range }}{{ . }} {{ end }}\
+ {{- end }}
+ {{- if hasKey .browse_type.alpha "nodata" }}
+ --alpha-nodata {{ .browse_type.alpha.nodata }} \
+ {{- end }}
+ {{- end }}
+ {{- end }}
+ --traceback
+{{- end -}}
- {{ .Values.initDb | nindent 6 }}
+# Check if collection exits in database and initialize database only if not
+if python3 manage.py id check "${COLLECTION}"; then
+ echo "Initialize database"
-{{- else }}
+ python3 manage.py coveragetype import /rgbnir_definition.json \
+ --traceback
- if [ "${COLLECTION}" == "VHR_IMAGE_2018" ]; then
- echo "Initializing collection '${COLLECTION}'."
+ echo "Initializing collection '${COLLECTION}'."
- # PL00
- python3 manage.py producttype create "${COLLECTION}"_Product_PL00 --traceback \
- --coverage-type "RGBNir"
- python3 manage.py browsetype create "${COLLECTION}"_Product_PL00 --traceback \
- --red "red" \
- --green "green" \
- --blue "blue" \
- --red-range 1000 15000 \
- --green-range 1000 15000 \
- --blue-range 1000 15000 \
- --red-nodata 0 \
- --green-nodata 0 \
- --blue-nodata 0
- python3 manage.py browsetype create "${COLLECTION}"_Product_PL00 "TRUE_COLOR" --traceback \
- --red "red" \
- --green "green" \
- --blue "blue" \
- --red-range 1000 15000 \
- --green-range 1000 15000 \
- --blue-range 1000 15000 \
- --red-nodata 0 \
- --green-nodata 0 \
- --blue-nodata 0
- python3 manage.py browsetype create "${COLLECTION}"_Product_PL00 "FALSE_COLOR" --traceback \
- --red "nir" \
- --green "red" \
- --blue "green" \
- --red-range 1000 15000 \
- --green-range 1000 15000 \
- --blue-range 1000 15000 \
- --red-nodata 0 \
- --green-nodata 0 \
- --blue-nodata 0
- python3 manage.py browsetype create "${COLLECTION}"_Product_PL00 "NDVI" --traceback \
- --grey "(nir-red)/(nir+red)" --grey-range -1 1
+ {{- range $product_type_name, $product_type := .Values.config.products.types | default dict }}
+ #
+ # {{ $product_type_name }}
+ #
- python3 manage.py collectiontype create "${COLLECTION}"_Collection --traceback \
- --coverage-type "RGBNir" \
- --product-type "${COLLECTION}"_Product_PL00
+ # create the product type
+ python3 manage.py producttype create {{ $product_type_name | quote }} \
+ {{ range $_, $coverage_type := $product_type.coverages }}--coverage-type { $coverage_type | quote }} \
+ {{- end }}
+ --traceback
- # Create collections for all products
- python3 manage.py collection create "${COLLECTION}" --type "${COLLECTION}"_Collection --traceback
+ {{- if hasKey $product_type "default_browse" }}
+ {{- template "browsetype.cli" dict "product_type_name" $product_type_name "browse_type_name" nil "browse_type" (get $product_type.browses $product_type.default_browse) -}}
+ {{- end }}
- # Register mask type
- python3 manage.py masktype create --validity "${COLLECTION}"_Product_PL00 validity
+ {{- range $browse_type_name, $browse_type := $product_type.browses }}
+ {{- template "browsetype.cli" dict "product_type_name" $product_type_name "browse_type_name" $browse_type_name "browse_type" $browse_type -}}
+ {{- end }}
- else
- echo "Provided collection '${COLLECTION}' not valid."
- fi
+ # create mask type
+ {{- range $mask_type_name, $mask_type := $product_type.masks }}
+ python3 manage.py masktype create {{ $product_type_name | quote }} {{ $mask_type_name | quote }} \
+ {{ if $mask_type.validity -}} --validity \ {{- end }}
+ --traceback
+ {{- end }}
+ {{- end }} {{/* range .Values.config.products.types */}}
- python3 manage.py storageauth create auth-cloud-ovh "${OS_AUTH_URL_SHORT}" \
- --type keystone \
- -p auth-version "${ST_AUTH_VERSION}" \
- -p identity-api-version="${ST_AUTH_VERSION}" \
- -p username "${OS_USERNAME}" \
- -p password "${OS_PASSWORD}" \
- -p tenant-name "${OS_TENANT_NAME}" \
- -p tenant-id "${OS_TENANT_ID}" \
- -p region-name "${OS_REGION_NAME}"
+ # set up collection type
+ {{- range $collection_name, $collection := .Values.config.collections }}
+ python3 manage.py collectiontype create "{{ $collection_name }}_type" \
+ {{- range $coverage_type := $collection.coverage_types }}
+ --coverage-type {{ $coverage_type | quote }} \
+ {{- end }}
+ {{- range $product_type_name := $collection.product_types }}
+ --product-type {{ $product_type_name | quote }} \
+ {{- end }}
+ --traceback
- python3 manage.py storage create \
- ${UPLOAD_CONTAINER} ${UPLOAD_CONTAINER} \
- --type swift \
- --storage-auth auth-cloud-ovh
+ # Instantiate a collection for the collection itself and all levels
+ python3 manage.py collection create {{ $collection_name | quote }} \
+ --type "{{ $collection_name }}_type" \
+ --traceback
-{{- end }}
+ {{- range $product_level := $collection.product_levels }}
+ python3 manage.py collection create "{{ $collection_name }}_{{ $product_level }}" \
+ --type "{{ $collection_name }}_type" \
+ --traceback
+ {{- end }}
+ {{- end }} {{/* range .collections */}}
else
echo "Using existing database"
diff --git a/chart/files/preprocessor-config.yaml b/chart/files/preprocessor-config.yaml
new file mode 100644
index 00000000..bfedb5c3
--- /dev/null
+++ b/chart/files/preprocessor-config.yaml
@@ -0,0 +1,26 @@
+{{- define "preprocessor.storage" }}
+ type: {{ .type }}
+ kwargs:
+{{- if eq .type "swift" }}
+ username: {{ .username }}
+ password: {{ .password }}
+ tenant_name: {{ .tenant_name }}
+ tenant_id: {{ .tenant_id }}
+ region_name: {{ .region_name }}
+ auth_url: {{ .auth_url }}
+ auth_version: {{ .auth_version }}
+ user_domain_name: {{ .user_domain_name }}
+{{- else if eq .type "s3" }}
+ bucket: {{ .bucket }}
+ endpoint_url: {{ .endpoint_url }}
+ access_key_id: {{ .access_key_id }}
+ secret_access_key: {{ .secret_access_key }}
+ region: {{ .region }}
+{{- end }}
+{{- end -}}
+source:
+ {{ template "preprocessor.storage" .Values.config.objectStorage.download | indent 2 }}
+target:
+ {{ template "preprocessor.storage" .Values.config.objectStorage.data | indent 2 }}
+
+{{ toYaml .Values.config.preprocessor | indent 2 }}
diff --git a/chart/files/registrar-config.yaml b/chart/files/registrar-config.yaml
new file mode 100644
index 00000000..e69de29b
diff --git a/chart/templates/preprocessor-config-configmap.yaml b/chart/templates/preprocessor-config-configmap.yaml
new file mode 100644
index 00000000..ab379486
--- /dev/null
+++ b/chart/templates/preprocessor-config-configmap.yaml
@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: {{ include "vs.fullname" . }}-preprocessor-config
+data:
+ {{ (tpl (.Files.Glob "files/preprocessor-config.yaml").AsConfig . ) | indent 2 }}
diff --git a/chart/templates/registrar-config-configmap.yaml b/chart/templates/registrar-config-configmap.yaml
new file mode 100644
index 00000000..1eaf7ab5
--- /dev/null
+++ b/chart/templates/registrar-config-configmap.yaml
@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: {{ include "vs.fullname" . }}-registrar-config
+data:
+ {{ (tpl (.Files.Glob "files/registrar-config.yaml").AsConfig . ) | nindent 2}}
diff --git a/chart/values.yaml b/chart/values.yaml
index 77a4950c..37f53ad8 100644
--- a/chart/values.yaml
+++ b/chart/values.yaml
@@ -18,32 +18,34 @@ config:
DJANGO_USER: djangouser
objectStorage:
download:
- OS_AUTH_URL_DOWNLOAD: https://auth.cloud.ovh.net/
- OS_PASSWORD_DOWNLOAD: ospw
- OS_REGION_NAME_DOWNLOAD: SERCO-DIAS1
- OS_TENANT_ID_DOWNLOAD: tenantid
- OS_TENANT_NAME_DOWNLOAD: "tenantname"
- OS_USERNAME_DOWNLOAD: osuser
- ST_AUTH_VERSION_DOWNLOAD: "3"
+ type: swift
+ username: "username"
+ password: "password"
+ tenant_name: "tenant_name"
+ tenant_id: "tenant_id"
+ region_name: "region_name"
+ auth_url: "auth_url"
+ auth_url_short: "auth_url_short"
+ auth_version: "auth_version"
+ user_domain_name: "user_domain_name"
data:
- OS_AUTH_URL: https://auth.cloud.ovh.net/v3/
- OS_AUTH_URL_SHORT: https://auth.cloud.ovh.net/
- OS_PASSWORD: ospw
- OS_REGION_NAME: SERCO-DIAS1
- OS_TENANT_ID: tenantid
- OS_TENANT_NAME: "tenantname"
- OS_USER_DOMAIN_NAME: default
- OS_USERNAME: osuser
- ST_AUTH_VERSION: "3"
+ type: swift
+ username: "username"
+ password: "password"
+ tenant_name: "tenant_name"
+ tenant_id: "tenant_id"
+ region_name: "region_name"
+ auth_url: "auth_url"
+ auth_url_short: "auth_url_short"
+ auth_version: "auth_version"
+ user_domain_name: "user_domain_name"
cache:
- S3_BUCKET: s3bucket
- S3_ID: s3id
- S3_SECRET: s3secret
- S3_REGION: eu-central-1
- preprocessing:
- UPLOAD_CONTAINER: container
- ENFORCE_FOUR_BANDS: "True"
- SPLIT_PARTS_CHECK: "False"
+ type: S3
+ bucket: "bucket:"
+ endpoint_url: "endpoint_url:"
+ access_key_id: "access_key_id:"
+ secret_access_key: "secret_access_key:"
+ region: "region:"
redis:
REDIS_HOST: redis
REDIS_PORT: "6379"
@@ -54,6 +56,217 @@ config:
REDIS_SEED_QUEUE_KEY: seed_queue
REDIS_SET_KEY: registered_set
+ client:
+ layers:
+ VHR_IMAGE_2018_Level_1:
+ display_color: '#eb3700'
+ title: VHR Image 2018 Level 1
+ layer: VHR_IMAGE_2018_Level_1__TRUE_COLOR
+ sub_layers:
+ VHR_IMAGE_2018_Level_1__TRUE_COLOR:
+ label: VHR Image 2018 True Color
+ VHR_IMAGE_2018_Level_1__masked_validity:
+ label: VHR Image 2018 True Color with masked validity
+ VHR_IMAGE_2018_Level_1__FALSE_COLOR:
+ label: VHR Image 2018 False Color
+ VHR_IMAGE_2018_Level_1__NDVI:
+ label: VHR Image 2018 NDVI
+ VHR_IMAGE_2018_Level_3:
+ display_color: '#eb3700'
+ title: VHR Image 2018 Level 3
+ layer: VHR_IMAGE_2018_Level_3__TRUE_COLOR
+ sub_layers:
+ VHR_IMAGE_2018_Level_3__TRUE_COLOR:
+ label: VHR Image 2018 True Color
+ VHR_IMAGE_2018_Level_3__masked_validity:
+ label: VHR Image 2018 True Color with masked validity
+ VHR_IMAGE_2018_Level_3__FALSE_COLOR:
+ label: VHR Image 2018 False Color
+ VHR_IMAGE_2018_Level_3__NDVI:
+ label: VHR Image 2018 NDVI
+ overlay_layers:
+ VHR_IMAGE_2018_Level_3__outlines:
+ display_color: '#187465'
+ title: VHR Image 2018 Level 3 Outlines
+ layer: VHR_IMAGE_2018_Level_3__outlines
+ VHR_IMAGE_2018_Level_3__masked_validity__Full:
+ display_color: '#187465'
+ title: VHR Image 2018 Level 3 True Color with masked validity Full Coverage
+ layer: VHR_IMAGE_2018_Level_3__masked_validity__Full
+ VHR_IMAGE_2018_Level_3__Full:
+ display_color: '#187465'
+ title: VHR Image 2018 Level 3 True Color Full Coverage
+ layer: VHR_IMAGE_2018_Level_3__Full
+
+ # cache related options
+ cache:
+ metadata:
+ title: PRISM Data Access Service (PASS) developed by EOX
+ abstract: PRISM Data Access Service (PASS) developed by EOX
+ url: https://vhr18.pvs.prism.eox.at/cache/ows
+ keyword: view service
+ accessconstraints: UNKNOWN
+ fees: UNKNOWN
+ contactname: Stephan Meissl
+ contactphone: Please contact via mail.
+ contactfacsimile: None
+ contactorganization: EOX IT Services GmbH
+ contactcity: Vienna
+ contactstateorprovince: Vienna
+ contactpostcode: 1090
+ contactcountry: Austria
+ contactelectronicmailaddress: office@eox.at
+ contactposition: CTO
+ providername: EOX
+ providerurl: https://eox.at
+ inspire_profile: true
+ inspire_metadataurl: TBD
+ defaultlanguage: eng
+ language: eng
+ services:
+ wms:
+ enabled: true
+ wmts:
+ enabled: true
+ connection_timeout: 10
+ timeout: 120
+ expires: 3600
+ key: /{tileset}/{grid}/{dim}/{z}/{x}/{y}.{ext}
+ tilesets:
+ VHR_IMAGE_2018__TRUE_COLOR:
+ title: VHR Image 2018 True Color
+ abstract: VHR Image 2018 True Color
+ VHR_IMAGE_2018__FALSE_COLOR:
+ title: VHR Image 2018 False Color
+ abstract: VHR Image 2018 False Color
+ VHR_IMAGE_2018__NDVI:
+ title: VHR Image 2018 NDVI
+ abstract: VHR Image 2018 NDVI
+ style: earth
+ VHR_IMAGE_2018_Level_1__TRUE_COLOR:
+ title: VHR Image 2018 Level 1 True Color
+ abstract: VHR Image 2018 Level 1 True Color
+ VHR_IMAGE_2018_Level_1__FALSE_COLOR:
+ title: VHR Image 2018 Level 1 False Color
+ abstract: VHR Image 2018 Level 1 False Color
+ VHR_IMAGE_2018_Level_1__NDVI:
+ title: VHR Image 2018 Level 1 NDVI
+ abstract: VHR Image 2018 Level 1 NDVI
+ style: earth
+ VHR_IMAGE_2018_Level_1__TRUE_COLOR:
+ title: VHR Image 2018 Level 3 True Color
+ abstract: VHR Image 2018 Level 3 True Color
+ VHR_IMAGE_2018_Level_1__FALSE_COLOR:
+ title: VHR Image 2018 Level 3 False Color
+ abstract: VHR Image 2018 Level 3 False Color
+ VHR_IMAGE_2018_Level_1__NDVI:
+ title: VHR Image 2018 Level 3 NDVI
+ abstract: VHR Image 2018 Level 3 NDVI
+ style: earth
+
+ preprocessor:
+ metadata_glob: '*GSC*.xml'
+ type_extractor:
+ xpath:
+ - /gsc:report/gsc:opt_metadata/gml:using/eop:EarthObservationEquipment/eop:platform/eop:Platform/eop:shortName/text()
+ level_extractor:
+ # xpath can also be a list of xpaths to be tried one after another
+ xpath: substring-after(substring-after(/gsc:report/gsc:opt_metadata/gml:metaDataProperty/gsc:EarthObservationMetaData/eop:parentIdentifier/text(), '/'), '/')
+ preprocessing:
+ defaults:
+ move_files: true
+ data_file_globs:
+ - '*.tif'
+ - '*.jp2'
+ output:
+ options:
+ format: COG
+ dstSRS: 'EPSG:4326'
+ dstNodata: 0
+ creationOptions:
+ - BLOCKSIZE=512
+ - COMPRESS=DEFLATE
+ - NUM_THREADS=8
+ - BIGTIFF=IF_SAFER
+ - OVERVIEWS=AUTO
+ types:
+ PH1B: # just to pass validation
+ nested: true
+
+ # mapping of collection name to objects
+ collections:
+ VHR_IMAGE_2018:
+ product_types:
+ - PL00
+ - DM02
+ - KS03
+ - KS04
+ - PH1A
+ - PH1B
+ - SP06
+ - SP07
+ - SW00
+ - TR00
+ product_levels:
+ - Level_1
+ - Level_3
+ coverage_types:
+ - RGBNir
+
+ products:
+ type_extractor:
+ xpath:
+ - /gsc:report/gsc:opt_metadata/gml:using/eop:EarthObservationEquipment/eop:platform/eop:Platform/eop:shortName/text()
+ - /gsc:report/gsc:sar_metadata/gml:using/eop:EarthObservationEquipment/eop:platform/eop:Platform/eop:shortName/text()
+ namespace_map:
+
+ level_extractor:
+ xpath:
+ namespace_map:
+
+ types:
+ PL00:
+ coverages:
+ PL00: RGBNir
+ default_browse: TRUE_COLOR
+ browses:
+ TRUE_COLOR:
+ red:
+ expression: red
+ range: [1000, 15000]
+ nodata: 0
+ green:
+ expression: green
+ range: [1000, 15000]
+ nodata: 0
+ blue:
+ expression: blue
+ range: [1000, 15000]
+ nodata: 0
+ FALSE_COLOR:
+ red:
+ expression: nir
+ range: [1000, 15000]
+ nodata: 0
+ green:
+ expression: red
+ range: [1000, 15000]
+ nodata: 0
+ blue:
+ expression: green
+ range: [1000, 15000]
+ nodata: 0
+ NDVI:
+ grey:
+ expression: (nir-red)/(nir+red)
+ range: [-1, 1]
+ masks:
+ validity:
+ validity: true
+
+ coverages:
+ # only RGBNir? SAR? complete list with all options here?
+
database:
persistence:
enabled: false
--
GitLab