EOX GitLab Instance

Skip to content
Snippets Groups Projects
Commit 6fbb8e8e authored by Fabian Schindler's avatar Fabian Schindler
Browse files

Improving Helm templates 

Improvin on init-db template
Adding initial templates for preprocessor- and registar-configs
Improved 'values' with new config structs
parent 831c082c
No related branches found
No related tags found
2 merge requests!37Chart,!36Staging to master to prepare 1.0.0 release
Hide whitespace changes
Inline Side-by-side
chart/files/init-db.sh
+ 131
− 68
View file @ 6fbb8e8e
# Check if collection exits in database and initialize database only if not
if python3 manage.py id check "${COLLECTION}"; then
echo "Initialize database"
{{/*
Template to create the invocation of a browse type from a given
product type name, browse type name and browse type definition
python3 manage.py coveragetype import /rgbnir_definition.json --traceback
Expects '.' to be a dictionary in the following shape:
{{- if .Values.initDb }}
{
product_type_name: "<product_type_name>",
browse_type_name: "<browse_type_name>", // or nil
browse_type: {
// either
grey: {
expression: "",
range: [low, high], // optional
nodata: nodatavalue, // optional
}
// or
red: {
expression: "",
range: [low, high], // optional
nodata: nodatavalue, // optional
},
green: {
expression: "",
range: [low, high], // optional
nodata: nodatavalue, // optional
},
blue: {
expression: "",
range: [low, high], // optional
nodata: nodatavalue, // optional
},
// optionally:
alpha: {
expression: "",
range: [low, high], // optional
nodata: nodatavalue, // optional
}
}
}
*/}}
{{- define "browsetype.cli" }}
python3 manage.py browsetype create "{{ .product_type_name }}" {{ if .browse_type_name }} "{{ .browse_type_name }}" {{- end }} \
{{- if hasKey .browse_type "grey" }}
--grey {{ .browse_type.grey.expression | quote }} \
{{- if hasKey .browse_type.grey "range" }}
--grey-range {{ range .browse_type.grey.range }}{{ . }} {{ end }}\
{{- end }}
{{- if hasKey .browse_type.grey "nodata" }}
--grey-nodata {{ .browse_type.grey.nodata }} \
{{- end }}
{{- else if and (.browse_type.red) (and .browse_type.green .browse_type.blue) }}
--red {{ .browse_type.red.expression | quote }} \
--green {{ .browse_type.green.expression | quote }} \
--blue {{ .browse_type.blue.expression | quote }} \
{{- if hasKey .browse_type.red "range" }}
--red-range {{ range .browse_type.red.range }}{{ . }} {{ end }}\
{{- end }}
{{- if hasKey .browse_type.green "range" }}
--green-range {{ range .browse_type.green.range }}{{ . }} {{ end }}\
{{- end }}
{{- if hasKey .browse_type.blue "range" }}
--blue-range {{ range .browse_type.blue.range }}{{ . }} {{ end }}\
{{- end }}
{{- if hasKey .browse_type.red "nodata" }}
--red-nodata {{ .browse_type.red.nodata }} \
{{- end }}
{{- if hasKey .browse_type.green "nodata" }}
--green-nodata {{ .browse_type.green.nodata }} \
{{- end }}
{{- if hasKey .browse_type.blue "nodata" }}
--blue-nodata {{ .browse_type.blue.nodata }} \
{{- end }}
{{- if hasKey .browse_type "alpha" }}
--grey {{ .browse_type.alpha.expression | quote }} \
{{- if hasKey .browse_type.alpha "range" }}
--alpha-range {{ range .browse_type.alpha.range }}{{ . }} {{ end }}\
{{- end }}
{{- if hasKey .browse_type.alpha "nodata" }}
--alpha-nodata {{ .browse_type.alpha.nodata }} \
{{- end }}
{{- end }}
{{- end }}
--traceback
{{- end -}}
{{ .Values.initDb | nindent 6 }}
# Check if collection exits in database and initialize database only if not
if python3 manage.py id check "${COLLECTION}"; then
echo "Initialize database"
{{- else }}
python3 manage.py coveragetype import /rgbnir_definition.json \
--traceback
if [ "${COLLECTION}" == "VHR_IMAGE_2018" ]; then
echo "Initializing collection '${COLLECTION}'."
echo "Initializing collection '${COLLECTION}'."
# PL00
python3 manage.py producttype create "${COLLECTION}"_Product_PL00 --traceback \
--coverage-type "RGBNir"
python3 manage.py browsetype create "${COLLECTION}"_Product_PL00 --traceback \
--red "red" \
--green "green" \
--blue "blue" \
--red-range 1000 15000 \
--green-range 1000 15000 \
--blue-range 1000 15000 \
--red-nodata 0 \
--green-nodata 0 \
--blue-nodata 0
python3 manage.py browsetype create "${COLLECTION}"_Product_PL00 "TRUE_COLOR" --traceback \
--red "red" \
--green "green" \
--blue "blue" \
--red-range 1000 15000 \
--green-range 1000 15000 \
--blue-range 1000 15000 \
--red-nodata 0 \
--green-nodata 0 \
--blue-nodata 0
python3 manage.py browsetype create "${COLLECTION}"_Product_PL00 "FALSE_COLOR" --traceback \
--red "nir" \
--green "red" \
--blue "green" \
--red-range 1000 15000 \
--green-range 1000 15000 \
--blue-range 1000 15000 \
--red-nodata 0 \
--green-nodata 0 \
--blue-nodata 0
python3 manage.py browsetype create "${COLLECTION}"_Product_PL00 "NDVI" --traceback \
--grey "(nir-red)/(nir+red)" --grey-range -1 1
{{- range $product_type_name, $product_type := .Values.config.products.types | default dict }}
#
# {{ $product_type_name }}
#
python3 manage.py collectiontype create "${COLLECTION}"_Collection --traceback \
--coverage-type "RGBNir" \
--product-type "${COLLECTION}"_Product_PL00
# create the product type
python3 manage.py producttype create {{ $product_type_name | quote }} \
{{ range $_, $coverage_type := $product_type.coverages }}--coverage-type { $coverage_type | quote }} \
{{- end }}
--traceback
# Create collections for all products
python3 manage.py collection create "${COLLECTION}" --type "${COLLECTION}"_Collection --traceback
{{- if hasKey $product_type "default_browse" }}
{{- template "browsetype.cli" dict "product_type_name" $product_type_name "browse_type_name" nil "browse_type" (get $product_type.browses $product_type.default_browse) -}}
{{- end }}
# Register mask type
python3 manage.py masktype create --validity "${COLLECTION}"_Product_PL00 validity
{{- range $browse_type_name, $browse_type := $product_type.browses }}
{{- template "browsetype.cli" dict "product_type_name" $product_type_name "browse_type_name" $browse_type_name "browse_type" $browse_type -}}
{{- end }}
else
echo "Provided collection '${COLLECTION}' not valid."
fi
# create mask type
{{- range $mask_type_name, $mask_type := $product_type.masks }}
python3 manage.py masktype create {{ $product_type_name | quote }} {{ $mask_type_name | quote }} \
{{ if $mask_type.validity -}} --validity \ {{- end }}
--traceback
{{- end }}
{{- end }} {{/* range .Values.config.products.types */}}
python3 manage.py storageauth create auth-cloud-ovh "${OS_AUTH_URL_SHORT}" \
--type keystone \
-p auth-version "${ST_AUTH_VERSION}" \
-p identity-api-version="${ST_AUTH_VERSION}" \
-p username "${OS_USERNAME}" \
-p password "${OS_PASSWORD}" \
-p tenant-name "${OS_TENANT_NAME}" \
-p tenant-id "${OS_TENANT_ID}" \
-p region-name "${OS_REGION_NAME}"
# set up collection type
{{- range $collection_name, $collection := .Values.config.collections }}
python3 manage.py collectiontype create "{{ $collection_name }}_type" \
{{- range $coverage_type := $collection.coverage_types }}
--coverage-type {{ $coverage_type | quote }} \
{{- end }}
{{- range $product_type_name := $collection.product_types }}
--product-type {{ $product_type_name | quote }} \
{{- end }}
--traceback
python3 manage.py storage create \
${UPLOAD_CONTAINER} ${UPLOAD_CONTAINER} \
--type swift \
--storage-auth auth-cloud-ovh
# Instantiate a collection for the collection itself and all levels
python3 manage.py collection create {{ $collection_name | quote }} \
--type "{{ $collection_name }}_type" \
--traceback
{{- end }}
{{- range $product_level := $collection.product_levels }}
python3 manage.py collection create "{{ $collection_name }}_{{ $product_level }}" \
--type "{{ $collection_name }}_type" \
--traceback
{{- end }}
{{- end }} {{/* range .collections */}}
else
echo "Using existing database"
......
chart/files/preprocessor-config.yaml 0 → 100644
+ 26
− 0
View file @ 6fbb8e8e
{{- define "preprocessor.storage" }}
type: {{ .type }}
kwargs:
{{- if eq .type "swift" }}
username: {{ .username }}
password: {{ .password }}
tenant_name: {{ .tenant_name }}
tenant_id: {{ .tenant_id }}
region_name: {{ .region_name }}
auth_url: {{ .auth_url }}
auth_version: {{ .auth_version }}
user_domain_name: {{ .user_domain_name }}
{{- else if eq .type "s3" }}
bucket: {{ .bucket }}
endpoint_url: {{ .endpoint_url }}
access_key_id: {{ .access_key_id }}
secret_access_key: {{ .secret_access_key }}
region: {{ .region }}
{{- end }}
{{- end -}}
source:
{{ template "preprocessor.storage" .Values.config.objectStorage.download | indent 2 }}
target:
{{ template "preprocessor.storage" .Values.config.objectStorage.data | indent 2 }}
{{ toYaml .Values.config.preprocessor | indent 2 }}
chart/files/registrar-config.yaml 0 → 100644
+ 0
− 0
View file @ 6fbb8e8e
chart/templates/preprocessor-config-configmap.yaml 0 → 100644
+ 6
− 0
View file @ 6fbb8e8e
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "vs.fullname" . }}-preprocessor-config
data:
{{ (tpl (.Files.Glob "files/preprocessor-config.yaml").AsConfig . ) | indent 2 }}
chart/templates/registrar-config-configmap.yaml 0 → 100644
+ 6
− 0
View file @ 6fbb8e8e
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "vs.fullname" . }}-registrar-config
data:
{{ (tpl (.Files.Glob "files/registrar-config.yaml").AsConfig . ) | nindent 2}}
chart/values.yaml
+ 237
− 24
View file @ 6fbb8e8e
......@@ -18,32 +18,34 @@ config:
DJANGO_USER: djangouser
objectStorage:
download:
OS_AUTH_URL_DOWNLOAD: https://auth.cloud.ovh.net/
OS_PASSWORD_DOWNLOAD: ospw
OS_REGION_NAME_DOWNLOAD: SERCO-DIAS1
OS_TENANT_ID_DOWNLOAD: tenantid
OS_TENANT_NAME_DOWNLOAD: "tenantname"
OS_USERNAME_DOWNLOAD: osuser
ST_AUTH_VERSION_DOWNLOAD: "3"
type: swift
username: "username"
password: "password"
tenant_name: "tenant_name"
tenant_id: "tenant_id"
region_name: "region_name"
auth_url: "auth_url"
auth_url_short: "auth_url_short"
auth_version: "auth_version"
user_domain_name: "user_domain_name"
data:
OS_AUTH_URL: https://auth.cloud.ovh.net/v3/
OS_AUTH_URL_SHORT: https://auth.cloud.ovh.net/
OS_PASSWORD: ospw
OS_REGION_NAME: SERCO-DIAS1
OS_TENANT_ID: tenantid
OS_TENANT_NAME: "tenantname"
OS_USER_DOMAIN_NAME: default
OS_USERNAME: osuser
ST_AUTH_VERSION: "3"
type: swift
username: "username"
password: "password"
tenant_name: "tenant_name"
tenant_id: "tenant_id"
region_name: "region_name"
auth_url: "auth_url"
auth_url_short: "auth_url_short"
auth_version: "auth_version"
user_domain_name: "user_domain_name"
cache:
S3_BUCKET: s3bucket
S3_ID: s3id
S3_SECRET: s3secret
S3_REGION: eu-central-1
preprocessing:
UPLOAD_CONTAINER: container
ENFORCE_FOUR_BANDS: "True"
SPLIT_PARTS_CHECK: "False"
type: S3
bucket: "bucket:"
endpoint_url: "endpoint_url:"
access_key_id: "access_key_id:"
secret_access_key: "secret_access_key:"
region: "region:"
redis:
REDIS_HOST: redis
REDIS_PORT: "6379"
......@@ -54,6 +56,217 @@ config:
REDIS_SEED_QUEUE_KEY: seed_queue
REDIS_SET_KEY: registered_set
client:
layers:
VHR_IMAGE_2018_Level_1:
display_color: '#eb3700'
title: VHR Image 2018 Level 1
layer: VHR_IMAGE_2018_Level_1__TRUE_COLOR
sub_layers:
VHR_IMAGE_2018_Level_1__TRUE_COLOR:
label: VHR Image 2018 True Color
VHR_IMAGE_2018_Level_1__masked_validity:
label: VHR Image 2018 True Color with masked validity
VHR_IMAGE_2018_Level_1__FALSE_COLOR:
label: VHR Image 2018 False Color
VHR_IMAGE_2018_Level_1__NDVI:
label: VHR Image 2018 NDVI
VHR_IMAGE_2018_Level_3:
display_color: '#eb3700'
title: VHR Image 2018 Level 3
layer: VHR_IMAGE_2018_Level_3__TRUE_COLOR
sub_layers:
VHR_IMAGE_2018_Level_3__TRUE_COLOR:
label: VHR Image 2018 True Color
VHR_IMAGE_2018_Level_3__masked_validity:
label: VHR Image 2018 True Color with masked validity
VHR_IMAGE_2018_Level_3__FALSE_COLOR:
label: VHR Image 2018 False Color
VHR_IMAGE_2018_Level_3__NDVI:
label: VHR Image 2018 NDVI
overlay_layers:
VHR_IMAGE_2018_Level_3__outlines:
display_color: '#187465'
title: VHR Image 2018 Level 3 Outlines
layer: VHR_IMAGE_2018_Level_3__outlines
VHR_IMAGE_2018_Level_3__masked_validity__Full:
display_color: '#187465'
title: VHR Image 2018 Level 3 True Color with masked validity Full Coverage
layer: VHR_IMAGE_2018_Level_3__masked_validity__Full
VHR_IMAGE_2018_Level_3__Full:
display_color: '#187465'
title: VHR Image 2018 Level 3 True Color Full Coverage
layer: VHR_IMAGE_2018_Level_3__Full
# cache related options
cache:
metadata:
title: PRISM Data Access Service (PASS) developed by EOX
abstract: PRISM Data Access Service (PASS) developed by EOX
url: https://vhr18.pvs.prism.eox.at/cache/ows
keyword: view service
accessconstraints: UNKNOWN
fees: UNKNOWN
contactname: Stephan Meissl
contactphone: Please contact via mail.
contactfacsimile: None
contactorganization: EOX IT Services GmbH
contactcity: Vienna
contactstateorprovince: Vienna
contactpostcode: 1090
contactcountry: Austria
contactelectronicmailaddress: office@eox.at
contactposition: CTO
providername: EOX
providerurl: https://eox.at
inspire_profile: true
inspire_metadataurl: TBD
defaultlanguage: eng
language: eng
services:
wms:
enabled: true
wmts:
enabled: true
connection_timeout: 10
timeout: 120
expires: 3600
key: /{tileset}/{grid}/{dim}/{z}/{x}/{y}.{ext}
tilesets:
VHR_IMAGE_2018__TRUE_COLOR:
title: VHR Image 2018 True Color
abstract: VHR Image 2018 True Color
VHR_IMAGE_2018__FALSE_COLOR:
title: VHR Image 2018 False Color
abstract: VHR Image 2018 False Color
VHR_IMAGE_2018__NDVI:
title: VHR Image 2018 NDVI
abstract: VHR Image 2018 NDVI
style: earth
VHR_IMAGE_2018_Level_1__TRUE_COLOR:
title: VHR Image 2018 Level 1 True Color
abstract: VHR Image 2018 Level 1 True Color
VHR_IMAGE_2018_Level_1__FALSE_COLOR:
title: VHR Image 2018 Level 1 False Color
abstract: VHR Image 2018 Level 1 False Color
VHR_IMAGE_2018_Level_1__NDVI:
title: VHR Image 2018 Level 1 NDVI
abstract: VHR Image 2018 Level 1 NDVI
style: earth
VHR_IMAGE_2018_Level_1__TRUE_COLOR:
title: VHR Image 2018 Level 3 True Color
abstract: VHR Image 2018 Level 3 True Color
VHR_IMAGE_2018_Level_1__FALSE_COLOR:
title: VHR Image 2018 Level 3 False Color
abstract: VHR Image 2018 Level 3 False Color
VHR_IMAGE_2018_Level_1__NDVI:
title: VHR Image 2018 Level 3 NDVI
abstract: VHR Image 2018 Level 3 NDVI
style: earth
preprocessor:
metadata_glob: '*GSC*.xml'
type_extractor:
xpath:
- /gsc:report/gsc:opt_metadata/gml:using/eop:EarthObservationEquipment/eop:platform/eop:Platform/eop:shortName/text()
level_extractor:
# xpath can also be a list of xpaths to be tried one after another
xpath: substring-after(substring-after(/gsc:report/gsc:opt_metadata/gml:metaDataProperty/gsc:EarthObservationMetaData/eop:parentIdentifier/text(), '/'), '/')
preprocessing:
defaults:
move_files: true
data_file_globs:
- '*.tif'
- '*.jp2'
output:
options:
format: COG
dstSRS: 'EPSG:4326'
dstNodata: 0
creationOptions:
- BLOCKSIZE=512
- COMPRESS=DEFLATE
- NUM_THREADS=8
- BIGTIFF=IF_SAFER
- OVERVIEWS=AUTO
types:
PH1B: # just to pass validation
nested: true
# mapping of collection name to objects
collections:
VHR_IMAGE_2018:
product_types:
- PL00
- DM02
- KS03
- KS04
- PH1A
- PH1B
- SP06
- SP07
- SW00
- TR00
product_levels:
- Level_1
- Level_3
coverage_types:
- RGBNir
products:
type_extractor:
xpath:
- /gsc:report/gsc:opt_metadata/gml:using/eop:EarthObservationEquipment/eop:platform/eop:Platform/eop:shortName/text()
- /gsc:report/gsc:sar_metadata/gml:using/eop:EarthObservationEquipment/eop:platform/eop:Platform/eop:shortName/text()
namespace_map:
level_extractor:
xpath:
namespace_map:
types:
PL00:
coverages:
PL00: RGBNir
default_browse: TRUE_COLOR
browses:
TRUE_COLOR:
red:
expression: red
range: [1000, 15000]
nodata: 0
green:
expression: green
range: [1000, 15000]
nodata: 0
blue:
expression: blue
range: [1000, 15000]
nodata: 0
FALSE_COLOR:
red:
expression: nir
range: [1000, 15000]
nodata: 0
green:
expression: red
range: [1000, 15000]
nodata: 0
blue:
expression: green
range: [1000, 15000]
nodata: 0
NDVI:
grey:
expression: (nir-red)/(nir+red)
range: [-1, 1]
masks:
validity:
validity: true
coverages:
# only RGBNir? SAR? complete list with all options here?
database:
persistence:
enabled: false
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment