EOX GitLab Instance

Skip to content
Snippets Groups Projects
Commit 7ffe69c0 authored by Stephan's avatar Stephan
Browse files

Adding core image files.

parent 2da250d2
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
core/Dockerfile 0 → 100644
+ 105
− 0
View file @ 7ffe69c0
#------------------------------------------------------------------------------
#
# Project: prism data access server
# Authors: Stephan Meissl <stephan.meissl@eox.at>
#
#------------------------------------------------------------------------------
# Copyright (C) 2019 EOX IT Services GmbH <https://eox.at>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to
# deal in the Software without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
# sell copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies of this Software or works derived from this Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
#-----------------------------------------------------------------------------
FROM ubuntu:18.04
MAINTAINER EOX
LABEL name="prism data access server core" \
vendor="EOX IT Services GmbH <https://eox.at>" \
license="MIT Copyright (C) 2019 EOX IT Services GmbH <https://eox.at>" \
type="prism data access server core" \
version="0.0.1-dev"
USER root
RUN apt update && \
DEBIAN_FRONTEND=noninteractive apt install -y software-properties-common && \
add-apt-repository -y ppa:ubuntugis/ppa && \
apt update && \
DEBIAN_FRONTEND=noninteractive apt install -y \
python \
python-pip \
libpq-dev \
python-mapscript \
python-gdal \
gdal-bin \
postgresql-client \
python-redis && \
apt autoremove -y && \
apt clean && \
rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
# Install EOxServer (TODO: Use final release package)
RUN mkdir /opt/eoxserver/
WORKDIR /opt/eoxserver
ADD eoxserver/eoxserver eoxserver
ADD eoxserver/tools tools
ADD eoxserver/setup.cfg eoxserver/setup.py eoxserver/MANIFEST.in eoxserver/README.rst eoxserver/requirements.txt ./
RUN pip install -r requirements.txt && \
pip install . && \
pip install python-keystoneclient python-swiftclient
ENV INSTANCE_ID="prism-data-access-server_core" \
COLLECTION= \
DB_USER= \
DB_PW= \
DB_HOST= \
DB_PORT= \
DB_NAME= \
INSTALL_DIR="/var/www/pdas/" \
DJANGO_USER="admin" \
DJANGO_MAIL="office@eox.at" \
DJANGO_PASSWORD="***REMOVED***" \
DATA_DIR="/data/" \
APACHE_CONF="/etc/httpd/conf.d/010_pdas.conf" \
APACHE_ServerName="pdas_instance" \
APACHE_ServerAdmin="office@eox.at" \
APACHE_ALIAS="pdas" \
REDIS_HOST= \
REDIS_PORT= \
REDIS_REGISTER_QUEUE_KEY= \
REDIS_REGISTERED_SET_KEY=
ADD rgbnir_definition.json \
configure.sh \
wait-for-database.sh \
run-httpd.sh \
run-registrar.sh \
registrar.py \
/
# run-redis-manager.sh \
# redis-manager.py \
RUN chmod -v +x \
/configure.sh \
/wait-for-database.sh \
/run-registrar.sh \
/run-httpd.sh
# /run-redis-manager.sh
EXPOSE 80
CMD ["/run-httpd.sh"]
core/configure.sh 0 → 100644
+ 614
− 0
View file @ 7ffe69c0
#!/bin/bash
echo "Running configure.sh"
echo "Generating PDAS directory"
mkdir -p "${INSTALL_DIR}"
cd "${INSTALL_DIR}"
if [ ! -d pdas_instance ] ; then
echo "Creating PRISM Data Access Server (PDAS) instance"
eoxserver-admin.py create_instance pdas_instance
echo "Configure PRISM Data Access Server (PDAS) instance"
cd "pdas_instance"
# Configure DBs
INSTALL_DIR_ESCAPED=`echo ${INSTALL_DIR} | sed -e 's,/$,,' | sed -e 's,/,\\\&,g'`
sed -e "s/'ENGINE': 'django.contrib.gis.db.backends.spatialite',/'ENGINE': 'django.contrib.gis.db.backends.postgis',/" -i pdas_instance/settings.py
sed -e "s/'NAME': join(PROJECT_DIR, 'data\/config.sqlite'),/'NAME': '${DB_NAME}',/" -i pdas_instance/settings.py
sed -e "/#'TEST_NAME': join(PROJECT_DIR, 'data\/test-config.sqlite'),/d" -i pdas_instance/settings.py
sed -e "s/'USER': '',/'USER': '${DB_USER}',/" -i pdas_instance/settings.py
sed -e "s/'PASSWORD': '',/'PASSWORD': '${DB_PW}',/" -i pdas_instance/settings.py
sed -e "s/'HOST': '',/'HOST': '${DB_HOST}',/" -i pdas_instance/settings.py
sed -e "s/'PORT': '',/'PORT': '${DB_PORT}',/" -i pdas_instance/settings.py
# Configure instance
sed -e "s/'disable_existing_loggers': True,/'disable_existing_loggers': False,/" -i pdas_instance/settings.py
HANDLERS="'handlers': {\n
'null': {\n
'level':'DEBUG',\n
'class':'logging.NullHandler',\n
},\n
'console': {\n
'level': 'DEBUG' if DEBUG else 'INFO',\n
'class': 'logging.StreamHandler',\n
'formatter': 'verbose' if DEBUG else 'simple',\n
'filters': [],\n
},\n
},"
LOGGERS="'loggers': {\n
'eoxserver': {\n
'handlers': ['console'],\n
'level': 'DEBUG' if DEBUG else 'INFO',\n
'propagate': False,\n
},
}"
sed -e "/^ 'handlers': {$/,/^ },$/c `echo ${HANDLERS}`" -i pdas_instance/settings.py
sed -e "/^ 'loggers': {$/,/^ }$/c `echo ${LOGGERS}`" -i pdas_instance/settings.py
sed -e "s,http_service_url=http://localhost:8000/ows,http_service_url=${APACHE_ALIAS}/ows," -i pdas_instance/conf/eoxserver.conf
sed -e "s/resampling_method=average/resampling_method=near/" -i pdas_instance/conf/eoxserver.conf
# TODO maxsize...
echo "EOXS_VALIDATE_IDS_NCNAME = False" >> pdas_instance/settings.py
echo "EOXS_OPENSEARCH_RECORD_MODEL = 'eoxserver.resources.coverages.models.Product'" >> pdas_instance/settings.py
echo "CACHES = {
'default': {
'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache',
'LOCATION': '/var/tmp/django_cache',
}
}" >> pdas_instance/settings.py
sed -e "/TEMPLATE_DEBUG = DEBUG/d" -i pdas_instance/settings.py
sed -e 's/DEBUG = True/DEBUG = False/' -i pdas_instance/settings.py
# Further configuration
echo "ALLOWED_HOSTS = ['*']" >> pdas_instance/settings.py
echo "USE_X_FORWARDED_HOST = True" >> pdas_instance/settings.py
echo "SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https')" >> pdas_instance/settings.py
python manage.py migrate --noinput
# Check if collection exits in database and initialize database only if not
if python manage.py id check "${COLLECTION}"; then
echo "Initialize database"
python manage.py coveragetype import /rgbnir_definition.json --traceback
if [ "${COLLECTION}" == "VHR_IMAGE_2018" ]; then
echo "Initializing collection '${COLLECTION}'."
# PL00
python manage.py producttype create "${COLLECTION}"_Product_PL00 --traceback \
--coverage-type "RGBNir"
python manage.py browsetype create "${COLLECTION}"_Product_PL00 --traceback \
--red "red" \
--green "green" \
--blue "blue" \
--red-range 1000 15000 \
--green-range 1000 15000 \
--blue-range 1000 15000 \
--red-nodata 0 \
--green-nodata 0 \
--blue-nodata 0
python manage.py browsetype create "${COLLECTION}"_Product_PL00 "TRUE_COLOR" --traceback \
--red "red" \
--green "green" \
--blue "blue" \
--red-range 1000 15000 \
--green-range 1000 15000 \
--blue-range 1000 15000 \
--red-nodata 0 \
--green-nodata 0 \
--blue-nodata 0
python manage.py browsetype create "${COLLECTION}"_Product_PL00 "FALSE_COLOR" --traceback \
--red "nir" \
--green "red" \
--blue "green" \
--red-range 1000 15000 \
--green-range 1000 15000 \
--blue-range 1000 15000 \
--red-nodata 0 \
--green-nodata 0 \
--blue-nodata 0
python manage.py browsetype create "${COLLECTION}"_Product_PL00 "NDVI" --traceback \
--grey "(nir-red)/(nir+red)" --grey-range -1 1
# DM02
python manage.py producttype create "${COLLECTION}"_Product_DM02 --traceback \
--coverage-type "RGBNir"
python manage.py browsetype create "${COLLECTION}"_Product_DM02 --traceback \
--red "red" \
--green "green" \
--blue "blue" \
--red-range 250 11500 \
--green-range 600 10000 \
--blue-range 1300 8500 \
--red-nodata 0 \
--green-nodata 0 \
--blue-nodata 0
python manage.py browsetype create "${COLLECTION}"_Product_DM02 "TRUE_COLOR" --traceback \
--red "red" \
--green "green" \
--blue "blue" \
--red-range 250 11500 \
--green-range 600 10000 \
--blue-range 1300 8500 \
--red-nodata 0 \
--green-nodata 0 \
--blue-nodata 0
python manage.py browsetype create "${COLLECTION}"_Product_DM02 "FALSE_COLOR" --traceback \
--red "nir" \
--green "red" \
--blue "green" \
--red-range 400 28000 \
--green-range 250 11500 \
--blue-range 600 10000 \
--red-nodata 0 \
--green-nodata 0 \
--blue-nodata 0
python manage.py browsetype create "${COLLECTION}"_Product_DM02 "NDVI" --traceback \
--grey "(nir-red)/(nir+red)" --grey-range -1 1
# KS03
python manage.py producttype create "${COLLECTION}"_Product_KS03 --traceback \
--coverage-type "RGBNir"
python manage.py browsetype create "${COLLECTION}"_Product_KS03 --traceback \
--red "red" \
--green "green" \
--blue "blue" \
--red-range 1 10000 \
--green-range 1 10000 \
--blue-range 1 10000 \
--red-nodata 0 \
--green-nodata 0 \
--blue-nodata 0
python manage.py browsetype create "${COLLECTION}"_Product_KS03 "TRUE_COLOR" --traceback \
--red "red" \
--green "green" \
--blue "blue" \
--red-range 1 10000 \
--green-range 1 10000 \
--blue-range 1 10000 \
--red-nodata 0 \
--green-nodata 0 \
--blue-nodata 0
python manage.py browsetype create "${COLLECTION}"_Product_KS03 "FALSE_COLOR" --traceback \
--red "nir" \
--green "red" \
--blue "green" \
--red-range 1 10000 \
--green-range 1 10000 \
--blue-range 1 10000 \
--red-nodata 0 \
--green-nodata 0 \
--blue-nodata 0
python manage.py browsetype create "${COLLECTION}"_Product_KS03 "NDVI" --traceback \
--grey "(nir-red)/(nir+red)" --grey-range -1 1
# KS04
python manage.py producttype create "${COLLECTION}"_Product_KS04 --traceback \
--coverage-type "RGBNir"
python manage.py browsetype create "${COLLECTION}"_Product_KS04 --traceback \
--red "red" \
--green "green" \
--blue "blue" \
--red-range 1 10000 \
--green-range 1 10000 \
--blue-range 1 10000 \
--red-nodata 0 \
--green-nodata 0 \
--blue-nodata 0
python manage.py browsetype create "${COLLECTION}"_Product_KS04 "TRUE_COLOR" --traceback \
--red "red" \
--green "green" \
--blue "blue" \
--red-range 1 10000 \
--green-range 1 10000 \
--blue-range 1 10000 \
--red-nodata 0 \
--green-nodata 0 \
--blue-nodata 0
python manage.py browsetype create "${COLLECTION}"_Product_KS04 "FALSE_COLOR" --traceback \
--red "nir" \
--green "red" \
--blue "green" \
--red-range 1 10000 \
--green-range 1 10000 \
--blue-range 1 10000 \
--red-nodata 0 \
--green-nodata 0 \
--blue-nodata 0
python manage.py browsetype create "${COLLECTION}"_Product_KS04 "NDVI" --traceback \
--grey "(nir-red)/(nir+red)" --grey-range -1 1
# PH1A
python manage.py producttype create "${COLLECTION}"_Product_PH1A --traceback \
--coverage-type "RGBNir"
python manage.py browsetype create "${COLLECTION}"_Product_PH1A --traceback \
--red "red" \
--green "green" \
--blue "blue" \
--red-range 1 2000 \
--green-range 1 2000 \
--blue-range 1 2000 \
--red-nodata 0 \
--green-nodata 0 \
--blue-nodata 0
python manage.py browsetype create "${COLLECTION}"_Product_PH1A "TRUE_COLOR" --traceback \
--red "red" \
--green "green" \
--blue "blue" \
--red-range 1 2000 \
--green-range 1 2000 \
--blue-range 1 2000 \
--red-nodata 0 \
--green-nodata 0 \
--blue-nodata 0
python manage.py browsetype create "${COLLECTION}"_Product_PH1A "FALSE_COLOR" --traceback \
--red "nir" \
--green "red" \
--blue "green" \
--red-range 1 2000 \
--green-range 1 2000 \
--blue-range 1 2000 \
--red-nodata 0 \
--green-nodata 0 \
--blue-nodata 0
python manage.py browsetype create "${COLLECTION}"_Product_PH1A "NDVI" --traceback \
--grey "(nir-red)/(nir+red)" --grey-range -1 1
# PH1B
python manage.py producttype create "${COLLECTION}"_Product_PH1B --traceback \
--coverage-type "RGBNir"
python manage.py browsetype create "${COLLECTION}"_Product_PH1B --traceback \
--red "red" \
--green "green" \
--blue "blue" \
--red-range 1 2000 \
--green-range 1 2000 \
--blue-range 1 2000 \
--red-nodata 0 \
--green-nodata 0 \
--blue-nodata 0
python manage.py browsetype create "${COLLECTION}"_Product_PH1B "TRUE_COLOR" --traceback \
--red "red" \
--green "green" \
--blue "blue" \
--red-range 1 2000 \
--green-range 1 2000 \
--blue-range 1 2000 \
--red-nodata 0 \
--green-nodata 0 \
--blue-nodata 0
python manage.py browsetype create "${COLLECTION}"_Product_PH1B "FALSE_COLOR" --traceback \
--red "nir" \
--green "red" \
--blue "green" \
--red-range 1 2000 \
--green-range 1 2000 \
--blue-range 1 2000 \
--red-nodata 0 \
--green-nodata 0 \
--blue-nodata 0
python manage.py browsetype create "${COLLECTION}"_Product_PH1B "NDVI" --traceback \
--grey "(nir-red)/(nir+red)" --grey-range -1 1
# SP06
python manage.py producttype create "${COLLECTION}"_Product_SP06 --traceback \
--coverage-type "RGBNir"
python manage.py browsetype create "${COLLECTION}"_Product_SP06 --traceback \
--red "red" \
--green "green" \
--blue "blue" \
--red-range 1000 20000 \
--green-range 1000 20000 \
--blue-range 1000 20000 \
--red-nodata 0 \
--green-nodata 0 \
--blue-nodata 0
python manage.py browsetype create "${COLLECTION}"_Product_SP06 "TRUE_COLOR" --traceback \
--red "red" \
--green "green" \
--blue "blue" \
--red-range 1000 20000 \
--green-range 1000 20000 \
--blue-range 1000 20000 \
--red-nodata 0 \
--green-nodata 0 \
--blue-nodata 0
python manage.py browsetype create "${COLLECTION}"_Product_SP06 "FALSE_COLOR" --traceback \
--red "nir" \
--green "red" \
--blue "green" \
--red-range 1000 20000 \
--green-range 1000 20000 \
--blue-range 1000 20000 \
--red-nodata 0 \
--green-nodata 0 \
--blue-nodata 0
python manage.py browsetype create "${COLLECTION}"_Product_SP06 "NDVI" --traceback \
--grey "(nir-red)/(nir+red)" --grey-range -1 1
# SP07
python manage.py producttype create "${COLLECTION}"_Product_SP07 --traceback \
--coverage-type "RGBNir"
python manage.py browsetype create "${COLLECTION}"_Product_SP07 --traceback \
--red "red" \
--green "green" \
--blue "blue" \
--red-range 1000 20000 \
--green-range 1000 20000 \
--blue-range 1000 20000 \
--red-nodata 0 \
--green-nodata 0 \
--blue-nodata 0
python manage.py browsetype create "${COLLECTION}"_Product_SP07 "TRUE_COLOR" --traceback \
--red "red" \
--green "green" \
--blue "blue" \
--red-range 1000 20000 \
--green-range 1000 20000 \
--blue-range 1000 20000 \
--red-nodata 0 \
--green-nodata 0 \
--blue-nodata 0
python manage.py browsetype create "${COLLECTION}"_Product_SP07 "FALSE_COLOR" --traceback \
--red "nir" \
--green "red" \
--blue "green" \
--red-range 1000 20000 \
--green-range 1000 20000 \
--blue-range 1000 20000 \
--red-nodata 0 \
--green-nodata 0 \
--blue-nodata 0
python manage.py browsetype create "${COLLECTION}"_Product_SP07 "NDVI" --traceback \
--grey "(nir-red)/(nir+red)" --grey-range -1 1
# SW00
python manage.py producttype create "${COLLECTION}"_Product_SW00 --traceback \
--coverage-type "RGBNir"
python manage.py browsetype create "${COLLECTION}"_Product_SW00 --traceback \
--red "red" \
--green "green" \
--blue "blue" \
--red-range 100 1500 \
--green-range 100 1500 \
--blue-range 100 1500 \
--red-nodata 0 \
--green-nodata 0 \
--blue-nodata 0
python manage.py browsetype create "${COLLECTION}"_Product_SW00 "TRUE_COLOR" --traceback \
--red "red" \
--green "green" \
--blue "blue" \
--red-range 100 1500 \
--green-range 100 1500 \
--blue-range 100 1500 \
--red-nodata 0 \
--green-nodata 0 \
--blue-nodata 0
python manage.py browsetype create "${COLLECTION}"_Product_SW00 "FALSE_COLOR" --traceback \
--red "nir" \
--green "red" \
--blue "green" \
--red-range 100 1500 \
--green-range 100 1500 \
--blue-range 100 1500 \
--red-nodata 0 \
--green-nodata 0 \
--blue-nodata 0
python manage.py browsetype create "${COLLECTION}"_Product_SW00 "NDVI" --traceback \
--grey "(nir-red)/(nir+red)" --grey-range -1 1
# TR00
python manage.py producttype create "${COLLECTION}"_Product_TR00 --traceback \
--coverage-type "RGBNir"
python manage.py browsetype create "${COLLECTION}"_Product_TR00 --traceback \
--red "red" \
--green "green" \
--blue "blue" \
--red-range 1 3000 \
--green-range 1 3000 \
--blue-range 1 3000 \
--red-nodata 0 \
--green-nodata 0 \
--blue-nodata 0
python manage.py browsetype create "${COLLECTION}"_Product_TR00 "TRUE_COLOR" --traceback \
--red "red" \
--green "green" \
--blue "blue" \
--red-range 1 3000 \
--green-range 1 3000 \
--blue-range 1 3000 \
--red-nodata 0 \
--green-nodata 0 \
--blue-nodata 0
python manage.py browsetype create "${COLLECTION}"_Product_TR00 "FALSE_COLOR" --traceback \
--red "nir" \
--green "red" \
--blue "green" \
--red-range 1 3000 \
--green-range 1 3000 \
--blue-range 1 3000 \
--red-nodata 0 \
--green-nodata 0 \
--blue-nodata 0
python manage.py browsetype create "${COLLECTION}"_Product_TR00 "NDVI" --traceback \
--grey "(nir-red)/(nir+red)" --grey-range -1 1
python manage.py collectiontype create "${COLLECTION}"_Collection --traceback \
--coverage-type "RGBNir" \
--product-type "${COLLECTION}"_Product_PL00 \
--product-type "${COLLECTION}"_Product_DM02 \
--product-type "${COLLECTION}"_Product_KS03 \
--product-type "${COLLECTION}"_Product_KS04 \
--product-type "${COLLECTION}"_Product_PH1A \
--product-type "${COLLECTION}"_Product_PH1B \
--product-type "${COLLECTION}"_Product_SP06 \
--product-type "${COLLECTION}"_Product_SP07 \
--product-type "${COLLECTION}"_Product_SW00 \
--product-type "${COLLECTION}"_Product_TR00
# Create collections for all/Level-1/Level-3 products
python manage.py collection create "${COLLECTION}" --type "${COLLECTION}"_Collection --traceback
python manage.py collection create "${COLLECTION}_Level_1" --type "${COLLECTION}"_Collection --traceback
python manage.py collection create "${COLLECTION}_Level_3" --type "${COLLECTION}"_Collection --traceback
# Register mask type
python manage.py masktype create --validity "${COLLECTION}"_Product_PL00 validity
python manage.py masktype create --validity "${COLLECTION}"_Product_DM02 validity
python manage.py masktype create --validity "${COLLECTION}"_Product_KS03 validity
python manage.py masktype create --validity "${COLLECTION}"_Product_KS04 validity
python manage.py masktype create --validity "${COLLECTION}"_Product_PH1A validity
python manage.py masktype create --validity "${COLLECTION}"_Product_PH1B validity
python manage.py masktype create --validity "${COLLECTION}"_Product_SP06 validity
python manage.py masktype create --validity "${COLLECTION}"_Product_SP07 validity
python manage.py masktype create --validity "${COLLECTION}"_Product_SW00 validity
python manage.py masktype create --validity "${COLLECTION}"_Product_TR00 validity
elif [ "${COLLECTION}" == "Emergency" ]; then
echo "Initializing collection '${COLLECTION}'."
# CS00
# CS01
# CS02
# CS03
# CS04
# DM01
# EQ02
# EW01
# EW02
# EW03
# GE01
# GY01
# IK02
# RE00
# RS02
# SP04
# SP05
# TX01
# DM02
# KS03
# SP06
# SP07
# PH1A
python manage.py producttype create "${COLLECTION}"_Product_PH1A --traceback \
--coverage-type "RGBNir"
python manage.py browsetype create "${COLLECTION}"_Product_PH1A --traceback \
--red "red" \
--green "green" \
--blue "blue" \
--red-range 1 2000 \
--green-range 1 2000 \
--blue-range 1 2000 \
--red-nodata 0 \
--green-nodata 0 \
--blue-nodata 0
python manage.py browsetype create "${COLLECTION}"_Product_PH1A "TRUE_COLOR" --traceback \
--red "red" \
--green "green" \
--blue "blue" \
--red-range 1 2000 \
--green-range 1 2000 \
--blue-range 1 2000 \
--red-nodata 0 \
--green-nodata 0 \
--blue-nodata 0
python manage.py browsetype create "${COLLECTION}"_Product_PH1A "FALSE_COLOR" --traceback \
--red "nir" \
--green "red" \
--blue "green" \
--red-range 1 2000 \
--green-range 1 2000 \
--blue-range 1 2000 \
--red-nodata 0 \
--green-nodata 0 \
--blue-nodata 0
python manage.py browsetype create "${COLLECTION}"_Product_PH1A "NDVI" --traceback \
--grey "(nir-red)/(nir+red)" --grey-range -1 1
# PH1B
python manage.py producttype create "${COLLECTION}"_Product_PH1B --traceback \
--coverage-type "RGBNir"
python manage.py browsetype create "${COLLECTION}"_Product_PH1B --traceback \
--red "red" \
--green "green" \
--blue "blue" \
--red-range 1 2000 \
--green-range 1 2000 \
--blue-range 1 2000 \
--red-nodata 0 \
--green-nodata 0 \
--blue-nodata 0
python manage.py browsetype create "${COLLECTION}"_Product_PH1B "TRUE_COLOR" --traceback \
--red "red" \
--green "green" \
--blue "blue" \
--red-range 1 2000 \
--green-range 1 2000 \
--blue-range 1 2000 \
--red-nodata 0 \
--green-nodata 0 \
--blue-nodata 0
python manage.py browsetype create "${COLLECTION}"_Product_PH1B "FALSE_COLOR" --traceback \
--red "nir" \
--green "red" \
--blue "green" \
--red-range 1 2000 \
--green-range 1 2000 \
--blue-range 1 2000 \
--red-nodata 0 \
--green-nodata 0 \
--blue-nodata 0
python manage.py browsetype create "${COLLECTION}"_Product_PH1B "NDVI" --traceback \
--grey "(nir-red)/(nir+red)" --grey-range -1 1
python manage.py collectiontype create "${COLLECTION}"_Collection --traceback \
--coverage-type "RGBNir" \
--product-type "${COLLECTION}"_Product_PH1A \
--product-type "${COLLECTION}"_Product_PH1B
python manage.py collection create "${COLLECTION}" --type "${COLLECTION}"_Collection --traceback
else
echo "Provided collection '${COLLECTION}' not valid."
fi
python manage.py storageauth create auth-cloud-ovh "${OS_AUTH_URL_SHORT}" \
--type keystone \
-p auth-version "${ST_AUTH_VERSION}" \
-p identity-api-version="${ST_AUTH_VERSION}" \
-p username "${OS_USERNAME}" \
-p password "${OS_PASSWORD}" \
-p tenant-name "${OS_TENANT_NAME}" \
-p tenant-id "${OS_TENANT_ID}" \
-p region-name "${OS_REGION_NAME}"
for bucket in "data10" "data11" "data12" "data13" "data14" "data15" "data16" "data17" "data18" "data20" "data21" "data22" "data23" "data24" "data25" "data26"; do
python manage.py storage create \
${bucket} ${bucket} \
--type swift \
--storage-auth auth-cloud-ovh
done
echo "Creating admin user"
python manage.py shell -c "from django.contrib.auth.models import User; User.objects.create_superuser('${DJANGO_USER}', '${DJANGO_MAIL}', '${DJANGO_PASSWORD}')"
else
echo "Using existing database"
fi
# Collect static files
python manage.py collectstatic --noinput
chmod g+w -R .
chgrp users -R .
else
echo "Using existing PRISM Data Access Server (PDAS) instance"
fi
core/registrar.py 0 → 100644
+ 345
− 0
View file @ 7ffe69c0
#!/usr/bin/env python
# -----------------------------------------------------------------------------
#
# Project: registrar.py
# Authors: Stephan Meissl <stephan.meissl@eox.at>
#
# -----------------------------------------------------------------------------
# Copyright (c) 2019 EOX IT Services GmbH
#
# Python script to register products.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to
# deal in the Software without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
# sell copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies of this Software or works derived from this Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
# -----------------------------------------------------------------------------
import sys
import os
import argparse
import textwrap
import logging
import traceback
import redis
import lxml.etree
from swiftclient.service import SwiftService
import django
from django.db import transaction
from django.contrib.gis.geos import GEOSGeometry
path = os.path.join(os.getenv('INSTALL_DIR', "/var/www/pdas"), "pdas_instance")
if path not in sys.path:
sys.path.append(path)
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "pdas_instance.settings")
django.setup()
from eoxserver.backends import access
from eoxserver.resources.coverages import models
from eoxserver.resources.coverages.registration.product import (
ProductRegistrator
)
from eoxserver.resources.coverages.registration.registrators.gdal import (
GDALRegistrator
)
# collection: [name]
COLLECTION_MAP = {
"VHR_IMAGE_2018": ["VHR IMAGE 2018", ],
"Emergency": ["Emergency", ],
}
logger = logging.getLogger(__name__)
def setup_logging(verbosity):
# start logging setup
# get command line level
verbosity = verbosity
if verbosity == 0:
level = logging.CRITICAL
elif verbosity == 1:
level = logging.ERROR
elif verbosity == 2:
level = logging.WARNING
elif verbosity == 3:
level = logging.INFO
else:
level = logging.DEBUG
logger.setLevel(level)
sh = logging.StreamHandler()
sh.setLevel(level)
formatter = logging.Formatter("%(asctime)s %(levelname)s: %(message)s")
sh.setFormatter(formatter)
logger.addHandler(sh)
# finished logging setup
def add_mask(product):
metadata_item = product.metadata_items.all()[0]
with access.vsi_open(metadata_item) as f:
tree = lxml.etree.parse(f)
root = tree.getroot()
wkt = tree.xpath(
'//gsc:opt_metadata/gml:metaDataProperty/gsc:EarthObservationMetaData/eop:vendorSpecific/eop:SpecificInformation[eop:localAttribute/text() = "CF_POLY"]/eop:localValue/text()',
namespaces=root.nsmap
)[0]
geometry = GEOSGeometry(wkt)
mask_type = models.MaskType.objects.get(product_type=product.product_type)
models.Mask.objects.create(
product=product,
mask_type=mask_type,
geometry=geometry,
)
def get_product_level(product):
try:
metadata_item = product.metadata_items.all()[0]
with access.vsi_open(metadata_item) as f:
tree = lxml.etree.parse(f)
root = tree.getroot()
xp = '/gsc:report/gsc:opt_metadata/gml:metaDataProperty/gsc:EarthObservationMetaData/eop:parentIdentifier/text()'
product_type_name = tree.xpath(xp, namespaces=root.nsmap)[0]
if product_type_name.endswith('Level_1'):
return 'Level_1'
if product_type_name.endswith('Level_3'):
return 'Level_3'
else:
raise Exception('Invalid product type name %s' % product_type_name)
except Exception as e:
logger.warning(
'Failed to determine product level for product %s, error was %s'
% (product.identifier, e)
)
class RegistrationError(Exception):
pass
@transaction.atomic
def registrar(
collection,
objects_prefix, replace=False, client=None, registered_set_key=None
):
logger.info("Starting registration of product '%s'." % objects_prefix)
container = objects_prefix.split("/")[1]
package = "/".join(objects_prefix.split("/")[2:])
metadata_package, data_package = None, None
with SwiftService() as swift:
list_parts_gen = swift.list(
container=container, options={"prefix": package},
)
for page in list_parts_gen:
if page["success"]:
for item in page["listing"]:
if item["name"].endswith(".xml"):
metadata_package = item["name"]
elif item["name"].endswith(".TIF") or \
item["name"].endswith(".tif"):
data_package = item["name"]
else:
raise RegistrationError(
"Product with objects prefix '%s' has "
"wrong content '%s'."
% (objects_prefix, item["name"])
)
else:
raise RegistrationError(
"No product found with objects prefix '%s'."
% objects_prefix
)
if metadata_package is None or data_package is None:
raise RegistrationError(
"Product with objects prefix '%s' has missing content."
% objects_prefix
)
product_type = data_package.split("/")[1]
product, replaced = ProductRegistrator().register(
metadata_locations=[[container,
metadata_package, ], ],
type_name="%s_Product_%s" % (collection, product_type),
replace=replace,
extended_metadata=True,
mask_locations=None,
package_path=None,
overrides={},
)
collection = models.Collection.objects.get(
identifier=collection
)
models.collection_insert_eo_object(collection, product)
level = get_product_level(product)
if level == 'Level_1':
collection_level_1 = models.Collection.objects.get(
identifier="%s_Level_1" % collection
)
models.collection_insert_eo_object(collection_level_1, product)
elif level == 'Level_3':
collection_level_3 = models.Collection.objects.get(
identifier="%s_Level_3" % collection
)
models.collection_insert_eo_object(collection_level_3, product)
report = GDALRegistrator().register(
data_locations=[[container, data_package, ], ],
metadata_locations=[[container,
metadata_package, ], ],
coverage_type_name="RGBNir",
overrides={"identifier": "%s__coverage" % product.identifier},
replace=replace,
)
models.product_add_coverage(product, report.coverage)
try:
add_mask(product)
except Exception as e:
logger.info("Couldn't add mask.")
logger.debug(traceback.format_exc())
logger.warning("%s: %s\n" % (type(e).__name__, str(e)))
if client is not None:
logger.debug(
"Storing times in redis queue '%s" % registered_set_key
)
client.sadd(
registered_set_key, "%s/%s"
% (
product.begin_time.strftime("%Y%m%dT%H%M%S"),
product.end_time.strftime("%Y%m%dT%H%M%S")
)
)
logger.info(
"Successfully finished registration of product '%s'." % objects_prefix
)
def registrar_redis_wrapper(
collection,
replace=False, host="localhost", port=6379,
register_queue_key="register_queue",
registered_set_key="registered_set",
):
client = redis.Redis(
host=host, port=port, charset="utf-8", decode_responses=True
)
while True:
logger.debug("waiting for redis queue '%s'..." % register_queue_key)
value = client.brpop(register_queue_key)
try:
registrar(
collection,
value[1],
replace=replace,
client=client,
registered_set_key=registered_set_key
)
except Exception as e:
logger.debug(traceback.format_exc())
logger.error("%s: %s\n" % (type(e).__name__, str(e)))
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.description = textwrap.dedent("""\
Register products.
""")
parser.add_argument(
"collection", default=None,
help=(
"Collection the registrar is run for."
)
)
parser.add_argument(
"--mode", default="standard", choices=["standard", "redis"],
help=(
"The mode to run the registrar. Either one-off (standard) or "
"reading from a redis queue."
)
)
parser.add_argument(
"--objects-prefix", default=None,
help=(
"Prefix to objects holding the metadata and data of product."
)
)
parser.add_argument(
"--replace", action="store_true",
help=(
"Replace existing products instead of skipping the registration."
)
)
parser.add_argument(
"--redis-register-queue-key", default="register_queue"
)
parser.add_argument(
"--redis-registered-set-key", default="registered_set"
)
parser.add_argument(
"--redis-host", default="localhost"
)
parser.add_argument(
"--redis-port", type=int, default=6379
)
parser.add_argument(
"-v", "--verbosity", type=int, default=3, choices=[0, 1, 2, 3, 4],
help=(
"Set verbosity of log output "
"(4=DEBUG, 3=INFO, 2=WARNING, 1=ERROR, 0=CRITICAL). (default: 3)"
)
)
arg_values = parser.parse_args()
setup_logging(arg_values.verbosity)
collection = arg_values.collection
if collection not in COLLECTION_MAP:
logger.critical("Provided collection '%s' is not valid." % collection)
sys.exit(1)
if arg_values.mode == "standard":
registrar(
collection,
arg_values.objects_prefix,
replace=arg_values.replace,
)
else:
registrar_redis_wrapper(
collection,
replace=arg_values.replace,
host=arg_values.redis_host,
port=arg_values.redis_port,
register_queue_key=arg_values.redis_register_queue_key,
registered_set_key=arg_values.redis_registered_set_key,
)
core/rgbnir_definition.json 0 → 100644
+ 78
− 0
View file @ 7ffe69c0
[{
"bands": [
{
"definition": "http://www.opengis.net/def/property/OGC/0/Radiance",
"description": "Red Channel",
"gdal_interpretation": "RedBand",
"identifier": "red",
"name": "red",
"nil_values": [
{
"reason": "http://www.opengis.net/def/nil/OGC/0/unknown",
"value": 0
}
],
"uom": "W.m-2.Sr-1",
"significant_figures": 5,
"allowed_value_ranges": [
[0, 65535]
]
},
{
"definition": "http://www.opengis.net/def/property/OGC/0/Radiance",
"description": "Green Channel",
"gdal_interpretation": "GreenBand",
"identifier": "green",
"name": "green",
"nil_values": [
{
"reason": "http://www.opengis.net/def/nil/OGC/0/unknown",
"value": 0
}
],
"uom": "W.m-2.Sr-1",
"significant_figures": 5,
"allowed_value_ranges": [
[0, 65535]
]
},
{
"definition": "http://www.opengis.net/def/property/OGC/0/Radiance",
"description": "Blue Channel",
"gdal_interpretation": "BlueBand",
"identifier": "blue",
"name": "blue",
"nil_values": [
{
"reason": "http://www.opengis.net/def/nil/OGC/0/unknown",
"value": 0
}
],
"uom": "W.m-2.Sr-1",
"significant_figures": 5,
"allowed_value_ranges": [
[0, 65535]
]
},
{
"definition": "http://www.opengis.net/def/property/OGC/0/Radiance",
"description": "Nir Channel",
"gdal_interpretation": "NirBand",
"identifier": "nir",
"name": "nir",
"nil_values": [
{
"reason": "http://www.opengis.net/def/nil/OGC/0/unknown",
"value": 0
}
],
"uom": "W.m-2.Sr-1",
"significant_figures": 5,
"allowed_value_ranges": [
[0, 65535]
]
}
],
"data_type": "Uint16",
"name": "RGBNir"
}]
core/run-httpd.sh 0 → 100644
+ 6
− 0
View file @ 7ffe69c0
#!/bin/bash
/configure.sh
echo "Running gunicorn"
exec gunicorn --chdir ${INSTALL_DIR}/pdas_instance/ --bind :80 pdas_instance.wsgi:application --workers 8 --timeout 600 --access-logfile - --error-logfile - --log-level warning --disable-redirect-access-to-syslog
core/run-registrar.sh 0 → 100644
+ 6
− 0
View file @ 7ffe69c0
#!/bin/sh
/configure.sh
echo "Running registrar"
python /registrar.py ${COLLECTION} --mode redis --redis-host ${REDIS_HOST} --redis-port ${REDIS_PORT} --redis-register-queue-key ${REDIS_REGISTER_QUEUE_KEY} --redis-registered-set-key ${REDIS_REGISTERED_SET_KEY}
core/wait-for-database.sh 0 → 100644
+ 14
− 0
View file @ 7ffe69c0
#!/bin/bash
# wait-for-database.sh
set -e
cmd="$@"
until PGPASSWORD=${DB_PW} psql "${DB_NAME}" -h "${DB_HOST}" -U "${DB_USER}" -c '\q'; do
>&2 echo "Database is unavailable - sleeping"
sleep 5
done
>&2 echo "Database is up - executing command ${cmd}"
exec ${cmd}
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment