Adding core image files.

core/Dockerfile

+#------------------------------------------------------------------------------
+#
+# Project: prism data access server
+# Authors: Stephan Meissl <stephan.meissl@eox.at>
+#
+#------------------------------------------------------------------------------
+# Copyright (C) 2019 EOX IT Services GmbH <https://eox.at>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to
+# deal in the Software without restriction, including without limitation the
+# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+# sell copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies of this Software or works derived from this Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+#-----------------------------------------------------------------------------
+
+FROM ubuntu:18.04
+MAINTAINER EOX
+LABEL name="prism data access server core" \
+      vendor="EOX IT Services GmbH <https://eox.at>" \
+      license="MIT Copyright (C) 2019 EOX IT Services GmbH <https://eox.at>" \
+      type="prism data access server core" \
+      version="0.0.1-dev"
+
+USER root
+
+RUN apt update && \
+    DEBIAN_FRONTEND=noninteractive apt install -y software-properties-common && \
+    add-apt-repository -y ppa:ubuntugis/ppa && \
+    apt update && \
+    DEBIAN_FRONTEND=noninteractive apt install -y \
+      python \
+      python-pip \
+      libpq-dev \
+      python-mapscript \
+      python-gdal \
+      gdal-bin \
+      postgresql-client \
+      python-redis && \
+  apt autoremove -y && \
+  apt clean && \
+  rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+# Install EOxServer (TODO: Use final release package)
+RUN mkdir /opt/eoxserver/
+
+WORKDIR /opt/eoxserver
+ADD eoxserver/eoxserver eoxserver
+ADD eoxserver/tools tools
+ADD eoxserver/setup.cfg eoxserver/setup.py eoxserver/MANIFEST.in eoxserver/README.rst eoxserver/requirements.txt ./
+RUN pip install -r requirements.txt && \
+    pip install . && \
+    pip install python-keystoneclient python-swiftclient
+
+ENV INSTANCE_ID="prism-data-access-server_core" \
+    COLLECTION= \
+    DB_USER= \
+    DB_PW= \
+    DB_HOST= \
+    DB_PORT= \
+    DB_NAME= \
+    INSTALL_DIR="/var/www/pdas/" \
+    DJANGO_USER="admin" \
+    DJANGO_MAIL="office@eox.at" \
+    DJANGO_PASSWORD="***REMOVED***" \
+    DATA_DIR="/data/" \
+    APACHE_CONF="/etc/httpd/conf.d/010_pdas.conf" \
+    APACHE_ServerName="pdas_instance" \
+    APACHE_ServerAdmin="office@eox.at" \
+    APACHE_ALIAS="pdas" \
+    REDIS_HOST= \
+    REDIS_PORT= \
+    REDIS_REGISTER_QUEUE_KEY= \
+    REDIS_REGISTERED_SET_KEY=
+
+ADD rgbnir_definition.json \
+    configure.sh \
+    wait-for-database.sh \
+    run-httpd.sh \
+    run-registrar.sh \
+    registrar.py \
+    /
+#    run-redis-manager.sh \
+#    redis-manager.py \
+
+RUN chmod -v +x \
+    /configure.sh \
+    /wait-for-database.sh \
+    /run-registrar.sh \
+    /run-httpd.sh
+#    /run-redis-manager.sh
+
+EXPOSE 80
+CMD ["/run-httpd.sh"]

core/registrar.py

+#!/usr/bin/env python
+# -----------------------------------------------------------------------------
+#
+# Project: registrar.py
+# Authors: Stephan Meissl <stephan.meissl@eox.at>
+#
+# -----------------------------------------------------------------------------
+# Copyright (c) 2019 EOX IT Services GmbH
+#
+# Python script to register products.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to
+# deal in the Software without restriction, including without limitation the
+# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+# sell copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies of this Software or works derived from this Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+# -----------------------------------------------------------------------------
+
+
+import sys
+import os
+import argparse
+import textwrap
+import logging
+import traceback
+import redis
+
+import lxml.etree
+from swiftclient.service import SwiftService
+
+import django
+from django.db import transaction
+from django.contrib.gis.geos import GEOSGeometry
+
+path = os.path.join(os.getenv('INSTALL_DIR', "/var/www/pdas"), "pdas_instance")
+if path not in sys.path:
+    sys.path.append(path)
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "pdas_instance.settings")
+django.setup()
+
+from eoxserver.backends import access
+from eoxserver.resources.coverages import models
+from eoxserver.resources.coverages.registration.product import (
+    ProductRegistrator
+)
+from eoxserver.resources.coverages.registration.registrators.gdal import (
+    GDALRegistrator
+)
+
+
+# collection: [name]
+COLLECTION_MAP = {
+    "VHR_IMAGE_2018": ["VHR IMAGE 2018", ],
+    "Emergency": ["Emergency", ],
+}
+
+logger = logging.getLogger(__name__)
+
+
+def setup_logging(verbosity):
+    # start logging setup
+    # get command line level
+    verbosity = verbosity
+    if verbosity == 0:
+        level = logging.CRITICAL
+    elif verbosity == 1:
+        level = logging.ERROR
+    elif verbosity == 2:
+        level = logging.WARNING
+    elif verbosity == 3:
+        level = logging.INFO
+    else:
+        level = logging.DEBUG
+    logger.setLevel(level)
+    sh = logging.StreamHandler()
+    sh.setLevel(level)
+    formatter = logging.Formatter("%(asctime)s %(levelname)s: %(message)s")
+    sh.setFormatter(formatter)
+    logger.addHandler(sh)
+    # finished logging setup
+
+
+def add_mask(product):
+    metadata_item = product.metadata_items.all()[0]
+    with access.vsi_open(metadata_item) as f:
+        tree = lxml.etree.parse(f)
+    root = tree.getroot()
+    wkt = tree.xpath(
+        '//gsc:opt_metadata/gml:metaDataProperty/gsc:EarthObservationMetaData/eop:vendorSpecific/eop:SpecificInformation[eop:localAttribute/text() = "CF_POLY"]/eop:localValue/text()',
+        namespaces=root.nsmap
+    )[0]
+    geometry = GEOSGeometry(wkt)
+    mask_type = models.MaskType.objects.get(product_type=product.product_type)
+    models.Mask.objects.create(
+        product=product,
+        mask_type=mask_type,
+        geometry=geometry,
+    )
+
+
+def get_product_level(product):
+    try:
+        metadata_item = product.metadata_items.all()[0]
+        with access.vsi_open(metadata_item) as f:
+            tree = lxml.etree.parse(f)
+        root = tree.getroot()
+        xp = '/gsc:report/gsc:opt_metadata/gml:metaDataProperty/gsc:EarthObservationMetaData/eop:parentIdentifier/text()'
+        product_type_name = tree.xpath(xp, namespaces=root.nsmap)[0]
+        if product_type_name.endswith('Level_1'):
+            return 'Level_1'
+        if product_type_name.endswith('Level_3'):
+            return 'Level_3'
+        else:
+            raise Exception('Invalid product type name %s' % product_type_name)
+    except Exception as e:
+        logger.warning(
+            'Failed to determine product level for product %s, error was %s'
+            % (product.identifier, e)
+        )
+
+
+class RegistrationError(Exception):
+    pass
+
+
+@transaction.atomic
+def registrar(
+    collection,
+    objects_prefix, replace=False, client=None, registered_set_key=None
+):
+    logger.info("Starting registration of product '%s'." % objects_prefix)
+
+    container = objects_prefix.split("/")[1]
+    package = "/".join(objects_prefix.split("/")[2:])
+    metadata_package, data_package = None, None
+
+    with SwiftService() as swift:
+        list_parts_gen = swift.list(
+            container=container, options={"prefix": package},
+        )
+        for page in list_parts_gen:
+            if page["success"]:
+                for item in page["listing"]:
+                    if item["name"].endswith(".xml"):
+                        metadata_package = item["name"]
+                    elif item["name"].endswith(".TIF") or \
+                            item["name"].endswith(".tif"):
+                        data_package = item["name"]
+                    else:
+                        raise RegistrationError(
+                            "Product with objects prefix '%s' has "
+                            "wrong content '%s'."
+                            % (objects_prefix, item["name"])
+                        )
+            else:
+                raise RegistrationError(
+                    "No product found with objects prefix '%s'."
+                    % objects_prefix
+                )
+
+    if metadata_package is None or data_package is None:
+        raise RegistrationError(
+            "Product with objects prefix '%s' has missing content."
+            % objects_prefix
+        )
+
+    product_type = data_package.split("/")[1]
+
+    product, replaced = ProductRegistrator().register(
+        metadata_locations=[[container,
+                             metadata_package, ], ],
+        type_name="%s_Product_%s" % (collection, product_type),
+        replace=replace,
+        extended_metadata=True,
+        mask_locations=None,
+        package_path=None,
+        overrides={},
+    )
+    collection = models.Collection.objects.get(
+        identifier=collection
+    )
+    models.collection_insert_eo_object(collection, product)
+
+    level = get_product_level(product)
+    if level == 'Level_1':
+        collection_level_1 = models.Collection.objects.get(
+            identifier="%s_Level_1" % collection
+        )
+        models.collection_insert_eo_object(collection_level_1, product)
+    elif level == 'Level_3':
+        collection_level_3 = models.Collection.objects.get(
+            identifier="%s_Level_3" % collection
+        )
+        models.collection_insert_eo_object(collection_level_3, product)
+
+    report = GDALRegistrator().register(
+        data_locations=[[container, data_package, ], ],
+        metadata_locations=[[container,
+                             metadata_package, ], ],
+        coverage_type_name="RGBNir",
+        overrides={"identifier": "%s__coverage" % product.identifier},
+        replace=replace,
+    )
+    models.product_add_coverage(product, report.coverage)
+
+    try:
+        add_mask(product)
+    except Exception as e:
+        logger.info("Couldn't add mask.")
+        logger.debug(traceback.format_exc())
+        logger.warning("%s: %s\n" % (type(e).__name__, str(e)))
+
+    if client is not None:
+        logger.debug(
+            "Storing times in redis queue '%s" % registered_set_key
+        )
+        client.sadd(
+            registered_set_key, "%s/%s"
+            % (
+                product.begin_time.strftime("%Y%m%dT%H%M%S"),
+                product.end_time.strftime("%Y%m%dT%H%M%S")
+            )
+        )
+
+    logger.info(
+        "Successfully finished registration of product '%s'." % objects_prefix
+    )
+
+
+def registrar_redis_wrapper(
+    collection,
+    replace=False, host="localhost", port=6379,
+    register_queue_key="register_queue",
+    registered_set_key="registered_set",
+):
+    client = redis.Redis(
+        host=host, port=port, charset="utf-8", decode_responses=True
+    )
+    while True:
+        logger.debug("waiting for redis queue '%s'..." % register_queue_key)
+        value = client.brpop(register_queue_key)
+        try:
+            registrar(
+                collection,
+                value[1],
+                replace=replace,
+                client=client,
+                registered_set_key=registered_set_key
+            )
+        except Exception as e:
+            logger.debug(traceback.format_exc())
+            logger.error("%s: %s\n" % (type(e).__name__, str(e)))
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.description = textwrap.dedent("""\
+    Register products.
+    """)
+
+    parser.add_argument(
+        "collection", default=None,
+        help=(
+            "Collection the registrar is run for."
+        )
+    )
+
+    parser.add_argument(
+        "--mode", default="standard", choices=["standard", "redis"],
+        help=(
+            "The mode to run the registrar. Either one-off (standard) or "
+            "reading from a redis queue."
+        )
+    )
+    parser.add_argument(
+        "--objects-prefix", default=None,
+        help=(
+            "Prefix to objects holding the metadata and data of product."
+        )
+    )
+    parser.add_argument(
+        "--replace", action="store_true",
+        help=(
+            "Replace existing products instead of skipping the registration."
+        )
+    )
+    parser.add_argument(
+        "--redis-register-queue-key", default="register_queue"
+    )
+    parser.add_argument(
+        "--redis-registered-set-key", default="registered_set"
+    )
+    parser.add_argument(
+        "--redis-host", default="localhost"
+    )
+    parser.add_argument(
+        "--redis-port", type=int, default=6379
+    )
+
+    parser.add_argument(
+        "-v", "--verbosity", type=int, default=3, choices=[0, 1, 2, 3, 4],
+        help=(
+            "Set verbosity of log output "
+            "(4=DEBUG, 3=INFO, 2=WARNING, 1=ERROR, 0=CRITICAL). (default: 3)"
+        )
+    )
+
+    arg_values = parser.parse_args()
+
+    setup_logging(arg_values.verbosity)
+
+    collection = arg_values.collection
+
+    if collection not in COLLECTION_MAP:
+        logger.critical("Provided collection '%s' is not valid." % collection)
+        sys.exit(1)
+
+    if arg_values.mode == "standard":
+        registrar(
+            collection,
+            arg_values.objects_prefix,
+            replace=arg_values.replace,
+        )
+    else:
+        registrar_redis_wrapper(
+            collection,
+            replace=arg_values.replace,
+            host=arg_values.redis_host,
+            port=arg_values.redis_port,
+            register_queue_key=arg_values.redis_register_queue_key,
+            registered_set_key=arg_values.redis_registered_set_key,
+        )

core/rgbnir_definition.json

+[{
+    "bands": [
+        {
+            "definition": "http://www.opengis.net/def/property/OGC/0/Radiance",
+            "description": "Red Channel",
+            "gdal_interpretation": "RedBand",
+            "identifier": "red",
+            "name": "red",
+            "nil_values": [
+                {
+                    "reason": "http://www.opengis.net/def/nil/OGC/0/unknown",
+                    "value": 0
+                }
+            ],
+            "uom": "W.m-2.Sr-1",
+            "significant_figures": 5,
+            "allowed_value_ranges": [
+                [0, 65535]
+            ]
+        },
+        {
+            "definition": "http://www.opengis.net/def/property/OGC/0/Radiance",
+            "description": "Green Channel",
+            "gdal_interpretation": "GreenBand",
+            "identifier": "green",
+            "name": "green",
+            "nil_values": [
+                {
+                    "reason": "http://www.opengis.net/def/nil/OGC/0/unknown",
+                    "value": 0
+                }
+            ],
+            "uom": "W.m-2.Sr-1",
+            "significant_figures": 5,
+            "allowed_value_ranges": [
+                [0, 65535]
+            ]
+        },
+        {
+            "definition": "http://www.opengis.net/def/property/OGC/0/Radiance",
+            "description": "Blue Channel",
+            "gdal_interpretation": "BlueBand",
+            "identifier": "blue",
+            "name": "blue",
+            "nil_values": [
+                {
+                    "reason": "http://www.opengis.net/def/nil/OGC/0/unknown",
+                    "value": 0
+                }
+            ],
+            "uom": "W.m-2.Sr-1",
+            "significant_figures": 5,
+            "allowed_value_ranges": [
+                [0, 65535]
+            ]
+        },
+        {
+            "definition": "http://www.opengis.net/def/property/OGC/0/Radiance",
+            "description": "Nir Channel",
+            "gdal_interpretation": "NirBand",
+            "identifier": "nir",
+            "name": "nir",
+            "nil_values": [
+                {
+                    "reason": "http://www.opengis.net/def/nil/OGC/0/unknown",
+                    "value": 0
+                }
+            ],
+            "uom": "W.m-2.Sr-1",
+            "significant_figures": 5,
+            "allowed_value_ranges": [
+                [0, 65535]
+            ]
+        }
+    ],
+    "data_type": "Uint16",
+    "name": "RGBNir"
+}]

core/run-httpd.sh

+#!/bin/bash
+
+/configure.sh
+
+echo "Running gunicorn"
+exec gunicorn --chdir ${INSTALL_DIR}/pdas_instance/ --bind :80 pdas_instance.wsgi:application --workers 8 --timeout 600 --access-logfile - --error-logfile - --log-level warning --disable-redirect-access-to-syslog

core/run-registrar.sh

+#!/bin/sh
+
+/configure.sh
+
+echo "Running registrar"
+python /registrar.py ${COLLECTION} --mode redis --redis-host ${REDIS_HOST} --redis-port ${REDIS_PORT} --redis-register-queue-key ${REDIS_REGISTER_QUEUE_KEY} --redis-registered-set-key ${REDIS_REGISTERED_SET_KEY}

core/wait-for-database.sh

+#!/bin/bash
+# wait-for-database.sh
+
+set -e
+
+cmd="$@"
+
+until PGPASSWORD=${DB_PW} psql "${DB_NAME}" -h "${DB_HOST}" -U "${DB_USER}" -c '\q'; do
+  >&2 echo "Database is unavailable - sleeping"
+  sleep 5
+done
+
+>&2 echo "Database is up - executing command ${cmd}"
+exec ${cmd}