Adding initial cache files.

cache/Dockerfile

+#------------------------------------------------------------------------------
+#
+# Project: prism data access server
+# Authors: Stephan Meissl <stephan.meissl@eox.at>
+#
+#------------------------------------------------------------------------------
+# Copyright (C) 2018 EOX IT Services GmbH <https://eox.at>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to
+# deal in the Software without restriction, including without limitation the
+# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+# sell copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies of this Software or works derived from this Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+#-----------------------------------------------------------------------------
+
+FROM ubuntu:18.04
+MAINTAINER EOX
+LABEL name="prism data access server cache" \
+      vendor="EOX IT Services GmbH <https://eox.at>" \
+      license="MIT Copyright (C) 2019 EOX IT Services GmbH <https://eox.at>" \
+      type="prism data access server cache" \
+      version="0.0.1-dev"
+
+USER root
+ADD install.sh \
+    /
+RUN ./install.sh
+
+ENV COLLECTION_ID= \
+    INSTANCE_ID="prism-data-access-server_cache" \
+    RENDERER_HOST= \
+    INSTALL_DIR="/var/www/pdas/" \
+    COLLECTION= \
+    APACHE_CONF="/etc/apache2/sites-enabled/010_prism_cache.conf" \
+    APACHE_ServerName="pdas_cache" \
+    APACHE_ServerAdmin="office@eox.at" \
+    APACHE_NGEO_CACHE_ALIAS="/ows" \
+    REDIS_HOST= \
+    REDIS_PORT="6379" \
+    REDIS_QUEUE_KEY="seed_queue" \
+    ST_AUTH_VERSION=3 \
+    OS_AUTH_URL= \
+    OS_USERNAME= \
+    OS_PASSWORD= \
+    OS_TENANT_NAME= \
+    OS_TENANT_ID= \
+    OS_REGION_NAME=
+
+ADD configure.sh \
+    run-httpd.sh \
+    run-seeder.sh \
+    seeder.py \
+    get-token-and-render.sh \
+    mapcache-template.xml \
+    /
+RUN chmod -v +x \
+    /configure.sh \
+    /run-httpd.sh \
+    /run-seeder.sh
+
+ADD reload-http /etc/cron.d/
+
+EXPOSE 80
+CMD ["/run-httpd.sh"]

cache/configure.sh

+#!/bin/bash -e
+echo "Running configure.sh"
+
+echo "Generating directory for seeding logs"
+mkdir -p "/cache-db/${COLLECTION}"
+
+chown -R www-data:www-data "${INSTALL_DIR}"
+
+if [ ! -f "${APACHE_CONF}" ] ; then
+    echo "Adding Apache configuration"
+
+    # Log to stderr
+    if ! grep -Fxq "ErrorLog /proc/self/fd/2" /etc/apache2/apache2.conf ; then
+        sed -e 's,^ErrorLog .*$,ErrorLog /proc/self/fd/2,' -i /etc/apache2/apache2.conf
+    fi
+
+    # Enable & configure Keepalive
+    if ! grep -Fxq "KeepAlive On" /etc/apache2/apache2.conf ; then
+        sed -e 's/^KeepAlive .*$/KeepAlive On/' -i /etc/apache2/apache2.conf
+    fi
+    if ! grep -Fxq "MaxKeepAliveRequests 0" /etc/apache2/apache2.conf ; then
+        sed -e 's/^MaxKeepAliveRequests .*$/MaxKeepAliveRequests 0/' -i /etc/apache2/apache2.conf
+    fi
+    if ! grep -Fxq "KeepAliveTimeout 5" /etc/apache2/apache2.conf ; then
+        sed -e 's/^KeepAliveTimeout .*$/KeepAliveTimeout 5/' -i /etc/apache2/apache2.conf
+    fi
+
+    # Enlarge timeout setting for ingestion of full resolution images
+    if ! grep -Fxq "Timeout 1800" /etc/apache2/apache2.conf ; then
+        sed -e 's/^Timeout .*$/Timeout 1800/' -i /etc/apache2/apache2.conf
+    fi
+
+    # TODO optimize Apache configuration like MPM in combination with Docker Swarm
+
+    a2dissite 000-default
+    a2enmod headers
+
+    MAPCACHE_CONF=`echo ${INSTALL_DIR}/mapcache.xml | sed -e 's;//;/;g'`
+    cat << EOF > "${APACHE_CONF}"
+<VirtualHost *:80>
+    ServerName ${APACHE_ServerName}
+    ServerAdmin ${APACHE_ServerAdmin}
+
+    DocumentRoot ${INSTALL_DIR}
+    <Directory "${INSTALL_DIR}">
+        Options -Indexes +FollowSymLinks
+        Require all granted
+        Header set Access-Control-Allow-Origin *
+    </Directory>
+
+    MapCacheAlias $APACHE_NGEO_CACHE_ALIAS "${MAPCACHE_CONF}"
+
+    ErrorLog /proc/self/fd/2
+    ServerSignature Off
+    LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\" %D" ngeo
+    CustomLog /proc/self/fd/1 ngeo
+</VirtualHost>
+EOF
+else
+    echo "Using existing Apache configuration"
+fi
+
+if [ ! -f "${INSTALL_DIR}/index.html" ] ; then
+    echo "Adding index.html to replace Apache HTTP server test page"
+    cat << EOF > "${INSTALL_DIR}/index.html"
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
+<html>
+    <head>
+        <title>PRISM Data Access Service (PDAS) - Cache</title>
+    </head>
+    <body>
+        <h1>PRISM Data Access Service (PDAS) - Cache Test Page<br><font size="-1">
+        <strong>powered by</font> <a href="https://eox.at">EOX</a></strong></h1>
+        <p>This page is used to test the proper operation of the PRISM Data Access Service (PDAS)
+        cache after it has been installed. If you can read
+        this page it means that the PRISM Data Access Service (PDAS) cache
+        installed at this site is working properly.</p>
+        <p>Links to services:</p>
+        <ul>
+            <li><a href="/cache${APACHE_NGEO_CACHE_ALIAS}/wmts/1.0.0/WMTSCapabilities.xml">PDAS WMTS</a></li>
+            <li><a href="/cache${APACHE_NGEO_CACHE_ALIAS}?SERVICE=WMS&REQUEST=GetCapabilities">PDAS WMS</a></li>
+        </ul>
+    </body>
+</html>
+EOF
+else
+    echo "Using existing index.html"
+fi
+
+echo "Store environment variables for cron."
+env > /etc/environment

cache/get-token-and-render.sh

+#!/bin/bash -e
+
+echo "Fetching auth token and storage URL"
+
+# fetch a new token
+eval $(swift auth)
+
+echo "Fetched auth token ${OS_AUTH_TOKEN} and storage URL ${OS_STORAGE_URL}"
+
+# render the template with the provided values
+echo "Copying and adjusting MapCache configuration file"
+mkdir -p "${INSTALL_DIR}"
+cd "${INSTALL_DIR}"
+cat /mapcache-template.xml \
+    | sed -e "s/{{OS_STORAGE_URL}}/$(echo ${OS_STORAGE_URL} | sed -e 's/[]\/$*.^[]/\\&/g')/g" \
+    | sed -e "s/{{OS_AUTH_TOKEN}}/$(echo ${OS_AUTH_TOKEN} | sed -e 's/[]\/$*.^[]/\\&/g')/g" \
+    | sed -e "s/{{BUCKET_NAME}}/$(echo ${BUCKET_NAME} | sed -e 's/[]\/$*.^[]/\\&/g')/g" > mapcache.xml
+sed -e "s;http://localhost/ows;http://${RENDERER_HOST}/ows;" -i mapcache.xml
+cd -

cache/install.sh

+#!/bin/bash
+echo "Running install.sh"
+
+apt update
+
+echo "Adding UbuntuGIS repo"
+DEBIAN_FRONTEND=noninteractive apt install -y software-properties-common
+add-apt-repository -y ppa:ubuntugis/ubuntugis-unstable
+apt update
+
+echo "Installing packages"
+DEBIAN_FRONTEND=noninteractive apt install -y libapache2-mod-mapcache \
+    mapcache-tools sqlite3 curl apache2 python3-dateutil python3-redis \
+    python3-boto3 cron \
+    python3-swiftclient python-swiftclient
+
+rm -rf /var/lib/apt/lists/*

cache/mapcache-template.xml

+<mapcache>
+  <default_format>mixed</default_format>
+  <format name="mypng" type="PNG">
+    <compression>fast</compression>
+  </format>
+  <format name="myjpeg" type="JPEG">
+    <quality>75</quality>
+    <photometric>ycbcr</photometric>
+  </format>
+  <format name="mixed" type="MIXED">
+    <transparent>mypng</transparent>
+    <opaque>myjpeg</opaque>
+  </format>
+  <service type="wms" enabled="true">
+    <full_wms>assemble</full_wms>
+    <resample_mode>bilinear</resample_mode>
+    <format>mixed</format>
+    <maxsize>4096</maxsize>
+    <!-- <forwarding_rule name="wms">
+      <param name="SERVICE" type="values">WMS</param>
+      <http>
+        <url>http://localhost/browse/ows</url>
+      </http>
+    </forwarding_rule> -->
+  </service>
+  <service type="wmts" enabled="true"/>
+  <metadata>
+    <title>Pre-rendered View Service (pdas) developed by EOX</title>
+    <abstract>Pre-rendered View Service (pdas) developed by EOX</abstract>
+    <keyword>view service</keyword>
+    <accessconstraints>UNKNOWN</accessconstraints>
+    <fees>UNKNOWN</fees>
+    <contactname>Stephan Meissl</contactname>
+    <contactphone>Please contact via mail.</contactphone>
+    <contactfacsimile>None</contactfacsimile>
+    <contactorganization>EOX IT Services GmbH</contactorganization>
+    <contactcity>Vienna</contactcity>
+    <contactstateorprovince>Vienna</contactstateorprovince>
+    <contactpostcode>1090</contactpostcode>
+    <contactcountry>Austria</contactcountry>
+    <contactelectronicmailaddress>office@eox.at</contactelectronicmailaddress>
+    <contactposition>CTO</contactposition>
+    <providername>EOX</providername>
+    <providerurl>https://eox.at</providerurl>
+    <inspire_profile>true</inspire_profile>
+    <inspire_metadataurl>TBD</inspire_metadataurl>
+    <defaultlanguage>eng</defaultlanguage>
+    <language>eng</language>
+  </metadata>
+  <errors>empty_img</errors>
+  <lock_dir>/tmp</lock_dir>
+  <threaded_fetching>true</threaded_fetching>
+</mapcache>

cache/reload-http

+42 * * * * root /get-token-and-render.sh > /proc/1/fd/1 && /usr/sbin/apache2ctl -k graceful > /proc/1/fd/1

cache/run-httpd.sh

+#!/bin/bash -e
+
+/get-token-and-render.sh
+/configure.sh
+
+echo "Running cron in background"
+service cron start
+
+echo "Running Apache server"
+rm -rf /run/apache2/* /var/run/apache2/* /tmp/apache2*
+exec /usr/sbin/apache2ctl -D FOREGROUND

cache/run-seeder.sh

+#!/bin/bash
+
+/configure.sh
+
+echo "Running seeder"
+
+python3 /seeder.py ${COLLECTION} --mode redis --redis-host ${REDIS_HOST} --redis-port ${REDIS_PORT} --redis-queue-key ${REDIS_QUEUE_KEY}

cache/seeder.py

+#!/usr/bin/env python
+#-----------------------------------------------------------------------------
+#
+# Project: seeder.py
+# Authors: Stephan Meissl <stephan.meissl@eox.at>
+#
+#-----------------------------------------------------------------------------
+# Copyright (c) 2018 EOX IT Services GmbH
+#
+# Python script to pre-seed PDAS cache.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to
+# deal in the Software without restriction, including without limitation the
+# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+# sell copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies of this Software or works derived from this Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+#-----------------------------------------------------------------------------
+
+
+import os
+import sys
+import argparse
+import textwrap
+import logging
+import traceback
+import signal
+import dateutil.parser
+import redis
+import subprocess
+
+import sqlite3
+import boto3
+
+
+# collection:
+COLLECTION_MAP = {
+    # TODO
+}
+
+
+logger = logging.getLogger(__name__)
+
+
+def setup_logging(verbosity):
+    # start logging setup
+    # get command line level
+    verbosity = verbosity
+    if verbosity == 0:
+        level = logging.CRITICAL
+    elif verbosity == 1:
+        level = logging.ERROR
+    elif verbosity == 2:
+        level = logging.WARNING
+    elif verbosity == 3:
+        level = logging.INFO
+    else:
+        level = logging.DEBUG
+    logger.setLevel(level)
+    sh = logging.StreamHandler()
+    sh.setLevel(level)
+    formatter = logging.Formatter('%(asctime)s %(levelname)s: %(message)s')
+    sh.setFormatter(formatter)
+    logger.addHandler(sh)
+    # finished logging setup
+
+
+CRS_BOUNDS = {
+    3857: (-20037508.3428, -20037508.3428, 20037508.3428, 20037508.3428),
+    4326: (-180, -90, 180, 90)
+}
+
+GRID_TO_SRID = {
+    "GoogleMapsCompatible": 3857,
+    "WGS84": 4326
+}
+
+
+def seed_mapcache(seed_command, config_file, tileset, grid,
+                  minx, miny, maxx, maxy, minzoom, maxzoom,
+                  start_time, end_time, threads, delete, metatile,
+                  timeout=1800, force=True):
+
+    bounds = CRS_BOUNDS[GRID_TO_SRID[grid]]
+    full = float(abs(bounds[0]) + abs(bounds[2]))
+
+    dateline_crossed = False
+    if maxx > bounds[2]:
+        dateline_crossed = True
+    # extent is always within [bounds[0],bounds[2]]
+    # where maxx can be >bounds[2] but <=full
+    if minx < bounds[0] or minx > bounds[2] or maxx < bounds[0] or maxx > full:
+        raise Exception("Invalid extent '%s,%s,%s,%s'."
+                        % (minx, miny, maxx, maxy))
+
+    if minzoom is None:
+        minzoom = 0
+    if maxzoom is None:
+        maxzoom = 10
+
+    # start- and end-time are expected to be UTC Zulu
+    start_time = start_time.replace(tzinfo=None)
+    end_time = end_time.replace(tzinfo=None)
+
+    logger.info("Starting mapcache seed with parameters: command='%s', "
+                "config_file='%s', tileset='%s', grid='%s', "
+                "extent='%f,%f,%f,%f', zoom='%d,%d', nthreads='%s', "
+                "mode='%s', dimension='TIME=%sZ/%sZ', metatile=%d,%d, "
+                "force=%s."
+                % (seed_command, config_file, tileset, grid,
+                   minx, miny, maxx, maxy, minzoom, maxzoom, threads,
+                   "seed" if not delete else "delete",
+                   start_time.isoformat(), end_time.isoformat(),
+                   metatile, metatile, force))
+
+    seed_args = [
+        seed_command,
+        "-c", config_file,
+        "-t", tileset,
+        "-g", grid,
+        "-e", "%f,%f,%f,%f"
+        % (minx, miny, bounds[2] if dateline_crossed else maxx, maxy),
+        "-n", str(threads),
+        "-z", "%d,%d" % (minzoom, maxzoom),
+        "-D", "TIME=%sZ/%sZ" % (start_time.isoformat(), end_time.isoformat()),
+        "-m", "seed" if not delete else "delete",
+        "-q",
+        "-M", "%d,%d" % (metatile, metatile),
+        "-L", "/cache-db/%s/failed_TIME_%sZ_%sZ_extent_%f,%f,%f,%f_zoom-%d-%d"
+        % (
+            collection,
+            start_time.strftime("%Y%m%dT%H%M%S"),
+            end_time.strftime("%Y%m%dT%H%M%S"),
+            minx, miny, maxx, maxy, minzoom, maxzoom
+        ),
+        "-P", "10",
+    ]
+
+    if not delete and force:
+        seed_args.append("-f")
+
+    logger.debug("MapCache seeding command: '%s'. raw: '%s'."
+                 % (" ".join(seed_args), seed_args))
+
+    process = subprocess.Popen(
+        seed_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+    )
+
+    try:
+        out = process.communicate(timeout=timeout)[0]
+    except subprocess.TimeoutExpired:
+        logger.error("Seeding timed out")
+        process.kill()
+        out = process.communicate()
+
+    if isinstance(out, tuple):
+        for string in out:
+            if string is not None:
+                for line in string.decode("utf-8").split("\n"):
+                    if line != "":
+                        logger.info(
+                            "MapCache output: %s" % line
+                        )
+    else:
+        out = out.decode("utf-8").strip()
+        if out != "":
+            logger.info(
+                "MapCache output: %s" % out
+            )
+
+    if process.returncode != 0:
+        raise Exception("'%s' failed. Returncode '%d'."
+                        % (seed_command, process.returncode))
+
+    logger.info("Seeding finished with returncode '%d'." % process.returncode)
+
+    return process.returncode
+
+
+def seeder(collection, start, end, leave_existing=False):
+    logger.info("Starting seeding from '%s' to '%s'." % (start, end))
+
+    if start > end:
+        logger.critical("Start '%s' is after end '%s'." % (start, end))
+        sys.exit(1)
+
+    # TODO
+
+    logger.info(
+        "Finished seeding from '%s' to '%s'." % (start, end)
+    )
+
+
+def seeder_redis_wrapper(
+        collection, leave_existing, host="localhost", port=6379,
+        queue_key='ingest_queue'
+):
+    client = redis.Redis(host=host, port=port)
+    while True:
+        logger.debug("waiting for redis queue '%s'..." % queue_key)
+        value = client.brpop(queue_key)
+        start, end = value[1].split(b"/")
+        seeder(
+            collection,
+            dateutil.parser.parse(start),
+            dateutil.parser.parse(end)
+        )
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.description = textwrap.dedent("""\
+    Pre-seeds cache of PRISM Data Access Service (pdas).
+    """)
+
+    parser.add_argument(
+        "collection", default=None,
+        help=(
+            "Collection the seeder is run for."
+        )
+    )
+    parser.add_argument(
+        "--mode", default="standard", choices=["standard", "redis"],
+        help=(
+            "The mode to run the seeder. Either one-off (standard) or "
+            "reading from a redis queue."
+        )
+    )
+    parser.add_argument(
+        "--start", default=None, type=dateutil.parser.parse,
+        help=(
+            "Mandatory argument indicating start date and time for "
+            "the seeding."
+        )
+    )
+    parser.add_argument(
+        "--end", default=None, type=dateutil.parser.parse,
+        help=(
+            "Mandatory argument indicating end date and time for the seeding."
+        )
+    )
+    parser.add_argument(
+        "--leave_existing", action="store_true",
+        help=(
+            "Don't delete existing images from cache."
+        )
+    )
+    parser.add_argument(
+        "--redis-queue-key", default="seed_queue"
+    )
+    parser.add_argument(
+        "--redis-host", default="localhost"
+    )
+    parser.add_argument(
+        "--redis-port", type=int, default=6379
+    )
+
+    parser.add_argument(
+        "-v", "--verbosity", type=int, default=3, choices=[0, 1, 2, 3, 4],
+        help=(
+            "Set verbosity of log output "
+            "(4=DEBUG, 3=INFO, 2=WARNING, 1=ERROR, 0=CRITICAL). (default: 3)"
+        )
+    )
+
+    arg_values = parser.parse_args()
+
+    setup_logging(arg_values.verbosity)
+
+    collection = arg_values.collection
+    if collection not in COLLECTION_MAP:
+        logger.critical("Provided collection '%s' is not valid." % collection)
+        sys.exit(1)
+
+    if arg_values.mode == "standard":
+        seeder(
+            collection,
+            arg_values.start,
+            arg_values.end,
+            arg_values.leave_existing,
+        )
+    else:
+        seeder_redis_wrapper(
+            collection,
+            arg_values.leave_existing,
+            host=arg_values.redis_host,
+            port=arg_values.redis_port,
+            queue_key=arg_values.redis_queue_key,
+        )