diff --git a/cache/Dockerfile b/cache/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..6f0e42e74cd1f4158363abdfcfb74f23b297ae33 --- /dev/null +++ b/cache/Dockerfile @@ -0,0 +1,76 @@ +#------------------------------------------------------------------------------ +# +# Project: prism data access server +# Authors: Stephan Meissl <stephan.meissl@eox.at> +# +#------------------------------------------------------------------------------ +# Copyright (C) 2018 EOX IT Services GmbH <https://eox.at> +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies of this Software or works derived from this Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +#----------------------------------------------------------------------------- + +FROM ubuntu:18.04 +MAINTAINER EOX +LABEL name="prism data access server cache" \ + vendor="EOX IT Services GmbH <https://eox.at>" \ + license="MIT Copyright (C) 2019 EOX IT Services GmbH <https://eox.at>" \ + type="prism data access server cache" \ + version="0.0.1-dev" + +USER root +ADD install.sh \ + / +RUN ./install.sh + +ENV COLLECTION_ID= \ + INSTANCE_ID="prism-data-access-server_cache" \ + RENDERER_HOST= \ + INSTALL_DIR="/var/www/pdas/" \ + COLLECTION= \ + APACHE_CONF="/etc/apache2/sites-enabled/010_prism_cache.conf" \ + APACHE_ServerName="pdas_cache" \ + APACHE_ServerAdmin="office@eox.at" \ + APACHE_NGEO_CACHE_ALIAS="/ows" \ + REDIS_HOST= \ + REDIS_PORT="6379" \ + REDIS_QUEUE_KEY="seed_queue" \ + ST_AUTH_VERSION=3 \ + OS_AUTH_URL= \ + OS_USERNAME= \ + OS_PASSWORD= \ + OS_TENANT_NAME= \ + OS_TENANT_ID= \ + OS_REGION_NAME= + +ADD configure.sh \ + run-httpd.sh \ + run-seeder.sh \ + seeder.py \ + get-token-and-render.sh \ + mapcache-template.xml \ + / +RUN chmod -v +x \ + /configure.sh \ + /run-httpd.sh \ + /run-seeder.sh + +ADD reload-http /etc/cron.d/ + +EXPOSE 80 +CMD ["/run-httpd.sh"] diff --git a/cache/configure.sh b/cache/configure.sh new file mode 100755 index 0000000000000000000000000000000000000000..23f7b87537f5e809f51b4079c144304f7fd8eb18 --- /dev/null +++ b/cache/configure.sh @@ -0,0 +1,91 @@ +#!/bin/bash -e +echo "Running configure.sh" + +echo "Generating directory for seeding logs" +mkdir -p "/cache-db/${COLLECTION}" + +chown -R www-data:www-data "${INSTALL_DIR}" + +if [ ! -f "${APACHE_CONF}" ] ; then + echo "Adding Apache configuration" + + # Log to stderr + if ! grep -Fxq "ErrorLog /proc/self/fd/2" /etc/apache2/apache2.conf ; then + sed -e 's,^ErrorLog .*$,ErrorLog /proc/self/fd/2,' -i /etc/apache2/apache2.conf + fi + + # Enable & configure Keepalive + if ! grep -Fxq "KeepAlive On" /etc/apache2/apache2.conf ; then + sed -e 's/^KeepAlive .*$/KeepAlive On/' -i /etc/apache2/apache2.conf + fi + if ! grep -Fxq "MaxKeepAliveRequests 0" /etc/apache2/apache2.conf ; then + sed -e 's/^MaxKeepAliveRequests .*$/MaxKeepAliveRequests 0/' -i /etc/apache2/apache2.conf + fi + if ! grep -Fxq "KeepAliveTimeout 5" /etc/apache2/apache2.conf ; then + sed -e 's/^KeepAliveTimeout .*$/KeepAliveTimeout 5/' -i /etc/apache2/apache2.conf + fi + + # Enlarge timeout setting for ingestion of full resolution images + if ! grep -Fxq "Timeout 1800" /etc/apache2/apache2.conf ; then + sed -e 's/^Timeout .*$/Timeout 1800/' -i /etc/apache2/apache2.conf + fi + + # TODO optimize Apache configuration like MPM in combination with Docker Swarm + + a2dissite 000-default + a2enmod headers + + MAPCACHE_CONF=`echo ${INSTALL_DIR}/mapcache.xml | sed -e 's;//;/;g'` + cat << EOF > "${APACHE_CONF}" +<VirtualHost *:80> + ServerName ${APACHE_ServerName} + ServerAdmin ${APACHE_ServerAdmin} + + DocumentRoot ${INSTALL_DIR} + <Directory "${INSTALL_DIR}"> + Options -Indexes +FollowSymLinks + Require all granted + Header set Access-Control-Allow-Origin * + </Directory> + + MapCacheAlias $APACHE_NGEO_CACHE_ALIAS "${MAPCACHE_CONF}" + + ErrorLog /proc/self/fd/2 + ServerSignature Off + LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\" %D" ngeo + CustomLog /proc/self/fd/1 ngeo +</VirtualHost> +EOF +else + echo "Using existing Apache configuration" +fi + +if [ ! -f "${INSTALL_DIR}/index.html" ] ; then + echo "Adding index.html to replace Apache HTTP server test page" + cat << EOF > "${INSTALL_DIR}/index.html" +<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd"> +<html> + <head> + <title>PRISM Data Access Service (PDAS) - Cache</title> + </head> + <body> + <h1>PRISM Data Access Service (PDAS) - Cache Test Page<br><font size="-1"> + <strong>powered by</font> <a href="https://eox.at">EOX</a></strong></h1> + <p>This page is used to test the proper operation of the PRISM Data Access Service (PDAS) + cache after it has been installed. If you can read + this page it means that the PRISM Data Access Service (PDAS) cache + installed at this site is working properly.</p> + <p>Links to services:</p> + <ul> + <li><a href="/cache${APACHE_NGEO_CACHE_ALIAS}/wmts/1.0.0/WMTSCapabilities.xml">PDAS WMTS</a></li> + <li><a href="/cache${APACHE_NGEO_CACHE_ALIAS}?SERVICE=WMS&REQUEST=GetCapabilities">PDAS WMS</a></li> + </ul> + </body> +</html> +EOF +else + echo "Using existing index.html" +fi + +echo "Store environment variables for cron." +env > /etc/environment diff --git a/cache/get-token-and-render.sh b/cache/get-token-and-render.sh new file mode 100755 index 0000000000000000000000000000000000000000..cf898357ac5509bb0a1b96be3735bd10ca620fb2 --- /dev/null +++ b/cache/get-token-and-render.sh @@ -0,0 +1,19 @@ +#!/bin/bash -e + +echo "Fetching auth token and storage URL" + +# fetch a new token +eval $(swift auth) + +echo "Fetched auth token ${OS_AUTH_TOKEN} and storage URL ${OS_STORAGE_URL}" + +# render the template with the provided values +echo "Copying and adjusting MapCache configuration file" +mkdir -p "${INSTALL_DIR}" +cd "${INSTALL_DIR}" +cat /mapcache-template.xml \ + | sed -e "s/{{OS_STORAGE_URL}}/$(echo ${OS_STORAGE_URL} | sed -e 's/[]\/$*.^[]/\\&/g')/g" \ + | sed -e "s/{{OS_AUTH_TOKEN}}/$(echo ${OS_AUTH_TOKEN} | sed -e 's/[]\/$*.^[]/\\&/g')/g" \ + | sed -e "s/{{BUCKET_NAME}}/$(echo ${BUCKET_NAME} | sed -e 's/[]\/$*.^[]/\\&/g')/g" > mapcache.xml +sed -e "s;http://localhost/ows;http://${RENDERER_HOST}/ows;" -i mapcache.xml +cd - diff --git a/cache/install.sh b/cache/install.sh new file mode 100755 index 0000000000000000000000000000000000000000..6048eb7a54cdbeb584a0fcec2df03651fbe5e4c6 --- /dev/null +++ b/cache/install.sh @@ -0,0 +1,17 @@ +#!/bin/bash +echo "Running install.sh" + +apt update + +echo "Adding UbuntuGIS repo" +DEBIAN_FRONTEND=noninteractive apt install -y software-properties-common +add-apt-repository -y ppa:ubuntugis/ubuntugis-unstable +apt update + +echo "Installing packages" +DEBIAN_FRONTEND=noninteractive apt install -y libapache2-mod-mapcache \ + mapcache-tools sqlite3 curl apache2 python3-dateutil python3-redis \ + python3-boto3 cron \ + python3-swiftclient python-swiftclient + +rm -rf /var/lib/apt/lists/* diff --git a/cache/mapcache-template.xml b/cache/mapcache-template.xml new file mode 100644 index 0000000000000000000000000000000000000000..fc9a642170d26a69dcdf74571beb646b9f6a8930 --- /dev/null +++ b/cache/mapcache-template.xml @@ -0,0 +1,53 @@ +<mapcache> + <default_format>mixed</default_format> + <format name="mypng" type="PNG"> + <compression>fast</compression> + </format> + <format name="myjpeg" type="JPEG"> + <quality>75</quality> + <photometric>ycbcr</photometric> + </format> + <format name="mixed" type="MIXED"> + <transparent>mypng</transparent> + <opaque>myjpeg</opaque> + </format> + <service type="wms" enabled="true"> + <full_wms>assemble</full_wms> + <resample_mode>bilinear</resample_mode> + <format>mixed</format> + <maxsize>4096</maxsize> + <!-- <forwarding_rule name="wms"> + <param name="SERVICE" type="values">WMS</param> + <http> + <url>http://localhost/browse/ows</url> + </http> + </forwarding_rule> --> + </service> + <service type="wmts" enabled="true"/> + <metadata> + <title>Pre-rendered View Service (pdas) developed by EOX</title> + <abstract>Pre-rendered View Service (pdas) developed by EOX</abstract> + <keyword>view service</keyword> + <accessconstraints>UNKNOWN</accessconstraints> + <fees>UNKNOWN</fees> + <contactname>Stephan Meissl</contactname> + <contactphone>Please contact via mail.</contactphone> + <contactfacsimile>None</contactfacsimile> + <contactorganization>EOX IT Services GmbH</contactorganization> + <contactcity>Vienna</contactcity> + <contactstateorprovince>Vienna</contactstateorprovince> + <contactpostcode>1090</contactpostcode> + <contactcountry>Austria</contactcountry> + <contactelectronicmailaddress>office@eox.at</contactelectronicmailaddress> + <contactposition>CTO</contactposition> + <providername>EOX</providername> + <providerurl>https://eox.at</providerurl> + <inspire_profile>true</inspire_profile> + <inspire_metadataurl>TBD</inspire_metadataurl> + <defaultlanguage>eng</defaultlanguage> + <language>eng</language> + </metadata> + <errors>empty_img</errors> + <lock_dir>/tmp</lock_dir> + <threaded_fetching>true</threaded_fetching> +</mapcache> diff --git a/cache/reload-http b/cache/reload-http new file mode 100755 index 0000000000000000000000000000000000000000..9653417c990bbbee03ecc67a21ac82ce5f173461 --- /dev/null +++ b/cache/reload-http @@ -0,0 +1 @@ +42 * * * * root /get-token-and-render.sh > /proc/1/fd/1 && /usr/sbin/apache2ctl -k graceful > /proc/1/fd/1 diff --git a/cache/run-httpd.sh b/cache/run-httpd.sh new file mode 100755 index 0000000000000000000000000000000000000000..d7e01bf7b92b92c9219dffe3cd142fc35e6112e3 --- /dev/null +++ b/cache/run-httpd.sh @@ -0,0 +1,11 @@ +#!/bin/bash -e + +/get-token-and-render.sh +/configure.sh + +echo "Running cron in background" +service cron start + +echo "Running Apache server" +rm -rf /run/apache2/* /var/run/apache2/* /tmp/apache2* +exec /usr/sbin/apache2ctl -D FOREGROUND diff --git a/cache/run-seeder.sh b/cache/run-seeder.sh new file mode 100644 index 0000000000000000000000000000000000000000..d7855841ee5d8508887b011ccd00add5813e2538 --- /dev/null +++ b/cache/run-seeder.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +/configure.sh + +echo "Running seeder" + +python3 /seeder.py ${COLLECTION} --mode redis --redis-host ${REDIS_HOST} --redis-port ${REDIS_PORT} --redis-queue-key ${REDIS_QUEUE_KEY} diff --git a/cache/seeder.py b/cache/seeder.py new file mode 100644 index 0000000000000000000000000000000000000000..2d9716fc2325f8a40a9693ac61f9f71cd40b0fe9 --- /dev/null +++ b/cache/seeder.py @@ -0,0 +1,300 @@ +#!/usr/bin/env python +#----------------------------------------------------------------------------- +# +# Project: seeder.py +# Authors: Stephan Meissl <stephan.meissl@eox.at> +# +#----------------------------------------------------------------------------- +# Copyright (c) 2018 EOX IT Services GmbH +# +# Python script to pre-seed PDAS cache. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies of this Software or works derived from this Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +#----------------------------------------------------------------------------- + + +import os +import sys +import argparse +import textwrap +import logging +import traceback +import signal +import dateutil.parser +import redis +import subprocess + +import sqlite3 +import boto3 + + +# collection: +COLLECTION_MAP = { + # TODO +} + + +logger = logging.getLogger(__name__) + + +def setup_logging(verbosity): + # start logging setup + # get command line level + verbosity = verbosity + if verbosity == 0: + level = logging.CRITICAL + elif verbosity == 1: + level = logging.ERROR + elif verbosity == 2: + level = logging.WARNING + elif verbosity == 3: + level = logging.INFO + else: + level = logging.DEBUG + logger.setLevel(level) + sh = logging.StreamHandler() + sh.setLevel(level) + formatter = logging.Formatter('%(asctime)s %(levelname)s: %(message)s') + sh.setFormatter(formatter) + logger.addHandler(sh) + # finished logging setup + + +CRS_BOUNDS = { + 3857: (-20037508.3428, -20037508.3428, 20037508.3428, 20037508.3428), + 4326: (-180, -90, 180, 90) +} + +GRID_TO_SRID = { + "GoogleMapsCompatible": 3857, + "WGS84": 4326 +} + + +def seed_mapcache(seed_command, config_file, tileset, grid, + minx, miny, maxx, maxy, minzoom, maxzoom, + start_time, end_time, threads, delete, metatile, + timeout=1800, force=True): + + bounds = CRS_BOUNDS[GRID_TO_SRID[grid]] + full = float(abs(bounds[0]) + abs(bounds[2])) + + dateline_crossed = False + if maxx > bounds[2]: + dateline_crossed = True + # extent is always within [bounds[0],bounds[2]] + # where maxx can be >bounds[2] but <=full + if minx < bounds[0] or minx > bounds[2] or maxx < bounds[0] or maxx > full: + raise Exception("Invalid extent '%s,%s,%s,%s'." + % (minx, miny, maxx, maxy)) + + if minzoom is None: + minzoom = 0 + if maxzoom is None: + maxzoom = 10 + + # start- and end-time are expected to be UTC Zulu + start_time = start_time.replace(tzinfo=None) + end_time = end_time.replace(tzinfo=None) + + logger.info("Starting mapcache seed with parameters: command='%s', " + "config_file='%s', tileset='%s', grid='%s', " + "extent='%f,%f,%f,%f', zoom='%d,%d', nthreads='%s', " + "mode='%s', dimension='TIME=%sZ/%sZ', metatile=%d,%d, " + "force=%s." + % (seed_command, config_file, tileset, grid, + minx, miny, maxx, maxy, minzoom, maxzoom, threads, + "seed" if not delete else "delete", + start_time.isoformat(), end_time.isoformat(), + metatile, metatile, force)) + + seed_args = [ + seed_command, + "-c", config_file, + "-t", tileset, + "-g", grid, + "-e", "%f,%f,%f,%f" + % (minx, miny, bounds[2] if dateline_crossed else maxx, maxy), + "-n", str(threads), + "-z", "%d,%d" % (minzoom, maxzoom), + "-D", "TIME=%sZ/%sZ" % (start_time.isoformat(), end_time.isoformat()), + "-m", "seed" if not delete else "delete", + "-q", + "-M", "%d,%d" % (metatile, metatile), + "-L", "/cache-db/%s/failed_TIME_%sZ_%sZ_extent_%f,%f,%f,%f_zoom-%d-%d" + % ( + collection, + start_time.strftime("%Y%m%dT%H%M%S"), + end_time.strftime("%Y%m%dT%H%M%S"), + minx, miny, maxx, maxy, minzoom, maxzoom + ), + "-P", "10", + ] + + if not delete and force: + seed_args.append("-f") + + logger.debug("MapCache seeding command: '%s'. raw: '%s'." + % (" ".join(seed_args), seed_args)) + + process = subprocess.Popen( + seed_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + ) + + try: + out = process.communicate(timeout=timeout)[0] + except subprocess.TimeoutExpired: + logger.error("Seeding timed out") + process.kill() + out = process.communicate() + + if isinstance(out, tuple): + for string in out: + if string is not None: + for line in string.decode("utf-8").split("\n"): + if line != "": + logger.info( + "MapCache output: %s" % line + ) + else: + out = out.decode("utf-8").strip() + if out != "": + logger.info( + "MapCache output: %s" % out + ) + + if process.returncode != 0: + raise Exception("'%s' failed. Returncode '%d'." + % (seed_command, process.returncode)) + + logger.info("Seeding finished with returncode '%d'." % process.returncode) + + return process.returncode + + +def seeder(collection, start, end, leave_existing=False): + logger.info("Starting seeding from '%s' to '%s'." % (start, end)) + + if start > end: + logger.critical("Start '%s' is after end '%s'." % (start, end)) + sys.exit(1) + + # TODO + + logger.info( + "Finished seeding from '%s' to '%s'." % (start, end) + ) + + +def seeder_redis_wrapper( + collection, leave_existing, host="localhost", port=6379, + queue_key='ingest_queue' +): + client = redis.Redis(host=host, port=port) + while True: + logger.debug("waiting for redis queue '%s'..." % queue_key) + value = client.brpop(queue_key) + start, end = value[1].split(b"/") + seeder( + collection, + dateutil.parser.parse(start), + dateutil.parser.parse(end) + ) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.description = textwrap.dedent("""\ + Pre-seeds cache of PRISM Data Access Service (pdas). + """) + + parser.add_argument( + "collection", default=None, + help=( + "Collection the seeder is run for." + ) + ) + parser.add_argument( + "--mode", default="standard", choices=["standard", "redis"], + help=( + "The mode to run the seeder. Either one-off (standard) or " + "reading from a redis queue." + ) + ) + parser.add_argument( + "--start", default=None, type=dateutil.parser.parse, + help=( + "Mandatory argument indicating start date and time for " + "the seeding." + ) + ) + parser.add_argument( + "--end", default=None, type=dateutil.parser.parse, + help=( + "Mandatory argument indicating end date and time for the seeding." + ) + ) + parser.add_argument( + "--leave_existing", action="store_true", + help=( + "Don't delete existing images from cache." + ) + ) + parser.add_argument( + "--redis-queue-key", default="seed_queue" + ) + parser.add_argument( + "--redis-host", default="localhost" + ) + parser.add_argument( + "--redis-port", type=int, default=6379 + ) + + parser.add_argument( + "-v", "--verbosity", type=int, default=3, choices=[0, 1, 2, 3, 4], + help=( + "Set verbosity of log output " + "(4=DEBUG, 3=INFO, 2=WARNING, 1=ERROR, 0=CRITICAL). (default: 3)" + ) + ) + + arg_values = parser.parse_args() + + setup_logging(arg_values.verbosity) + + collection = arg_values.collection + if collection not in COLLECTION_MAP: + logger.critical("Provided collection '%s' is not valid." % collection) + sys.exit(1) + + if arg_values.mode == "standard": + seeder( + collection, + arg_values.start, + arg_values.end, + arg_values.leave_existing, + ) + else: + seeder_redis_wrapper( + collection, + arg_values.leave_existing, + host=arg_values.redis_host, + port=arg_values.redis_port, + queue_key=arg_values.redis_queue_key, + )