EOX GitLab Instance

Commit 1ea1e96d authored by Mussab Abdalla's avatar Mussab Abdalla
Browse files

initial commit

parent 727841c0
#------------------------------------------------------------------------------
#
# Project: prism view server
# Authors: Stephan Meissl <stephan.meissl@eox.at>
#
#------------------------------------------------------------------------------
# Copyright (C) 2018 EOX IT Services GmbH <https://eox.at>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to
# deal in the Software without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
# sell copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies of this Software or works derived from this Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
#-----------------------------------------------------------------------------
FROM ubuntu:18.04
MAINTAINER EOX
LABEL name="prism view server cache" \
vendor="EOX IT Services GmbH <https://eox.at>" \
license="MIT Copyright (C) 2019 EOX IT Services GmbH <https://eox.at>" \
type="prism view server cache"
USER root
ADD install.sh \
/
RUN ./install.sh
ENV COLLECTION_ID= \
INSTANCE_ID="prism-view-server_cache" \
RENDERER_HOST= \
INSTALL_DIR="/var/www/pvs/" \
COLLECTION= \
APACHE_CONF="/etc/apache2/sites-enabled/010_prism_cache.conf" \
APACHE_ServerName="pvs_cache" \
APACHE_ServerAdmin="office@eox.at" \
APACHE_NGEO_CACHE_ALIAS="/ows" \
REDIS_HOST="redis" \
REDIS_PORT="6379" \
REDIS_SEED_QUEUE_KEY="seed_queue" \
ST_AUTH_VERSION=3 \
OS_AUTH_URL= \
OS_USERNAME= \
OS_PASSWORD= \
OS_TENANT_NAME= \
OS_TENANT_ID= \
OS_REGION_NAME= \
BUCKET_NAME= \
BUCKET_NAME_FULL=
ADD configure.sh \
run-httpd.sh \
run-seeder.sh \
seeder.py \
mapcache-template.xml \
entrypoint.sh \
mapcache_items_view.sql \
/
RUN chmod -v +x \
/configure.sh \
/run-httpd.sh \
/run-seeder.sh \
/entrypoint.sh
EXPOSE 80
ENTRYPOINT ["/entrypoint.sh"]
CMD ["/run-httpd.sh"]
LABEL version="1.5.2"
#!/bin/bash -e
echo "Running configure.sh"
echo "Generating directory for seeding logs"
mkdir -p "${INSTALL_DIR}"
cd "${INSTALL_DIR}"
cat /mapcache-template.xml \
| sed -e "s/{{ST_AUTH_VERSION}}/$(echo ${ST_AUTH_VERSION} | sed -e 's/[]\/$*.^[]/\\&/g')/g" \
| sed -e "s/{{OS_AUTH_URL_SHORT}}/$(echo ${OS_AUTH_URL_SHORT} | sed -e 's/[]\/$*.^[]/\\&/g')/g" \
| sed -e "s/{{OS_TENANT_NAME}}/$(echo ${OS_TENANT_NAME} | sed -e 's/[]\/$*.^[]/\\&/g')/g" \
| sed -e "s/{{OS_TENANT_ID}}/$(echo ${OS_TENANT_ID} | sed -e 's/[]\/$*.^[]/\\&/g')/g" \
| sed -e "s/{{OS_USERNAME}}/$(echo ${OS_USERNAME} | sed -e 's/[]\/$*.^[]/\\&/g')/g" \
| sed -e "s/{{OS_PASSWORD}}/$(echo ${OS_PASSWORD} | sed -e 's/[]\/$*.^[]/\\&/g')/g" \
| sed -e "s/{{BUCKET_NAME}}/$(echo ${BUCKET_NAME} | sed -e 's/[]\/$*.^[]/\\&/g')/g" \
| sed -e "s/{{BUCKET_NAME_FULL}}/$(echo ${BUCKET_NAME_FULL} | sed -e 's/[]\/$*.^[]/\\&/g')/g" \
| sed -e "s/{{DB_USER}}/$(echo ${DB_USER} | sed -e 's/[]\/$*.^[]/\\&/g')/g" \
| sed -e "s/{{DB_PW}}/$(echo ${DB_PW} | sed -e 's/[]\/$*.^[]/\\&/g')/g" \
| sed -e "s/{{DB_HOST}}/$(echo ${DB_HOST} | sed -e 's/[]\/$*.^[]/\\&/g')/g" \
| sed -e "s/{{DB_PORT}}/$(echo ${DB_PORT} | sed -e 's/[]\/$*.^[]/\\&/g')/g" \
| sed -e "s/{{DB_NAME}}/$(echo ${DB_NAME} | sed -e 's/[]\/$*.^[]/\\&/g')/g" \
| sed -e "s;http://localhost/ows;http://${RENDERER_HOST}/ows;" > mapcache.xml
cd -
chown -R www-data:www-data "${INSTALL_DIR}"
if [ ! -f "${APACHE_CONF}" ] ; then
echo "Adding Apache configuration"
# Log to stderr
if ! grep -Fxq "ErrorLog /proc/self/fd/2" /etc/apache2/apache2.conf ; then
sed -e 's,^ErrorLog .*$,ErrorLog /proc/self/fd/2,' -i /etc/apache2/apache2.conf
fi
# Enable & configure Keepalive
if ! grep -Fxq "KeepAlive On" /etc/apache2/apache2.conf ; then
sed -e 's/^KeepAlive .*$/KeepAlive On/' -i /etc/apache2/apache2.conf
fi
if ! grep -Fxq "MaxKeepAliveRequests 0" /etc/apache2/apache2.conf ; then
sed -e 's/^MaxKeepAliveRequests .*$/MaxKeepAliveRequests 0/' -i /etc/apache2/apache2.conf
fi
if ! grep -Fxq "KeepAliveTimeout 5" /etc/apache2/apache2.conf ; then
sed -e 's/^KeepAliveTimeout .*$/KeepAliveTimeout 5/' -i /etc/apache2/apache2.conf
fi
# Enlarge timeout setting for ingestion of full resolution images
if ! grep -Fxq "Timeout 1800" /etc/apache2/apache2.conf ; then
sed -e 's/^Timeout .*$/Timeout 1800/' -i /etc/apache2/apache2.conf
fi
# TODO optimize Apache configuration like MPM in combination with Docker Swarm
a2dissite 000-default
a2enmod headers
MAPCACHE_CONF=`echo ${INSTALL_DIR}/mapcache.xml | sed -e 's;//;/;g'`
cat << EOF > "${APACHE_CONF}"
<VirtualHost *:80>
ServerName ${APACHE_ServerName}
ServerAdmin ${APACHE_ServerAdmin}
DocumentRoot ${INSTALL_DIR}
<Directory "${INSTALL_DIR}">
Options -Indexes +FollowSymLinks
Require all granted
Header set Access-Control-Allow-Origin *
</Directory>
MapCacheAlias $APACHE_NGEO_CACHE_ALIAS "${MAPCACHE_CONF}"
ErrorLog /proc/self/fd/2
ServerSignature Off
LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\" %D" ngeo
CustomLog /proc/self/fd/1 ngeo
</VirtualHost>
EOF
else
echo "Using existing Apache configuration"
fi
if [ ! -f "${INSTALL_DIR}/index.html" ] ; then
echo "Adding index.html to replace Apache HTTP server test page"
cat << EOF > "${INSTALL_DIR}/index.html"
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
<html>
<head>
<title>PRISM View Server (PVS) - Cache</title>
</head>
<body>
<h1>PRISM View Server (PVS) - Cache Test Page<br><font size="-1">
<strong>powered by</font> <a href="https://eox.at">EOX</a></strong></h1>
<p>This page is used to test the proper operation of the PRISM View Server (PVS)
cache after it has been installed. If you can read
this page it means that the PRISM View Server (PVS) cache
installed at this site is working properly.</p>
<p>Links to services:</p>
<ul>
<li><a href="/cache${APACHE_NGEO_CACHE_ALIAS}/wmts/1.0.0/WMTSCapabilities.xml">PVS WMTS</a></li>
<li><a href="/cache${APACHE_NGEO_CACHE_ALIAS}?SERVICE=WMS&REQUEST=GetCapabilities">PVS WMS</a></li>
</ul>
</body>
</html>
EOF
else
echo "Using existing index.html"
fi
# create a database view for the mapcache items
echo "Creating database view for mapcache items"
psql "host=${DB_HOST} user=${DB_USER} password=${DB_PW} dbname=${DB_NAME} port=${DB_PORT}" -f /mapcache_items_view.sql
#!/bin/bash -e
set -eo pipefail
SERVICES=${WAIT_SERVICES:=''}
TIMEOUT=${WAIT_TIMEOUT:='15'}
if [[ ! -z $SERVICES ]] ; then
for service in $SERVICES ; do
wait-for-it -t $TIMEOUT $service
done
fi
# usage: file_env VAR [DEFAULT]
# ie: file_env 'XYZ_DB_PASSWORD' 'example'
# (will allow for "$XYZ_DB_PASSWORD_FILE" to fill in the value of
# "$XYZ_DB_PASSWORD" from a file, especially for Docker's secrets feature)
file_env() {
local var="$1"
local fileVar="${var}_FILE"
local def="${2:-}"
if [ "${!var:-}" ] && [ "${!fileVar:-}" ]; then
echo "Both $var and $fileVar are set (but are exclusive)" >&2
fi
local val="$def"
if [ "${!var:-}" ]; then
val="${!var}"
elif [ "${!fileVar:-}" ]; then
val="$(< "${!fileVar}")"
fi
cat >> /etc/bash.bashrc <<EOF
export "${var}"="${val}"
EOF
echo "the value of variable ${var} is set" >&2
# make them also available in preparatory steps until container starts
export "${var}"="${val}"
unset "$fileVar"
}
file_env "OS_PASSWORD"
eval "$@"
#!/bin/bash -e
echo "Running install.sh"
apt update
echo "Adding UbuntuGIS repo"
DEBIAN_FRONTEND=noninteractive apt install -y software-properties-common
add-apt-repository -y ppa:ubuntugis/ubuntugis-unstable
add-apt-repository -y ppa:schpidi2/eox
apt update
echo "Installing packages"
VERSION=1.10.0-1~bionic0eox5
DEBIAN_FRONTEND=noninteractive apt install -y \
libmapcache1=${VERSION} libapache2-mod-mapcache=${VERSION} mapcache-tools=${VERSION} \
sqlite3 curl ca-certificates apache2 python3-dateutil python3-redis wait-for-it postgresql-client
rm -rf /var/lib/apt/lists/*
<mapcache>
<default_format>mixed</default_format>
<format name="mypng" type="PNG">
<compression>fast</compression>
</format>
<format name="myjpeg" type="JPEG">
<quality>75</quality>
<photometric>ycbcr</photometric>
</format>
<format name="mixed" type="MIXED">
<transparent>mypng</transparent>
<opaque>myjpeg</opaque>
</format>
<service type="wms" enabled="true">
<full_wms>assemble</full_wms>
<resample_mode>bilinear</resample_mode>
<format>mixed</format>
<maxsize>4096</maxsize>
<!-- <forwarding_rule name="wms">
<param name="SERVICE" type="values">WMS</param>
<http>
<url>http://localhost/browse/ows</url>
</http>
</forwarding_rule> -->
</service>
<service type="wmts" enabled="true"/>
<metadata>
<title>PRISM View Server (PVS) developed by EOX</title>
<abstract>PRISM View Server (PVS) developed by EOX</abstract>
<keyword>View Server</keyword>
<accessconstraints>UNKNOWN</accessconstraints>
<fees>UNKNOWN</fees>
<contactname>Stephan Meissl</contactname>
<contactphone>Please contact via mail.</contactphone>
<contactfacsimile>None</contactfacsimile>
<contactorganization>EOX IT Services GmbH</contactorganization>
<contactcity>Vienna</contactcity>
<contactstateorprovince>Vienna</contactstateorprovince>
<contactpostcode>1090</contactpostcode>
<contactcountry>Austria</contactcountry>
<contactelectronicmailaddress>office@eox.at</contactelectronicmailaddress>
<contactposition>CTO</contactposition>
<providername>EOX</providername>
<providerurl>https://eox.at</providerurl>
<inspire_profile>true</inspire_profile>
<inspire_metadataurl>TBD</inspire_metadataurl>
<defaultlanguage>eng</defaultlanguage>
<language>eng</language>
</metadata>
<errors>empty_img</errors>
<lock_dir>/tmp</lock_dir>
<threaded_fetching>true</threaded_fetching>
</mapcache>
CREATE OR REPLACE VIEW mapcache_items AS
SELECT
date_trunc('second', product_eoobject."begin_time") AS "begin_time",
CASE
WHEN date_trunc('second', product_eoobject."end_time") = product_eoobject."end_time"
THEN product_eoobject."end_time"
ELSE date_trunc('second', product_eoobject."end_time") + '1 second'::interval
END AS "end_time",
product_eoobject."footprint" AS "footprint",
to_char(product_eoobject."begin_time", 'YYYY-MM-DD"T"HH24:MI:SS"Z"') || '/' || to_char(
CASE
WHEN date_trunc('second', product_eoobject."end_time") = product_eoobject."end_time"
THEN product_eoobject."end_time"
ELSE date_trunc('second', product_eoobject."end_time") + '1 second'::interval
END, 'YYYY-MM-DD"T"HH24:MI:SS"Z"'
) AS "interval",
collection_eoobject.identifier AS "collection"
FROM "coverages_product"
INNER JOIN "coverages_product_collections" ON ("coverages_product"."eoobject_ptr_id" = "coverages_product_collections"."product_id")
INNER JOIN "coverages_collection" collection ON ("coverages_product_collections"."collection_id" = collection."eoobject_ptr_id")
INNER JOIN "coverages_eoobject" ON (collection."eoobject_ptr_id" = "coverages_eoobject"."id")
INNER JOIN "coverages_eoobject" product_eoobject ON ("coverages_product"."eoobject_ptr_id" = product_eoobject."id")
INNER JOIN "coverages_eoobject" collection_eoobject ON (collection."eoobject_ptr_id" = collection_eoobject."id")
;
#!/bin/bash -e
/configure.sh >&2
echo "Running Apache server" >&2
rm -rf /run/apache2/* /var/run/apache2/* /tmp/apache2*
exec /usr/sbin/apache2ctl -D FOREGROUND
#!/bin/bash
/configure.sh
echo "Running seeder"
python3 /seeder.py --mode redis --redis-host ${REDIS_HOST} --redis-port ${REDIS_PORT} --redis-queue-key ${REDIS_SEED_QUEUE_KEY}
#!/usr/bin/env python
#-----------------------------------------------------------------------------
#
# Project: seeder.py
# Authors: Stephan Meissl <stephan.meissl@eox.at>
#
#-----------------------------------------------------------------------------
# Copyright (c) 2018 EOX IT Services GmbH
#
# Python script to pre-seed PVS cache.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to
# deal in the Software without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
# sell copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies of this Software or works derived from this Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
#-----------------------------------------------------------------------------
import os
import sys
import argparse
import textwrap
import logging
import traceback
import signal
import dateutil.parser
import redis
import subprocess
import sqlite3
import boto3
logger = logging.getLogger(__name__)
def setup_logging(verbosity):
# start logging setup
# get command line level
verbosity = verbosity
if verbosity == 0:
level = logging.CRITICAL
elif verbosity == 1:
level = logging.ERROR
elif verbosity == 2:
level = logging.WARNING
elif verbosity == 3:
level = logging.INFO
else:
level = logging.DEBUG
logger.setLevel(level)
sh = logging.StreamHandler()
sh.setLevel(level)
formatter = logging.Formatter('%(asctime)s %(levelname)s: %(message)s')
sh.setFormatter(formatter)
logger.addHandler(sh)
# finished logging setup
CRS_BOUNDS = {
3857: (-20037508.3428, -20037508.3428, 20037508.3428, 20037508.3428),
4326: (-180, -90, 180, 90)
}
GRID_TO_SRID = {
"GoogleMapsCompatible": 3857,
"WGS84": 4326
}
def seed_mapcache(seed_command, config_file, tileset, grid,
minx, miny, maxx, maxy, minzoom, maxzoom,
start_time, end_time, threads, delete, metatile,
timeout=1800, force=True):
bounds = CRS_BOUNDS[GRID_TO_SRID[grid]]
full = float(abs(bounds[0]) + abs(bounds[2]))
dateline_crossed = False
if maxx > bounds[2]:
dateline_crossed = True
# extent is always within [bounds[0],bounds[2]]
# where maxx can be >bounds[2] but <=full
if minx < bounds[0] or minx > bounds[2] or maxx < bounds[0] or maxx > full:
raise Exception("Invalid extent '%s,%s,%s,%s'."
% (minx, miny, maxx, maxy))
if minzoom is None:
minzoom = 0
if maxzoom is None:
maxzoom = 10
# start- and end-time are expected to be UTC Zulu
start_time = start_time.replace(tzinfo=None)
end_time = end_time.replace(tzinfo=None)
logger.info("Starting mapcache seed with parameters: command='%s', "
"config_file='%s', tileset='%s', grid='%s', "
"extent='%f,%f,%f,%f', zoom='%d,%d', nthreads='%s', "
"mode='%s', dimension='TIME=%sZ/%sZ', metatile=%d,%d, "
"force=%s."
% (seed_command, config_file, tileset, grid,
minx, miny, maxx, maxy, minzoom, maxzoom, threads,
"seed" if not delete else "delete",
start_time.isoformat(), end_time.isoformat(),
metatile, metatile, force))
seed_args = [
seed_command,
"-c", config_file,
"-t", tileset,
"-g", grid,
"-e", "%f,%f,%f,%f"
% (minx, miny, bounds[2] if dateline_crossed else maxx, maxy),
"-n", str(threads),
"-z", "%d,%d" % (minzoom, maxzoom),
"-D", "TIME=%sZ/%sZ" % (start_time.isoformat(), end_time.isoformat()),
"-m", "seed" if not delete else "delete",
"-q",
"-M", "%d,%d" % (metatile, metatile),
"-L", "/cache-db/%s/failed_TIME_%sZ_%sZ_extent_%f,%f,%f,%f_zoom-%d-%d"
% (
collection,
start_time.strftime("%Y%m%dT%H%M%S"),
end_time.strftime("%Y%m%dT%H%M%S"),
minx, miny, maxx, maxy, minzoom, maxzoom
),
"-P", "10",
]
if not delete and force:
seed_args.append("-f")
logger.debug("MapCache seeding command: '%s'. raw: '%s'."
% (" ".join(seed_args), seed_args))
process = subprocess.Popen(
seed_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
)
try:
out = process.communicate(timeout=timeout)[0]
except subprocess.TimeoutExpired:
logger.error("Seeding timed out")
process.kill()
out = process.communicate()
if isinstance(out, tuple):
for string in out:
if string is not None:
for line in string.decode("utf-8").split("\n"):
if line != "":
logger.info(
"MapCache output: %s" % line
)
else:
out = out.decode("utf-8").strip()
if out != "":
logger.info(
"MapCache output: %s" % out
)
if process.returncode != 0:
raise Exception("'%s' failed. Returncode '%d'."
% (seed_command, process.returncode))
logger.info("Seeding finished with returncode '%d'." % process.returncode)
return process.returncode
def seeder(collection, start, end, leave_existing=False):
logger.info("Starting seeding from '%s' to '%s'." % (start, end))
if start > end:
logger.critical("Start '%s' is after end '%s'." % (start, end))
sys.exit(1)
# TODO
logger.info(
"Finished seeding from '%s' to '%s'." % (start, end)
)
def seeder_redis_wrapper(
collection, leave_existing, host="localhost", port=6379,
queue_key='ingest_queue'
):
client = redis.Redis(host=host, port=port)
while True:
logger.debug("waiting for redis queue '%s'..." % queue_key)
value = client.brpop(queue_key)
start, end = value[1].split(b"/")
seeder(
collection,
dateutil.parser.parse(start),
dateutil.parser.parse(end)
)
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.description = textwrap.dedent("""\
Pre-seeds cache of PRISM View Server (pvs).
""")
parser.add_argument(
"--mode", default="standard", choices=["standard", "redis"],
help=(
"The mode to run the seeder. Either one-off (standard) or "
"reading from a redis queue."
)
)
parser.add_argument(
"--start", default=None, type=dateutil.parser.parse,
help=(
"Mandatory argument indicating start date and time for "
"the seeding."
)
)
parser.add_argument(
"--end", default=None, type=dateutil.parser.parse,
help=(
"Mandatory argument indicating end date and time for the seeding."
)
)
parser.add_argument(
"--leave_existing", action="store_true",
help=(
"Don't delete existing images from cache."
)
)
parser.add_argument(
"--redis-queue-key", default="seed_queue"
)
parser.add_argument(
"--redis-host", default="localhost"
)
parser.add_argument(
"--redis-port", type=int, default=6379
)
parser.add_argument(
"-v", "--verbosity", type=int, default=3, choices=[0, 1, 2, 3, 4],
help=(
"Set verbosity of log output "
"(4=DEBUG, 3=INFO, 2=WARNING, 1=ERROR, 0=CRITICAL). (default: 3)"
)
)
arg_values = parser.parse_args()
setup_logging(arg_values.verbosity)
collection = os.environ.get('Collection')
if collection is None:
logger.critical("Collection environment variable not set.")
sys.exit(1)
if arg_values.mode == "standard":
seeder(
collection,
arg_values.start,
arg_values.end,
arg_values.leave_existing,