EOX GitLab Instance

Commit 02f66ab2 authored by Nikola Jankovic's avatar Nikola Jankovic 💻
Browse files

added basics for seeder

parent 1f2f6db4
Pipeline #19062 passed with stage
in 3 minutes and 15 seconds
# Temporary and binary files
*~
*.py[cod]
*.so
!.isort.cfg
!setup.cfg
*.orig
*.log
*.pot
__pycache__/*
.cache/*
.*.swp
*/.ipynb_checkpoints/*
.DS_Store
.pylintrc
# Project files
.ropeproject
.project
.pydevproject
.settings
.idea
.vscode
.vscode/
tags
# Package files
*.egg
*.eggs/
.installed.cfg
*.egg-info
# Unittest and coverage
htmlcov/*
.coverage
.coverage.*
.tox
junit*.xml
coverage.xml
.pytest_cache/
# Build and docs folder/files
build/*
dist/*
sdist/*
docs/api/*
docs/_rst/*
docs/_build/*
cover/*
MANIFEST
# Per-project virtualenvs
.venv*/
.conda*/
-i https://__token__:${CI_JOB_TOKEN}@gitlab.eox.at/api/v4/projects/324/packages/pypi/simple
vsq
redis<4.0
boto3<2.0
#!/usr/bin/env python
#-----------------------------------------------------------------------------
# -----------------------------------------------------------------------------
#
# Project: seeder.py
# Authors: Stephan Meissl <stephan.meissl@eox.at>
#
#-----------------------------------------------------------------------------
# -----------------------------------------------------------------------------
# Copyright (c) 2018 EOX IT Services GmbH
#
# Python script to pre-seed VS cache.
......@@ -26,7 +26,7 @@
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
#-----------------------------------------------------------------------------
# -----------------------------------------------------------------------------
import os
......@@ -34,13 +34,10 @@ import sys
import argparse
import textwrap
import logging
import traceback
import signal
import dateutil.parser
import redis
import subprocess
import sqlite3
import redis
import boto3
......@@ -64,7 +61,7 @@ def setup_logging(verbosity):
logger.setLevel(level)
sh = logging.StreamHandler()
sh.setLevel(level)
formatter = logging.Formatter('%(asctime)s %(levelname)s: %(message)s')
formatter = logging.Formatter("%(asctime)s %(levelname)s: %(message)s")
sh.setFormatter(formatter)
logger.addHandler(sh)
# finished logging setup
......@@ -72,19 +69,31 @@ def setup_logging(verbosity):
CRS_BOUNDS = {
3857: (-20037508.3428, -20037508.3428, 20037508.3428, 20037508.3428),
4326: (-180, -90, 180, 90)
4326: (-180, -90, 180, 90),
}
GRID_TO_SRID = {
"GoogleMapsCompatible": 3857,
"WGS84": 4326
}
GRID_TO_SRID = {"GoogleMapsCompatible": 3857, "WGS84": 4326}
def seed_mapcache(seed_command, config_file, tileset, grid,
minx, miny, maxx, maxy, minzoom, maxzoom,
start_time, end_time, threads, delete, metatile,
timeout=1800, force=True):
def seed_mapcache(
seed_command,
config_file,
tileset,
grid,
minx,
miny,
maxx,
maxy,
minzoom,
maxzoom,
start_time,
end_time,
threads,
delete,
metatile,
timeout=1800,
force=True,
):
bounds = CRS_BOUNDS[GRID_TO_SRID[grid]]
full = float(abs(bounds[0]) + abs(bounds[2]))
......@@ -95,8 +104,7 @@ def seed_mapcache(seed_command, config_file, tileset, grid,
# extent is always within [bounds[0],bounds[2]]
# where maxx can be >bounds[2] but <=full
if minx < bounds[0] or minx > bounds[2] or maxx < bounds[0] or maxx > full:
raise Exception("Invalid extent '%s,%s,%s,%s'."
% (minx, miny, maxx, maxy))
raise Exception("Invalid extent '%s,%s,%s,%s'." % (minx, miny, maxx, maxy))
if minzoom is None:
minzoom = 0
......@@ -107,48 +115,82 @@ def seed_mapcache(seed_command, config_file, tileset, grid,
start_time = start_time.replace(tzinfo=None)
end_time = end_time.replace(tzinfo=None)
logger.info("Starting mapcache seed with parameters: command='%s', "
logger.info(
"Starting mapcache seed with parameters: command='%s', "
"config_file='%s', tileset='%s', grid='%s', "
"extent='%f,%f,%f,%f', zoom='%d,%d', nthreads='%s', "
"mode='%s', dimension='TIME=%sZ/%sZ', metatile=%d,%d, "
"force=%s."
% (seed_command, config_file, tileset, grid,
minx, miny, maxx, maxy, minzoom, maxzoom, threads,
% (
seed_command,
config_file,
tileset,
grid,
minx,
miny,
maxx,
maxy,
minzoom,
maxzoom,
threads,
"seed" if not delete else "delete",
start_time.isoformat(), end_time.isoformat(),
metatile, metatile, force))
start_time.isoformat(),
end_time.isoformat(),
metatile,
metatile,
force,
)
)
seed_args = [
seed_command,
"-c", config_file,
"-t", tileset,
"-g", grid,
"-e", "%f,%f,%f,%f"
% (minx, miny, bounds[2] if dateline_crossed else maxx, maxy),
"-n", str(threads),
"-z", "%d,%d" % (minzoom, maxzoom),
"-D", "TIME=%sZ/%sZ" % (start_time.isoformat(), end_time.isoformat()),
"-m", "seed" if not delete else "delete",
"-c",
config_file,
"-t",
tileset,
"-g",
grid,
"-e",
"%f,%f,%f,%f" % (minx, miny, bounds[2] if dateline_crossed else maxx, maxy),
"-n",
str(threads),
"-z",
"%d,%d" % (minzoom, maxzoom),
"-D",
"TIME=%sZ/%sZ" % (start_time.isoformat(), end_time.isoformat()),
"-m",
"seed" if not delete else "delete",
"-q",
"-M", "%d,%d" % (metatile, metatile),
"-L", "/cache-db/%s/failed_TIME_%sZ_%sZ_extent_%f,%f,%f,%f_zoom-%d-%d"
"-M",
"%d,%d" % (metatile, metatile),
"-L",
"/cache-db/%s/failed_TIME_%sZ_%sZ_extent_%f,%f,%f,%f_zoom-%d-%d"
% (
collection,
start_time.strftime("%Y%m%dT%H%M%S"),
end_time.strftime("%Y%m%dT%H%M%S"),
minx, miny, maxx, maxy, minzoom, maxzoom
minx,
miny,
maxx,
maxy,
minzoom,
maxzoom,
),
"-P", "10",
"-P",
"10",
]
if not delete and force:
seed_args.append("-f")
logger.debug("MapCache seeding command: '%s'. raw: '%s'."
% (" ".join(seed_args), seed_args))
logger.debug(
"MapCache seeding command: '%s'. raw: '%s'." % (" ".join(seed_args), seed_args)
)
process = subprocess.Popen(
seed_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
seed_args,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
try:
......@@ -163,19 +205,16 @@ def seed_mapcache(seed_command, config_file, tileset, grid,
if string is not None:
for line in string.decode("utf-8").split("\n"):
if line != "":
logger.info(
"MapCache output: %s" % line
)
logger.info("MapCache output: %s" % line)
else:
out = out.decode("utf-8").strip()
if out != "":
logger.info(
"MapCache output: %s" % out
)
logger.info("MapCache output: %s" % out)
if process.returncode != 0:
raise Exception("'%s' failed. Returncode '%d'."
% (seed_command, process.returncode))
raise Exception(
"'%s' failed. Returncode '%d'." % (seed_command, process.returncode)
)
logger.info("Seeding finished with returncode '%d'." % process.returncode)
......@@ -191,82 +230,75 @@ def seeder(collection, start, end, leave_existing=False):
# TODO
logger.info(
"Finished seeding from '%s' to '%s'." % (start, end)
)
logger.info("Finished seeding from '%s' to '%s'." % (start, end))
def seeder_redis_wrapper(
collection, leave_existing, host="localhost", port=6379,
queue_key='ingest_queue'
collection, leave_existing, host="localhost", port=6379, queue_key="ingest_queue"
):
client = redis.Redis(host=host, port=port)
while True:
logger.debug("waiting for redis queue '%s'..." % queue_key)
value = client.brpop(queue_key)
start, end = value[1].split(b"/")
seeder(
collection,
dateutil.parser.parse(start),
dateutil.parser.parse(end)
)
seeder(collection, dateutil.parser.parse(start), dateutil.parser.parse(end))
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.description = textwrap.dedent("""\
parser.description = textwrap.dedent(
"""\
Pre-seeds cache of View Server (vs).
""")
"""
)
parser.add_argument(
"--mode", default="standard", choices=["standard", "redis"],
"--mode",
default="standard",
choices=["standard", "redis"],
help=(
"The mode to run the seeder. Either one-off (standard) or "
"reading from a redis queue."
)
)
parser.add_argument(
"--start", default=None, type=dateutil.parser.parse,
help=(
"Mandatory argument indicating start date and time for "
"the seeding."
)
)
parser.add_argument(
"--end", default=None, type=dateutil.parser.parse,
help=(
"Mandatory argument indicating end date and time for the seeding."
)
)
parser.add_argument(
"--leave_existing", action="store_true",
help=(
"Don't delete existing images from cache."
)
),
)
parser.add_argument(
"--redis-queue-key", default="seed_queue"
"--start",
default=None,
type=dateutil.parser.parse,
help=("Mandatory argument indicating start date and time for " "the seeding."),
)
parser.add_argument(
"--redis-host", default="localhost"
"--end",
default=None,
type=dateutil.parser.parse,
help=("Mandatory argument indicating end date and time for the seeding."),
)
parser.add_argument(
"--redis-port", type=int, default=6379
"--leave_existing",
action="store_true",
help=("Don't delete existing images from cache."),
)
parser.add_argument("--redis-queue-key", default="seed_queue")
parser.add_argument("--redis-host", default="localhost")
parser.add_argument("--redis-port", type=int, default=6379)
parser.add_argument(
"-v", "--verbosity", type=int, default=3, choices=[0, 1, 2, 3, 4],
"-v",
"--verbosity",
type=int,
default=3,
choices=[0, 1, 2, 3, 4],
help=(
"Set verbosity of log output "
"(4=DEBUG, 3=INFO, 2=WARNING, 1=ERROR, 0=CRITICAL). (default: 3)"
)
),
)
arg_values = parser.parse_args()
setup_logging(arg_values.verbosity)
collection = os.environ.get('Collection')
collection = os.environ.get("Collection")
if collection is None:
logger.critical("Collection environment variable not set.")
sys.exit(1)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment