EOX GitLab Instance
Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Menu
Open sidebar
View Server 2
cache
Commits
02f66ab2
Commit
02f66ab2
authored
Nov 26, 2021
by
Nikola Jankovic
💻
Browse files
added basics for seeder
parent
1f2f6db4
Pipeline
#19062
passed with stage
in 3 minutes and 15 seconds
Changes
3
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
.gitignore
View file @
02f66ab2
# Temporary and binary files
*~
*.py[cod]
*.so
!.isort.cfg
!setup.cfg
*.orig
*.log
*.pot
__pycache__/*
.cache/*
.*.swp
*/.ipynb_checkpoints/*
.DS_Store
.pylintrc
# Project files
.ropeproject
.project
.pydevproject
.settings
.idea
.vscode
.vscode/
tags
# Package files
*.egg
*.eggs/
.installed.cfg
*.egg-info
# Unittest and coverage
htmlcov/*
.coverage
.coverage.*
.tox
junit*.xml
coverage.xml
.pytest_cache/
# Build and docs folder/files
build/*
dist/*
sdist/*
docs/api/*
docs/_rst/*
docs/_build/*
cover/*
MANIFEST
# Per-project virtualenvs
.venv*/
.conda*/
seeder/requirements.txt
View file @
02f66ab2
-i https://__token__:${CI_JOB_TOKEN}@gitlab.eox.at/api/v4/projects/324/packages/pypi/simple
vsq
\ No newline at end of file
vsq
redis<4.0
boto3<2.0
seeder/seeder.py
View file @
02f66ab2
#!/usr/bin/env python
#-----------------------------------------------------------------------------
#
-----------------------------------------------------------------------------
#
# Project: seeder.py
# Authors: Stephan Meissl <stephan.meissl@eox.at>
#
#-----------------------------------------------------------------------------
#
-----------------------------------------------------------------------------
# Copyright (c) 2018 EOX IT Services GmbH
#
# Python script to pre-seed VS cache.
...
...
@@ -26,7 +26,7 @@
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
#-----------------------------------------------------------------------------
#
-----------------------------------------------------------------------------
import
os
...
...
@@ -34,13 +34,10 @@ import sys
import
argparse
import
textwrap
import
logging
import
traceback
import
signal
import
dateutil.parser
import
redis
import
subprocess
import
sqlite3
import
redis
import
boto3
...
...
@@ -64,7 +61,7 @@ def setup_logging(verbosity):
logger
.
setLevel
(
level
)
sh
=
logging
.
StreamHandler
()
sh
.
setLevel
(
level
)
formatter
=
logging
.
Formatter
(
'
%(asctime)s %(levelname)s: %(message)s
'
)
formatter
=
logging
.
Formatter
(
"
%(asctime)s %(levelname)s: %(message)s
"
)
sh
.
setFormatter
(
formatter
)
logger
.
addHandler
(
sh
)
# finished logging setup
...
...
@@ -72,19 +69,31 @@ def setup_logging(verbosity):
CRS_BOUNDS
=
{
3857
:
(
-
20037508.3428
,
-
20037508.3428
,
20037508.3428
,
20037508.3428
),
4326
:
(
-
180
,
-
90
,
180
,
90
)
4326
:
(
-
180
,
-
90
,
180
,
90
)
,
}
GRID_TO_SRID
=
{
"GoogleMapsCompatible"
:
3857
,
"WGS84"
:
4326
}
def
seed_mapcache
(
seed_command
,
config_file
,
tileset
,
grid
,
minx
,
miny
,
maxx
,
maxy
,
minzoom
,
maxzoom
,
start_time
,
end_time
,
threads
,
delete
,
metatile
,
timeout
=
1800
,
force
=
True
):
GRID_TO_SRID
=
{
"GoogleMapsCompatible"
:
3857
,
"WGS84"
:
4326
}
def
seed_mapcache
(
seed_command
,
config_file
,
tileset
,
grid
,
minx
,
miny
,
maxx
,
maxy
,
minzoom
,
maxzoom
,
start_time
,
end_time
,
threads
,
delete
,
metatile
,
timeout
=
1800
,
force
=
True
,
):
bounds
=
CRS_BOUNDS
[
GRID_TO_SRID
[
grid
]]
full
=
float
(
abs
(
bounds
[
0
])
+
abs
(
bounds
[
2
]))
...
...
@@ -95,8 +104,7 @@ def seed_mapcache(seed_command, config_file, tileset, grid,
# extent is always within [bounds[0],bounds[2]]
# where maxx can be >bounds[2] but <=full
if
minx
<
bounds
[
0
]
or
minx
>
bounds
[
2
]
or
maxx
<
bounds
[
0
]
or
maxx
>
full
:
raise
Exception
(
"Invalid extent '%s,%s,%s,%s'."
%
(
minx
,
miny
,
maxx
,
maxy
))
raise
Exception
(
"Invalid extent '%s,%s,%s,%s'."
%
(
minx
,
miny
,
maxx
,
maxy
))
if
minzoom
is
None
:
minzoom
=
0
...
...
@@ -107,48 +115,82 @@ def seed_mapcache(seed_command, config_file, tileset, grid,
start_time
=
start_time
.
replace
(
tzinfo
=
None
)
end_time
=
end_time
.
replace
(
tzinfo
=
None
)
logger
.
info
(
"Starting mapcache seed with parameters: command='%s', "
"config_file='%s', tileset='%s', grid='%s', "
"extent='%f,%f,%f,%f', zoom='%d,%d', nthreads='%s', "
"mode='%s', dimension='TIME=%sZ/%sZ', metatile=%d,%d, "
"force=%s."
%
(
seed_command
,
config_file
,
tileset
,
grid
,
minx
,
miny
,
maxx
,
maxy
,
minzoom
,
maxzoom
,
threads
,
"seed"
if
not
delete
else
"delete"
,
start_time
.
isoformat
(),
end_time
.
isoformat
(),
metatile
,
metatile
,
force
))
logger
.
info
(
"Starting mapcache seed with parameters: command='%s', "
"config_file='%s', tileset='%s', grid='%s', "
"extent='%f,%f,%f,%f', zoom='%d,%d', nthreads='%s', "
"mode='%s', dimension='TIME=%sZ/%sZ', metatile=%d,%d, "
"force=%s."
%
(
seed_command
,
config_file
,
tileset
,
grid
,
minx
,
miny
,
maxx
,
maxy
,
minzoom
,
maxzoom
,
threads
,
"seed"
if
not
delete
else
"delete"
,
start_time
.
isoformat
(),
end_time
.
isoformat
(),
metatile
,
metatile
,
force
,
)
)
seed_args
=
[
seed_command
,
"-c"
,
config_file
,
"-t"
,
tileset
,
"-g"
,
grid
,
"-e"
,
"%f,%f,%f,%f"
%
(
minx
,
miny
,
bounds
[
2
]
if
dateline_crossed
else
maxx
,
maxy
),
"-n"
,
str
(
threads
),
"-z"
,
"%d,%d"
%
(
minzoom
,
maxzoom
),
"-D"
,
"TIME=%sZ/%sZ"
%
(
start_time
.
isoformat
(),
end_time
.
isoformat
()),
"-m"
,
"seed"
if
not
delete
else
"delete"
,
"-c"
,
config_file
,
"-t"
,
tileset
,
"-g"
,
grid
,
"-e"
,
"%f,%f,%f,%f"
%
(
minx
,
miny
,
bounds
[
2
]
if
dateline_crossed
else
maxx
,
maxy
),
"-n"
,
str
(
threads
),
"-z"
,
"%d,%d"
%
(
minzoom
,
maxzoom
),
"-D"
,
"TIME=%sZ/%sZ"
%
(
start_time
.
isoformat
(),
end_time
.
isoformat
()),
"-m"
,
"seed"
if
not
delete
else
"delete"
,
"-q"
,
"-M"
,
"%d,%d"
%
(
metatile
,
metatile
),
"-L"
,
"/cache-db/%s/failed_TIME_%sZ_%sZ_extent_%f,%f,%f,%f_zoom-%d-%d"
"-M"
,
"%d,%d"
%
(
metatile
,
metatile
),
"-L"
,
"/cache-db/%s/failed_TIME_%sZ_%sZ_extent_%f,%f,%f,%f_zoom-%d-%d"
%
(
collection
,
start_time
.
strftime
(
"%Y%m%dT%H%M%S"
),
end_time
.
strftime
(
"%Y%m%dT%H%M%S"
),
minx
,
miny
,
maxx
,
maxy
,
minzoom
,
maxzoom
minx
,
miny
,
maxx
,
maxy
,
minzoom
,
maxzoom
,
),
"-P"
,
"10"
,
"-P"
,
"10"
,
]
if
not
delete
and
force
:
seed_args
.
append
(
"-f"
)
logger
.
debug
(
"MapCache seeding command: '%s'. raw: '%s'."
%
(
" "
.
join
(
seed_args
),
seed_args
))
logger
.
debug
(
"MapCache seeding command: '%s'. raw: '%s'."
%
(
" "
.
join
(
seed_args
),
seed_args
)
)
process
=
subprocess
.
Popen
(
seed_args
,
stdout
=
subprocess
.
PIPE
,
stderr
=
subprocess
.
STDOUT
,
seed_args
,
stdout
=
subprocess
.
PIPE
,
stderr
=
subprocess
.
STDOUT
,
)
try
:
...
...
@@ -163,19 +205,16 @@ def seed_mapcache(seed_command, config_file, tileset, grid,
if
string
is
not
None
:
for
line
in
string
.
decode
(
"utf-8"
).
split
(
"
\n
"
):
if
line
!=
""
:
logger
.
info
(
"MapCache output: %s"
%
line
)
logger
.
info
(
"MapCache output: %s"
%
line
)
else
:
out
=
out
.
decode
(
"utf-8"
).
strip
()
if
out
!=
""
:
logger
.
info
(
"MapCache output: %s"
%
out
)
logger
.
info
(
"MapCache output: %s"
%
out
)
if
process
.
returncode
!=
0
:
raise
Exception
(
"'%s' failed. Returncode '%d'."
%
(
seed_command
,
process
.
returncode
))
raise
Exception
(
"'%s' failed. Returncode '%d'."
%
(
seed_command
,
process
.
returncode
)
)
logger
.
info
(
"Seeding finished with returncode '%d'."
%
process
.
returncode
)
...
...
@@ -191,82 +230,75 @@ def seeder(collection, start, end, leave_existing=False):
# TODO
logger
.
info
(
"Finished seeding from '%s' to '%s'."
%
(
start
,
end
)
)
logger
.
info
(
"Finished seeding from '%s' to '%s'."
%
(
start
,
end
))
def
seeder_redis_wrapper
(
collection
,
leave_existing
,
host
=
"localhost"
,
port
=
6379
,
queue_key
=
'ingest_queue'
collection
,
leave_existing
,
host
=
"localhost"
,
port
=
6379
,
queue_key
=
"ingest_queue"
):
client
=
redis
.
Redis
(
host
=
host
,
port
=
port
)
while
True
:
logger
.
debug
(
"waiting for redis queue '%s'..."
%
queue_key
)
value
=
client
.
brpop
(
queue_key
)
start
,
end
=
value
[
1
].
split
(
b
"/"
)
seeder
(
collection
,
dateutil
.
parser
.
parse
(
start
),
dateutil
.
parser
.
parse
(
end
)
)
seeder
(
collection
,
dateutil
.
parser
.
parse
(
start
),
dateutil
.
parser
.
parse
(
end
))
if
__name__
==
"__main__"
:
parser
=
argparse
.
ArgumentParser
()
parser
.
description
=
textwrap
.
dedent
(
"""
\
parser
.
description
=
textwrap
.
dedent
(
"""
\
Pre-seeds cache of View Server (vs).
"""
)
"""
)
parser
.
add_argument
(
"--mode"
,
default
=
"standard"
,
choices
=
[
"standard"
,
"redis"
],
"--mode"
,
default
=
"standard"
,
choices
=
[
"standard"
,
"redis"
],
help
=
(
"The mode to run the seeder. Either one-off (standard) or "
"reading from a redis queue."
)
)
parser
.
add_argument
(
"--start"
,
default
=
None
,
type
=
dateutil
.
parser
.
parse
,
help
=
(
"Mandatory argument indicating start date and time for "
"the seeding."
)
)
parser
.
add_argument
(
"--end"
,
default
=
None
,
type
=
dateutil
.
parser
.
parse
,
help
=
(
"Mandatory argument indicating end date and time for the seeding."
)
)
parser
.
add_argument
(
"--leave_existing"
,
action
=
"store_true"
,
help
=
(
"Don't delete existing images from cache."
)
),
)
parser
.
add_argument
(
"--redis-queue-key"
,
default
=
"seed_queue"
"--start"
,
default
=
None
,
type
=
dateutil
.
parser
.
parse
,
help
=
(
"Mandatory argument indicating start date and time for "
"the seeding."
),
)
parser
.
add_argument
(
"--redis-host"
,
default
=
"localhost"
"--end"
,
default
=
None
,
type
=
dateutil
.
parser
.
parse
,
help
=
(
"Mandatory argument indicating end date and time for the seeding."
),
)
parser
.
add_argument
(
"--redis-port"
,
type
=
int
,
default
=
6379
"--leave_existing"
,
action
=
"store_true"
,
help
=
(
"Don't delete existing images from cache."
),
)
parser
.
add_argument
(
"--redis-queue-key"
,
default
=
"seed_queue"
)
parser
.
add_argument
(
"--redis-host"
,
default
=
"localhost"
)
parser
.
add_argument
(
"--redis-port"
,
type
=
int
,
default
=
6379
)
parser
.
add_argument
(
"-v"
,
"--verbosity"
,
type
=
int
,
default
=
3
,
choices
=
[
0
,
1
,
2
,
3
,
4
],
"-v"
,
"--verbosity"
,
type
=
int
,
default
=
3
,
choices
=
[
0
,
1
,
2
,
3
,
4
],
help
=
(
"Set verbosity of log output "
"(4=DEBUG, 3=INFO, 2=WARNING, 1=ERROR, 0=CRITICAL). (default: 3)"
)
)
,
)
arg_values
=
parser
.
parse_args
()
setup_logging
(
arg_values
.
verbosity
)
collection
=
os
.
environ
.
get
(
'
Collection
'
)
collection
=
os
.
environ
.
get
(
"
Collection
"
)
if
collection
is
None
:
logger
.
critical
(
"Collection environment variable not set."
)
sys
.
exit
(
1
)
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment