diff --git a/.gitignore b/.gitignore index 3af0ccb687b45753a5c8b3c917d0ebe7cc4ab466..907583ae264c330b973f58ecc2bccf5d98658923 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ /data +__pycache__ +*.pyc diff --git a/README.md b/README.md index eeaa2a18135694c492cc1f699948851ebf596549..dac0b81791e16e725c2b2215cb94d3c809cb0f3f 100644 --- a/README.md +++ b/README.md @@ -150,7 +150,7 @@ see new service in #7 * provides external service for OpenSearch, WMS, & WCS * renders WMS requests received from cache or seeder -## TODO: ELK stack +### TODO: ELK stack see #9 @@ -201,6 +201,40 @@ docker volume rm vhr18-pvs_cache-db docker volume rm vhr18-pvs_instance-data ``` +Generate mapcache.sqlite + +```bash +docker exec -it $(docker ps -qf "name=vhr18-pvs_renderer") python3 /var/www/pvs/dev/pvs_instance/manage.py mapcache sync -f +docker exec -it $(docker ps -qf "name=vhr18-pvs_renderer") mv VHR_IMAGE_2018.sqlite /cache-db/vhr18_mapcache_cache.sqlite + +docker exec -it $(docker ps -qf "name=emg-pvs_renderer") python3 /var/www/pvs/dev/pvs_instance/manage.py mapcache sync -f +docker exec -it $(docker ps -qf "name=emg-pvs_renderer") mv Emergency.sqlite /cache-db/emg_mapcache_cache.sqlite +``` + +# Documentation + +## Installation + +```bash +python3 -m pip install sphinx recommonmark sphinx-autobuild +``` + +## Generation + +```bash +make html + +# For watched html automatic building +make html-watch + +# For pdf output +make latexpdf +# To shrink size of pdf +gs -sDEVICE=pdfwrite -dPDFSETTINGS=/ebook -dPrinted=false -q -o View-Server_-_User-Guide_small.pdf View-Server_-_User-Guide.pdf +``` + +The documentation is generated in the respective *_build/html* directory. + # Create software releases ## Source code release diff --git a/documentation/.gitignore b/documentation/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..136ea96bb3e3375c9d6bbd02997a2a13dbd3d5b5 --- /dev/null +++ b/documentation/.gitignore @@ -0,0 +1 @@ +/*/_build/* diff --git a/documentation/operator-guide/Makefile b/documentation/operator-guide/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..115276079038a54fd7b649c3dbf384333981ec59 --- /dev/null +++ b/documentation/operator-guide/Makefile @@ -0,0 +1,24 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SPHINXAUTOBUILD ?= sphinx-autobuild +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +html-watch: + @$(SPHINXAUTOBUILD) -i "$(BUILDDIR)/*" "$(SOURCEDIR)" "$(BUILDDIR)/html" $(SPHINXOPTS) $(O) + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/documentation/operator-guide/conf.py b/documentation/operator-guide/conf.py new file mode 100644 index 0000000000000000000000000000000000000000..bd0fda7971468c754f8672eed9da12a702a31439 --- /dev/null +++ b/documentation/operator-guide/conf.py @@ -0,0 +1,74 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + + +# -- Project information ----------------------------------------------------- + +project = u'View Server - Operator Guide' +copyright = u'2019, 2020, EOX IT Services GmbH' +author = u'EOX IT Services GmbH' + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'alabaster' + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +html_favicon = 'images/favicon.ico' + +numfig = True +numfig_format = { + 'figure': 'Figure %s', + 'table': 'Table %s', + 'code-block': 'Listing %s', + 'section': 'Section', +} +numfig_secnum_depth = 2 + +latex_documents = [( + 'index', + 'View-Server_-_Operator-Guide.tex', + u'View Server - Operator Guide', + u'EOX IT Services GmbH', + 'manual', + True +)] +latex_elements = { + 'papersize': 'a4paper', +} diff --git a/documentation/operator-guide/configuration.rst b/documentation/operator-guide/configuration.rst new file mode 100644 index 0000000000000000000000000000000000000000..327832e46be3b71202293e4df20d59354518231b --- /dev/null +++ b/documentation/operator-guide/configuration.rst @@ -0,0 +1,257 @@ +.. _configuration: + +Configuration +============= + +This chapter details how a running VS stack can be configured. And what steps +are necessary to deploy the configuration. + +In order for these configuration changes to be picked up by a running VS +stack and to take effect some steps need to be performed. These steps are +either a "re-deploy" of the running stack or a complete re-creation of it. + +Stack Re-deploy +--------------- + +As will be further described, for some configurations it is sufficient to +"re-deploy" the stack which automatically re-starts any service with changed +configuration. This is done re-using the stack deployment command: + +.. code-block:: bash + + docker stack deploy -c docker-compose..yml -c docker-compose..dev.yml + +.. warning:: + + When calling the ``docker stack deploy`` command, it is vital to use the + command with the same files and name the stack was originally created. + +Stack Re-creation +----------------- + +In some cases a stack re-redeploy is not enough, as the configuration was used +for a materialized instance which needs to be reverted. The easiest way to do +this is to delete the volume in question. If, for example, the +renderer/registrar configuration was updated, the ``instance-data`` volume +needs to be re-created. + +First, the stack needs to be shut down. This is done using the following +command: + +.. code-block:: bash + + docker stack rm + + +When that command has completed (it is advisable to wait for some time until +all containers have actually stopped) the next step is to delete the +``instance-data`` volume: + +.. code-block:: bash + + docker volume rm _instance-data + +.. note:: + + It is possible that this command fails, with the error message that the + volume is still in use. In this case, it is advisable to wait for a minute + and to try the deletion again. + +Now that the volume was deleted, the stack can be re-deployed as described +above, which will trigger the automatic re-creation and initialization of the +volume. For the ``instance-data``, it means that the instance will be +re-created and all database models with it. + + +Docker Compose Settings +----------------------- + +These configurations are altering the behavior of the stack itself and its +contained services. A complete reference of the configuration file structure +can be found in the `Docker Compose documentation +`_. + +Environment Variables +--------------------- + +These variables are passed to their respective containers environment and +change the behavior of certain functionality. They can be declared in the +Docker Compose configuration file directly, but typically they are bundled by +field of interest and then placed into ``.env`` files and then passed to the +containers. So for example, there will be a ``_obs.env`` file +to store the access parameters for the object storage. +All those files are placed in the ``env/`` directory in the instances +directory. + +Environment variables and ``.env`` files are passed to the services via the +``docker-compose.yml`` directives. The following example shows how to pass +``.env`` files and direct environment variables: + +.. code-block:: yaml + + services: + # .... + registrar: + env_file: + - env/stack.env + - env/stack_db.env + - env/stack_obs.env + - env/stack_redis.env + environment: + INSTANCE_ID: "prism-view-server_registrar" + INSTALL_DIR: "/var/www/pvs/dev/" + SCALEFACTOR: "1" + IN_MEMORY: "false" + INIT_SCRIPTS: "/configure.sh /init-db.sh" + # ... + +``.env`` Files +~~~~~~~~~~~~~~ + +The following ``.env`` files are typically used: + +* ``.env``: The general ``.env`` file used for all services +* ``_db.env``: The database access credentials, for all services + interacting with the database. +* ``_django.env``: This env files defines the credentials for the + django admin user to be used with the admin GUI. +* ``_obs.env``: This contains access parameters for the object + storage(s). +* ``_preprocessor.env``: Preprocessor related environment + variables +* ``_redis.env``: Redis access credentials and queue names + + +Groups of Environment Variables +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +GDAL Environment Variables +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This group of environment variables controls the intricacies of GDAL. They +control how GDAL interacts with its supported files. As GDAL supports a +variety of formats and backend access, most of the full `list of env +variables `_ are not applicable and +only a handful are actually relevant for the VS. + +* ``GDAL_DISABLE_READDIR_ON_OPEN`` - + Especially when using an Object Storage backend with a very large number of + files, it is vital to activate this setting (``=TRUE``) in order to + suppress to read the whole directory contents which is very slow for some + OBS backends. +* ``CPL_VSIL_CURL_ALLOWED_EXTENSIONS`` - + This limits the file extensions to disable the lookup of so called sidecar + files which are not used for VS. By default this value is used: + ``=.TIF,.tif,.xml``. + +OpenStack Swift Environment Variables +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +These variables define the access coordinates and credentials for the +OpenStack Swift Object storage backend. + +This set of variables define the credentials for the object storage to +place the preprocessed results: + +* ``ST_AUTH_VERSION`` +* ``OS_AUTH_URL_SHORT`` +* ``OS_AUTH_URL`` +* ``OS_USERNAME`` +* ``OS_PASSWORD`` +* ``OS_TENANT_NAME`` +* ``OS_TENANT_ID`` +* ``OS_REGION_NAME`` +* ``OS_USER_DOMAIN_NAME`` + +This set of variables define the credentials for the object storage to +retrieve the original product files: + +* ``OS_USERNAME_DOWNLOAD`` +* ``OS_PASSWORD_DOWNLOAD`` +* ``OS_TENANT_NAME_DOWNLOAD`` +* ``OS_TENANT_ID_DOWNLOAD`` +* ``OS_REGION_NAME_DOWNLOAD`` +* ``OS_AUTH_URL_DOWNLOAD`` +* ``ST_AUTH_VERSION_DOWNLOAD`` + +VS Environment Variables +^^^^^^^^^^^^^^^^^^^^^^^^ + +These environment variables are used by the VS itself to configure various +parts. + +.. note:: + These variables are used during the initial stack setup. When these + variables are changed, they will not be reflected unless the instance + volume is re-created. + +* ``COLLECTION`` - + This defines the main collections name. This is used in various parts of + the VS and serves as the layer base name. +* ``UPLOAD_CONTAINER`` - + This controls the bucket name where the preprocessed images are uploaded + to. +* ``DJANGO_USER``, ``DJANGO_MAIL``, ``DJANGO_PASSWORD`` - + The Django admin user account credentials to use the Admin GUI. + +.. note:: + These variables are used during the initial stack setup. When these + variables are changed, they will not be reflected unless the database + volume is re-created. + +These are the internal access credentials for the database: + +* ``POSTGRES_USER`` +* ``POSTGRES_PASSWORD`` +* ``POSTGRES_DB`` +* ``DB`` +* ``DB_USER`` +* ``DB_PW`` +* ``DB_HOST`` +* ``DB_PORT`` +* ``DB_NAME`` + +Configuration Files +------------------- + +Such files are passed to the containers in a similar way as environment +variables, but usually contain more settings at once and are placed at a +specific path in the container at runtime. + +Configuration files are passed into the containers using the ``configs`` +section of the ``docker-compose.yaml`` file. The following example shows how +such a configuration file is defined and the used in a service: + + +.. code-block:: yaml + + # ... + configs: + my-config: + file: ./config/example.cfg + # ... + services: + myservice: + # ... + configs: + - source: my-config + target: /example.cfg + + +The following configuration files are used throughout the VS: + +* ``_init-db.sh``: This shell script file's purpose is to set up + the EOxServer instance used by both the renderer and registrar. +* ``_index-dev.html``/``_index-ops.html``: The + clients main HTML page, containing various client settings. The ``dev`` one + is used for development only, whereas the ``ops`` one is used for operational + deployment. +* ``_mapcache-dev.xml``/``_mapcache-ops.xml``: The + configuration file for MapCache, the software powering the cache service. + Similarly to the client configuration files, the ``dev`` and ``ops`` files + used for development and operational usage respectively. Further + documentation can be found at `the official site + `_. + +The next section :ref:`management` describes how an operator interacts with a +deployed VS stack. diff --git a/documentation/operator-guide/images/architecture.png b/documentation/operator-guide/images/architecture.png new file mode 100644 index 0000000000000000000000000000000000000000..5c4ac8b16a10be403674baa0201dff3c6e96481a Binary files /dev/null and b/documentation/operator-guide/images/architecture.png differ diff --git a/documentation/operator-guide/images/favicon.ico b/documentation/operator-guide/images/favicon.ico new file mode 100644 index 0000000000000000000000000000000000000000..b4d1efa50d197e85eb86c8d60c53db49393b3254 Binary files /dev/null and b/documentation/operator-guide/images/favicon.ico differ diff --git a/documentation/operator-guide/index.rst b/documentation/operator-guide/index.rst new file mode 100644 index 0000000000000000000000000000000000000000..249dfe69a680b8d7b323f4a674e9d65de7a1ec11 --- /dev/null +++ b/documentation/operator-guide/index.rst @@ -0,0 +1,30 @@ +.. _contents: + +View Server - Operator Guide +============================ + +.. toctree:: + :maxdepth: 3 + + intro + initialization + setup + configuration + management + ingestion + +.. + Indices and tables + ================== + + .. only:: builder_html + + * :ref:`genindex` + * :ref:`modindex` + * :ref:`search` + .. * :ref:`glossary` + + .. only:: not builder_html + + * :ref:`modindex` + .. * :ref:`glossary` diff --git a/documentation/operator-guide/ingestion.rst b/documentation/operator-guide/ingestion.rst new file mode 100644 index 0000000000000000000000000000000000000000..dc1b31aaab837ab4514ec8949c3047d389a7cb34 --- /dev/null +++ b/documentation/operator-guide/ingestion.rst @@ -0,0 +1,275 @@ +.. _ingestion: + +Data Ingestion +============== + +This section details the data ingestion and later management in the VS. + +Redis Queues +------------ + +The central synchronization component in the VS is the ``redis`` key-value +store. It provides various queues, which the services are listening to. For +operators it provides a high-level interface through which data products can be +registered and managed. + +Via the Redis, the ingestion can be triggered and observed. In order to +eventually start the preprocessing of a product, its path on the configured +object storage has to be pushed onto the ``preprocess_queue``, as will be +explained in detail in this chapter. + +As the Redis store is not publicly accessible from outside of the stack. So to +interact with it, the operator has to run a command from one of the services. +Conveniently, the service running Redis also has the ``redis-cli`` tool +installed that lets users interact with the store. + +When doing one off commands, it is maybe more convenient to execute it on a +running service. For this, the ``docker ps`` command can be used to select the +identifier of the running docker container of the redis service. + +.. code-block:: bash + + container_id=$(docker ps -qf "name=_redis") + +With this identifier, a command can be issued: + +.. code-block:: bash + + docker exec -it $container_id redis-cli ... + +When performing more than one command, it can be simpler to open a shell on the +service instead: + +.. code-block:: bash + + docker exec -it $container_id bash + +As the container ID may change (for example when the replica is restarted) it +is better to retrieve it for every command instead of relying on a variable: + +.. code-block:: bash + + docker exec -it $(docker ps -qf "name=_redis") + +For the sake of brevity, the subsequent commands in this chapter can be used +with either of the above techniques and will just print the final commands that +are run inside the redis container. + +.. note:: + + For the VS, only the ``List`` and ``Set`` `Redis data types + `_ are really used. + + ``Sets`` are an unordered collection of string elements. In the VS it is + used to denote that an element is part of a particular group, e.g: being + preprocessed, or having failed registration. + + ``Lists`` are used as a task queue. It is possible to add items to either + end of the queue, but by convention items are pushed on the "left" and + popped from the "right" end of the list resulting in a first-in-first-out + (FIFO) queue. It is entirely possible to push elements to the "right" end + as-well, and an operator may want to do so in order to add an element to be + processed as soon as possible instead of waiting before all other elements + before it are processed. + + The full list of available commands can be found for both `Lists + `_ and `Sets + `_. + +For a more concrete example the following command executes a +``redis-cli lpush`` command to add a new path of an object to preprocess on the +``preprocess_queue``: + +.. code-block:: bash + + redis-cli lpush preprocess_queue "/data25/OA/PL00/1.0/00/urn:eop:DOVE:MULTISPECTRAL_4m:20180811_081455_1054_3be7/0001/PL00_DOV_MS_L3A_20180811T081455_20180811T081455_TOU_1234_3be7.DIMA.tar" + +Usually, with a preprocessor service running and no other items in the +``preprocess_queue`` this value will be immediately popped from the list and +processed. For the sake of demonstration this command would print the contents +of the ``preprocess_queue``: + +.. code-block:: bash + + $ redis-cli lrange preprocess_queue 0 -1 + /data25/OA/PL00/1.0/00/urn:eop:DOVE:MULTISPECTRAL_4m:20180811_081455_1054_3be7/0001/PL00_DOV_MS_L3A_20180811T081455_20180811T081455_TOU_1234_3be7.DIMA.tar + +Now that the product is being preprocessed, it should be visible in the +``preprocessing_set``. As the name indicates, this is using the ``Set`` +datatype, thus requiring the ``SMEMBERS`` subcommand to list: + +.. code-block:: bash + + $ redis-cli smembers preprocessing_set 0 -1 + /data25/OA/PL00/1.0/00/urn:eop:DOVE:MULTISPECTRAL_4m:20180811_081455_1054_3be7/0001/PL00_DOV_MS_L3A_20180811T081455_20180811T081455_TOU_1234_3be7.DIMA.tar + +Once the preprocessing of the product is finished, the preprocessor will remove +the currently worked on path from the ``preprocessing_set`` and add it either +to the ``preprocess-success_set`` or the ``preprocess-failure_set`` depending +on whether the processing succeeded or not. They can be inspected using the +same ``SMEMBERS`` subcommand with one of set names as a parameter. + +Additionally, upon success, the preprocessor places the same product path on +the ``register_queue``, where it can be inspected with the following command. + +.. code-block:: bash + + $ redis-cli lrange register_queue 0 -1 + /data25/OA/PL00/1.0/00/urn:eop:DOVE:MULTISPECTRAL_4m:20180811_081455_1054_3be7/0001/PL00_DOV_MS_L3A_20180811T081455_20180811T081455_TOU_1234_3be7.DIMA.tar + +If an operator wants to trigger only the re-registration of a product without +preprocessing the product path needs to be pushed to this queue: + +.. code-block:: bash + + redis-cli lpush register_queue "/data25/OA/PL00/1.0/00/urn:eop:DOVE:MULTISPECTRAL_4m:20180811_081455_1054_3be7/0001/PL00_DOV_MS_L3A_20180811T081455_20180811T081455_TOU_1234_3be7.DIMA.tar" + +Very similar to the preprocessing, during the registration the product path is +added to the ``registering_set``, afterwards the path is placed to either the +``register-success_set`` or ``register-failure_set``. Again, these queues or +sets can be inspected by the ``LRANGE`` or ``SMEMBERS`` subcommands +respectively. + +Direct Data Management +---------------------- + +Sometimes it is necessary to directly interact with the preprocessor or +registrar. The following section shows what tasks on the preprocessor and +registrar can be accomplished. + +.. warning:: + + This approach is not recommended for everyday use, as it circumvents the + Redis sets to track what products have been registered and where the + registration failed. + +Preprocessing +~~~~~~~~~~~~~ + +In this section all command examples are assumed to be run from within a running +preprocessor container. To open a shell on a preprocessor, the following +command can be used. + +.. code-block:: bash + + docker exec -it $(docker ps -qf "name=_preprocessor") bash + +The preprocessor can be used in two modes. The first (and default mode when +used as a service) is to be run as a daemon: it listens to the Redis queue for +new items, which will be preprocessed one by one. The second mode is to run the +preprocessor in a "one-off" mode: instead of pulling an item from the queue, +it is passed as a command line argument, which is then processed normally. + +.. code-block:: bash + + python3 /preprocessor.py \ + --mode standard \ + --replace \ + --tar-object-path /data25/OA/PL00/1.0/00/urn:eop:DOVE:MULTISPECTRAL_4m:20180811_081455_1054_3be7/0001/PL00_DOV_MS_L3A_20180811T081455_20180811T081455_TOU_1234_3be7.DIMA.tar + +In this mode, the item will not be placed in the resulting set +(``preprocessing_set``, ``preprocess-success_set``, and +``preprocess-failure_set``). + + +Registration Handling +~~~~~~~~~~~~~~~~~~~~~ + +For all intents and purposes in this section it is assumed, that the operator +is logged into a shell on the ``registrar`` service. This can be achieved via +the following command (assuming at least one registrar replica is running): + +.. code-block:: bash + + docker exec -it $(docker ps -qf "name=_registrar") bash + +The contents of the shared registrar/renderer database can be managed using +the registrars instance ``manage.py`` script. For brevity, the following bash +alias is assumed: + +.. code-block:: bash + + alias manage.py='python3 /var/www/pvs/dev/pvs_instance/manage.py' + + +A collection is a grouping of earth observation products, accessible as a +single entity via various service endpoints. Depending on the configuration, +multiple collections are created when the service is set up. They can be listed +using the ``collection list`` command. + +New collections can be created using the ``collection create`` command. This +can refer to a ``Collection Type``, which will restrict the collection in terms +of insertable products: only products of an allowed ``Product Type`` can be +added. Detailed information about the available Collection management commands +can be found in the `CLI documentation `__. + +Collections can be deleted, without affecting the contained products. + +.. warning:: + + As some other services have fixed configurations and depend on specific + collections, deleting said collections without a replacement can lead to + configuration inconsistencies and ultimately service disruptions. + +In certain scenarios it may be useful to add specific products to or exclude +them from a collection. For this, the Product identifier needs to be known. To +find out the Product identifier, either a query of the existing +collection via OpenSearch or the CLI command ``id list`` can be used. + +When the identifier is obtained, the following management command inserts a +product into a collection: + +.. code-block:: bash + + manage.py collection insert + +Multiple products can be inserted in one pass by providing more than one +identifier. + +The reverse command excludes a product from a collection: + +.. code-block:: bash + + manage.py collection exclude + +Again, multiple products can be excluded in a single call. + + +Product Handling +~~~~~~~~~~~~~~~~ + +Registration + Products can be registered using the EOxServer CLI tools as well. + + .. code-block:: bash + + manage.py product register \ + --metadata-file data25 /OA/PL00/1.0/00/urn:eop:DOVE:MULTISPECTRAL_4m:20180811_081455_1054_3be7/0001/PL00_DOV_MS_L3A_20180811T081455_20180811T081455_TOU_1234_3be7.DIMA.tar/metadata.xml \ + --print-identifier \ + --type PL00 + + The identifier of the newly registered product is printed to the console and + can be used to put it into a collection. Additionally, it is necessary to add + a coverage to it, which can be registered like: + + .. code-block:: bash + + manage.py coverage register \ + -d data25 /OA/PL00/1.0/00/urn:eop:DOVE:MULTISPECTRAL_4m:20180811_081455_1054_3be7/0001/PL00_DOV_MS_L3A_20180811T081455_20180811T081455_TOU_1234_3be7.DIMA.tar/some.tif \ + -m data25 /OA/PL00/1.0/00/urn:eop:DOVE:MULTISPECTRAL_4m:20180811_081455_1054_3be7/0001/PL00_DOV_MS_L3A_20180811T081455_20180811T081455_TOU_1234_3be7.DIMA.tar/metadata.xml \ + --identifier "${product_id}_coverage" \ + --type RGBNir + +Deregistration + Products and coverages need to be derigestered when no longer in use. A + product can be deregistered using its identifier: + + .. code-block:: bash + + manage.py product deregister "${product_id}" + + The contained coverage must also be deregistered manually: + + .. code-block:: bash + + manage.py coverage deregister "${product_id}_coverage" diff --git a/documentation/operator-guide/initialization.rst b/documentation/operator-guide/initialization.rst new file mode 100644 index 0000000000000000000000000000000000000000..437592a29dc2b588713d68c1dda09217de0dca33 --- /dev/null +++ b/documentation/operator-guide/initialization.rst @@ -0,0 +1,277 @@ +.. _initialization: + +Initialization +============== + +In order to set up an instance of the View Server (VS), the separate +``pvs_starter`` utility is recommended. + +Running the Initialization +-------------------------- + +The ``pvs_starter`` utility is distributed as a Python package and easily +installed via ``pip``. + +.. code-block:: bash + + pip3 install pvs_starter git+git@gitlab.eox.at:esa/prism/pvs_starter.git + +Now a new VS instance can be set up like this: + +.. code-block:: bash + + python3 -m pvs_starter.cli config.yaml out/ -f + +This takes the initialization configuration ``config.yaml`` to generate +the required structure of a new VS instance in the ``out/`` directory. + +Configuration of the Initialization +----------------------------------- + +The important part of the initialization is the configuration. The file is +structured in YAML as detailed below. It contains the following +sections: + +``database`` +~~~~~~~~~~~~ + +Here, access details and credentials of the database are stored. It defines the +internal database name, user, and password that will be created when the stack +is deployed. Note that there is no ``host`` setting, as this will be handled +automatically within the Docker Swarm. + +.. code-block:: yaml + + database: + name: vs_db + user: vs_user + password: Go-J_eOUvj2k + +``django_admin`` +~~~~~~~~~~~~~~~~ + +This section deals with the setup of a Django admin account. This is used to +later access the admin panel to inspect the registered data. + +.. code-block:: yaml + + django_admin: + user: admin + mail: office@eox.at + password: jvLwv_20x-69 + +``preprocessor`` +~~~~~~~~~~~~~~~~ + +Here, the preprocessing can be configured in detail. + +.. TODO + + +``products`` +~~~~~~~~~~~~ + +This section defines ``product_type`` related information. The two most +important settings here are the ``type_extractor`` and ``level_extractor`` +structures which specify how the product type and product level should be +extracted from the metadata. For this, an XPath (or multiple) can be specified +to retrieve that information. + +The ``types`` section defines the available ``product_types`` and which +``browse`` and ``mask`` types are to be generated. + +.. code-block:: yaml + + products: + type_extractor: + xpath: + namespace_map: + level_extractor: + xpath: + namespace_map: + types: + PL00: + default_browse: TRUE_COLOR + browses: + TRUE_COLOR: + red: + expression: red + range: [1000, 15000] + nodata: 0 + green: + expression: green + range: [1000, 15000] + nodata: 0 + blue: + expression: blue + range: [1000, 15000] + nodata: 0 + FALSE_COLOR: + red: + expression: nir + range: [1000, 15000] + nodata: 0 + green: + expression: red + range: [1000, 15000] + nodata: 0 + blue: + expression: green + range: [1000, 15000] + nodata: 0 + NDVI: + grey: + expression: (nir-red)/(nir+red) + range: [-1, 1] + masks: + validity: + validity: true + + + +``collections`` +~~~~~~~~~~~~~~~ + +In the ``collections`` section, the collections are set up and it is defined +which products based on ``product_type`` and ``product_level`` will be inserted +into them. The ``product_types`` must list types defined in the ``products`` +section. + +.. code-block:: yaml + + collections: + COLLECTION: + product_types: + - PL00 + product_levels: + - Level_1 + - Level_3 + +``storages`` +~~~~~~~~~~~~ + +Here, the three relevant storages can be configured: the ``source``, +``preprocessed``, and ``cache`` storages. + +The ``source`` storage defines the location from which the original files will +be downloaded to be preprocessed. Preprocessed images and metadata will then be +uploaded to the ``preprocessed`` storage. The cache service will cache images on +the ``cache`` storage. + +Each storage definition uses the same structure and can target various types +of storages, such as OpenStack swift. + +These storage definitions will be used in the appropriate sections. + +.. code-block:: yaml + + storages: + source: + auth_type: keystone + auth_url: + version: 3 + username: + password: + tenant_name: + tenant_id: + region_name: + container: + preprocessed: + auth_type: keystone + auth_url: + version: 3 + username: + password: + tenant_name: + tenant_id: + region_name: + container: + cache: + type: swift + auth_type: keystone + auth_url: https://auth.cloud.ovh.net/v3/ + auth_url_short: https://auth.cloud.ovh.net/ + version: 3 + username: + password: + tenant_name: + tenant_id: + region_name: + container: + + +``cache`` +~~~~~~~~~ + +This section defines the exposed services, and how the layers shall be cached +internally. + +.. code-block:: yaml + + cache: + metadata: + title: PRISM Data Access Service (PASS) developed by EOX + abstract: PRISM Data Access Service (PASS) developed by EOX + url: https://vhr18.pvs.prism.eox.at/cache/ows + keyword: view service + accessconstraints: UNKNOWN + fees: UNKNOWN + contactname: Stephan Meissl + contactphone: Please contact via mail. + contactfacsimile: None + contactorganization: EOX IT Services GmbH + contactcity: Vienna + contactstateorprovince: Vienna + contactpostcode: 1090 + contactcountry: Austria + contactelectronicmailaddress: office@eox.at + contactposition: CTO + providername: EOX + providerurl: https://eox.at + inspire_profile: true + inspire_metadataurl: TBD + defaultlanguage: eng + language: eng + services: + wms: + enabled: true + wmts: + enabled: true + connection_timeout: 10 + timeout: 120 + expires: 3600 + key: /{tileset}/{grid}/{dim}/{z}/{x}/{y}.{ext} + tilesets: + VHR_IMAGE_2018__TRUE_COLOR: + title: VHR Image 2018 True Color + abstract: VHR Image 2018 True Color + VHR_IMAGE_2018__FALSE_COLOR: + title: VHR Image 2018 False Color + abstract: VHR Image 2018 False Color + VHR_IMAGE_2018__NDVI: + title: VHR Image 2018 NDVI + abstract: VHR Image 2018 NDVI + style: earth + VHR_IMAGE_2018_Level_1__TRUE_COLOR: + title: VHR Image 2018 Level 1 True Color + abstract: VHR Image 2018 Level 1 True Color + VHR_IMAGE_2018_Level_1__FALSE_COLOR: + title: VHR Image 2018 Level 1 False Color + abstract: VHR Image 2018 Level 1 False Color + VHR_IMAGE_2018_Level_1__NDVI: + title: VHR Image 2018 Level 1 NDVI + abstract: VHR Image 2018 Level 1 NDVI + style: earth + VHR_IMAGE_2018_Level_1__TRUE_COLOR: + title: VHR Image 2018 Level 3 True Color + abstract: VHR Image 2018 Level 3 True Color + VHR_IMAGE_2018_Level_1__FALSE_COLOR: + title: VHR Image 2018 Level 3 False Color + abstract: VHR Image 2018 Level 3 False Color + VHR_IMAGE_2018_Level_1__NDVI: + title: VHR Image 2018 Level 3 NDVI + abstract: VHR Image 2018 Level 3 NDVI + style: earth + +Once the initialization is finished the next step is to deploy the Docker Swarm +stack as described in the section :ref:`setup`. diff --git a/documentation/operator-guide/intro.rst b/documentation/operator-guide/intro.rst new file mode 100644 index 0000000000000000000000000000000000000000..a0802d660e26ea0654118670eb3e0cf2714074e8 --- /dev/null +++ b/documentation/operator-guide/intro.rst @@ -0,0 +1,81 @@ +.. _introduction: + +Introduction +============ + +This guide details the operation of a View Server and all of its +components. + +Since the View Server is a Docker based software and all of its components are +distributed and executed in context of Docker images and containers, basic +knowledge of Docker and Docker Swarm is a prerequisite. + +Components +---------- + +.. figure:: images/architecture.png + :alt: View Server Architecture + + *View Server Architecture* + +The View Server consists of the following service components (with their +respective Docker image in parenthesis): + +- Reverse proxy (traefik) +- Web Client (client) +- Cache (cache) +- Renderer (core) +- Registrar (core) +- Seeder (cache) +- Preprocessor (preprocessor) +- Ingestor (ingestor) +- Database (postgis) +- Queue Manager (redis) +- Log collector (fluentd) + +These services are bundled and managed together in a Docker Swarm via +Docker Compose configuration files. + +Docker Images +------------- + +The software is distributed as Docker images, which can be instantiated +and run in their intended role. Some images are hosted on +`docker hub `_, +the official and default repository for Docker images. Other images reside +on an `EOX hosted registry `_. Images from the official registry +are only identified via their name, whereas images from the EOX registry +conventionally use the full URL, including the domain name. Below is a list of +the used images: + +- mdillon/postgis:10 +- redis +- traefik:2.1 +- fluent/fluentd +- registry.gitlab.eox.at/esa/prism/vs/pvs_core:latest +- registry.gitlab.eox.at/esa/prism/vs/pvs_cache:latest +- registry.gitlab.eox.at/esa/prism/vs/pvs_preprocessor:latest +- registry.gitlab.eox.at/esa/prism/vs/pvs_client:latest + +Configuration Files +------------------- + +The following configuration files impact the behavior of the View Server: + +- `index.html`: This is the main file to configure the client. In this file, + the viewing layer, search and download endpoints are configured. Usually + this is associated with additional backdrop and overlay layers. +- `preprocessor.yml`: This file configures the preprocessing steps. +- `mapcache.xml`: This file defines the input sources of the cache and its + published layers. +- `init-db.sh`: This file sets up the registrar and renderer side of the VS. + +Initialization and Setup +------------------------ + +In order to help with the initial setup of a VS, the ``pvs_starter`` package +described in the section :ref:`initialization` allows to quickly establish the +required structure of configuration files. + +The section :ref:`setup` describes how to deploy a Docker Swarm stack using the +configuration files generated in the initialization step. diff --git a/documentation/operator-guide/make.bat b/documentation/operator-guide/make.bat new file mode 100644 index 0000000000000000000000000000000000000000..2119f51099bf37e4fdb6071dce9f451ea44c62dd --- /dev/null +++ b/documentation/operator-guide/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/documentation/operator-guide/management.rst b/documentation/operator-guide/management.rst new file mode 100644 index 0000000000000000000000000000000000000000..3125157ca58c1d4e4201841b22e7545aa5d265bd --- /dev/null +++ b/documentation/operator-guide/management.rst @@ -0,0 +1,77 @@ +.. _management: + +Service Management +================== + +This section shows how a deployed VS stack can and should be interacted with. + + +Scaling +------- + +Scaling is a handy tool to ensure stable performance, even when dealing with +higher usage on any service. For example, the preprocessor and registrar can +be scaled to a higher replica count to enable a better throughput when +ingesting data into the VS. + +The following command scales the ``renderer`` service to 5 replicas: + +.. code-block:: bash + + docker service scale _renderer=5 + +A service can also be scaled to zero replicas, effectively disabling the +service. + +.. warning:: + + The ``redis`` and ``database`` should never be scaled (their replica count + should remain 1) as this can lead to service disruptions and corrupted data. + + +Updating Images +--------------- + +Updating the service software is done using previously established tools. To +update the service in question, it needs to be scaled to zero replicas. Then +the new image can be pulled, and the service can be scaled back to its original +value. This forces the start of the service from the newly fetched image. +Another option to keep the service running during the upgrade procedure is to +sequentially restart the individual instances of the services after pulling a +newer image using a command: + +.. code-block:: bash + + docker service update --force _ + +Updating configurations or environment files +-------------------------------------------- + +Updating the service configurations or environment files used can not be done +just by rescaling the impacted services to 0 and rerunning. The whole stack +needs to be shut down using the command: + +.. code-block:: bash + + docker stack rm + +A new deployment of the stack will use the updated configuration. The above +mentioned process necessarily involves a certain service downtime between +shutting down of the stack and new deployment. + +Cleaning up +----------- +Current configuration of the services does not have any log rotation set up, which means that service logs can grow significantly over time, if left not maintained and set on verbose logging levels. In order to delete logs older than 7 days from a single node, a following command can be run + +.. code-block:: bash + + journalctl --vacuum-time=7d + +Additionally in order to delete older logs from docker containers present on a node, keeping only a certain number of newest rows, a following command can be run. + +.. code-block:: bash + + truncate -s $(docker inspect -f '{{.LogPath}}' $container 2> /dev/null) + + +The final section :ref:`ingestion` explains how to get data into the VS. diff --git a/documentation/operator-guide/setup.rst b/documentation/operator-guide/setup.rst new file mode 100644 index 0000000000000000000000000000000000000000..fbafa38bda800a7f858170a7776c605f87862885 --- /dev/null +++ b/documentation/operator-guide/setup.rst @@ -0,0 +1,150 @@ +.. _setup: + +Setup +===== + +In this chapter the setup of a new VS stack is detailed. Before this step can +be done, the configuration and environment files need to be present. These +files can be added manually or be created as described in the +:ref:`initialization` step. + + +Docker +------ + +In order to deploy the Docker Swarm stack to the target machine, Docker and +its facilities need to be installed. This step depends on the systems +architecture. On a Debian based system this may look like this: + +.. code-block:: bash + + sudo apt-get install \ + apt-transport-https \ + ca-certificates \ + curl \ + gnupg-agent \ + software-properties-common + + curl -fsSL https://download.docker.com/linux/debian/gpg | sudo apt-key add - + + # add the apt repository + sudo add-apt-repository \ + "deb [arch=amd64] https://download.docker.com/linux/debian \ + $(lsb_release -cs) \ + stable" + + # fetch the package index and install Docker + sudo apt-get update + sudo apt-get install docker-ce docker-ce-cli containerd.io + +.. ## TODO: RHEL? + +Docker Swarm +------------ + +Now that Docker is installed, the machine can either create a new swarm or join +an existing one. + +To create a new Swarm, the following command is used: + +.. code-block:: bash + + docker swarm init + + +Alternatively, an existing Swarm can be joined. The easiest way to do this, is +to obtain a ``join-token``. On an existing Swarm manager (where a Swarm was +initialized or already joined as manager) run this command: + + +.. code-block:: bash + + docker swarm join-token worker + + +This prints out a command that can be run on a machine to join the swarm: + +.. code-block:: bash + + docker swarm join --token + +It is possible to dedicate certain workers for example to contribute to ingestion exclusively, while others can take care only for rendering. This setup has benefits, when a mixed setup of nodes with different parameters is available. + +In order to set a node for example as `external`, to contribute in rendering only, one can simply run: + +.. code-block:: bash + + docker node update --label-add type=external + +Additionally, it is necessary to modify `placement` parameter in the docker compose file. + +.. code-block:: yaml + + renderer: + deploy: + placement: + constraints: + - node.labels.type == external + +Additional information for swarm management can be obtained in the official +`documentation of the project +`_. + + +Docker Image Retrieval +---------------------- + +Before the Docker images can be used, they have to be retrieved first. With +images from the default repository, this happens automatically. When private +repositories are used, they need to be configured beforehand. +Currently, all images used in VS that are not off-the-shelf are hosted on the +``registry.gitlab.eox.at`` registry. It can be configured to be used with this +command with the correct ``username`` and ``password`` filled in: + +.. code-block:: bash + + docker login -u -p registry.gitlab.eox.at + +Now the relevant images can be pulled: + +.. code-block:: bash + + docker pull registry.gitlab.eox.at/esa/prism/vs/pvs_core + docker pull registry.gitlab.eox.at/esa/prism/vs/pvs_cache + docker pull registry.gitlab.eox.at/esa/prism/vs/pvs_preprocessor + docker pull registry.gitlab.eox.at/esa/prism/vs/pvs_client + +.. # TODO: ingestor image? + +Stack Deployment +---------------- + +Now that a Docker Swarm is established, it is time to deploy the VS as a stack. +This is done using the created Docker Compose configuration files. In order to +enhance the re-usability, these files are split into multiple parts to be used +for both development and final service deployment. + +For a development deployment one would do (replace ``name`` with the actual +service identifier: + +.. code-block:: bash + + docker stack deploy -c docker-compose..yml -c docker-compose..dev.yml + + +This command actually performs a variety of tasks. First off, it obtains any +missing images, such as the image for the reverse proxy, the database, or the +redis key-value store. + +When all relevant images are pulled from their respective repository the +services of the stack are initialized. In the default setting, each service is +represented by a single container of its respective service type. When starting +for the first time, the startup procedure takes some time, as everything needs +to be initialized. This includes the creation of the database, user, +required tables, and the Django instance. + +That process can be supervised using the ``docker service ls`` command, which +lists all available services and their respective status. + +Continue to the next section :ref:`configuration` to read how a running VS +stack can be configured. diff --git a/documentation/user-guide/Makefile b/documentation/user-guide/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..115276079038a54fd7b649c3dbf384333981ec59 --- /dev/null +++ b/documentation/user-guide/Makefile @@ -0,0 +1,24 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SPHINXAUTOBUILD ?= sphinx-autobuild +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +html-watch: + @$(SPHINXAUTOBUILD) -i "$(BUILDDIR)/*" "$(SOURCEDIR)" "$(BUILDDIR)/html" $(SPHINXOPTS) $(O) + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/documentation/user-guide/conf.py b/documentation/user-guide/conf.py new file mode 100644 index 0000000000000000000000000000000000000000..efa8dcae64cb60ba2017aee8004a8bd3c163f37c --- /dev/null +++ b/documentation/user-guide/conf.py @@ -0,0 +1,76 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + + +# -- Project information ----------------------------------------------------- + +project = u'View Server - User Guide' +copyright = u'2019, 2020, EOX IT Services GmbH' +author = u'EOX IT Services GmbH' + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'recommonmark', + +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'alabaster' + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +html_favicon = 'images/favicon.ico' + +numfig = True +numfig_format = { + 'figure': 'Figure %s', + 'table': 'Table %s', + 'code-block': 'Listing %s', + 'section': 'Section', +} +numfig_secnum_depth = 2 + +latex_documents = [( + 'index', + 'View-Server_-_User-Guide.tex', + u'View Server - User Guide', + u'EOX IT Services GmbH', + 'manual', + True +)] +latex_elements = { + 'papersize': 'a4paper', +} diff --git a/documentation/user-guide/downloading.rst b/documentation/user-guide/downloading.rst new file mode 100644 index 0000000000000000000000000000000000000000..be753b8c61826872ec27fc8297b6d4f8c417721e --- /dev/null +++ b/documentation/user-guide/downloading.rst @@ -0,0 +1,62 @@ +.. _downloading: + +Downloading +=========== + +The download interface is implemented following the `Web Coverage Service (WCS) +`_ as well as the `Download Service for Earth +Observation Products Best Practice (DSEO) +`_ standards. + +These interfaces support both, the simple download of entire products via DSEO +as well as flexible customized downloads via WCS. Both services can be accessed +at the path ``/ows?``. + +The WCS offers the following customizations in order to optimize downloads and +minimize unnecessarily used bandwidth: + +* `Earth Observation Application Profile (EO-WCS) + `_ +* Meaningful default configuration +* Spatial subset to retrieve only the geographic area of interest +* Range subset to retrieve only the band(s) of interest +* Scaling to required resolution +* Format selection +* Interpolation selection if needed +* Projection selection + +Note that the download size in WCS is restricted in order to not block the +service. Clients are advised to download bigger coverages in chunks and put them +together afterwards. + +Downloading in QGIS +------------------- + +In order to use the download via WCS 1.0, that QGIS supports, a WCS layer needs to be added. + +Open the Data Source Manager and add a WCS Layer, configuring the URL as shown in :numref:`fig_wcs_qgis1` and create a new WCS Connection using the URL of the service and specifying the product id to be downloaded via CQL= query. It is necessary to tick the **Ignore GetCoverage URI reported in capabilities**, as the reference URI usually does not contain the additional CQL or any other request parameters. + +.. _fig_wcs_qgis1: +.. figure:: images/wcs_qgis1.* + :alt: QGIS - Create WCS Connection + + *QGIS - Create WCS Connection* + +A user can be prompted to specify the transformation between coordinate reference systems in case that the default output image is in a different coordinate reference system than current map projection - as shown in :numref:`fig_wcs_qgis2` + +.. _fig_wcs_qgis2: +.. figure:: images/wcs_qgis2.* + :alt: QGIS - CRS transformation confirmation + + *QGIS - CRS transformation confirmation* + +Currently, QGIS supports specifying the output coordinate reference system and image format through the relevant input fields as shown in :numref:`fig_wcs_qgis3`. All other optional parameters for WCS request need to be added to the URL manually clicking on Edit button. + +.. _fig_wcs_qgis3: +.. figure:: images/wcs_qgis3.* + :alt: QGIS WCS layer options + + *QGIS WCS layer options* + +The final chapter :ref:`sample_requests` holds a list of sample requests against +the various services described in this user guide. diff --git a/documentation/user-guide/images/arcgis_0.png b/documentation/user-guide/images/arcgis_0.png new file mode 100644 index 0000000000000000000000000000000000000000..1c550b186137e066379b1d989e8dbfb839344a57 Binary files /dev/null and b/documentation/user-guide/images/arcgis_0.png differ diff --git a/documentation/user-guide/images/arcgis_1.png b/documentation/user-guide/images/arcgis_1.png new file mode 100644 index 0000000000000000000000000000000000000000..0f2c9346231f797a2b6025a8cbf2096b6467f891 Binary files /dev/null and b/documentation/user-guide/images/arcgis_1.png differ diff --git a/documentation/user-guide/images/arcgis_2.png b/documentation/user-guide/images/arcgis_2.png new file mode 100644 index 0000000000000000000000000000000000000000..39d76e3dda36f264901c6c7af68f3065c38d34ba Binary files /dev/null and b/documentation/user-guide/images/arcgis_2.png differ diff --git a/documentation/user-guide/images/arcgis_3.png b/documentation/user-guide/images/arcgis_3.png new file mode 100644 index 0000000000000000000000000000000000000000..77bd1806e8354c2da2c4c300c05055a6af7c7127 Binary files /dev/null and b/documentation/user-guide/images/arcgis_3.png differ diff --git a/documentation/user-guide/images/arcgis_4.png b/documentation/user-guide/images/arcgis_4.png new file mode 100644 index 0000000000000000000000000000000000000000..d86011c28708db2f71c2935ca0a4b973a0c30b55 Binary files /dev/null and b/documentation/user-guide/images/arcgis_4.png differ diff --git a/documentation/user-guide/images/arcgis_5.png b/documentation/user-guide/images/arcgis_5.png new file mode 100644 index 0000000000000000000000000000000000000000..4f420fdca47e647f46e52276cfbf620b4a7f4d94 Binary files /dev/null and b/documentation/user-guide/images/arcgis_5.png differ diff --git a/documentation/user-guide/images/arcgis_6.png b/documentation/user-guide/images/arcgis_6.png new file mode 100644 index 0000000000000000000000000000000000000000..e7c29cd2985c3ab79d325fedf2f56916bb40032b Binary files /dev/null and b/documentation/user-guide/images/arcgis_6.png differ diff --git a/documentation/user-guide/images/arcgis_7.png b/documentation/user-guide/images/arcgis_7.png new file mode 100644 index 0000000000000000000000000000000000000000..3bf3b08f07e849d47d7f9be14959ac6929e89bde Binary files /dev/null and b/documentation/user-guide/images/arcgis_7.png differ diff --git a/documentation/user-guide/images/arcgis_8.png b/documentation/user-guide/images/arcgis_8.png new file mode 100644 index 0000000000000000000000000000000000000000..4fd8cd51ea6a59fd1b243c8c61177be3090a80b3 Binary files /dev/null and b/documentation/user-guide/images/arcgis_8.png differ diff --git a/documentation/user-guide/images/arcgis_9.png b/documentation/user-guide/images/arcgis_9.png new file mode 100644 index 0000000000000000000000000000000000000000..012a7b86ba55560f77d02cb1f96b45ea2212d4d3 Binary files /dev/null and b/documentation/user-guide/images/arcgis_9.png differ diff --git a/documentation/user-guide/images/favicon.ico b/documentation/user-guide/images/favicon.ico new file mode 100644 index 0000000000000000000000000000000000000000..b4d1efa50d197e85eb86c8d60c53db49393b3254 Binary files /dev/null and b/documentation/user-guide/images/favicon.ico differ diff --git a/documentation/user-guide/images/wcs_qgis1.png b/documentation/user-guide/images/wcs_qgis1.png new file mode 100644 index 0000000000000000000000000000000000000000..7902d0fe32b3cab7db98825032ad29d3ffb70f99 Binary files /dev/null and b/documentation/user-guide/images/wcs_qgis1.png differ diff --git a/documentation/user-guide/images/wcs_qgis2.png b/documentation/user-guide/images/wcs_qgis2.png new file mode 100644 index 0000000000000000000000000000000000000000..7c67f23aa6d58ac33680f7eaa30a0c098e9a2773 Binary files /dev/null and b/documentation/user-guide/images/wcs_qgis2.png differ diff --git a/documentation/user-guide/images/wcs_qgis3.png b/documentation/user-guide/images/wcs_qgis3.png new file mode 100644 index 0000000000000000000000000000000000000000..db7fdada96711769ad8b436549f6279e82bc1c27 Binary files /dev/null and b/documentation/user-guide/images/wcs_qgis3.png differ diff --git a/documentation/user-guide/images/webclient_add-to-basket.png b/documentation/user-guide/images/webclient_add-to-basket.png new file mode 100644 index 0000000000000000000000000000000000000000..fd776d30f82f586638c712aeaa7c8fbd15d45544 Binary files /dev/null and b/documentation/user-guide/images/webclient_add-to-basket.png differ diff --git a/documentation/user-guide/images/webclient_basket1.png b/documentation/user-guide/images/webclient_basket1.png new file mode 100644 index 0000000000000000000000000000000000000000..78850f0858792396ffae974b866a9194cabf4ec7 Binary files /dev/null and b/documentation/user-guide/images/webclient_basket1.png differ diff --git a/documentation/user-guide/images/webclient_basket2.png b/documentation/user-guide/images/webclient_basket2.png new file mode 100644 index 0000000000000000000000000000000000000000..e807e1da33ee7df8991c4bc8b3d877bc8f88f23b Binary files /dev/null and b/documentation/user-guide/images/webclient_basket2.png differ diff --git a/documentation/user-guide/images/webclient_coordinates.png b/documentation/user-guide/images/webclient_coordinates.png new file mode 100644 index 0000000000000000000000000000000000000000..270f9f4c5b4bd2a88dc3f7c9e02c7588e07d27a3 Binary files /dev/null and b/documentation/user-guide/images/webclient_coordinates.png differ diff --git a/documentation/user-guide/images/webclient_current-view.png b/documentation/user-guide/images/webclient_current-view.png new file mode 100644 index 0000000000000000000000000000000000000000..013a4fa386e7a3860b040b9d676a0d384ef3fb98 Binary files /dev/null and b/documentation/user-guide/images/webclient_current-view.png differ diff --git a/documentation/user-guide/images/webclient_details_display.png b/documentation/user-guide/images/webclient_details_display.png new file mode 100644 index 0000000000000000000000000000000000000000..e2b72eb7fc7f266fbdd8cc46426aef450d8eebc2 Binary files /dev/null and b/documentation/user-guide/images/webclient_details_display.png differ diff --git a/documentation/user-guide/images/webclient_download-buttons.png b/documentation/user-guide/images/webclient_download-buttons.png new file mode 100644 index 0000000000000000000000000000000000000000..7e86887096970e0f21240df27cf32b59a4eec32e Binary files /dev/null and b/documentation/user-guide/images/webclient_download-buttons.png differ diff --git a/documentation/user-guide/images/webclient_download_multiple.png b/documentation/user-guide/images/webclient_download_multiple.png new file mode 100644 index 0000000000000000000000000000000000000000..f28edbe3daec774b2c0877b1213d32dbc06e49aa Binary files /dev/null and b/documentation/user-guide/images/webclient_download_multiple.png differ diff --git a/documentation/user-guide/images/webclient_highlighting.png b/documentation/user-guide/images/webclient_highlighting.png new file mode 100644 index 0000000000000000000000000000000000000000..2f8957b7287904f778650d95174158b257b40fe5 Binary files /dev/null and b/documentation/user-guide/images/webclient_highlighting.png differ diff --git a/documentation/user-guide/images/webclient_info1.png b/documentation/user-guide/images/webclient_info1.png new file mode 100644 index 0000000000000000000000000000000000000000..7e1235fa875088da0ce7b0bd30a96fba2bab1e7e Binary files /dev/null and b/documentation/user-guide/images/webclient_info1.png differ diff --git a/documentation/user-guide/images/webclient_info2.png b/documentation/user-guide/images/webclient_info2.png new file mode 100644 index 0000000000000000000000000000000000000000..19eb8c3e1c3528c24b7452caaaf86ac3046e7dcb Binary files /dev/null and b/documentation/user-guide/images/webclient_info2.png differ diff --git a/documentation/user-guide/images/webclient_layer-configuration.png b/documentation/user-guide/images/webclient_layer-configuration.png new file mode 100644 index 0000000000000000000000000000000000000000..78446813587d1feabde4077d69a97ac66f6a8a0c Binary files /dev/null and b/documentation/user-guide/images/webclient_layer-configuration.png differ diff --git a/documentation/user-guide/images/webclient_loading-indicator.png b/documentation/user-guide/images/webclient_loading-indicator.png new file mode 100644 index 0000000000000000000000000000000000000000..47c3f642ed697bcb37915491718d7133e28fb67d Binary files /dev/null and b/documentation/user-guide/images/webclient_loading-indicator.png differ diff --git a/documentation/user-guide/images/webclient_search-buttons.png b/documentation/user-guide/images/webclient_search-buttons.png new file mode 100644 index 0000000000000000000000000000000000000000..53d38edeca6f047c40fc14a73d17387a5910d024 Binary files /dev/null and b/documentation/user-guide/images/webclient_search-buttons.png differ diff --git a/documentation/user-guide/images/webclient_select-on-map.png b/documentation/user-guide/images/webclient_select-on-map.png new file mode 100644 index 0000000000000000000000000000000000000000..75d21fc35383cb3e52f810667f3e6a44eb6068c9 Binary files /dev/null and b/documentation/user-guide/images/webclient_select-on-map.png differ diff --git a/documentation/user-guide/images/webclient_start.png b/documentation/user-guide/images/webclient_start.png new file mode 100644 index 0000000000000000000000000000000000000000..89ae6721d39d4364f13cdd2179213820037ae96b Binary files /dev/null and b/documentation/user-guide/images/webclient_start.png differ diff --git a/documentation/user-guide/images/webclient_timeslider-dots.png b/documentation/user-guide/images/webclient_timeslider-dots.png new file mode 100644 index 0000000000000000000000000000000000000000..9d97e414eb44cc535294c4621da1aa9ed5de3773 Binary files /dev/null and b/documentation/user-guide/images/webclient_timeslider-dots.png differ diff --git a/documentation/user-guide/images/webclient_timeslider-toggle.png b/documentation/user-guide/images/webclient_timeslider-toggle.png new file mode 100644 index 0000000000000000000000000000000000000000..2cf14e103381e369250654c8594756f2a3326e8d Binary files /dev/null and b/documentation/user-guide/images/webclient_timeslider-toggle.png differ diff --git a/documentation/user-guide/images/webclient_toggle-search-single.png b/documentation/user-guide/images/webclient_toggle-search-single.png new file mode 100644 index 0000000000000000000000000000000000000000..97f5f3dbf823716218f57319c172220622c21c59 Binary files /dev/null and b/documentation/user-guide/images/webclient_toggle-search-single.png differ diff --git a/documentation/user-guide/images/webclient_toggle-search.png b/documentation/user-guide/images/webclient_toggle-search.png new file mode 100644 index 0000000000000000000000000000000000000000..8d8adb0f2aad8e6e19747070650de53e14d5e824 Binary files /dev/null and b/documentation/user-guide/images/webclient_toggle-search.png differ diff --git a/documentation/user-guide/images/webclient_tutorial-button.png b/documentation/user-guide/images/webclient_tutorial-button.png new file mode 100644 index 0000000000000000000000000000000000000000..b7c7ffe26661d3f7f71f8d0339d955e044cd6369 Binary files /dev/null and b/documentation/user-guide/images/webclient_tutorial-button.png differ diff --git a/documentation/user-guide/images/webclient_tutorial_01_start.png b/documentation/user-guide/images/webclient_tutorial_01_start.png new file mode 100644 index 0000000000000000000000000000000000000000..75f957e35fc93074f61dbabf99d4a521f3f5baa3 Binary files /dev/null and b/documentation/user-guide/images/webclient_tutorial_01_start.png differ diff --git a/documentation/user-guide/images/webclient_tutorial_02_map.png b/documentation/user-guide/images/webclient_tutorial_02_map.png new file mode 100644 index 0000000000000000000000000000000000000000..cee5ce8142c3eeb34ce991e783499bbe3e712f01 Binary files /dev/null and b/documentation/user-guide/images/webclient_tutorial_02_map.png differ diff --git a/documentation/user-guide/images/webclient_tutorial_03_timeslider.png b/documentation/user-guide/images/webclient_tutorial_03_timeslider.png new file mode 100644 index 0000000000000000000000000000000000000000..2b7fa2c648f20fcb4bb15d366b1f4a56ada62aba Binary files /dev/null and b/documentation/user-guide/images/webclient_tutorial_03_timeslider.png differ diff --git a/documentation/user-guide/images/webclient_tutorial_04_search-results.png b/documentation/user-guide/images/webclient_tutorial_04_search-results.png new file mode 100644 index 0000000000000000000000000000000000000000..257ea4aea775bfc7c8f09f7e1f0345856f52bb8a Binary files /dev/null and b/documentation/user-guide/images/webclient_tutorial_04_search-results.png differ diff --git a/documentation/user-guide/images/webclient_tutorial_05_filters.png b/documentation/user-guide/images/webclient_tutorial_05_filters.png new file mode 100644 index 0000000000000000000000000000000000000000..06ca3d3650311939f1e57b10680931211fba56a2 Binary files /dev/null and b/documentation/user-guide/images/webclient_tutorial_05_filters.png differ diff --git a/documentation/user-guide/images/webclient_tutorial_06_map-layers.png b/documentation/user-guide/images/webclient_tutorial_06_map-layers.png new file mode 100644 index 0000000000000000000000000000000000000000..6c54ca38c5fbb24b8e0cbd7bacc312630e679688 Binary files /dev/null and b/documentation/user-guide/images/webclient_tutorial_06_map-layers.png differ diff --git a/documentation/user-guide/images/wms_cloud-mask.png b/documentation/user-guide/images/wms_cloud-mask.png new file mode 100644 index 0000000000000000000000000000000000000000..4075f5fa211bec1d98ad3a4d608ca858ccfe7ff3 Binary files /dev/null and b/documentation/user-guide/images/wms_cloud-mask.png differ diff --git a/documentation/user-guide/images/wms_cql-filtering.png b/documentation/user-guide/images/wms_cql-filtering.png new file mode 100644 index 0000000000000000000000000000000000000000..570312ec3730faccef10d5a97347cda481df107b Binary files /dev/null and b/documentation/user-guide/images/wms_cql-filtering.png differ diff --git a/documentation/user-guide/images/wms_custom-stretch.png b/documentation/user-guide/images/wms_custom-stretch.png new file mode 100644 index 0000000000000000000000000000000000000000..7257de1cda32c97e8e782000da5cd07a76ea37bd Binary files /dev/null and b/documentation/user-guide/images/wms_custom-stretch.png differ diff --git a/documentation/user-guide/images/wms_false-color.png b/documentation/user-guide/images/wms_false-color.png new file mode 100644 index 0000000000000000000000000000000000000000..1ada5f34b600c9d838b300134e5752024fbf7e31 Binary files /dev/null and b/documentation/user-guide/images/wms_false-color.png differ diff --git a/documentation/user-guide/images/wms_full.png b/documentation/user-guide/images/wms_full.png new file mode 100644 index 0000000000000000000000000000000000000000..44cb9c6e4a089b17658a4dfef97a6c98a2b85bd7 Binary files /dev/null and b/documentation/user-guide/images/wms_full.png differ diff --git a/documentation/user-guide/images/wms_masked-clouds.png b/documentation/user-guide/images/wms_masked-clouds.png new file mode 100644 index 0000000000000000000000000000000000000000..dfc53bde6a99005fafbea09949b3233eed8ba80a Binary files /dev/null and b/documentation/user-guide/images/wms_masked-clouds.png differ diff --git a/documentation/user-guide/images/wms_ndvi-style.png b/documentation/user-guide/images/wms_ndvi-style.png new file mode 100644 index 0000000000000000000000000000000000000000..9a0329e7a8d463fa7e71680106f70a9ec2404a62 Binary files /dev/null and b/documentation/user-guide/images/wms_ndvi-style.png differ diff --git a/documentation/user-guide/images/wms_qgis1.png b/documentation/user-guide/images/wms_qgis1.png new file mode 100644 index 0000000000000000000000000000000000000000..621d041a04147ab73500bb848838e9d2127da663 Binary files /dev/null and b/documentation/user-guide/images/wms_qgis1.png differ diff --git a/documentation/user-guide/images/wms_qgis2.png b/documentation/user-guide/images/wms_qgis2.png new file mode 100644 index 0000000000000000000000000000000000000000..224f04c915bc7f84b30076a724993ed90e8bd820 Binary files /dev/null and b/documentation/user-guide/images/wms_qgis2.png differ diff --git a/documentation/user-guide/images/wms_qgis3.png b/documentation/user-guide/images/wms_qgis3.png new file mode 100644 index 0000000000000000000000000000000000000000..728dd21030f3dc6c41ba340360f588c135faa0cc Binary files /dev/null and b/documentation/user-guide/images/wms_qgis3.png differ diff --git a/documentation/user-guide/images/wms_qgis4.png b/documentation/user-guide/images/wms_qgis4.png new file mode 100644 index 0000000000000000000000000000000000000000..32d9ff281392deb59d9810dddff0e7a4edd08240 Binary files /dev/null and b/documentation/user-guide/images/wms_qgis4.png differ diff --git a/documentation/user-guide/images/wms_quicklook.png b/documentation/user-guide/images/wms_quicklook.png new file mode 100644 index 0000000000000000000000000000000000000000..5684526b8939e29ef9aa7ae641b4ad1f20cda019 Binary files /dev/null and b/documentation/user-guide/images/wms_quicklook.png differ diff --git a/documentation/user-guide/images/wmts_animation.gif b/documentation/user-guide/images/wmts_animation.gif new file mode 100644 index 0000000000000000000000000000000000000000..f65ed604cfd541f0c077eafee02d987d7f83ca3a Binary files /dev/null and b/documentation/user-guide/images/wmts_animation.gif differ diff --git a/documentation/user-guide/images/wmts_full.png b/documentation/user-guide/images/wmts_full.png new file mode 100644 index 0000000000000000000000000000000000000000..8a247c1ac8ab08e17d3abdc6741ffd33a0d1f44c Binary files /dev/null and b/documentation/user-guide/images/wmts_full.png differ diff --git a/documentation/user-guide/images/wmts_masked.png b/documentation/user-guide/images/wmts_masked.png new file mode 100644 index 0000000000000000000000000000000000000000..bba306b53bb888b0643407012bc4eec4528d952e Binary files /dev/null and b/documentation/user-guide/images/wmts_masked.png differ diff --git a/documentation/user-guide/images/wmts_tiles.png b/documentation/user-guide/images/wmts_tiles.png new file mode 100644 index 0000000000000000000000000000000000000000..14e363a46e05587933f52a3ea4d33c4bb7b0b202 Binary files /dev/null and b/documentation/user-guide/images/wmts_tiles.png differ diff --git a/documentation/user-guide/index.rst b/documentation/user-guide/index.rst new file mode 100644 index 0000000000000000000000000000000000000000..0c7cca566e6f78506eba506ac8a8a59f5de5196d --- /dev/null +++ b/documentation/user-guide/index.rst @@ -0,0 +1,32 @@ +.. _contents: + +View Server - User Guide +======================== + +.. toctree:: + :maxdepth: 3 + + intro + + web-client + viewing + searching + downloading + + sample-requests + +.. + Indices and tables + ================== + + .. only:: builder_html + + * :ref:`genindex` + * :ref:`modindex` + * :ref:`search` + .. * :ref:`glossary` + + .. only:: not builder_html + + * :ref:`modindex` + .. * :ref:`glossary` diff --git a/documentation/user-guide/intro.rst b/documentation/user-guide/intro.rst new file mode 100644 index 0000000000000000000000000000000000000000..b9951ff69edd36c3a8a0daa24ef7e3265415c36f --- /dev/null +++ b/documentation/user-guide/intro.rst @@ -0,0 +1,16 @@ +Introduction +============ + +This is the User Guide of the View Server (VS). The intended readers are users +of the web client shown below and external services provided by the View Server. + +These provided external services are services for searching, viewing, and +downloading of Earth Observation (EO) data. Service endpoints optimized for +performance as well as for flexibility are provided alongside each other. + +.. figure:: images/webclient_start.* + :alt: Web Client + + *Web Client* + +Continue reading to understand the details of the provided :ref:`web_client`. diff --git a/documentation/user-guide/make.bat b/documentation/user-guide/make.bat new file mode 100644 index 0000000000000000000000000000000000000000..2119f51099bf37e4fdb6071dce9f451ea44c62dd --- /dev/null +++ b/documentation/user-guide/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/documentation/user-guide/sample-requests.rst b/documentation/user-guide/sample-requests.rst new file mode 100644 index 0000000000000000000000000000000000000000..0e41826eb7f61903a0f69430f88cfaf7e7d427ae --- /dev/null +++ b/documentation/user-guide/sample-requests.rst @@ -0,0 +1,1101 @@ +.. _sample_requests: + +Sample Service Requests +======================= + +This chapter provides various sample requests. Those requests are against a +production service and might get invalid over time. + +Please note that the production service used requires user authentication. +Please ask the European Space Agency (ESA) to get access granted. + +Web Client +---------- + +The web and services are accessible at two production URLs: + +* pdas.prism.eox.at +* pass.copernicus.eu + +Currently as of writing two collections/datasets are available, the `VHR IMAGE +2018 `_ and the `Emergency +`_ dataset. + +Performance optimized viewing +----------------------------- + +`WMTS Capabilities `_ + +.. code-block:: xml + + + + + PRISM Data Access Service (PASS) developed by EOX + PRISM Data Access Service (PASS) developed by EOX + + view service + + OGC WMTS + 1.0.0 + UNKNOWN + UNKNOWN + + + + + + + + + + + + KVP + + + + + + + + + + + + + KVP + + + + + + + + + + + + + KVP + + + + + + + + + TBD + application/vnd.iso.19139+xml + + + + eng + + + + eng + + + + + + VHR Image 2018 Level 3 True Color with masked validity + VHR Image 2018 Level 3 True Color with masked validity + + -180.000000 -90.000000 + 180.000000 90.000000 + + VHR_IMAGE_2018_Level_3__masked_validity + + image/unknown + + time + 2017/2019 + 2017-05-01T09:54:28Z/2019-10-06T07:03:34Z + + + WGS84 + + + + + VHR Image 2018 Level 3 True Color with masked validity Full Coverage + VHR Image 2018 Level 3 True Color with masked validity Full Coverage + + -24.700000 27.500000 + 45.000000 71.300000 + + VHR_IMAGE_2018_Level_3__masked_validity__Full + + image/unknown + + WGS84 + + + + + + + + + + +`WMS Capabilities `_ + +.. code-block:: xml + + + ]> + + + OGC:WMS + PRISM Data Access Service (PASS) developed by EOX + + + + + + application/vnd.ogc.wms_xml + + + + + + + + + + image/png + image/jpeg + + + + + + + + + + text/plain + application/vnd.ogc.gml + + + + + + + + + + + text/plain + + + + + + PRISM Data Access Service (PASS) developed by EOX + EPSG:4326 + + VHR_IMAGE_2018_Level_3__masked_validity + VHR Image 2018 Level 3 True Color with masked validity + VHR Image 2018 Level 3 True Color with masked validity + + 2017-05-01T09:54:28Z/2019-10-06T07:03:34Z + + EPSG:4326 + + + VHR_IMAGE_2018_Level_3__masked_validity__Full + VHR Image 2018 Level 3 True Color with masked validity Full Coverage + VHR Image 2018 Level 3 True Color with masked validity Full Coverage + + + EPSG:4326 + + + + + + +`WMTS True color tile `_, +`WMTS False color tile `_, and +`WMTS NDVI tile `_. +Same area via `WMS request `_. + +.. figure:: images/wmts_tiles.* + :alt: WMTS Tiles - True color, false color, and NDVI + + *WMTS Tiles - True color, false color, and NDVI* + +`WMTS True color masked validity tile `_ + +.. figure:: images/wmts_masked.* + :alt: WMTS Tiles - Unmasked and masked + + *WMTS Tiles - Unmasked and masked* + +`WMTS Full layer `_ + +.. figure:: images/wmts_full.* + :alt: WMTS Tiles - Full layer + + *WMTS Tiles - Full layer* + +`WMS Full layer `_ + +.. figure:: images/wms_full.* + :alt: WMS - Full layer + + *WMS - Full layer* + +Flexible viewing +---------------- + +`Capabilities for collection layers `_ +(example below) or `Layer capabilities for one product `_ + +.. code-block:: xml + + + + + PRISM Data Access Service (PASS) developed by EOX + PRISM Data Access Service (PASS) developed by EOX + PRISM Data Access Service (PASS) developed by EOX + + <KEYWORDLIST> + + http://eoxserver.org + + + <CONTACTPERSON> + <CONTACTORGANIZATION> + + <CONTACTPOSITION> + + postal +
<ADDRESS>
+ <CITY> + <STATEORPROVINCE> + <POSTCODE> + <COUNTRY> + + <CONTACTVOICETELEPHONE> + <CONTACTFACSIMILETELEPHONE> + <CONTACTELECTRONICMAILADDRESS> + + None + None + + + + + text/xml + + + + + + + + + + image/png + image/jpeg + image/gif + image/tiff + + + + + + + + + + + + + + + + + + + + + XML + INIMAGE + BLANK + + + PRISM Data Access Service (PASS) developed by EOX + EPSG:4326 + EPSG:3857 + EPSG:3035 + + -180 + 180 + -90 + 90 + + + + VHR_IMAGE_2018_Level_3 + + -24.6397992939 + 44.8308373419 + 27.5748429858 + 71.2306420611 + + + VHR_IMAGE_2018_Level_3__outlines + + + + + + + VHR_IMAGE_2018_Level_3__masked_validity + + 2017-05-01T09:54:28Z/2018-09-30T13:26:34Z/PT1S + + + + + +`Quicklook `_ + +.. figure:: images/wms_quicklook.* + :alt: WMS - Quicklook + + *WMS - Quicklook* + +`False color `_ + +.. figure:: images/wms_false-color.* + :alt: WMS - False color + + *WMS - False color* + +`NDVI including style `_ + +.. figure:: images/wms_ndvi-style.* + :alt: WMS - NDVI custom style + + *WMS - NDVI custom style* + +`Custom color stretch `_ + +.. figure:: images/wms_custom-stretch.* + :alt: WMS - Custom color stretch + + *WMS - Custom color stretch* + +`CQL filtering on Cloud Coverage `_ + +.. figure:: images/wms_cql-filtering.* + :alt: WMS - CQL filtering + + *WMS - CQL filtering* + +`Clouds mask `_ + +.. figure:: images/wms_cloud-mask.* + :alt: WMS - Clouds mask + + *WMS - Clouds mask* + +`Masked clouds `_ + +.. figure:: images/wms_masked-clouds.* + :alt: WMS - Masked clouds + + *WMS - Masked clouds* + +.. _samples_searching: + +Searching +--------- + +`1st step OSDD `_ (stripped to only +contain the Atom URL template) + +.. code-block:: xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +`1st step search for collections `_ + +.. code-block:: xml + + + https://vhr18.pdas.prism.eox.at/opensearch/atom/ + None Search + + 3 + 0 + 3 + + + + + + + VHR_IMAGE_2018_Level_3 + https://vhr18.pdas.prism.eox.at/opensearch/atom/?uid=VHR_IMAGE_2018_Level_3 + VHR_IMAGE_2018_Level_3 + 2017-05-01T09:54:28Z/2018-09-30T13:26:34Z + 27.574843 -24.639799 71.230642 44.830837 + + + + + + + 27.57484299 -24.63979929 71.23064206 -24.63979929 71.23064206 44.83083734 27.57484299 44.83083734 27.57484299 -24.63979929 + + + + + + + + +
Date 2017-05-01T09:54:28+00:00 / 2018-09-30T13:26:34+00:00
MetadataATOM

OGC cross links

Metadata

]]> + + + + +`2nd step OSDD `_ +(stripped to only contain the Atom URL template) + +.. code-block:: xml + + + + VHR_IMAGE_2018_Level_3 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +`2nd step search `_ and +`2nd step search returning one product `_ + +.. code-block:: xml + + + https://vhr18.pdas.prism.eox.at/opensearch/collections/VHR_IMAGE_2018_Level_3/atom/?uid=urn:eop:DOVE:MULTISPECTRAL_4m:20180811_081455_1054_3be7 + VHR_IMAGE_2018_Level_3 Search + + 1 + 0 + 1 + + + + + + + urn:eop:DOVE:MULTISPECTRAL_4m:20180811_081455_1054_3be7 + https://vhr18.pdas.prism.eox.at/opensearch/collections/VHR_IMAGE_2018_Level_3/atom/?uid=urn:eop:DOVE:MULTISPECTRAL_4m:20180811_081455_1054_3be7 + urn:eop:DOVE:MULTISPECTRAL_4m:20180811_081455_1054_3be7 + 2018-08-11T08:14:55Z + 47.297788 25.978201 47.372606 26.047511 + + + + + + + 47.29778796 26.00330821 47.29784395 26.00300545 47.29852855 26.00098039 47.33176050 25.98961353 47.35564999 25.98157482 47.35572189 25.98204011 47.36693204 25.97820059 47.37260630 26.01455420 47.37197192 26.01658883 47.37098788 26.01863079 47.36976416 26.02036682 47.36834411 26.02173541 47.36677806 26.02268812 47.31293985 26.04698765 47.31114307 26.04751063 47.30931486 26.04748521 47.30753117 26.04691252 47.30586607 26.04581637 47.30438870 26.04424235 47.30409927 26.04377423 47.29778796 26.00330821 + + + + + + + + + QUICKLOOK + + + THUMBNAIL + + + + + + + + + + + +
Date 2018-08-11T08:14:55+00:00 / 2018-08-11T08:14:55+00:00
MetadataATOM
DownloadPackage

OGC cross links

Metadata

Cloud Cover0.0
]]> + + + + +Downloading +----------- + +WCS +^^^ + +`WCS Capabilities `_ + +.. code-block:: xml + + + + + PRISM Data Access Service (PASS) developed by EOX + PRISM Data Access Service (PASS) developed by EOX + + <KEYWORDLIST> + + OGC WCS + 2.0.1 + 2.0.0 + 1.1.2 + 1.1.1 + 1.1.0 + 1.0.0 + http://www.opengis.net/spec/WCS_application-profile_earth-observation/1.0/conf/eowcs + http://www.opengis.net/spec/WCS_application-profile_earth-observation/1.0/conf/eowcs_get-kvp + http://www.opengis.net/spec/WCS_service-extension_crs/1.0/conf/crs + http://www.opengis.net/spec/WCS/2.0/conf/core + http://www.opengis.net/spec/WCS_protocol-binding_get-kvp/1.0/conf/get-kvp + http://www.opengis.net/spec/WCS_protocol-binding_post-xml/1.0/conf/post-xml + http://www.opengis.net/spec/GMLCOV/1.0/conf/gml-coverage + http://www.opengis.net/spec/GMLCOV/1.0/conf/multipart + http://www.opengis.net/spec/GMLCOV/1.0/conf/special-format + http://www.opengis.net/spec/GMLCOV_geotiff-coverages/1.0/conf/geotiff-coverage + http://www.opengis.net/spec/WCS_geotiff-coverages/1.0/conf/geotiff-coverage + http://www.opengis.net/spec/WCS_service-model_crs-predefined/1.0/conf/crs-predefined + http://www.opengis.net/spec/WCS_service-extension_interpolation/1.0/conf/interpolation + http://www.opengis.net/spec/WCS_service-extension_range-subsetting/1.0/conf/record-subsetting + http://www.opengis.net/spec/WCS_service-extension_scaling/1.0/conf/scaling + None + None + + + <CONTACTORGANIZATION> + + + <CONTACTPERSON> + <CONTACTPOSITION> + + + <CONTACTVOICETELEPHONE> + <CONTACTFACSIMILETELEPHONE> + + + <ADDRESS> + <CITY> + <STATEORPROVINCE> + <POSTCODE> + <COUNTRY> + <CONTACTELECTRONICMAILADDRESS> + + + <HOURSOFSERVICE> + <CONTACTINSTRUCTIONS> + + Service provider + + + + + + + + + + + XML + + + + + + + + + + + + + + XML + + + + + + + + + + + + + + XML + + + + + + + + + + + + + + XML + + + + + + + + 10 + + + + + + + + + + XML + + + + + + + + 10 + + + + + image/tiff + image/jp2 + application/x-netcdf + application/x-hdf + + + http://www.opengis.net/def/crs/EPSG/0/4326 + http://www.opengis.net/def/crs/EPSG/0/3857 + http://www.opengis.net/def/crs/EPSG/0/3035 + + + http://www.opengis.net/def/interpolation/OGC/1/average + http://www.opengis.net/def/interpolation/OGC/1/nearest-neighbour + http://www.opengis.net/def/interpolation/OGC/1/bilinear + http://www.opengis.net/def/interpolation/OGC/1/cubic + http://www.opengis.net/def/interpolation/OGC/1/cubic-spline + http://www.opengis.net/def/interpolation/OGC/1/lanczos + http://www.opengis.net/def/interpolation/OGC/1/mode + + + + + + + + 12.136296 -24.639799 + 71.230642 52.293404 + + VHR_IMAGE_2018 + + 2017-05-01T09:54:28Z + 2019-10-06T07:03:34Z + + + + + 27.574843 -24.639799 + 71.230642 44.830837 + + VHR_IMAGE_2018_Level_1 + + 2017-05-01T09:54:28Z + 2018-09-30T13:26:34Z + + + + + 27.574843 -24.639799 + 71.230642 44.830837 + + VHR_IMAGE_2018_Level_3 + + 2017-05-01T09:54:28Z + 2018-09-30T13:26:34Z + + + + + + +`EO Coverage set description `_ and +`Coverage description `_ + +.. code-block:: xml + + + + + + + 2817620.000 5518044.000 + 2825972.000 5524488.000 + + + urn:eop:DOVE:MULTISPECTRAL_4m:20180811_081455_1054_3be7__coverage + + + + + + + 2018-08-11T08:14:55Z + 2018-08-11T08:14:55Z + + + + + 2018-08-11T08:14:55Z + + + + + + + + + + + + + 47.29778796 26.00330821 47.29784395 26.00300545 47.29852855 26.00098039 47.33176050 25.98961353 47.35564999 25.98157482 47.35572189 25.98204011 47.36693204 25.97820059 47.37260630 26.01455420 47.37197192 26.01658883 47.37098788 26.01863079 47.36976416 26.02036682 47.36834411 26.02173541 47.36677806 26.02268812 47.31293985 26.04698765 47.31114307 26.04751063 47.30931486 26.04748521 47.30753117 26.04691252 47.30586607 26.04581637 47.30438870 26.04424235 47.30409927 26.04377423 47.29778796 26.00330821 + + + + + + + + + + + + urn:eop:DOVE:MULTISPECTRAL_4m:20180811_081455_1054_3be7__coverage + NOMINAL + ARCHIVED + + + + + + + + + + + 0 0 + 1610 2087 + + + y x + + + 2825972.000 5518044.000 + + + 0.000 4.000 + -4.000 0.000 + + + + + + + Red Channel + + + 0 + + + + + + 0 65535 + 5 + + + + + + + Green Channel + + + 0 + + + + + + 0 65535 + 5 + + + + + + + Blue Channel + + + 0 + + + + + + 0 65535 + 5 + + + + + + + Nir Channel + + + 0 + + + + + + 0 65535 + 5 + + + + + + + + RectifiedDataset + image/tiff + + + + +`Full coverage in default configuration `_ + +`Spatial subset `_ + +`Range subset `_ + +`Scaled down `_ + +`Specific format and interpolation `_ + +`Specific projection `_ + +DSEO +^^^^ + +`One entire product `_ diff --git a/documentation/user-guide/searching.rst b/documentation/user-guide/searching.rst new file mode 100644 index 0000000000000000000000000000000000000000..a422e0579cb03769d0e422cf689746d6e440ae64 --- /dev/null +++ b/documentation/user-guide/searching.rst @@ -0,0 +1,97 @@ +.. _searching: + +Searching +========= + +The main interface for searching is provided by `OpenSearch +`_ using various extensions including the `Geo and Time +Extensions `_, the +`Extension for Earth Observation +`_ as well as the +`CEOS OpenSearch Best Practice Document +`_. + +The OpenSearch URL endpoints are: + +* ``/opensearch``: Main service description +* ``/opensearch/``: Search for collections in a specific format +* ``/opensearch/collections/``: Collection specific description +* ``/opensearch/collections//``: Search for products in + a collection in that format + +Responses +--------- + +The description related endpoints provide an OpenSearch Description Document +(OSDD). This XML based file format describes general service metadata and +various access URL templates. + +These URL objects provide templates that can be filled with parameters to submit +actual metadata queries. Depending on the used URL, the search result is +returned in one of the following formats: + +* ``atom``: Atom syndication format +* ``rss``: RSS +* ``html``: HTML +* ``kml``: Keyhole Markup Language +* ``json``: GeoJSON + +The resulting items in that result document can be paged. Specific pages can be +retrieved using the ``offset`` and ``count`` parameters. Additionally, the +response contains links to the first, last, next, and previous pages of the +result set. + +The results of the search depend on the type of object being searched. For the +collection search, the result items are collection descriptions of all matched +collections encoded in the selected format. Each description contains a link to +its own specific OSDD document. + +In product searches, the result document items are descriptions of products, +containing links to thumbnails, browses, and product specific services such as +WMS or WCS. Additionally it contains the products acquisition footprint and +time stamp. + +Filters +------- + +The URL templates for both the collection and the record search allow to pass +parameters that control the query and shape of the search. The following table +shows all parameters available for both collection and product searches: + +=============== ================================================================ +Parameter Description +=============== ================================================================ +``startIndex`` The index offset of the items. Used to get subsets of a result + set +``count`` The number of records for each page +``start`` The start time stamp for the time range +``end`` The end time stamp for the time range +``timerel`` The semantic of the time range. Either ``intersects``, + ``contains``, ``disjoint``, or ``equals`` +``bbox`` The geographic bounding box expressed as ``min-lon``, + ``min-lat``, ``max-lon``, ``max-lat`` in WGS84 coordinates +``geom`` A WKT encoded geometry for a geographic search +``lon`` The longitude component of point/radius searches +``lat`` The latitude component of point/radius searches +``r`` The radious in metres of point/radius searches +``georel`` The semantic of the geometry filter. One of ``intersects``, + ``contains``, or ``disjoint`` +=============== ================================================================ + +The following parameters are only available in product searches in +addition to the general search parameters: + +=============== ================================================================ +Parameter Description +=============== ================================================================ +``uid`` Filter on identifier +``cloudCover`` Cloud cover filter. Either a specific value or an interval of + values. Values between 0-100 +=============== ================================================================ + +The available parameters depend on the configuration made by the operator. + +Samples responses for each of the four URL endpoints are provided in the +:ref:`samples_searching` section of the :ref:`sample_requests` chapter. + +Continue reading about the provided :ref:`downloading` services. diff --git a/documentation/user-guide/viewing.rst b/documentation/user-guide/viewing.rst new file mode 100644 index 0000000000000000000000000000000000000000..ad874561274f5728abb94c02c8b0e32de5ec4a62 --- /dev/null +++ b/documentation/user-guide/viewing.rst @@ -0,0 +1,309 @@ +.. _viewing: + +Viewing +======= + +The viewing interface is implemented following the `Web Map Tile Service (WMTS) +`_ as well as the `Web Map Service (WMS) +`_ standards. Both services are implemented +by a wide range of client software which makes it as simple as possible to +integrate the viewing service as needed. + +Service endpoints optimized for performance as well as for flexibility are +provided alongside each other. Concrete example URLs demonstrating the +capabilities of the various endpoints are provided in the chapter +:ref:`sample_requests`. + + +Performance optimized service +----------------------------- + +The performance optimized service variant is provided under the path +``/cache/ows?``. + +This endpoint offers access to predefined visualizations via WMTS and WMS. The +GetTile as well as GetMap requests are answered from image tiles cached on +object storage when possible. :numref:`fig_wmts_tiles` below shows the three +predefined visualizations, true color, false color, and NDVI of the same image +tile and :numref:`fig_wmts_masked` the unmasked and masked versions using the +validity mask of one tile. + +.. _fig_wmts_tiles: +.. figure:: images/wmts_tiles.* + :alt: WMTS Tiles - True color, false color, and NDVI + + *WMTS Tiles - True color, false color, and NDVI* + +.. _fig_wmts_masked: +.. figure:: images/wmts_masked.* + :alt: WMTS Tiles - Unmasked and masked + + *WMTS Tiles - Unmasked and masked* + +The operator might configure additional layers besides the default ones. These +layers can offer a specific customization described below as performance +optimized variant. + +For example for the VHR Image 2018 dataset not only the three default layers for +True Color, False Color, and NDVI are configured but also the Masked Validity +one as shown in the figures above. + +In addition the VHR Image 2018 dataset contains level 1 and 3 products. Thus +individual layers aggregating each level individually are added making the total +of offered layers twelve. + +Finally two layers are configured offering a quick true color visualization of +the entire level 3 datasets without support of the time dimension both unmasked +as well as masked. The masked layer is shown in +:numref:`fig_webclient_fulllayer` below. + +.. _fig_webclient_fulllayer: +.. figure:: images/webclient_start.* + :alt: Web Client - Full coverage layer + + *Web Client - Full coverage layer* + +Image tiles not yet present in the cache are stored upon first hit for +consecutive requests. This is the so-called on-demand caching. The operator +might decide for optimal performance, even for the first request, to pre-seed +the cache. Also a mixed variant with a partly pre-seeded cache for example up to +a certain zoom level is possible. + +However, this is done fully transparent to the users and the only impact is that +they might need to wait slightly longer when they are the first to visualize a +certain product. + +This service offers access on collection level only. Individual products can be +addressed using the ``time`` parameter given that they do not cover one another +on the time axis. + +The best performance is offered at the WMTS interface which serves images tiles +as they are cached without the need of any reprocessing. WMS responses are +generated on-the-fly from image tiles and might thus need some reprocessing to +stitch together and scale image tiles. + +More sophisticated reprocessing like reprojection or custom color adjustments +are not offered at this endpoint. + +Flexible service +---------------- + +The other service variant offered is optimized for the best possible flexibility +and can be accessed at the path ``/ows?``. + +This endpoint offers the following customizations via WMS: + +* Access to individual products via ID (``cql=identifier='ID'``) +* Access to entire collections/datasets supporting ``time`` parameter +* Various layers as defined by the `Earth Observation Application Profile of WMS + (EO-WMS) `_ like + one showing outlines of products (``__outlines``) +* Additional useful layers like one showing both, products and outlines, + correctly rendering topology of outlines (``__outlined``) or one showing a + validity mask (``__validity``) or applied validity mask + (``___masked_validity``) +* Numerous styles for layer where feasible like outlines or NDVI + (``layers=ID__NDVI&STYLES=coolwarm``) +* Reprojection to operator configured coordinate reference systems +* Color adjustment via custom linear stretch ``dim_range= + [, , ]`` +* Filtering of products to render into collection/dataset visualizations via + Common Query Language (CQL) for example ``&cql=cloudCover<1`` +* All other standard WMS parameters like ``transparent``, ``format``, etc. + +All these customization parameters can of course be combined as needed. + +Please see the chapter :ref:`sample_requests` for detailed examples of all +supported parameters. + +.. _loading_qgis: + +Loading in QGIS +--------------- + +Any of the above described layers can be loaded in client software supporting +either the WMTS or WMS standard like for example `QGIS `_. + +Open the Data Source Manager as shown in :numref:`fig_wms_qgis1` and create a +new WMS/WMTS Connection using the URL to the layer of interest. + +When using parameters like for example ``cql=identifier='ID'`` to view one +individual product make sure to tick the "Ignore GetMap/GetTile URI reported in +capabilities" as these parameters are overwritten otherwise. + +.. _fig_wms_qgis1: +.. figure:: images/wms_qgis1.* + :alt: QGIS - Create WMS/WMTS Connection + + *QGIS - Create WMS/WMTS Connection* + +The next step is to retrieve the capabilities of the created WMS/WMTS Connection +using the Connect button. Selected layers from the list are added using the Add +button as shown in :numref:`fig_wms_qgis2`. + +.. _fig_wms_qgis2: +.. figure:: images/wms_qgis2.* + :alt: WMS loading in QGIS + + *WMS loading in QGIS* + +An important parameter to consider while viewing the layers is the requested +time interval, allowing for example to observe changes over years. One option is +to manually add a custom **time=** parameter to the URL after checking the +`Ignore GetMap URI` checkbox. + +For QGIS versions lower than 3.14, the only way how to use WMS layers with time +support (WMS-T) is through a maintained plugin called `TimeManager +`_. It offers a convenient way to +navigate through the time for raster and vector layers with custom defined steps +as shown in :numref:`fig_wms_qgis3`. + +.. _fig_wms_qgis3: +.. figure:: images/wms_qgis3.* + :alt: WMS loading in QGIS with TimeManager + + *WMS loading in QGIS with TimeManager* + +The plugin needs to be installed first and then set up with the desired layer. +The raster layer to be added to the TimeManager must be first added to the +workspace as shown in :ref:`loading_qgis`. By clicking the `Settings` button and +following up with the `Add raster` button as can be seen in +:numref:`fig_wms_qgis4` it can be then chosen from the list of layers. It is +also suggested to add Start and End time constrains to narrow down the interval +which TimeManager will consider. + +.. _fig_wms_qgis4: +.. figure:: images/wms_qgis4.* + :alt: Setup of TimeManager + + *Setup of TimeManager* + +After setting the layer up, it is possible to change the time interval length +(for example one month) and view all products in that interval. After clicking +the `play` button, it displays the layer iteratively from the defined Start time +to the defined End time, always displaying each frame for a set number of +milliseconds. + +.. only:: builder_html + + It even allows exporting an animation as shown in :numref:`fig_wms_qgis5` below. + + .. _fig_wms_qgis5: + .. figure:: images/wmts_animation.gif + :alt: GIF animation from TimeManager + + *GIF animation created from TimeManager* + +New support for time aware layers was added in QGIS Version 3.14 called +**Temporal Controller**. `This video +`_ directly from the author of the +feature shows the usage. However, due to the way that Capabilities of layers are +provided by both View Server components Cache and Renderer, the QGIS Temporal +Controller can not parse the time dimension metadata correctly. + +For performance reasons the View Server lists only start and end of the time +interval, in which the collection contains some products. It would not be +suitable to list all individual time entries, as there can be easily hundreds of +thousands of them. Both QGIS 3.14 and ArcMap 10.4 expect individual time entries +listing and then determine its own time related behavior while handling these +layers. QGIS 3.14 does not recognize capabilities time interval as time aware +layer and does not add the `TIME=` parameter to the WMS/WMTS query, so the +TimeManager plugin still stays an ideal choice for QGIS users to view the layers +provided by the View Server. + +.. _loading_arcmap: + +Loading in ArcMap 10 +-------------------- + +Any of the above described layers can be loaded in ArcMap 10, but the inbuilt +time support does have the same issues as in the case of QGIS 3.14 Time +Controller. Therefore a custom `TIME` parameter has to be added manually in the +Layer settings. + +To add a WMS/WMTS layer, click on the `Add Layer` symbol. Select the `Add GIS +Server` option and follow up with either `Add WMS Server` or `Add WMTS Server`, +as can be seen on :numref:`fig_wms_arcgis0`, :numref:`fig_wms_arcgis1`, and +:numref:`fig_wms_arcgis2`. + +.. _fig_wms_arcgis0: +.. figure:: images/arcgis_0.* + :alt: ArcMap Add Data Buttton + + *ArcMap Add Data Buttton* + + +.. _fig_wms_arcgis1: +.. figure:: images/arcgis_1.* + :alt: ArcMap Choose GIS Servers + + *Choose GIS Servers* + +.. _fig_wms_arcgis2: +.. figure:: images/arcgis_2.* + :alt: ArcMap WMS/WMTS loading in ArcMap + + *WMS/WMTS loading in ArcMap* + +Then user is required to insert the service `URL` and click `Get Layers`. If the +service is secured with some sort of authentication, the user must provide a +valid `username` and `password` on the login attempt. After they are supplied, +the list of available layers appears, as seen on :numref:`fig_wms_arcgis3`, and +:numref:`fig_wms_arcgis4`. + +.. _fig_wms_arcgis3: +.. figure:: images/arcgis_3.* + :alt: ArcMap add WMS Server panel + + *ArcMap add WMS Server panel* + +.. _fig_wms_arcgis4: +.. figure:: images/arcgis_4.* + :alt: ArcMap add WMS Server panel + + *ArcMap add WMS Server panel* + +This layer then gets added into the list of layers to choose from for adding +into the workspace, as seen on :numref:`fig_wms_arcgis5`. A user than follows +into the nested structure of sublayers to select the one to explore, as seen on +:numref:`fig_wms_arcgis6`, :numref:`fig_wms_arcgis7`, and +:numref:`fig_wms_arcgis8`. + +.. _fig_wms_arcgis5: +.. figure:: images/arcgis_5.* + :alt: Select layers + + *Select layers* + +.. _fig_wms_arcgis6: +.. figure:: images/arcgis_6.* + :alt: Select layers + + *Select layers* + +.. _fig_wms_arcgis7: +.. figure:: images/arcgis_7.* + :alt: Select layers + + *Select layers* + +.. _fig_wms_arcgis8: +.. figure:: images/arcgis_8.* + :alt: Select sublayers + + *Select sublayers* + +In order to limit the displayed time interval, a user needs to add the `TIME` +parameter to the requests by accessing `Layer Properties`, expand the Tab +**Parameters** and enter the time interval as can be seen on +:numref:`fig_wms_arcgis9`. If the TIME parameter was already added in the +previous step of adding a WMS Server, it can not be added for the request twice +and needs to be changed for the whole Data Source. + +.. _fig_wms_arcgis9: +.. figure:: images/arcgis_9.* + :alt: Set custom TIME parameter + + *Set custom TIME parameter* + +Continue reading to learn about the provided :ref:`searching` services. diff --git a/documentation/user-guide/web-client.rst b/documentation/user-guide/web-client.rst new file mode 100644 index 0000000000000000000000000000000000000000..0e1dcc55f3bc6c6382fb5335f5b098b836ff15f8 --- /dev/null +++ b/documentation/user-guide/web-client.rst @@ -0,0 +1,406 @@ +.. _web_client: + +Web Client +========== + +The most comfortable way to exploit the functionalities of the View Server (VS) +is via the built-in `web client `_. This client +is based on the open source `eoxc `_ library and +provides an intuitively to use interactive graphical user interface running in +the user's web browser. + +Interactive Tutorial +-------------------- + +When the web client is opened for the first time a short interactive tutorial +starts as shown in :numref:`fig_webclient_tutorial` below. It guides the user +through the main features and elements. + +.. _fig_webclient_tutorial: +.. figure:: images/webclient_tutorial_01_start.* + :alt: Web Client Tutorial - Start + + *Web Client Tutorial - Start* + +The tutorial can be started again +anytime by clicking the *book* icon in the bottom right of the window. + +.. figure:: images/webclient_tutorial-button.* + :alt: Web Client Tutorial Button + + *Web Client Tutorial Button* + +Map +--- + +The main map view is for visualization and interactively browsing through +available satellite data. Navigating to the area of interest is done by: + +* *panning*: left click + drag or one finger drag and +* *zooming*: mouse wheel scroll, double click, two finger pinch, or the plus + and minus icons in the bottom right corner of the map. + +.. figure:: images/webclient_tutorial_02_map.* + :alt: Web Client Tutorial - Map + + *Web Client Tutorial - Map* + +While hovering over footprints on the map, the respective product(s) are +highlighted in the *map*, *timeslider*, and the *Search Results* menu. The same +behavior is applied vice versa while hovering over dot groups in the +*timeslider* and individual items in the *Search Results*. + +.. figure:: images/webclient_highlighting.* + :alt: Web Client - 3-way highlighting + + *Web Client - 3-way highlighting* + +Clicking on a footprint in the map followed by clicking on the *plus* or *minus* +button can be used to add or remove a product to or from the *basket*, holding +the list of products selected for download (see :ref:`weblient_download` +below). + +.. figure:: images/webclient_select-on-map.* + :alt: Web Client - Select on map + + *Web Client - Select on map* + +In the top most part of the map, there is a colored bar spanning the full width +of the web client acting as loading indicator. It makes it easier to see if the +loading of all requested map tiles is already finished or still in progress. + +.. figure:: images/webclient_loading-indicator.* + :alt: Web Client - Loading indicator + + *Web Client - Loading indicator* + +The geographic coordinates of the current cursor location can be seen as +longitude, latitude pair in the bottom left corner of the map. + +.. figure:: images/webclient_coordinates.* + :alt: Web Client - Coordinates + + *Web Client - Coordinates* + +Clicking on the *landscape map* icon in the bottom right of the map window shows +a list of available layers. Clicking on one of the layers opens a new browser +tab showing this layer as currently visible on the map by generating a *WMS* +request copying the current map parameters (zoom, bounding box, selected time, +styling, etc.). + +.. figure:: images/webclient_current-view.* + :alt: Web Client - Current view + + *Web Client - Current view* + +.. _timeslider: + +Timeslider +---------- + +The *timeslider* presents the distribution of products in time via a *bar* (see +:numref:`fig_webclient_timeslider`) or an *aggregated dots* (see +:numref:`fig_webclient_timeslider_dots`) graph. The visualization mode used +(*bars* or *dots*) is based on the total number of products available for the +shown time interval. + +.. _fig_webclient_timeslider: +.. figure:: images/webclient_tutorial_03_timeslider.* + :alt: Web Client Tutorial - Timeslider + + *Web Client Tutorial - Timeslider* + +.. _fig_webclient_timeslider_dots: +.. figure:: images/webclient_timeslider-dots.* + :alt: Web Client - Timeslider dots + + *Web Client - Timeslider dots* + +The interactive *timeslider* can be used in the following ways: + +* *panning*: left click + drag or one finger drag on the bottom area (below the + horizontal line) or left and right arrow icons on the sides +* *zooming*: mouse wheel scroll, two finger pinch, or plus and minus icons in + the middle +* *reloading*: spinner icon in the middle +* *selecting a time interval*: left click + drag or one finger drag in the upper + area (above the horizontal line) or click on a histogram bucket +* *showing the total number of available products in a histogram bucket*: hover + over the histogram bucket +* *showing the footprints of all products, the first product id, and the total + number of additional products in a dot group*: hover over the dot group +* *showing start and end of the current selection*: left click + hold on + selection rectangle +* *zooming the map to a combined minimum bounding box of all products in a dot + group*: click on the dot group + +There are currently two ways how timeslider works connected to spatial filters: + +* Spatially un-aware: Always shows time distribution of all available products. +* Spatially aware: Takes currently set spatial filter (or current map view) in account when displaying histograms. That way, time distribution of products over a specific area can be evaluated. + +In order to change between the modes, use the location switch in top right edge of the timeslider panel. + +.. _fig_webclient_timeslider_toggle: +.. figure:: images/webclient_timeslider-toggle.* + :alt: Web Client - Timeslider histogram spatial toggle + + *Web Client - Timeslider spatial toggle* + +.. _search_results: + +Search Results +-------------- + +The panel in the right part of the map window has a tab showing the *Search +Results* grouped by enabled layer(s). The search is defined by the currently +visible geographic area or spatial filter, selected time interval, and +additional filters if set. Items in the list are sorted as the catalog returns +them which typically is from the oldest to the newest. + +.. figure:: images/webclient_tutorial_04_search-results.* + :alt: Web Client Tutorial - Search Results + + *Web Client Tutorial - Search Results* + +Searching can be disabled and enabled again for each displayed layer +individually by clicking on the large button labeled *N layer(s) selected to +show n/n* and clicking on the corresponding layer entry (see +:numref:`fig_webclient_toggle_search`) or by clicking on the *Search* toggle +(see :numref:`fig_webclient_toggle_search_single`) if only a single layer is +configured. + +.. _fig_webclient_toggle_search: +.. figure:: images/webclient_toggle-search.* + :alt: Web Client - Toggle search + + *Web Client - Toggle search* + +.. _fig_webclient_toggle_search_single: +.. figure:: images/webclient_toggle-search-single.* + :alt: Web Client - Toggle search single layer + + *Web Client - Toggle search single layer* + +Metadata and alternative visualizations of individual products can be inspected +by hovering over an item and hitting the *info* icon either in the search +results (see :numref:`fig_webclient_info1`) or on the map (see +:numref:`fig_webclient_info2`). That opens a new overlay panel showing *details +display* (see :ref:`details_display` below). + +.. _fig_webclient_info1: +.. figure:: images/webclient_info1.* + :alt: Web Client - Details button 1 + + *Web Client - Details button 1* + +.. _fig_webclient_info2: +.. figure:: images/webclient_info2.* + :alt: Web Client - Details button 2 + + *Web Client - Details button 2* + +By clicking on individual items, they can be inserted to or removed from the +*basket* holding the list of items selected for download. + +.. figure:: images/webclient_add-to-basket.* + :alt: Web Client - Add item to basket + + *Web Client - Add item to basket* + +Searching from the web client is throttled for performance and only a limited +number of items is shown from the first search. Additional items, if available, +are shown by clicking the button labeled *Load n more*. + +The *Select all* button adds all currently listed items to the *basket*. + +The *link* button next to the layer name shows the current search query. To +download the results XML file click with the right mouse button on the link and +use the *Save Link As...* menu entry. + +.. figure:: images/webclient_search-buttons.* + :alt: Web Client - Buttons in search panel + + *Web Client - Buttons in search panel* + +Items selected for download can be viewed in the *Basket* tab (see +:numref:`fig_webclient_basket1`) or *Selected* button (see +:numref:`fig_webclient_basket2`, single layer mode) on the right panel. To +remove an item from the selection use the *minus* icon visible while hovering +over it. + +.. _fig_webclient_basket1: +.. figure:: images/webclient_basket1.* + :alt: Web Client - Basket + + *Web Client - Basket* + +.. _fig_webclient_basket2: +.. figure:: images/webclient_basket2.* + :alt: Web Client - Selected + + *Web Client - Selected* + +Filters +------- + +There are several options to add explicit filters to product queries in the +*Filters* tab in the left panel: + +* *Spatial filter*: allows drawing a Point, Rectangle, or Polygon directly on + the map. By default when no explicit spatial filter is set, the current map + window extent is used as a spatial filter. +* *Time filter*: by default the time filter, a simple time interval, is + configured via the :ref:`timeslider`. If enabled, the time filter tool allows + a fine grained configuration of the time interval to use for searching. +* *Additional filters*: based on the configuration provided by the operator + additional filters for each layer based on available metadata like "Cloud + Coverage" might be available. + +.. figure:: images/webclient_tutorial_05_filters.* + :alt: Web Client Tutorial - Filters + + *Web Client Tutorial - Filters* + +Map Layers +---------- + +The *Layers* tab in the left panel enables configuring *Overlays*, *Layers*, +and *Base Layers* to be shown. + +.. figure:: images/webclient_tutorial_06_map-layers.* + :alt: Web Client Tutorial - Map Layers + + *Web Client Tutorial - Map Layers* + +*Base Layers* provide various backgrounds whereas *Overlays* are text labels or +other information like footprints to view on top of products for easier +orientation on the *map*. They can be enabled and disabled individually as +needed via the *eye* icon next to their name. + +*Layers* correspond to product collections or datasets. Searching is performed +on enabled layers. + +The order of *layers* can be changed by dragging the arrows next to each layer +up and down. This influences the order of the rendering on the *map* as well as +in the *timeslider*. + +The visualization (styling) of *layers* can be changed by clicking on the *paint +brush* icon next to the layer name. The displayed overlay enables setting the +layer's opacity and selecting one of the predefined styles for the layer. + +.. figure:: images/webclient_layer-configuration.* + :alt: Web Client - Layer configuration + + *Web Client - Layer configuration* + +.. _details_display: + +Details Display +--------------- + +In order to view details of individual products, either click on the *info* (i) +icon after clicking on the product footprint on the map or the product entry in +the *Search Results* panel. + +.. figure:: images/webclient_details_display.* + :alt: Web Client Details Display panel + + *Web Client Details Display* + +An overlay display will open showing the product metadata as well as the product +itself on a map. The product metadata contains links to visualize and download +the metadata and product. Those links are OGC standards compliant service +requests for OpenSearch (see chapter :ref:`searching`), WMS (see chapter +:ref:`viewing`), WCS, and DSEO (both see chapter :ref:`downloading`). The +chapter :ref:`sample_requests` provides sample requests for all those services. + +The following links are provided: + +* *Metadata ATOM*: download the product's metadata in ATOM format +* *Download Package*: download the entire product as stored on object storage +* *WMS: GetCapabilities*: get the description of the WMS for the individual + product for example for inclusion in QGIS +* *WMS: GetMap*: visualize the individual product using WMS +* *WCS: DescribeEOCoverageSet* or *WCS: DescribeCoverage*: get the description + of the product's coverage in WCS +* *WCS: GetCoverage*: download the product as coverage potentially after + adjusting the URL parameters + +The visualization (opacity and style) of the product layer for this map window +can be changed via the *Visualization* button in the top left corner. + +The product itself can be added to the *basket* by ticking the check box in the +bottom right of the panel. + +.. _weblient_download: + +Download +-------- + +Several methods for downloading products or only parts of them are exploited by +the web client. + +After putting one or more products in the *basket* or selecting them for +download, they are listed in the right panel, either on the *Search Results* tab +via the *Selected* button or on the *Basket* tab (see section +:ref:`search_results` particularly :numref:`fig_webclient_basket1` and +:numref:`fig_webclient_basket2` above). + +To download a full product (image and linked metadata) or a full coverage (just +the image), the *Details Display* of the respective product can be used (see +chapter :ref:`details_display` above). + +To download all selected products in parallel the *Download* or *Download as* +buttons on the right panel of the web client can be used. + +.. figure:: images/webclient_download-buttons.* + :alt: Web Client - Download multiple products + + *Web Client - Download multiple products* + +* *Download as*: offers a *Download as URL-List* option which downloads a *.txt* + file with a list of download links (*Get Coverage* requests) and a *Download + as Metalink* option allowing for possibly increased speed via segment + downloading by metalink-aware software. +* *Download*: opens an overlay panel allowing to specify download parameters + for subsetting, projection, format, interpolation, and resolution or scaling + as shown in :numref:`webclient_download_multiple` below. When multiple + products are to be downloaded, the configured options are applied to all of + them. Clicking the *Download* button confirms the selected options and + initiates the downloads. Note, the browser might open multiple confirmation + dialogs or issue a warning. Besides, modern browsers have a limit of 6 + concurrent connections. Popups might need to be explicitly allowed for this + site. + +.. _webclient_download_multiple: +.. figure:: images/webclient_download_multiple.* + :alt: Web Client - Download multiple products via WCS + + *Web Client - Download multiple products via WCS* + +Saving the session +------------------ + +The web client allows to save some parts of its current state by setting the +URL search parameters. That enables users to revisit the same view in time and +space later on or share the direct link with someone, overruling the default +client setup. Please note that certain characters are automatically URL encoded, +as URLs sent over the Internet only use the ASCII character set. This means that +if user manually enters for example **:** sign, while entering the website, +it will automatically transform into **%3A**, making the URL less readable. + +* Pre-set the map position by setting **x**, **y** as coordinates of the center + of the map and the zoom with the **z** parameter. `Map position example + `_. +* The time selection is set by `timestart` and `timeend`, and should be entered + as ISO 8601 UTC time format without milliseconds. `Time selection example + `_. +* In case only one main layer is configured (like in the Emergency collection), + additionally custom filters can be saved via their name. In order to + investigate this option, the fastest way is to use the filter manually within + the client and copy/modify the URL parameters. `Filter example + `_. + +The following chapters describe the services offered for :ref:`viewing`, +:ref:`searching`, as well as :ref:`downloading`.