diff --git a/README.md b/README.md index 5eac58361ab188aa0cbc9c71760af7c15e30d926..734ddd8ac41799920d6ecdd4a93fa5e39590f8a9 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,13 @@ The following services are defined via docker compose files. * provides the endpoint for external access * configured via docker labels +### shibauth + +* based on the external unicon/shibboleth-sp:3.0.4 Apache + Shibboleth SP3 image +* provides authentication and authorization via SAML2 +* docker configuration files set access control rules +* traefik labels determine which services are protected via Shib + ### database * based on external postgis:10 image @@ -226,6 +233,18 @@ docker secret create BASIC_AUTH_USERS_AUTH auth_list.txt docker secret create BASIC_AUTH_USERS_APIAUTH auth_list_api.txt ``` +In case **shibauth** service will be used, for production deployment, two more secrets need to be created for each stack, where **shibauth** is deployed. These ensure that the SP is recognized and its identity confirmed by the IDP. They are configured as **stack-name-capitalized_SHIB_KEY** and **stack-name-capitalized_SHIB_CERT**. In order to create them, use the attached **keygen.sh** command-line tool in */config* folder. +```bash +SPURL="https://emg.pass.copernicus.eu" # service initial access point made accessible by traefik +./config/keygen.sh -h $SPURL -y 20 -e https://$SPURL/shibboleth -n sp-signing -f +docker secret create EMG_SHIB_CERT sp-signing-cert.pem +docker secret create EMG_SHIB_KEY sp-signing-key.pem +``` +Additionally a docker config `idp-metadata` containing the metadata of the used IDP needs to be added: +```bash +docker config create idp_metadata idp-metadata-received.xml +``` + Deploy the stack in dev environment: ``` docker stack deploy -c docker-compose.vhr18.yml -c docker-compose.vhr18.dev.yml -c docker-compose.logging.yml -c docker-compose.logging.dev.yml vhr18-pvs # start VHR_IMAGE_2018 stack in dev mode, for example to use local sources diff --git a/config/shibboleth/attribute-map.xml b/config/shibboleth/attribute-map.xml new file mode 100755 index 0000000000000000000000000000000000000000..ddc1eeef9aebada40cd6b76e958c7127c1946cec --- /dev/null +++ b/config/shibboleth/attribute-map.xml @@ -0,0 +1,4 @@ + + + + diff --git a/config/shibboleth/dem-ac-cache.xml b/config/shibboleth/dem-ac-cache.xml new file mode 100644 index 0000000000000000000000000000000000000000..29cc048b186121be86f341375cebf85342976565 --- /dev/null +++ b/config/shibboleth/dem-ac-cache.xml @@ -0,0 +1,9 @@ + + + .+ + + Copernicus_Services Union_Inst Union_Research_Projects_space Union_Research_Projects_non-space TP_Data_Providers Data_Access_Services Ops_Space_Inf_Services Public_Auth Int_Org_NGO + + + diff --git a/config/shibboleth/dem-ac.xml b/config/shibboleth/dem-ac.xml new file mode 100644 index 0000000000000000000000000000000000000000..ef16ad42fa237e77b916f2e8bda6c78f8af2ca5d --- /dev/null +++ b/config/shibboleth/dem-ac.xml @@ -0,0 +1,9 @@ + + + .+ + + Copernicus_Services Union_Inst Union_Research_Projects_space Union_Research_Projects_non-space TP_Data_Providers Data_Access_Services Ops_Space_Inf_Services + + + diff --git a/config/shibboleth/dem-shibboleth2.xml b/config/shibboleth/dem-shibboleth2.xml new file mode 100755 index 0000000000000000000000000000000000000000..80564fad501cd585ae0f72b3d88f4ef9405e7280 --- /dev/null +++ b/config/shibboleth/dem-shibboleth2.xml @@ -0,0 +1,31 @@ + + + + + SAML2 + + SAML2 Local + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/config/shibboleth/emg-ac-cache.xml b/config/shibboleth/emg-ac-cache.xml new file mode 100644 index 0000000000000000000000000000000000000000..91372acfb7f6d0583d8048d4143fcd52dde38954 --- /dev/null +++ b/config/shibboleth/emg-ac-cache.xml @@ -0,0 +1,9 @@ + + + .+ + + Copernicus_Services Union_Inst Union_Research_Projects_space Union_Research_Projects_non-space Public_Auth + + + diff --git a/config/shibboleth/emg-ac.xml b/config/shibboleth/emg-ac.xml new file mode 100644 index 0000000000000000000000000000000000000000..91372acfb7f6d0583d8048d4143fcd52dde38954 --- /dev/null +++ b/config/shibboleth/emg-ac.xml @@ -0,0 +1,9 @@ + + + .+ + + Copernicus_Services Union_Inst Union_Research_Projects_space Union_Research_Projects_non-space Public_Auth + + + diff --git a/config/shibboleth/emg-shibboleth2.xml b/config/shibboleth/emg-shibboleth2.xml new file mode 100644 index 0000000000000000000000000000000000000000..a659df737b0c0413a0e3b030cd068e40af00cf54 --- /dev/null +++ b/config/shibboleth/emg-shibboleth2.xml @@ -0,0 +1,31 @@ + + + + + SAML2 + + SAML2 Local + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/config/shibboleth/index.html b/config/shibboleth/index.html new file mode 100644 index 0000000000000000000000000000000000000000..d1b182504b3959bb246951fad8f652018dd9572d --- /dev/null +++ b/config/shibboleth/index.html @@ -0,0 +1,11 @@ + + + + + Authentication Success + + +

Your login was successful and you were granted access to the service. + Please access the URL, which you originally requested. Proper redirection is not implemented yet.

+ + diff --git a/config/shibboleth/keygen.sh b/config/shibboleth/keygen.sh new file mode 100755 index 0000000000000000000000000000000000000000..b5378fd36f74c2c13f9c8328e79a393960fe0f60 --- /dev/null +++ b/config/shibboleth/keygen.sh @@ -0,0 +1,91 @@ +#! /bin/sh + +while getopts n:h:u:g:o:e:y:bf c + do + case $c in + u) USER=$OPTARG;; + g) GROUP=$OPTARG;; + o) OUT=$OPTARG;; + b) BATCH=1;; + f) FORCE=1;; + h) FQDN=$OPTARG;; + e) ENTITYID=$OPTARG;; + y) YEARS=$OPTARG;; + n) PREFIX=$OPTARG;; + \?) echo "keygen [-o output directory (default .)] [-u username to own keypair] [-g owning groupname] [-h hostname for cert] [-y years to issue cert] [-e entityID to embed in cert] [-n filename prefix (default 'sp')]" + exit 1;; + esac + done + +if [ -z "$OUT" ] ; then + OUT=. +fi + +if [ -z "$PREFIX" ]; then + PREFIX="sp" +fi + +if [ -n "$FORCE" ] ; then + rm $OUT/${PREFIX}-key.pem $OUT/${PREFIX}-cert.pem +fi + +if [ -s $OUT/${PREFIX}-key.pem -o -s $OUT/${PREFIX}-cert.pem ] ; then + if [ -z "$BATCH" ] ; then + echo The files $OUT/${PREFIX}-key.pem and/or $OUT/${PREFIX}-cert.pem already exist! + echo Use -f option to force recreation of keypair. + exit 2 + fi + exit 0 +fi + +if [ -z "$FQDN" ] ; then + FQDN=`hostname` +fi + +if [ -z "$YEARS" ] ; then + YEARS=10 +fi + +DAYS=`expr $YEARS \* 365` + +if [ -z "$ENTITYID" ] ; then + ALTNAME=DNS:$FQDN +else + ALTNAME=DNS:$FQDN,URI:$ENTITYID +fi + +SSLCNF=$OUT/${PREFIX}-cert.cnf +cat >$SSLCNF < /dev/null +fi +rm $SSLCNF + +if [ -s $OUT/${PREFIX}-key.pem -a -n "$USER" ] ; then + chown $USER $OUT/${PREFIX}-key.pem $OUT/${PREFIX}-cert.pem +fi + +if [ -s $OUT/${PREFIX}-key.pem -a -n "$GROUP" ] ; then + chgrp $GROUP $OUT/${PREFIX}-key.pem $OUT/${PREFIX}-cert.pem +fi diff --git a/config/shibboleth/native.logger b/config/shibboleth/native.logger new file mode 100644 index 0000000000000000000000000000000000000000..1a854391ae2d4e2386f10c706d4736a423a432d6 --- /dev/null +++ b/config/shibboleth/native.logger @@ -0,0 +1,41 @@ +# set overall behavior +log4j.rootCategory=INFO, native_log + +# fairly verbose for DEBUG, so generally leave at WARN/INFO +log4j.category.XMLTooling.XMLObject=WARN +log4j.category.XMLTooling.KeyInfoResolver=WARN +log4j.category.Shibboleth.IPRange=WARN +log4j.category.Shibboleth.PropertySet=WARN + +# raise for low-level tracing of SOAP client HTTP/SSL behavior +log4j.category.XMLTooling.libcurl=WARN + +# useful categories to tune independently: +# +# tracing of SAML messages and security policies +#log4j.category.OpenSAML.MessageDecoder=DEBUG +#log4j.category.OpenSAML.MessageEncoder=DEBUG +#log4j.category.OpenSAML.SecurityPolicyRule=DEBUG +# interprocess message remoting +#log4j.category.Shibboleth.Listener=DEBUG +# mapping of requests to applicationId +#log4j.category.Shibboleth.RequestMapper=DEBUG +# high level session cache operations +#log4j.category.Shibboleth.SessionCache=DEBUG +# persistent storage and caching +#log4j.category.XMLTooling.StorageService=DEBUG + +# define the appender + +log4j.appender.native_log=org.apache.log4j.RollingFileAppender +log4j.appender.native_log.fileName=/dev/stdout +log4j.appender.native_log.maxFileSize=0 +log4j.appender.native_log.maxBackupIndex=0 +log4j.appender.native_log.layout=org.apache.log4j.PatternLayout +log4j.appender.native_log.layout.ConversionPattern=sp-native %d{%Y-%m-%d %H:%M:%S} %p %c %x: %m%n + +#log4j.appender.warn_log=org.apache.log4j.RollingFileAppender +#log4j.appender.warn_log.fileName=/var/log/shibboleth-www/native_warn.log +#log4j.appender.warn_log.layout=org.apache.log4j.PatternLayout +#log4j.appender.warn_log.layout.ConversionPattern=%d{%Y-%m-%d %H:%M:%S} %p %c %x: %m%n +#log4j.appender.warn_log.threshold=WARN diff --git a/config/shibboleth/shib-apache.conf b/config/shibboleth/shib-apache.conf new file mode 100755 index 0000000000000000000000000000000000000000..8e5d486bda696885b9cb23f703aaff37b458f4f2 --- /dev/null +++ b/config/shibboleth/shib-apache.conf @@ -0,0 +1,41 @@ +LoadModule mod_shib /usr/lib64/shibboleth/mod_shib_24.so +ShibCompatValidUser On +UseCanonicalName On +DocumentRoot "/var/www/html" + + + SetHandler shib + + + + PassEnv APACHE_SERVERNAME + ServerName "${APACHE_SERVERNAME}" + + + Require valid-user + AuthType Basic + AuthBasicProvider file + AuthName "/secure" + AuthUserFile /run/secrets/BASIC_AUTH_USERS_AUTH + + + AuthType shibboleth + ShibRequestSetting requireSession 1 + Require shib-plugin /etc/shibboleth/pass-ac.xml + + + + + Require valid-user + AuthType Basic + AuthBasicProvider file + AuthName "/secure" + AuthUserFile /run/secrets/BASIC_AUTH_USERS_AUTH + + + AuthType shibboleth + ShibRequestSetting requireSession 1 + Require shib-plugin /etc/shibboleth/pass-ac-cache.xml + + + diff --git a/config/shibboleth/shibd.logger b/config/shibboleth/shibd.logger new file mode 100644 index 0000000000000000000000000000000000000000..909609dfcbb274cac05f838129cfc49e1ae2fd37 --- /dev/null +++ b/config/shibboleth/shibd.logger @@ -0,0 +1,75 @@ +# set overall behavior +log4j.rootCategory=INFO, shibd_log, warn_log + +# fairly verbose for DEBUG, so generally leave at INFO +log4j.category.XMLTooling.XMLObject=INFO +log4j.category.XMLTooling.KeyInfoResolver=INFO +log4j.category.Shibboleth.IPRange=INFO +log4j.category.Shibboleth.PropertySet=INFO + +# raise for low-level tracing of SOAP client HTTP/SSL behavior +log4j.category.XMLTooling.libcurl=INFO + +# useful categories to tune independently: +# +# tracing of SAML messages and security policies +#log4j.category.OpenSAML.MessageDecoder=DEBUG +#log4j.category.OpenSAML.MessageEncoder=DEBUG +#log4j.category.OpenSAML.SecurityPolicyRule=DEBUG +#log4j.category.XMLTooling.SOAPClient=DEBUG +# interprocess message remoting +#log4j.category.Shibboleth.Listener=DEBUG +# mapping of requests to applicationId +#log4j.category.Shibboleth.RequestMapper=DEBUG +# high level session cache operations +#log4j.category.Shibboleth.SessionCache=DEBUG +# persistent storage and caching +#log4j.category.XMLTooling.StorageService=DEBUG + +# logs XML being signed or verified if set to DEBUG +log4j.category.XMLTooling.Signature.Debugger=INFO, sig_log +log4j.additivity.XMLTooling.Signature.Debugger=false +log4j.ownAppenders.XMLTooling.Signature.Debugger=true + +# the tran log blocks the "default" appender(s) at runtime +# Level should be left at INFO for this category +log4j.category.Shibboleth-TRANSACTION=INFO, tran_log +log4j.additivity.Shibboleth-TRANSACTION=false +log4j.ownAppenders.Shibboleth-TRANSACTION=true + +# uncomment to suppress particular event types +#log4j.category.Shibboleth-TRANSACTION.AuthnRequest=WARN +#log4j.category.Shibboleth-TRANSACTION.Login=WARN +#log4j.category.Shibboleth-TRANSACTION.Logout=WARN + +# define the appenders + +log4j.appender.shibd_log=org.apache.log4j.RollingFileAppender +log4j.appender.shibd_log.fileName=/dev/stdout +log4j.appender.shibd_log.maxFileSize=0 +log4j.appender.shibd_log.maxBackupIndex=0 +log4j.appender.shibd_log.layout=org.apache.log4j.PatternLayout +log4j.appender.shibd_log.layout.ConversionPattern=sp-shibd %d{%Y-%m-%d %H:%M:%S} %p %c %x: %m%n + +#log4j.appender.warn_log=org.apache.log4j.RollingFileAppender +#log4j.appender.warn_log.fileName=/var/log/shibboleth/shibd_warn.log +#log4j.appender.warn_log.maxFileSize=0 +#log4j.appender.warn_log.maxBackupIndex=0 +#log4j.appender.warn_log.layout=org.apache.log4j.PatternLayout +#log4j.appender.warn_log.layout.ConversionPattern=%d{%Y-%m-%d %H:%M:%S} %p %c %x: %m%n +#log4j.appender.warn_log.threshold=WARN + +log4j.appender.tran_log=org.apache.log4j.RollingFileAppender +log4j.appender.tran_log.fileName=/dev/stdout +log4j.appender.tran_log.maxFileSize=0 +log4j.appender.tran_log.maxBackupIndex=0 +log4j.appender.tran_log.layout=org.apache.log4j.PatternLayout +log4j.appender.tran_log.layout.ConversionPattern=sp-transaction %d{%Y-%m-%d %H:%M:%S} %p %c %x: %m%n + +log4j.appender.sig_log=org.apache.log4j.FileAppender +log4j.appender.sig_log.fileName=/dev/stdout +log4j.appender.sig_log.maxFileSize=0 +log4j.appender.sig_log.maxBackupIndex=0 +log4j.appender.sig_log.layout=org.apache.log4j.PatternLayout +log4j.appender.sig_log.layout.ConversionPattern=sp-signature %m + diff --git a/config/shibboleth/vhr18-ac-cache.xml b/config/shibboleth/vhr18-ac-cache.xml new file mode 100644 index 0000000000000000000000000000000000000000..6a1528120ba8f56d59220d3d9a4cd4948044dafa --- /dev/null +++ b/config/shibboleth/vhr18-ac-cache.xml @@ -0,0 +1,9 @@ + + + .+ + + Copernicus_Services Union_Inst Union_Research_Projects_space Union_Research_Projects_non-space Public_Auth Int_Org_NGO Public + + + diff --git a/config/shibboleth/vhr18-ac.xml b/config/shibboleth/vhr18-ac.xml new file mode 100644 index 0000000000000000000000000000000000000000..2a0fb352d37a80065fc5b9100b99221d91b111ab --- /dev/null +++ b/config/shibboleth/vhr18-ac.xml @@ -0,0 +1,9 @@ + + + .+ + + Copernicus_Services Union_Inst Union_Research_Projects_space Union_Research_Projects_non-space Public_Auth Int_Org_NGO + + + diff --git a/config/shibboleth/vhr18-shibboleth2.xml b/config/shibboleth/vhr18-shibboleth2.xml new file mode 100644 index 0000000000000000000000000000000000000000..c063012bef8389f26d1d60466af9f72ee7754078 --- /dev/null +++ b/config/shibboleth/vhr18-shibboleth2.xml @@ -0,0 +1,31 @@ + + + + + SAML2 + + SAML2 Local + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/docker-compose.base.ops.yml b/docker-compose.base.ops.yml index 2d6ade931593401487ae790393608215522ba3f9..7b31c830c595c85fe1e0eeb81416300cc0feabdc 100644 --- a/docker-compose.base.ops.yml +++ b/docker-compose.base.ops.yml @@ -20,7 +20,7 @@ services: environment: HTTP_PROXY: "http://172.30.252.68:3128" HTTPS_PROXY: "http://172.30.252.68:3128" - NO_PROXY: "172.0.0.0/8,192.168.0.0/16,10.0.0.0/8" + NO_PROXY: "172.0.0.0/8,192.168.0.0/16,10.0.0.0/8,shibauth" deploy: placement: constraints: [node.role == manager] @@ -47,4 +47,4 @@ secrets: BASIC_AUTH_USERS_APIAUTH: external: true BASIC_AUTH_USERS_AUTH: - external: true + external: true \ No newline at end of file diff --git a/docker-compose.dem.ops.yml b/docker-compose.dem.ops.yml index 012016bd1a39eaa1a22fb9dc0d6b769998fa404d..0fa2a23ec7772392ed549d677a2b950bd507b92c 100644 --- a/docker-compose.dem.ops.yml +++ b/docker-compose.dem.ops.yml @@ -14,16 +14,16 @@ services: INSTANCE_DIR: "/var/www/pvs/ops/pvs_instance/" deploy: labels: - # router for basic auth based access (https) - - "traefik.http.routers.dem-renderer.rule=Host(`dem.pdas.prism.eox.at`, `a.dem.pdas.prism.eox.at`, `b.dem.pdas.prism.eox.at`, `c.dem.pdas.prism.eox.at`, `d.dem.pdas.prism.eox.at`, `e.dem.pdas.prism.eox.at`, `f.dem.pdas.prism.eox.at`, `g.dem.pdas.prism.eox.at`, `h.dem.pdas.prism.eox.at`, `dem.pass.copernicus.eu`, `a.dem.pass.copernicus.eu`, `b.dem.pass.copernicus.eu`, `c.dem.pass.copernicus.eu`, `d.dem.pass.copernicus.eu`, `e.dem.pass.copernicus.eu`, `f.dem.pass.copernicus.eu`, `g.dem.pass.copernicus.eu`, `h.dem.pass.copernicus.eu`) && PathPrefix(`/ows`, `/opensearch`, `/admin`)" - - "traefik.http.routers.dem-renderer.middlewares=auth@file,compress@file,cors@file" - - "traefik.http.routers.dem-renderer.tls=true" - - "traefik.http.routers.dem-renderer.tls.certresolver=default" - - "traefik.http.routers.dem-renderer.entrypoints=https" - # router for basic auth based access (http) - - "traefik.http.routers.dem-renderer-redirect.rule=Host(`dem.pdas.prism.eox.at`, `a.dem.pdas.prism.eox.at`, `b.dem.pdas.prism.eox.at`, `c.dem.pdas.prism.eox.at`, `d.dem.pdas.prism.eox.at`, `e.dem.pdas.prism.eox.at`, `f.dem.pdas.prism.eox.at`, `g.dem.pdas.prism.eox.at`, `h.dem.pdas.prism.eox.at`, `dem.pass.copernicus.eu`, `a.dem.pass.copernicus.eu`, `b.dem.pass.copernicus.eu`, `c.dem.pass.copernicus.eu`, `d.dem.pass.copernicus.eu`, `e.dem.pass.copernicus.eu`, `f.dem.pass.copernicus.eu`, `g.dem.pass.copernicus.eu`, `h.dem.pass.copernicus.eu`) && PathPrefix(`/ows`, `/opensearch`, `/admin`)" - - "traefik.http.routers.dem-renderer-redirect.middlewares=redirect@file" - - "traefik.http.routers.dem-renderer-redirect.entrypoints=http" + # router for shib auth based access (https) + - "traefik.http.routers.dem-renderer-shib.rule=Host(`dem.pass.copernicus.eu`, `a.dem.pass.copernicus.eu`, `b.dem.pass.copernicus.eu`, `c.dem.pass.copernicus.eu`, `d.dem.pass.copernicus.eu`, `e.dem.pass.copernicus.eu`, `f.dem.pass.copernicus.eu`, `g.dem.pass.copernicus.eu`, `h.dem.pass.copernicus.eu`) && PathPrefix(`/ows`, `/opensearch`, `/admin`)" + - "traefik.http.routers.dem-renderer-shib.middlewares=shibAuth@file,compress@file,cors@file" + - "traefik.http.routers.dem-renderer-shib.tls=true" + - "traefik.http.routers.dem-renderer-shib.tls.certresolver=default" + - "traefik.http.routers.dem-renderer-shib.entrypoints=https" + # router for shib auth based access (http) + - "traefik.http.routers.dem-renderer-redirect-shib.rule=Host(`dem.pass.copernicus.eu`, `a.dem.pass.copernicus.eu`, `b.dem.pass.copernicus.eu`, `c.dem.pass.copernicus.eu`, `d.dem.pass.copernicus.eu`, `e.dem.pass.copernicus.eu`, `f.dem.pass.copernicus.eu`, `g.dem.pass.copernicus.eu`, `h.dem.pass.copernicus.eu`) && PathPrefix(`/ows`, `/opensearch`, `/admin`)" + - "traefik.http.routers.dem-renderer-redirect-shib.middlewares=redirect@file" + - "traefik.http.routers.dem-renderer-redirect-shib.entrypoints=http" # router for referrer based access (https) - "traefik.http.routers.dem-renderer_referer.rule=Host(`dem.pdas.prism.eox.at`, `a.dem.pdas.prism.eox.at`, `b.dem.pdas.prism.eox.at`, `c.dem.pdas.prism.eox.at`, `d.dem.pdas.prism.eox.at`, `e.dem.pdas.prism.eox.at`, `f.dem.pdas.prism.eox.at`, `g.dem.pdas.prism.eox.at`, `h.dem.pdas.prism.eox.at`, `dem.pass.copernicus.eu`, `a.dem.pass.copernicus.eu`, `b.dem.pass.copernicus.eu`, `c.dem.pass.copernicus.eu`, `d.dem.pass.copernicus.eu`, `e.dem.pass.copernicus.eu`, `f.dem.pass.copernicus.eu`, `g.dem.pass.copernicus.eu`, `h.dem.pass.copernicus.eu`) && PathPrefix(`/ows`, `/opensearch`, `/admin`) && HeadersRegexp(`Referer`, `(https?://)?(panda.copernicus.eu|panda.cdsv3.eu|panda-demo.ondaprism.eu|panda-demo.copernicus.eu|cdsportal-demo.copernicus.eu|ocqc-demo.copernicus.eu|spdm-intservices.cds.esa.int|spdm-intservices-adm.cds.esa.int|dem.pdas.prism.eox.at|dem.pass.copernicus.eu)/?`)" - "traefik.http.routers.dem-renderer_referer.middlewares=compress@file,cors@file" @@ -34,6 +34,16 @@ services: - "traefik.http.routers.dem-renderer_referer-redirect.rule=Host(`dem.pdas.prism.eox.at`, `a.dem.pdas.prism.eox.at`, `b.dem.pdas.prism.eox.at`, `c.dem.pdas.prism.eox.at`, `d.dem.pdas.prism.eox.at`, `e.dem.pdas.prism.eox.at`, `f.dem.pdas.prism.eox.at`, `g.dem.pdas.prism.eox.at`, `h.dem.pdas.prism.eox.at`, `dem.pass.copernicus.eu`, `a.dem.pass.copernicus.eu`, `b.dem.pass.copernicus.eu`, `c.dem.pass.copernicus.eu`, `d.dem.pass.copernicus.eu`, `e.dem.pass.copernicus.eu`, `f.dem.pass.copernicus.eu`, `g.dem.pass.copernicus.eu`, `h.dem.pass.copernicus.eu`) && PathPrefix(`/ows`, `/opensearch`, `/admin`) && HeadersRegexp(`Referer`, `(https?://)?(panda.copernicus.eu|panda.cdsv3.eu|panda-demo.ondaprism.eu|panda-demo.copernicus.eu|cdsportal-demo.copernicus.eu|ocqc-demo.copernicus.eu|spdm-intservices.cds.esa.int|spdm-intservices-adm.cds.esa.int|dem.pdas.prism.eox.at|dem.pass.copernicus.eu)/?`)" - "traefik.http.routers.dem-renderer_referer-redirect.middlewares=redirect@file" - "traefik.http.routers.dem-renderer_referer-redirect.entrypoints=http" + # router for basic auth based access (https) + - "traefik.http.routers.dem-renderer.rule=Host(`dem.pdas.prism.eox.at`, `a.dem.pdas.prism.eox.at`, `b.dem.pdas.prism.eox.at`, `c.dem.pdas.prism.eox.at`, `d.dem.pdas.prism.eox.at`, `e.dem.pdas.prism.eox.at`, `f.dem.pdas.prism.eox.at`, `g.dem.pdas.prism.eox.at`, `h.dem.pdas.prism.eox.at`) && PathPrefix(`/ows`, `/opensearch`, `/admin`)" + - "traefik.http.routers.dem-renderer.middlewares=auth@file,compress@file,cors@file" + - "traefik.http.routers.dem-renderer.tls=true" + - "traefik.http.routers.dem-renderer.tls.certresolver=default" + - "traefik.http.routers.dem-renderer.entrypoints=https" + # router for basic auth based access (http) + - "traefik.http.routers.dem-renderer-redirect.rule=Host(`dem.pdas.prism.eox.at`, `a.dem.pdas.prism.eox.at`, `b.dem.pdas.prism.eox.at`, `c.dem.pdas.prism.eox.at`, `d.dem.pdas.prism.eox.at`, `e.dem.pdas.prism.eox.at`, `f.dem.pdas.prism.eox.at`, `g.dem.pdas.prism.eox.at`, `h.dem.pdas.prism.eox.at`) && PathPrefix(`/ows`, `/opensearch`, `/admin`)" + - "traefik.http.routers.dem-renderer-redirect.middlewares=redirect@file" + - "traefik.http.routers.dem-renderer-redirect.entrypoints=http" # general - "traefik.http.services.dem-renderer.loadbalancer.sticky=false" - "traefik.http.services.dem-renderer.loadbalancer.server.port=80" @@ -57,16 +67,16 @@ services: deploy: labels: - "traefik.http.middlewares.cache-stripprefix.stripprefix.prefixes=/cache" - # router for basic auth based access (https) - - "traefik.http.routers.dem-cache.rule=Host(`dem.pdas.prism.eox.at`, `a.dem.pdas.prism.eox.at`, `b.dem.pdas.prism.eox.at`, `c.dem.pdas.prism.eox.at`, `d.dem.pdas.prism.eox.at`, `e.dem.pdas.prism.eox.at`, `f.dem.pdas.prism.eox.at`, `g.dem.pdas.prism.eox.at`, `h.dem.pdas.prism.eox.at`, `dem.pass.copernicus.eu`, `a.dem.pass.copernicus.eu`, `b.dem.pass.copernicus.eu`, `c.dem.pass.copernicus.eu`, `d.dem.pass.copernicus.eu`, `e.dem.pass.copernicus.eu`, `f.dem.pass.copernicus.eu`, `g.dem.pass.copernicus.eu`, `h.dem.pass.copernicus.eu`) && PathPrefix(`/cache`)" - - "traefik.http.routers.dem-cache.middlewares=auth@file,cache-stripprefix,compress@file,cors@file" - - "traefik.http.routers.dem-cache.tls=true" - - "traefik.http.routers.dem-cache.tls.certresolver=default" - - "traefik.http.routers.dem-cache.entrypoints=https" - # router for basic auth based access (http) - - "traefik.http.routers.dem-cache-redirect.rule=Host(`dem.pdas.prism.eox.at`, `a.dem.pdas.prism.eox.at`, `b.dem.pdas.prism.eox.at`, `c.dem.pdas.prism.eox.at`, `d.dem.pdas.prism.eox.at`, `e.dem.pdas.prism.eox.at`, `f.dem.pdas.prism.eox.at`, `g.dem.pdas.prism.eox.at`, `h.dem.pdas.prism.eox.at`, `dem.pass.copernicus.eu`, `a.dem.pass.copernicus.eu`, `b.dem.pass.copernicus.eu`, `c.dem.pass.copernicus.eu`, `d.dem.pass.copernicus.eu`, `e.dem.pass.copernicus.eu`, `f.dem.pass.copernicus.eu`, `g.dem.pass.copernicus.eu`, `h.dem.pass.copernicus.eu`) && PathPrefix(`/cache`)" - - "traefik.http.routers.dem-cache-redirect.middlewares=redirect@file" - - "traefik.http.routers.dem-cache-redirect.entrypoints=http" + # router for shib auth based access (https) + - "traefik.http.routers.dem-cache-shib.rule=Host(`dem.pass.copernicus.eu`, `a.dem.pass.copernicus.eu`, `b.dem.pass.copernicus.eu`, `c.dem.pass.copernicus.eu`, `d.dem.pass.copernicus.eu`, `e.dem.pass.copernicus.eu`, `f.dem.pass.copernicus.eu`, `g.dem.pass.copernicus.eu`, `h.dem.pass.copernicus.eu`) && PathPrefix(`/cache`)" + - "traefik.http.routers.dem-cache-shib.middlewares=shibAuthCache@file,cache-stripprefix,compress@file,cors@file" + - "traefik.http.routers.dem-cache-shib.tls=true" + - "traefik.http.routers.dem-cache-shib.tls.certresolver=default" + - "traefik.http.routers.dem-cache-shib.entrypoints=https" + # router for shib auth based access (http) + - "traefik.http.routers.dem-cache-redirect-shib.rule=Host(`dem.pass.copernicus.eu`, `a.dem.pass.copernicus.eu`, `b.dem.pass.copernicus.eu`, `c.dem.pass.copernicus.eu`, `d.dem.pass.copernicus.eu`, `e.dem.pass.copernicus.eu`, `f.dem.pass.copernicus.eu`, `g.dem.pass.copernicus.eu`, `h.dem.pass.copernicus.eu`) && PathPrefix(`/cache`)" + - "traefik.http.routers.dem-cache-redirect-shib.middlewares=redirect@file" + - "traefik.http.routers.dem-cache-redirect-shib.entrypoints=http" # router for referrer based access (https) - "traefik.http.routers.dem-cache_referer.rule=Host(`dem.pdas.prism.eox.at`, `a.dem.pdas.prism.eox.at`, `b.dem.pdas.prism.eox.at`, `c.dem.pdas.prism.eox.at`, `d.dem.pdas.prism.eox.at`, `e.dem.pdas.prism.eox.at`, `f.dem.pdas.prism.eox.at`, `g.dem.pdas.prism.eox.at`, `h.dem.pdas.prism.eox.at`, `dem.pass.copernicus.eu`, `a.dem.pass.copernicus.eu`, `b.dem.pass.copernicus.eu`, `c.dem.pass.copernicus.eu`, `d.dem.pass.copernicus.eu`, `e.dem.pass.copernicus.eu`, `f.dem.pass.copernicus.eu`, `g.dem.pass.copernicus.eu`, `h.dem.pass.copernicus.eu`) && PathPrefix(`/cache`) && HeadersRegexp(`Referer`, `(https?://)?(panda.copernicus.eu|panda.cdsv3.eu|panda-demo.ondaprism.eu|panda-demo.copernicus.eu|cdsportal-demo.copernicus.eu|ocqc-demo.copernicus.eu|spdm-intservices.cds.esa.int|spdm-intservices-adm.cds.esa.int|dem.pdas.prism.eox.at|dem.pass.copernicus.eu)/?`)" - "traefik.http.routers.dem-cache_referer.middlewares=cache-stripprefix,compress@file,cors@file" @@ -77,6 +87,16 @@ services: - "traefik.http.routers.dem-cache_referer-redirect.rule=Host(`dem.pdas.prism.eox.at`, `a.dem.pdas.prism.eox.at`, `b.dem.pdas.prism.eox.at`, `c.dem.pdas.prism.eox.at`, `d.dem.pdas.prism.eox.at`, `e.dem.pdas.prism.eox.at`, `f.dem.pdas.prism.eox.at`, `g.dem.pdas.prism.eox.at`, `h.dem.pdas.prism.eox.at`, `dem.pass.copernicus.eu`, `a.dem.pass.copernicus.eu`, `b.dem.pass.copernicus.eu`, `c.dem.pass.copernicus.eu`, `d.dem.pass.copernicus.eu`, `e.dem.pass.copernicus.eu`, `f.dem.pass.copernicus.eu`, `g.dem.pass.copernicus.eu`, `h.dem.pass.copernicus.eu`) && PathPrefix(`/cache`) && HeadersRegexp(`Referer`, `(https?://)?(panda.copernicus.eu|panda.cdsv3.eu|panda-demo.ondaprism.eu|panda-demo.copernicus.eu|cdsportal-demo.copernicus.eu|ocqc-demo.copernicus.eu|spdm-intservices.cds.esa.int|spdm-intservices-adm.cds.esa.int|dem.pdas.prism.eox.at|dem.pass.copernicus.eu)/?`)" - "traefik.http.routers.dem-cache_referer-redirect.middlewares=redirect@file" - "traefik.http.routers.dem-cache_referer-redirect.entrypoints=http" + # router for basic auth based access (https) + - "traefik.http.routers.dem-cache.rule=Host(`dem.pdas.prism.eox.at`, `a.dem.pdas.prism.eox.at`, `b.dem.pdas.prism.eox.at`, `c.dem.pdas.prism.eox.at`, `d.dem.pdas.prism.eox.at`, `e.dem.pdas.prism.eox.at`, `f.dem.pdas.prism.eox.at`, `g.dem.pdas.prism.eox.at`, `h.dem.pdas.prism.eox.at`) && PathPrefix(`/cache`)" + - "traefik.http.routers.dem-cache.middlewares=auth@file,cache-stripprefix,compress@file,cors@file" + - "traefik.http.routers.dem-cache.tls=true" + - "traefik.http.routers.dem-cache.tls.certresolver=default" + - "traefik.http.routers.dem-cache.entrypoints=https" + # router for basic auth based access (http) + - "traefik.http.routers.dem-cache-redirect.rule=Host(`dem.pdas.prism.eox.at`, `a.dem.pdas.prism.eox.at`, `b.dem.pdas.prism.eox.at`, `c.dem.pdas.prism.eox.at`, `d.dem.pdas.prism.eox.at`, `e.dem.pdas.prism.eox.at`, `f.dem.pdas.prism.eox.at`, `g.dem.pdas.prism.eox.at`, `h.dem.pdas.prism.eox.at`) && PathPrefix(`/cache`)" + - "traefik.http.routers.dem-cache-redirect.middlewares=redirect@file" + - "traefik.http.routers.dem-cache-redirect.entrypoints=http" # general - "traefik.http.services.dem-cache.loadbalancer.sticky=false" - "traefik.http.services.dem-cache.loadbalancer.server.port=80" @@ -109,14 +129,24 @@ services: target: /usr/share/nginx/html/index.html deploy: labels: + # router for shib auth based access (https) + - "traefik.http.routers.dem-client-shib.rule=Host(`dem.pass.copernicus.eu`)" + - "traefik.http.routers.dem-client-shib.middlewares=shibAuthCache@file,compress@file" + - "traefik.http.routers.dem-client-shib.tls=true" + - "traefik.http.routers.dem-client-shib.tls.certresolver=default" + - "traefik.http.routers.dem-client-shib.entrypoints=https" + # router for shib auth based access (http) + - "traefik.http.routers.dem-client-redirect-shib.rule=Host(`dem.pass.copernicus.eu`)" + - "traefik.http.routers.dem-client-redirect-shib.middlewares=redirect@file" + - "traefik.http.routers.dem-client-redirect-shib.entrypoints=http" # router for basic auth based access (https) - - "traefik.http.routers.dem-client.rule=Host(`dem.pdas.prism.eox.at`, `dem.pass.copernicus.eu`)" + - "traefik.http.routers.dem-client.rule=Host(`dem.pdas.prism.eox.at`)" - "traefik.http.routers.dem-client.middlewares=auth@file,compress@file" - "traefik.http.routers.dem-client.tls=true" - "traefik.http.routers.dem-client.tls.certresolver=default" - "traefik.http.routers.dem-client.entrypoints=https" # router for basic auth based access (http) - - "traefik.http.routers.dem-client-redirect.rule=Host(`dem.pdas.prism.eox.at`, `dem.pass.copernicus.eu`)" + - "traefik.http.routers.dem-client-redirect.rule=Host(`dem.pdas.prism.eox.at`)" - "traefik.http.routers.dem-client-redirect.middlewares=redirect@file" - "traefik.http.routers.dem-client-redirect.entrypoints=http" # general @@ -141,9 +171,89 @@ services: placement: constraints: - node.labels.type == internal + shibauth: + image: unicon/shibboleth-sp:3.0.4 + environment: + APACHE_SERVERNAME: "https://dem.pass.copernicus.eu:443" + secrets: + - source: DEM_SHIB_CERT + target: SHIB_CERT + - source: DEM_SHIB_KEY + target: SHIB_KEY + - BASIC_AUTH_USERS_AUTH + deploy: + replicas: 1 + placement: + constraints: [node.role == manager] + labels: + # router for basic auth based access (https) + - "traefik.http.routers.shibauth.rule=Host(`dem.pass.copernicus.eu`, `a.dem.pass.copernicus.eu`, `b.dem.pass.copernicus.eu`, `c.dem.pass.copernicus.eu`, `d.dem.pass.copernicus.eu`, `e.dem.pass.copernicus.eu`, `f.dem.pass.copernicus.eu`, `g.dem.pass.copernicus.eu`, `h.dem.pass.copernicus.eu`) && PathPrefix(`/secure`, `/secure-cache`, `/Shibboleth.sso`)" + - "traefik.http.routers.shibauth.middlewares=compress@file,cors@file" + - "traefik.http.routers.shibauth.tls=true" + - "traefik.http.routers.shibauth.tls.certresolver=default" + - "traefik.http.routers.shibauth.entrypoints=https" + # router for basic auth based access (http) + - "traefik.http.routers.shibauth-redirect.rule=Host(`dem.pass.copernicus.eu`, `a.dem.pass.copernicus.eu`, `b.dem.pass.copernicus.eu`, `c.dem.pass.copernicus.eu`, `d.dem.pass.copernicus.eu`, `e.dem.pass.copernicus.eu`, `f.dem.pass.copernicus.eu`, `g.dem.pass.copernicus.eu`, `h.dem.pass.copernicus.eu`) && PathPrefix(`/secure`, `/secure-cache`, `/Shibboleth.sso`)" + - "traefik.http.routers.shibauth-redirect.middlewares=redirect@file" + - "traefik.http.routers.shibauth-redirect.entrypoints=http" + # general + - "traefik.http.services.shibauth.loadbalancer.sticky=false" + - "traefik.http.services.shibauth.loadbalancer.server.port=80" + - "traefik.docker.network=dem-extnet" + - "traefik.docker.lbswarm=true" + - "traefik.enable=true" + networks: + - extnet + configs: + - source: shib-access-control-conf + target: /etc/shibboleth/pass-ac.xml + - source: shib-access-control-conf-cache + target: /etc/shibboleth/pass-ac-cache.xml + - source: shib-shibboleth2 + target: /etc/shibboleth/shibboleth2.xml + - source: shib-apache + target: /etc/httpd/conf.d/shib.conf + - source: shib-attribute-map + target: /etc/shibboleth/attribute-map.xml + - source: idp-metadata + target: /etc/shibboleth/idp-metadata.xml + - source: shib-index + target: /var/www/html/secure/index.html + - source: shib-index + target: /var/www/html/secure-cache/index.html + - source: shibd-logger + target: /etc/shibboleth/shibd.logger + - source: native-logger + target: /etc/shibboleth/native.logger ingestor: image: registry.gitlab.eox.at/esa/prism/vs/pvs_ingestor:release-1.0.0-rc.2 # bumpversion networks: extnet: name: dem-extnet external: true +configs: + shib-access-control-conf: + file: ./config/shibboleth/dem-ac.xml + shib-access-control-conf-cache: + file: ./config/shibboleth/dem-ac-cache.xml + shib-shibboleth2: + file: ./config/shibboleth/dem-shibboleth2.xml + shib-apache: + file: ./config/shibboleth/shib-apache.conf + shib-attribute-map: + file: ./config/shibboleth/attribute-map.xml + shib-index: + file: ./config/shibboleth/index.html + native-logger: + file: ./config/shibboleth/native.logger + shibd-logger: + file: ./config/shibboleth/shibd.logger + idp-metadata: + external: true +secrets: + DEM_SHIB_CERT: + external: true + DEM_SHIB_KEY: + external: true + BASIC_AUTH_USERS_AUTH: + external: true diff --git a/docker-compose.dem.yml b/docker-compose.dem.yml index 598bfbfc0063957c1a688bf51592224f10d2a29d..61059026ef9c5f0089ed89a1ad3d2ba85118b059 100644 --- a/docker-compose.dem.yml +++ b/docker-compose.dem.yml @@ -227,4 +227,3 @@ secrets: external: true DJANGO_PASSWORD: external: true - \ No newline at end of file diff --git a/docker-compose.emg.ops.yml b/docker-compose.emg.ops.yml index 94ec7a1840009a817f1749e534adc5233f5a13ab..98a5ba55a6493981f1f5a70c74f9c924168178c7 100644 --- a/docker-compose.emg.ops.yml +++ b/docker-compose.emg.ops.yml @@ -14,16 +14,16 @@ services: INSTANCE_DIR: "/var/www/pvs/ops/pvs_instance/" deploy: labels: - # router for basic auth based access (https) - - "traefik.http.routers.emg-renderer.rule=Host(`emg.pdas.prism.eox.at`, `a.emg.pdas.prism.eox.at`, `b.emg.pdas.prism.eox.at`, `c.emg.pdas.prism.eox.at`, `d.emg.pdas.prism.eox.at`, `e.emg.pdas.prism.eox.at`, `f.emg.pdas.prism.eox.at`, `g.emg.pdas.prism.eox.at`, `h.emg.pdas.prism.eox.at`, `emg.pass.copernicus.eu`, `a.emg.pass.copernicus.eu`, `b.emg.pass.copernicus.eu`, `c.emg.pass.copernicus.eu`, `d.emg.pass.copernicus.eu`, `e.emg.pass.copernicus.eu`, `f.emg.pass.copernicus.eu`, `g.emg.pass.copernicus.eu`, `h.emg.pass.copernicus.eu`) && PathPrefix(`/ows`, `/opensearch`, `/admin`)" - - "traefik.http.routers.emg-renderer.middlewares=auth@file,compress@file,cors@file" - - "traefik.http.routers.emg-renderer.tls=true" - - "traefik.http.routers.emg-renderer.tls.certresolver=default" - - "traefik.http.routers.emg-renderer.entrypoints=https" - # router for basic auth based access (http) - - "traefik.http.routers.emg-renderer-redirect.rule=Host(`emg.pdas.prism.eox.at`, `a.emg.pdas.prism.eox.at`, `b.emg.pdas.prism.eox.at`, `c.emg.pdas.prism.eox.at`, `d.emg.pdas.prism.eox.at`, `e.emg.pdas.prism.eox.at`, `f.emg.pdas.prism.eox.at`, `g.emg.pdas.prism.eox.at`, `h.emg.pdas.prism.eox.at`, `emg.pass.copernicus.eu`, `a.emg.pass.copernicus.eu`, `b.emg.pass.copernicus.eu`, `c.emg.pass.copernicus.eu`, `d.emg.pass.copernicus.eu`, `e.emg.pass.copernicus.eu`, `f.emg.pass.copernicus.eu`, `g.emg.pass.copernicus.eu`, `h.emg.pass.copernicus.eu`) && PathPrefix(`/ows`, `/opensearch`, `/admin`)" - - "traefik.http.routers.emg-renderer-redirect.middlewares=redirect@file" - - "traefik.http.routers.emg-renderer-redirect.entrypoints=http" + # router for shib auth based access (https) + - "traefik.http.routers.emg-renderer-shib.rule=Host(`emg.pass.copernicus.eu`, `a.emg.pass.copernicus.eu`, `b.emg.pass.copernicus.eu`, `c.emg.pass.copernicus.eu`, `d.emg.pass.copernicus.eu`, `e.emg.pass.copernicus.eu`, `f.emg.pass.copernicus.eu`, `g.emg.pass.copernicus.eu`, `h.emg.pass.copernicus.eu`) && PathPrefix(`/ows`, `/opensearch`, `/admin`)" + - "traefik.http.routers.emg-renderer-shib.middlewares=shibAuth@file,compress@file,cors@file" + - "traefik.http.routers.emg-renderer-shib.tls=true" + - "traefik.http.routers.emg-renderer-shib.tls.certresolver=default" + - "traefik.http.routers.emg-renderer-shib.entrypoints=https" + # router for shib auth based access (http) + - "traefik.http.routers.emg-renderer-redirect-shib.rule=Host(`emg.pass.copernicus.eu`, `a.emg.pass.copernicus.eu`, `b.emg.pass.copernicus.eu`, `c.emg.pass.copernicus.eu`, `d.emg.pass.copernicus.eu`, `e.emg.pass.copernicus.eu`, `f.emg.pass.copernicus.eu`, `g.emg.pass.copernicus.eu`, `h.emg.pass.copernicus.eu`) && PathPrefix(`/ows`, `/opensearch`, `/admin`)" + - "traefik.http.routers.emg-renderer-redirect-shib.middlewares=redirect@file" + - "traefik.http.routers.emg-renderer-redirect-shib.entrypoints=http" # router for referrer based access (https) - "traefik.http.routers.emg-renderer_referer.rule=Host(`emg.pdas.prism.eox.at`, `a.emg.pdas.prism.eox.at`, `b.emg.pdas.prism.eox.at`, `c.emg.pdas.prism.eox.at`, `d.emg.pdas.prism.eox.at`, `e.emg.pdas.prism.eox.at`, `f.emg.pdas.prism.eox.at`, `g.emg.pdas.prism.eox.at`, `h.emg.pdas.prism.eox.at`, `emg.pass.copernicus.eu`, `a.emg.pass.copernicus.eu`, `b.emg.pass.copernicus.eu`, `c.emg.pass.copernicus.eu`, `d.emg.pass.copernicus.eu`, `e.emg.pass.copernicus.eu`, `f.emg.pass.copernicus.eu`, `g.emg.pass.copernicus.eu`, `h.emg.pass.copernicus.eu`) && PathPrefix(`/ows`, `/opensearch`, `/admin`) && HeadersRegexp(`Referer`, `(https?://)?(panda.copernicus.eu|panda.cdsv3.eu|panda-demo.ondaprism.eu|panda-demo.copernicus.eu|cdsportal-demo.copernicus.eu|ocqc-demo.copernicus.eu|spdm-intservices.cds.esa.int|spdm-intservices-adm.cds.esa.int|emg.pdas.prism.eox.at|emg.pass.copernicus.eu)/?`)" - "traefik.http.routers.emg-renderer_referer.middlewares=compress@file,cors@file" @@ -34,6 +34,16 @@ services: - "traefik.http.routers.emg-renderer_referer-redirect.rule=Host(`emg.pdas.prism.eox.at`, `a.emg.pdas.prism.eox.at`, `b.emg.pdas.prism.eox.at`, `c.emg.pdas.prism.eox.at`, `d.emg.pdas.prism.eox.at`, `e.emg.pdas.prism.eox.at`, `f.emg.pdas.prism.eox.at`, `g.emg.pdas.prism.eox.at`, `h.emg.pdas.prism.eox.at`, `emg.pass.copernicus.eu`, `a.emg.pass.copernicus.eu`, `b.emg.pass.copernicus.eu`, `c.emg.pass.copernicus.eu`, `d.emg.pass.copernicus.eu`, `e.emg.pass.copernicus.eu`, `f.emg.pass.copernicus.eu`, `g.emg.pass.copernicus.eu`, `h.emg.pass.copernicus.eu`) && PathPrefix(`/ows`, `/opensearch`, `/admin`) && HeadersRegexp(`Referer`, `(https?://)?(panda.copernicus.eu|panda.cdsv3.eu|panda-demo.ondaprism.eu|panda-demo.copernicus.eu|cdsportal-demo.copernicus.eu|ocqc-demo.copernicus.eu|spdm-intservices.cds.esa.int|spdm-intservices-adm.cds.esa.int|emg.pdas.prism.eox.at|emg.pass.copernicus.eu)/?`)" - "traefik.http.routers.emg-renderer_referer-redirect.middlewares=redirect@file" - "traefik.http.routers.emg-renderer_referer-redirect.entrypoints=http" + # router for basic auth based access (https) + - "traefik.http.routers.emg-renderer.rule=Host(`emg.pdas.prism.eox.at`, `a.emg.pdas.prism.eox.at`, `b.emg.pdas.prism.eox.at`, `c.emg.pdas.prism.eox.at`, `d.emg.pdas.prism.eox.at`, `e.emg.pdas.prism.eox.at`, `f.emg.pdas.prism.eox.at`, `g.emg.pdas.prism.eox.at`, `h.emg.pdas.prism.eox.at`) && PathPrefix(`/ows`, `/opensearch`, `/admin`)" + - "traefik.http.routers.emg-renderer.middlewares=auth@file,compress@file,cors@file" + - "traefik.http.routers.emg-renderer.tls=true" + - "traefik.http.routers.emg-renderer.tls.certresolver=default" + - "traefik.http.routers.emg-renderer.entrypoints=https" + # router for basic auth based access (http) + - "traefik.http.routers.emg-renderer-redirect.rule=Host(`emg.pdas.prism.eox.at`, `a.emg.pdas.prism.eox.at`, `b.emg.pdas.prism.eox.at`, `c.emg.pdas.prism.eox.at`, `d.emg.pdas.prism.eox.at`, `e.emg.pdas.prism.eox.at`, `f.emg.pdas.prism.eox.at`, `g.emg.pdas.prism.eox.at`, `h.emg.pdas.prism.eox.at`) && PathPrefix(`/ows`, `/opensearch`, `/admin`)" + - "traefik.http.routers.emg-renderer-redirect.middlewares=redirect@file" + - "traefik.http.routers.emg-renderer-redirect.entrypoints=http" # general - "traefik.http.services.emg-renderer.loadbalancer.sticky=false" - "traefik.http.services.emg-renderer.loadbalancer.server.port=80" @@ -57,16 +67,16 @@ services: deploy: labels: - "traefik.http.middlewares.cache-stripprefix.stripprefix.prefixes=/cache" - # router for basic auth based access (https) - - "traefik.http.routers.emg-cache.rule=Host(`emg.pdas.prism.eox.at`, `a.emg.pdas.prism.eox.at`, `b.emg.pdas.prism.eox.at`, `c.emg.pdas.prism.eox.at`, `d.emg.pdas.prism.eox.at`, `e.emg.pdas.prism.eox.at`, `f.emg.pdas.prism.eox.at`, `g.emg.pdas.prism.eox.at`, `h.emg.pdas.prism.eox.at`, `emg.pass.copernicus.eu`, `a.emg.pass.copernicus.eu`, `b.emg.pass.copernicus.eu`, `c.emg.pass.copernicus.eu`, `d.emg.pass.copernicus.eu`, `e.emg.pass.copernicus.eu`, `f.emg.pass.copernicus.eu`, `g.emg.pass.copernicus.eu`, `h.emg.pass.copernicus.eu`) && PathPrefix(`/cache`)" - - "traefik.http.routers.emg-cache.middlewares=auth@file,cache-stripprefix,compress@file,cors@file" - - "traefik.http.routers.emg-cache.tls=true" - - "traefik.http.routers.emg-cache.tls.certresolver=default" - - "traefik.http.routers.emg-cache.entrypoints=https" - # router for basic auth based access (http) - - "traefik.http.routers.emg-cache-redirect.rule=Host(`emg.pdas.prism.eox.at`, `a.emg.pdas.prism.eox.at`, `b.emg.pdas.prism.eox.at`, `c.emg.pdas.prism.eox.at`, `d.emg.pdas.prism.eox.at`, `e.emg.pdas.prism.eox.at`, `f.emg.pdas.prism.eox.at`, `g.emg.pdas.prism.eox.at`, `h.emg.pdas.prism.eox.at`, `emg.pass.copernicus.eu`, `a.emg.pass.copernicus.eu`, `b.emg.pass.copernicus.eu`, `c.emg.pass.copernicus.eu`, `d.emg.pass.copernicus.eu`, `e.emg.pass.copernicus.eu`, `f.emg.pass.copernicus.eu`, `g.emg.pass.copernicus.eu`, `h.emg.pass.copernicus.eu`) && PathPrefix(`/cache`)" - - "traefik.http.routers.emg-cache-redirect.middlewares=redirect@file" - - "traefik.http.routers.emg-cache-redirect.entrypoints=http" + # router for shib auth based access (https) + - "traefik.http.routers.emg-cache-shib.rule=Host(`emg.pass.copernicus.eu`, `a.emg.pass.copernicus.eu`, `b.emg.pass.copernicus.eu`, `c.emg.pass.copernicus.eu`, `d.emg.pass.copernicus.eu`, `e.emg.pass.copernicus.eu`, `f.emg.pass.copernicus.eu`, `g.emg.pass.copernicus.eu`, `h.emg.pass.copernicus.eu`) && PathPrefix(`/cache`)" + - "traefik.http.routers.emg-cache-shib.middlewares=shibAuthCache@file,cache-stripprefix,compress@file,cors@file" + - "traefik.http.routers.emg-cache-shib.tls=true" + - "traefik.http.routers.emg-cache-shib.tls.certresolver=default" + - "traefik.http.routers.emg-cache-shib.entrypoints=https" + # router for shib auth based access (http) + - "traefik.http.routers.emg-cache-redirect-shib.rule=Host(`emg.pass.copernicus.eu`, `a.emg.pass.copernicus.eu`, `b.emg.pass.copernicus.eu`, `c.emg.pass.copernicus.eu`, `d.emg.pass.copernicus.eu`, `e.emg.pass.copernicus.eu`, `f.emg.pass.copernicus.eu`, `g.emg.pass.copernicus.eu`, `h.emg.pass.copernicus.eu`) && PathPrefix(`/cache`)" + - "traefik.http.routers.emg-cache-redirect-shib.middlewares=redirect@file" + - "traefik.http.routers.emg-cache-redirect-shib.entrypoints=http" # router for referrer based access (https) - "traefik.http.routers.emg-cache_referer.rule=Host(`emg.pdas.prism.eox.at`, `a.emg.pdas.prism.eox.at`, `b.emg.pdas.prism.eox.at`, `c.emg.pdas.prism.eox.at`, `d.emg.pdas.prism.eox.at`, `e.emg.pdas.prism.eox.at`, `f.emg.pdas.prism.eox.at`, `g.emg.pdas.prism.eox.at`, `h.emg.pdas.prism.eox.at`, `emg.pass.copernicus.eu`, `a.emg.pass.copernicus.eu`, `b.emg.pass.copernicus.eu`, `c.emg.pass.copernicus.eu`, `d.emg.pass.copernicus.eu`, `e.emg.pass.copernicus.eu`, `f.emg.pass.copernicus.eu`, `g.emg.pass.copernicus.eu`, `h.emg.pass.copernicus.eu`) && PathPrefix(`/cache`) && HeadersRegexp(`Referer`, `(https?://)?(panda.copernicus.eu|panda.cdsv3.eu|panda-demo.ondaprism.eu|panda-demo.copernicus.eu|cdsportal-demo.copernicus.eu|ocqc-demo.copernicus.eu|spdm-intservices.cds.esa.int|spdm-intservices-adm.cds.esa.int|emg.pdas.prism.eox.at|emg.pass.copernicus.eu)/?`)" - "traefik.http.routers.emg-cache_referer.middlewares=cache-stripprefix,compress@file,cors@file" @@ -77,6 +87,16 @@ services: - "traefik.http.routers.emg-cache_referer-redirect.rule=Host(`emg.pdas.prism.eox.at`, `a.emg.pdas.prism.eox.at`, `b.emg.pdas.prism.eox.at`, `c.emg.pdas.prism.eox.at`, `d.emg.pdas.prism.eox.at`, `e.emg.pdas.prism.eox.at`, `f.emg.pdas.prism.eox.at`, `g.emg.pdas.prism.eox.at`, `h.emg.pdas.prism.eox.at`, `emg.pass.copernicus.eu`, `a.emg.pass.copernicus.eu`, `b.emg.pass.copernicus.eu`, `c.emg.pass.copernicus.eu`, `d.emg.pass.copernicus.eu`, `e.emg.pass.copernicus.eu`, `f.emg.pass.copernicus.eu`, `g.emg.pass.copernicus.eu`, `h.emg.pass.copernicus.eu`) && PathPrefix(`/cache`) && HeadersRegexp(`Referer`, `(https?://)?(panda.copernicus.eu|panda.cdsv3.eu|panda-demo.ondaprism.eu|panda-demo.copernicus.eu|cdsportal-demo.copernicus.eu|ocqc-demo.copernicus.eu|spdm-intservices.cds.esa.int|spdm-intservices-adm.cds.esa.int|emg.pdas.prism.eox.at|emg.pass.copernicus.eu)/?`)" - "traefik.http.routers.emg-cache_referer-redirect.middlewares=redirect@file" - "traefik.http.routers.emg-cache_referer-redirect.entrypoints=http" + # router for basic auth based access (https) + - "traefik.http.routers.emg-cache.rule=Host(`emg.pdas.prism.eox.at`, `a.emg.pdas.prism.eox.at`, `b.emg.pdas.prism.eox.at`, `c.emg.pdas.prism.eox.at`, `d.emg.pdas.prism.eox.at`, `e.emg.pdas.prism.eox.at`, `f.emg.pdas.prism.eox.at`, `g.emg.pdas.prism.eox.at`, `h.emg.pdas.prism.eox.at`) && PathPrefix(`/cache`)" + - "traefik.http.routers.emg-cache.middlewares=auth@file,cache-stripprefix,compress@file,cors@file" + - "traefik.http.routers.emg-cache.tls=true" + - "traefik.http.routers.emg-cache.tls.certresolver=default" + - "traefik.http.routers.emg-cache.entrypoints=https" + # router for basic auth based access (http) + - "traefik.http.routers.emg-cache-redirect.rule=Host(`emg.pdas.prism.eox.at`, `a.emg.pdas.prism.eox.at`, `b.emg.pdas.prism.eox.at`, `c.emg.pdas.prism.eox.at`, `d.emg.pdas.prism.eox.at`, `e.emg.pdas.prism.eox.at`, `f.emg.pdas.prism.eox.at`, `g.emg.pdas.prism.eox.at`, `h.emg.pdas.prism.eox.at`) && PathPrefix(`/cache`)" + - "traefik.http.routers.emg-cache-redirect.middlewares=redirect@file" + - "traefik.http.routers.emg-cache-redirect.entrypoints=http" # general - "traefik.http.services.emg-cache.loadbalancer.sticky=false" - "traefik.http.services.emg-cache.loadbalancer.server.port=80" @@ -109,14 +129,24 @@ services: target: /usr/share/nginx/html/index.html deploy: labels: + # router for shib auth based access (https) + - "traefik.http.routers.emg-client-shib.rule=Host(`emg.pass.copernicus.eu`)" + - "traefik.http.routers.emg-client-shib.middlewares=shibAuthCache@file,compress@file" + - "traefik.http.routers.emg-client-shib.tls=true" + - "traefik.http.routers.emg-client-shib.tls.certresolver=default" + - "traefik.http.routers.emg-client-shib.entrypoints=https" + # router for shib auth based access (http) + - "traefik.http.routers.emg-client-redirect-shib.rule=Host(`emg.pass.copernicus.eu`)" + - "traefik.http.routers.emg-client-redirect-shib.middlewares=redirect@file" + - "traefik.http.routers.emg-client-redirect-shib.entrypoints=http" # router for basic auth based access (https) - - "traefik.http.routers.emg-client.rule=Host(`emg.pdas.prism.eox.at`, `emg.pass.copernicus.eu`)" + - "traefik.http.routers.emg-client.rule=Host(`emg.pdas.prism.eox.at`)" - "traefik.http.routers.emg-client.middlewares=auth@file,compress@file" - "traefik.http.routers.emg-client.tls=true" - "traefik.http.routers.emg-client.tls.certresolver=default" - "traefik.http.routers.emg-client.entrypoints=https" # router for basic auth based access (http) - - "traefik.http.routers.emg-client-redirect.rule=Host(`emg.pdas.prism.eox.at`, `emg.pass.copernicus.eu`)" + - "traefik.http.routers.emg-client-redirect.rule=Host(`emg.pdas.prism.eox.at`)" - "traefik.http.routers.emg-client-redirect.middlewares=redirect@file" - "traefik.http.routers.emg-client-redirect.entrypoints=http" # general @@ -141,9 +171,89 @@ services: placement: constraints: - node.labels.type == internal + shibauth: + image: unicon/shibboleth-sp:3.0.4 + environment: + APACHE_SERVERNAME: "https://emg.pass.copernicus.eu:443" + secrets: + - source: EMG_SHIB_CERT + target: SHIB_CERT + - source: EMG_SHIB_KEY + target: SHIB_KEY + - BASIC_AUTH_USERS_AUTH + deploy: + replicas: 1 + placement: + constraints: [node.role == manager] + labels: + # router for basic auth based access (https) + - "traefik.http.routers.shibauth.rule=Host(`emg.pass.copernicus.eu`, `a.emg.pass.copernicus.eu`, `b.emg.pass.copernicus.eu`, `c.emg.pass.copernicus.eu`, `d.emg.pass.copernicus.eu`, `e.emg.pass.copernicus.eu`, `f.emg.pass.copernicus.eu`, `g.emg.pass.copernicus.eu`, `h.emg.pass.copernicus.eu`) && PathPrefix(`/secure`, `/secure-cache`, `/Shibboleth.sso`)" + - "traefik.http.routers.shibauth.middlewares=compress@file,cors@file" + - "traefik.http.routers.shibauth.tls=true" + - "traefik.http.routers.shibauth.tls.certresolver=default" + - "traefik.http.routers.shibauth.entrypoints=https" + # router for basic auth based access (http) + - "traefik.http.routers.shibauth-redirect.rule=Host(`emg.pass.copernicus.eu`, `a.emg.pass.copernicus.eu`, `b.emg.pass.copernicus.eu`, `c.emg.pass.copernicus.eu`, `d.emg.pass.copernicus.eu`, `e.emg.pass.copernicus.eu`, `f.emg.pass.copernicus.eu`, `g.emg.pass.copernicus.eu`, `h.emg.pass.copernicus.eu`) && PathPrefix(`/secure`, `/secure-cache`, `/Shibboleth.sso`)" + - "traefik.http.routers.shibauth-redirect.middlewares=redirect@file" + - "traefik.http.routers.shibauth-redirect.entrypoints=http" + # general + - "traefik.http.services.shibauth.loadbalancer.sticky=false" + - "traefik.http.services.shibauth.loadbalancer.server.port=80" + - "traefik.docker.network=emg-extnet" + - "traefik.docker.lbswarm=true" + - "traefik.enable=true" + networks: + - extnet + configs: + - source: shib-access-control-conf + target: /etc/shibboleth/pass-ac.xml + - source: shib-access-control-conf-cache + target: /etc/shibboleth/pass-ac-cache.xml + - source: shib-shibboleth2 + target: /etc/shibboleth/shibboleth2.xml + - source: shib-apache + target: /etc/httpd/conf.d/shib.conf + - source: shib-attribute-map + target: /etc/shibboleth/attribute-map.xml + - source: idp-metadata + target: /etc/shibboleth/idp-metadata.xml + - source: shib-index + target: /var/www/html/secure/index.html + - source: shib-index + target: /var/www/html/secure-cache/index.html + - source: shibd-logger + target: /etc/shibboleth/shibd.logger + - source: native-logger + target: /etc/shibboleth/native.logger ingestor: image: registry.gitlab.eox.at/esa/prism/vs/pvs_ingestor:release-1.0.0-rc.2 # bumpversion networks: extnet: name: emg-extnet external: true +configs: + shib-access-control-conf: + file: ./config/shibboleth/emg-ac.xml + shib-access-control-conf-cache: + file: ./config/shibboleth/emg-ac-cache.xml + shib-shibboleth2: + file: ./config/shibboleth/emg-shibboleth2.xml + shib-apache: + file: ./config/shibboleth/shib-apache.conf + shib-attribute-map: + file: ./config/shibboleth/attribute-map.xml + shib-index: + file: ./config/shibboleth/index.html + native-logger: + file: ./config/shibboleth/native.logger + shibd-logger: + file: ./config/shibboleth/shibd.logger + idp-metadata: + external: true +secrets: + EMG_SHIB_CERT: + external: true + EMG_SHIB_KEY: + external: true + BASIC_AUTH_USERS_AUTH: + external: true diff --git a/docker-compose.vhr18.ops.yml b/docker-compose.vhr18.ops.yml index 024ac46c9d69b5e81176bb0a9daf4646cb453fd0..1c8a5f4ce07ed0811ec5f9de7aa66bf812c960f4 100644 --- a/docker-compose.vhr18.ops.yml +++ b/docker-compose.vhr18.ops.yml @@ -15,16 +15,16 @@ services: deploy: replicas: 3 labels: - # router for basic auth based access (https) - - "traefik.http.routers.vhr18-renderer.rule=Host(`vhr18.pdas.prism.eox.at`, `a.vhr18.pdas.prism.eox.at`, `b.vhr18.pdas.prism.eox.at`, `c.vhr18.pdas.prism.eox.at`, `d.vhr18.pdas.prism.eox.at`, `e.vhr18.pdas.prism.eox.at`, `f.vhr18.pdas.prism.eox.at`, `g.vhr18.pdas.prism.eox.at`, `h.vhr18.pdas.prism.eox.at`, `vhr18.pass.copernicus.eu`, `a.vhr18.pass.copernicus.eu`, `b.vhr18.pass.copernicus.eu`, `c.vhr18.pass.copernicus.eu`, `d.vhr18.pass.copernicus.eu`, `e.vhr18.pass.copernicus.eu`, `f.vhr18.pass.copernicus.eu`, `g.vhr18.pass.copernicus.eu`, `h.vhr18.pass.copernicus.eu`) && PathPrefix(`/ows`, `/opensearch`, `/admin`)" - - "traefik.http.routers.vhr18-renderer.middlewares=auth@file,compress@file,cors@file" - - "traefik.http.routers.vhr18-renderer.tls=true" - - "traefik.http.routers.vhr18-renderer.tls.certresolver=default" - - "traefik.http.routers.vhr18-renderer.entrypoints=https" - # router for basic auth based access (http) - - "traefik.http.routers.vhr18-renderer-redirect.rule=Host(`vhr18.pdas.prism.eox.at`, `a.vhr18.pdas.prism.eox.at`, `b.vhr18.pdas.prism.eox.at`, `c.vhr18.pdas.prism.eox.at`, `d.vhr18.pdas.prism.eox.at`, `e.vhr18.pdas.prism.eox.at`, `f.vhr18.pdas.prism.eox.at`, `g.vhr18.pdas.prism.eox.at`, `h.vhr18.pdas.prism.eox.at`, `vhr18.pass.copernicus.eu`, `a.vhr18.pass.copernicus.eu`, `b.vhr18.pass.copernicus.eu`, `c.vhr18.pass.copernicus.eu`, `d.vhr18.pass.copernicus.eu`, `e.vhr18.pass.copernicus.eu`, `f.vhr18.pass.copernicus.eu`, `g.vhr18.pass.copernicus.eu`, `h.vhr18.pass.copernicus.eu`) && PathPrefix(`/ows`, `/opensearch`, `/admin`)" - - "traefik.http.routers.vhr18-renderer-redirect.middlewares=redirect@file" - - "traefik.http.routers.vhr18-renderer-redirect.entrypoints=http" + # router for shib auth based access (https) + - "traefik.http.routers.vhr18-renderer-shib.rule=Host(`vhr18.pass.copernicus.eu`, `a.vhr18.pass.copernicus.eu`, `b.vhr18.pass.copernicus.eu`, `c.vhr18.pass.copernicus.eu`, `d.vhr18.pass.copernicus.eu`, `e.vhr18.pass.copernicus.eu`, `f.vhr18.pass.copernicus.eu`, `g.vhr18.pass.copernicus.eu`, `h.vhr18.pass.copernicus.eu`) && PathPrefix(`/ows`, `/opensearch`, `/admin`)" + - "traefik.http.routers.vhr18-renderer-shib.middlewares=shibAuth@file,compress@file,cors@file" + - "traefik.http.routers.vhr18-renderer-shib.tls=true" + - "traefik.http.routers.vhr18-renderer-shib.tls.certresolver=default" + - "traefik.http.routers.vhr18-renderer-shib.entrypoints=https" + # router for shib auth based access (http) + - "traefik.http.routers.vhr18-renderer-redirect-shib.rule=Host(`vhr18.pass.copernicus.eu`, `a.vhr18.pass.copernicus.eu`, `b.vhr18.pass.copernicus.eu`, `c.vhr18.pass.copernicus.eu`, `d.vhr18.pass.copernicus.eu`, `e.vhr18.pass.copernicus.eu`, `f.vhr18.pass.copernicus.eu`, `g.vhr18.pass.copernicus.eu`, `h.vhr18.pass.copernicus.eu`) && PathPrefix(`/ows`, `/opensearch`, `/admin`)" + - "traefik.http.routers.vhr18-renderer-redirect-shib.middlewares=redirect@file" + - "traefik.http.routers.vhr18-renderer-redirect-shib.entrypoints=http" # router for referrer based access (https) - "traefik.http.routers.vhr18-renderer_referer.rule=Host(`vhr18.pdas.prism.eox.at`, `a.vhr18.pdas.prism.eox.at`, `b.vhr18.pdas.prism.eox.at`, `c.vhr18.pdas.prism.eox.at`, `d.vhr18.pdas.prism.eox.at`, `e.vhr18.pdas.prism.eox.at`, `f.vhr18.pdas.prism.eox.at`, `g.vhr18.pdas.prism.eox.at`, `h.vhr18.pdas.prism.eox.at`, `vhr18.pass.copernicus.eu`, `a.vhr18.pass.copernicus.eu`, `b.vhr18.pass.copernicus.eu`, `c.vhr18.pass.copernicus.eu`, `d.vhr18.pass.copernicus.eu`, `e.vhr18.pass.copernicus.eu`, `f.vhr18.pass.copernicus.eu`, `g.vhr18.pass.copernicus.eu`, `h.vhr18.pass.copernicus.eu`) && PathPrefix(`/ows`, `/opensearch`, `/admin`) && HeadersRegexp(`Referer`, `(https?://)?(panda.copernicus.eu|panda.cdsv3.eu|panda-demo.ondaprism.eu|panda-demo.copernicus.eu|cdsportal-demo.copernicus.eu|ocqc-demo.copernicus.eu|spdm-intservices.cds.esa.int|spdm-intservices-adm.cds.esa.int|vhr18.pdas.prism.eox.at|vhr18.pass.copernicus.eu)/?`)" - "traefik.http.routers.vhr18-renderer_referer.middlewares=compress@file,cors@file" @@ -35,6 +35,16 @@ services: - "traefik.http.routers.vhr18-renderer_referer-redirect.rule=Host(`vhr18.pdas.prism.eox.at`, `a.vhr18.pdas.prism.eox.at`, `b.vhr18.pdas.prism.eox.at`, `c.vhr18.pdas.prism.eox.at`, `d.vhr18.pdas.prism.eox.at`, `e.vhr18.pdas.prism.eox.at`, `f.vhr18.pdas.prism.eox.at`, `g.vhr18.pdas.prism.eox.at`, `h.vhr18.pdas.prism.eox.at`, `vhr18.pass.copernicus.eu`, `a.vhr18.pass.copernicus.eu`, `b.vhr18.pass.copernicus.eu`, `c.vhr18.pass.copernicus.eu`, `d.vhr18.pass.copernicus.eu`, `e.vhr18.pass.copernicus.eu`, `f.vhr18.pass.copernicus.eu`, `g.vhr18.pass.copernicus.eu`, `h.vhr18.pass.copernicus.eu`) && PathPrefix(`/ows`, `/opensearch`, `/admin`) && HeadersRegexp(`Referer`, `(https?://)?(panda.copernicus.eu|panda.cdsv3.eu|panda-demo.ondaprism.eu|panda-demo.copernicus.eu|cdsportal-demo.copernicus.eu|ocqc-demo.copernicus.eu|spdm-intservices.cds.esa.int|spdm-intservices-adm.cds.esa.int|vhr18.pdas.prism.eox.at|vhr18.pass.copernicus.eu)/?`)" - "traefik.http.routers.vhr18-renderer_referer-redirect.middlewares=redirect@file" - "traefik.http.routers.vhr18-renderer_referer-redirect.entrypoints=http" + # router for basic auth based access (https) + - "traefik.http.routers.vhr18-renderer.rule=Host(`vhr18.pdas.prism.eox.at`, `a.vhr18.pdas.prism.eox.at`, `b.vhr18.pdas.prism.eox.at`, `c.vhr18.pdas.prism.eox.at`, `d.vhr18.pdas.prism.eox.at`, `e.vhr18.pdas.prism.eox.at`, `f.vhr18.pdas.prism.eox.at`, `g.vhr18.pdas.prism.eox.at`, `h.vhr18.pdas.prism.eox.at`) && PathPrefix(`/ows`, `/opensearch`, `/admin`)" + - "traefik.http.routers.vhr18-renderer.middlewares=auth@file,compress@file,cors@file" + - "traefik.http.routers.vhr18-renderer.tls=true" + - "traefik.http.routers.vhr18-renderer.tls.certresolver=default" + - "traefik.http.routers.vhr18-renderer.entrypoints=https" + # router for basic auth based access (http) + - "traefik.http.routers.vhr18-renderer-redirect.rule=Host(`vhr18.pdas.prism.eox.at`, `a.vhr18.pdas.prism.eox.at`, `b.vhr18.pdas.prism.eox.at`, `c.vhr18.pdas.prism.eox.at`, `d.vhr18.pdas.prism.eox.at`, `e.vhr18.pdas.prism.eox.at`, `f.vhr18.pdas.prism.eox.at`, `g.vhr18.pdas.prism.eox.at`, `h.vhr18.pdas.prism.eox.at`) && PathPrefix(`/ows`, `/opensearch`, `/admin`)" + - "traefik.http.routers.vhr18-renderer-redirect.middlewares=redirect@file" + - "traefik.http.routers.vhr18-renderer-redirect.entrypoints=http" # general - "traefik.http.services.vhr18-renderer.loadbalancer.sticky=false" - "traefik.http.services.vhr18-renderer.loadbalancer.server.port=80" @@ -57,16 +67,16 @@ services: deploy: labels: - "traefik.http.middlewares.cache-stripprefix.stripprefix.prefixes=/cache" - # router for basic auth based access (https) - - "traefik.http.routers.vhr18-cache.rule=Host(`vhr18.pdas.prism.eox.at`, `a.vhr18.pdas.prism.eox.at`, `b.vhr18.pdas.prism.eox.at`, `c.vhr18.pdas.prism.eox.at`, `d.vhr18.pdas.prism.eox.at`, `e.vhr18.pdas.prism.eox.at`, `f.vhr18.pdas.prism.eox.at`, `g.vhr18.pdas.prism.eox.at`, `h.vhr18.pdas.prism.eox.at`, `vhr18.pass.copernicus.eu`, `a.vhr18.pass.copernicus.eu`, `b.vhr18.pass.copernicus.eu`, `c.vhr18.pass.copernicus.eu`, `d.vhr18.pass.copernicus.eu`, `e.vhr18.pass.copernicus.eu`, `f.vhr18.pass.copernicus.eu`, `g.vhr18.pass.copernicus.eu`, `h.vhr18.pass.copernicus.eu`) && PathPrefix(`/cache`)" - - "traefik.http.routers.vhr18-cache.middlewares=auth@file,cache-stripprefix,compress@file,cors@file" - - "traefik.http.routers.vhr18-cache.tls=true" - - "traefik.http.routers.vhr18-cache.tls.certresolver=default" - - "traefik.http.routers.vhr18-cache.entrypoints=https" - # router for basic auth based access (http) - - "traefik.http.routers.vhr18-cache-redirect.rule=Host(`vhr18.pdas.prism.eox.at`, `a.vhr18.pdas.prism.eox.at`, `b.vhr18.pdas.prism.eox.at`, `c.vhr18.pdas.prism.eox.at`, `d.vhr18.pdas.prism.eox.at`, `e.vhr18.pdas.prism.eox.at`, `f.vhr18.pdas.prism.eox.at`, `g.vhr18.pdas.prism.eox.at`, `h.vhr18.pdas.prism.eox.at`, `vhr18.pass.copernicus.eu`, `a.vhr18.pass.copernicus.eu`, `b.vhr18.pass.copernicus.eu`, `c.vhr18.pass.copernicus.eu`, `d.vhr18.pass.copernicus.eu`, `e.vhr18.pass.copernicus.eu`, `f.vhr18.pass.copernicus.eu`, `g.vhr18.pass.copernicus.eu`, `h.vhr18.pass.copernicus.eu`) && PathPrefix(`/cache`)" - - "traefik.http.routers.vhr18-cache-redirect.middlewares=redirect@file" - - "traefik.http.routers.vhr18-cache-redirect.entrypoints=http" + # router for shib auth based access (https) + - "traefik.http.routers.vhr18-cache-shib.rule=Host(`vhr18.pass.copernicus.eu`, `a.vhr18.pass.copernicus.eu`, `b.vhr18.pass.copernicus.eu`, `c.vhr18.pass.copernicus.eu`, `d.vhr18.pass.copernicus.eu`, `e.vhr18.pass.copernicus.eu`, `f.vhr18.pass.copernicus.eu`, `g.vhr18.pass.copernicus.eu`, `h.vhr18.pass.copernicus.eu`) && PathPrefix(`/cache`)" + - "traefik.http.routers.vhr18-cache-shib.middlewares=shibAuthCache@file,cache-stripprefix,compress@file,cors@file" + - "traefik.http.routers.vhr18-cache-shib.tls=true" + - "traefik.http.routers.vhr18-cache-shib.tls.certresolver=default" + - "traefik.http.routers.vhr18-cache-shib.entrypoints=https" + # router for shib auth based access (http) + - "traefik.http.routers.vhr18-cache-redirect-shib.rule=Host(`vhr18.pass.copernicus.eu`, `a.vhr18.pass.copernicus.eu`, `b.vhr18.pass.copernicus.eu`, `c.vhr18.pass.copernicus.eu`, `d.vhr18.pass.copernicus.eu`, `e.vhr18.pass.copernicus.eu`, `f.vhr18.pass.copernicus.eu`, `g.vhr18.pass.copernicus.eu`, `h.vhr18.pass.copernicus.eu`) && PathPrefix(`/cache`)" + - "traefik.http.routers.vhr18-cache-redirect-shib.middlewares=redirect@file" + - "traefik.http.routers.vhr18-cache-redirect-shib.entrypoints=http" # router for referrer based access (https) - "traefik.http.routers.vhr18-cache_referer.rule=Host(`vhr18.pdas.prism.eox.at`, `a.vhr18.pdas.prism.eox.at`, `b.vhr18.pdas.prism.eox.at`, `c.vhr18.pdas.prism.eox.at`, `d.vhr18.pdas.prism.eox.at`, `e.vhr18.pdas.prism.eox.at`, `f.vhr18.pdas.prism.eox.at`, `g.vhr18.pdas.prism.eox.at`, `h.vhr18.pdas.prism.eox.at`, `vhr18.pass.copernicus.eu`, `a.vhr18.pass.copernicus.eu`, `b.vhr18.pass.copernicus.eu`, `c.vhr18.pass.copernicus.eu`, `d.vhr18.pass.copernicus.eu`, `e.vhr18.pass.copernicus.eu`, `f.vhr18.pass.copernicus.eu`, `g.vhr18.pass.copernicus.eu`, `h.vhr18.pass.copernicus.eu`) && PathPrefix(`/cache`) && HeadersRegexp(`Referer`, `(https?://)?(panda.copernicus.eu|panda.cdsv3.eu|panda-demo.ondaprism.eu|panda-demo.copernicus.eu|cdsportal-demo.copernicus.eu|ocqc-demo.copernicus.eu|spdm-intservices.cds.esa.int|spdm-intservices-adm.cds.esa.int|vhr18.pdas.prism.eox.at|vhr18.pass.copernicus.eu)/?`)" - "traefik.http.routers.vhr18-cache_referer.middlewares=cache-stripprefix,compress@file,cors@file" @@ -77,6 +87,16 @@ services: - "traefik.http.routers.vhr18-cache_referer-redirect.rule=Host(`vhr18.pdas.prism.eox.at`, `a.vhr18.pdas.prism.eox.at`, `b.vhr18.pdas.prism.eox.at`, `c.vhr18.pdas.prism.eox.at`, `d.vhr18.pdas.prism.eox.at`, `e.vhr18.pdas.prism.eox.at`, `f.vhr18.pdas.prism.eox.at`, `g.vhr18.pdas.prism.eox.at`, `h.vhr18.pdas.prism.eox.at`, `vhr18.pass.copernicus.eu`, `a.vhr18.pass.copernicus.eu`, `b.vhr18.pass.copernicus.eu`, `c.vhr18.pass.copernicus.eu`, `d.vhr18.pass.copernicus.eu`, `e.vhr18.pass.copernicus.eu`, `f.vhr18.pass.copernicus.eu`, `g.vhr18.pass.copernicus.eu`, `h.vhr18.pass.copernicus.eu`) && PathPrefix(`/cache`) && HeadersRegexp(`Referer`, `(https?://)?(panda.copernicus.eu|panda.cdsv3.eu|panda-demo.ondaprism.eu|panda-demo.copernicus.eu|cdsportal-demo.copernicus.eu|ocqc-demo.copernicus.eu|spdm-intservices.cds.esa.int|spdm-intservices-adm.cds.esa.int|vhr18.pdas.prism.eox.at|vhr18.pass.copernicus.eu)/?`)" - "traefik.http.routers.vhr18-cache_referer-redirect.middlewares=redirect@file" - "traefik.http.routers.vhr18-cache_referer-redirect.entrypoints=http" + # router for basic auth based access (https) + - "traefik.http.routers.vhr18-cache.rule=Host(`vhr18.pdas.prism.eox.at`, `a.vhr18.pdas.prism.eox.at`, `b.vhr18.pdas.prism.eox.at`, `c.vhr18.pdas.prism.eox.at`, `d.vhr18.pdas.prism.eox.at`, `e.vhr18.pdas.prism.eox.at`, `f.vhr18.pdas.prism.eox.at`, `g.vhr18.pdas.prism.eox.at`, `h.vhr18.pdas.prism.eox.at`) && PathPrefix(`/cache`)" + - "traefik.http.routers.vhr18-cache.middlewares=auth@file,cache-stripprefix,compress@file,cors@file" + - "traefik.http.routers.vhr18-cache.tls=true" + - "traefik.http.routers.vhr18-cache.tls.certresolver=default" + - "traefik.http.routers.vhr18-cache.entrypoints=https" + # router for basic auth based access (http) + - "traefik.http.routers.vhr18-cache-redirect.rule=Host(`vhr18.pdas.prism.eox.at`, `a.vhr18.pdas.prism.eox.at`, `b.vhr18.pdas.prism.eox.at`, `c.vhr18.pdas.prism.eox.at`, `d.vhr18.pdas.prism.eox.at`, `e.vhr18.pdas.prism.eox.at`, `f.vhr18.pdas.prism.eox.at`, `g.vhr18.pdas.prism.eox.at`, `h.vhr18.pdas.prism.eox.at`) && PathPrefix(`/cache`)" + - "traefik.http.routers.vhr18-cache-redirect.middlewares=redirect@file" + - "traefik.http.routers.vhr18-cache-redirect.entrypoints=http" # general - "traefik.http.services.vhr18-cache.loadbalancer.sticky=false" - "traefik.http.services.vhr18-cache.loadbalancer.server.port=80" @@ -101,7 +121,7 @@ services: replicas: 0 placement: constraints: - - node.labels.type == internal + - node.labels.type == internal client: image: registry.gitlab.eox.at/esa/prism/vs/pvs_client:release-1.0.0-rc.2 # bumpversion configs: @@ -109,14 +129,24 @@ services: target: /usr/share/nginx/html/index.html deploy: labels: + # router for shib auth based access (https) + - "traefik.http.routers.vhr18-client-shib.rule=Host(`vhr18.pass.copernicus.eu`)" + - "traefik.http.routers.vhr18-client-shib.middlewares=shibAuthCache@file,compress@file" + - "traefik.http.routers.vhr18-client-shib.tls=true" + - "traefik.http.routers.vhr18-client-shib.tls.certresolver=default" + - "traefik.http.routers.vhr18-client-shib.entrypoints=https" + # router for shib auth based access (http) + - "traefik.http.routers.vhr18-client-redirect-shib.rule=Host(`vhr18.pass.copernicus.eu`)" + - "traefik.http.routers.vhr18-client-redirect-shib.middlewares=redirect@file" + - "traefik.http.routers.vhr18-client-redirect-shib.entrypoints=http" # router for basic auth based access (https) - - "traefik.http.routers.vhr18-client.rule=Host(`vhr18.pdas.prism.eox.at`, `vhr18.pass.copernicus.eu`)" - - "traefik.http.routers.vhr18-client.middlewares=auth@file,compress@file" + - "traefik.http.routers.vhr18-client.rule=Host(`vhr18.pdas.prism.eox.at`)" + - "traefik.http.routers.vhr18-client.middlewares=shibAuthCache@file,compress@file" - "traefik.http.routers.vhr18-client.tls=true" - "traefik.http.routers.vhr18-client.tls.certresolver=default" - "traefik.http.routers.vhr18-client.entrypoints=https" # router for basic auth based access (http) - - "traefik.http.routers.vhr18-client-redirect.rule=Host(`vhr18.pdas.prism.eox.at`, `vhr18.pass.copernicus.eu`)" + - "traefik.http.routers.vhr18-client-redirect.rule=Host(`vhr18.pdas.prism.eox.at`)" - "traefik.http.routers.vhr18-client-redirect.middlewares=redirect@file" - "traefik.http.routers.vhr18-client-redirect.entrypoints=http" # general @@ -127,7 +157,7 @@ services: - "traefik.enable=true" placement: constraints: - - node.labels.type == external + - node.labels.type == external networks: - extnet preprocessor: @@ -141,9 +171,89 @@ services: placement: constraints: - node.labels.type == internal + shibauth: + image: unicon/shibboleth-sp:3.0.4 + environment: + APACHE_SERVERNAME: "https://vhr18.pass.copernicus.eu:443" + secrets: + - source: VHR18_SHIB_CERT + target: SHIB_CERT + - source: VHR18_SHIB_KEY + target: SHIB_KEY + - BASIC_AUTH_USERS_AUTH + deploy: + replicas: 1 + placement: + constraints: [node.role == manager] + labels: + # router for basic auth based access (https) + - "traefik.http.routers.shibauth.rule=Host(`vhr18.pass.copernicus.eu`, `a.vhr18.pass.copernicus.eu`, `b.vhr18.pass.copernicus.eu`, `c.vhr18.pass.copernicus.eu`, `d.vhr18.pass.copernicus.eu`, `e.vhr18.pass.copernicus.eu`, `f.vhr18.pass.copernicus.eu`, `g.vhr18.pass.copernicus.eu`, `h.vhr18.pass.copernicus.eu`) && PathPrefix(`/secure`, `/secure-cache`, `/Shibboleth.sso`)" + - "traefik.http.routers.shibauth.middlewares=compress@file,cors@file" + - "traefik.http.routers.shibauth.tls=true" + - "traefik.http.routers.shibauth.tls.certresolver=default" + - "traefik.http.routers.shibauth.entrypoints=https" + # router for basic auth based access (http) + - "traefik.http.routers.shibauth-redirect.rule=Host(`vhr18.pass.copernicus.eu`, `a.vhr18.pass.copernicus.eu`, `b.vhr18.pass.copernicus.eu`, `c.vhr18.pass.copernicus.eu`, `d.vhr18.pass.copernicus.eu`, `e.vhr18.pass.copernicus.eu`, `f.vhr18.pass.copernicus.eu`, `g.vhr18.pass.copernicus.eu`, `h.vhr18.pass.copernicus.eu`) && PathPrefix(`/secure`, `/secure-cache`, `/Shibboleth.sso`)" + - "traefik.http.routers.shibauth-redirect.middlewares=redirect@file" + - "traefik.http.routers.shibauth-redirect.entrypoints=http" + # general + - "traefik.http.services.shibauth.loadbalancer.sticky=false" + - "traefik.http.services.shibauth.loadbalancer.server.port=80" + - "traefik.docker.network=vhr18-extnet" + - "traefik.docker.lbswarm=true" + - "traefik.enable=true" + networks: + - extnet + configs: + - source: shib-access-control-conf + target: /etc/shibboleth/pass-ac.xml + - source: shib-access-control-conf-cache + target: /etc/shibboleth/pass-ac-cache.xml + - source: shib-shibboleth2 + target: /etc/shibboleth/shibboleth2.xml + - source: shib-apache + target: /etc/httpd/conf.d/shib.conf + - source: shib-attribute-map + target: /etc/shibboleth/attribute-map.xml + - source: idp-metadata + target: /etc/shibboleth/idp-metadata.xml + - source: shib-index + target: /var/www/html/secure/index.html + - source: shib-index + target: /var/www/html/secure-cache/index.html + - source: shibd-logger + target: /etc/shibboleth/shibd.logger + - source: native-logger + target: /etc/shibboleth/native.logger ingestor: image: registry.gitlab.eox.at/esa/prism/vs/pvs_ingestor:release-1.0.0-rc.2 # bumpversion networks: extnet: name: vhr18-extnet external: true +configs: + shib-access-control-conf: + file: ./config/shibboleth/vhr18-ac.xml + shib-access-control-conf-cache: + file: ./config/shibboleth/vhr18-ac-cache.xml + shib-shibboleth2: + file: ./config/shibboleth/vhr18-shibboleth2.xml + shib-apache: + file: ./config/shibboleth/shib-apache.conf + shib-attribute-map: + file: ./config/shibboleth/attribute-map.xml + native-logger: + file: ./config/shibboleth/native.logger + shibd-logger: + file: ./config/shibboleth/shibd.logger + shib-index: + file: ./config/shibboleth/index.html + idp-metadata: + external: true +secrets: + VHR18_SHIB_CERT: + external: true + VHR18_SHIB_KEY: + external: true + BASIC_AUTH_USERS_AUTH: + external: true diff --git a/documentation/operator-guide/access.rst b/documentation/operator-guide/access.rst new file mode 100644 index 0000000000000000000000000000000000000000..b96c281e63c29a686b2574f52f2d00a5ef5ccb54 --- /dev/null +++ b/documentation/operator-guide/access.rst @@ -0,0 +1,177 @@ +.. _access: + +Access +====== +This chapter describes general concepts of how external access to each component is provided and how authentication and authorization layer based on `Shibboleth SP3 `_. is configured. + +General overview +~~~~~~~~~~~~~~~~ +Each individual docker **stack** has its own internal network ``intnet`` where services can communicate between each other. This network is not exposed to the public and provides most of the necessary communication. Additionally external user access to some services (client, renderer, cache) is provided via external network ``extnet`` and reverse-proxy (traefik) with load balancer. + +These services can have a set of authentication and authorization rules applied both on traefik level and Shibboleth SP level. Kibana and Traefik dashboard are also accessible externally, but through a different set of default credentials. + + +Routing with traefik +~~~~~~~~~~~~~~~~~~~~ +``Reverse-proxy`` service in base stack provides central access endpoint to the VS. It exposes ports 80 and 443 for HTTP and HTTPS access. Configuration of the reverse-proxy is done on three places. + +First two are static and dynamic configuration files ``traefik.yml`` and ``traefik-dynamic.yml``. Static configuration sets up connections to providers and define the entrypoints that Traefik will listen to. Dynamic configuration defines how the requests are handled. This configuration can change and is seamlessly hot-reloaded, without any request interruption or connection loss. Third part are docker ``labels`` on individual services which Traefik provides access to. + +For example following configuration snippet enables access to certain paths of ``renderer`` service under a given hostname. It also sets externally set basic authentication and other rules via ``@file`` identifier, which references configurations from ``traefik-dynamic.yml``. + +.. code-block:: yaml + + renderer: + deploy: + labels: + # router for basic auth based access (https) + - "traefik.http.routers.vhr18-renderer.rule=Host(`vhr18.pdas.prism.eox.at`) && PathPrefix(`/ows`, `/opensearch`, `/admin`)" + - "traefik.http.routers.vhr18-renderer.middlewares=auth@file,compress@file,cors@file" + - "traefik.http.routers.vhr18-renderer.tls=true" + - "traefik.http.routers.vhr18-renderer.tls.certresolver=default" + - "traefik.http.routers.vhr18-renderer.entrypoints=https" + # general rules + - "traefik.http.services.vhr18-renderer.loadbalancer.sticky=false" + - "traefik.http.services.vhr18-renderer.loadbalancer.server.port=80" + - "traefik.docker.network=vhr18-extnet" + - "traefik.docker.lbswarm=true" + - "traefik.enable=true" + +An example of such auth@file configuration from ``traefik-dynamic.yml`` would be following snippet, where ``BASIC_AUTH_USERS_AUTH`` is referencing a docker secret configured earlier: + +.. code-block:: yaml + + http: + middlewares: + auth: + basicAuth: + realm: "PRISM View Server (PVS)" + usersFile: "/run/secrets/BASIC_AUTH_USERS_AUTH" + +Unsecured HTTP access is configured to be redirected to the HTTPS endpoint but inside the swarm among the services, only HTTP is used internally. + +Authentication and Authorization +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Authentication of access to external parts of VS is made up of two options: + +- Traefik provided basic authentication - configured as ``file@auth`` and ``file@apiAuth`` + +Here, access on such endpoint requires basic authentication credentials (username, password) to be inserted, while web browsers are usually prompted for input. After inserting valid credentials, access is granted. + +- Shibboleth Service Provider 3 + Apache 2 instance, to which requests are forwarded by `Traefik ForwardAuth middleware `_. + +Middleware delegates the authentication to Shibboleth. If Shibboleth response code is 2XX, access is granted and the original request is performed. Otherwise, the response from the Shibboleth is returned. + +In order to authenticate with Shibboleth, a user must log in with valid credentials on the side of Identity Provider (IdP), if doing so, the IdP informs the SP about successful login, accompanied by relevant user attributes and a session is created for the user. SP then saves the information about a created session into a cookie and based on user attributes can authorize access to the services. If the user was already logged in, he is automatically + +Currently setting individual authorization rules on a ``Collection`` and ``Service`` level is possible with current approach. It is yet not clearly possible to separate viewing and download, as both of these parts are handled by ``renderer`` service. + +Configuration +~~~~~~~~~~~~~ +For correct configuration of Shibboleth SP3 on a new stack, several steps need to be done. Most of these configurations are usually done in the :ref:`_initialization` step using ``pvs_starter`` tool. Still, it is advised to check following steps, understand them and change if necessary. +Briefly summarized, SP and IdP need to exchange metadata and certificates to trust each other, SP needs to know which attributes the IdP will be sending about the logged-in user and respective access-control rules are configured based on those attributes. Most of the configurations are done via docker configs defined in the docker compose + +- Create a pair of key, certificate using attached Shibboleth ``config/shibboleth/keygen.sh`` in the cloned vs repository and save them as respective docker secrets. + +.. code-block:: bash + + SPURL="https://emg.pass.copernicus.eu" # service initial access point made accessible by traefik + ./config/shibboleth/keygen.sh -h $SPURL -y 20 -e https://$SPURL/shibboleth -n sp-signing -f + docker secret create EMG_SHIB_CERT sp-signing-cert.pem + docker secret create _SHIB_KEY sp-signing-key.pem + +- Get IDP metadata and save it as a docker config. Also read the entityID of the IdP for further use in referencing it in your ``shibboleth2.xml`` configuration. +.. code-block:: bash + + docker config create idp-metadata idp-metadata-received.xml + +- Configure Apache ServerName used inside the ``shibauth`` service by modifying ``APACHE_SERVERNAME`` environment variable of corresponding ``shibauth`` service in ``docker-compose..ops.yml``. This URL should resolve to the actual service URL. + +- Modify shibboleth2.xml content by setting your "entityID" in Additionally edit the "entityID" value inside ``SSO`` element to match the IdP "entityID". Note that "entityID" does not need to resolve to an actual service URL. +.. code-block:: xml + + + SAML2 + + +- Deploy your shibauth service and exchange your SP metadata with the IdP provider and have them recognize your SP. Necessary metadata needs to be downloaded from url ``/Shibboleth.sso/Metadata``. + +- Get information about attributes provided by IdP and update ``config/shibboleth/attribute-map.xml`` by adding individual entries mapping ``name`` provided by IdP to ``id`` used by SP internally. Example configuration: + +.. code-block:: xml + + + + + + +- Create custom access rules based on these attributes and map these access controls to different internal apache routes to which Traefik ForwardAuth middleware will point. Access rules are created in ``config/shibboleth/-ac.xml``. +Example of external Access control rules configuration: + +.. code-block:: xml + + + + .+ + + Privileged_Access Public_Access + + + + +- Check configured link between Apache configuration for ``shibauth`` service, access rules, Traefik ForwardAuth middleware and per-service Traefik labels. Following simplified examples show the links in more detail: + +``APACHE_SERVERNAME`` environment variable needs to be set and same as the hostname, that Traefik will be serving as main entry point. Part of docker compose of shibauth service in ``docker-compose.emg.ops.yml``: + +.. code-block:: yaml + + services: + shibauth: + environment: + APACHE_SERVERNAME: "https://emg.pass.copernicus.eu:443" + deploy: + labels: + - "traefik.http.routers.shibauth.rule=Host(`emg.pass.copernicus.eu`) && PathPrefix(`/secure`, `/Shibboleth.sso`)" + ... + +Relevant Apache configuration in ``config/shibboleth/shib-apache.conf``, enabling Shibboleth authentication and authorization of the ``/secure`` endpoint. + +.. code-block:: apacheconf + + + SetHandler shib + + + PassEnv APACHE_SERVERNAME + ServerName "${APACHE_SERVERNAME}" + + AuthType shibboleth + ShibRequestSetting requireSession 1 + Require shib-plugin /etc/shibboleth/pass-ac.xml + + ... + +Part of Traefik ForwardAuth middleware configuration from ``traefik-dynamic.yml``, defining the internal address pointing to the ``shibauth`` service and ``/secure`` endpoint in it: + +.. code-block:: yaml + + http: + middlewares: + shibAuth: + forwardAuth: + address: http://shibauth/secure + trustForwardHeader: true + +Part of renderer service Traefik labels from ``docker-compose.emg.ops.yml``, where access through the middleware is configured. + +.. code-block:: yaml + + services: + renderer: + deploy: + labels: + # router for shib auth based access (https) + - "traefik.http.routers.emg-renderer-shib.rule=Host(`emg.pass.copernicus.eu`) && PathPrefix(`/ows`, `/opensearch`, `/admin`)" + - "traefik.http.routers.emg-renderer-shib.middlewares=shibAuth@file" diff --git a/documentation/operator-guide/configuration.rst b/documentation/operator-guide/configuration.rst index 27b245cd6b4f87d8fbf490cd2ef86e7823e1ebbb..47c1ea100fd21298edad78e39e5222a3e885d026 100644 --- a/documentation/operator-guide/configuration.rst +++ b/documentation/operator-guide/configuration.rst @@ -277,6 +277,8 @@ source/target Here, the source file storage and the target file storage are configured. This can either be a local directory or an OpenStack Swift object storage. + If Swift is used for source, download container can be left unset. In that case, + container can be inferred from the given path in format /. workdir @@ -397,7 +399,7 @@ preprocessing force_north_up - TODO + Circumvents the naming of corner names and assumes a north-up orientation of the image. tps @@ -531,5 +533,7 @@ An example of creating ``BASIC_AUTH_USERS_AUTH`` secret: htpasswd -nb user2 YyuN9bYRvBUUU6COx7itWw5qyyARus >> auth_list.txt docker secret create BASIC_AUTH_USERS_AUTH auth_list.txt +For configuration of the ``shibauth`` service, please consult a separate chapter :ref:`access`. + The next section :ref:`management` describes how an operator interacts with a deployed VS stack. diff --git a/documentation/operator-guide/images/kibana_1.png b/documentation/operator-guide/images/kibana_1.png new file mode 100644 index 0000000000000000000000000000000000000000..40efe5567ff541250adc10f91d1a23a1c96223b0 Binary files /dev/null and b/documentation/operator-guide/images/kibana_1.png differ diff --git a/documentation/operator-guide/images/kibana_2.png b/documentation/operator-guide/images/kibana_2.png new file mode 100644 index 0000000000000000000000000000000000000000..bfc60b5b3958778a3770c201d543dfc152b3cdd6 Binary files /dev/null and b/documentation/operator-guide/images/kibana_2.png differ diff --git a/documentation/operator-guide/index.rst b/documentation/operator-guide/index.rst index 249dfe69a680b8d7b323f4a674e9d65de7a1ec11..4aa4e923c76cd5fcb41800e9118066d60bcbf656 100644 --- a/documentation/operator-guide/index.rst +++ b/documentation/operator-guide/index.rst @@ -12,6 +12,7 @@ View Server - Operator Guide configuration management ingestion + access .. Indices and tables diff --git a/documentation/operator-guide/ingestion.rst b/documentation/operator-guide/ingestion.rst index 16be9b8cf531ba9df398debe962f7c313d9b7c27..456d9ca7fbbcdb51fb9168a5e052da1a9e0bf920 100644 --- a/documentation/operator-guide/ingestion.rst +++ b/documentation/operator-guide/ingestion.rst @@ -82,7 +82,7 @@ For a more concrete example the following command executes a .. code-block:: bash - redis-cli lpush preprocess_queue "/data25/OA/PL00/1.0/00/urn:eop:DOVE:MULTISPECTRAL_4m:20180811_081455_1054_3be7/0001/PL00_DOV_MS_L3A_20180811T081455_20180811T081455_TOU_1234_3be7.DIMA.tar" + redis-cli lpush preprocess_queue "data25/OA/PL00/1.0/00/urn:eop:DOVE:MULTISPECTRAL_4m:20180811_081455_1054_3be7/0001/PL00_DOV_MS_L3A_20180811T081455_20180811T081455_TOU_1234_3be7.DIMA.tar" Usually, with a preprocessor service running and no other items in the ``preprocess_queue`` this value will be immediately popped from the list and @@ -92,7 +92,7 @@ of the ``preprocess_queue``: .. code-block:: bash $ redis-cli lrange preprocess_queue 0 -1 - /data25/OA/PL00/1.0/00/urn:eop:DOVE:MULTISPECTRAL_4m:20180811_081455_1054_3be7/0001/PL00_DOV_MS_L3A_20180811T081455_20180811T081455_TOU_1234_3be7.DIMA.tar + data25/OA/PL00/1.0/00/urn:eop:DOVE:MULTISPECTRAL_4m:20180811_081455_1054_3be7/0001/PL00_DOV_MS_L3A_20180811T081455_20180811T081455_TOU_1234_3be7.DIMA.tar Now that the product is being preprocessed, it should be visible in the ``preprocessing_set``. As the name indicates, this is using the ``Set`` @@ -101,7 +101,7 @@ datatype, thus requiring the ``SMEMBERS`` subcommand to list: .. code-block:: bash $ redis-cli smembers preprocessing_set 0 -1 - /data25/OA/PL00/1.0/00/urn:eop:DOVE:MULTISPECTRAL_4m:20180811_081455_1054_3be7/0001/PL00_DOV_MS_L3A_20180811T081455_20180811T081455_TOU_1234_3be7.DIMA.tar + data25/OA/PL00/1.0/00/urn:eop:DOVE:MULTISPECTRAL_4m:20180811_081455_1054_3be7/0001/PL00_DOV_MS_L3A_20180811T081455_20180811T081455_TOU_1234_3be7.DIMA.tar Once the preprocessing of the product is finished, the preprocessor will remove the currently worked on path from the ``preprocessing_set`` and add it either @@ -130,6 +130,27 @@ added to the ``registering_set``, afterwards the path is placed to either the sets can be inspected by the ``LRANGE`` or ``SMEMBERS`` subcommands respectively. +Ingestor and sftp +~~~~~~~~~~~~~~~~~ +Triggering preprocessing and registration via pushing to the redis queues is very convenient for single ingestion campaigns, but not optimal for continuous ingestion of new products from "live" sources. +``Ingestor`` service, together optionally with ``sftp`` service allow data ingestion to be initiated by external means. + +``Ingestor`` can work in two modes: + +- Default: Exposing a simple ``/`` endpoint, and listening for ``POST`` requests containing ``data`` with either a Browse Report JSON or a string with path to the object storage with product to be ingested. It then parses this informatio and internally puts it into configured redis queue (preprocess or register). +- Alternative: Listening for newly added Browse Report files on a configured path on a file system via ``inotify``. +These Browse Report files need to be in an agreed XML schema to be correctly handled. +``Sftp`` service enables a secure access to a configured folder via sftp, while this folder can be mounted to other vs services. This way, ``Ingestor`` can listen for newly created files by the sftp access. +If the filedaemon alternative mode should be used, ``INOTIFY_WATCH_DIR`` environment variable needs to be set and a ``command`` used in the docker-compose..ops.yml for ``ingestor`` service needs to be set to ``python3 filedaemon.py``: + +.. code-block:: yaml + + ingestor: + environment: + REDIS_PREPROCESS_MD_QUEUE_KEY: "preprocess_queue" # to override md_queue (json) and instead use (string) + command: + ["python3", "/filedaemon.py"] + Direct Data Management ---------------------- @@ -162,12 +183,23 @@ it is passed as a command line argument, which is then processed normally. .. code-block:: bash - python3 /preprocessor.py \ - --mode standard \ - --replace \ - --tar-object-path /data25/OA/PL00/1.0/00/urn:eop:DOVE:MULTISPECTRAL_4m:20180811_081455_1054_3be7/0001/PL00_DOV_MS_L3A_20180811T081455_20180811T081455_TOU_1234_3be7.DIMA.tar + preprocess \ + --config-file /preprocessor_config.yml \ + --validate \ + --use-dir /tmp \ + data25/OA/PL00/1.0/00/urn:eop:DOVE:MULTISPECTRAL_4m:20180811_081455_1054_3be7/0001/PL00_DOV_MS_L3A_20180811T081455_20180811T081455_TOU_1234_3be7.DIMA.tar + +In order to preprocess a ngEO Ingest Browse Report, an additonal ``--browse-report`` parameter needs to be added: -In this mode, the item will not be placed in the resulting set +.. code-block:: bash + + preprocess \ + --config-file /preprocessor_config.yml \ + --browse-report \ + --use-dir /tmp \ + browse_report_test1.json + +In this "one-off" mode, the item will not be placed in the resulting set (``preprocessing_set``, ``preprocess-success_set``, and ``preprocess-failure_set``). @@ -273,3 +305,14 @@ Deregistration .. code-block:: bash manage.py coverage deregister "${product_id}_coverage" + +Preprocessing vs registration +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The preprocessing step aims to ensure that cloud optimized GeoTIFF (COG) files are created in order to significantly speed up the viewing of large amount of data in lower zoom levels. There are several cases, where such preprocessing is not necessary or wanted. + +- If data are already in COGs and in favorable projection, which will be presented to the user for most of the times, direct registration should be used. This means, paths to individual products will be pushed directly to the register-queue. + +- Also for cases, where preprocessing step would take too much time, direct registration allowing access to the metadata and catalog functions, while justifying slower rendering times can be preferred. + +Next chapter :ref:`access` describes used authorization and authentication concepts and lines out how the external access to individual components and service as such is configured. \ No newline at end of file diff --git a/documentation/operator-guide/intro.rst b/documentation/operator-guide/intro.rst index 3ee54989f7b75e516c1385b080225ac8247dfa6a..6e8184953e672efa72ed24cd507b58f932a80b93 100644 --- a/documentation/operator-guide/intro.rst +++ b/documentation/operator-guide/intro.rst @@ -32,6 +32,9 @@ respective Docker image in parenthesis): - Database (postgis) - Queue Manager (redis) - Log collector (fluentd) +- Kibana (kibana) +- Elasticsearch (elasticsearch) +- Shibboleth SP3 (unicon/shibboleth-sp) These services are bundled and managed together in a Docker Swarm via Docker Compose configuration files. @@ -50,15 +53,17 @@ the used images: - mdillon/postgis:10 - redis -- traefik:2.1 -- elasticsearch:7.9.0 -- kibana:7.9.0 +- traefik +- elasticsearch +- kibana +- unicon/shibboleth-sp - atmoz/sftp -- registry.gitlab.eox.at/esa/prism/vs/fluentd:latest -- registry.gitlab.eox.at/esa/prism/vs/pvs_core:latest -- registry.gitlab.eox.at/esa/prism/vs/pvs_cache:latest -- registry.gitlab.eox.at/esa/prism/vs/pvs_preprocessor:latest -- registry.gitlab.eox.at/esa/prism/vs/pvs_client:latest +- registry.gitlab.eox.at/esa/prism/vs/fluentd +- registry.gitlab.eox.at/esa/prism/vs/pvs_core +- registry.gitlab.eox.at/esa/prism/vs/pvs_cache +- registry.gitlab.eox.at/esa/prism/vs/pvs_preprocessor +- registry.gitlab.eox.at/esa/prism/vs/pvs_client +- registry.gitlab.eox.at/esa/prism/vs/pvs_ingestor Configuration Files ------------------- diff --git a/documentation/operator-guide/management.rst b/documentation/operator-guide/management.rst index d453d28cb4e76964885c70d10e2d4e9d692e5ff5..bbb33703d4a4d9b33bca29bf7797be2161192fc9 100644 --- a/documentation/operator-guide/management.rst +++ b/documentation/operator-guide/management.rst @@ -62,9 +62,8 @@ shutting down of the stack and new deployment. Inspecting reports ------------------ -Once registered, a xml report containes wcs and wms getcapabilities of the registered product is generated and can be accessed by connecting to the `SFTP` image -via the sftp protocol. -In order to log into the logging folders through port 2222 on the hosting ip (e.g. localhost if you are running the dev stack ) The following command can be used: +Once a product is registered, a xml report containing wcs and wms getcapabilities of the registered product is generated and can be accessed by connecting to the `SFTP` service via the sftp protocol. +In order to log into the logging folders through port 2222 on the hosting ip (e.g. localhost if you are running the dev stack) The following command can be used: .. code-block:: bash @@ -74,8 +73,8 @@ this will direct the user into `/home//data` directory which contains .. Note:: The mounted directory that the user is directed into is *`/home/user`*, where `user` is the username, hence when changing the username in the `.conf` file, the `sftp` mounted volumes path in `docker-compose..yml` must be changed respectively. -Inspecting logs ---------------- +Inspecting logs in development +------------------------------ All service components are running inside docker containers and it is therefore possible to inspect the logs for anomalies via standard docker logs calls redirected for example to less command to allow paging through them. @@ -95,8 +94,35 @@ It is possible to show logs of all containers belonging to a service from a mast docker service logs _ -t 2>&1 | sort -k 1 2>&1 | tail -n 2>&1 | less -The docker service logs is intended as a quick way to view the latest log entries of all tasks of a service, but should not be used as a main way to collect these logs. For that it would be appropriate to use a logging driver, and extract the logs from the nodes to a central log aggregator. -The docker service logs is intended as a quick way to view the latest log entries of all tasks of a service, but should not be used as a main way to collect these logs. For that it would be appropriate to use a logging driver, and extract the logs from the nodes to a central log aggregator. +The docker service logs is intended as a quick way to view the latest log entries of all tasks of a service, but should not be used as a main way to collect these logs. For that, on production setup, an additional EFK (Elasticsearch, Fluentd, Kibana) stack is deployed. + +Inspecting logs in production +----------------------------- + +Fluentd is configured as main logging driver of the Docker daemon on Virtual machine level. Therefore for other services to run, Fluentd service must be running too. To access the logs, interactive and multi-purpose Kibana interface is available and exposed externally by traefik. + +For simple listing of the filtered time-sorted logs as an equivalent to `docker service logs` command, a basic ``Discover`` app can be used. The main panel to interact with the logs is the ``Search`` bar, allowing filtered field-data and free-text searches, modyfing time range etc. The individual log results will then appear in the ``Document table`` panel in the bottom of the page. + + +.. _fig_kibana_discover: +.. figure:: images/kibana_2.* + :alt: Kibana discover + + *Kibana discover panel* + +For specific help with ``Discover`` panel, please consult `Kibana official documentation `_ + +In order to select any other option from the Kibana toolkit, click the horizontal lines selection on the top left and pick a tool. + +.. _fig_kibana_menu: +.. figure:: images/kibana_1.* + :alt: Kibana menu + + *Kibana menu* + +Kibana also allows to aggregate log data based on a search query in two modes of operation: ``Bucketing`` and ``Metrics`` being applied on all buckets. + +These aggregations then are used in ``Visualisations`` with various chart modes like vertical bar chart, horizontal line chart. Using saved searches improves the performance of the charts due to limiting the results list. Increasing logging level ------------------------ @@ -112,7 +138,9 @@ A restart of respective service for the change to be applied is also necessary. cd ${INSTALL_DIR}/pvs_instance sed -i 's/DEBUG = False/DEBUG = True/g' settings.py -In order to increase logging level of registrar and preprocessor services to `debug`, the respective Python scripts need to be run with an optional parameter **-v 4**. +In order to increase logging level of registrar and preprocessor services to `DEBUG`, the respective Python commands need to be run with an optional parameter **--debug**. + +Ingestor service by default logs its messages in DEBUG mode. The cache services internally uses a Mapcache software, which usually incorporates an Apache 2 HTTP Server. Due to that, logging level is shared throughout the whole service and is based on Apache `.conf` file, which is stored in $APACHE_CONF environment variable. To change the logging level, edit this file, by setting a **LogLevel debug** and then gracefully restart the Apache component (this way, the cache service itself will not restart and renew default configuration). diff --git a/testing/preprocessor_test.sh b/testing/preprocessor_test.sh index 60334fb9ca8ff7890ee9659d02072d816ef0c8f9..c86f3e8493e921a77310419998aa012e46fffce5 100755 --- a/testing/preprocessor_test.sh +++ b/testing/preprocessor_test.sh @@ -1,5 +1,6 @@ #!/bin/bash product_list_file=$1 +echo "Starting preprocessor test" while read product; do echo $product diff --git a/testing/registrar_test.sh b/testing/registrar_test.sh index 0e0b6ea8ba2d49301d89721e6c0f6ad9be90f39c..7659f177f2156c726dc222a4485d4ec5deca6eec 100755 --- a/testing/registrar_test.sh +++ b/testing/registrar_test.sh @@ -1,11 +1,12 @@ #!/bin/bash product_list_file=$1 -echo "inside registrar_test" +echo "Starting registrar test" OS_PASSWORD=$(docker exec -i $(docker ps -qf "name=emg-pvs_registrar") cat /run/secrets/OS_PASSWORD) IFS="," docker exec -i $(docker ps -qf "name=emg-pvs_registrar") /wait-initialized.sh +docker exec -i $(docker ps -qf "name=emg-pvs_renderer") /wait-initialized.sh while read product; do docker exec -e OS_PASSWORD=$OS_PASSWORD -i $(docker ps -qf "name=emg-pvs_registrar") \ python3 /registrar.py \ diff --git a/traefik-dynamic.yml b/traefik-dynamic.yml index c839637c84bfe949813a78412634147b5a201955..932dc5fc0872f5469ba274790c91dab0733bcdb1 100644 --- a/traefik-dynamic.yml +++ b/traefik-dynamic.yml @@ -18,6 +18,14 @@ http: basicAuth: realm: "PRISM View Server (PVS)" usersFile: "/run/secrets/BASIC_AUTH_USERS_AUTH" + shibAuth: + forwardAuth: + address: http://shibauth/secure + trustForwardHeader: true + shibAuthCache: + forwardAuth: + address: http://shibauth/secure-cache + trustForwardHeader: true compress: compress: {} redirect: diff --git a/traefik.yml b/traefik.yml index 2986bbf23539296df30957b5adb8376143d77adf..39a93c19019a09cfdd7e5d5b0e7413b65494ec5b 100644 --- a/traefik.yml +++ b/traefik.yml @@ -19,7 +19,7 @@ providers: api: dashboard: true log: - level: WARN + level: INFO accessLog: {} certificatesResolvers: default: