EOX GitLab Instance

Skip to content
Snippets Groups Projects
Commit 8e450ae2 authored by Lubomir Dolezal's avatar Lubomir Dolezal
Browse files

Merge branch 'shib-configs-update' into 'staging' 

Shib configs update

See merge request !54
parents dc4def2f 7a28feef
No related branches found
No related tags found
2 merge requests!55Production release 1.2.0,!54Shib configs update
Hide whitespace changes
Inline Side-by-side
Showing
with 182 additions and 188 deletions
config/emg_preprocessor-config.yml
+ 0
− 13
View file @ 8e450ae2
......@@ -200,37 +200,24 @@ preprocessing:
GY01:
# throw away Panchromatic *-P3D*
data_file_globs:
- "*-M3D*.TIF"
- "*-M3D*.tif"
- "*-S3D*.TIF"
- "*-S3D*.tif"
- "*-M2A*.TIF"
- "*-M2A*.tif"
- "*-S2A*.TIF"
- "*-S2A*.tif"
EW03:
data_file_globs:
- "*-M3D*.TIF"
- "*-M3D*.tif"
- "*-S3D*.TIF"
- "*-S3D*.tif"
- "*-M2A*.TIF"
- "*-M2A*.tif"
- "*-S2A*.TIF"
- "*-S2A*.tif"
EW02:
data_file_globs:
- "*-M3D*.TIF"
- "*-M3D*.tif"
- "*-S3D*.TIF"
- "*-S3D*.tif"
- "*-M2A*.TIF"
- "*-M2A*.tif"
- "*-S2A*.TIF"
- "*-S2A*.tif"
EW01:
data_file_globs:
- "*.TIF"
- "*.tif"
DM02:
data_file_globs:
......
config/shibboleth/shib-apache.conf
+ 29
− 1
View file @ 8e450ae2
......@@ -7,11 +7,37 @@
PassEnv APACHE_SERVERNAME
ServerName "${APACHE_SERVERNAME}"
PassEnv PROXY_USER_CATEGORY_ALLOW_RENDERER
PassEnv PROXY_USER_CATEGORY_ALLOW_CACHE
<Location "/Shibboleth.sso">
SetHandler shib
</Location>
# Internally redirected to here in case of Panda proxy access to renderer
<Location /proxy-renderer>
<If "%{HTTP:Oa-User-Category} !~ /${PROXY_USER_CATEGORY_ALLOW_RENDERER}/">
Require all denied
</If>
<Else>
Require all granted
RewriteEngine On
RewriteRule ^.*$ - [R=200]
</Else>
</Location>
# Internally redirected to here in case of Panda proxy access to cache
<Location /proxy-cache>
<If "%{HTTP:Oa-User-Category} !~ /${PROXY_USER_CATEGORY_ALLOW_CACHE}/">
Require all denied
</If>
<Else>
Require all granted
RewriteEngine On
RewriteRule ^.*$ - [R=200]
</Else>
</Location>
# Internally redirected to here. Rewrite for proper relaystate in shib
<Location /secure>
<If "-n req('Authorization')">
......@@ -20,6 +46,8 @@
AuthBasicProvider file
AuthName "/secure"
AuthUserFile /run/secrets/BASIC_AUTH_USERS_AUTH
RewriteEngine On
RewriteRule ^.*$ - [R=200]
</If>
<Else>
RewriteEngine On
......@@ -38,7 +66,7 @@
</LocationMatch>
# Match everything not above like /cache or the client
<LocationMatch "^(?!/(Shibboleth.sso|secure|admin|ows|opensearch))">
<LocationMatch "^(?!/(Shibboleth.sso|secure|admin|ows|opensearch|proxy-renderer|proxy-cache))">
RewriteEngine On
AuthType shibboleth
ShibRequestSetting requireSession 1
......
config/shibboleth/emg-shibboleth2.xml → config/shibboleth/shibboleth2_template.xml
+ 3
− 7
View file @ 8e450ae2
......@@ -4,21 +4,18 @@
xmlns:samlp="urn:oasis:names:tc:SAML:2.0:protocol"
xmlns:md="urn:oasis:names:tc:SAML:2.0:metadata"
clockSkew="180">
<ApplicationDefaults entityID="https://emg.pass.copernicus.eu/shibboleth"
<ApplicationDefaults entityID="{{SPEntityID}}"
REMOTE_USER="eppn uid persistent-id targeted-id">
<Sessions lifetime="28800" timeout="3600" relayState="ss:mem" sameSiteSession="None"
checkAddress="false" handlerSSL="true" cookieProps="https">
<SSO entityID="https://umssoidp.cdsv3.eu:443/shibboleth">
SAML2
</SSO>
<SSO entityID="{{IDPEntityID}}"> SAML2 </SSO>
<Logout>SAML2 Local</Logout>
<Handler type="MetadataGenerator" Location="/Metadata" signing="false"/>
<Handler type="Status" Location="/Status" acl="127.0.0.1 ::1"/>
<Handler type="Session" Location="/Session" showAttributeValues="false"/>
<Handler type="DiscoveryFeed" Location="/DiscoFeed"/>
</Sessions>
<Errors supportContact="admin@eox.at"
helpLocation="/about.html"/>
<Errors supportContact="admin@eox.at" helpLocation="/about.html"/>
<MetadataProvider type="XML" validate="false" path="idp-metadata.xml"/>
<AttributeExtractor type="XML" validate="true" reloadChanges="false" path="attribute-map.xml"/>
<AttributeResolver type="Query" subjectMatch="true"/>
......@@ -27,5 +24,4 @@
</ApplicationDefaults>
<SecurityPolicyProvider type="XML" validate="true" path="security-policy.xml"/>
<ProtocolProvider type="XML" validate="true" reloadChanges="false" path="protocols.xml"/>
</SPConfig>
config/shibboleth/vhr18-shibboleth2.xml deleted 100644 → 0
+ 0
− 31
View file @ dc4def2f
<SPConfig xmlns="urn:mace:shibboleth:3.0:native:sp:config"
xmlns:conf="urn:mace:shibboleth:3.0:native:sp:config"
xmlns:saml="urn:oasis:names:tc:SAML:2.0:assertion"
xmlns:samlp="urn:oasis:names:tc:SAML:2.0:protocol"
xmlns:md="urn:oasis:names:tc:SAML:2.0:metadata"
clockSkew="180">
<ApplicationDefaults entityID="https://vhr18.pass.copernicus.eu/shibboleth"
REMOTE_USER="eppn uid persistent-id targeted-id">
<Sessions lifetime="28800" timeout="3600" relayState="ss:mem"
checkAddress="false" handlerSSL="true" cookieProps="https">
<SSO entityID="https://umssoidp.cdsv3.eu:443/shibboleth">
SAML2
</SSO>
<Logout>SAML2 Local</Logout>
<Handler type="MetadataGenerator" Location="/Metadata" signing="false"/>
<Handler type="Status" Location="/Status" acl="127.0.0.1 ::1"/>
<Handler type="Session" Location="/Session" showAttributeValues="false"/>
<Handler type="DiscoveryFeed" Location="/DiscoFeed"/>
</Sessions>
<Errors supportContact="admin@eox.at"
helpLocation="/about.html"/>
<MetadataProvider type="XML" validate="false" path="idp-metadata.xml"/>
<AttributeExtractor type="XML" validate="true" reloadChanges="false" path="attribute-map.xml"/>
<AttributeResolver type="Query" subjectMatch="true"/>
<AttributeFilter type="XML" validate="true" path="attribute-policy.xml"/>
<CredentialResolver type="File" key="/run/secrets/SHIB_KEY" certificate="/run/secrets/SHIB_CERT"/>
</ApplicationDefaults>
<SecurityPolicyProvider type="XML" validate="true" path="security-policy.xml"/>
<ProtocolProvider type="XML" validate="true" reloadChanges="false" path="protocols.xml"/>
</SPConfig>
\ No newline at end of file
docker-compose.base.ops.yml
+ 2
− 2
View file @ 8e450ae2
......@@ -2,7 +2,7 @@ version: "3.6"
x-vs-version: :release-1.1.1 # bumpversion
services:
reverse-proxy:
image: traefik:2.1
image: traefik:2.4
ports:
- target: 80
published: 80
......@@ -20,7 +20,7 @@ services:
environment:
HTTP_PROXY: "http://172.30.252.68:3128"
HTTPS_PROXY: "http://172.30.252.68:3128"
NO_PROXY: "172.0.0.0/8,192.168.0.0/16,10.0.0.0/8,shibauth"
NO_PROXY: "172.0.0.0/8,192.168.0.0/16,10.0.0.0/8,shibauth-emg,shibauth-dem,shibauth-vhr18"
deploy:
placement:
constraints: [node.role == manager]
......
docker-compose.dem.staging.yml
+ 3
− 3
View file @ 8e450ae2
......@@ -15,7 +15,7 @@ services:
labels:
# router for basic auth access (https)
- "traefik.http.routers.dem-renderer.rule=Host(`dem.pass.copernicus.eu`, `a.dem.pass.copernicus.eu`, `b.dem.pass.copernicus.eu`, `c.dem.pass.copernicus.eu`, `d.dem.pass.copernicus.eu`, `e.dem.pass.copernicus.eu`, `f.dem.pass.copernicus.eu`, `g.dem.pass.copernicus.eu`, `h.dem.pass.copernicus.eu`, `dem.pdas.prism.eox.at`, `a.dem.pdas.prism.eox.at`, `b.dem.pdas.prism.eox.at`, `c.dem.pdas.prism.eox.at`, `d.dem.pdas.prism.eox.at`, `e.dem.pdas.prism.eox.at`, `f.dem.pdas.prism.eox.at`, `g.dem.pdas.prism.eox.at`, `h.dem.pdas.prism.eox.at`) && PathPrefix(`/ows`, `/opensearch`, `/admin`)"
- "traefik.http.routers.dem-renderer.middlewares=auth@file,compress@file,cors@file"
- "traefik.http.routers.dem-renderer.middlewares=compress@file,cors@file"
- "traefik.http.routers.dem-renderer.tls=true"
- "traefik.http.routers.dem-renderer.tls.certresolver=default"
- "traefik.http.routers.dem-renderer.entrypoints=https"
......@@ -45,7 +45,7 @@ services:
- "traefik.http.middlewares.cache-stripprefix.stripprefix.prefixes=/cache"
# router for basic auth access (https)
- "traefik.http.routers.dem-cache.rule=Host(`dem.pass.copernicus.eu`, `a.dem.pass.copernicus.eu`, `b.dem.pass.copernicus.eu`, `c.dem.pass.copernicus.eu`, `d.dem.pass.copernicus.eu`, `e.dem.pass.copernicus.eu`, `f.dem.pass.copernicus.eu`, `g.dem.pass.copernicus.eu`, `h.dem.pass.copernicus.eu`, `dem.pdas.prism.eox.at`, `a.dem.pdas.prism.eox.at`, `b.dem.pdas.prism.eox.at`, `c.dem.pdas.prism.eox.at`, `d.dem.pdas.prism.eox.at`, `e.dem.pdas.prism.eox.at`, `f.dem.pdas.prism.eox.at`, `g.dem.pdas.prism.eox.at`, `h.dem.pdas.prism.eox.at`) && PathPrefix(`/cache`)"
- "traefik.http.routers.dem-cache.middlewares=cache-stripprefix,auth@file,compress@file,cors@file"
- "traefik.http.routers.dem-cache.middlewares=cache-stripprefix,compress@file,cors@file"
- "traefik.http.routers.dem-cache.tls=true"
- "traefik.http.routers.dem-cache.tls.certresolver=default"
- "traefik.http.routers.dem-cache.entrypoints=https"
......@@ -88,7 +88,7 @@ services:
labels:
# router for basic auth access (https)
- "traefik.http.routers.dem-client.rule=Host(`dem.pdas.prism.eox.at`, `dem.pass.copernicus.eu`)"
- "traefik.http.routers.dem-client.middlewares=auth@file,compress@file"
- "traefik.http.routers.dem-client.middlewares=compress@file"
- "traefik.http.routers.dem-client.tls=true"
- "traefik.http.routers.dem-client.tls.certresolver=default"
- "traefik.http.routers.dem-client.entrypoints=https"
......
docker-compose.emg.ops.yml
+ 26
− 20
View file @ 8e450ae2
......@@ -16,7 +16,8 @@ services:
labels:
# router for shib auth based access (https)
- "traefik.http.routers.emg-renderer-shib.rule=Host(`emg.pass.copernicus.eu`) && PathPrefix(`/ows`, `/opensearch`, `/admin`)"
- "traefik.http.routers.emg-renderer-shib.middlewares=shibAuth@file,compress@file,cors@file"
- "traefik.http.middlewares.emg-renderer-shib-fa.forwardauth.address=http://shibauth-emg/secure"
- "traefik.http.routers.emg-renderer-shib.middlewares=emg-renderer-shib-fa,compress@file,cors@file"
- "traefik.http.routers.emg-renderer-shib.tls=true"
- "traefik.http.routers.emg-renderer-shib.tls.certresolver=default"
- "traefik.http.routers.emg-renderer-shib.entrypoints=https"
......@@ -25,20 +26,17 @@ services:
- "traefik.http.routers.emg-renderer-redirect-shib.middlewares=redirect@file"
- "traefik.http.routers.emg-renderer-redirect-shib.entrypoints=http"
# router for internal proxy based access (https)
- "traefik.http.middlewares.emg-pass-whitelist.ipwhitelist.sourcerange=<insert-proxy-url>"
- "traefik.http.routers.emg-renderer-proxy.rule=Host(`proxy.emg.pass.copernicus.eu`) && PathPrefix(`/ows`, `/opensearch`, `/admin`) && HeadersRegexp(`HTTP_Oa-User-Category`,`(Copernicus_Services|Union_Inst|Union_Research_Projects_space|Union_Research_Projects_non-space|Public_Auth)`)"
- "traefik.http.routers.emg-renderer-proxy.middlewares=emg-pass-whitelist,compress@file,cors@file"
- "traefik.http.middlewares.emg-pass-whitelist.ipwhitelist.sourcerange=172.30.78.20"
- "traefik.http.middlewares.emg-renderer-proxy-fa.forwardauth.address=http://shibauth-emg/proxy-renderer"
- "traefik.http.routers.emg-renderer-proxy.rule=Host(`proxy.emg.pass.copernicus.eu`) && PathPrefix(`/ows`, `/opensearch`, `/admin`)"
- "traefik.http.routers.emg-renderer-proxy.middlewares=emg-renderer-proxy-wl,emg-renderer-proxy-fa,compress@file,cors@file"
- "traefik.http.routers.emg-renderer-proxy.tls=true"
- "traefik.http.routers.emg-renderer-proxy.tls.certresolver=default"
- "traefik.http.routers.emg-renderer-proxy.entrypoints=https"
# router for internal proxy based access (http)
- "traefik.http.routers.emg-renderer-redirect-proxy.rule=Host(`proxy.emg.pass.copernicus.eu`) && PathPrefix(`/ows`, `/opensearch`, `/admin`) && HeadersRegexp(`HTTP_Oa-User-Category`,`(Copernicus_Services|Union_Inst|Union_Research_Projects_space|Union_Research_Projects_non-space|Public_Auth)`)"
- "traefik.http.routers.emg-renderer-redirect-proxy.middlewares=emg-pass-whitelist,redirect@file"
- "traefik.http.routers.emg-renderer-redirect-proxy.rule=Host(`proxy.emg.pass.copernicus.eu`) && PathPrefix(`/ows`, `/opensearch`, `/admin`)"
- "traefik.http.routers.emg-renderer-redirect-proxy.middlewares=emg-renderer-proxy-wl,redirect@file"
- "traefik.http.routers.emg-renderer-redirect-proxy.entrypoints=http"
# router for shib auth based access (http)
- "traefik.http.routers.emg-renderer-redirect-shib.rule=Host(`emg.pass.copernicus.eu`) && PathPrefix(`/ows`, `/opensearch`, `/admin`)"
- "traefik.http.routers.emg-renderer-redirect-shib.middlewares=redirect@file"
- "traefik.http.routers.emg-renderer-redirect-shib.entrypoints=http"
# router for basic auth based access (https)
- "traefik.http.routers.emg-renderer.rule=Host(`emg.pdas.prism.eox.at`, `a.emg.pdas.prism.eox.at`, `b.emg.pdas.prism.eox.at`, `c.emg.pdas.prism.eox.at`, `d.emg.pdas.prism.eox.at`, `e.emg.pdas.prism.eox.at`, `f.emg.pdas.prism.eox.at`, `g.emg.pdas.prism.eox.at`, `h.emg.pdas.prism.eox.at`) && PathPrefix(`/ows`, `/opensearch`, `/admin`)"
- "traefik.http.routers.emg-renderer.middlewares=auth@file,compress@file,cors@file"
......@@ -74,8 +72,9 @@ services:
- "traefik.http.middlewares.cache-stripprefix.stripprefix.prefixes=/cache"
# router for shib auth based access (https)
- "traefik.http.routers.emg-cache-shib.rule=Host(`emg.pass.copernicus.eu`) && PathPrefix(`/cache`)"
- "traefik.http.middlewares.emg-cache-shib-fa.forwardauth.address=http://shibauth-emg/secure"
- "traefik.http.middlewares.emg-cache-shib-chain.chain.middlewares=emg-cache-shib-fa,cache-stripprefix,compress@file,cors@file"
- "traefik.http.routers.emg-cache-shib.middlewares=emg-cache-shib-chain"
- "traefik.http.middlewares.emg-cache-shib-chain.chain.middlewares=shibAuth@file,cache-stripprefix,compress@file,cors@file"
- "traefik.http.routers.emg-cache-shib.tls=true"
- "traefik.http.routers.emg-cache-shib.tls.certresolver=default"
- "traefik.http.routers.emg-cache-shib.entrypoints=https"
......@@ -84,15 +83,17 @@ services:
- "traefik.http.routers.emg-cache-redirect-shib.middlewares=redirect@file"
- "traefik.http.routers.emg-cache-redirect-shib.entrypoints=http"
# router for internal proxy based access (https)
- "traefik.http.middlewares.emg-pass-whitelist-cache.ipwhitelist.sourcerange=<insert-proxy-url>"
- "traefik.http.routers.emg-cache-proxy.rule=Host(`proxy.emg.pass.copernicus.eu`) && PathPrefix(`/cache`) && HeadersRegexp(`HTTP_Oa-User-Category`,`(Copernicus_Services|Union_Inst|Union_Research_Projects_space|Union_Research_Projects_non-space|Public_Auth)`)"
- "traefik.http.routers.emg-cache-proxy.middlewares=emg-pass-whitelist-cache,compress@file,cors@file"
- "traefik.http.middlewares.emg-cache-proxy-wl.ipwhitelist.sourcerange=172.30.78.20"
- "traefik.http.routers.emg-cache-proxy.rule=Host(`proxy.emg.pass.copernicus.eu`) && PathPrefix(`/cache`)"
- "traefik.http.middlewares.emg-cache-proxy-fa.forwardauth.address=http://shibauth-emg/proxy-cache"
- "traefik.http.routers.emg-cache-proxy.middlewares=emg-cache-proxy-chain"
- "traefik.http.middlewares.emg-cache-proxy-chain.chain.middlewares=emg-cache-proxy-wl,emg-cache-proxy-fa,cache-stripprefix,compress@file,cors@file"
- "traefik.http.routers.emg-cache-proxy.tls=true"
- "traefik.http.routers.emg-cache-proxy.tls.certresolver=default"
- "traefik.http.routers.emg-cache-proxy.entrypoints=https"
# router for internal proxy based access (http)
- "traefik.http.routers.emg-cache-redirect-proxy.rule=Host(`proxy.emg.pass.copernicus.eu`) && PathPrefix(`/cache`) && HeadersRegexp(`HTTP_Oa-User-Category`,`(Copernicus_Services|Union_Inst|Union_Research_Projects_space|Union_Research_Projects_non-space|Public_Auth)`)"
- "traefik.http.routers.emg-cache-redirect-proxy.middlewares=emg-pass-whitelist-cache,redirect@file"
- "traefik.http.routers.emg-cache-redirect-proxy.rule=Host(`proxy.emg.pass.copernicus.eu`) && PathPrefix(`/cache`)"
- "traefik.http.routers.emg-cache-redirect-proxy.middlewares=emg-cache-proxy-wl,redirect@file"
- "traefik.http.routers.emg-cache-redirect-proxy.entrypoints=http"
# router for basic auth based access (https)
- "traefik.http.routers.emg-cache.rule=Host(`emg.pdas.prism.eox.at`, `a.emg.pdas.prism.eox.at`, `b.emg.pdas.prism.eox.at`, `c.emg.pdas.prism.eox.at`, `d.emg.pdas.prism.eox.at`, `e.emg.pdas.prism.eox.at`, `f.emg.pdas.prism.eox.at`, `g.emg.pdas.prism.eox.at`, `h.emg.pdas.prism.eox.at`) && PathPrefix(`/cache`)"
......@@ -150,7 +151,8 @@ services:
labels:
# router for shib auth based access (https)
- "traefik.http.routers.emg-client-shib.rule=Host(`emg.pass.copernicus.eu`)"
- "traefik.http.routers.emg-client-shib.middlewares=shibAuth@file,compress@file"
- "traefik.http.middlewares.emg-client-shib-fa.forwardauth.address=http://shibauth-emg/secure"
- "traefik.http.routers.emg-client-shib.middlewares=emg-client-shib-fa,compress@file"
- "traefik.http.routers.emg-client-shib.tls=true"
- "traefik.http.routers.emg-client-shib.tls.certresolver=default"
- "traefik.http.routers.emg-client-shib.entrypoints=https"
......@@ -190,10 +192,14 @@ services:
placement:
constraints:
- node.labels.type == internal
shibauth:
shibauth-emg:
image: registry.gitlab.eox.at/esa/prism/vs/pvs_shibauth:release-1.1.1 # bumpversion
environment:
APACHE_SERVERNAME: "https://emg.pass.copernicus.eu:443"
PROXY_USER_CATEGORY_ALLOW_RENDERER: "(Copernicus_Services|Union_Inst|Union_Research_Projects_space|Union_Research_Projects_non-space|Public_Auth)"
PROXY_USER_CATEGORY_ALLOW_CACHE: "(Copernicus_Services|Union_Inst|Union_Research_Projects_space|Union_Research_Projects_non-space|Public_Auth)"
SPEntityID: "https://emg.pass.copernicus.eu/shibboleth"
IDPEntityID: "https://umssoidp.cdsv3.eu:443/shibboleth"
secrets:
- source: EMG_SHIB_CERT
target: SHIB_CERT
......@@ -229,7 +235,7 @@ services:
- source: shib-access-control-conf-cache
target: /etc/shibboleth/pass-ac-cache.xml
- source: shib-shibboleth2
target: /etc/shibboleth/shibboleth2.xml
target: /shibboleth2_template.xml
- source: shib-apache
target: /etc/httpd/conf.d/shib.conf
- source: shib-attribute-map
......@@ -250,7 +256,7 @@ configs:
shib-access-control-conf-cache:
file: ./config/shibboleth/emg-ac-cache.xml
shib-shibboleth2:
file: ./config/shibboleth/emg-shibboleth2.xml
file: ./config/shibboleth/shibboleth2_template.xml
shib-apache:
file: ./config/shibboleth/shib-apache.conf
shib-attribute-map:
......
docker-compose.emg.staging.yml
+ 26
− 20
View file @ 8e450ae2
......@@ -15,7 +15,8 @@ services:
labels:
# router for shib auth based access (https)
- "traefik.http.routers.emg-renderer-shib.rule=Host(`emg.pass.copernicus.eu`) && PathPrefix(`/ows`, `/opensearch`, `/admin`)"
- "traefik.http.routers.emg-renderer-shib.middlewares=shibAuth@file,compress@file,cors@file"
- "traefik.http.middlewares.emg-renderer-shib-fa.forwardauth.address=http://shibauth-emg/secure"
- "traefik.http.routers.emg-renderer-shib.middlewares=emg-renderer-shib-fa,compress@file,cors@file"
- "traefik.http.routers.emg-renderer-shib.tls=true"
- "traefik.http.routers.emg-renderer-shib.tls.certresolver=default"
- "traefik.http.routers.emg-renderer-shib.entrypoints=https"
......@@ -24,20 +25,17 @@ services:
- "traefik.http.routers.emg-renderer-redirect-shib.middlewares=redirect@file"
- "traefik.http.routers.emg-renderer-redirect-shib.entrypoints=http"
# router for internal proxy based access (https)
- "traefik.http.middlewares.emg-pass-whitelist.ipwhitelist.sourcerange=178.248.89.10"
- "traefik.http.routers.emg-renderer-proxy.rule=Host(`proxy.emg.pass.copernicus.eu`) && PathPrefix(`/ows`, `/opensearch`, `/admin`) && HeadersRegexp(`HTTP_Oa-User-Category`,`(Copernicus_Services|Union_Inst|Union_Research_Projects_space|Union_Research_Projects_non-space|Public_Auth|CDS Operations)`)"
- "traefik.http.routers.emg-renderer-proxy.middlewares=emg-pass-whitelist,compress@file,cors@file"
- "traefik.http.middlewares.emg-renderer-proxy-wl.ipwhitelist.sourcerange=178.248.89.10"
- "traefik.http.middlewares.emg-renderer-proxy-fa.forwardauth.address=http://shibauth-emg/proxy-renderer"
- "traefik.http.routers.emg-renderer-proxy.rule=Host(`proxy.emg.pass.copernicus.eu`) && PathPrefix(`/ows`, `/opensearch`, `/admin`)"
- "traefik.http.routers.emg-renderer-proxy.middlewares=emg-renderer-proxy-wl,emg-renderer-proxy-fa,compress@file,cors@file"
- "traefik.http.routers.emg-renderer-proxy.tls=true"
- "traefik.http.routers.emg-renderer-proxy.tls.certresolver=default"
- "traefik.http.routers.emg-renderer-proxy.entrypoints=https"
# router for internal proxy based access (http)
- "traefik.http.routers.emg-renderer-redirect-proxy.rule=Host(`proxy.emg.pass.copernicus.eu`) && PathPrefix(`/ows`, `/opensearch`, `/admin`) && HeadersRegexp(`HTTP_Oa-User-Category`,`(Copernicus_Services|Union_Inst|Union_Research_Projects_space|Union_Research_Projects_non-space|Public_Auth|CDS Operations)`)"
- "traefik.http.routers.emg-renderer-redirect-proxy.middlewares=emg-pass-whitelist,redirect@file"
- "traefik.http.routers.emg-renderer-redirect-proxy.rule=Host(`proxy.emg.pass.copernicus.eu`) && PathPrefix(`/ows`, `/opensearch`, `/admin`)"
- "traefik.http.routers.emg-renderer-redirect-proxy.middlewares=emg-renderer-proxy-wl,redirect@file"
- "traefik.http.routers.emg-renderer-redirect-proxy.entrypoints=http"
# router for shib auth based access (http)
- "traefik.http.routers.emg-renderer-redirect-shib.rule=Host(`emg.pass.copernicus.eu`) && PathPrefix(`/ows`, `/opensearch`, `/admin`)"
- "traefik.http.routers.emg-renderer-redirect-shib.middlewares=redirect@file"
- "traefik.http.routers.emg-renderer-redirect-shib.entrypoints=http"
# router for basic auth based access (https)
- "traefik.http.routers.emg-renderer.rule=Host(`emg.pdas.prism.eox.at`, `a.emg.pdas.prism.eox.at`, `b.emg.pdas.prism.eox.at`, `c.emg.pdas.prism.eox.at`, `d.emg.pdas.prism.eox.at`, `e.emg.pdas.prism.eox.at`, `f.emg.pdas.prism.eox.at`, `g.emg.pdas.prism.eox.at`, `h.emg.pdas.prism.eox.at`) && PathPrefix(`/ows`, `/opensearch`, `/admin`)"
- "traefik.http.routers.emg-renderer.middlewares=auth@file,compress@file,cors@file"
......@@ -70,8 +68,9 @@ services:
- "traefik.http.middlewares.cache-stripprefix.stripprefix.prefixes=/cache"
# router for shib auth based access (https)
- "traefik.http.routers.emg-cache-shib.rule=Host(`emg.pass.copernicus.eu`) && PathPrefix(`/cache`)"
- "traefik.http.middlewares.emg-cache-shib-fa.forwardauth.address=http://shibauth-emg/secure"
- "traefik.http.middlewares.emg-cache-shib-chain.chain.middlewares=emg-cache-shib-fa,cache-stripprefix,compress@file,cors@file"
- "traefik.http.routers.emg-cache-shib.middlewares=emg-cache-shib-chain"
- "traefik.http.middlewares.emg-cache-shib-chain.chain.middlewares=shibAuth@file,cache-stripprefix,compress@file,cors@file"
- "traefik.http.routers.emg-cache-shib.tls=true"
- "traefik.http.routers.emg-cache-shib.tls.certresolver=default"
- "traefik.http.routers.emg-cache-shib.entrypoints=https"
......@@ -80,15 +79,17 @@ services:
- "traefik.http.routers.emg-cache-redirect-shib.middlewares=redirect@file"
- "traefik.http.routers.emg-cache-redirect-shib.entrypoints=http"
# router for internal proxy based access (https)
- "traefik.http.middlewares.emg-pass-whitelist-cache.ipwhitelist.sourcerange=178.248.89.10"
- "traefik.http.routers.emg-cache-proxy.rule=Host(`proxy.emg.pass.copernicus.eu`) && PathPrefix(`/cache`) && HeadersRegexp(`HTTP_Oa-User-Category`,`(Copernicus_Services|Union_Inst|Union_Research_Projects_space|Union_Research_Projects_non-space|Public_Auth|CDS Operations)`)"
- "traefik.http.routers.emg-cache-proxy.middlewares=emg-pass-whitelist-cache,compress@file,cors@file"
- "traefik.http.middlewares.emg-cache-proxy-wl.ipwhitelist.sourcerange=178.248.89.10"
- "traefik.http.routers.emg-cache-proxy.rule=Host(`proxy.emg.pass.copernicus.eu`) && PathPrefix(`/cache`)"
- "traefik.http.middlewares.emg-cache-proxy-fa.forwardauth.address=http://shibauth-emg/proxy-cache"
- "traefik.http.routers.emg-cache-proxy.middlewares=emg-cache-proxy-chain"
- "traefik.http.middlewares.emg-cache-proxy-chain.chain.middlewares=emg-cache-proxy-wl,emg-cache-proxy-fa,cache-stripprefix,compress@file,cors@file"
- "traefik.http.routers.emg-cache-proxy.tls=true"
- "traefik.http.routers.emg-cache-proxy.tls.certresolver=default"
- "traefik.http.routers.emg-cache-proxy.entrypoints=https"
# router for internal proxy based access (http)
- "traefik.http.routers.emg-cache-redirect-proxy.rule=Host(`proxy.emg.pass.copernicus.eu`) && PathPrefix(`/cache`) && HeadersRegexp(`HTTP_Oa-User-Category`,`(Copernicus_Services|Union_Inst|Union_Research_Projects_space|Union_Research_Projects_non-space|Public_Auth|CDS Operations)`)"
- "traefik.http.routers.emg-cache-redirect-proxy.middlewares=emg-pass-whitelist-cache,redirect@file"
- "traefik.http.routers.emg-cache-redirect-proxy.rule=Host(`proxy.emg.pass.copernicus.eu`) && PathPrefix(`/cache`)"
- "traefik.http.routers.emg-cache-redirect-proxy.middlewares=emg-cache-proxy-wl,redirect@file"
- "traefik.http.routers.emg-cache-redirect-proxy.entrypoints=http"
# router for basic auth based access (https)
- "traefik.http.routers.emg-cache.rule=Host(`emg.pdas.prism.eox.at`, `a.emg.pdas.prism.eox.at`, `b.emg.pdas.prism.eox.at`, `c.emg.pdas.prism.eox.at`, `d.emg.pdas.prism.eox.at`, `e.emg.pdas.prism.eox.at`, `f.emg.pdas.prism.eox.at`, `g.emg.pdas.prism.eox.at`, `h.emg.pdas.prism.eox.at`) && PathPrefix(`/cache`)"
......@@ -135,7 +136,8 @@ services:
labels:
# router for shib auth based access (https)
- "traefik.http.routers.emg-client-shib.rule=Host(`emg.pass.copernicus.eu`)"
- "traefik.http.routers.emg-client-shib.middlewares=shibAuth@file,compress@file"
- "traefik.http.middlewares.emg-client-shib-fa.forwardauth.address=http://shibauth-emg/secure"
- "traefik.http.routers.emg-client-shib.middlewares=emg-client-shib-fa,compress@file"
- "traefik.http.routers.emg-client-shib.tls=true"
- "traefik.http.routers.emg-client-shib.tls.certresolver=default"
- "traefik.http.routers.emg-client-shib.entrypoints=https"
......@@ -171,10 +173,14 @@ services:
replicas: 1
environment:
UPLOAD_CONTAINER: "emg-data-staging"
shibauth:
shibauth-emg:
image: registry.gitlab.eox.at/esa/prism/vs/pvs_shibauth:staging
environment:
APACHE_SERVERNAME: "https://emg.pass.copernicus.eu:443"
PROXY_USER_CATEGORY_ALLOW_RENDERER: "(Copernicus_Services|Union_Inst|Union_Research_Projects_space|Union_Research_Projects_non-space|Public_Auth|CDS Operations)"
PROXY_USER_CATEGORY_ALLOW_CACHE: "(Copernicus_Services|Union_Inst|Union_Research_Projects_space|Union_Research_Projects_non-space|Public_Auth|CDS Operations)"
SPEntityID: "https://emg.pass.copernicus.eu/shibboleth"
IDPEntityID: "https://umssoidp.cdsv3.eu:443/shibboleth"
secrets:
- source: EMG_SHIB_CERT
target: SHIB_CERT
......@@ -208,7 +214,7 @@ services:
- source: shib-access-control-conf-cache
target: /etc/shibboleth/pass-ac-cache.xml
- source: shib-shibboleth2
target: /etc/shibboleth/shibboleth2.xml
target: /shibboleth2_template.xml
- source: shib-apache
target: /etc/httpd/conf.d/shib.conf
- source: shib-attribute-map
......@@ -229,7 +235,7 @@ configs:
shib-access-control-conf-cache:
file: ./config/shibboleth/emg-ac-cache.xml
shib-shibboleth2:
file: ./config/shibboleth/emg-shibboleth2.xml
file: ./config/shibboleth/shibboleth2_template.xml
shib-apache:
file: ./config/shibboleth/shib-apache.conf
shib-attribute-map:
......
docker-compose.vhr18.staging.yml
+ 3
− 6
View file @ 8e450ae2
......@@ -16,7 +16,7 @@ services:
labels:
# router for basic auth access (https)
- "traefik.http.routers.vhr18-renderer.rule=Host(`vhr18.pdas.prism.eox.at`, `a.vhr18.pdas.prism.eox.at`, `b.vhr18.pdas.prism.eox.at`, `c.vhr18.pdas.prism.eox.at`, `d.vhr18.pdas.prism.eox.at`, `e.vhr18.pdas.prism.eox.at`, `f.vhr18.pdas.prism.eox.at`, `g.vhr18.pdas.prism.eox.at`, `h.vhr18.pdas.prism.eox.at`, `vhr18.pass.copernicus.eu`, `a.vhr18.pass.copernicus.eu`, `b.vhr18.pass.copernicus.eu`, `c.vhr18.pass.copernicus.eu`, `d.vhr18.pass.copernicus.eu`, `e.vhr18.pass.copernicus.eu`, `f.vhr18.pass.copernicus.eu`, `g.vhr18.pass.copernicus.eu`, `h.vhr18.pass.copernicus.eu`) && PathPrefix(`/ows`, `/opensearch`, `/admin`)"
- "traefik.http.routers.vhr18-renderer.middlewares=auth@file,compress@file,cors@file"
- "traefik.http.routers.vhr18-renderer.middlewares=compress@file,cors@file"
- "traefik.http.routers.vhr18-renderer.tls=true"
- "traefik.http.routers.vhr18-renderer.tls.certresolver=default"
- "traefik.http.routers.vhr18-renderer.entrypoints=https"
......@@ -33,9 +33,6 @@ services:
resources:
limits:
memory: 8G
placement:
constraints:
- node.labels.type == external
networks:
- extnet
cache:
......@@ -48,7 +45,7 @@ services:
- "traefik.http.middlewares.cache-stripprefix.stripprefix.prefixes=/cache"
# router for basic auth access (https)
- "traefik.http.routers.vhr18-cache.rule=Host(`vhr18.pdas.prism.eox.at`, `a.vhr18.pdas.prism.eox.at`, `b.vhr18.pdas.prism.eox.at`, `c.vhr18.pdas.prism.eox.at`, `d.vhr18.pdas.prism.eox.at`, `e.vhr18.pdas.prism.eox.at`, `f.vhr18.pdas.prism.eox.at`, `g.vhr18.pdas.prism.eox.at`, `h.vhr18.pdas.prism.eox.at`, `vhr18.pass.copernicus.eu`, `a.vhr18.pass.copernicus.eu`, `b.vhr18.pass.copernicus.eu`, `c.vhr18.pass.copernicus.eu`, `d.vhr18.pass.copernicus.eu`, `e.vhr18.pass.copernicus.eu`, `f.vhr18.pass.copernicus.eu`, `g.vhr18.pass.copernicus.eu`, `h.vhr18.pass.copernicus.eu`) && PathPrefix(`/cache`)"
- "traefik.http.routers.vhr18-cache.middlewares=cache-stripprefix,auth@file,compress@file,cors@file"
- "traefik.http.routers.vhr18-cache.middlewares=cache-stripprefix,compress@file,cors@file"
- "traefik.http.routers.vhr18-cache.tls=true"
- "traefik.http.routers.vhr18-cache.tls.certresolver=default"
- "traefik.http.routers.vhr18-cache.entrypoints=https"
......@@ -91,7 +88,7 @@ services:
labels:
# router for basic auth access (https)
- "traefik.http.routers.vhr18-client.rule=Host(`vhr18.pdas.prism.eox.at`, `vhr18.pass.copernicus.eu`)"
- "traefik.http.routers.vhr18-client.middlewares=auth@file,compress@file"
- "traefik.http.routers.vhr18-client.middlewares=compress@file"
- "traefik.http.routers.vhr18-client.tls=true"
- "traefik.http.routers.vhr18-client.tls.certresolver=default"
- "traefik.http.routers.vhr18-client.entrypoints=https"
......
documentation/operator-guide/access.rst
+ 40
− 55
View file @ 8e450ae2
......@@ -13,11 +13,11 @@ These services can have a set of authentication and authorization rules applied
Routing with traefik
~~~~~~~~~~~~~~~~~~~~
``Reverse-proxy`` service in base stack provides central access endpoint to the VS. It exposes ports 80 and 443 for HTTP and HTTPS access. Configuration of the reverse-proxy is done on three places.
``Reverse-proxy`` service in base stack provides central access endpoint to the VS. It exposes ports 80 and 443 for HTTP and HTTPS access in the host mode. Configuration of the reverse-proxy is done on three places.
First two are static and dynamic configuration files ``traefik.yml`` and ``traefik-dynamic.yml``. Static configuration sets up connections to providers and define the entrypoints that Traefik will listen to. Dynamic configuration defines how the requests are handled. This configuration can change and is seamlessly hot-reloaded, without any request interruption or connection loss. Third part are docker ``labels`` on individual services which Traefik provides access to.
First two are static and dynamic configuration files ``traefik.yml`` and ``traefik-dynamic.yml``. Static configuration sets up connections to providers and define the entrypoints that Traefik will listen to. Dynamic configuration defines how the requests are handled. This configuration can change and is seamlessly hot-reloaded, without any request interruption or connection loss. Third part are docker ``labels`` on individual services which Traefik provides access to, for which an update requires removing and re-creating the stack.
For example following configuration snippet enables access to certain paths of ``renderer`` service under a given hostname. It also sets externally set basic authentication and other rules via ``@file`` identifier, which references configurations from ``traefik-dynamic.yml``.
For example following configuration snippet enables access to certain paths of ``renderer`` service under a given hostname. It also sets externally set basic authentication and other rules via ``@file`` identifier, which references global configurations from ``traefik-dynamic.yml``.
.. code-block:: yaml
......@@ -48,7 +48,7 @@ An example of such auth@file configuration from ``traefik-dynamic.yml`` would be
realm: "PRISM View Server (PVS)"
usersFile: "/run/secrets/BASIC_AUTH_USERS_AUTH"
Unsecured HTTP access is configured to be redirected to the HTTPS endpoint but inside the swarm among the services, only HTTP is used internally.
Unsecured HTTP access is configured to be redirected to the HTTPS endpoint. Inside the swarm among the services, only HTTP is used internally.
Authentication and Authorization
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
......@@ -60,27 +60,27 @@ Here, access on such endpoint requires basic authentication credentials (usernam
- Shibboleth Service Provider 3 + Apache 2 instance, to which requests are forwarded by `Traefik ForwardAuth middleware <https://doc.traefik.io/traefik/middlewares/forwardauth/>`_.
Middleware delegates the authentication to Shibboleth. If Shibboleth response code is 2XX, access is granted and the original request is performed. Otherwise, the response from the Shibboleth is returned.
Middleware delegates the authentication to Shibboleth. If Shibboleth response code is 2XX, access is granted and the original request is performed. Otherwise, the error response from Shibboleth is returned.
In order to authenticate with Shibboleth, a user must log in with valid credentials on the side of Identity Provider (IdP), if doing so, the IdP informs the SP about successful login, accompanied by relevant user attributes and a session is created for the user. SP then saves the information about a created session into a cookie and based on user attributes can authorize access to the services. If the user was already logged in, he is automatically offered the requested resource.
Currently setting individual authorization rules on a ``Collection`` and ``Service`` level is possible with current approach. It is yet not clearly possible to separate viewing and download, as both of these parts are handled by ``renderer`` service.
Currently setting individual authorization rules on a ``Collection`` (docker stack) and ``Service`` (docker service) level is possible. It is yet not clearly possible to separate viewing and download functionality, as both of these parts are handled by ``renderer`` service.
Configuration
~~~~~~~~~~~~~
For correct configuration of Shibboleth SP3 on a new stack, several steps need to be done. Most of these configurations are usually done in the :ref:`initialization` step using ``pvs_starter`` tool. Still, it is advised to check following steps, understand them and change if necessary.
Briefly summarized, SP and IdP need to exchange metadata and certificates to trust each other, SP needs to know which attributes the IdP will be sending about the logged-in user and respective access-control rules are configured based on those attributes. Most of the configurations are done via docker configs defined in the docker compose
Briefly summarized, SP and IdP need to exchange metadata and certificates to trust each other, SP needs to know which attributes the IdP will be sending about the logged-in user and respective access-control rules are configured based on those attributes. Most of the configurations are done via docker configs defined in the docker compose files.
- Create a pair of key, certificate using attached Shibboleth ``config/shibboleth/keygen.sh`` in the cloned vs repository and save them as respective docker secrets.
- Create a pair of key, certificate using attached Shibboleth utility ``config/shibboleth/keygen.sh`` in the cloned ``vs`` repository and save them as respective docker secrets.
.. code-block:: bash
SPURL="https://emg.pass.copernicus.eu" # service initial access point made accessible by traefik
./config/shibboleth/keygen.sh -h $SPURL -y 20 -e https://$SPURL/shibboleth -n sp-signing -f
docker secret create EMG_SHIB_CERT sp-signing-cert.pem
SP_URL="https://emg.pass.copernicus.eu" # service initial access point made accessible by traefik
./config/shibboleth/keygen.sh -h $SPURL -y 20 -e https://$SP_URL/shibboleth -n sp-signing -f
docker secret create <stack-name>_SHIB_CERT sp-signing-cert.pem
docker secret create <stack-name>_SHIB_KEY sp-signing-key.pem
- Get IDP metadata and save it as a docker config. Also read the entityID of the IdP for further use in referencing it in your ``shibboleth2.xml`` configuration.
- Get IDP metadata and save it as a docker config. Also save the entityID of the IdP for further use in filling the ``shibboleth2.xml`` template.
.. code-block:: bash
......@@ -88,17 +88,9 @@ Briefly summarized, SP and IdP need to exchange metadata and certificates to tru
- Configure Apache ServerName used inside the ``shibauth`` service by modifying ``APACHE_SERVERNAME`` environment variable of corresponding ``shibauth`` service in ``docker-compose.<stack>.ops.yml``. This URL should resolve to the actual service URL.
- Modify shibboleth2.xml content by setting your "entityID" in <ApplicationDefaults> Additionally edit the "entityID" value inside ``SSO`` element to match the IdP "entityID". Note that "entityID" does not need to resolve to an actual service URL.
- Configure SP and IdP EntityIDs used inside the ``shibauth`` service by modifying ``SPEntityID`` and ``IDPEntityID`` environment variables of corresponding ``shibauth`` service in ``docker-compose.<stack>.ops.yml``. ``SPEntityID`` can be chosen in any way, IDPEntityID should be extracted from received IDP metadata.
.. code-block:: xml
<ApplicationDefaults entityID="https://testing-sp/shibboleth">
<SSO entityID="https://testing-idp:443/shibboleth">
SAML2
</SSO>
</ApplicationDefaults>
- Deploy your shibauth service and exchange your SP metadata with the IdP provider and have them recognize your SP. Necessary metadata needs to be downloaded from url ``<service-url>/Shibboleth.sso/Metadata``.
- Deploy your shibauth service and exchange your SP metadata with the IdP provider and have them register your SP. Necessary metadata can be downloaded from url ``<service-url>/Shibboleth.sso/Metadata``.
- Get information about attributes provided by IdP and update ``config/shibboleth/attribute-map.xml`` by adding individual entries mapping ``name`` provided by IdP to ``id`` used by SP internally. Example configuration:
......@@ -109,7 +101,7 @@ Briefly summarized, SP and IdP need to exchange metadata and certificates to tru
<Attribute name="urn:mace:dir:attribute-def:primary-group" id="user_group_primary"/>
</Attributes>
- Create custom access rules based on these attributes and map these access controls to different internal apache routes to which Traefik ForwardAuth middleware will point. Access rules are created in ``config/shibboleth/<stack-name>-ac.xml``.
- Create custom access rules based on these attributes and map these access controls to different internal Apache routes to which Traefik ForwardAuth middleware will point. Access rules are created in ``config/shibboleth/<stack-name>-ac.xml`` and ``config/shibboleth/<stack-name>-ac-cache.xml``.
Example of external Access control rules configuration:
......@@ -136,45 +128,38 @@ Example of external Access control rules configuration:
APACHE_SERVERNAME: "https://emg.pass.copernicus.eu:443"
deploy:
labels:
- "traefik.http.routers.shibauth.rule=Host(`emg.pass.copernicus.eu`) && PathPrefix(`/secure`, `/Shibboleth.sso`)"
- "traefik.http.routers.shibauth.rule=Host(`emg.pass.copernicus.eu`) && PathPrefix(`/Shibboleth.sso`)"
...
Relevant Apache configuration in ``config/shibboleth/shib-apache.conf``, enabling Shibboleth authentication and authorization of the ``/secure`` endpoint.
Relevant Apache configuration in ``config/shibboleth/shib-apache.conf``, enabling Shibboleth authentication and authorization of the renderer service on the ``/secure`` endpoint.
.. code-block:: apacheconf
<Location />
SetHandler shib
# Internally redirected to here. Rewrite for proper relaystate in shib
<Location /secure>
<If "-n req('Authorization')">
# rules for Basic Auth fallback
</If>
<Else>
RewriteEngine On
RewriteCond %{HTTP:X-Forwarded-Uri} ^(.*)$ [NC]
RewriteRule ^.*$ %1 [PT]
</Else>
</Location>
<VirtualHost *:80>
PassEnv APACHE_SERVERNAME
ServerName "${APACHE_SERVERNAME}"
<Location /secure>
AuthType shibboleth
ShibRequestSetting requireSession 1
Require shib-plugin /etc/shibboleth/pass-ac.xml
</Location>
...
Part of Traefik ForwardAuth middleware configuration from ``traefik-dynamic.yml``, defining the internal address pointing to the ``shibauth`` service and ``/secure`` endpoint in it:
.. code-block:: yaml
http:
middlewares:
shibAuth:
forwardAuth:
address: http://shibauth/secure
trustForwardHeader: true
<LocationMatch "^/(admin|ows|opensearch)">
RewriteEngine On
AuthType shibboleth
ShibRequestSetting requireSession 1
Require shib-plugin /etc/shibboleth/pass-ac.xml
RewriteRule ^.*$ - [R=200]
</LocationMatch>
Part of renderer service Traefik labels from ``docker-compose.emg.ops.yml``, where access through the middleware is configured.
Part of Traefik ForwardAuth middleware configuration from ``docker-compose.emg.ops.yml``, defining the internal address pointing to the ``shibauth-emg`` service and ``/secure`` endpoint in it:
.. code-block:: yaml
services:
renderer:
deploy:
labels:
# router for shib auth based access (https)
- "traefik.http.routers.emg-renderer-shib.rule=Host(`emg.pass.copernicus.eu`) && PathPrefix(`/ows`, `/opensearch`, `/admin`)"
- "traefik.http.routers.emg-renderer-shib.middlewares=shibAuth@file"
renderer:
deploy:
labels:
- "traefik.http.middlewares.emg-renderer-shib-fa.forwardauth.address=http://shibauth-emg/secure"
- "traefik.http.routers.emg-renderer-shib.middlewares=emg-renderer-shib-fa,compress@file,cors@file"
ingestor/ingestor/util.py
+ 1
− 1
View file @ 8e450ae2
......@@ -62,4 +62,4 @@ def save_endpoint_report(filename: str, data, success: bool):
def browse_name(report):
return '_'.join(browse["browse_identifier"] for browse in report["browses"])
\ No newline at end of file
return '_'.join(browse["browse_identifier"] for browse in report["browses"])
preprocessor/Dockerfile
+ 1
− 1
View file @ 8e450ae2
......@@ -25,7 +25,7 @@
# IN THE SOFTWARE.
#-----------------------------------------------------------------------------
FROM osgeo/gdal:ubuntu-full-latest
FROM osgeo/gdal:ubuntu-full-3.2.1
MAINTAINER EOX
LABEL name="prism view server preprocessor" \
......
preprocessor/preprocessor/config-schema.yaml
+ 2
− 1
View file @ 8e450ae2
......@@ -41,7 +41,8 @@ properties:
type: string
glob_case:
description: If all file globs will use case-sensitive match.
type: boolean
type: boolean
default: false
type_extractor:
description: How the product type is to be extracted from the metadata file.
type: object
......
preprocessor/preprocessor/steps/georeference.py
+ 2
− 5
View file @ 8e450ae2
......@@ -5,7 +5,7 @@ from glob import glob
import shutil
from typing import List, Tuple
from ..util import gdal, osr, replace_ext
from ..util import gdal, osr, replace_ext, get_all_data_files
logger = logging.getLogger(__name__)
......@@ -31,10 +31,7 @@ def georeference_step(source_dir: os.PathLike, target_dir: os.PathLike, preproce
else:
raise Exception('Invalid georeference type %s' % type_name)
try:
filenames = []
for dataglob in preprocessor_config.get('data_file_globs', '*'):
for p in [path for path in glob(join(source_dir, '**', dataglob), recursive=True) if not os.path.isdir(path)]:
filenames.append(p)
filenames = get_all_data_files(source_dir, preprocessor_config)
for filename in filenames:
target_filename = join(target_dir, basename(filename))
georef_func(filename, target_filename, **opts_dict)
......
preprocessor/preprocessor/steps/output.py
+ 2
− 6
View file @ 8e450ae2
import os
from os.path import join, basename
from uuid import uuid4
from glob import glob
from ..util import replace_ext, gdal
from ..util import replace_ext, gdal, get_all_data_files
import logging
logger = logging.getLogger(__name__)
......@@ -19,10 +18,7 @@ def output_step(source_dir: os.PathLike, target_dir: os.PathLike, preprocessor_c
extension = driver.GetMetadata().get('DMD_EXTENSIONS', 'tif').split(' ')[0]
# warp each individual file
warped_files = []
filenames = []
for dataglob in preprocessor_config.get('data_file_globs', '*'):
for p in [path for path in glob(join(source_dir, '**', dataglob), recursive=True) if not os.path.isdir(path)]:
filenames.append(p)
filenames = get_all_data_files(source_dir, preprocessor_config)
for filename in filenames:
target_filename = join(target_dir, replace_ext(basename(filename), extension))
logger.debug('Warping file %s' % filename)
......
preprocessor/preprocessor/steps/stack.py
+ 3
− 7
View file @ 8e450ae2
import os
from os.path import basename, join, splitext
from os.path import basename, join
from itertools import groupby
import re
from glob import glob
from typing import List
from ..util import replace_ext, gdal
from ..util import replace_ext, gdal, get_all_data_files
def stack_bands_step(source_dir: os.PathLike, target_dir: os.PathLike, preprocessor_config: dict, group_by: str=None, sort_by: str=None, order: List[str]=None):
""" Stack bands of the individual images
"""
filenames = []
for dataglob in preprocessor_config.get('data_file_globs', '*'):
for p in [path for path in glob(join(source_dir, '**', dataglob), recursive=True) if not os.path.isdir(path)]:
filenames.append(p)
filenames = get_all_data_files(source_dir, preprocessor_config)
# check if we have a group_by regex. If yes, use the first
# re-group to group by.
# Fallback is basename of file as groupname
......
preprocessor/preprocessor/steps/subdataset.py
+ 4
− 8
View file @ 8e450ae2
import os
from os.path import join, splitext, basename, dirname, isdir
from glob import glob
from os.path import join, basename
from typing import Dict
from ..util import replace_ext, gdal
from ..util import replace_ext, gdal, get_all_data_files
def extract_subdataset_step(source_dir: os.PathLike, target_dir: os.PathLike, preprocessor_config: dict, subdataset_types: Dict[str, str]=None):
filenames = []
for dataglob in preprocessor_config.get('data_file_globs', '*'):
for p in [path for path in glob(join(source_dir, '**', dataglob), recursive=True) if not isdir(path)]:
filenames.append(p)
filenames = get_all_data_files(source_dir, preprocessor_config)
if len(filenames) == 0:
raise Exception('No datafiles were matched by the provided glob')
for filename in datafiles:
for filename in filenames:
extract_subdatasets(
filename,
target_dir,
......
preprocessor/preprocessor/util.py
+ 17
− 1
View file @ 8e450ae2
import os
from os.path import splitext
from os.path import splitext, join
from contextlib import contextmanager
from tempfile import TemporaryDirectory, mkdtemp
from time import time
from glob import glob
from .archive import filter_filenames
try:
from osgeo import gdal
......@@ -86,3 +89,16 @@ def get_size_in_bytes(file_path, unit):
""" Get size of file at given path in bytes"""
size = os.path.getsize(file_path)
return convert_unit(size, unit)
def get_all_data_files(source_dir, preprocessor_config):
""" Based on 'data_file_globs' configuration, gets all unique data file paths from folder matching any of the globs"""
# get all file paths recursively
file_paths = [p for p in glob(join(source_dir, '**'), recursive=True) if not os.path.isdir(p)]
# filter them by data_globs
file_paths_filt = []
for dataglob in preprocessor_config.get('data_file_globs', ['*']):
file_paths_filt += filter_filenames(file_paths, dataglob, preprocessor_config.get('glob_case', False))
# get only unique files to compensate for possibly bad glob yielding doubles, keeping order
file_paths_filt = list(dict.fromkeys(file_paths_filt))
return file_paths
shibauth/Dockerfile
+ 10
− 0
View file @ 8e450ae2
......@@ -36,3 +36,13 @@ LABEL name="prism view server shibauth" \
RUN yum -y update \
&& yum -y install shibboleth-3.2.0-2.1 \
&& yum -y clean all
ADD configure.sh \
shibboleth2_template.xml \
run-shibboleth.sh \
/
RUN chmod -v +x \
/configure.sh \
/run-shibboleth.sh
CMD ["/run-shibboleth.sh"]
shibauth/configure.sh 0 → 100644
+ 8
− 0
View file @ 8e450ae2
#!/bin/bash -e
echo "Running configure.sh" >&2
# substitute template values
cat /shibboleth2_template.xml \
| sed -e "s/{{SPEntityID}}/$(echo ${SPEntityID} | sed -e 's/[]\/$*.^[]/\\&/g')/g" \
| sed -e "s/{{IDPEntityID}}/$(echo ${IDPEntityID} | sed -e 's/[]\/$*.^[]/\\&/g')/g" \
> /etc/shibboleth/shibboleth2.xml
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment