Prototype Thredds verify twitcher

birdhouse/config/catalog/config/magpie/providers.cfg.template

@@ -8,7 +8,9 @@ providers:
     sync_type: wps
 
   ${CATALOG_THREDDS_SERVICE}:
-    url: http://${PAVICS_FQDN}:8083/twitcher/ows/proxy/thredds
+    # below URL is only used to fill in the required location in Magpie
+    # actual auth validation is performed with Twitcher 'verify' endpoint without accessing this proxied URL
+    url: http://proxy:80
     title: Thredds
     public: true
     c4i: false

birdhouse/config/thredds/.gitignore

@@ -1,6 +1,7 @@
 catalog.xml
 threddsConfig.xml
 config/proxy/conf.extra-service.d/thredds.conf
+config/proxy/conf.extra-service.d.bypasstwitcher/thredds_bypass_twitcher.conf
 config/canarie-api/canarie_api_monitoring.py
 
 # Old paths. Keep these so that old config files remain uncommittable after updates.

birdhouse/config/thredds/catalog.xml.template

@@ -4,14 +4,14 @@
          xmlns:xlink="http://www.w3.org/1999/xlink" >
 
     <service name="all" serviceType="Compound" base="" >
-        <service name="http" serviceType="HTTPServer" base="/twitcher/ows/proxy/thredds/fileServer/" />
-        <service name="odap" serviceType="OpenDAP" base="/twitcher/ows/proxy/thredds/dodsC/" />
-        <service name="ncml" serviceType="NCML" base="/twitcher/ows/proxy/thredds/ncml/"/>
-        <service name="uddc" serviceType="UDDC" base="/twitcher/ows/proxy/thredds/uddc/"/>
-        <service name="iso" serviceType="ISO" base="/twitcher/ows/proxy/thredds/iso/"/>
-        <service name="wcs" serviceType="WCS" base="/twitcher/ows/proxy/thredds/wcs/" />
-        <service name="wms" serviceType="WMS" base="/twitcher/ows/proxy/thredds/wms/" />
-        <service name="subsetServer" serviceType="NetcdfSubset" base="/twitcher/ows/proxy/thredds/ncss/" />
+        <service name="http" serviceType="HTTPServer" base="${THREDDS_CONTEXT_ROOT}/fileServer/" />
+        <service name="odap" serviceType="OpenDAP" base="${THREDDS_CONTEXT_ROOT}/dodsC/" />
+        <service name="ncml" serviceType="NCML" base="${THREDDS_CONTEXT_ROOT}/ncml/"/>
+        <service name="uddc" serviceType="UDDC" base="${THREDDS_CONTEXT_ROOT}/uddc/"/>
+        <service name="iso" serviceType="ISO" base="${THREDDS_CONTEXT_ROOT}/iso/"/>
+        <service name="wcs" serviceType="WCS" base="${THREDDS_CONTEXT_ROOT}/wcs/" />
+        <service name="wms" serviceType="WMS" base="${THREDDS_CONTEXT_ROOT}/wms/" />
+        <service name="subsetServer" serviceType="NetcdfSubset" base="${THREDDS_CONTEXT_ROOT}/ncss/" />
     </service>
 
     <datasetScan name="Birdhouse" ID="birdhouse" path="birdhouse" location="/pavics-data">

birdhouse/config/thredds/config/canarie-api/canarie_api_monitoring.py.template

@@ -15,7 +15,7 @@ SERVICES['renderer'] = {
     },
     'stats': {
         'method': '.*',
-        'route': '/thredds/.*'
+        'route': '${THREDDS_CONTEXT_ROOT}/.*'
     },
     'redirect': {
         'doc': 'https://ouranosinc.github.io/pavics-sdi/arch/frontend.html#gridded-data-rendering',
@@ -29,7 +29,7 @@ SERVICES['renderer'] = {
     'monitoring': {
         'ncWMS': {
             'request': {
-                'url': 'https://${PAVICS_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/thredds/wms/birdhouse/testdata/ta_Amon_MRI-CGCM3_decadal1980_r1i1p1_199101-200012.nc?service=WMS&version=1.3.0&request=GetCapabilities'
+                'url': 'https://${PAVICS_FQDN_PUBLIC}${THREDDS_CONTEXT_ROOT}/wms/birdhouse/testdata/ta_Amon_MRI-CGCM3_decadal1980_r1i1p1_199101-200012.nc?service=WMS&version=1.3.0&request=GetCapabilities'
             }
         },
     }
@@ -49,22 +49,22 @@ SERVICES['Thredds'] = {
     },
     'stats': {
         'method': '.*',
-        'route': '${TWITCHER_PROTECTED_PATH}/thredds/.*'
+        'route': '${THREDDS_CONTEXT_ROOT}/.*'
     },
     'redirect': {
         'doc': 'https://www.unidata.ucar.edu/software/tds/',
         'releasenotes': 'https://docs.unidata.ucar.edu/tds/current/userguide/upgrade.html',
         'support': 'https://www.unidata.ucar.edu/software/tds/#help',
         'source': 'https://github.com/Unidata/tds',
-        'tryme': 'https://${PAVICS_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/thredds/',
+        'tryme': 'https://${PAVICS_FQDN_PUBLIC}${THREDDS_CONTEXT_ROOT}/',
         'licence': 'https://github.com/Unidata/tds/blob/main/LICENSE',
         'provenance': 'https://downloads.unidata.ucar.edu/tds/'
     },
     "monitoring": {
         "Thredds": {
             'request': {
                 # FIXME: remove port by design (https://github.com/bird-house/birdhouse-deploy/issues/222)
-                'url': 'http://${PAVICS_FQDN}:8083${TWITCHER_PROTECTED_PATH}/thredds/catalog.html'
+                'url': 'http://${PAVICS_FQDN}:8083${THREDDS_CONTEXT_ROOT}/catalog.html'
             }
         }
     }

birdhouse/config/thredds/config/proxy/conf.extra-service.d.bypasstwitcher/thredds_bypass_twitcher.conf.template

@@ -0,0 +1,11 @@
+
+    location ${THREDDS_CONTEXT_ROOT}/ {
+        #auth_request /secure-thredds-auth;
+        #auth_request_set $auth_status $upstream_status;
+        proxy_pass http://thredds:8080${THREDDS_CONTEXT_ROOT}/;
+
+        proxy_set_header Host $host;
+        proxy_set_header X-Forwarded-Proto $real_scheme;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    }
+

birdhouse/config/thredds/config/proxy/conf.extra-service.d/thredds.conf.template

@@ -1,10 +1,22 @@
 
     location /thredds/ {
-        #return 302 /twitcher/ows/proxy$request_uri;
-        proxy_pass https://${PAVICS_FQDN}${TWITCHER_PROTECTED_PATH}/thredds/;
-        # direct hit Thredds, bypassing twitcher, for debugging only
-        # proxy_pass http://thredds:8080${TWITCHER_PROTECTED_PATH}/thredds/;
+        auth_request /secure-thredds-auth;
+        auth_request_set $auth_status $upstream_status;
+        proxy_pass http://thredds:8080${THREDDS_CONTEXT_ROOT}/;
+
+        proxy_set_header Host $host;
+        proxy_set_header X-Forwarded-Proto $real_scheme;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    }
+
+    location = /secure-thredds-auth {
+        internal;
+        proxy_pass https://${PAVICS_FQDN_PUBLIC}${TWITCHER_VERIFY_PATH}/thredds$request_uri;
+        proxy_pass_request_body off;
         proxy_set_header Host $host;
+        proxy_set_header Content-Length "";
+        proxy_set_header X-Original-URI $request_uri;
         proxy_set_header X-Forwarded-Proto $real_scheme;
         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Host $host:$server_port;
     }

birdhouse/config/thredds/config/proxy/docker-compose-extra.yml

@@ -3,5 +3,6 @@ services:
   proxy:
     volumes:
     - ./config/thredds/config/proxy/conf.extra-service.d:/etc/nginx/conf.extra-service.d/thredds:ro
+    - ./config/thredds/config/proxy/conf.extra-service.d.bypasstwitcher:/etc/nginx/conf.extra-service.d/_1_thredds_bypass_twitcher:ro
     links:
       - thredds

birdhouse/config/thredds/default.env

@@ -6,16 +6,35 @@ export THREDDS_DOCKER=pavics/thredds-docker
 export THREDDS_IMAGE="${THREDDS_DOCKER}:${THREDDS_VERSION}"
 export THREDDS_ORGANIZATION="Birdhouse"
 
+export THREDDS_CONTEXT_ROOT='${TWITCHER_PROTECTED_PATH}/thredds'
+
 # add any new variables not already in 'VARS' or 'OPTIONAL_VARS' that must be replaced in templates here
 VARS="
   $VARS
+  \$TWITCHER_VERIFY_PATH
   \$CMIP5_THREDDS_ROOT
 "
 
 OPTIONAL_VARS="
   $OPTIONAL_VARS
   \$THREDDS_ORGANIZATION
-  \$TWITCHER_PROTECTED_PATH
   \$THREDDS_DOCKER
   \$THREDDS_VERSION
+  \$THREDDS_CONTEXT_ROOT
+"
+
+# Endpoint to verify Magpie/Twitcher authorization to a service/resource by a user without proxy request
+# Requires Twitcher>=0.8.0, Required for 'optional-compontents/secure-data-proxy'
+export TWITCHER_VERIFY_PATH='$(echo "${TWITCHER_PROTECTED_PATH}" | sed "s/proxy/verify/")'
+
+export DELAYED_EVAL="
+  $DELAYED_EVAL
+  TWITCHER_VERIFY_PATH
+  THREDDS_CONTEXT_ROOT
+"
+
+# add any component that this component requires to run
+COMPONENT_DEPENDENCIES="
+  ./config/magpie
+  ./config/twitcher
 "

birdhouse/config/thredds/docker-compose-extra.yml

@@ -19,6 +19,7 @@ services:
       # for reconstructing proper URL back to user when Thredds behind proxy
       # because Twitcher eats the "Host" http header set by Nginx
       PAVICS_FQDN_PUBLIC: $PAVICS_FQDN_PUBLIC
+      THREDDS_CONTEXT_ROOT: $THREDDS_CONTEXT_ROOT
     volumes:
       - thredds_persistence:/usr/local/tomcat/content/thredds
       - ${DATA_PERSIST_ROOT}/datasets:/pavics-data
@@ -37,7 +38,7 @@ services:
           "CMD",
           "curl",
           "--fail",
-          "http://localhost:8080${TWITCHER_PROTECTED_PATH}/thredds/catalog.html",
+          "http://localhost:8080${THREDDS_CONTEXT_ROOT}/catalog.html",
         ]
 
 volumes:

birdhouse/config/thredds/entrypointwrapper

@@ -21,10 +21,12 @@ fi
 WEBAPPS_ROOT="/usr/local/tomcat/webapps"
 EXISTING_CONTEXT_ROOT="thredds"
 if [ -z "$WANTED_CONTEXT_ROOT" ]; then
-    WANTED_CONTEXT_ROOT="twitcher/ows/proxy/thredds"
+    WANTED_CONTEXT_ROOT="${THREDDS_CONTEXT_ROOT}"
+    # Remove front slash '/'.
+    WANTED_CONTEXT_ROOT="$(echo "$WANTED_CONTEXT_ROOT" | sed 's@^/@@')"
 fi
 if [ -z "$WANTED_CONTEXT_ROOT_WARFILE_NAME" ]; then
-    WANTED_CONTEXT_ROOT_WARFILE_NAME="twitcher#ows#proxy#thredds"
+    WANTED_CONTEXT_ROOT_WARFILE_NAME="$(echo "$WANTED_CONTEXT_ROOT" | sed 's@/@#@g')"
 fi
 
 if [ -d "$WEBAPPS_ROOT/$EXISTING_CONTEXT_ROOT" ]; then

birdhouse/config/thredds/threddsConfig.xml.template

@@ -6,7 +6,7 @@ https://www.unidata.ucar.edu/software/tds/current/reference/ThreddsConfigXMLFile
 <threddsConfig>
   <serverInformation>
     <name>${THREDDS_ORGANIZATION} Thredds Data Server</name>
-    <logoUrl>/twitcher/ows/proxy/thredds/threddsIcon.gif</logoUrl>
+    <logoUrl>${THREDDS_CONTEXT_ROOT}/threddsIcon.gif</logoUrl>
     <logoAltText>TDS</logoAltText>
 
     <abstract>Scientific Climate Data</abstract>