Merge pull request #18 from jhiemstrawisc/pr-16

Add path-style hosting to plugin
PelicanPlatform · Feb 29, 2024 · 91c03b4 · 91c03b4
2 parents 6c240c8 + af9b857
commit 91c03b4
Show file tree

Hide file tree

Showing 6 changed files with 54 additions and 15 deletions.
diff --git a/src/S3Commands.cc b/src/S3Commands.cc
@@ -47,7 +47,12 @@ bool AmazonRequest::parseURL(	const std::string & url,
 
     auto j = url.find( "/", i + 3 );
     if( j == std::string::npos ) {
-        host = bucket + "." + substring( url, i + 3 );
+        if (style == "path") {
+            host = substring( url, i + 3 );
+        } else {
+            host = bucket + "." + substring( url, i + 3 );
+        }
+
         path = "/" + object;
         return true;
     }

diff --git a/src/S3Commands.hh b/src/S3Commands.hh
@@ -11,14 +11,16 @@ public:
         const std::string & skf,
         const std::string & b,
         const std::string & o,
+        const std::string & style,
         int sv = 4
     ) :
     HTTPRequest( s ),
     accessKeyFile(akf),
     secretKeyFile(skf),
     signatureVersion(sv),
     bucket(b),
-    object(o)
+    object(o),
+    style(style)
     { 
         requiresSignature = true;
         // Start off by parsing the hostUrl, which we use in conjunction with the bucket to fill in the host (for setting host header).
@@ -33,8 +35,11 @@ public:
 
         // Now that we have the host and canonicalURI, we can build the actual url we perform the curl against.
         // Using the previous example, we'd get a new hostUrl of "https://my-bucket.my-url.com:443/my-object".
-        hostUrl = protocol + "://" +
-            host + canonicalURI;
+        if (style == "path") {
+            hostUrl = protocol + "://" + host + "/" + b + canonicalURI;
+        } else {
+            hostUrl = protocol + "://" + host + canonicalURI;            
+        }
 
         // If we can, set the region based on the host.
         size_t secondDot = host.find( ".", 2 + 1 );
@@ -77,6 +82,7 @@ protected:
     std::string region;
     std::string service;
 
+    std::string style;
 private:
     bool createV4Signature( const std::string & payload, std::string & authorizationHeader, bool sendContentSHA = false );
 
@@ -91,9 +97,10 @@ public:
         const std::string & akf,
         const std::string & skf,
         const std::string & b,
-        const std::string & o
+        const std::string & o,
+        const std::string & style
     ) :
-    AmazonRequest(s, akf, skf, b, o){}
+    AmazonRequest(s, akf, skf, b, o, style){}
 
     virtual ~AmazonS3Upload();
 
@@ -111,9 +118,10 @@ public:
         const std::string & akf,
         const std::string & skf,
         const std::string & b,
-        const std::string & o
+        const std::string & o,
+        const std::string & style
     ) :
-    AmazonRequest(s, akf, skf, b, o){}
+    AmazonRequest(s, akf, skf, b, o, style){}
 
     virtual ~AmazonS3Download();
 
@@ -128,9 +136,10 @@ public:
         const std::string & akf,
         const std::string & skf,
         const std::string & b,
-        const std::string & o
+        const std::string & o,
+        const std::string & style
     ) :
-    AmazonRequest(s, akf, skf, b, o){}
+    AmazonRequest(s, akf, skf, b, o, style){}
 
     virtual ~AmazonS3Head();
 

diff --git a/src/S3File.cc b/src/S3File.cc
@@ -6,6 +6,7 @@
 #include "XrdVersion.hh"
 #include "S3FileSystem.hh"
 #include "S3File.hh"
+#include "stl_string_utils.hh"
 
 #include <curl/curl.h>
 
@@ -95,6 +96,7 @@ S3File::Open(const char *path, int Oflag, mode_t Mode, XrdOucEnv &env)
     std::string configured_s3_service_url = m_oss->getS3ServiceURL();
     std::string configured_s3_access_key = m_oss->getS3AccessKeyFile();
     std::string configured_s3_secret_key = m_oss->getS3SecretKeyFile();
+    std::string configured_s3_url_style = m_oss->getS3URLStyle();
 
 
     // We used to query S3 here to see if the object existed, but of course
@@ -105,6 +107,7 @@ S3File::Open(const char *path, int Oflag, mode_t Mode, XrdOucEnv &env)
     this->s3_service_url = configured_s3_service_url;
     this->s3_access_key = configured_s3_access_key;
     this->s3_secret_key = configured_s3_secret_key;
+    this->s3_url_style = configured_s3_url_style;
     return 0;
 }
 
@@ -117,7 +120,8 @@ S3File::Read(void *buffer, off_t offset, size_t size)
         this->s3_access_key,
         this->s3_secret_key,
         this->s3_bucket_name,
-        this->s3_object_name
+        this->s3_object_name,
+        this->s3_url_style
     );
 
 
@@ -140,7 +144,8 @@ S3File::Fstat(struct stat *buff)
         this->s3_access_key,
         this->s3_secret_key,
         this->s3_bucket_name,
-        this->s3_object_name
+        this->s3_object_name,
+        this->s3_url_style
     );
 
     if(! head.SendRequest()) {
@@ -169,10 +174,11 @@ S3File::Fstat(struct stat *buff)
             std::string attr = substring( line, 0, colon );
             std::string value = substring( line, colon + 1 );
             trim(value);
+            toLower(attr);
 
-            if( attr == "Content-Length" ) {
+            if( attr == "content-length" ) {
                 this->content_length = std::stol(value);
-            } else if( attr == "Last-Modified" ) {
+            } else if( attr == "last-modified" ) {
                 struct tm t;
                 char * eos = strptime( value.c_str(),
                     "%a, %d %b %Y %T %Z",
@@ -213,7 +219,8 @@ S3File::Write(const void *buffer, off_t offset, size_t size)
         this->s3_access_key,
         this->s3_secret_key,
         this->s3_bucket_name,
-        this->s3_object_name
+        this->s3_object_name,
+        this->s3_url_style
     );
 
     std::string payload( (char *)buffer, size );

diff --git a/src/S3File.hh b/src/S3File.hh
@@ -125,6 +125,7 @@ private:
     std::string s3_object_name;
     std::string s3_access_key;
     std::string s3_secret_key;
+    std::string s3_url_style;
 
     size_t content_length;
     time_t last_modified;

diff --git a/src/S3FileSystem.cc b/src/S3FileSystem.cc
@@ -5,6 +5,7 @@
 #include "S3FileSystem.hh"
 #include "S3Directory.hh"
 #include "S3File.hh"
+#include "stl_string_utils.hh"
 
 #include <memory>
 #include <vector>
@@ -85,6 +86,8 @@ S3FileSystem::Config(XrdSysLogger *lp, const char *configfn)
             value, this->s3_access_key_file ) ) { Config.Close(); return false; }
         if(! handle_required_config( attribute, "s3.secret_key_file",
             value, this->s3_secret_key_file ) ) { Config.Close(); return false; }
+        if(! handle_required_config( attribute, "s3.url_style",
+            value, this->s3_url_style ) ) { Config.Close(); return false; }
     }
 
     if( this->s3_service_name.empty() ) {
@@ -95,6 +98,17 @@ S3FileSystem::Config(XrdSysLogger *lp, const char *configfn)
         m_log.Emsg("Config", "s3.region not specified");
         return false;
     }
+    if( this->s3_url_style.empty() ) {
+        m_log.Emsg("Config", "s3.url_style not specified");
+        return false;
+    } else {
+        // We want this to be case-insensitive.
+        toLower( this->s3_url_style );
+    }
+    if( this->s3_url_style != "virtual" && this->s3_url_style != "path" ) {
+        m_log.Emsg("Config", "invalid s3.url_style specified. Must be 'virtual' or 'path'");
+        return false;
+    }
 
     int retc = Config.LastError();
     if( retc ) {

diff --git a/src/S3FileSystem.hh b/src/S3FileSystem.hh
@@ -60,6 +60,7 @@ public:
     const std::string & getS3ServiceName() const { return s3_service_name; }
     const std::string & getS3Region() const { return s3_region; }
     const std::string & getS3ServiceURL() const { return s3_service_url; }
+    const std::string & getS3URLStyle() const { return s3_url_style; }
 
     const std::string & getS3AccessKeyFile() const { return s3_access_key_file; }
     const std::string & getS3SecretKeyFile() const { return s3_secret_key_file; }
@@ -81,4 +82,6 @@ private:
 
     std::string s3_access_key_file;
     std::string s3_secret_key_file;
+
+    std::string s3_url_style;
 };