Skip to content

S3 Multibucket

S3 Multibucket #31

Triggered via pull request March 29, 2024 00:59
@rw2rw2
synchronize #23
Status Success
Total duration 31s
Artifacts

linter.yml

on: pull_request_target
Run linters
24s
Run linters
Fit to window
Zoom out
Zoom in

Annotations

1 error and 8 warnings
Run linters
This action does not have permission to push to forks. You may want to run it only on `push` events.
Run linters: src/S3Commands.cc#L56
[lint] reported by reviewdog 🐶 Raw Output: src/S3Commands.cc:56:-// Takes in the configured `s3.service_url` and uses the bucket/object requested to generate src/S3Commands.cc:57:-// the virtual host URL, as well as the canonical URI (which is the path to the object). src/S3Commands.cc:58:-bool AmazonRequest::parseURL( const std::string & url, src/S3Commands.cc:59:- std::string & path ) { src/S3Commands.cc:60:- auto i = url.find( "://" ); src/S3Commands.cc:61:- if( i == std::string::npos ) { return false; } src/S3Commands.cc:62:- //protocol = substring( url, 0, i ); src/S3Commands.cc:63:- src/S3Commands.cc:64:- auto j = url.find( "/", i + 3 ); src/S3Commands.cc:65:- if( j == std::string::npos ) { src/S3Commands.cc:66:- if (style == "path") { src/S3Commands.cc:67:- host = substring( url, i + 3 ); src/S3Commands.cc:68:- } else { src/S3Commands.cc:69:- host = bucket + "." + substring( url, i + 3 ); src/S3Commands.cc:70:- } src/S3Commands.cc:71:- src/S3Commands.cc:72:- path = "/" + object; src/S3Commands.cc:73:- return true; src/S3Commands.cc:57:+// Takes in the configured `s3.service_url` and uses the bucket/object requested src/S3Commands.cc:58:+// to generate the virtual host URL, as well as the canonical URI (which is the src/S3Commands.cc:59:+// path to the object). src/S3Commands.cc:60:+bool AmazonRequest::parseURL(const std::string &url, std::string &path) { src/S3Commands.cc:61:+ auto i = url.find("://"); src/S3Commands.cc:62:+ if (i == std::string::npos) { src/S3Commands.cc:63:+ return false; src/S3Commands.cc:64:+ } src/S3Commands.cc:65:+ // protocol = substring( url, 0, i ); src/S3Commands.cc:66:+ src/S3Commands.cc:67:+ auto j = url.find("/", i + 3); src/S3Commands.cc:68:+ if (j == std::string::npos) { src/S3Commands.cc:69:+ if (style == "path") { src/S3Commands.cc:70:+ host = substring(url, i + 3); src/S3Commands.cc:71:+ } else { src/S3Commands.cc:72:+ host = bucket + "." + substring(url, i + 3);
Run linters: src/S3File.cc#L48
[lint] reported by reviewdog 🐶 Raw Output: src/S3File.cc:48:-S3File::S3File(XrdSysError &log, S3FileSystem *oss) : src/S3File.cc:49:- m_log(log), src/S3File.cc:50:- m_oss(oss), src/S3File.cc:51:- content_length(0), src/S3File.cc:52:- last_modified(0) src/S3File.cc:53:-{} src/S3File.cc:54:- src/S3File.cc:55:- src/S3File.cc:56:-int src/S3File.cc:57:-parse_path( const S3FileSystem & fs, const char * fullPath, std::string & exposedPath, std::string & object ) { src/S3File.cc:58:- // src/S3File.cc:59:- // Check the path for validity. src/S3File.cc:60:- // src/S3File.cc:61:- std::filesystem::path p(fullPath); src/S3File.cc:62:- auto pathComponents = p.begin(); src/S3File.cc:63:- src/S3File.cc:64:- ++pathComponents; src/S3File.cc:65:- if( pathComponents == p.end() ) { return -ENOENT; } src/S3File.cc:66:- exposedPath = *pathComponents; src/S3File.cc:67:- src/S3File.cc:68:- // Objects names may contain path separators. src/S3File.cc:69:- ++pathComponents; src/S3File.cc:70:- if( pathComponents == p.end() ) { return -ENOENT; } src/S3File.cc:71:- src/S3File.cc:72:- std::filesystem::path objectPath = *pathComponents++; src/S3File.cc:73:- for( ; pathComponents != p.end(); ++pathComponents ) { src/S3File.cc:74:- objectPath /= (* pathComponents); src/S3File.cc:75:- } src/S3File.cc:76:- object = objectPath.string(); src/S3File.cc:77:- src/S3File.cc:78:- fprintf( stderr, "object = %s\n", object.c_str() ); src/S3File.cc:79:- src/S3File.cc:80:- return 0; src/S3File.cc:48:+S3File::S3File(XrdSysError &log, S3FileSystem *oss) src/S3File.cc:49:+ : m_log(log), m_oss(oss), content_length(0), last_modified(0) {} src/S3File.cc:50:+ src/S3File.cc:51:+int parse_path(const S3FileSystem &fs, const char *fullPath, src/S3File.cc:52:+ std::string &exposedPath, std::string &object) { src/S3File.cc:53:+ // src/S3File.cc:54:+ // Check the path for validity. src/S3File.cc:55:+ // src/S3File.cc:56:+ std::filesystem::path p(fullPath); src/S3File.cc:57:+ auto pathComponents = p.begin(); src/S3File.cc:58:+ src/S3File.cc:59:+ ++pathComponents; src/S3File.cc:60:+ if (pathComponents == p.end()) { src/S3File.cc:61:+ return -ENOENT; src/S3File.cc:62:+ } src/S3File.cc:63:+ exposedPath = *pathComponents; src/S3File.cc:64:+ src/S3File.cc:65:+ // Objects names may contain path separators. src/S3File.cc:66:+ ++pathComponents; src/S3File.cc:67:+ if (pathComponents == p.end()) { src/S3File.cc:68:+ return -ENOENT; src/S3File.cc:69:+ } src/S3File.cc:70:+ src/S3File.cc:71:+ std::filesystem::path objectPath = *pathComponents++; src/S3File.cc:72:+ for (; pathComponents != p.end(); ++pathComponents) { src/S3File.cc:73:+ objectPath /= (*pathComponents); src/S3File.cc:74:+ } src/S3File.cc:75:+ object = objectPath.string(); src/S3File.cc:76:+ src/S3File.cc:77:+ fprintf(stderr, "object = %s\n", object.c_str()); src/S3File.cc:78:+ src/S3File.cc:79:+ return 0;
Run linters: src/S3File.cc#L83
[lint] reported by reviewdog 🐶 Raw Output: src/S3File.cc:83:- src/S3File.cc:84:-int src/S3File.cc:85:-S3File::Open(const char *path, int Oflag, mode_t Mode, XrdOucEnv &env) src/S3File.cc:86:-{ src/S3File.cc:87:- std::string exposedPath, object; src/S3File.cc:88:- int rv = parse_path( * m_oss, path, exposedPath, object ); src/S3File.cc:89:- if( rv != 0 ) { return rv; } src/S3File.cc:90:- if(!m_oss->exposedPathExists(exposedPath)) return -ENOENT; src/S3File.cc:91:- src/S3File.cc:92:- std::string configured_s3_region = m_oss->getS3Region(exposedPath); src/S3File.cc:93:- std::string configured_s3_service_url = m_oss->getS3ServiceURL(exposedPath); src/S3File.cc:94:- std::string configured_s3_access_key = m_oss->getS3AccessKeyFile(exposedPath); src/S3File.cc:95:- std::string configured_s3_secret_key = m_oss->getS3SecretKeyFile(exposedPath); src/S3File.cc:96:- std::string configured_s3_bucket_name = m_oss->getS3BucketName(exposedPath); src/S3File.cc:97:- src/S3File.cc:98:- // We used to query S3 here to see if the object existed, but of course src/S3File.cc:99:- // if you're creating a file on upload, you don't care. src/S3File.cc:100:- src/S3File.cc:101:- this->s3_object_name = object; src/S3File.cc:102:- this->s3_bucket_name = configured_s3_bucket_name; src/S3File.cc:103:- this->s3_service_url = configured_s3_service_url; src/S3File.cc:104:- this->s3_access_key = configured_s3_access_key; src/S3File.cc:105:- this->s3_secret_key = configured_s3_secret_key; src/S3File.cc:106:- std::string configured_s3_url_style = m_oss->getS3URLStyle(); src/S3File.cc:107:- src/S3File.cc:108:- src/S3File.cc:109:- // We used to query S3 here to see if the object existed, but of course src/S3File.cc:110:- // if you're creating a file on upload, you don't care. src/S3File.cc:111:- src/S3File.cc:112:- this->s3_object_name = object; src/S3File.cc:113:- this->s3_bucket_name = configured_s3_bucket_name; src/S3File.cc:114:- this->s3_service_url = configured_s3_service_url; src/S3File.cc:115:- this->s3_access_key = configured_s3_access_key; src/S3File.cc:116:- this->s3_secret_key = configured_s3_secret_key; src/S3File.cc:117:- this->s3_url_style = configured_s3_url_style; src/S3File.cc:118:- return 0; src/S3File.cc:82:+int S3File::Open(const char *path, int Oflag, mode_t Mode, XrdOucEnv &env) { src/S3File.cc:83:+ std::string exposedPath, object; src/S3File.cc:84:+ int rv = parse_path(*m_oss, path, exposedPath, object); src/S3File.cc:85:+ if (rv != 0) { src/S3File.cc:86:+ return rv; src/S3File.cc:87:+ } src/S3File.cc:88:+ if (!m_oss->exposedPathExists(exposedPath)) src/S3File.cc:89:+ return -ENOENT; src/S3File.cc:90:+ src/S3File.cc:91:+ std::string configured_s3_region = m_oss->getS3Region(exposedPath); src/S3File.cc:92:+ std::string configured_s3_service_url = m_oss->getS3ServiceURL(exposedPath); src/S3File.cc:93:+ std::string configured_s3_access_key = m_oss->getS3AccessKeyFile(exposedPath); src/S3File.cc:94:+ std::string configured_s3_secret_key = m_oss->getS3SecretKeyFile(exposedPath); src/S3File.cc:95:+ std::string configured_s3_bucket_name = m_oss->getS3BucketName(exposedPath); src/S3File.cc:96:+ src/S3File.cc:97:+ // We used to query S3 here to see if the object existed, but of course src/S3File.cc:98:+ // if you're creating a file on upload, you don't care. src/S3File.cc:99:+ src/S3File.cc:100:+ this->s3_object_name = object; src/S3File.cc:101:+ this->s3_bucket_name = configured_s3_bucket_name; src/S3File.cc:102:+ this->s3_service_url = configured_s3_service_url; src/S3File.cc:103:+ this->s3_access_key = configured_s3_access_key; src/S3File.cc:104:+ this->s3_secret_key = configured_s3_secret_key; src/S3File.cc:105:+ std::string configured_s3_url_style = m_oss->getS3URLStyle(); src/S3File.cc:106:+ src/S3File.cc:107:+ // We used to query S3 here to see if the object existed, but of course src/S3File.cc:108:+ // if you're creating a file on upload, you don't care. src/S3File.cc:109:+ src/S3File.cc:110:+ this->s3_object_name = object; src/S3Fil
Run linters: src/S3File.cc#L147
[lint] reported by reviewdog 🐶 Raw Output: src/S3File.cc:147:- src/S3File.cc:148:-int src/S3File.cc:149:-S3File::Fstat(struct stat *buff) src/S3File.cc:150:-{ src/S3File.cc:151:- AmazonS3Head head( src/S3File.cc:152:- this->s3_service_url, src/S3File.cc:153:- this->s3_access_key, src/S3File.cc:154:- this->s3_secret_key, src/S3File.cc:155:- this->s3_bucket_name, src/S3File.cc:156:- this->s3_object_name, src/S3File.cc:157:- this->s3_url_style, src/S3File.cc:158:- m_log src/S3File.cc:159:- ); src/S3File.cc:160:- src/S3File.cc:161:- if(! head.SendRequest()) { src/S3File.cc:162:- // SendRequest() returns false for all errors, including ones src/S3File.cc:163:- // where the server properly responded with something other src/S3File.cc:164:- // than code 200. If xrootd wants us to distinguish between src/S3File.cc:165:- // these cases, head.getResponseCode() is initialized to 0, so src/S3File.cc:166:- // we can check. src/S3File.cc:167:- std::stringstream ss; src/S3File.cc:168:- ss << "Failed to send HeadObject command: " << head.getResponseCode() << "'" << head.getResultString() << "'"; src/S3File.cc:169:- m_log.Log(LogMask::Warning, "S3File::Fstat", ss.str().c_str()); src/S3File.cc:170:- return -ENOENT; src/S3File.cc:171:- } src/S3File.cc:172:- src/S3File.cc:173:- src/S3File.cc:174:- std::string headers = head.getResultString(); src/S3File.cc:175:- src/S3File.cc:176:- std::string line; src/S3File.cc:177:- size_t current_newline = 0; src/S3File.cc:178:- size_t next_newline = std::string::npos; src/S3File.cc:179:- size_t last_character = headers.size(); src/S3File.cc:180:- while( current_newline != std::string::npos && current_newline != last_character - 1 ) { src/S3File.cc:181:- next_newline = headers.find( "\r\n", current_newline + 2); src/S3File.cc:182:- line = substring( headers, current_newline + 2, next_newline ); src/S3File.cc:183:- src/S3File.cc:184:- size_t colon = line.find(":"); src/S3File.cc:185:- if( colon != std::string::npos && colon != line.size() ) { src/S3File.cc:186:- std::string attr = substring( line, 0, colon ); src/S3File.cc:187:- std::string value = substring( line, colon + 1 ); src/S3File.cc:188:- trim(value); src/S3File.cc:189:- toLower(attr); src/S3File.cc:190:- src/S3File.cc:191:- if( attr == "content-length" ) { src/S3File.cc:192:- this->content_length = std::stol(value); src/S3File.cc:193:- } else if( attr == "last-modified" ) { src/S3File.cc:194:- struct tm t; src/S3File.cc:195:- char * eos = strptime( value.c_str(), src/S3File.cc:196:- "%a, %d %b %Y %T %Z", src/S3File.cc:197:- & t ); src/S3File.cc:198:- if( eos == & value.c_str()[value.size()] ) { src/S3File.cc:199:- time_t epoch = timegm(& t); src/S3File.cc:200:- if( epoch != -1 ) { src/S3File.cc:201:- this->last_modified = epoch; src/S3File.cc:202:- } src/S3File.cc:203:- } src/S3File.cc:204:- } src/S3File.cc:137:+int S3File::Fstat(struct stat *buff) { src/S3File.cc:138:+ AmazonS3Head head(this->s3_service_url, this->s3_access_key, src/S3File.cc:139:+ this->s3_secret_key, this->s3_bucket_name, src/S3File.cc:140:+ this->s3_object_name, this->s3_url_style, m_log); src/S3File.cc:141:+ src/S3File.cc:142:+ if (!head.SendRequest()) { src/S3File.cc:143:+ // SendRequest() returns false for all errors, including ones src/S3File.cc:144:+ // where the server properly responded with something other src/S3File.cc:145:+ // than code 200. If xrootd wants us to distinguish between src/S3File.cc:146:+ // these cases, head.getResponseCode() is initialized to 0, so src/S3File.cc:147:+ // we can check. src/S3File.cc:148:+ std::stringstream ss; src/S3File.cc:149:+ ss <<
Run linters: src/S3FileSystem.cc#L63
[lint] reported by reviewdog 🐶 Raw Output: src/S3FileSystem.cc:63:- } src/S3FileSystem.cc:64:- return true; src/S3FileSystem.cc:65:-} src/S3FileSystem.cc:66:- src/S3FileSystem.cc:67:- src/S3FileSystem.cc:68:-bool src/S3FileSystem.cc:69:-S3FileSystem::Config(XrdSysLogger *lp, const char *configfn) src/S3FileSystem.cc:70:-{ src/S3FileSystem.cc:71:- XrdOucEnv myEnv; src/S3FileSystem.cc:72:- XrdOucStream Config(&m_log, getenv("XRDINSTANCE"), &myEnv, "=====> "); src/S3FileSystem.cc:73:- src/S3FileSystem.cc:74:- int cfgFD = open(configfn, O_RDONLY, 0); src/S3FileSystem.cc:75:- if (cfgFD < 0) { src/S3FileSystem.cc:76:- m_log.Emsg("Config", errno, "open config file", configfn); src/S3FileSystem.cc:85:+ } src/S3FileSystem.cc:86:+ if (newAccessInfo->getS3Region().empty()) { src/S3FileSystem.cc:87:+ m_log.Emsg("Config", "s3.region not specified");
Run linters: src/S3FileSystem.cc#L79
[lint] reported by reviewdog 🐶 Raw Output: src/S3FileSystem.cc:79:- src/S3FileSystem.cc:80:- char * temporary; src/S3FileSystem.cc:81:- std::string value; src/S3FileSystem.cc:82:- std::string attribute; src/S3FileSystem.cc:83:- Config.Attach(cfgFD); src/S3FileSystem.cc:84:- S3AccessInfo *newAccessInfo = new S3AccessInfo(); src/S3FileSystem.cc:85:- std::string exposedPath; src/S3FileSystem.cc:86:- while ((temporary = Config.GetMyFirstWord())) { src/S3FileSystem.cc:87:- attribute = temporary; src/S3FileSystem.cc:88:- temporary = Config.GetWord(); src/S3FileSystem.cc:89:- if(attribute == "s3.end") { src/S3FileSystem.cc:90:- s3_access_map[exposedPath] = newAccessInfo; src/S3FileSystem.cc:91:- if(newAccessInfo->getS3ServiceName().empty()) { src/S3FileSystem.cc:92:- m_log.Emsg("Config", "s3.service_name not specified"); src/S3FileSystem.cc:93:- return false; src/S3FileSystem.cc:94:- } src/S3FileSystem.cc:95:- if(newAccessInfo->getS3Region().empty()) { src/S3FileSystem.cc:96:- m_log.Emsg("Config", "s3.region not specified"); src/S3FileSystem.cc:97:- return false; src/S3FileSystem.cc:98:- } src/S3FileSystem.cc:99:- newAccessInfo = new S3AccessInfo(); src/S3FileSystem.cc:100:- exposedPath = ""; src/S3FileSystem.cc:101:- continue; src/S3FileSystem.cc:102:- } src/S3FileSystem.cc:103:- if(! temporary) { continue; } src/S3FileSystem.cc:104:- value = temporary; src/S3FileSystem.cc:105:- src/S3FileSystem.cc:106:- if(!handle_required_config("s3.path_name",value) ) { Config.Close(); return false; } src/S3FileSystem.cc:107:- if(!handle_required_config("s3.bucket_name",value) ) { Config.Close(); return false; } src/S3FileSystem.cc:108:- if(!handle_required_config("s3.service_name",value) ) { Config.Close(); return false; } src/S3FileSystem.cc:109:- if(!handle_required_config("s3.region", value ) ) { Config.Close(); return false; } src/S3FileSystem.cc:110:- if(!handle_required_config("s3.service_url", value) ) { Config.Close(); return false; } src/S3FileSystem.cc:111:- if(!handle_required_config("s3.access_key_file", value) ) { Config.Close(); return false; } src/S3FileSystem.cc:112:- if(!handle_required_config("s3.secret_key_file", value) ) { Config.Close(); return false; } src/S3FileSystem.cc:113:- if(!handle_required_config("s3.url_style", value) ) { Config.Close(); return false; } src/S3FileSystem.cc:114:- src/S3FileSystem.cc:115:- if(attribute == "s3.path_name") exposedPath = value; src/S3FileSystem.cc:116:- else if(attribute == "s3.bucket_name") newAccessInfo->setS3BucketName(value); src/S3FileSystem.cc:117:- else if(attribute == "s3.service_name") newAccessInfo->setS3ServiceName(value); src/S3FileSystem.cc:118:- else if(attribute == "s3.region") newAccessInfo->setS3Region(value); src/S3FileSystem.cc:119:- else if(attribute == "s3.access_key_file") newAccessInfo->setS3AccessKeyFile(value); src/S3FileSystem.cc:120:- else if(attribute == "s3.secret_key_file") newAccessInfo->setS3SecretKeyFile(value); src/S3FileSystem.cc:121:- else if(attribute == "s3.service_url") newAccessInfo->setS3ServiceUrl(value); src/S3FileSystem.cc:122:- else if(attribute == "s3.url_style") this->s3_url_style = value; src/S3FileSystem.cc:123:- src/S3FileSystem.cc:94:+ if (!temporary) { src/S3FileSystem.cc:95:+ continue;
Run linters: src/S3FileSystem.cc#L126
[lint] reported by reviewdog 🐶 Raw Output: src/S3FileSystem.cc:126:- if( this->s3_url_style.empty() ) { src/S3FileSystem.cc:127:- m_log.Emsg("Config", "s3.url_style not specified"); src/S3FileSystem.cc:128:- return false; src/S3FileSystem.cc:129:- } else { src/S3FileSystem.cc:130:- // We want this to be case-insensitive. src/S3FileSystem.cc:131:- toLower( this->s3_url_style ); src/S3FileSystem.cc:99:+ if (!handle_required_config("s3.path_name", value)) { src/S3FileSystem.cc:100:+ Config.Close(); src/S3FileSystem.cc:101:+ return false;
Run linters
Node.js 16 actions are deprecated. Please update the following actions to use Node.js 20: actions/checkout@v3, wearerequired/lint-action@v2. For more information see: https://github.blog/changelog/2023-09-22-github-actions-transitioning-from-node-16-to-node-20/.