S3 Multibucket #33
Annotations
1 error and 8 warnings
Run linters
This action does not have permission to push to forks. You may want to run it only on `push` events.
|
Run linters:
src/S3Commands.cc#L56
[lint] reported by reviewdog 🐶
Raw Output:
src/S3Commands.cc:56:-// Takes in the configured `s3.service_url` and uses the bucket/object requested to generate
src/S3Commands.cc:57:-// the virtual host URL, as well as the canonical URI (which is the path to the object).
src/S3Commands.cc:58:-bool AmazonRequest::parseURL( const std::string & url,
src/S3Commands.cc:59:- std::string & path ) {
src/S3Commands.cc:60:- auto i = url.find( "://" );
src/S3Commands.cc:61:- if( i == std::string::npos ) { return false; }
src/S3Commands.cc:62:- //protocol = substring( url, 0, i );
src/S3Commands.cc:63:-
src/S3Commands.cc:64:- auto j = url.find( "/", i + 3 );
src/S3Commands.cc:65:- if( j == std::string::npos ) {
src/S3Commands.cc:66:- if (style == "path") {
src/S3Commands.cc:67:- host = substring( url, i + 3 );
src/S3Commands.cc:68:- } else {
src/S3Commands.cc:69:- host = bucket + "." + substring( url, i + 3 );
src/S3Commands.cc:70:- }
src/S3Commands.cc:71:-
src/S3Commands.cc:72:- path = "/" + object;
src/S3Commands.cc:73:- return true;
src/S3Commands.cc:57:+// Takes in the configured `s3.service_url` and uses the bucket/object requested
src/S3Commands.cc:58:+// to generate the virtual host URL, as well as the canonical URI (which is the
src/S3Commands.cc:59:+// path to the object).
src/S3Commands.cc:60:+bool AmazonRequest::parseURL(const std::string &url, std::string &path) {
src/S3Commands.cc:61:+ auto i = url.find("://");
src/S3Commands.cc:62:+ if (i == std::string::npos) {
src/S3Commands.cc:63:+ return false;
src/S3Commands.cc:64:+ }
src/S3Commands.cc:65:+ // protocol = substring( url, 0, i );
src/S3Commands.cc:66:+
src/S3Commands.cc:67:+ auto j = url.find("/", i + 3);
src/S3Commands.cc:68:+ if (j == std::string::npos) {
src/S3Commands.cc:69:+ if (style == "path") {
src/S3Commands.cc:70:+ host = substring(url, i + 3);
src/S3Commands.cc:71:+ } else {
src/S3Commands.cc:72:+ host = bucket + "." + substring(url, i + 3);
|
Run linters:
src/S3File.cc#L48
[lint] reported by reviewdog 🐶
Raw Output:
src/S3File.cc:48:-S3File::S3File(XrdSysError &log, S3FileSystem *oss) :
src/S3File.cc:49:- m_log(log),
src/S3File.cc:50:- m_oss(oss),
src/S3File.cc:51:- content_length(0),
src/S3File.cc:52:- last_modified(0)
src/S3File.cc:53:-{}
src/S3File.cc:54:-
src/S3File.cc:55:-
src/S3File.cc:56:-int
src/S3File.cc:57:-parse_path( const S3FileSystem & fs, const char * fullPath, std::string & exposedPath, std::string & object ) {
src/S3File.cc:58:- //
src/S3File.cc:59:- // Check the path for validity.
src/S3File.cc:60:- //
src/S3File.cc:61:- std::filesystem::path p(fullPath);
src/S3File.cc:62:- auto pathComponents = p.begin();
src/S3File.cc:63:-
src/S3File.cc:64:- ++pathComponents;
src/S3File.cc:65:- if( pathComponents == p.end() ) { return -ENOENT; }
src/S3File.cc:66:- exposedPath = *pathComponents;
src/S3File.cc:67:-
src/S3File.cc:68:- // Objects names may contain path separators.
src/S3File.cc:69:- ++pathComponents;
src/S3File.cc:70:- if( pathComponents == p.end() ) { return -ENOENT; }
src/S3File.cc:71:-
src/S3File.cc:72:- std::filesystem::path objectPath = *pathComponents++;
src/S3File.cc:73:- for( ; pathComponents != p.end(); ++pathComponents ) {
src/S3File.cc:74:- objectPath /= (* pathComponents);
src/S3File.cc:75:- }
src/S3File.cc:76:- object = objectPath.string();
src/S3File.cc:77:-
src/S3File.cc:78:- fprintf( stderr, "object = %s\n", object.c_str() );
src/S3File.cc:79:-
src/S3File.cc:80:- return 0;
src/S3File.cc:48:+S3File::S3File(XrdSysError &log, S3FileSystem *oss)
src/S3File.cc:49:+ : m_log(log), m_oss(oss), content_length(0), last_modified(0) {}
src/S3File.cc:50:+
src/S3File.cc:51:+int parse_path(const S3FileSystem &fs, const char *fullPath,
src/S3File.cc:52:+ std::string &exposedPath, std::string &object) {
src/S3File.cc:53:+ //
src/S3File.cc:54:+ // Check the path for validity.
src/S3File.cc:55:+ //
src/S3File.cc:56:+ std::filesystem::path p(fullPath);
src/S3File.cc:57:+ auto pathComponents = p.begin();
src/S3File.cc:58:+
src/S3File.cc:59:+ ++pathComponents;
src/S3File.cc:60:+ if (pathComponents == p.end()) {
src/S3File.cc:61:+ return -ENOENT;
src/S3File.cc:62:+ }
src/S3File.cc:63:+ exposedPath = *pathComponents;
src/S3File.cc:64:+
src/S3File.cc:65:+ // Objects names may contain path separators.
src/S3File.cc:66:+ ++pathComponents;
src/S3File.cc:67:+ if (pathComponents == p.end()) {
src/S3File.cc:68:+ return -ENOENT;
src/S3File.cc:69:+ }
src/S3File.cc:70:+
src/S3File.cc:71:+ std::filesystem::path objectPath = *pathComponents++;
src/S3File.cc:72:+ for (; pathComponents != p.end(); ++pathComponents) {
src/S3File.cc:73:+ objectPath /= (*pathComponents);
src/S3File.cc:74:+ }
src/S3File.cc:75:+ object = objectPath.string();
src/S3File.cc:76:+
src/S3File.cc:77:+ fprintf(stderr, "object = %s\n", object.c_str());
src/S3File.cc:78:+
src/S3File.cc:79:+ return 0;
|
Run linters:
src/S3File.cc#L83
[lint] reported by reviewdog 🐶
Raw Output:
src/S3File.cc:83:-
src/S3File.cc:84:-int
src/S3File.cc:85:-S3File::Open(const char *path, int Oflag, mode_t Mode, XrdOucEnv &env)
src/S3File.cc:86:-{
src/S3File.cc:87:- std::string exposedPath, object;
src/S3File.cc:88:- int rv = parse_path( * m_oss, path, exposedPath, object );
src/S3File.cc:89:- if( rv != 0 ) { return rv; }
src/S3File.cc:90:- if(!m_oss->exposedPathExists(exposedPath)) return -ENOENT;
src/S3File.cc:91:-
src/S3File.cc:92:- std::string configured_s3_region = m_oss->getS3Region(exposedPath);
src/S3File.cc:93:- std::string configured_s3_service_url = m_oss->getS3ServiceURL(exposedPath);
src/S3File.cc:94:- std::string configured_s3_access_key = m_oss->getS3AccessKeyFile(exposedPath);
src/S3File.cc:95:- std::string configured_s3_secret_key = m_oss->getS3SecretKeyFile(exposedPath);
src/S3File.cc:96:- std::string configured_s3_bucket_name = m_oss->getS3BucketName(exposedPath);
src/S3File.cc:97:-
src/S3File.cc:98:- // We used to query S3 here to see if the object existed, but of course
src/S3File.cc:99:- // if you're creating a file on upload, you don't care.
src/S3File.cc:100:-
src/S3File.cc:101:- this->s3_object_name = object;
src/S3File.cc:102:- this->s3_bucket_name = configured_s3_bucket_name;
src/S3File.cc:103:- this->s3_service_url = configured_s3_service_url;
src/S3File.cc:104:- this->s3_access_key = configured_s3_access_key;
src/S3File.cc:105:- this->s3_secret_key = configured_s3_secret_key;
src/S3File.cc:106:- std::string configured_s3_url_style = m_oss->getS3URLStyle();
src/S3File.cc:107:-
src/S3File.cc:108:-
src/S3File.cc:109:- // We used to query S3 here to see if the object existed, but of course
src/S3File.cc:110:- // if you're creating a file on upload, you don't care.
src/S3File.cc:111:-
src/S3File.cc:112:- this->s3_object_name = object;
src/S3File.cc:113:- this->s3_bucket_name = configured_s3_bucket_name;
src/S3File.cc:114:- this->s3_service_url = configured_s3_service_url;
src/S3File.cc:115:- this->s3_access_key = configured_s3_access_key;
src/S3File.cc:116:- this->s3_secret_key = configured_s3_secret_key;
src/S3File.cc:117:- this->s3_url_style = configured_s3_url_style;
src/S3File.cc:118:- return 0;
src/S3File.cc:82:+int S3File::Open(const char *path, int Oflag, mode_t Mode, XrdOucEnv &env) {
src/S3File.cc:83:+ std::string exposedPath, object;
src/S3File.cc:84:+ int rv = parse_path(*m_oss, path, exposedPath, object);
src/S3File.cc:85:+ if (rv != 0) {
src/S3File.cc:86:+ return rv;
src/S3File.cc:87:+ }
src/S3File.cc:88:+ if (!m_oss->exposedPathExists(exposedPath))
src/S3File.cc:89:+ return -ENOENT;
src/S3File.cc:90:+
src/S3File.cc:91:+ std::string configured_s3_region = m_oss->getS3Region(exposedPath);
src/S3File.cc:92:+ std::string configured_s3_service_url = m_oss->getS3ServiceURL(exposedPath);
src/S3File.cc:93:+ std::string configured_s3_access_key = m_oss->getS3AccessKeyFile(exposedPath);
src/S3File.cc:94:+ std::string configured_s3_secret_key = m_oss->getS3SecretKeyFile(exposedPath);
src/S3File.cc:95:+ std::string configured_s3_bucket_name = m_oss->getS3BucketName(exposedPath);
src/S3File.cc:96:+
src/S3File.cc:97:+ // We used to query S3 here to see if the object existed, but of course
src/S3File.cc:98:+ // if you're creating a file on upload, you don't care.
src/S3File.cc:99:+
src/S3File.cc:100:+ this->s3_object_name = object;
src/S3File.cc:101:+ this->s3_bucket_name = configured_s3_bucket_name;
src/S3File.cc:102:+ this->s3_service_url = configured_s3_service_url;
src/S3File.cc:103:+ this->s3_access_key = configured_s3_access_key;
src/S3File.cc:104:+ this->s3_secret_key = configured_s3_secret_key;
src/S3File.cc:105:+ std::string configured_s3_url_style = m_oss->getS3URLStyle();
src/S3File.cc:106:+
src/S3File.cc:107:+ // We used to query S3 here to see if the object existed, but of course
src/S3File.cc:108:+ // if you're creating a file on upload, you don't care.
src/S3File.cc:109:+
src/S3File.cc:110:+ this->s3_object_name = object;
src/S3Fil
|
Run linters:
src/S3File.cc#L147
[lint] reported by reviewdog 🐶
Raw Output:
src/S3File.cc:147:-
src/S3File.cc:148:-int
src/S3File.cc:149:-S3File::Fstat(struct stat *buff)
src/S3File.cc:150:-{
src/S3File.cc:151:- AmazonS3Head head(
src/S3File.cc:152:- this->s3_service_url,
src/S3File.cc:153:- this->s3_access_key,
src/S3File.cc:154:- this->s3_secret_key,
src/S3File.cc:155:- this->s3_bucket_name,
src/S3File.cc:156:- this->s3_object_name,
src/S3File.cc:157:- this->s3_url_style,
src/S3File.cc:158:- m_log
src/S3File.cc:159:- );
src/S3File.cc:160:-
src/S3File.cc:161:- if(! head.SendRequest()) {
src/S3File.cc:162:- // SendRequest() returns false for all errors, including ones
src/S3File.cc:163:- // where the server properly responded with something other
src/S3File.cc:164:- // than code 200. If xrootd wants us to distinguish between
src/S3File.cc:165:- // these cases, head.getResponseCode() is initialized to 0, so
src/S3File.cc:166:- // we can check.
src/S3File.cc:167:- std::stringstream ss;
src/S3File.cc:168:- ss << "Failed to send HeadObject command: " << head.getResponseCode() << "'" << head.getResultString() << "'";
src/S3File.cc:169:- m_log.Log(LogMask::Warning, "S3File::Fstat", ss.str().c_str());
src/S3File.cc:170:- return -ENOENT;
src/S3File.cc:171:- }
src/S3File.cc:172:-
src/S3File.cc:173:-
src/S3File.cc:174:- std::string headers = head.getResultString();
src/S3File.cc:175:-
src/S3File.cc:176:- std::string line;
src/S3File.cc:177:- size_t current_newline = 0;
src/S3File.cc:178:- size_t next_newline = std::string::npos;
src/S3File.cc:179:- size_t last_character = headers.size();
src/S3File.cc:180:- while( current_newline != std::string::npos && current_newline != last_character - 1 ) {
src/S3File.cc:181:- next_newline = headers.find( "\r\n", current_newline + 2);
src/S3File.cc:182:- line = substring( headers, current_newline + 2, next_newline );
src/S3File.cc:183:-
src/S3File.cc:184:- size_t colon = line.find(":");
src/S3File.cc:185:- if( colon != std::string::npos && colon != line.size() ) {
src/S3File.cc:186:- std::string attr = substring( line, 0, colon );
src/S3File.cc:187:- std::string value = substring( line, colon + 1 );
src/S3File.cc:188:- trim(value);
src/S3File.cc:189:- toLower(attr);
src/S3File.cc:190:-
src/S3File.cc:191:- if( attr == "content-length" ) {
src/S3File.cc:192:- this->content_length = std::stol(value);
src/S3File.cc:193:- } else if( attr == "last-modified" ) {
src/S3File.cc:194:- struct tm t;
src/S3File.cc:195:- char * eos = strptime( value.c_str(),
src/S3File.cc:196:- "%a, %d %b %Y %T %Z",
src/S3File.cc:197:- & t );
src/S3File.cc:198:- if( eos == & value.c_str()[value.size()] ) {
src/S3File.cc:199:- time_t epoch = timegm(& t);
src/S3File.cc:200:- if( epoch != -1 ) {
src/S3File.cc:201:- this->last_modified = epoch;
src/S3File.cc:202:- }
src/S3File.cc:203:- }
src/S3File.cc:204:- }
src/S3File.cc:137:+int S3File::Fstat(struct stat *buff) {
src/S3File.cc:138:+ AmazonS3Head head(this->s3_service_url, this->s3_access_key,
src/S3File.cc:139:+ this->s3_secret_key, this->s3_bucket_name,
src/S3File.cc:140:+ this->s3_object_name, this->s3_url_style, m_log);
src/S3File.cc:141:+
src/S3File.cc:142:+ if (!head.SendRequest()) {
src/S3File.cc:143:+ // SendRequest() returns false for all errors, including ones
src/S3File.cc:144:+ // where the server properly responded with something other
src/S3File.cc:145:+ // than code 200. If xrootd wants us to distinguish between
src/S3File.cc:146:+ // these cases, head.getResponseCode() is initialized to 0, so
src/S3File.cc:147:+ // we can check.
src/S3File.cc:148:+ std::stringstream ss;
src/S3File.cc:149:+ ss <<
|
Run linters:
src/S3FileSystem.cc#L65
[lint] reported by reviewdog 🐶
Raw Output:
src/S3FileSystem.cc:65:- }
src/S3FileSystem.cc:66:- return true;
src/S3FileSystem.cc:67:-}
src/S3FileSystem.cc:68:-
src/S3FileSystem.cc:69:-
src/S3FileSystem.cc:70:-bool
src/S3FileSystem.cc:71:-S3FileSystem::Config(XrdSysLogger *lp, const char *configfn)
src/S3FileSystem.cc:72:-{
src/S3FileSystem.cc:73:- XrdOucEnv myEnv;
src/S3FileSystem.cc:74:- XrdOucStream Config(&m_log, getenv("XRDINSTANCE"), &myEnv, "=====> ");
src/S3FileSystem.cc:75:-
src/S3FileSystem.cc:76:- int cfgFD = open(configfn, O_RDONLY, 0);
src/S3FileSystem.cc:77:- if (cfgFD < 0) {
src/S3FileSystem.cc:78:- m_log.Emsg("Config", errno, "open config file", configfn);
src/S3FileSystem.cc:85:+ }
src/S3FileSystem.cc:86:+ if (newAccessInfo->getS3Region().empty()) {
src/S3FileSystem.cc:87:+ m_log.Emsg("Config", "s3.region not specified");
|
Run linters:
src/S3FileSystem.cc#L81
[lint] reported by reviewdog 🐶
Raw Output:
src/S3FileSystem.cc:81:-
src/S3FileSystem.cc:82:- char * temporary;
src/S3FileSystem.cc:83:- std::string value;
src/S3FileSystem.cc:84:- std::string attribute;
src/S3FileSystem.cc:85:- Config.Attach(cfgFD);
src/S3FileSystem.cc:86:- S3AccessInfo *newAccessInfo = new S3AccessInfo();
src/S3FileSystem.cc:87:- std::string exposedPath;
src/S3FileSystem.cc:88:- while ((temporary = Config.GetMyFirstWord())) {
src/S3FileSystem.cc:89:- attribute = temporary;
src/S3FileSystem.cc:90:- temporary = Config.GetWord();
src/S3FileSystem.cc:91:- if(attribute == "s3.end") {
src/S3FileSystem.cc:92:- s3_access_map[exposedPath] = newAccessInfo;
src/S3FileSystem.cc:93:- if(newAccessInfo->getS3ServiceName().empty()) {
src/S3FileSystem.cc:94:- m_log.Emsg("Config", "s3.service_name not specified");
src/S3FileSystem.cc:95:- return false;
src/S3FileSystem.cc:96:- }
src/S3FileSystem.cc:97:- if(newAccessInfo->getS3Region().empty()) {
src/S3FileSystem.cc:98:- m_log.Emsg("Config", "s3.region not specified");
src/S3FileSystem.cc:99:- return false;
src/S3FileSystem.cc:100:- }
src/S3FileSystem.cc:101:- newAccessInfo = new S3AccessInfo();
src/S3FileSystem.cc:102:- exposedPath = "";
src/S3FileSystem.cc:103:- continue;
src/S3FileSystem.cc:104:- }
src/S3FileSystem.cc:105:- if(! temporary) { continue; }
src/S3FileSystem.cc:106:- value = temporary;
src/S3FileSystem.cc:107:-
src/S3FileSystem.cc:108:- if(!handle_required_config("s3.path_name",value) ) { Config.Close(); return false; }
src/S3FileSystem.cc:109:- if(!handle_required_config("s3.bucket_name",value) ) { Config.Close(); return false; }
src/S3FileSystem.cc:110:- if(!handle_required_config("s3.service_name",value) ) { Config.Close(); return false; }
src/S3FileSystem.cc:111:- if(!handle_required_config("s3.region", value ) ) { Config.Close(); return false; }
src/S3FileSystem.cc:112:- if(!handle_required_config("s3.service_url", value) ) { Config.Close(); return false; }
src/S3FileSystem.cc:113:- if(!handle_required_config("s3.access_key_file", value) ) { Config.Close(); return false; }
src/S3FileSystem.cc:114:- if(!handle_required_config("s3.secret_key_file", value) ) { Config.Close(); return false; }
src/S3FileSystem.cc:115:- if(!handle_required_config("s3.url_style", value) ) { Config.Close(); return false; }
src/S3FileSystem.cc:116:-
src/S3FileSystem.cc:117:- if(attribute == "s3.path_name") exposedPath = value;
src/S3FileSystem.cc:118:- else if(attribute == "s3.bucket_name") newAccessInfo->setS3BucketName(value);
src/S3FileSystem.cc:119:- else if(attribute == "s3.service_name") newAccessInfo->setS3ServiceName(value);
src/S3FileSystem.cc:120:- else if(attribute == "s3.region") newAccessInfo->setS3Region(value);
src/S3FileSystem.cc:121:- else if(attribute == "s3.access_key_file") newAccessInfo->setS3AccessKeyFile(value);
src/S3FileSystem.cc:122:- else if(attribute == "s3.secret_key_file") newAccessInfo->setS3SecretKeyFile(value);
src/S3FileSystem.cc:123:- else if(attribute == "s3.service_url") newAccessInfo->setS3ServiceUrl(value);
src/S3FileSystem.cc:124:- else if(attribute == "s3.url_style") this->s3_url_style = value;
src/S3FileSystem.cc:125:-
src/S3FileSystem.cc:94:+ if (!temporary) {
src/S3FileSystem.cc:95:+ continue;
|
Run linters:
src/S3FileSystem.cc#L128
[lint] reported by reviewdog 🐶
Raw Output:
src/S3FileSystem.cc:128:- if( this->s3_url_style.empty() ) {
src/S3FileSystem.cc:129:- m_log.Emsg("Config", "s3.url_style not specified");
src/S3FileSystem.cc:130:- return false;
src/S3FileSystem.cc:131:- } else {
src/S3FileSystem.cc:132:- // We want this to be case-insensitive.
src/S3FileSystem.cc:133:- toLower( this->s3_url_style );
src/S3FileSystem.cc:99:+ if (!handle_required_config("s3.path_name", value)) {
src/S3FileSystem.cc:100:+ Config.Close();
src/S3FileSystem.cc:101:+ return false;
|
Run linters
Node.js 16 actions are deprecated. Please update the following actions to use Node.js 20: actions/checkout@v3, wearerequired/lint-action@v2. For more information see: https://github.blog/changelog/2023-09-22-github-actions-transitioning-from-node-16-to-node-20/.
|