diff --git a/CMakeLists.txt b/CMakeLists.txt index 5c57d32a..5eeeb5a0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 3.5.0 FATAL_ERROR) #CPACK_DEBIAN__PACK find_package(IRODS 4.2.11 EXACT REQUIRED) -set(IRODS_PLUGIN_REVISION "0") +set(IRODS_PLUGIN_REVISION "1") set(IRODS_PLUGIN_VERSION "${IRODS_VERSION}.${IRODS_PLUGIN_REVISION}") set(IRODS_PACKAGE_REVISION "1") diff --git a/s3/s3_transport/include/s3_transport.hpp b/s3/s3_transport/include/s3_transport.hpp index 1ac3c0a3..fce07e9b 100644 --- a/s3/s3_transport/include/s3_transport.hpp +++ b/s3/s3_transport/include/s3_transport.hpp @@ -630,6 +630,7 @@ namespace irods::experimental::io::s3_transport } else { + int64_t existing_object_size = config_.object_size; switch (_dir) { case std::ios_base::beg: set_file_offset(_offset); @@ -640,7 +641,19 @@ namespace irods::experimental::io::s3_transport break; case std::ios_base::end: - set_file_offset(config_.object_size + _offset); + + if (existing_object_size == config::UNKNOWN_OBJECT_SIZE) { + // do a stat to get object size + object_s3_status object_status = object_s3_status::DOES_NOT_EXIST; + irods::error ret = get_object_s3_status(object_key_, bucket_context_, existing_object_size, object_status); + if (!ret.ok() || object_status == object_s3_status::DOES_NOT_EXIST) { + rodsLog(LOG_ERROR, "%s:%d (%s) [[%lu]] seek failed because object size is unknown and HEAD failed", + __FILE__, __LINE__, __FUNCTION__, get_thread_identifier()); + return seek_error; + } + } + + set_file_offset(existing_object_size + _offset); break; default: diff --git a/s3/s3_transport/unit_tests/CMakeLists.txt b/s3/s3_transport/unit_tests/CMakeLists.txt index f53b5a69..524207af 100644 --- a/s3/s3_transport/unit_tests/CMakeLists.txt +++ b/s3/s3_transport/unit_tests/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION ${CMAKE_VERSION}) project(unit_tests LANGUAGES C CXX) -find_package(IRODS 4.2.10 EXACT REQUIRED) +find_package(IRODS 4.2.11 EXACT REQUIRED) set(IRODS_EXTERNALS_FULLPATH_S3 "/opt/irods-externals/libs3e4674774-0/") # without verbosity diff --git a/s3/s3_transport/unit_tests/src/test_s3_transport.cpp b/s3/s3_transport/unit_tests/src/test_s3_transport.cpp index 5cadf1de..72722d37 100644 --- a/s3/s3_transport/unit_tests/src/test_s3_transport.cpp +++ b/s3/s3_transport/unit_tests/src/test_s3_transport.cpp @@ -682,6 +682,54 @@ void do_read_write_thread(const std::string& bucket_name, check_read_write_results(bucket_name, filename, object_prefix); } +void test_seek_end(const std::string& bucket_name, + const std::string& filename, + const std::string& object_prefix, + const std::string& keyfile) +{ + std::string access_key, secret_access_key; + read_keys(keyfile, access_key, secret_access_key); + + // stage file to s3 + std::stringstream ss; + ss << "aws --endpoint-url http://" << hostname << " s3 cp " << filename << " s3://" << bucket_name << "/" << object_prefix + << filename; + rodsLog(LOG_NOTICE, "%s\n", ss.str().c_str()); + system(ss.str().c_str()); + + // get the size of the file + std::ifstream in(filename, std::ifstream::ate | std::ifstream::binary); + off_t file_size = in.tellg(); + in.close(); + + // open object and seek to end + s3_transport_config s3_config; + s3_config.hostname = hostname; + s3_config.number_of_cache_transfer_threads = 1; + s3_config.number_of_client_transfer_threads = 1; + s3_config.bucket_name = bucket_name; + s3_config.access_key = access_key; + s3_config.secret_access_key = secret_access_key; + s3_config.shared_memory_timeout_in_seconds = 20; + s3_config.put_repl_flag = true; + s3_config.developer_messages_log_level = LOG_NOTICE; + s3_config.region_name = "us-east-1"; + + std::ios_base::openmode open_modes = std::ios_base::in; + s3_transport tp{s3_config}; + dstream ds{tp, std::string(object_prefix)+filename, open_modes}; + + ds.seekp(0, std::ios_base::end); + off_t offset = ds.tellg(); + REQUIRE(offset == file_size); + + ds.seekp(-1, std::ios_base::end); + offset = ds.tellg(); + REQUIRE(offset == file_size - 1); + + ds.close(); + rodsLog(LOG_NOTICE, "CLOSE DONE\n"); +} TEST_CASE("quick test upload", "[quick_test][quick_test_upload]") { @@ -1041,6 +1089,30 @@ TEST_CASE("s3_transport_readwrite_thread", "[rw][thread]") remove_bucket(bucket_name); } +TEST_CASE("test_seek_end_existing_file", "[seek_end]") +{ + rodsLogLevel(log_level); + + std::string bucket_name = create_bucket(); + + std::string filename = "medium_file"; + std::string object_prefix = "dir1/dir2/"; + + SECTION("seek end small file") + { + filename = "small_file"; + test_seek_end(bucket_name, filename, object_prefix, keyfile); + + } + + SECTION("seek end medium file") + { + test_seek_end(bucket_name, filename, object_prefix, keyfile); + + } + remove_bucket(bucket_name); +} + TEST_CASE("test_part_splits", "[part_splits]") { rodsLogLevel(log_level);