[#2020] Do a head to get the object size when seeking to end if size …

…is unknown.
irods · Jan 7, 2022 · 2800b36 · 2800b36
1 parent 5207083
commit 2800b36
Show file tree

Hide file tree

Showing 4 changed files with 88 additions and 3 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 3.5.0 FATAL_ERROR) #CPACK_DEBIAN_<COMPONENT>_PACK
 
 find_package(IRODS 4.2.11 EXACT REQUIRED)
 
-set(IRODS_PLUGIN_REVISION "0")
+set(IRODS_PLUGIN_REVISION "1")
 set(IRODS_PLUGIN_VERSION "${IRODS_VERSION}.${IRODS_PLUGIN_REVISION}")
 
 set(IRODS_PACKAGE_REVISION "1")

diff --git a/s3/s3_transport/include/s3_transport.hpp b/s3/s3_transport/include/s3_transport.hpp
@@ -630,6 +630,7 @@ namespace irods::experimental::io::s3_transport
 
             } else {
 
+                int64_t existing_object_size = config_.object_size;
                 switch (_dir) {
                     case std::ios_base::beg:
                         set_file_offset(_offset);
@@ -640,7 +641,19 @@ namespace irods::experimental::io::s3_transport
                         break;
 
                     case std::ios_base::end:
-                        set_file_offset(config_.object_size + _offset);
+
+                        if (existing_object_size == config::UNKNOWN_OBJECT_SIZE) {
+                            // do a stat to get object size
+                            object_s3_status object_status = object_s3_status::DOES_NOT_EXIST;
+                            irods::error ret = get_object_s3_status(object_key_, bucket_context_, existing_object_size, object_status);
+                            if (!ret.ok() || object_status == object_s3_status::DOES_NOT_EXIST) {
+                                rodsLog(LOG_ERROR, "%s:%d (%s) [[%lu]] seek failed because object size is unknown and HEAD failed",
+                                         __FILE__, __LINE__, __FUNCTION__, get_thread_identifier());
+                                return seek_error;
+                            }
+                        }
+
+                        set_file_offset(existing_object_size + _offset);
                         break;
 
                     default:

diff --git a/s3/s3_transport/unit_tests/CMakeLists.txt b/s3/s3_transport/unit_tests/CMakeLists.txt
@@ -1,7 +1,7 @@
 cmake_minimum_required(VERSION ${CMAKE_VERSION})
 project(unit_tests LANGUAGES C CXX)
 
-find_package(IRODS 4.2.10 EXACT REQUIRED)
+find_package(IRODS 4.2.11 EXACT REQUIRED)
 
 set(IRODS_EXTERNALS_FULLPATH_S3 "/opt/irods-externals/libs3e4674774-0/")    # without verbosity
 

diff --git a/s3/s3_transport/unit_tests/src/test_s3_transport.cpp b/s3/s3_transport/unit_tests/src/test_s3_transport.cpp
@@ -682,6 +682,54 @@ void do_read_write_thread(const std::string& bucket_name,
     check_read_write_results(bucket_name, filename, object_prefix);
 }
 
+void test_seek_end(const std::string& bucket_name,
+                          const std::string& filename,
+                          const std::string& object_prefix,
+                          const std::string& keyfile)
+{
+    std::string access_key, secret_access_key;
+    read_keys(keyfile, access_key, secret_access_key);
+
+    // stage file to s3
+    std::stringstream ss;
+    ss << "aws --endpoint-url http://" << hostname << " s3 cp " << filename << " s3://" << bucket_name << "/" << object_prefix
+       << filename;
+    rodsLog(LOG_NOTICE, "%s\n", ss.str().c_str());
+    system(ss.str().c_str());
+
+    // get the size of the file
+    std::ifstream in(filename, std::ifstream::ate | std::ifstream::binary);
+    off_t file_size = in.tellg(); 
+    in.close();
+
+    // open object and seek to end
+    s3_transport_config s3_config;
+    s3_config.hostname = hostname;
+    s3_config.number_of_cache_transfer_threads = 1;
+    s3_config.number_of_client_transfer_threads = 1;
+    s3_config.bucket_name = bucket_name;
+    s3_config.access_key = access_key;
+    s3_config.secret_access_key = secret_access_key;
+    s3_config.shared_memory_timeout_in_seconds = 20;
+    s3_config.put_repl_flag = true;
+    s3_config.developer_messages_log_level = LOG_NOTICE;
+    s3_config.region_name = "us-east-1";
+
+    std::ios_base::openmode open_modes = std::ios_base::in;
+    s3_transport tp{s3_config};
+    dstream ds{tp, std::string(object_prefix)+filename, open_modes};
+
+    ds.seekp(0, std::ios_base::end);
+    off_t offset = ds.tellg();
+    REQUIRE(offset == file_size);
+
+    ds.seekp(-1, std::ios_base::end);
+    offset = ds.tellg();
+    REQUIRE(offset == file_size - 1);
+
+    ds.close();
+    rodsLog(LOG_NOTICE, "CLOSE DONE\n");
+}
 
 TEST_CASE("quick test upload", "[quick_test][quick_test_upload]")
 {
@@ -1041,6 +1089,30 @@ TEST_CASE("s3_transport_readwrite_thread", "[rw][thread]")
     remove_bucket(bucket_name);
 }
 
+TEST_CASE("test_seek_end_existing_file", "[seek_end]")
+{
+    rodsLogLevel(log_level);
+
+    std::string bucket_name = create_bucket();
+
+    std::string filename = "medium_file";
+    std::string object_prefix = "dir1/dir2/";
+
+    SECTION("seek end small file")
+    {
+        filename = "small_file";
+        test_seek_end(bucket_name, filename, object_prefix, keyfile);
+
+    }
+
+    SECTION("seek end medium file")
+    {
+        test_seek_end(bucket_name, filename, object_prefix, keyfile);
+
+    }
+    remove_bucket(bucket_name);
+}
+
 TEST_CASE("test_part_splits", "[part_splits]")
 {
     rodsLogLevel(log_level);