diff --git a/CMakeLists.txt b/CMakeLists.txt index fdb26b7e..20d14069 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.5.0 FATAL_ERROR) #CPACK_DEBIAN__PACKAGE_NAME find_package(IRODS 4.2.8 EXACT REQUIRED) -set(IRODS_PLUGIN_REVISION "0") +set(IRODS_PLUGIN_REVISION "1") if (NOT IRODS_EXTERNALS_PACKAGE_ROOT) set(IRODS_EXTERNALS_PACKAGE_ROOT "/opt/irods-externals" CACHE STRING "Choose the location of iRODS external packages." FORCE) @@ -23,7 +23,7 @@ macro(IRODS_MACRO_CHECK_DEPENDENCY_SET_FULLPATH_ADD_TO_IRODS_PACKAGE_DEPENDENCIE IRODS_MACRO_CHECK_DEPENDENCY_SET_FULLPATH(${DEPENDENCY_NAME} ${DEPENDENCY_SUBDIRECTORY}) list(APPEND IRODS_PACKAGE_DEPENDENCIES_LIST irods-externals-${DEPENDENCY_SUBDIRECTORY}) endmacro() -IRODS_MACRO_CHECK_DEPENDENCY_SET_FULLPATH_ADD_TO_IRODS_PACKAGE_DEPENDENCIES_LIST(S3 libs3a30e55e8-1) +IRODS_MACRO_CHECK_DEPENDENCY_SET_FULLPATH_ADD_TO_IRODS_PACKAGE_DEPENDENCIES_LIST(S3 libs34e684077-0) string(REPLACE ";" ", " IRODS_PACKAGE_DEPENDENCIES_STRING "${IRODS_PACKAGE_DEPENDENCIES_LIST}") set(CMAKE_C_COMPILER ${IRODS_EXTERNALS_FULLPATH_CLANG}/bin/clang) @@ -52,6 +52,7 @@ add_compile_options(-nostdinc++ -Wall -Wextra -Werror) add_compile_options(-Wno-unused-function) # due to OPENSSL_NO_DEPRECATED_1_1_0 add_compile_options(-Wno-unused-parameter) # Until fix of irods_re_plugin.hpp add_compile_options(-Wno-unneeded-internal-declaration) # Until fix of https://github.com/irods/irods/issues/3396 +add_compile_options(-Wno-unused-function) # Due to openssl_unused_function link_libraries(c++abi curl xml2 crypto pthread) include_directories(${IRODS_EXTERNALS_FULLPATH_CLANG}/include/c++/v1 ${IRODS_EXTERNALS_FULLPATH_JSON}/include) @@ -114,7 +115,10 @@ install( FILES ${CMAKE_SOURCE_DIR}/packaging/test_irods_resource_plugin_s3.py ${CMAKE_SOURCE_DIR}/packaging/test_irods_resource_plugin_s3_for_gcp.py + ${CMAKE_SOURCE_DIR}/packaging/test_irods_resource_plugin_s3_ceph.py + ${CMAKE_SOURCE_DIR}/packaging/test_irods_resource_plugin_s3_minio.py ${CMAKE_SOURCE_DIR}/packaging/resource_suite_s3_nocache.py + ${CMAKE_SOURCE_DIR}/packaging/resource_suite_s3_cache.py DESTINATION ${IRODS_HOME_DIRECTORY}/scripts/irods/test PERMISSIONS OWNER_READ OWNER_WRITE GROUP_READ WORLD_READ ) diff --git a/packaging/resource_suite_s3_cache.py b/packaging/resource_suite_s3_cache.py new file mode 100644 index 00000000..5c04870e --- /dev/null +++ b/packaging/resource_suite_s3_cache.py @@ -0,0 +1,582 @@ +try: + from minio import Minio + from minio.error import ResponseError +except ImportError: + print('This test requires minio: perhaps try pip install minio') + exit() + +import commands +import datetime +import os +import platform +import random +import re +import shutil +import string +import subprocess +import urllib3 + +from resource_suite_s3_nocache import Test_S3_NoCache_Base + +import sys +if sys.version_info >= (2,7): + import unittest +else: + import unittest2 as unittest + +from .. import lib +from . import session +from ..configuration import IrodsConfig +from .resource_suite import ResourceSuite +from .test_chunkydevtest import ChunkyDevTest + + + +class Test_S3_Cache_Base(ResourceSuite, ChunkyDevTest): + def __init__(self, *args, **kwargs): + + # if self.proto is defined use it else default to HTTPS + try: + self.proto = self.proto + except AttributeError: + self.proto = 'HTTPS' + + # if self.archive_naming_policy is defined use it + # else default to 'consistent' + try: + self.archive_naming_policy = self.archive_naming_policy + except AttributeError: + self.archive_naming_policy = 'consistent' + + super(Test_S3_Cache_Base, self).__init__(*args, **kwargs) + + def setUp(self): + # skip ssl tests on ub12 + distro_str = ''.join(platform.linux_distribution()[:2]).replace(' ','') + if self._testMethodName.startswith('test_ssl') and distro_str.lower().startswith('ubuntu12'): + self.skipTest("skipping ssl tests on ubuntu 12") + + # set up aws configuration + self.read_aws_keys() + + # set up s3 bucket + try: + httpClient = urllib3.poolmanager.ProxyManager( + os.environ['http_proxy'], + timeout=urllib3.Timeout.DEFAULT_TIMEOUT, + cert_reqs='CERT_REQUIRED', + retries=urllib3.Retry( + total=5, + backoff_factor=0.2, + status_forcelist=[500, 502, 503, 504] + ) + ) + except KeyError: + httpClient = None + + if self.proto == 'HTTPS': + s3_client = Minio(self.s3endPoint, + access_key=self.aws_access_key_id, + secret_key=self.aws_secret_access_key, + http_client=httpClient) + else: + s3_client = Minio(self.s3endPoint, + access_key=self.aws_access_key_id, + secret_key=self.aws_secret_access_key, + http_client=httpClient, + secure=False) + + self.s3bucketname = 'irods-ci-' + distro_str + datetime.datetime.utcnow().strftime('-%Y-%m-%d.%H-%M-%S-%f-') + self.s3bucketname += ''.join(random.choice(string.letters) for i in xrange(10)) + self.s3bucketname = self.s3bucketname[:63].lower() # bucket names can be no more than 63 characters long + s3_client.make_bucket(self.s3bucketname, location=self.s3region) + + # set up resources + + hostname = lib.get_hostname() + s3params = 'S3_RETRY_COUNT=15;S3_WAIT_TIME_SEC=1;S3_PROTO=%s;S3_MPU_CHUNK=10;S3_MPU_THREADS=4;S3_ENABLE_MD5=1' % self.proto + s3params += ';S3_STSDATE=' + self.s3stsdate + s3params += ';S3_DEFAULT_HOSTNAME=' + self.s3endPoint + s3params += ';S3_AUTH_FILE=' + self.keypairfile + s3params += ';S3_REGIONNAME=' + self.s3region + s3params += ';ARCHIVE_NAMING_POLICY=' + self.archive_naming_policy + try: + s3params += ';S3_SERVER_ENCRYPT=' + str(self.s3sse) + except AttributeError: + pass + + s3params=os.environ.get('S3PARAMS', s3params); + + with session.make_session_for_existing_admin() as admin_session: + irods_config = IrodsConfig() + admin_session.assert_icommand("iadmin modresc demoResc name origResc", 'STDOUT_SINGLELINE', 'rename', input='yes\n') + admin_session.assert_icommand("iadmin mkresc demoResc compound", 'STDOUT_SINGLELINE', 'compound') + admin_session.assert_icommand("iadmin mkresc cacheResc 'unixfilesystem' " + hostname + ":" + irods_config.irods_directory + "/cacheRescVault", 'STDOUT_SINGLELINE', 'cacheResc') + admin_session.assert_icommand('iadmin mkresc archiveResc s3 '+hostname+':/'+self.s3bucketname+'/irods/Vault "'+s3params+'"', 'STDOUT_SINGLELINE', 'archiveResc') + admin_session.assert_icommand("iadmin addchildtoresc demoResc cacheResc cache") + admin_session.assert_icommand("iadmin addchildtoresc demoResc archiveResc archive") + + super(Test_S3_Cache_Base, self).setUp() + + def tearDown(self): + super(Test_S3_Cache_Base, self).tearDown() + + # delete s3 bucket + try: + httpClient = urllib3.poolmanager.ProxyManager( + os.environ['http_proxy'], + timeout=urllib3.Timeout.DEFAULT_TIMEOUT, + cert_reqs='CERT_REQUIRED', + retries=urllib3.Retry( + total=5, + backoff_factor=0.2, + status_forcelist=[500, 502, 503, 504] + ) + ) + except KeyError: + httpClient = None + + if self.proto == 'HTTPS': + s3_client = Minio(self.s3endPoint, + access_key=self.aws_access_key_id, + secret_key=self.aws_secret_access_key, + http_client=httpClient) + else: + s3_client = Minio(self.s3endPoint, + access_key=self.aws_access_key_id, + secret_key=self.aws_secret_access_key, + http_client=httpClient, + secure=False) + + objects = s3_client.list_objects_v2(self.s3bucketname, recursive=True) + try: + for del_err in s3_client.remove_objects(self.s3bucketname, [object.object_name for object in objects]): + print("Deletion Error: {}".format(del_err)) + except ResponseError as err: + print(err) + s3_client.remove_bucket(self.s3bucketname) + + # tear down resources + with session.make_session_for_existing_admin() as admin_session: + admin_session.assert_icommand("iadmin rmchildfromresc demoResc archiveResc") + admin_session.assert_icommand("iadmin rmchildfromresc demoResc cacheResc") + admin_session.assert_icommand("iadmin rmresc archiveResc") + admin_session.assert_icommand("iadmin rmresc cacheResc") + admin_session.assert_icommand("iadmin rmresc demoResc") + admin_session.assert_icommand("iadmin modresc origResc name demoResc", 'STDOUT_SINGLELINE', 'rename', input='yes\n') + + shutil.rmtree(IrodsConfig().irods_directory + "/cacheRescVault", ignore_errors=True) + + def read_aws_keys(self): + # read access keys from keypair file + with open(self.keypairfile) as f: + self.aws_access_key_id = f.readline().rstrip() + self.aws_secret_access_key = f.readline().rstrip() + + # read the endpoint address from the file endpointfile + def read_endpoint(self, endpointfile): + # read endpoint file + with open(endpointfile) as f: + return f.readline().rstrip() + + def test_irm_specific_replica(self): + self.admin.assert_icommand("ils -L "+self.testfile,'STDOUT_SINGLELINE',self.testfile) # should be listed + self.admin.assert_icommand("irepl -R "+self.testresc+" "+self.testfile) # creates replica + self.admin.assert_icommand("ils -L "+self.testfile,'STDOUT_SINGLELINE',self.testfile) # should be listed twice + self.admin.assert_icommand("irm -n 0 "+self.testfile, 'STDOUT_SINGLELINE','deprecated') # remove original from cacheResc only + self.admin.assert_icommand("ils -L "+self.testfile,'STDOUT_SINGLELINE',["2 "+self.testresc,self.testfile]) # replica 2 should still be there + self.admin.assert_icommand_fail("ils -L "+self.testfile,'STDOUT_SINGLELINE',["0 "+self.admin.default_resource,self.testfile]) # replica 0 should be gone + trashpath = self.admin.session_collection_trash + self.admin.assert_icommand_fail("ils -L "+trashpath+"/"+self.testfile,'STDOUT_SINGLELINE',["0 "+self.admin.default_resource,self.testfile]) # replica should not be in trash + + @unittest.skip("--wlock has possible race condition due to Compound/Replication PDMO") + def test_local_iput_collision_with_wlock(self): + pass + + @unittest.skip("NOTSURE / FIXME ... -K not supported, perhaps") + def test_local_iput_checksum(self): + pass + + @unittest.skip("EMPTY_RESC_PATH - no vault path for coordinating resources") + def test_ireg_as_rodsuser_in_vault(self): + pass + + @unittest.skip("No Vault for S3 archive resource") + def test_iput_overwrite_others_file__ticket_2086(self): + pass + + def test_local_iput_with_force_and_destination_resource__ticket_1706(self): + # local setup + filename = "iputwithforceanddestination.txt" + filepath = lib.create_local_testfile(filename) + doublefile = "doublefile.txt" + os.system("cat %s %s > %s" % (filename, filename, doublefile)) + doublesize = str(os.stat(doublefile).st_size) + # assertions + self.admin.assert_icommand("ils -L "+filename,'STDERR_SINGLELINE',"does not exist") # should not be listed + self.admin.assert_icommand("iput "+filename) # put file + self.admin.assert_icommand("irepl -R "+self.testresc+" "+filename) # replicate to test resource + self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',filename) # + self.admin.assert_icommand("iput -f -R %s %s %s" % (self.testresc, doublefile, filename) ) # overwrite test repl with different data + self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 0 "," "+filename]) # default resource cache should have dirty copy + self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 1 "," "+filename]) # default resource archive should have dirty copy + self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 0 "," "+doublesize+" "," "+filename]) # default resource cache should not have doublesize file + self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 1 "," "+doublesize+" "," "+filename]) # default resource archive should not have doublesize file + self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 2 "," "+doublesize+" ","& "+filename]) # targeted resource should have new double clean copy + # local cleanup + os.remove(filepath) + os.remove(doublefile) + + ################### + # irepl + ################### + + def test_irepl_update_replicas(self): + # local setup + filename = "updatereplicasfile.txt" + filepath = lib.create_local_testfile(filename) + hostname = lib.get_hostname() + doublefile = "doublefile.txt" + os.system("cat %s %s > %s" % (filename, filename, doublefile)) + doublesize = str(os.stat(doublefile).st_size) + + # assertions + self.admin.assert_icommand("iadmin mkresc thirdresc unixfilesystem %s:/tmp/thirdrescVault" % hostname, 'STDOUT_SINGLELINE', "Creating") # create third resource + self.admin.assert_icommand("iadmin mkresc fourthresc unixfilesystem %s:/tmp/fourthrescVault" % hostname, 'STDOUT_SINGLELINE', "Creating") # create fourth resource + self.admin.assert_icommand("ils -L "+filename,'STDERR_SINGLELINE',"does not exist") # should not be listed + self.admin.assert_icommand("iput "+filename) # put file + self.admin.assert_icommand("irepl -R "+self.testresc+" "+filename) # replicate to test resource + self.admin.assert_icommand("irepl -R thirdresc "+filename) # replicate to third resource + self.admin.assert_icommand("irepl -R fourthresc "+filename) # replicate to fourth resource + self.admin.assert_icommand("iput -f -R "+self.testresc+" "+doublefile+" "+filename) # repave overtop test resource + self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',filename) # for debugging + + self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 0 "," & "+filename]) # should have a dirty copy + self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 1 "," & "+filename]) # should have a dirty copy + self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 2 "," & "+filename]) # should have a clean copy + self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 3 "," & "+filename]) # should have a dirty copy + self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 4 "," & "+filename]) # should have a dirty copy + + self.admin.assert_icommand("irepl -U "+filename) # update last replica + + self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 0 "," & "+filename]) # should have a dirty copy + self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 1 "," & "+filename]) # should have a dirty copy + self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 2 "," & "+filename]) # should have a clean copy + self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 3 "," & "+filename]) # should have a dirty copy + self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 4 "," & "+filename]) # should have a clean copy + + self.admin.assert_icommand("irepl -aU "+filename) # update all replicas + + self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 0 "," & "+filename]) # should have a clean copy + self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 1 "," & "+filename]) # should have a clean copy + self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 2 "," & "+filename]) # should have a clean copy + self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 3 "," & "+filename]) # should have a clean copy + self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 4 "," & "+filename]) # should have a clean copy + + self.admin.assert_icommand("irm -f "+filename) # cleanup file + self.admin.assert_icommand("iadmin rmresc thirdresc") # remove third resource + self.admin.assert_icommand("iadmin rmresc fourthresc") # remove third resource + + # local cleanup + os.remove(filepath) + os.remove(doublefile) + + def test_irepl_over_existing_second_replica__ticket_1705(self): + # local setup + filename = "secondreplicatest.txt" + filepath = lib.create_local_testfile(filename) + # assertions + self.admin.assert_icommand("ils -L "+filename,'STDERR_SINGLELINE',"does not exist") # should not be listed + self.admin.assert_icommand("iput -R "+self.testresc+" "+filename) # put file + self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',filename) # for debugging + self.admin.assert_icommand("irepl "+filename) # replicate to default resource + self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',filename) # for debugging + self.admin.assert_icommand("irepl "+filename) # replicate overtop default resource + self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 3 "," & "+filename]) # should not have a replica 3 + self.admin.assert_icommand("irepl -R "+self.testresc+" "+filename) # replicate overtop test resource + self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 3 "," & "+filename]) # should not have a replica 3 + # local cleanup + os.remove(filepath) + + def test_irepl_over_existing_third_replica__ticket_1705(self): + # local setup + filename = "thirdreplicatest.txt" + filepath = lib.create_local_testfile(filename) + hostname = lib.get_hostname() + # assertions + self.admin.assert_icommand("iadmin mkresc thirdresc unixfilesystem %s:/tmp/thirdrescVault" % hostname, 'STDOUT_SINGLELINE', "Creating") # create third resource + self.admin.assert_icommand("ils -L "+filename,'STDERR_SINGLELINE',"does not exist") # should not be listed + self.admin.assert_icommand("iput "+filename) # put file + self.admin.assert_icommand("irepl -R "+self.testresc+" "+filename) # replicate to test resource + self.admin.assert_icommand("irepl -R thirdresc "+filename) # replicate to third resource + self.admin.assert_icommand("irepl "+filename) # replicate overtop default resource + self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',filename) # for debugging + self.admin.assert_icommand("irepl -R "+self.testresc+" "+filename) # replicate overtop test resource + self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',filename) # for debugging + self.admin.assert_icommand("irepl -R thirdresc "+filename) # replicate overtop third resource + self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',filename) # for debugging + self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 4 "," & "+filename]) # should not have a replica 4 + self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 5 "," & "+filename]) # should not have a replica 5 + self.admin.assert_icommand("irm -f "+filename) # cleanup file + self.admin.assert_icommand("iadmin rmresc thirdresc") # remove third resource + # local cleanup + os.remove(filepath) + + def test_irepl_over_existing_bad_replica__ticket_1705(self): + # local setup + filename = "reploverwritebad.txt" + filepath = lib.create_local_testfile(filename) + doublefile = "doublefile.txt" + os.system("cat %s %s > %s" % (filename, filename, doublefile)) + doublesize = str(os.stat(doublefile).st_size) + # assertions + self.admin.assert_icommand("ils -L "+filename,'STDERR_SINGLELINE',"does not exist") # should not be listed + self.admin.assert_icommand("iput "+filename) # put file + self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',filename) # for debugging + self.admin.assert_icommand("irepl -R "+self.testresc+" "+filename) # replicate to test resource + self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',filename) # for debugging + self.admin.assert_icommand("iput -f %s %s" % (doublefile, filename) ) # overwrite default repl with different data + self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 0 "," & "+filename]) # default resource cache should have clean copy + self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 0 "," "+doublesize+" "," & "+filename]) # default resource cache should have new double clean copy + self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 1 "," & "+filename]) # default resource archive should have clean copy + self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 1 "," "+doublesize+" "," & "+filename]) # default resource archive should have new double clean copy + self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 2 "+self.testresc," "+doublesize+" "," "+filename]) # test resource should not have doublesize file + self.admin.assert_icommand("irepl -R "+self.testresc+" "+filename) # replicate back onto test resource + self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 2 "+self.testresc," "+doublesize+" "," & "+filename]) # test resource should have new clean doublesize file + self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 3 "," & "+filename]) # should not have a replica 3 + # local cleanup + os.remove(filepath) + os.remove(doublefile) + + def test_iput_with_purgec(self): + # local setup + filename = "purgecfile.txt" + filepath = os.path.abspath(filename) + f = open(filepath,'wb') + f.write("TESTFILE -- ["+filepath+"]") + f.close() + + # assertions + self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',filename) # should not be listed + self.admin.assert_icommand("iput -f --purgec "+filename, 'STDOUT', 'Specifying a minimum number of replicas to keep is deprecated.') # get file and purge 'cached' replica + self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 0 ",filename]) # should not be listed (trimmed) + self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 1 ",filename]) # should be listed once - replica 1 + self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 2 ",filename]) # should be listed only once + + # local cleanup + output = commands.getstatusoutput( 'rm '+filepath ) + + def test_iget_with_purgec(self): + # local setup + filename = "purgecgetfile.txt" + filepath = os.path.abspath(filename) + f = open(filepath,'wb') + f.write("TESTFILE -- ["+filepath+"]") + f.close() + + # assertions + self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',filename) # should not be listed + self.admin.assert_icommand("iput "+filename) # put file + self.admin.assert_icommand("iget -f --purgec "+filename, 'STDOUT', 'Specifying a minimum number of replicas to keep is deprecated.') # get file and purge 'cached' replica + self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 0 ",filename]) # should not be listed (trimmed) + self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 1 ",filename]) # should be listed once + self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 2 ",filename]) # should not be listed + + # local cleanup + output = commands.getstatusoutput( 'rm '+filepath ) + + def test_irepl_with_purgec(self): + # local setup + filename = "purgecreplfile.txt" + filepath = os.path.abspath(filename) + f = open(filepath,'wb') + f.write("TESTFILE -- ["+filepath+"]") + f.close() + + # assertions + self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',filename) # should not be listed + self.admin.assert_icommand("iput "+filename) # put file + self.admin.assert_icommand("irepl -R " + self.testresc + " --purgec " + filename, 'STDOUT', 'Specifying a minimum number of replicas to keep is deprecated.') # replicate to test resource + self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 0 ",filename]) # should not be listed (trimmed) + self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 1 ",filename]) # should be listed twice - 2 of 3 + self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 2 ",filename]) # should be listed twice - 1 of 3 + + # local cleanup + output = commands.getstatusoutput( 'rm '+filepath ) + + def test_decoupled_naming_policy(self): + if self.archive_naming_policy != 'decoupled': + self.skipTest("Archive naming policy is not set to 'decoupled'") + + # local setup + filename = self.testfile + + # run as regular user + session = self.user0 + collection = session.session_collection + + # iquest to get the object id of the replica on the S3 archive + id_query = ( "select DATA_ID where COLL_NAME =" + "'" + collection + "'" + + " and DATA_NAME =" + "'" + filename + "'" + + " and DATA_REPL_NUM ='1'" ) + + # iquest to get the pysical path of the replica on the S3 archive + path_query = ( "select DATA_PATH where COLL_NAME =" + "'" + collection + "'" + + " and DATA_NAME =" + "'" + filename + "'" + + " and DATA_REPL_NUM ='1'" ) + + # assertions + session.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',filename) # should not be listed + session.assert_icommand("iput "+filename) # put file + + # get object id + object_id = session.run_icommand('iquest "%s" ' + '"' + id_query + '"')[0].strip() + + # physical path we expect to see: /{bucket_name}/{reversed_id}/{obj_name} + target_path = '/' + self.s3bucketname + '/' + object_id[::-1] + '/' + filename + + # get object path + physical_path = session.run_icommand('iquest "%s" ' + '"' + path_query + '"')[0].strip() + + # verify object path + self.assertEqual(target_path, physical_path) + + # move the file + new_filename = "%s.new" % filename + session.assert_icommand("imv %s %s" % (filename, new_filename)) + + # get and purge cache replica + session.assert_icommand("iget -f --purgec %s" % new_filename, 'STDOUT', 'Specifying a minimum number of replicas to keep is deprecated.') # get file and purge 'cached' replica + + # get again now that it is not in cache + session.assert_icommand("iget -f %s" % new_filename) # get file + + # cleanup + session.run_icommand('irm -f ' + new_filename) + + def test_decoupled_naming_policy_issue1855(self): + if self.archive_naming_policy != 'decoupled': + self.skipTest("Archive naming policy is not set to 'decoupled'") + + # local setup + filename = self.testfile + + # run as regular user + session = self.user0 + collection = session.session_collection + + # modify the s3 archive resource so that it only has the bucket name in the context + self.admin.assert_icommand('iadmin modresc archiveResc path /%s' % self.s3bucketname, 'STDOUT_SINGLELINE', 'Previous resource path:') + + # iquest to get the object id of the replica on the S3 archive + id_query = ( "select DATA_ID where COLL_NAME =" + "'" + collection + "'" + + " and DATA_NAME =" + "'" + filename + "'" + + " and DATA_REPL_NUM ='1'" ) + + # iquest to get the pysical path of the replica on the S3 archive + path_query = ( "select DATA_PATH where COLL_NAME =" + "'" + collection + "'" + + " and DATA_NAME =" + "'" + filename + "'" + + " and DATA_REPL_NUM ='1'" ) + + # assertions + session.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',filename) # should not be listed + session.assert_icommand("iput "+filename) # put file + + # get object id + object_id = session.run_icommand('iquest "%s" ' + '"' + id_query + '"')[0].strip() + + # physical path we expect to see: /{bucket_name}/{reversed_id}/{obj_name} + target_path = '/' + self.s3bucketname + '/' + object_id[::-1] + '/' + filename + + # get object path + physical_path = session.run_icommand('iquest "%s" ' + '"' + path_query + '"')[0].strip() + + # verify object path + self.assertEqual(target_path, physical_path) + + # move the file + new_filename = "%s.new" % filename + session.assert_icommand("imv %s %s" % (filename, new_filename)) + + # get and purge cache replica + session.assert_icommand("iget -f --purgec %s" % new_filename, 'STDOUT', 'Specifying a minimum number of replicas to keep is deprecated.') # get file and purge 'cached' replica + + # get again now that it is not in cache + session.assert_icommand("iget -f %s" % new_filename) # get file + + # cleanup + session.run_icommand('irm -f ' + filename) + + @unittest.skip("skip until minio added to CI") + def test_multiple_s3_endpoints_replication_issue1858(self): + + # local setup + filename = self.testfile + + # run as regular user + session = self.user0 + collection = session.session_collection + + # set up resources + + # TODO change these as necessary + minio_auth_file = '/var/lib/irods/s3.keypair' + minio_bucket_name = 'irods-bucket' + + hostname = lib.get_hostname() + s3params_aws = 'S3_RETRY_COUNT=1;S3_WAIT_TIME_SEC=1;S3_PROTO=%s;S3_MPU_CHUNK=10;S3_MPU_THREADS=4;S3_ENABLE_MD5=1' % self.proto + s3params_aws += ';S3_DEFAULT_HOSTNAME=%s' % self.s3endPoint + s3params_aws += ';S3_AUTH_FILE=%s' % self.keypairfile + s3params_aws += ';S3_REGIONNAME=%s' % self.s3region + s3params_aws += ';ARCHIVE_NAMING_POLICY=%s' % self.archive_naming_policy + + s3params_minio = 'S3_RETRY_COUNT=1;S3_WAIT_TIME_SEC=1;S3_PROTO=%s;S3_MPU_CHUNK=10;S3_MPU_THREADS=4;S3_ENABLE_MD5=1' % self.proto + s3params_minio += ';S3_DEFAULT_HOSTNAME=%s:9000' % hostname + s3params_minio += ';S3_AUTH_FILE=%s' % minio_auth_file + s3params_minio += ';S3_REGIONNAME=%s' % self.s3region + s3params_minio += ';ARCHIVE_NAMING_POLICY=%s' % self.archive_naming_policy + + try: + + # make resource tree with repl and two compound resources underneath + self.admin.assert_icommand('iadmin mkresc s3repl_1858 replication', 'STDOUT_SINGLELINE', 'Creating') + self.admin.assert_icommand('iadmin mkresc s3compound1_1858 compound', 'STDOUT_SINGLELINE', 'Creating') + self.admin.assert_icommand('iadmin mkresc s3compound2_1858 compound', 'STDOUT_SINGLELINE', 'Creating') + self.admin.assert_icommand('iadmin mkresc s3cache1_1858 unixfilesystem %s:/tmp/s3cache1_1858 unixfilesystem' % hostname, 'STDOUT_SINGLELINE', 'Creating') + self.admin.assert_icommand('iadmin mkresc s3archive1_1858 s3 %s:/%s/irods/Vault %s' % (hostname, self.s3bucketname, s3params_aws), 'STDOUT_SINGLELINE', 's3archive1_1858') + self.admin.assert_icommand('iadmin mkresc s3cache2_1858 unixfilesystem %s:/tmp/s3cache2_1858 unixfilesystem' % hostname, 'STDOUT_SINGLELINE', 'Creating') + self.admin.assert_icommand('iadmin mkresc s3archive2_1858 s3 %s:/%s/irods/s3archive2_1858_vault %s' % (hostname, minio_bucket_name, s3params_minio), 'STDOUT_SINGLELINE', 's3archive2_1858') + self.admin.assert_icommand('iadmin addchildtoresc s3repl_1858 s3compound1_1858') + self.admin.assert_icommand('iadmin addchildtoresc s3repl_1858 s3compound2_1858') + self.admin.assert_icommand('iadmin addchildtoresc s3compound1_1858 s3cache1_1858 cache') + self.admin.assert_icommand('iadmin addchildtoresc s3compound1_1858 s3archive1_1858 archive') + self.admin.assert_icommand('iadmin addchildtoresc s3compound2_1858 s3cache2_1858 cache') + self.admin.assert_icommand('iadmin addchildtoresc s3compound2_1858 s3archive2_1858 archive') + + # put a file to this tree + session.assert_icommand('iput -R s3repl_1858 %s' % filename) # put file + + # make sure we have four replicas + session.assert_icommand('ils -L %s' % filename, 'STDOUT_MULTILINE', ['s3repl_1858;s3compound1_1858;s3cache1_1858', + 's3repl_1858;s3compound1_1858;s3archive1_1858', + 's3repl_1858;s3compound2_1858;s3cache2_1858', + 's3repl_1858;s3compound2_1858;s3archive2_1858']) + finally: + + # remove the file + session.assert_icommand('irm -f %s' % filename) # remove file + + # cleanup + self.admin.assert_icommand('iadmin rmchildfromresc s3repl_1858 s3compound1_1858') + self.admin.assert_icommand('iadmin rmchildfromresc s3repl_1858 s3compound2_1858') + self.admin.assert_icommand('iadmin rmchildfromresc s3compound1_1858 s3cache1_1858 cache') + self.admin.assert_icommand('iadmin rmchildfromresc s3compound1_1858 s3archive1_1858 archive') + self.admin.assert_icommand('iadmin rmchildfromresc s3compound2_1858 s3cache2_1858 cache') + self.admin.assert_icommand('iadmin rmchildfromresc s3compound2_1858 s3archive2_1858 archive') + self.admin.assert_icommand('iadmin rmresc s3repl_1858') + self.admin.assert_icommand('iadmin rmresc s3compound1_1858') + self.admin.assert_icommand('iadmin rmresc s3compound2_1858') + self.admin.assert_icommand('iadmin rmresc s3cache1_1858') + self.admin.assert_icommand('iadmin rmresc s3archive1_1858') + self.admin.assert_icommand('iadmin rmresc s3cache2_1858') + self.admin.assert_icommand('iadmin rmresc s3archive2_1858') diff --git a/packaging/resource_suite_s3_nocache.py b/packaging/resource_suite_s3_nocache.py index b21afd75..89c9fc27 100644 --- a/packaging/resource_suite_s3_nocache.py +++ b/packaging/resource_suite_s3_nocache.py @@ -33,10 +33,28 @@ from . import session -class ResourceBase(session.make_sessions_mixin([('otherrods', 'rods')], [('alice', 'apass'), ('bobby', 'bpass')])): +class Test_S3_NoCache_Base(session.make_sessions_mixin([('otherrods', 'rods')], [('alice', 'apass'), ('bobby', 'bpass')])): + + def __init__(self, *args, **kwargs): + + # if self.proto is defined use it else default to HTTPS + try: + self.proto = self.proto + except AttributeError: + self.proto = 'HTTPS' + + # if self.archive_naming_policy is defined use it + # else default to 'consistent' + try: + self.archive_naming_policy = self.archive_naming_policy + except AttributeError: + self.archive_naming_policy = 'consistent' + + super(Test_S3_NoCache_Base, self).__init__(*args, **kwargs) def setUp(self): - super(ResourceBase, self).setUp() + + super(Test_S3_NoCache_Base, self).setUp() self.admin = self.admin_sessions[0] self.user0 = self.user_sessions[0] self.user1 = self.user_sessions[1] @@ -51,7 +69,11 @@ def setUp(self): self.read_aws_keys() # set up s3 bucket - s3_client = Minio(self.s3endPoint, access_key=self.aws_access_key_id, secret_key=self.aws_secret_access_key) + if self.proto == 'HTTPS': + s3_client = Minio(self.s3endPoint, access_key=self.aws_access_key_id, secret_key=self.aws_secret_access_key) + else: + s3_client = Minio(self.s3endPoint, access_key=self.aws_access_key_id, secret_key=self.aws_secret_access_key, secure=False) + distro_str = ''.join(platform.linux_distribution()[:2]).replace(' ','') self.s3bucketname = 'irods-ci-' + distro_str + datetime.datetime.utcnow().strftime('-%Y-%m-%d.%H-%M-%S-%f-') @@ -64,7 +86,7 @@ def setUp(self): self.anotherresc = "AnotherResc" self.anothervault = "/tmp/" + self.anotherresc - self.s3_context = "S3_DEFAULT_HOSTNAME=%s;S3_AUTH_FILE=%s;S3_REGIONNAME=%s;S3_RETRY_COUNT=2;S3_WAIT_TIME_SEC=3;S3_PROTO=HTTPS;ARCHIVE_NAMING_POLICY=consistent;HOST_MODE=cacheless_attached;S3_ENABLE_MD5=1;S3_ENABLE_MPU=%d;S3_SIGNATURE_VERSION=%d" % (self.s3endPoint, self.keypairfile, self.s3region, self.s3EnableMPU, self.s3signature_version) + self.s3_context = "S3_DEFAULT_HOSTNAME=%s;S3_AUTH_FILE=%s;S3_REGIONNAME=%s;S3_RETRY_COUNT=2;S3_WAIT_TIME_SEC=3;S3_PROTO=%s;ARCHIVE_NAMING_POLICY=consistent;HOST_MODE=cacheless_attached;S3_ENABLE_MD5=1;S3_ENABLE_MPU=%d" % (self.s3endPoint, self.keypairfile, self.s3region, self.proto, self.s3EnableMPU) self.admin.assert_icommand("iadmin modresc demoResc name origResc", 'STDOUT_SINGLELINE', 'rename', input='yes\n') @@ -91,7 +113,7 @@ def tearDown(self): self.admin.run_icommand(['irm', self.testfile, '../public/' + self.testfile]) self.admin.run_icommand('irm -rf ../../bundle') - super(ResourceBase, self).tearDown() + super(Test_S3_NoCache_Base, self).tearDown() with session.make_session_for_existing_admin() as admin_session: admin_session.run_icommand('irmtrash -M') admin_session.run_icommand(['iadmin', 'rmresc', self.testresc]) @@ -101,7 +123,10 @@ def tearDown(self): print("run_resource_teardown - END") # delete s3 bucket - s3_client = Minio(self.s3endPoint, access_key=self.aws_access_key_id, secret_key=self.aws_secret_access_key) + if self.proto == 'HTTPS': + s3_client = Minio(self.s3endPoint, access_key=self.aws_access_key_id, secret_key=self.aws_secret_access_key) + else: + s3_client = Minio(self.s3endPoint, access_key=self.aws_access_key_id, secret_key=self.aws_secret_access_key, secure=False) objects = s3_client.list_objects_v2(self.s3bucketname, recursive=True) try: @@ -118,6 +143,12 @@ def read_aws_keys(self): self.aws_access_key_id = f.readline().rstrip() self.aws_secret_access_key = f.readline().rstrip() + # read the endpoint address from the file endpointfile + def read_endpoint(self, endpointfile): + # read endpoint file + with open(endpointfile) as f: + return f.readline().rstrip() + def set_up_aws_config_dir(self): # read access keys from keypair file with open(self.keypairfile) as f: @@ -143,9 +174,6 @@ def set_up_aws_config_dir(self): cred_file.write('aws_access_key_id = ' + aws_access_key_id + '\n') cred_file.write('aws_secret_access_key = ' + aws_secret_access_key + '\n') -class ResourceSuite_S3_NoCache(ResourceBase): - - ################### # iget ################### @@ -1121,8 +1149,7 @@ def test_itrim_returns_on_negative_status__ticket_3531(self): os.unlink(filepath) -# specific test cases unique to cacheless S3 -class Test_S3_NoCache_Base(ResourceSuite_S3_NoCache): + # tests add for cacheless S3 def test_iput_large_file_over_smaller(self): @@ -1455,7 +1482,8 @@ def test_detached_mode(self): resource_host = "irods.org" # change demoResc to use detached mode - s3_context_detached = "S3_DEFAULT_HOSTNAME=%s;S3_AUTH_FILE=%s;S3_REGIONNAME=%s;S3_RETRY_COUNT=2;S3_WAIT_TIME_SEC=3;S3_PROTO=HTTPS;ARCHIVE_NAMING_POLICY=consistent;HOST_MODE=cacheless_detached;S3_ENABLE_MD5=1;S3_ENABLE_MPU=%d;S3_SIGNATURE_VERSION=%d" % (self.s3endPoint, self.keypairfile, self.s3region, self.s3EnableMPU, self.s3signature_version) + + s3_context_detached = "S3_DEFAULT_HOSTNAME=%s;S3_AUTH_FILE=%s;S3_REGIONNAME=%s;S3_RETRY_COUNT=2;S3_WAIT_TIME_SEC=3;S3_PROTO=%s;ARCHIVE_NAMING_POLICY=consistent;HOST_MODE=cacheless_detached;S3_ENABLE_MD5=1;S3_ENABLE_MPU=%d" % (self.s3endPoint, self.keypairfile, self.s3region, self.proto, self.s3EnableMPU) self.admin.assert_icommand("iadmin modresc demoResc context %s" % s3_context_detached , 'EMPTY') self.admin.assert_icommand("iadmin modresc demoResc host %s" % resource_host, 'EMPTY') @@ -1517,7 +1545,10 @@ def test_attached_mode_invalid_resource_host(self): def recursive_register_from_s3_bucket(self): # create some files on s3 - s3_client = Minio(self.s3endPoint, access_key=self.aws_access_key_id, secret_key=self.aws_secret_access_key) + if self.proto == 'HTTPS': + s3_client = Minio(self.s3endPoint, access_key=self.aws_access_key_id, secret_key=self.aws_secret_access_key) + else: + s3_client = Minio(self.s3endPoint, access_key=self.aws_access_key_id, secret_key=self.aws_secret_access_key, secure=False) file_contents = b'random test data' f = io.BytesIO(file_contents) size = len(file_contents) diff --git a/packaging/test_irods_resource_plugin_s3.py b/packaging/test_irods_resource_plugin_s3.py index ed42f50e..43bf3af2 100644 --- a/packaging/test_irods_resource_plugin_s3.py +++ b/packaging/test_irods_resource_plugin_s3.py @@ -17,6 +17,7 @@ import urllib3 from resource_suite_s3_nocache import Test_S3_NoCache_Base +from resource_suite_s3_cache import Test_S3_Cache_Base import sys if sys.version_info >= (2,7): @@ -31,528 +32,17 @@ from .test_chunkydevtest import ChunkyDevTest - -class Test_Compound_With_S3_Resource(ResourceSuite, ChunkyDevTest, unittest.TestCase): +class Test_Compound_With_S3_Resource(Test_S3_Cache_Base, unittest.TestCase): def __init__(self, *args, **kwargs): self.keypairfile='/projects/irods/vsphere-testing/externals/amazon_web_services-CI.keypair' self.archive_naming_policy='decoupled' self.s3stsdate='' self.s3region='us-east-1' self.s3endPoint='s3.amazonaws.com' - self.s3signature_version=2 self.s3sse = 0 # server side encryption super(Test_Compound_With_S3_Resource, self).__init__(*args, **kwargs) - def setUp(self): - # skip ssl tests on ub12 - distro_str = ''.join(platform.linux_distribution()[:2]).replace(' ','') - if self._testMethodName.startswith('test_ssl') and distro_str.lower().startswith('ubuntu12'): - self.skipTest("skipping ssl tests on ubuntu 12") - - # set up aws configuration - self.read_aws_keys() - - # set up s3 bucket - try: - httpClient = urllib3.poolmanager.ProxyManager( - os.environ['http_proxy'], - timeout=urllib3.Timeout.DEFAULT_TIMEOUT, - cert_reqs='CERT_REQUIRED', - retries=urllib3.Retry( - total=5, - backoff_factor=0.2, - status_forcelist=[500, 502, 503, 504] - ) - ) - except KeyError: - httpClient = None - s3_client = Minio(self.s3endPoint, - access_key=self.aws_access_key_id, - secret_key=self.aws_secret_access_key, - http_client=httpClient) - - self.s3bucketname = 'irods-ci-' + distro_str + datetime.datetime.utcnow().strftime('-%Y-%m-%d.%H-%M-%S-%f-') - self.s3bucketname += ''.join(random.choice(string.letters) for i in xrange(10)) - self.s3bucketname = self.s3bucketname[:63].lower() # bucket names can be no more than 63 characters long - s3_client.make_bucket(self.s3bucketname, location=self.s3region) - - # set up resources - hostname = lib.get_hostname() - s3params = 'S3_RETRY_COUNT=15;S3_WAIT_TIME_SEC=1;S3_PROTO=HTTPS;S3_MPU_CHUNK=10;S3_MPU_THREADS=4;S3_ENABLE_MD5=1' - s3params += ';S3_STSDATE=' + self.s3stsdate - s3params += ';S3_DEFAULT_HOSTNAME=' + self.s3endPoint - s3params += ';S3_AUTH_FILE=' + self.keypairfile - s3params += ';S3_REGIONNAME=' + self.s3region - s3params += ';S3_SIGNATURE_VERSION=' + str(self.s3signature_version) - s3params += ';ARCHIVE_NAMING_POLICY=' + self.archive_naming_policy - try: - s3params += ';S3_SERVER_ENCRYPT=' + str(self.s3sse) - except AttributeError: - pass - - s3params=os.environ.get('S3PARAMS', s3params); - - with session.make_session_for_existing_admin() as admin_session: - irods_config = IrodsConfig() - admin_session.assert_icommand("iadmin modresc demoResc name origResc", 'STDOUT_SINGLELINE', 'rename', input='yes\n') - admin_session.assert_icommand("iadmin mkresc demoResc compound", 'STDOUT_SINGLELINE', 'compound') - admin_session.assert_icommand("iadmin mkresc cacheResc 'unixfilesystem' " + hostname + ":" + irods_config.irods_directory + "/cacheRescVault", 'STDOUT_SINGLELINE', 'cacheResc') - admin_session.assert_icommand('iadmin mkresc archiveResc s3 '+hostname+':/'+self.s3bucketname+'/irods/Vault "'+s3params+'"', 'STDOUT_SINGLELINE', 'archiveResc') - admin_session.assert_icommand("iadmin addchildtoresc demoResc cacheResc cache") - admin_session.assert_icommand("iadmin addchildtoresc demoResc archiveResc archive") - - super(Test_Compound_With_S3_Resource, self).setUp() - - def tearDown(self): - super(Test_Compound_With_S3_Resource, self).tearDown() - - # delete s3 bucket - try: - httpClient = urllib3.poolmanager.ProxyManager( - os.environ['http_proxy'], - timeout=urllib3.Timeout.DEFAULT_TIMEOUT, - cert_reqs='CERT_REQUIRED', - retries=urllib3.Retry( - total=5, - backoff_factor=0.2, - status_forcelist=[500, 502, 503, 504] - ) - ) - except KeyError: - httpClient = None - s3_client = Minio(self.s3endPoint, - access_key=self.aws_access_key_id, - secret_key=self.aws_secret_access_key, - http_client=httpClient) - - objects = s3_client.list_objects_v2(self.s3bucketname, recursive=True) - try: - for del_err in s3_client.remove_objects(self.s3bucketname, [object.object_name for object in objects]): - print("Deletion Error: {}".format(del_err)) - except ResponseError as err: - print(err) - s3_client.remove_bucket(self.s3bucketname) - - # tear down resources - with session.make_session_for_existing_admin() as admin_session: - admin_session.assert_icommand("iadmin rmchildfromresc demoResc archiveResc") - admin_session.assert_icommand("iadmin rmchildfromresc demoResc cacheResc") - admin_session.assert_icommand("iadmin rmresc archiveResc") - admin_session.assert_icommand("iadmin rmresc cacheResc") - admin_session.assert_icommand("iadmin rmresc demoResc") - admin_session.assert_icommand("iadmin modresc origResc name demoResc", 'STDOUT_SINGLELINE', 'rename', input='yes\n') - - shutil.rmtree(IrodsConfig().irods_directory + "/cacheRescVault", ignore_errors=True) - - def read_aws_keys(self): - # read access keys from keypair file - with open(self.keypairfile) as f: - self.aws_access_key_id = f.readline().rstrip() - self.aws_secret_access_key = f.readline().rstrip() - - def test_irm_specific_replica(self): - self.admin.assert_icommand("ils -L "+self.testfile,'STDOUT_SINGLELINE',self.testfile) # should be listed - self.admin.assert_icommand("irepl -R "+self.testresc+" "+self.testfile) # creates replica - self.admin.assert_icommand("ils -L "+self.testfile,'STDOUT_SINGLELINE',self.testfile) # should be listed twice - self.admin.assert_icommand("irm -n 0 "+self.testfile, 'STDOUT_SINGLELINE','deprecated') # remove original from cacheResc only - self.admin.assert_icommand("ils -L "+self.testfile,'STDOUT_SINGLELINE',["2 "+self.testresc,self.testfile]) # replica 2 should still be there - self.admin.assert_icommand_fail("ils -L "+self.testfile,'STDOUT_SINGLELINE',["0 "+self.admin.default_resource,self.testfile]) # replica 0 should be gone - trashpath = self.admin.session_collection_trash - self.admin.assert_icommand_fail("ils -L "+trashpath+"/"+self.testfile,'STDOUT_SINGLELINE',["0 "+self.admin.default_resource,self.testfile]) # replica should not be in trash - - @unittest.skip("--wlock has possible race condition due to Compound/Replication PDMO") - def test_local_iput_collision_with_wlock(self): - pass - - @unittest.skip("NOTSURE / FIXME ... -K not supported, perhaps") - def test_local_iput_checksum(self): - pass - - @unittest.skip("EMPTY_RESC_PATH - no vault path for coordinating resources") - def test_ireg_as_rodsuser_in_vault(self): - pass - - @unittest.skip("No Vault for S3 archive resource") - def test_iput_overwrite_others_file__ticket_2086(self): - pass - - def test_local_iput_with_force_and_destination_resource__ticket_1706(self): - # local setup - filename = "iputwithforceanddestination.txt" - filepath = lib.create_local_testfile(filename) - doublefile = "doublefile.txt" - os.system("cat %s %s > %s" % (filename, filename, doublefile)) - doublesize = str(os.stat(doublefile).st_size) - # assertions - self.admin.assert_icommand("ils -L "+filename,'STDERR_SINGLELINE',"does not exist") # should not be listed - self.admin.assert_icommand("iput "+filename) # put file - self.admin.assert_icommand("irepl -R "+self.testresc+" "+filename) # replicate to test resource - self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',filename) # - self.admin.assert_icommand("iput -f -R %s %s %s" % (self.testresc, doublefile, filename) ) # overwrite test repl with different data - self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 0 "," "+filename]) # default resource cache should have dirty copy - self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 1 "," "+filename]) # default resource archive should have dirty copy - self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 0 "," "+doublesize+" "," "+filename]) # default resource cache should not have doublesize file - self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 1 "," "+doublesize+" "," "+filename]) # default resource archive should not have doublesize file - self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 2 "," "+doublesize+" ","& "+filename]) # targeted resource should have new double clean copy - # local cleanup - os.remove(filepath) - os.remove(doublefile) - - ################### - # irepl - ################### - - def test_irepl_update_replicas(self): - # local setup - filename = "updatereplicasfile.txt" - filepath = lib.create_local_testfile(filename) - hostname = lib.get_hostname() - doublefile = "doublefile.txt" - os.system("cat %s %s > %s" % (filename, filename, doublefile)) - doublesize = str(os.stat(doublefile).st_size) - - # assertions - self.admin.assert_icommand("iadmin mkresc thirdresc unixfilesystem %s:/tmp/thirdrescVault" % hostname, 'STDOUT_SINGLELINE', "Creating") # create third resource - self.admin.assert_icommand("iadmin mkresc fourthresc unixfilesystem %s:/tmp/fourthrescVault" % hostname, 'STDOUT_SINGLELINE', "Creating") # create fourth resource - self.admin.assert_icommand("ils -L "+filename,'STDERR_SINGLELINE',"does not exist") # should not be listed - self.admin.assert_icommand("iput "+filename) # put file - self.admin.assert_icommand("irepl -R "+self.testresc+" "+filename) # replicate to test resource - self.admin.assert_icommand("irepl -R thirdresc "+filename) # replicate to third resource - self.admin.assert_icommand("irepl -R fourthresc "+filename) # replicate to fourth resource - self.admin.assert_icommand("iput -f -R "+self.testresc+" "+doublefile+" "+filename) # repave overtop test resource - self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',filename) # for debugging - - self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 0 "," & "+filename]) # should have a dirty copy - self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 1 "," & "+filename]) # should have a dirty copy - self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 2 "," & "+filename]) # should have a clean copy - self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 3 "," & "+filename]) # should have a dirty copy - self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 4 "," & "+filename]) # should have a dirty copy - - self.admin.assert_icommand("irepl -U "+filename) # update last replica - - self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 0 "," & "+filename]) # should have a dirty copy - self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 1 "," & "+filename]) # should have a dirty copy - self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 2 "," & "+filename]) # should have a clean copy - self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 3 "," & "+filename]) # should have a dirty copy - self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 4 "," & "+filename]) # should have a clean copy - - self.admin.assert_icommand("irepl -aU "+filename) # update all replicas - - self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 0 "," & "+filename]) # should have a clean copy - self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 1 "," & "+filename]) # should have a clean copy - self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 2 "," & "+filename]) # should have a clean copy - self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 3 "," & "+filename]) # should have a clean copy - self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 4 "," & "+filename]) # should have a clean copy - - self.admin.assert_icommand("irm -f "+filename) # cleanup file - self.admin.assert_icommand("iadmin rmresc thirdresc") # remove third resource - self.admin.assert_icommand("iadmin rmresc fourthresc") # remove third resource - - # local cleanup - os.remove(filepath) - os.remove(doublefile) - - def test_irepl_over_existing_second_replica__ticket_1705(self): - # local setup - filename = "secondreplicatest.txt" - filepath = lib.create_local_testfile(filename) - # assertions - self.admin.assert_icommand("ils -L "+filename,'STDERR_SINGLELINE',"does not exist") # should not be listed - self.admin.assert_icommand("iput -R "+self.testresc+" "+filename) # put file - self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',filename) # for debugging - self.admin.assert_icommand("irepl "+filename) # replicate to default resource - self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',filename) # for debugging - self.admin.assert_icommand("irepl "+filename) # replicate overtop default resource - self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 3 "," & "+filename]) # should not have a replica 3 - self.admin.assert_icommand("irepl -R "+self.testresc+" "+filename) # replicate overtop test resource - self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 3 "," & "+filename]) # should not have a replica 3 - # local cleanup - os.remove(filepath) - - def test_irepl_over_existing_third_replica__ticket_1705(self): - # local setup - filename = "thirdreplicatest.txt" - filepath = lib.create_local_testfile(filename) - hostname = lib.get_hostname() - # assertions - self.admin.assert_icommand("iadmin mkresc thirdresc unixfilesystem %s:/tmp/thirdrescVault" % hostname, 'STDOUT_SINGLELINE', "Creating") # create third resource - self.admin.assert_icommand("ils -L "+filename,'STDERR_SINGLELINE',"does not exist") # should not be listed - self.admin.assert_icommand("iput "+filename) # put file - self.admin.assert_icommand("irepl -R "+self.testresc+" "+filename) # replicate to test resource - self.admin.assert_icommand("irepl -R thirdresc "+filename) # replicate to third resource - self.admin.assert_icommand("irepl "+filename) # replicate overtop default resource - self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',filename) # for debugging - self.admin.assert_icommand("irepl -R "+self.testresc+" "+filename) # replicate overtop test resource - self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',filename) # for debugging - self.admin.assert_icommand("irepl -R thirdresc "+filename) # replicate overtop third resource - self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',filename) # for debugging - self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 4 "," & "+filename]) # should not have a replica 4 - self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 5 "," & "+filename]) # should not have a replica 5 - self.admin.assert_icommand("irm -f "+filename) # cleanup file - self.admin.assert_icommand("iadmin rmresc thirdresc") # remove third resource - # local cleanup - os.remove(filepath) - - def test_irepl_over_existing_bad_replica__ticket_1705(self): - # local setup - filename = "reploverwritebad.txt" - filepath = lib.create_local_testfile(filename) - doublefile = "doublefile.txt" - os.system("cat %s %s > %s" % (filename, filename, doublefile)) - doublesize = str(os.stat(doublefile).st_size) - # assertions - self.admin.assert_icommand("ils -L "+filename,'STDERR_SINGLELINE',"does not exist") # should not be listed - self.admin.assert_icommand("iput "+filename) # put file - self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',filename) # for debugging - self.admin.assert_icommand("irepl -R "+self.testresc+" "+filename) # replicate to test resource - self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',filename) # for debugging - self.admin.assert_icommand("iput -f %s %s" % (doublefile, filename) ) # overwrite default repl with different data - self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 0 "," & "+filename]) # default resource cache should have clean copy - self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 0 "," "+doublesize+" "," & "+filename]) # default resource cache should have new double clean copy - self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 1 "," & "+filename]) # default resource archive should have clean copy - self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 1 "," "+doublesize+" "," & "+filename]) # default resource archive should have new double clean copy - self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 2 "+self.testresc," "+doublesize+" "," "+filename]) # test resource should not have doublesize file - self.admin.assert_icommand("irepl -R "+self.testresc+" "+filename) # replicate back onto test resource - self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 2 "+self.testresc," "+doublesize+" "," & "+filename]) # test resource should have new clean doublesize file - self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 3 "," & "+filename]) # should not have a replica 3 - # local cleanup - os.remove(filepath) - os.remove(doublefile) - - def test_iput_with_purgec(self): - # local setup - filename = "purgecfile.txt" - filepath = os.path.abspath(filename) - f = open(filepath,'wb') - f.write("TESTFILE -- ["+filepath+"]") - f.close() - - # assertions - self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',filename) # should not be listed - self.admin.assert_icommand("iput -f --purgec "+filename, 'STDOUT', 'Specifying a minimum number of replicas to keep is deprecated.') # get file and purge 'cached' replica - self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 0 ",filename]) # should not be listed (trimmed) - self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 1 ",filename]) # should be listed once - replica 1 - self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 2 ",filename]) # should be listed only once - - # local cleanup - output = commands.getstatusoutput( 'rm '+filepath ) - - def test_iget_with_purgec(self): - # local setup - filename = "purgecgetfile.txt" - filepath = os.path.abspath(filename) - f = open(filepath,'wb') - f.write("TESTFILE -- ["+filepath+"]") - f.close() - - # assertions - self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',filename) # should not be listed - self.admin.assert_icommand("iput "+filename) # put file - self.admin.assert_icommand("iget -f --purgec "+filename, 'STDOUT', 'Specifying a minimum number of replicas to keep is deprecated.') # get file and purge 'cached' replica - self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 0 ",filename]) # should not be listed (trimmed) - self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 1 ",filename]) # should be listed once - self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 2 ",filename]) # should not be listed - - # local cleanup - output = commands.getstatusoutput( 'rm '+filepath ) - - def test_irepl_with_purgec(self): - # local setup - filename = "purgecreplfile.txt" - filepath = os.path.abspath(filename) - f = open(filepath,'wb') - f.write("TESTFILE -- ["+filepath+"]") - f.close() - - # assertions - self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',filename) # should not be listed - self.admin.assert_icommand("iput "+filename) # put file - self.admin.assert_icommand("irepl -R " + self.testresc + " --purgec " + filename, 'STDOUT', 'Specifying a minimum number of replicas to keep is deprecated.') # replicate to test resource - self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 0 ",filename]) # should not be listed (trimmed) - self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 1 ",filename]) # should be listed twice - 2 of 3 - self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 2 ",filename]) # should be listed twice - 1 of 3 - - # local cleanup - output = commands.getstatusoutput( 'rm '+filepath ) - - def test_decoupled_naming_policy(self): - if self.archive_naming_policy != 'decoupled': - self.skipTest("Archive naming policy is not set to 'decoupled'") - - # local setup - filename = self.testfile - - # run as regular user - session = self.user0 - collection = session.session_collection - - # iquest to get the object id of the replica on the S3 archive - id_query = ( "select DATA_ID where COLL_NAME =" + "'" + collection + "'" + - " and DATA_NAME =" + "'" + filename + "'" + - " and DATA_REPL_NUM ='1'" ) - - # iquest to get the pysical path of the replica on the S3 archive - path_query = ( "select DATA_PATH where COLL_NAME =" + "'" + collection + "'" + - " and DATA_NAME =" + "'" + filename + "'" + - " and DATA_REPL_NUM ='1'" ) - - # assertions - session.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',filename) # should not be listed - session.assert_icommand("iput "+filename) # put file - - # get object id - object_id = session.run_icommand('iquest "%s" ' + '"' + id_query + '"')[0].strip() - - # physical path we expect to see: /{bucket_name}/{reversed_id}/{obj_name} - target_path = '/' + self.s3bucketname + '/' + object_id[::-1] + '/' + filename - - # get object path - physical_path = session.run_icommand('iquest "%s" ' + '"' + path_query + '"')[0].strip() - - # verify object path - self.assertEqual(target_path, physical_path) - - # move the file - new_filename = "%s.new" % filename - session.assert_icommand("imv %s %s" % (filename, new_filename)) - - # get and purge cache replica - session.assert_icommand("iget -f --purgec %s" % new_filename, 'STDOUT', 'Specifying a minimum number of replicas to keep is deprecated.') # get file and purge 'cached' replica - - # get again now that it is not in cache - session.assert_icommand("iget -f %s" % new_filename) # get file - - # cleanup - session.run_icommand('irm -f ' + new_filename) - - def test_decoupled_naming_policy_issue1855(self): - if self.archive_naming_policy != 'decoupled': - self.skipTest("Archive naming policy is not set to 'decoupled'") - - # local setup - filename = self.testfile - - # run as regular user - session = self.user0 - collection = session.session_collection - - # modify the s3 archive resource so that it only has the bucket name in the context - self.admin.assert_icommand('iadmin modresc archiveResc path /%s' % self.s3bucketname, 'STDOUT_SINGLELINE', 'Previous resource path:') - - # iquest to get the object id of the replica on the S3 archive - id_query = ( "select DATA_ID where COLL_NAME =" + "'" + collection + "'" + - " and DATA_NAME =" + "'" + filename + "'" + - " and DATA_REPL_NUM ='1'" ) - - # iquest to get the pysical path of the replica on the S3 archive - path_query = ( "select DATA_PATH where COLL_NAME =" + "'" + collection + "'" + - " and DATA_NAME =" + "'" + filename + "'" + - " and DATA_REPL_NUM ='1'" ) - - # assertions - session.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',filename) # should not be listed - session.assert_icommand("iput "+filename) # put file - - # get object id - object_id = session.run_icommand('iquest "%s" ' + '"' + id_query + '"')[0].strip() - - # physical path we expect to see: /{bucket_name}/{reversed_id}/{obj_name} - target_path = '/' + self.s3bucketname + '/' + object_id[::-1] + '/' + filename - - # get object path - physical_path = session.run_icommand('iquest "%s" ' + '"' + path_query + '"')[0].strip() - - # verify object path - self.assertEqual(target_path, physical_path) - - # move the file - new_filename = "%s.new" % filename - session.assert_icommand("imv %s %s" % (filename, new_filename)) - - # get and purge cache replica - session.assert_icommand("iget -f --purgec %s" % new_filename, 'STDOUT', 'Specifying a minimum number of replicas to keep is deprecated.') # get file and purge 'cached' replica - - # get again now that it is not in cache - session.assert_icommand("iget -f %s" % new_filename) # get file - - # cleanup - session.run_icommand('irm -f ' + filename) - - @unittest.skip("skip until minio added to CI") - def test_multiple_s3_endpoints_replication_issue1858(self): - - # local setup - filename = self.testfile - - # run as regular user - session = self.user0 - collection = session.session_collection - - # set up resources - - # TODO change these as necessary - minio_auth_file = '/var/lib/irods/s3.keypair' - minio_bucket_name = 'irods-bucket' - - hostname = lib.get_hostname() - s3params_aws = 'S3_RETRY_COUNT=1;S3_WAIT_TIME_SEC=1;S3_PROTO=HTTPS;S3_MPU_CHUNK=10;S3_MPU_THREADS=4;S3_ENABLE_MD5=1' - s3params_aws += ';S3_DEFAULT_HOSTNAME=%s' % self.s3endPoint - s3params_aws += ';S3_AUTH_FILE=%s' % self.keypairfile - s3params_aws += ';S3_REGIONNAME=%s' % self.s3region - s3params_aws += ';ARCHIVE_NAMING_POLICY=%s' % self.archive_naming_policy - - s3params_minio = 'S3_RETRY_COUNT=1;S3_WAIT_TIME_SEC=1;S3_PROTO=HTTP;S3_MPU_CHUNK=10;S3_MPU_THREADS=4;S3_ENABLE_MD5=1' - s3params_minio += ';S3_DEFAULT_HOSTNAME=%s:9000' % hostname - s3params_minio += ';S3_AUTH_FILE=%s' % minio_auth_file - s3params_minio += ';S3_REGIONNAME=%s' % self.s3region - s3params_minio += ';ARCHIVE_NAMING_POLICY=%s' % self.archive_naming_policy - - try: - - # make resource tree with repl and two compound resources underneath - self.admin.assert_icommand('iadmin mkresc s3repl_1858 replication', 'STDOUT_SINGLELINE', 'Creating') - self.admin.assert_icommand('iadmin mkresc s3compound1_1858 compound', 'STDOUT_SINGLELINE', 'Creating') - self.admin.assert_icommand('iadmin mkresc s3compound2_1858 compound', 'STDOUT_SINGLELINE', 'Creating') - self.admin.assert_icommand('iadmin mkresc s3cache1_1858 unixfilesystem %s:/tmp/s3cache1_1858 unixfilesystem' % hostname, 'STDOUT_SINGLELINE', 'Creating') - self.admin.assert_icommand('iadmin mkresc s3archive1_1858 s3 %s:/%s/irods/Vault %s' % (hostname, self.s3bucketname, s3params_aws), 'STDOUT_SINGLELINE', 's3archive1_1858') - self.admin.assert_icommand('iadmin mkresc s3cache2_1858 unixfilesystem %s:/tmp/s3cache2_1858 unixfilesystem' % hostname, 'STDOUT_SINGLELINE', 'Creating') - self.admin.assert_icommand('iadmin mkresc s3archive2_1858 s3 %s:/%s/irods/s3archive2_1858_vault %s' % (hostname, minio_bucket_name, s3params_minio), 'STDOUT_SINGLELINE', 's3archive2_1858') - self.admin.assert_icommand('iadmin addchildtoresc s3repl_1858 s3compound1_1858') - self.admin.assert_icommand('iadmin addchildtoresc s3repl_1858 s3compound2_1858') - self.admin.assert_icommand('iadmin addchildtoresc s3compound1_1858 s3cache1_1858 cache') - self.admin.assert_icommand('iadmin addchildtoresc s3compound1_1858 s3archive1_1858 archive') - self.admin.assert_icommand('iadmin addchildtoresc s3compound2_1858 s3cache2_1858 cache') - self.admin.assert_icommand('iadmin addchildtoresc s3compound2_1858 s3archive2_1858 archive') - - # put a file to this tree - session.assert_icommand('iput -R s3repl_1858 %s' % filename) # put file - - # make sure we have four replicas - session.assert_icommand('ils -L %s' % filename, 'STDOUT_MULTILINE', ['s3repl_1858;s3compound1_1858;s3cache1_1858', - 's3repl_1858;s3compound1_1858;s3archive1_1858', - 's3repl_1858;s3compound2_1858;s3cache2_1858', - 's3repl_1858;s3compound2_1858;s3archive2_1858']) - finally: - - # remove the file - session.assert_icommand('irm -f %s' % filename) # remove file - - # cleanup - self.admin.assert_icommand('iadmin rmchildfromresc s3repl_1858 s3compound1_1858') - self.admin.assert_icommand('iadmin rmchildfromresc s3repl_1858 s3compound2_1858') - self.admin.assert_icommand('iadmin rmchildfromresc s3compound1_1858 s3cache1_1858 cache') - self.admin.assert_icommand('iadmin rmchildfromresc s3compound1_1858 s3archive1_1858 archive') - self.admin.assert_icommand('iadmin rmchildfromresc s3compound2_1858 s3cache2_1858 cache') - self.admin.assert_icommand('iadmin rmchildfromresc s3compound2_1858 s3archive2_1858 archive') - self.admin.assert_icommand('iadmin rmresc s3repl_1858') - self.admin.assert_icommand('iadmin rmresc s3compound1_1858') - self.admin.assert_icommand('iadmin rmresc s3compound2_1858') - self.admin.assert_icommand('iadmin rmresc s3cache1_1858') - self.admin.assert_icommand('iadmin rmresc s3archive1_1858') - self.admin.assert_icommand('iadmin rmresc s3cache2_1858') - self.admin.assert_icommand('iadmin rmresc s3archive2_1858') - -class Test_Compound_With_S3_Resource_EU_Central_1(Test_Compound_With_S3_Resource): +class Test_Compound_With_S3_Resource_EU_Central_1(Test_S3_Cache_Base, unittest.TestCase): ''' This also tests signature V4 with the x-amz-date header. ''' @@ -562,30 +52,27 @@ def __init__(self, *args, **kwargs): self.s3stsdate='' self.s3region='eu-central-1' self.s3endPoint='s3.eu-central-1.amazonaws.com' - self.s3signature_version=4 - super(Test_Compound_With_S3_Resource, self).__init__(*args, **kwargs) + super(Test_Compound_With_S3_Resource_EU_Central_1, self).__init__(*args, **kwargs) -class Test_Compound_With_S3_Resource_STSDate_Header(Test_Compound_With_S3_Resource): +class Test_Compound_With_S3_Resource_STSDate_Header(Test_S3_Cache_Base, unittest.TestCase): def __init__(self, *args, **kwargs): self.keypairfile='/projects/irods/vsphere-testing/externals/amazon_web_services-CI.keypair' self.archive_naming_policy='decoupled' self.s3stsdate='date' self.s3region='us-east-1' self.s3endPoint='s3.amazonaws.com' - self.s3signature_version=2 - super(Test_Compound_With_S3_Resource, self).__init__(*args, **kwargs) + super(Test_Compound_With_S3_Resource_STSDate_Header, self).__init__(*args, **kwargs) -class Test_Compound_With_S3_Resource_STSDate_Header_V4(Test_Compound_With_S3_Resource): +class Test_Compound_With_S3_Resource_STSDate_Header_V4(Test_S3_Cache_Base, unittest.TestCase): def __init__(self, *args, **kwargs): self.keypairfile='/projects/irods/vsphere-testing/externals/amazon_web_services-CI.keypair' self.archive_naming_policy='decoupled' self.s3stsdate='date' self.s3region='us-east-1' self.s3endPoint='s3.amazonaws.com' - self.s3signature_version=4 - super(Test_Compound_With_S3_Resource, self).__init__(*args, **kwargs) + super(Test_Compound_With_S3_Resource_STSDate_Header_V4, self).__init__(*args, **kwargs) -class Test_Compound_With_S3_Resource_V4_SSE(Test_Compound_With_S3_Resource): +class Test_Compound_With_S3_Resource_V4_SSE(Test_S3_Cache_Base, unittest.TestCase): def __init__(self, *args, **kwargs): self.keypairfile='/projects/irods/vsphere-testing/externals/amazon_web_services-CI.keypair' self.archive_naming_policy='decoupled' @@ -593,8 +80,7 @@ def __init__(self, *args, **kwargs): self.s3sse=1 self.s3region='us-east-1' self.s3endPoint='s3.amazonaws.com' - self.s3signature_version=4 - super(Test_Compound_With_S3_Resource, self).__init__(*args, **kwargs) + super(Test_Compound_With_S3_Resource_V4_SSE, self).__init__(*args, **kwargs) class Test_S3_NoCache_V4(Test_S3_NoCache_Base, unittest.TestCase): @@ -602,9 +88,8 @@ def __init__(self, *args, **kwargs): self.keypairfile='/projects/irods/vsphere-testing/externals/amazon_web_services-CI.keypair' self.s3region='us-east-1' self.s3endPoint='s3.amazonaws.com' - self.s3signature_version=4 self.s3EnableMPU=1 - super(Test_S3_NoCache_Base, self).__init__(*args, **kwargs) + super(Test_S3_NoCache_V4, self).__init__(*args, **kwargs) class Test_S3_NoCache_EU_Central_1(Test_S3_NoCache_Base, unittest.TestCase): ''' @@ -614,25 +99,13 @@ def __init__(self, *args, **kwargs): self.keypairfile='/projects/irods/vsphere-testing/externals/amazon_web_services-CI.keypair' self.s3region='eu-central-1' self.s3endPoint='s3.eu-central-1.amazonaws.com' - self.s3signature_version=4 - self.s3EnableMPU=1 - super(Test_S3_NoCache_Base, self).__init__(*args, **kwargs) - -class Test_S3_NoCache_V2(Test_S3_NoCache_Base, unittest.TestCase): - def __init__(self, *args, **kwargs): - self.keypairfile='/projects/irods/vsphere-testing/externals/amazon_web_services-CI.keypair' - self.s3region='us-east-1' - self.s3endPoint='s3.amazonaws.com' - self.s3signature_version=2 self.s3EnableMPU=1 - super(Test_S3_NoCache_Base, self).__init__(*args, **kwargs) - + super(Test_S3_NoCache_EU_Central_1, self).__init__(*args, **kwargs) class Test_S3_NoCache_MPU_Disabled(Test_S3_NoCache_Base, unittest.TestCase): def __init__(self, *args, **kwargs): self.keypairfile='/projects/irods/vsphere-testing/externals/amazon_web_services-CI.keypair' self.s3region='us-east-1' self.s3endPoint='s3.amazonaws.com' - self.s3signature_version=2 self.s3EnableMPU=0 - super(Test_S3_NoCache_Base, self).__init__(*args, **kwargs) + super(Test_S3_NoCache_MPU_Disabled, self).__init__(*args, **kwargs) diff --git a/packaging/test_irods_resource_plugin_s3_ceph.py b/packaging/test_irods_resource_plugin_s3_ceph.py new file mode 100644 index 00000000..1016db8b --- /dev/null +++ b/packaging/test_irods_resource_plugin_s3_ceph.py @@ -0,0 +1,60 @@ +try: + from minio import Minio + from minio.error import ResponseError +except ImportError: + print('This test requires minio: perhaps try pip install minio') + exit() + +import commands +import datetime +import os +import platform +import random +import re +import shutil +import string +import subprocess +import urllib3 + +from resource_suite_s3_nocache import Test_S3_NoCache_Base +from resource_suite_s3_cache import Test_S3_Cache_Base + +import sys +if sys.version_info >= (2,7): + import unittest +else: + import unittest2 as unittest + +from .. import lib +from . import session +from ..configuration import IrodsConfig +from .resource_suite import ResourceSuite +from .test_chunkydevtest import ChunkyDevTest + + +class Test_Compound_With_S3_Resource(Test_S3_Cache_Base, unittest.TestCase): + def __init__(self, *args, **kwargs): + self.keypairfile='/var/lib/irods/ceph_s3_key.keypair' + self.archive_naming_policy='decoupled' + self.s3stsdate='' + self.s3region='us-east-1' + self.s3endPoint = self.read_endpoint('/var/lib/irods/ceph_endpoint.txt') + self.s3sse = 0 # server side encryption + super(Test_Compound_With_S3_Resource, self).__init__(*args, **kwargs) + +class Test_S3_NoCache_V4(Test_S3_NoCache_Base, unittest.TestCase): + + def __init__(self, *args, **kwargs): + self.keypairfile='/var/lib/irods/ceph_s3_key.keypair' + self.s3region='us-east-1' + self.s3endPoint = self.read_endpoint('/var/lib/irods/ceph_endpoint.txt') + self.s3EnableMPU=1 + super(Test_S3_NoCache_V4, self).__init__(*args, **kwargs) + +class Test_S3_NoCache_MPU_Disabled(Test_S3_NoCache_Base, unittest.TestCase): + def __init__(self, *args, **kwargs): + self.keypairfile='/var/lib/irods/ceph_s3_key.keypair' + self.s3region='us-east-1' + self.s3endPoint = self.read_endpoint('/var/lib/irods/ceph_endpoint.txt') + self.s3EnableMPU=0 + super(Test_S3_NoCache_MPU_Disabled, self).__init__(*args, **kwargs) diff --git a/packaging/test_irods_resource_plugin_s3_for_cloudian.py b/packaging/test_irods_resource_plugin_s3_for_cloudian.py index cd422d29..6e07a6fe 100644 --- a/packaging/test_irods_resource_plugin_s3_for_cloudian.py +++ b/packaging/test_irods_resource_plugin_s3_for_cloudian.py @@ -34,7 +34,6 @@ def __init__(self, *args, **kwargs): self.s3stsdate='' self.s3region='demoreg1' self.s3endPoint='s3.cloudianhyperstore.com' - self.s3signature_version=2 self.s3sse = 0 # server side encryption super(Test_Compound_With_S3_Resource, self).__init__(*args, **kwargs) @@ -63,7 +62,6 @@ def setUp(self): s3params += ';S3_DEFAULT_HOSTNAME=' + self.s3endPoint s3params += ';S3_AUTH_FILE=' + self.keypairfile s3params += ';S3_REGIONNAME=' + self.s3region - s3params += ';S3_SIGNATURE_VERSION=' + str(self.s3signature_version) s3params += ';ARCHIVE_NAMING_POLICY=' + self.archive_naming_policy try: s3params += ';S3_SERVER_ENCRYPT=' + str(self.s3sse) diff --git a/packaging/test_irods_resource_plugin_s3_for_gcp.py b/packaging/test_irods_resource_plugin_s3_for_gcp.py index 46700e63..6cab0846 100644 --- a/packaging/test_irods_resource_plugin_s3_for_gcp.py +++ b/packaging/test_irods_resource_plugin_s3_for_gcp.py @@ -37,7 +37,6 @@ def setUp(self): self.s3stsdate='' self.s3region='us-east1-a' self.s3endPoint='https://storage.googleapis.com' - self.s3signature_version=4 self.s3sse = 0 # server side encryption # skip ssl tests on ub12 @@ -77,7 +76,6 @@ def setUp(self): s3params += ';S3_DEFAULT_HOSTNAME=' + 'storage.googleapis.com' s3params += ';S3_AUTH_FILE=' + self.keypairfile s3params += ';S3_REGIONNAME=' + self.s3region - s3params += ';S3_SIGNATURE_VERSION=' + str(self.s3signature_version) s3params += ';ARCHIVE_NAMING_POLICY=' + self.archive_naming_policy try: s3params += ';S3_SERVER_ENCRYPT=' + str(self.s3sse) @@ -402,7 +400,6 @@ def test_decoupled_naming_policy(self): # self.s3stsdate='' # self.s3region='eu-central-1' # self.s3endPoint='s3.eu-central-1.amazonaws.com' -# self.s3signature_version=4 # super(Test_Compound_With_S3_GCS_Resource, self).__init__(*args, **kwargs) class Test_Compound_With_S3_GCS_Resource_STSDate_Header(Test_Compound_With_S3_GCS_Resource): @@ -412,7 +409,6 @@ def __init__(self, *args, **kwargs): self.s3stsdate='date' self.s3region='us-east1-a' self.s3endPoint='https://storage.googleapis.com' - self.s3signature_version=2 super(Test_Compound_With_S3_GCS_Resource, self).__init__(*args, **kwargs) class Test_Compound_With_S3_GCS_Resource_STSDate_Header_V4(Test_Compound_With_S3_GCS_Resource): @@ -422,7 +418,6 @@ def __init__(self, *args, **kwargs): self.s3stsdate='date' self.s3region='us-east1-a' self.s3endPoint='https://storage.googleapis.com' - self.s3signature_version=4 super(Test_Compound_With_S3_GCS_Resource, self).__init__(*args, **kwargs) class Test_Compound_With_S3_GCS_Resource_V4_SSE(Test_Compound_With_S3_GCS_Resource): @@ -433,5 +428,4 @@ def __init__(self, *args, **kwargs): self.s3sse=1 self.s3region='us-east1-a' self.s3endPoint='https://storage.googleapis.com' - self.s3signature_version=4 super(Test_Compound_With_S3_GCS_Resource, self).__init__(*args, **kwargs) diff --git a/packaging/test_irods_resource_plugin_s3_minio.py b/packaging/test_irods_resource_plugin_s3_minio.py new file mode 100644 index 00000000..4411fc0b --- /dev/null +++ b/packaging/test_irods_resource_plugin_s3_minio.py @@ -0,0 +1,63 @@ +try: + from minio import Minio + from minio.error import ResponseError +except ImportError: + print('This test requires minio: perhaps try pip install minio') + exit() + +import commands +import datetime +import os +import platform +import random +import re +import shutil +import string +import subprocess +import urllib3 + +from resource_suite_s3_nocache import Test_S3_NoCache_Base +from resource_suite_s3_cache import Test_S3_Cache_Base + +import sys +if sys.version_info >= (2,7): + import unittest +else: + import unittest2 as unittest + +from .. import lib +from . import session +from ..configuration import IrodsConfig +from .resource_suite import ResourceSuite +from .test_chunkydevtest import ChunkyDevTest + + +class Test_Compound_With_S3_Resource(Test_S3_Cache_Base, unittest.TestCase): + def __init__(self, *args, **kwargs): + self.proto = 'HTTP' + self.keypairfile='/var/lib/irods/minio.keypair' + self.archive_naming_policy='decoupled' + self.s3stsdate='' + self.s3region='us-east-1' + self.s3endPoint = 'localhost:9000' + self.s3sse = 0 # server side encryption + super(Test_Compound_With_S3_Resource, self).__init__(*args, **kwargs) + +class Test_S3_NoCache_V4(Test_S3_NoCache_Base, unittest.TestCase): + + def __init__(self, *args, **kwargs): + self.proto = 'HTTP' + self.keypairfile='/var/lib/irods/minio.keypair' + self.s3region='us-east-1' + self.s3endPoint = 'localhost:9000' + self.s3EnableMPU=1 + super(Test_S3_NoCache_V4, self).__init__(*args, **kwargs) + +class Test_S3_NoCache_MPU_Disabled(Test_S3_NoCache_Base, unittest.TestCase): + def __init__(self, *args, **kwargs): + self.proto = 'HTTP' + self.keypairfile='/var/lib/irods/minio.keypair' + self.s3region='us-east-1' + self.s3endPoint = 'localhost:9000' + self.s3EnableMPU=0 + super(Test_S3_NoCache_MPU_Disabled, self).__init__(*args, **kwargs) diff --git a/s3/libirods_s3.cpp b/s3/libirods_s3.cpp index e935e24f..61202aba 100644 --- a/s3/libirods_s3.cpp +++ b/s3/libirods_s3.cpp @@ -117,7 +117,7 @@ S3ResponseProperties savedProperties; // gets the attached_mode and cacheless_mode from the host_mode_str // return value of 0 means host_mode_str was valid // return value of -1 means defaults were set because host_mode_str was invalid -int get_booleans_from_host_mode(const std::string& host_mode_str, +int get_booleans_from_host_mode(const std::string& host_mode_str, bool& attached_mode, bool& cacheless_mode) { if ( host_mode_str == "archive_attached" ) { @@ -149,17 +149,25 @@ std::string get_resource_name(irods::plugin_property_map& _prop_map) { } } -void get_modes_from_properties(irods::plugin_property_map& _prop_map, +std::string get_region_name(irods::plugin_property_map& _prop_map) { + std::string region_name = "us-east-1"; + if (!_prop_map.get< std::string >(s3_region_name, region_name ).ok()) { + rodsLog( LOG_ERROR, "[resource_name=%s] Failed to retrieve S3 region name from resource plugin properties, using 'us-east-1'", get_resource_name(_prop_map).c_str()); + } + return region_name; +} + +void get_modes_from_properties(irods::plugin_property_map& _prop_map, bool& attached_mode, bool& cacheless_mode) { // defaults attached_mode = true; cacheless_mode = false; - std::string host_mode_str; + std::string host_mode_str; irods::error ret = _prop_map.get< std::string >(host_mode, host_mode_str); - if( ret.ok() ) { + if( ret.ok() ) { if ( get_booleans_from_host_mode(host_mode_str, attached_mode, cacheless_mode) < 0 ) { rodsLog(LOG_ERROR, "[resource_name=%s] Invalid HOST_MODE for S3 plugin [%s]. Setting to default - archive_attached.", get_resource_name(_prop_map).c_str(), host_mode_str.c_str()); _prop_map.set(host_mode, "archive_attached"); @@ -170,7 +178,7 @@ void get_modes_from_properties(irods::plugin_property_map& _prop_map, _prop_map.set(host_mode, "archive_attached"); } } - + // Sleep for *at least* the given time, plus some up to 1s additional // The random addition ensures that threads don't all cluster up and retry @@ -456,25 +464,10 @@ irods::error parseS3Path ( return result; } -// Get S3 Signature version from plugin property map -S3SignatureVersion s3GetSignatureVersion (irods::plugin_property_map& _prop_map) -{ - std::string version_str; - - irods::error ret = _prop_map.get< std::string >(s3_signature_version, version_str); - if (ret.ok()) { - if (version_str == "4" || boost::iequals(version_str, "V4")) { - return S3SignatureV4; - } - } - - return S3SignatureV2; // default -} - irods::error readS3AuthInfo ( const std::string& _filename, std::string& _rtn_key_id, - std::string& _rtn_access_key, + std::string& _rtn_access_key, irods::plugin_property_map& _prop_map ) { irods::error result = SUCCESS(); @@ -638,13 +631,13 @@ irods::error s3InitPerOperation ( irods::error result = SUCCESS(); std::string resource_name = get_resource_name(_prop_map); - + size_t retry_count = 10; std::string retry_count_str; result = _prop_map.get< size_t >( s3_retry_count, retry_count ); - + size_t wait_time = S3_DEFAULT_RETRY_WAIT_SEC; _prop_map.get(s3_wait_time_sec_size_t, wait_time); @@ -652,11 +645,6 @@ irods::error s3InitPerOperation ( while( ctr < retry_count ) { int status = 0; int flags = S3_INIT_ALL; - S3SignatureVersion signature_version = s3GetSignatureVersion(_prop_map); - - if (signature_version == S3SignatureV4) { - flags |= S3_INIT_SIGNATURE_V4; - } std::string&& hostname = s3GetHostname(_prop_map); const char* host_name = hostname.c_str(); // Iterate through on each try @@ -672,25 +660,6 @@ irods::error s3InitPerOperation ( result = ASSERT_ERROR(status == S3StatusOK, status, "[resource_name=%s] Error initializing the S3 library. Status = %d.", resource_name.c_str(), status, msg.str().c_str()); if( result.ok() ) { - - // If using V4 we also need to set the S3 region name - if (signature_version == S3SignatureV4) { - std::string region_name = "us-east-1"; - - // Get S3 region name from plugin property map - if (!_prop_map.get< std::string >(s3_region_name, region_name ).ok()) { - rodsLog( LOG_ERROR, "[resource_name=%s] Failed to retrieve S3 region name from resource plugin properties, using 'us-east-1'", resource_name.c_str()); - } - - S3Status status = S3_set_region_name(region_name.c_str()); - if (status != S3StatusOK) { - std::string error_str = boost::str(boost::format("[resource_name=%s] failed to set region name to %s: %s") % resource_name.c_str() % - region_name.c_str() % S3_get_status_name(status)); - rodsLog(LOG_ERROR, error_str.c_str()); - return ERROR(S3_INIT_ERROR, error_str.c_str()); - } - } - break; } @@ -779,30 +748,6 @@ ssize_t s3GetMPUThreads ( return threads; } -bool s3GetEnableMD5 ( - irods::plugin_property_map& _prop_map ) -{ - irods::error ret; - std::string enable_str; - bool enable = false; - - // Don't send md5 digest when using signature V4 - if (s3GetSignatureVersion(_prop_map) == S3SignatureV4) { - return false; - } - - ret = _prop_map.get< std::string >( - s3_enable_md5, - enable_str ); - if (ret.ok()) { - // Only 0 = no, 1 = yes. Adding in strings would require localization I think - int parse = atol(enable_str.c_str()); - if (parse != 0) - enable = true; - } - return enable; -} - bool s3GetEnableMultiPartUpload ( irods::plugin_property_map& _prop_map ) @@ -933,7 +878,8 @@ static void mrdWorkerThread ( std::string&& hostname = s3GetHostname(_prop_map); bucketContext.hostName = hostname.c_str(); // Safe to do, this is a local copy of the data structure S3_get_object( &bucketContext, g_mrdKey, NULL, rangeData.get_object_data.offset, - rangeData.get_object_data.contentLength, 0, &getObjectHandler, &rangeData ); + rangeData.get_object_data.contentLength, 0, 0, + &getObjectHandler, &rangeData ); unsigned long long usEnd = usNow(); double bw = (g_mrdData[seq-1].get_object_data.contentLength / (1024.0*1024.0)) / ( (usEnd - usStart) / 1000000.0 ); msg << " -- END -- BW=" << bw << " MB/s"; @@ -942,7 +888,7 @@ static void mrdWorkerThread ( } while ((rangeData.status != S3StatusOK) && S3_status_is_retryable(rangeData.status) && (++retry_cnt < retry_count_limit)); if (rangeData.status != S3StatusOK) { msg.str( std::string() ); // Clear - msg << "[resource_name=" << resource_name << "] " << __FUNCTION__ + msg << "[resource_name=" << resource_name << "] " << __FUNCTION__ << " - Error getting the S3 object: \"" << g_mrdKey << "\" range " << seq; if (rangeData.status >= 0) { msg << " - \"" << S3_get_status_name( rangeData.status ) << "\""; @@ -1010,6 +956,8 @@ irods::error s3GetFile( if((result = ASSERT_ERROR(cache_fd != -1, UNIX_FILE_OPEN_ERR, "[resource_name=%s] Failed to open the cache file: \"%s\".", resource_name.c_str(), _filename.c_str())).ok()) { + std::string region_name = get_region_name(_prop_map); + callback_data_t data; S3BucketContext bucketContext; @@ -1020,6 +968,7 @@ irods::error s3GetFile( bucketContext.uriStyle = S3UriStylePath; bucketContext.accessKeyId = _key_id.c_str(); bucketContext.secretAccessKey = _access_key.c_str(); + bucketContext.authRegion = region_name.c_str(); long chunksize = s3GetMPUChunksize( _prop_map ); @@ -1039,7 +988,7 @@ irods::error s3GetFile( std::string&& hostname = s3GetHostname(_prop_map); bucketContext.hostName = hostname.c_str(); // Safe to do, this is a local copy of the data structure data.pCtx = &bucketContext; - S3_get_object (&bucketContext, key.c_str(), NULL, 0, _fileSize, 0, &getObjectHandler, &data); + S3_get_object (&bucketContext, key.c_str(), NULL, 0, _fileSize, 0, 0, &getObjectHandler, &data); unsigned long long usEnd = usNow(); double bw = (_fileSize / (1024.0*1024.0)) / ( (usEnd - usStart) / 1000000.0 ); rodsLog( LOG_DEBUG, "GETBW=%lf", bw); @@ -1292,7 +1241,7 @@ static void mpuCancel( S3BucketContext *bucketContext, const char *key, const ch rodsLog( LOG_ERROR, msg.str().c_str() ); g_mpuCancelRespCompCB_status = S3StatusOK; g_mpuCancelRespCompCB_pCtx = bucketContext; - S3_abort_multipart_upload(bucketContext, key, upload_id, &abortHandler); + S3_abort_multipart_upload(bucketContext, key, upload_id, 0, &abortHandler); status = g_mpuCancelRespCompCB_status; if (status != S3StatusOK) { msg.str( std::string() ); // Clear @@ -1360,8 +1309,6 @@ static void mpuWorkerThread ( S3PutProperties *putProps = NULL; putProps = (S3PutProperties*)calloc( sizeof(S3PutProperties), 1 ); - if ( putProps && partData.enable_md5 ) - putProps->md5 = s3CalcMD5( partData.put_object_data.fd, partData.put_object_data.offset, partData.put_object_data.contentLength, resource_name ); putProps->expires = -1; unsigned long long usStart = usNow(); std::string&& hostname = s3GetHostname(_prop_map); @@ -1378,9 +1325,10 @@ static void mpuWorkerThread ( startOffset, count, putProps, &lastModified, 512 /*TBD - magic # */, partData.manager->etags[seq-1], 0, - ©ResponseHandler, &partData); + 0, ©ResponseHandler, &partData); } else { - S3_upload_part(&bucketContext, g_mpuKey, putProps, &putObjectHandler, seq, g_mpuUploadId, partData.put_object_data.contentLength, 0, &partData); + S3_upload_part(&bucketContext, g_mpuKey, putProps, &putObjectHandler, seq, + g_mpuUploadId, partData.put_object_data.contentLength, 0, 0, &partData); } unsigned long long usEnd = usNow(); double bw = (g_mpuData[seq-1].put_object_data.contentLength / (1024.0 * 1024.0)) / ( (usEnd - usStart) / 1000000.0 ); @@ -1426,7 +1374,6 @@ irods::error s3PutCopyFile( int err_status = 0; long chunksize = s3GetMPUChunksize( _prop_map ); size_t retry_cnt = 0; - bool enable_md5 = s3GetEnableMD5 ( _prop_map ); bool server_encrypt = s3GetServerEncrypt ( _prop_map ); std::stringstream msg; @@ -1460,6 +1407,8 @@ irods::error s3PutCopyFile( if((result = ASSERT_ERROR(cache_fd != -1, err_status, "[resource_name=%s] Failed to open the cache file: \"%s\".", resource_name.c_str(), _filename.c_str())).ok()) { + std::string region_name = get_region_name(_prop_map); + callback_data_t data; S3BucketContext bucketContext; @@ -1470,12 +1419,10 @@ irods::error s3PutCopyFile( bucketContext.uriStyle = S3UriStylePath; bucketContext.accessKeyId = _key_id.c_str(); bucketContext.secretAccessKey = _access_key.c_str(); - + bucketContext.authRegion = region_name.c_str(); S3PutProperties *putProps = NULL; putProps = (S3PutProperties*)calloc( sizeof(S3PutProperties), 1 ); - if ( putProps && enable_md5 ) - putProps->md5 = s3CalcMD5( cache_fd, 0, _fileSize, get_resource_name(_prop_map) ); if ( putProps && server_encrypt ) putProps->useServerSideEncryption = true; putProps->expires = -1; @@ -1494,11 +1441,11 @@ irods::error s3PutCopyFile( data.fd = cache_fd; data.contentLength = data.originalContentLength = _fileSize; data.pCtx = &bucketContext; - + unsigned long long usStart = usNow(); std::string&& hostname = s3GetHostname(_prop_map); bucketContext.hostName = hostname.c_str(); // Safe to do, this is a local copy of the data structure - S3_put_object (&bucketContext, key.c_str(), _fileSize, putProps, 0, &putObjectHandler, &data); + S3_put_object (&bucketContext, key.c_str(), _fileSize, putProps, 0, 0, &putObjectHandler, &data); unsigned long long usEnd = usNow(); double bw = (_fileSize / (1024.0*1024.0)) / ( (usEnd - usStart) / 1000000.0 ); rodsLog( LOG_DEBUG, "BW=%lf", bw); @@ -1592,7 +1539,7 @@ irods::error s3PutCopyFile( std::string&& hostname = s3GetHostname(_prop_map); bucketContext.hostName = hostname.c_str(); // Safe to do, this is a local copy of the data structure manager.pCtx = &bucketContext; - S3_initiate_multipart(&bucketContext, key.c_str(), putProps, &mpuInitialHandler, NULL, &manager); + S3_initiate_multipart(&bucketContext, key.c_str(), putProps, &mpuInitialHandler, NULL, 0, &manager); if (manager.status != S3StatusOK) s3_sleep( retry_wait, 0 ); } while ( (manager.status != S3StatusOK) && S3_status_is_retryable(manager.status) && ( ++retry_cnt < retry_count_limit)); if (manager.upload_id == NULL || manager.status != S3StatusOK) { @@ -1626,6 +1573,8 @@ irods::error s3PutCopyFile( srcBucketContext.uriStyle = S3UriStylePath; srcBucketContext.accessKeyId = _key_id.c_str(); srcBucketContext.secretAccessKey = _access_key.c_str(); + srcBucketContext.authRegion = region_name.c_str(); + srcBucketContext.authRegion = get_region_name(_prop_map).c_str(); } g_mpuNext = 0; @@ -1645,7 +1594,6 @@ irods::error s3PutCopyFile( partContentLength = (data.contentLength > chunksize)?chunksize:data.contentLength; partData.put_object_data.contentLength = partContentLength; partData.put_object_data.offset = (seq-1) * chunksize; - partData.enable_md5 = s3GetEnableMD5( _prop_map ); partData.server_encrypt = s3GetServerEncrypt( _prop_map ); g_mpuData[seq-1] = partData; data.contentLength -= partContentLength; @@ -1705,7 +1653,7 @@ irods::error s3PutCopyFile( std::string&& hostname = s3GetHostname(_prop_map); bucketContext.hostName = hostname.c_str(); // Safe to do, this is a local copy of the data structure manager.pCtx = &bucketContext; - S3_complete_multipart_upload(&bucketContext, key.c_str(), &commit_handler, manager.upload_id, manager.remaining, NULL, &manager); + S3_complete_multipart_upload(&bucketContext, key.c_str(), &commit_handler, manager.upload_id, manager.remaining, NULL, 0, &manager); if (manager.status != S3StatusOK) s3_sleep( retry_wait, 0 ); } while ((manager.status != S3StatusOK) && S3_status_is_retryable(manager.status) && ( ++retry_cnt < retry_count_limit)); if (manager.status != S3StatusOK) { @@ -1799,6 +1747,8 @@ irods::error s3CopyFile( int64_t lastModified; char eTag[256]; + std::string region_name = get_region_name(_src_ctx.prop_map()); + bzero (&bucketContext, sizeof (bucketContext)); bucketContext.bucketName = src_bucket.c_str(); bucketContext.protocol = _proto; @@ -1806,6 +1756,7 @@ irods::error s3CopyFile( bucketContext.uriStyle = S3UriStylePath; bucketContext.accessKeyId = _key_id.c_str(); bucketContext.secretAccessKey = _access_key.c_str(); + bucketContext.authRegion = region_name.c_str(); S3ResponseHandler responseHandler = { &responsePropertiesCallback, @@ -1824,7 +1775,7 @@ irods::error s3CopyFile( std::string&& hostname = s3GetHostname(_src_ctx.prop_map()); bucketContext.hostName = hostname.c_str(); // Safe to do, this is a local copy of the data structure data.pCtx = &bucketContext; - S3_copy_object(&bucketContext, src_key.c_str(), dest_bucket.c_str(), dest_key.c_str(), &putProps, &lastModified, sizeof(eTag), eTag, 0, + S3_copy_object(&bucketContext, src_key.c_str(), dest_bucket.c_str(), dest_key.c_str(), &putProps, &lastModified, sizeof(eTag), eTag, 0, 0, &responseHandler, &data); if (data.status != S3StatusOK) s3_sleep( retry_wait, 0 ); } while ( (data.status != S3StatusOK) && S3_status_is_retryable(data.status) && (++retry_cnt < retry_count_limit) ); @@ -1906,7 +1857,7 @@ irods:: error s3StartOperation(irods::plugin_property_map& _prop_map) } bool attached_mode = true, cacheless_mode = false; - get_modes_from_properties(_prop_map, attached_mode, cacheless_mode); + get_modes_from_properties(_prop_map, attached_mode, cacheless_mode); if (!attached_mode) { char resource_location[MAX_NAME_LEN]; @@ -1917,26 +1868,26 @@ irods:: error s3StartOperation(irods::plugin_property_map& _prop_map) if (cacheless_mode) { xmlInitParser(); - + // Load SSE environment if(!S3fsCurl::LoadEnvSse()) { std::string error_str = boost::str(boost::format("[resource_name=%s] something wrong about SSE environment.") % resource_name.c_str()); rodsLog(LOG_ERROR, error_str.c_str()); return ERROR(S3_INIT_ERROR, error_str.c_str()); - } - + } + // ssl init else if(!s3fs_init_global_ssl()){ std::string error_str = boost::str(boost::format("[resource_name=%s] could not initialize for ssl libraries.") % resource_name.c_str()); rodsLog(LOG_ERROR, error_str.c_str()); return ERROR(S3_INIT_ERROR, error_str.c_str()); } - - + + // init curl S3fsCurl::InitS3fsCurl("/etc/mime.types"); } - + return result; } @@ -2120,7 +2071,7 @@ irods::error register_archive_object( if(phy_path.empty()) { return ERROR( INVALID_OBJECT_NAME, - boost::str(boost::format("[resource_name=%s] no matching phy path for [%s], [%s], [%s]") % resc_name.c_str() % + boost::str(boost::format("[resource_name=%s] no matching phy path for [%s], [%s], [%s]") % resc_name.c_str() % _file_obj->logical_path() % vault_path % resc_name)); @@ -2188,8 +2139,8 @@ irods::error register_archive_object( strncpy( dst_data_obj.statusString, obj.status( ).c_str(), NAME_LEN ); dst_data_obj.dataId = obj.id(); dst_data_obj.collId = obj.coll_id(); - dst_data_obj.dataMapId = 0; - dst_data_obj.flags = 0; + dst_data_obj.dataMapId = 0; + dst_data_obj.flags = 0; strncpy( dst_data_obj.dataComments, obj.r_comment( ).c_str(), MAX_NAME_LEN ); strncpy( dst_data_obj.dataMode, obj.mode( ).c_str(), SHORT_STR_LEN ); strncpy( dst_data_obj.dataExpiry, obj.expiry_ts( ).c_str(), TIME_LEN ); @@ -2212,7 +2163,7 @@ irods::error register_archive_object( reg_inp.destDataObjInfo = &dst_data_obj; int reg_status = rsRegReplica( _comm, ®_inp ); if( reg_status < 0 ) { - std::string error_str = boost::str(boost::format("[resource_name=%s] failed register data object") % resc_name.c_str()); + std::string error_str = boost::str(boost::format("[resource_name=%s] failed register data object") % resc_name.c_str()); return ERROR( reg_status, error_str.c_str() ); } @@ -2267,7 +2218,7 @@ irods::error s3RedirectOpen( if( INT_RESC_STATUS_DOWN == resc_status ) { _out_vote = 0.0; } - else if( _curr_host == host_name) { + else if( _curr_host == host_name) { // =-=-=-=-=-=-=- // vote higher if we are on the same host irods::error get_ret = register_archive_object( @@ -2336,19 +2287,19 @@ class s3_resource : public irods::resource { extern "C" irods::resource* plugin_factory( const std::string& _inst_name, const std::string& _context ) { - + s3_resource* resc = new s3_resource(_inst_name, _context); // default modes bool cacheless_mode = false; std::string host_mode_str; - irods::error ret = resc->get_property(host_mode , host_mode_str); + irods::error ret = resc->get_property(host_mode , host_mode_str); if (ret.ok()) { bool attached_mode = true; get_booleans_from_host_mode(host_mode_str, attached_mode, cacheless_mode); - } + } if (cacheless_mode) { @@ -2465,7 +2416,7 @@ irods::resource* plugin_factory( const std::string& _inst_name, const std::strin resc->add_operation( irods::RESOURCE_OP_NOTIFY, std::function( - irods_s3_cacheless::s3FileNotifyPlugin ) ); + irods_s3_cacheless::s3FileNotifyPlugin ) ); } else { @@ -2583,8 +2534,8 @@ irods::resource* plugin_factory( const std::string& _inst_name, const std::strin resc->add_operation( irods::RESOURCE_OP_NOTIFY, std::function( - irods_s3_archive::s3FileNotifyPlugin ) ); - } + irods_s3_archive::s3FileNotifyPlugin ) ); + } // set some properties necessary for backporting to iRODS legacy code resc->set_property< int >( irods::RESOURCE_CHECK_PATH_PERM, DO_CHK_PATH_PERM ); diff --git a/s3/libirods_s3.hpp b/s3/libirods_s3.hpp index 945f397f..0b9434b9 100644 --- a/s3/libirods_s3.hpp +++ b/s3/libirods_s3.hpp @@ -41,7 +41,6 @@ const std::string s3_mpu_chunk{"S3_MPU_CHUNK"}; const std::string s3_mpu_threads{"S3_MPU_THREADS"}; const std::string s3_enable_md5{"S3_ENABLE_MD5"}; const std::string s3_server_encrypt{"S3_SERVER_ENCRYPT"}; -const std::string s3_signature_version{"S3_SIGNATURE_VERSION"}; const std::string s3_region_name{"S3_REGIONNAME"}; const std::string REPL_POLICY_KEY{"repl_policy"}; const std::string REPL_POLICY_VAL{"reg_repl"}; @@ -51,10 +50,8 @@ const size_t S3_DEFAULT_RETRY_WAIT_SEC = 1; const size_t S3_DEFAULT_RETRY_COUNT = 1; std::string s3GetHostname(irods::plugin_property_map& _prop_map); -S3SignatureVersion s3GetSignatureVersion(irods::plugin_property_map& _prop_map); long s3GetMPUChunksize(irods::plugin_property_map& _prop_map); ssize_t s3GetMPUThreads(irods::plugin_property_map& _prop_map); -bool s3GetEnableMD5(irods::plugin_property_map& _prop_map); bool s3GetEnableMultiPartUpload (irods::plugin_property_map& _prop_map ); typedef struct S3Auth { @@ -187,6 +184,7 @@ void get_modes_from_properties(irods::plugin_property_map& _prop_map, bool& attached_mode, bool& cacheless_mode); std::string get_resource_name(irods::plugin_property_map& _prop_map); +std::string get_region_name(irods::plugin_property_map& _prop_map); bool determine_unlink_for_repl_policy( rsComm_t* _comm, diff --git a/s3/s3_archive_operations.cpp b/s3/s3_archive_operations.cpp index 3de4340d..5512c72e 100644 --- a/s3/s3_archive_operations.cpp +++ b/s3/s3_archive_operations.cpp @@ -156,6 +156,8 @@ namespace irods_s3_archive { return PASS(ret); } + std::string region_name = get_region_name(_ctx.prop_map()); + std::string key_id; std::string access_key; ret = s3GetAuthCredentials(_ctx.prop_map(), key_id, access_key); @@ -171,6 +173,7 @@ namespace irods_s3_archive { bucketContext.uriStyle = S3UriStylePath; bucketContext.accessKeyId = key_id.c_str(); bucketContext.secretAccessKey = access_key.c_str(); + bucketContext.authRegion = region_name.c_str(); callback_data_t data; S3ResponseHandler responseHandler = { 0, &responseCompleteCallback }; @@ -183,6 +186,7 @@ namespace irods_s3_archive { S3_delete_object( &bucketContext, key.c_str(), 0, + 0, // timeout &responseHandler, &data); if(data.status != S3StatusOK) { @@ -206,9 +210,9 @@ namespace irods_s3_archive { } return ERROR(S3_FILE_UNLINK_ERR, msg.str()); } - + return SUCCESS(); - } // s3FileUnlinkPlugin + } // s3FileUnlinkPlugin // =-=-=-=-=-=-=- // interface for POSIX Stat @@ -257,6 +261,8 @@ namespace irods_s3_archive { ret = s3GetAuthCredentials(_ctx.prop_map(), key_id, access_key); if((result = ASSERT_PASS(ret, "[resource_name=%s] Failed to get the S3 credentials properties.", get_resource_name(_ctx.prop_map()).c_str())).ok()) { + std::string region_name = get_region_name(_ctx.prop_map()); + callback_data_t data; S3BucketContext bucketContext; @@ -267,6 +273,7 @@ namespace irods_s3_archive { bucketContext.uriStyle = S3UriStylePath; bucketContext.accessKeyId = key_id.c_str(); bucketContext.secretAccessKey = access_key.c_str(); + bucketContext.authRegion = region_name.c_str(); S3ResponseHandler headObjectHandler = { &responsePropertiesCallback, &responseCompleteCallback }; size_t retry_cnt = 0; @@ -275,7 +282,7 @@ namespace irods_s3_archive { std::string&& hostname = s3GetHostname(_ctx.prop_map()); bucketContext.hostName = hostname.c_str(); data.pCtx = &bucketContext; - S3_head_object(&bucketContext, key.c_str(), 0, &headObjectHandler, &data); + S3_head_object(&bucketContext, key.c_str(), 0, 0, &headObjectHandler, &data); if (data.status != S3StatusOK) s3_sleep( retry_wait, 0 ); } while ( (data.status != S3StatusOK) && S3_status_is_retryable(data.status) && (++retry_cnt < retry_count_limit ) ); @@ -642,7 +649,7 @@ namespace irods_s3_archive { } // s3FileRebalance - irods::error s3FileNotifyPlugin( irods::plugin_context& _ctx, + irods::error s3FileNotifyPlugin( irods::plugin_context& _ctx, const std::string* str ) { return SUCCESS(); } // s3FileNotifyPlugin diff --git a/s3/s3_cacheless_operations.cpp b/s3/s3_cacheless_operations.cpp index 3ba25b4c..5c0d4991 100644 --- a/s3/s3_cacheless_operations.cpp +++ b/s3/s3_cacheless_operations.cpp @@ -156,14 +156,6 @@ namespace irods_s3_cacheless { s3_protocol_str = ""; } - S3SignatureVersion signature_version = s3GetSignatureVersion(_prop_map); - - if (signature_version == S3SignatureV4) { - S3fsCurl::SetSignatureV4(true); - } else { - S3fsCurl::SetSignatureV4(false); - } - nomultipart = !s3GetEnableMultiPartUpload(_prop_map); // set multipart size @@ -173,9 +165,6 @@ namespace irods_s3_cacheless { // set number of simultaneous threads S3fsCurl::SetMaxParallelCount(s3GetMPUThreads(_prop_map)); - // set the MD5 flag - S3fsCurl::SetContentMd5(s3GetEnableMD5(_prop_map)); - //service_path = ""; strncpy(host, s3GetHostname(_prop_map).c_str(), MAX_NAME_LEN-1);