From d1db90caee05a359fcd2e59eb3a58f5755495dbf Mon Sep 17 00:00:00 2001 From: karajan1001 Date: Sat, 27 Nov 2021 16:04:10 +0800 Subject: [PATCH] ci: fix ci failure(#58) fix: #58 Currently some of the tests relied on the test bucket status. Because the test files were created manually several months ago. We need to decouple this to make our tests more robust. Create test files before the tests session begin. --- tests/conftest.py | 63 ++++++++++++++++++++++++++++----- tests/test_file.py | 84 ++++++++++++++++++++------------------------ tests/test_login.py | 6 ++-- tests/test_object.py | 39 ++++++++++---------- 4 files changed, 114 insertions(+), 78 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index b7391f3..8a9a5f7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,10 +4,12 @@ # pylint: disable=missing-function-docstring # pylint: disable=redefined-outer-name import os +import pathlib import subprocess import time import uuid +import oss2 import pytest import requests @@ -16,29 +18,38 @@ PORT = 5555 AccessKeyId = os.environ.get("OSS_ACCESS_KEY_ID", "") AccessKeySecret = os.environ.get("OSS_SECRET_ACCESS_KEY", "") +LICENSE_PATH = os.path.join( + pathlib.Path(__file__).parent.parent.resolve(), "LICENSE" +) +NUMBERS = b"1234567890\n" test_id = uuid.uuid4() -@pytest.fixture() +@pytest.fixture(scope="session") def emulator_endpoint(): - return "http://127.0.0.1:%s/" % PORT + return f"http://127.0.0.1:{PORT}/" -@pytest.fixture() +@pytest.fixture(scope="session") def endpoint(): return os.environ.get("OSS_ENDPOINT") -@pytest.fixture() +@pytest.fixture(scope="session") def test_bucket_name(): return os.environ.get("OSS_TEST_BUCKET_NAME") -@pytest.fixture() -def test_path(test_bucket_name): - return f"/{test_bucket_name}/ossfs_test/{test_id}" +@pytest.fixture(scope="session") +def test_directory(): + return f"ossfs_test/{test_id}" + + +@pytest.fixture(scope="session") +def test_path(test_bucket_name, test_directory): + return f"/{test_bucket_name}/{test_directory}" @pytest.fixture() @@ -46,7 +57,7 @@ def oss_emulator_server_start(emulator_endpoint): """ Start a local emulator server """ - with subprocess.Popen("ruby bin/emulator -r store -p {}".format(PORT)): + with subprocess.Popen(f"ruby bin/emulator -r store -p {PORT}"): timeout = 5 while timeout > 0: @@ -61,7 +72,7 @@ def oss_emulator_server_start(emulator_endpoint): yield -@pytest.fixture() +@pytest.fixture(scope="session") def init_config(endpoint): result = {} result["key"] = AccessKeyId @@ -73,3 +84,37 @@ def init_config(endpoint): @pytest.fixture() def ossfs(init_config): return OSSFileSystem(**init_config) + + +def put_object(endpoint, bucket_name, filename, contents): + auth = oss2.Auth(AccessKeyId, AccessKeySecret) + bucket = oss2.Bucket(auth, endpoint, bucket_name) + bucket.put_object(filename, contents) + + +def put_file(endpoint, bucket_name, key, filename): + auth = oss2.Auth(AccessKeyId, AccessKeySecret) + bucket = oss2.Bucket(auth, endpoint, bucket_name) + bucket.put_object_from_file(key, filename) + + +@pytest.fixture(scope="session") +def file_in_anonymous(endpoint, test_directory): + bucket = "dvc-anonymous" + file = f"{test_directory}/file" + put_object(endpoint, bucket, file, "foobar") + return f"/{bucket}/{file}" + + +@pytest.fixture(scope="session") +def number_file(test_bucket_name, endpoint, test_directory): + filename = f"{test_directory}/number" + put_object(endpoint, test_bucket_name, filename, NUMBERS) + return f"/{test_bucket_name}/{filename}" + + +@pytest.fixture(scope="session") +def license_file(test_bucket_name, endpoint, test_directory): + filename = f"{test_directory}/LICENSE" + put_file(endpoint, test_bucket_name, filename, LICENSE_PATH) + return f"/{test_bucket_name}/{filename}" diff --git a/tests/test_file.py b/tests/test_file.py index c7e5986..2725c55 100644 --- a/tests/test_file.py +++ b/tests/test_file.py @@ -8,17 +8,7 @@ import os import pytest - -files = { - "LICENSE": ( - b" Apache License\n" - b" Version 2.0, January 2004\n" - b" http://www.apache.org/licenses/\n" - ), - "number": (b"1234567890\n"), -} - -glob_files = {"file.dat": b"", "filexdat": b""} +from conftest import LICENSE_PATH, NUMBERS def test_simple(ossfs, test_path): @@ -62,35 +52,36 @@ def test_seek(ossfs, test_path): assert f.seek(i) == i -def test_read_small(ossfs, test_bucket_name): - fn = test_bucket_name + "/number" - with ossfs.open(fn, "rb", block_size=3) as f: +def test_read_small(ossfs, number_file): + with ossfs.open(number_file, "rb", block_size=3) as f: out = [] while True: data = f.read(2) if data == b"": break out.append(data) - assert ossfs.cat(fn) == b"".join(out) + assert ossfs.cat(number_file) == b"".join(out) -def test_read_ossfs_block(ossfs, test_bucket_name): - data = files["LICENSE"] +def test_read_ossfs_block(ossfs, license_file, number_file): + with open(LICENSE_PATH, "rb") as f_r: + data = f_r.read() lines = io.BytesIO(data).readlines() - path = test_bucket_name + "/LICENSE" - assert ossfs.read_block(path, 0, 10, b"\n") == lines[0] - assert ossfs.read_block(path, 40, 10, b"\n") == lines[1] - assert ossfs.read_block(path, 0, 80, b"\n") == lines[0] + lines[1] - assert ossfs.read_block(path, 0, 120, b"\n") == data + assert ossfs.read_block(license_file, 0, 10, b"\n") == lines[0] + assert ossfs.read_block(license_file, 40, 10, b"\n") == lines[1] + assert ossfs.read_block(license_file, 0, 80, b"\n") == lines[0] + lines[1] + assert ossfs.read_block(license_file, 0, 120, b"\n") == b"".join( + [lines[0], lines[1], lines[2]] + ) - data = files["number"] - lines = io.BytesIO(data).readlines() - path = test_bucket_name + "/number" - assert len(ossfs.read_block(path, 0, 5)) == 5 - assert len(ossfs.read_block(path, 4, 150)) == len(data) - 4 - assert ossfs.read_block(path, 20, 25) == b"" + lines = io.BytesIO(NUMBERS).readlines() + assert len(ossfs.read_block(number_file, 0, 5)) == 5 + assert len(ossfs.read_block(number_file, 4, 150)) == len(NUMBERS) - 4 + assert ossfs.read_block(number_file, 20, 25) == b"" - assert ossfs.read_block(path, 5, None) == ossfs.read_block(path, 5, 25) + assert ossfs.read_block(number_file, 5, None) == ossfs.read_block( + number_file, 5, 25 + ) @pytest.mark.parametrize("size", [2 ** 10, 2 ** 20, 10 * 2 ** 20]) @@ -134,15 +125,18 @@ def test_write_blocks(ossfs, test_path): assert ossfs.info(file)["Size"] == 15 * 2 ** 20 -def test_readline(ossfs, test_bucket_name): - all_items = files.items() - for k, data in all_items: - with ossfs.open("/".join([test_bucket_name, k]), "rb") as f: - result = f.readline() - expected = data.split(b"\n")[0] + ( - b"\n" if data.count(b"\n") else b"" - ) - assert result == expected +def test_readline(ossfs, number_file, license_file): + with ossfs.open("/".join([number_file]), "rb") as f_r: + result = f_r.readline() + expected = NUMBERS + assert result == expected + + with ossfs.open("/".join([license_file]), "rb") as f_r, open( + LICENSE_PATH, "rb" + ) as f_l: + result = f_r.readline() + expected = f_l.readline() + assert result == expected def test_readline_empty(ossfs, test_path): @@ -174,10 +168,10 @@ def test_readline_blocksize(ossfs, test_path): assert result == expected -def test_next(ossfs, test_bucket_name): - expected = files["LICENSE"].split(b"\n")[0] + b"\n" - with ossfs.open(test_bucket_name + "/LICENSE") as f: - result = next(f) +def test_next(ossfs, license_file): + with open(LICENSE_PATH, "rb") as f_l, ossfs.open(license_file) as f_r: + expected = f_l.readline() + result = next(f_r) assert result == expected @@ -219,7 +213,7 @@ def test_file_status(ossfs, test_path): @pytest.mark.parametrize("data_size", [0, 20, 10 * 2 ** 20]) @pytest.mark.parametrize("append_size", [0, 20, 10 * 2 ** 20]) def test_append(ossfs, test_path, data_size, append_size): - file = test_path + "/test_append/file_{}_{}".format(data_size, append_size) + file = test_path + f"/test_append/file_{data_size}_{append_size}" data = os.urandom(data_size) extra = os.urandom(append_size) with ossfs.open(file, "wb") as f: @@ -230,8 +224,8 @@ def test_append(ossfs, test_path, data_size, append_size): assert ossfs.cat(file) == data + extra -def test_bigger_than_block_read(ossfs, test_bucket_name): - with ossfs.open(test_bucket_name + "/number", "rb", block_size=3) as f: +def test_bigger_than_block_read(ossfs, number_file): + with ossfs.open(number_file, "rb", block_size=3) as f: out = [] while True: data = f.read(4) diff --git a/tests/test_login.py b/tests/test_login.py index 0c88913..d2064f1 100644 --- a/tests/test_login.py +++ b/tests/test_login.py @@ -73,6 +73,6 @@ def test_env_endpoint(endpoint, test_bucket_name, monkeypatch): ossfs.ls(test_bucket_name) -def test_anonymous_login(): - ossfs = OSSFileSystem(endpoint="http://oss-cn-hangzhou.aliyuncs.com") - ossfs.cat("/dvc-test-anonymous/LICENSE", connect_timeout=600) +def test_anonymous_login(file_in_anonymous, endpoint): + ossfs = OSSFileSystem(endpoint=endpoint) + ossfs.cat(f"{file_in_anonymous}") diff --git a/tests/test_object.py b/tests/test_object.py index 8480ea5..ae9ce92 100644 --- a/tests/test_object.py +++ b/tests/test_object.py @@ -132,14 +132,13 @@ def test_bulk_delete(ossfs, test_path): ) -def test_ossfs_file_access(ossfs, test_bucket_name): - fn = test_bucket_name + "/number" +def test_ossfs_file_access(ossfs, number_file): data = b"1234567890\n" - assert ossfs.cat(fn) == data - assert ossfs.head(fn, 3) == data[:3] - assert ossfs.tail(fn, 3) == data[-3:] - assert ossfs.tail(fn, 10000) == data - assert ossfs.info(fn)["Size"] == len(data) + assert ossfs.cat(number_file) == data + assert ossfs.head(number_file, 3) == data[:3] + assert ossfs.tail(number_file, 3) == data[-3:] + assert ossfs.tail(number_file, 10000) == data + assert ossfs.info(number_file)["Size"] == len(data) def test_du(ossfs, test_path): @@ -173,16 +172,16 @@ def test_ossfs_ls(ossfs, test_path): assert all(isinstance(item, dict) for item in L) -def test_ossfs_big_ls(ossfs, test_bucket_name): - path = test_bucket_name + "/test_ossfs_big_ls" +def test_ossfs_big_ls(ossfs, test_path): + path = test_path + "/test_ossfs_big_ls" if not ossfs.exists(path): for x in range(1200): - ossfs.touch(path + "/%i.part" % x) + ossfs.touch(path + f"/{x}.part") files = ossfs.find(path, connect_timeout=600) for x in range(1200): - file = path + "/%i.part" % x + file = path + f"/{x}.part" if file not in files: - ossfs.touch(path + "/%i.part" % x) + ossfs.touch(path + f"/{x}.part") assert len(ossfs.find(path, connect_timeout=600)) == 1200 @@ -210,15 +209,13 @@ def test_ossfs_glob(ossfs, test_path): assert fn2 not in ossfs.glob(path + "/nested/file.*") -def test_copy(ossfs, test_bucket_name, test_path): - number_file = test_bucket_name + "/number" +def test_copy(ossfs, number_file, test_path): new_file = test_path + "/test_copy/file" ossfs.copy(number_file, new_file) assert ossfs.cat(number_file) == ossfs.cat(new_file) -def test_move(ossfs, test_bucket_name, test_path): - number_file = test_bucket_name + "/number" +def test_move(ossfs, number_file, test_path): from_file = test_path + "/test_move/from" to_file = test_path + "/test_move/to" ossfs.copy(number_file, from_file) @@ -378,7 +375,7 @@ def test_get_file_info_with_selector(ossfs, test_path): elif info["name"].rstrip("/").endswith(dir_a): assert info["type"] == "directory" else: - raise ValueError("unexpected path {}".format(info["name"])) + raise ValueError(f"unexpected path {info['name']}") def test_same_name_but_no_exact(ossfs, test_path): @@ -419,8 +416,8 @@ def test_leading_forward_slash(ossfs, test_path): assert ossfs.exists("/" + path + "/some/file") -def test_find_with_prefix(ossfs, test_bucket_name): - path = test_bucket_name + "/test_find_with_prefix" +def test_find_with_prefix(ossfs, test_path): + path = test_path + "/test_find_with_prefix" if not ossfs.exists(path): for cursor in range(100): ossfs.touch(path + f"/prefixes/test_{cursor}") @@ -448,10 +445,10 @@ def test_find_with_prefix(ossfs, test_bucket_name): READ_BLOCK_SIZE = 2 ** 14 # 16KB blocks -def test_get_put_file(ossfs, tmpdir, test_bucket_name): +def test_get_put_file(ossfs, tmpdir, test_path): src_file = str(tmpdir / "source") src2_file = str(tmpdir / "source_2") - dest_file = test_bucket_name + "/get_put_file/dest" + dest_file = test_path + "/get_put_file/dest" data = b"test" * 2 ** 20