From 627c675117aa9785e1fda0ddc3c1f899940c3881 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eddy=20=E2=88=86?= Date: Wed, 25 Sep 2024 12:34:10 +0100 Subject: [PATCH] Upgrades proposal --- .github/dependabot.yml | 4 +++ .github/workflows/python-package.yml | 48 ++++++++++++++++------------ setup.cfg | 1 + src/target_s3_json/s3.py | 17 ++++------ tests/conftest.py | 6 ++-- tests/test_s3.py | 14 ++++---- 6 files changed, 50 insertions(+), 40 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 5d739e9..b199f22 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -14,3 +14,7 @@ updates: reviewers: - ome9ax open-pull-requests-limit: 9 + groups: + python-package: + patterns: + - "*" diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 3b69c1a..41ed5ec 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -20,26 +20,31 @@ jobs: matrix: project: ['target-s3-jsonl'] os: [ubuntu-latest] #, macos-latest, windows-latest - python-version: [3.9, '3.10', 3.11] - exclude: - - os: macos-latest - python-version: 3.9 - - os: macos-latest - python-version: 3.10 - - os: windows-latest - python-version: 3.9 - - os: windows-latest - python-version: 3.10 + python-version: [3.9, '3.10', 3.11, 3.12] + # exclude: + # - os: macos-latest + # python-version: 3.9 + # - os: macos-latest + # python-version: 3.10 + # - os: macos-latest + # python-version: 3.12 + # - os: windows-latest + # python-version: 3.9 + # - os: windows-latest + # python-version: 3.10 + # - os: windows-latest + # python-version: 3.12 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - name: Cache pip Linux - uses: actions/cache@v3 + uses: actions/cache@v4 + id: cache-${{ runner.os }}-pip if: startsWith(runner.os, 'Linux') with: path: ~/.cache/pip @@ -47,7 +52,8 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - name: Cache pip macOS - uses: actions/cache@v3 + uses: actions/cache@v4 + id: cache-${{ runner.os }}-pip if: startsWith(runner.os, 'macOS') with: path: ~/Library/Caches/pip @@ -55,7 +61,8 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - name: Cache pip Windows - uses: actions/cache@v3 + uses: actions/cache@v4 + id: cache-${{ runner.os }}-pip if: startsWith(runner.os, 'Windows') with: path: ~\AppData\Local\pip\Cache @@ -64,6 +71,7 @@ jobs: ${{ runner.os }}-pip- - name: Install dependencies + if: steps.cache-${{ runner.os }}-pip.outputs.cache-hit != 'true' run: | python -m venv venv || virtualenv venv . venv/bin/activate @@ -71,10 +79,10 @@ jobs: # pip install .[test,lint,static,dist] pip install tox - - name: Get pip cache dir - id: pip-cache - run: | - echo "::set-output name=dir::$(pip cache dir)" + # - name: Get pip cache dir + # id: pip-cache + # run: | + # echo "::set-output name=dir::$(pip cache dir)" # - name: Lint with flake8 # run: | @@ -91,7 +99,7 @@ jobs: . venv/bin/activate TOX_PARALLEL_NO_SPINNER=1 tox --parallel -e lint,static - name: pip cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ${{ steps.pip-cache.outputs.dir }} key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.cfg') }} diff --git a/setup.cfg b/setup.cfg index 1d65a9b..37c116e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -17,6 +17,7 @@ classifiers = Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 Programming Language :: Python :: 3.11 + Programming Language :: Python :: 3.12 project_urls = # Documentation = https://ome9ax.github.io/target-s3-jsonl Releases = https://github.com/ome9ax/target-s3-jsonl/releases diff --git a/src/target_s3_json/s3.py b/src/target_s3_json/s3.py index 423d33f..9f3ba51 100644 --- a/src/target_s3_json/s3.py +++ b/src/target_s3_json/s3.py @@ -7,8 +7,7 @@ import gzip import lzma from typing import Callable, Dict, Any, List, TextIO -from asyncio import to_thread -from concurrent.futures import ThreadPoolExecutor, Future +from asyncio import to_thread, get_running_loop import backoff from boto3.session import Session @@ -155,13 +154,12 @@ def upload_file(config: Dict[str, Any], file_metadata: Dict) -> None: file_metadata['absolute_path'].unlink() # missing_ok=False -async def upload_thread(config: Dict[str, Any], file_metadata: Dict) -> Future: +async def upload_thread(config: Dict[str, Any], file_metadata: Dict) -> None: - return await to_thread( - *([config['executor'].submit] if config.get('thread_pool', True) else []), - upload_file, - config, - file_metadata) + if config.get('pool_executor', True): + return await get_running_loop().run_in_executor(None, upload_file, config, file_metadata) + else: + return await to_thread(upload_file, config, file_metadata) def config_s3(config_default: Dict[str, Any], datetime_format: Dict[str, str] = { @@ -200,8 +198,7 @@ def main(lines: TextIO = sys.stdin) -> None: client: BaseClient = create_session(config).client('s3', **({'endpoint_url': config.get('aws_endpoint_url')} if config.get('aws_endpoint_url') else {})) - with ThreadPoolExecutor() as executor: - Loader(config | {'client': client, 'executor': executor}, writeline=save_s3).run(lines) + Loader(config | {'client': client}, writeline=save_s3).run(lines) # from io import BytesIO diff --git a/tests/conftest.py b/tests/conftest.py index 0acf26b..79ee8bd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -25,9 +25,9 @@ def now(cls, tz: tzinfo = None): d: dt = dt.strptime('2022-04-29 07:39:38.321056+01:00', '%Y-%m-%d %H:%M:%S.%f%z') return d.astimezone(tz) if tz else d - @classmethod - def utcnow(cls): - return cls.now(timezone.utc).replace(tzinfo=None) + # @classmethod + # def utcnow(cls): + # return cls.now(timezone.utc).replace(tzinfo=None) monkeypatch.setattr(datetime, 'datetime', mydatetime) diff --git a/tests/test_s3.py b/tests/test_s3.py index a6e164f..8dff740 100644 --- a/tests/test_s3.py +++ b/tests/test_s3.py @@ -142,7 +142,7 @@ def config(patch_datetime, config_raw): return config_raw | { # 'date_time': dt.strptime('2021-08-11 06:39:38.321056+00:00', '%Y-%m-%d %H:%M:%S.%f%z'), - 'date_time': datetime.datetime.utcnow(), + 'date_time': datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None), 'work_path': Path(config_raw['work_dir']), 'open_func': open } @@ -160,7 +160,7 @@ def config_assume_role(config): # @fixture # @mock_s3 # def bucket(config): -# conn = boto3.resource('s3', region_name='us-east-1', endpoint_url='https://s3.amazonaws.com') +# conn = boto3.resource('s3', region_name='eu-west-1', endpoint_url='https://s3.amazonaws.com') # # We need to create the bucket since this is all in Moto's 'virtual' AWS account # conn.create_bucket(Bucket=config['s3_bucket']) # return conn @@ -220,11 +220,11 @@ def test_create_client_with_assumed_role(caplog, config_assume_role: dict): def test_create_session(aws_credentials, config): '''TEST : simple create_session call''' - conn = boto3.resource('s3', region_name='us-east-1', endpoint_url='https://s3.amazonaws.com') + conn = boto3.resource('s3', region_name='eu-west-1', endpoint_url='https://s3.amazonaws.com') # We need to create the bucket since this is all in Moto's 'virtual' AWS account conn.create_bucket(Bucket=config['s3_bucket']) - # async with get_session().create_client('s3', region_name='us-east-1', end_point_url=s3_server) as client: + # async with get_session().create_client('s3', region_name='eu-west-1', end_point_url=s3_server) as client: # with patch('aiobotocore.AioSession.create_client') as mock: # mock.return_value = client @@ -303,7 +303,7 @@ def test_put_object(config): '''TEST : simple put_object call''' config |= {'compression': 'gzip', 'open_func': gzip.compress} - conn = boto3.resource('s3', region_name='us-east-1') + conn = boto3.resource('s3', region_name='eu-west-1') conn.create_bucket(Bucket=config['s3_bucket']) client: BaseClient = create_session(config).client('s3') @@ -367,7 +367,7 @@ def test_put_object(config): def test_upload_file(config, temp_path): '''TEST : simple upload_file call''' - conn = boto3.resource('s3', region_name='us-east-1') + conn = boto3.resource('s3', region_name='eu-west-1') conn.create_bucket(Bucket=config['s3_bucket']) client: BaseClient = create_session(config).client('s3') @@ -458,7 +458,7 @@ def test_config_s3(config_raw): def test_main(capsys, patch_datetime, patch_sys_stdin, patch_argument_parser, config_raw, state, file_metadata): '''TEST : simple main call''' - conn = boto3.resource('s3', region_name='us-east-1', endpoint_url='https://s3.amazonaws.com') + conn = boto3.resource('s3', region_name='eu-west-1', endpoint_url='https://s3.amazonaws.com') # We need to create the bucket since this is all in Moto's 'virtual' AWS account conn.create_bucket(Bucket=config_raw['s3_bucket'])