Upgrades proposal

.github/dependabot.yml

@@ -14,3 +14,7 @@ updates:
     reviewers:
       - ome9ax
     open-pull-requests-limit: 9
+    groups:
+      python-package:
+        patterns:
+          - "*"

.github/workflows/python-package.yml

@@ -20,42 +20,49 @@ jobs:
       matrix:
         project: ['target-s3-jsonl']
         os: [ubuntu-latest] #, macos-latest, windows-latest
-        python-version: [3.9, '3.10', 3.11]
-        exclude:
-          - os: macos-latest
-            python-version: 3.9
-          - os: macos-latest
-            python-version: 3.10
-          - os: windows-latest
-            python-version: 3.9
-          - os: windows-latest
-            python-version: 3.10
+        python-version: [3.9, '3.10', 3.11, 3.12]
+        # exclude:
+        #   - os: macos-latest
+        #     python-version: 3.9
+        #   - os: macos-latest
+        #     python-version: 3.10
+        #   - os: macos-latest
+        #     python-version: 3.12
+        #   - os: windows-latest
+        #     python-version: 3.9
+        #   - os: windows-latest
+        #     python-version: 3.10
+        #   - os: windows-latest
+        #     python-version: 3.12
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v3
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
 
     - name: Cache pip Linux
-      uses: actions/cache@v3
+      uses: actions/cache@v4
+      id: cache-${{ runner.os }}-pip
       if: startsWith(runner.os, 'Linux')
       with:
         path: ~/.cache/pip
         key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.cfg') }}
         restore-keys: |
           ${{ runner.os }}-pip-
     - name: Cache pip macOS
-      uses: actions/cache@v3
+      uses: actions/cache@v4
+      id: cache-${{ runner.os }}-pip
       if: startsWith(runner.os, 'macOS')
       with:
         path: ~/Library/Caches/pip
         key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.cfg') }}
         restore-keys: |
           ${{ runner.os }}-pip-
     - name: Cache pip Windows
-      uses: actions/cache@v3
+      uses: actions/cache@v4
+      id: cache-${{ runner.os }}-pip
       if: startsWith(runner.os, 'Windows')
       with:
         path: ~\AppData\Local\pip\Cache
@@ -64,17 +71,18 @@ jobs:
           ${{ runner.os }}-pip-
 
     - name: Install dependencies
+      if: steps.cache-${{ runner.os }}-pip.outputs.cache-hit != 'true'
       run: |
         python -m venv venv || virtualenv venv
         . venv/bin/activate
         pip install --upgrade pip # setuptools
         # pip install .[test,lint,static,dist]
         pip install tox
 
-    - name: Get pip cache dir
-      id: pip-cache
-      run: |
-        echo "::set-output name=dir::$(pip cache dir)"
+    # - name: Get pip cache dir
+    #   id: pip-cache
+    #   run: |
+    #     echo "::set-output name=dir::$(pip cache dir)"
 
     # - name: Lint with flake8
     #   run: |
@@ -91,7 +99,7 @@ jobs:
         . venv/bin/activate
         TOX_PARALLEL_NO_SPINNER=1 tox --parallel -e lint,static
     - name: pip cache
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: ${{ steps.pip-cache.outputs.dir }}
         key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.cfg') }}

setup.cfg

@@ -17,6 +17,7 @@ classifiers =
     Programming Language :: Python :: 3.9
     Programming Language :: Python :: 3.10
     Programming Language :: Python :: 3.11
+    Programming Language :: Python :: 3.12
 project_urls =
     # Documentation = https://ome9ax.github.io/target-s3-jsonl
     Releases = https://github.com/ome9ax/target-s3-jsonl/releases

src/target_s3_json/s3.py

@@ -7,8 +7,7 @@
 import gzip
 import lzma
 from typing import Callable, Dict, Any, List, TextIO
-from asyncio import to_thread
-from concurrent.futures import ThreadPoolExecutor, Future
+from asyncio import to_thread, get_running_loop
 
 import backoff
 from boto3.session import Session
@@ -155,13 +154,12 @@ def upload_file(config: Dict[str, Any], file_metadata: Dict) -> None:
             file_metadata['absolute_path'].unlink()  # missing_ok=False
 
 
-async def upload_thread(config: Dict[str, Any], file_metadata: Dict) -> Future:
+async def upload_thread(config: Dict[str, Any], file_metadata: Dict) -> None:
 
-    return await to_thread(
-        *([config['executor'].submit] if config.get('thread_pool', True) else []),
-        upload_file,
-        config,
-        file_metadata)
+    if config.get('pool_executor', True):
+        return await get_running_loop().run_in_executor(None, upload_file, config, file_metadata)
+    else:
+        return await to_thread(upload_file, config, file_metadata)
 
 
 def config_s3(config_default: Dict[str, Any], datetime_format: Dict[str, str] = {
@@ -200,8 +198,7 @@ def main(lines: TextIO = sys.stdin) -> None:
     client: BaseClient = create_session(config).client('s3', **({'endpoint_url': config.get('aws_endpoint_url')}
                                                        if config.get('aws_endpoint_url') else {}))
 
-    with ThreadPoolExecutor() as executor:
-        Loader(config | {'client': client, 'executor': executor}, writeline=save_s3).run(lines)
+    Loader(config | {'client': client}, writeline=save_s3).run(lines)
 
 
 # from io import BytesIO

tests/conftest.py

@@ -25,9 +25,9 @@ def now(cls, tz: tzinfo = None):
             d: dt = dt.strptime('2022-04-29 07:39:38.321056+01:00', '%Y-%m-%d %H:%M:%S.%f%z')
             return d.astimezone(tz) if tz else d
 
-        @classmethod
-        def utcnow(cls):
-            return cls.now(timezone.utc).replace(tzinfo=None)
+        # @classmethod
+        # def utcnow(cls):
+        #     return cls.now(timezone.utc).replace(tzinfo=None)
 
     monkeypatch.setattr(datetime, 'datetime', mydatetime)
 

tests/test_s3.py

@@ -142,7 +142,7 @@ def config(patch_datetime, config_raw):
 
     return config_raw | {
         # 'date_time': dt.strptime('2021-08-11 06:39:38.321056+00:00', '%Y-%m-%d %H:%M:%S.%f%z'),
-        'date_time': datetime.datetime.utcnow(),
+        'date_time': datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None),
         'work_path': Path(config_raw['work_dir']),
         'open_func': open
     }
@@ -160,7 +160,7 @@ def config_assume_role(config):
 # @fixture
 # @mock_s3
 # def bucket(config):
-#     conn = boto3.resource('s3', region_name='us-east-1', endpoint_url='https://s3.amazonaws.com')
+#     conn = boto3.resource('s3', region_name='eu-west-1', endpoint_url='https://s3.amazonaws.com')
 #     # We need to create the bucket since this is all in Moto's 'virtual' AWS account
 #     conn.create_bucket(Bucket=config['s3_bucket'])
 #     return conn
@@ -220,11 +220,11 @@ def test_create_client_with_assumed_role(caplog, config_assume_role: dict):
 def test_create_session(aws_credentials, config):
     '''TEST : simple create_session call'''
 
-    conn = boto3.resource('s3', region_name='us-east-1', endpoint_url='https://s3.amazonaws.com')
+    conn = boto3.resource('s3', region_name='eu-west-1', endpoint_url='https://s3.amazonaws.com')
     # We need to create the bucket since this is all in Moto's 'virtual' AWS account
     conn.create_bucket(Bucket=config['s3_bucket'])
 
-    # async with get_session().create_client('s3', region_name='us-east-1', end_point_url=s3_server) as client:
+    # async with get_session().create_client('s3', region_name='eu-west-1', end_point_url=s3_server) as client:
     #     with patch('aiobotocore.AioSession.create_client') as mock:
     #         mock.return_value = client
 
@@ -303,7 +303,7 @@ def test_put_object(config):
     '''TEST : simple put_object call'''
     config |= {'compression': 'gzip', 'open_func': gzip.compress}
 
-    conn = boto3.resource('s3', region_name='us-east-1')
+    conn = boto3.resource('s3', region_name='eu-west-1')
     conn.create_bucket(Bucket=config['s3_bucket'])
 
     client: BaseClient = create_session(config).client('s3')
@@ -367,7 +367,7 @@ def test_put_object(config):
 def test_upload_file(config, temp_path):
     '''TEST : simple upload_file call'''
 
-    conn = boto3.resource('s3', region_name='us-east-1')
+    conn = boto3.resource('s3', region_name='eu-west-1')
     conn.create_bucket(Bucket=config['s3_bucket'])
 
     client: BaseClient = create_session(config).client('s3')
@@ -458,7 +458,7 @@ def test_config_s3(config_raw):
 def test_main(capsys, patch_datetime, patch_sys_stdin, patch_argument_parser, config_raw, state, file_metadata):
     '''TEST : simple main call'''
 
-    conn = boto3.resource('s3', region_name='us-east-1', endpoint_url='https://s3.amazonaws.com')
+    conn = boto3.resource('s3', region_name='eu-west-1', endpoint_url='https://s3.amazonaws.com')
     # We need to create the bucket since this is all in Moto's 'virtual' AWS account
     conn.create_bucket(Bucket=config_raw['s3_bucket'])