Merge branch 'main' into fix/configurable-dynamic-catalog

meltano · Jan 19, 2024 · 8ba50d3 · 8ba50d3
2 parents f16bef0 + 596323b
commit 8ba50d3
Show file tree

Hide file tree

Showing 64 changed files with 982 additions and 323 deletions.
diff --git a/.github/workflows/constraints.txt b/.github/workflows/constraints.txt
@@ -1,6 +1,7 @@
 pip==23.3.2
 poetry==1.7.1
 poetry-plugin-export==1.6.0
+poetry-dynamic-versioning==1.2.0
 pre-commit==3.6.0
 nox==2023.4.22
 nox-poetry==1.0.3
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -2,51 +2,33 @@ name: Release
 
 on:
   push:
-    tags:
-      - v*
 
 permissions:
   contents: write  # Needed to upload artifacts to the release
   id-token: write  # Needed for OIDC PyPI publishing
 
 jobs:
-  release:
-    name: Publish to PyPI
+  build:
     runs-on: ubuntu-latest
-    environment: publishing
-
     steps:
-    - name: Checkout code
-      uses: actions/checkout@v4
-
-    - name: Set up Python
-      uses: actions/setup-python@v5
+    - uses: actions/checkout@v4
       with:
-        python-version: "3.11"
-
-    - name: Upgrade pip
-      env:
-        PIP_CONSTRAINT: .github/workflows/constraints.txt
-      run: |
-        pip install pip
-        pip --version
-
-    - name: Install Poetry
-      env:
-        PIP_CONSTRAINT: .github/workflows/constraints.txt
-      run: |
-        pipx install poetry
-        poetry --version
-
-    - name: Check version
-      run: |
-        version=$(poetry version | awk '{print $2}')
-        tag=$(echo "${{ github.ref }}" | awk '{split($0,p,"/"); print p[3]}')
-        if [ "v$version" != $tag ]; then echo "Release tag and package version do not match!"; exit 1; fi;
-
-    - name: Build
-      run: poetry build
+        fetch-depth: 0
+    - uses: hynek/build-and-inspect-python-package@v2
 
+  publish:
+    name: Publish to PyPI
+    runs-on: ubuntu-latest
+    needs: [build]
+    environment:
+      name: publishing
+      url: https://pypi.org/p/singer-sdk
+    if: startsWith(github.ref, 'refs/tags/')
+    steps:
+    - uses: actions/download-artifact@v4
+      with:
+        name: Packages
+        path: dist
     - name: Upload wheel to release
       uses: svenstaro/upload-release-action@v2
       with:

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -56,13 +56,16 @@ jobs:
 
     steps:
     - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
 
     - name: Install Poetry
       env:
         PIP_CONSTRAINT: .github/workflows/constraints.txt
       run: |
         pipx install poetry
         pipx inject poetry poetry-plugin-export
+        pipx inject poetry poetry-dynamic-versioning[plugin]
         poetry --version
         poetry self show plugins
 
@@ -117,13 +120,16 @@ jobs:
 
     steps:
     - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
 
     - name: Install Poetry
       env:
         PIP_CONSTRAINT: .github/workflows/constraints.txt
       run: |
         pipx install poetry
         pipx inject poetry poetry-plugin-export
+        pipx inject poetry poetry-dynamic-versioning[plugin]
         poetry --version
         poetry self show plugins
 

diff --git a/.gitignore b/.gitignore
@@ -1,3 +1,7 @@
+# Local Poetry configuration file
+
+poetry.toml
+
 # CI
 
 _changelog_fragment.md

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -45,7 +45,7 @@ repos:
   - id: check-readthedocs
 
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.1.11
+  rev: v0.1.13
   hooks:
   - id: ruff
     args: [--fix, --exit-non-zero-on-fix, --show-fixes]

diff --git a/cookiecutter/mapper-template/cookiecutter.json b/cookiecutter/mapper-template/cookiecutter.json
@@ -7,13 +7,15 @@
   "variant": "None (Skip)",
   "include_ci_files": ["GitHub", "None (Skip)"],
   "license": ["Apache-2.0"],
+  "ide": ["VSCode", "None"],
   "__prompts__": {
     "name": "The name of the mapper, in CamelCase",
     "admin_name": "Provide your [bold yellow]full name[/]",
     "admin_email": "Provide your [bold yellow]email[/]",
     "mapper_id": "The ID of the tap, in kebab-case",
     "library_name": "The name of the library, in snake_case. This is how the library will be imported in Python.",
     "include_ci_files": "Whether to include CI files for a common CI services",
-    "license": "The license for the project"
+    "license": "The license for the project",
+    "ide": "Add configuration files for your preferred IDE"
   }
 }
diff --git a/cookiecutter/mapper-template/hooks/post_gen_project.py b/cookiecutter/mapper-template/hooks/post_gen_project.py
@@ -0,0 +1,17 @@
+#!/usr/bin/env python
+from pathlib import Path
+import shutil
+
+
+BASE_PATH = Path("{{cookiecutter.library_name}}")
+
+
+if __name__ == "__main__":
+    if "{{ cookiecutter.license }}" != "Apache-2.0":
+        Path("LICENSE").unlink()
+
+    if "{{ cookiecutter.include_ci_files }}" != "GitHub":
+        shutil.rmtree(Path(".github"))
+
+    if "{{ cookiecutter.ide }}" != "VSCode":
+        shutil.rmtree(".vscode")
diff --git a/...les == 'GitHub' %}dependabot.yml{%endif%} → ...cutter.mapper_id}}/.github/dependabot.yml b/...les == 'GitHub' %}dependabot.yml{%endif%} → ...cutter.mapper_id}}/.github/dependabot.yml
diff --git a/..._ci_files == 'GitHub' %}test.yml{%endif%} → ...er.mapper_id}}/.github/workflows/test.yml b/..._ci_files == 'GitHub' %}test.yml{%endif%} → ...er.mapper_id}}/.github/workflows/test.yml
diff --git a/...= cookiecutter.license %}LICENSE{%endif%} → ...mplate/{{cookiecutter.mapper_id}}/LICENSE b/...= cookiecutter.license %}LICENSE{%endif%} → ...mplate/{{cookiecutter.mapper_id}}/LICENSE
diff --git a/...tter/mapper-template/{{cookiecutter.mapper_id}}/{{cookiecutter.library_name}}/__main__.py b/...tter/mapper-template/{{cookiecutter.mapper_id}}/{{cookiecutter.library_name}}/__main__.py
@@ -0,0 +1,7 @@
+"""{{ cookiecutter.name }} entry point."""
+
+from __future__ import annotations
+
+from {{ cookiecutter.library_name }}.mapper import {{ cookiecutter.name }}Mapper
+
+{{ cookiecutter.name }}Mapper.cli()
diff --git a/cookiecutter/tap-template/cookiecutter.json b/cookiecutter/tap-template/cookiecutter.json
@@ -16,6 +16,7 @@
   ],
   "include_ci_files": ["GitHub", "None"],
   "license": ["Apache-2.0", "None"],
+  "ide": ["VSCode", "None"],
   "__prompts__": {
     "source_name": "The name of the source, in CamelCase",
     "admin_name": "Provide your [bold yellow]full name[/]",
@@ -25,6 +26,7 @@
     "stream_type": "The type of stream the source provides",
     "auth_method": "The [bold red]authentication[/] method used by the source, for REST and GraphQL sources",
     "include_ci_files": "Whether to include CI files for a common CI services",
-    "license": "The license for the project"
+    "license": "The license for the project",
+    "ide": "Add configuration files for your preferred IDE"
   }
 }
diff --git a/cookiecutter/tap-template/hooks/post_gen_project.py b/cookiecutter/tap-template/hooks/post_gen_project.py
@@ -26,3 +26,6 @@
 
     if "{{ cookiecutter.include_ci_files }}" != "GitHub":
         shutil.rmtree(".github")
+
+    if "{{ cookiecutter.ide }}" != "VSCode":
+        shutil.rmtree(".vscode")
diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/__main__.py b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/__main__.py
@@ -0,0 +1,7 @@
+"""{{ cookiecutter.source_name }} entry point."""
+
+from __future__ import annotations
+
+from {{ cookiecutter.library_name }}.tap import Tap{{ cookiecutter.source_name }}
+
+Tap{{ cookiecutter.source_name }}.cli()
diff --git a/cookiecutter/target-template/cookiecutter.json b/cookiecutter/target-template/cookiecutter.json
@@ -8,6 +8,7 @@
   "serialization_method": ["Per record", "Per batch", "SQL"],
   "include_ci_files": ["GitHub", "None (Skip)"],
   "license": ["Apache-2.0"],
+  "ide": ["VSCode", "None"],
   "__prompts__": {
     "name": "The name of the mapper, in CamelCase",
     "admin_name": "Provide your [bold yellow]full name[/]",
@@ -16,6 +17,7 @@
     "library_name": "The name of the library, in snake_case. This is how the library will be imported in Python.",
     "serialization_method": "The serialization method to use for loading data",
     "include_ci_files": "Whether to include CI files for a common CI services",
-    "license": "The license for the project"
+    "license": "The license for the project",
+    "ide": "Add configuration files for your preferred IDE"
   }
 }
diff --git a/cookiecutter/target-template/hooks/post_gen_project.py b/cookiecutter/target-template/hooks/post_gen_project.py
@@ -12,3 +12,6 @@
 
     if "{{ cookiecutter.include_ci_files }}" != "GitHub":
         shutil.rmtree(Path(".github"))
+
+    if "{{ cookiecutter.ide }}" != "VSCode":
+        shutil.rmtree(".vscode")
diff --git a/...tter/target-template/{{cookiecutter.target_id}}/{{cookiecutter.library_name}}/__main__.py b/...tter/target-template/{{cookiecutter.target_id}}/{{cookiecutter.library_name}}/__main__.py
@@ -0,0 +1,7 @@
+"""{{ cookiecutter.destination_name }} entry point."""
+
+from __future__ import annotations
+
+from {{ cookiecutter.library_name }}.target import Target{{ cookiecutter.destination_name }}
+
+Target{{ cookiecutter.destination_name }}.cli()
diff --git a/docs/parent_streams.md b/docs/parent_streams.md
@@ -8,18 +8,21 @@ from a parent record each time the child stream is invoked.
 
 1. Set `parent_stream_type` in the child-stream's class to the class of the parent.
 2. Implement one of the below methods to pass context from the parent to the child:
-   1. If using `get_records(context)` you can simply return a tuple instead of a `record`
+   1. If using [`get_records`](singer_sdk.Stream.get_child_context) you can simply return a tuple instead of a `record`
       dictionary. A tuple return value will be interpreted by the SDK as
       `(record: dict, child_context: dict)`.
-   1. Override `get_child_context(record, context: Dict) -> dict` to return a new
+   2. Override [`get_child_context`](singer_sdk.Stream.get_child_context) to return a new
       child context object based on records and any existing context from the parent stream.
+   3. If you need to sync more than one child stream per parent record, you can override
+      [`generate_child_contexts`](singer_sdk.Stream.generate_child_contexts) to yield as many
+      contexts as you need.
 3. If the parent stream's replication key won't get updated when child items are changed,
    indicate this by adding `ignore_parent_replication_key = True` in the child stream
    class declaration.
 4. If the number of _parent_ items is very large (thousands or tens of thousands), you can
-   optionally set `state_partitioning_keys` on the child stream to specify a subset of context keys to use
+   optionally set [`state_partitioning_keys`](singer_sdk.Stream.state_partitioning_keys) on the child stream to specify a subset of context keys to use
    in state bookmarks. (When not set, the number of bookmarks will be equal to the number
-   of parent items.) If you do not wish to store any state bookmarks for the child stream, set `state_partitioning_keys` to `[]`.
+   of parent items.) If you do not wish to store any state bookmarks for the child stream, set[`state_partitioning_keys`](singer_sdk.Stream.state_partitioning_keys) to `[]`.
 
 ## Example parent-child implementation
 

diff --git a/e2e-tests/cookiecutters/mapper-base.json b/e2e-tests/cookiecutters/mapper-base.json
@@ -8,6 +8,7 @@
       "variant": "None (Skip)",
       "include_ci_files": "None (Skip)",
       "license": "Apache-2.0",
+      "ide": "VSCode",
       "_template": "../mapper-template/",
       "_output_dir": "."
     }

diff --git a/e2e-tests/cookiecutters/tap-graphql-jwt.json b/e2e-tests/cookiecutters/tap-graphql-jwt.json
@@ -10,6 +10,7 @@
       "auth_method": "JWT",
       "include_ci_files": "None (Skip)",
       "license": "Apache-2.0",
+      "ide": "VSCode",
       "_template": "../tap-template/",
       "_output_dir": "."
     }

diff --git a/e2e-tests/cookiecutters/tap-other-custom.json b/e2e-tests/cookiecutters/tap-other-custom.json
@@ -10,6 +10,7 @@
       "auth_method": "Custom or N/A",
       "include_ci_files": "None (Skip)",
       "license": "Apache-2.0",
+      "ide": "VSCode",
       "_template": "../tap-template/",
       "_output_dir": "."
     }

diff --git a/e2e-tests/cookiecutters/tap-rest-api_key-github.json b/e2e-tests/cookiecutters/tap-rest-api_key-github.json
@@ -10,6 +10,7 @@
       "auth_method": "API Key",
       "include_ci_files": "GitHub",
       "license": "Apache-2.0",
+      "ide": "VSCode",
       "_template": "../tap-template/",
       "_output_dir": "."
     }

diff --git a/e2e-tests/cookiecutters/tap-rest-basic_auth.json b/e2e-tests/cookiecutters/tap-rest-basic_auth.json
@@ -10,6 +10,7 @@
       "auth_method": "Basic Auth",
       "include_ci_files": "None (Skip)",
       "license": "Apache-2.0",
+      "ide": "VSCode",
       "_template": "../tap-template/",
       "_output_dir": "."
     }

diff --git a/e2e-tests/cookiecutters/tap-rest-bearer_token.json b/e2e-tests/cookiecutters/tap-rest-bearer_token.json
@@ -10,6 +10,7 @@
       "auth_method": "Bearer Token",
       "include_ci_files": "None (Skip)",
       "license": "Apache-2.0",
+      "ide": "VSCode",
       "_template": "../tap-template/",
       "_output_dir": "."
     }

diff --git a/e2e-tests/cookiecutters/tap-rest-custom.json b/e2e-tests/cookiecutters/tap-rest-custom.json
@@ -10,6 +10,7 @@
       "auth_method": "Custom or N/A",
       "include_ci_files": "None (Skip)",
       "license": "Apache-2.0",
+      "ide": "VSCode",
       "_template": "../tap-template/",
       "_output_dir": "."
     }

diff --git a/e2e-tests/cookiecutters/tap-rest-jwt.json b/e2e-tests/cookiecutters/tap-rest-jwt.json
@@ -10,6 +10,7 @@
       "auth_method": "JWT",
       "include_ci_files": "None (Skip)",
       "license": "Apache-2.0",
+      "ide": "VSCode",
       "_template": "../tap-template/",
       "_output_dir": "."
     }

diff --git a/e2e-tests/cookiecutters/tap-rest-oauth2.json b/e2e-tests/cookiecutters/tap-rest-oauth2.json
@@ -10,6 +10,7 @@
       "auth_method": "OAuth2",
       "include_ci_files": "None (Skip)",
       "license": "Apache-2.0",
+      "ide": "VSCode",
       "_template": "../tap-template/",
       "_output_dir": "."
     }

diff --git a/e2e-tests/cookiecutters/tap-sql-custom.json b/e2e-tests/cookiecutters/tap-sql-custom.json
@@ -10,6 +10,7 @@
       "auth_method": "Custom or N/A",
       "include_ci_files": "None (Skip)",
       "license": "Apache-2.0",
+      "ide": "VSCode",
       "_template": "../tap-template/",
       "_output_dir": "."
     }

diff --git a/e2e-tests/cookiecutters/target-per_record.json b/e2e-tests/cookiecutters/target-per_record.json
@@ -9,6 +9,7 @@
       "serialization_method": "Per record",
       "include_ci_files": "None (Skip)",
       "license": "Apache-2.0",
+      "ide": "VSCode",
       "_template": "./sdk/cookiecutter/target-template",
       "_output_dir": "."
     }

diff --git a/e2e-tests/cookiecutters/target-sql.json b/e2e-tests/cookiecutters/target-sql.json
@@ -9,6 +9,7 @@
       "serialization_method": "SQL",
       "include_ci_files": "None (Skip)",
       "license": "Apache-2.0",
+      "ide": "VSCode",
       "_template": "./sdk/cookiecutter/target-template",
       "_output_dir": "."
     }