From 65d9dbacb6a000c7ff2058b7b578f57fec56b725 Mon Sep 17 00:00:00 2001 From: Pat Nadolny Date: Thu, 3 Aug 2023 11:19:07 -0400 Subject: [PATCH] feat: Add a mapper cookiecutter template (#1892) --- .github/workflows/cookiecutter-e2e.yml | 3 + cookiecutter/mapper-template/README.md | 24 +++ .../mapper-template/cookiecutter.json | 9 + ...e_ci_files == 'GitHub' %}test.yml{%endif%} | 30 +++ ...iles == 'GitHub' %}dependabot.yml{%endif%} | 26 +++ .../{{cookiecutter.mapper_id}}/.gitignore | 136 ++++++++++++ .../.pre-commit-config.yaml | 36 ++++ .../.secrets/.gitignore | 10 + .../{{cookiecutter.mapper_id}}/README.md | 128 +++++++++++ .../{{cookiecutter.mapper_id}}/meltano.yml | 31 +++ .../output/.gitignore | 4 + .../{{cookiecutter.mapper_id}}/pyproject.toml | 64 ++++++ .../tests/__init__.py | 1 + .../tests/conftest.py | 3 + .../{{cookiecutter.mapper_id}}/tox.ini | 19 ++ ...== cookiecutter.license %}LICENSE{%endif%} | 202 ++++++++++++++++++ .../{{cookiecutter.library_name}}/__init__.py | 1 + .../{{cookiecutter.library_name}}/mapper.py | 96 +++++++++ e2e-tests/cookiecutters/mapper-base.json | 13 ++ noxfile.py | 19 +- 20 files changed, 846 insertions(+), 9 deletions(-) create mode 100644 cookiecutter/mapper-template/README.md create mode 100644 cookiecutter/mapper-template/cookiecutter.json create mode 100644 cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.github/workflows/{% if cookiecutter.include_ci_files == 'GitHub' %}test.yml{%endif%} create mode 100644 cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.github/{% if cookiecutter.include_ci_files == 'GitHub' %}dependabot.yml{%endif%} create mode 100644 cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.gitignore create mode 100644 cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.pre-commit-config.yaml create mode 100644 cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.secrets/.gitignore create mode 100644 cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/README.md create mode 100644 cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/meltano.yml create mode 100644 cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/output/.gitignore create mode 100644 cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml create mode 100644 cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/tests/__init__.py create mode 100644 cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/tests/conftest.py create mode 100644 cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/tox.ini create mode 100644 cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/{%if 'Apache-2.0' == cookiecutter.license %}LICENSE{%endif%} create mode 100644 cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/{{cookiecutter.library_name}}/__init__.py create mode 100644 cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/{{cookiecutter.library_name}}/mapper.py create mode 100644 e2e-tests/cookiecutters/mapper-base.json diff --git a/.github/workflows/cookiecutter-e2e.yml b/.github/workflows/cookiecutter-e2e.yml index a88fd6bdd..a33989df4 100644 --- a/.github/workflows/cookiecutter-e2e.yml +++ b/.github/workflows/cookiecutter-e2e.yml @@ -74,7 +74,10 @@ jobs: path: | /tmp/tap-* /tmp/target-* + /tmp/mapper-* !/tmp/tap-*/.mypy_cache/ !/tmp/target-*/.mypy_cache/ + !/tmp/mapper-*/.mypy_cache/ !/tmp/tap-*/.tox/ !/tmp/target-*/.tox/ + !/tmp/mapper-*/.tox/ diff --git a/cookiecutter/mapper-template/README.md b/cookiecutter/mapper-template/README.md new file mode 100644 index 000000000..70e2e47e8 --- /dev/null +++ b/cookiecutter/mapper-template/README.md @@ -0,0 +1,24 @@ +# Singer Mapper Template + +To use this cookie cutter template: + +```bash +pip3 install pipx +pipx ensurepath +# You may need to reopen your shell at this point +pipx install cookiecutter +``` + +Initialize Cookiecutter template directly from Git: + +```bash +cookiecutter https://github.com/meltano/sdk --directory="cookiecutter/mapper-template" +``` + +Or locally from an already-cloned `sdk` repo: + +```bash +cookiecutter ./sdk/cookiecutter/mapper-template +``` + +See the [dev guide](https://sdk.meltano.com/en/latest/dev_guide.html). diff --git a/cookiecutter/mapper-template/cookiecutter.json b/cookiecutter/mapper-template/cookiecutter.json new file mode 100644 index 000000000..267e45fb9 --- /dev/null +++ b/cookiecutter/mapper-template/cookiecutter.json @@ -0,0 +1,9 @@ +{ + "name": "MyMapperName", + "admin_name": "FirstName LastName", + "mapper_id": "mapper-{{ cookiecutter.name.lower() }}", + "library_name": "{{ cookiecutter.mapper_id.replace('-', '_') }}", + "variant": "None (Skip)", + "include_ci_files": ["GitHub", "None (Skip)"], + "license": ["Apache-2.0"] +} diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.github/workflows/{% if cookiecutter.include_ci_files == 'GitHub' %}test.yml{%endif%} b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.github/workflows/{% if cookiecutter.include_ci_files == 'GitHub' %}test.yml{%endif%} new file mode 100644 index 000000000..0cfc81005 --- /dev/null +++ b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.github/workflows/{% if cookiecutter.include_ci_files == 'GitHub' %}test.yml{%endif%} @@ -0,0 +1,30 @@ +### A CI workflow template that runs linting and python testing +### TODO: Modify as needed or as desired. + +name: Test {{cookiecutter.mapper_id}} + +on: [push] + +jobs: + pytest: + runs-on: ubuntu-latest + env: + GITHUB_TOKEN: {{ '${{secrets.GITHUB_TOKEN}}' }} + strategy: + matrix: + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] + steps: + - uses: actions/checkout@v3 + - name: Set up Python {{ '${{ matrix.python-version }}' }} + uses: actions/setup-python@v4 + with: + python-version: {{ '${{ matrix.python-version }}' }} + - name: Install Poetry + run: | + pip install poetry + - name: Install dependencies + run: | + poetry install + - name: Test with pytest + run: | + poetry run pytest diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.github/{% if cookiecutter.include_ci_files == 'GitHub' %}dependabot.yml{%endif%} b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.github/{% if cookiecutter.include_ci_files == 'GitHub' %}dependabot.yml{%endif%} new file mode 100644 index 000000000..933e6b1c2 --- /dev/null +++ b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.github/{% if cookiecutter.include_ci_files == 'GitHub' %}dependabot.yml{%endif%} @@ -0,0 +1,26 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates + +version: 2 +updates: + - package-ecosystem: pip + directory: "/" + schedule: + interval: "daily" + commit-message: + prefix: "chore(deps): " + prefix-development: "chore(deps-dev): " + - package-ecosystem: pip + directory: "/.github/workflows" + schedule: + interval: daily + commit-message: + prefix: "ci: " + - package-ecosystem: github-actions + directory: "/" + schedule: + interval: "weekly" + commit-message: + prefix: "ci: " diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.gitignore b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.gitignore new file mode 100644 index 000000000..475019c31 --- /dev/null +++ b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.gitignore @@ -0,0 +1,136 @@ +# Secrets and internal config files +**/.secrets/* + +# Ignore meltano internal cache and sqlite systemdb + +.meltano/ + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.pre-commit-config.yaml b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.pre-commit-config.yaml new file mode 100644 index 000000000..6d9bbbfd5 --- /dev/null +++ b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.pre-commit-config.yaml @@ -0,0 +1,36 @@ +ci: + autofix_prs: true + autoupdate_schedule: weekly + autoupdate_commit_msg: 'chore: pre-commit autoupdate' + +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: check-json + - id: check-toml + - id: check-yaml + - id: end-of-file-fixer + - id: trailing-whitespace + +- repo: https://github.com/python-jsonschema/check-jsonschema + rev: 0.23.3 + hooks: + - id: check-dependabot + - id: check-github-workflows + +- repo: https://github.com/charliermarsh/ruff-pre-commit + rev: v0.0.282 + hooks: + - id: ruff + args: [--fix, --exit-non-zero-on-fix, --show-fixes] + +- repo: https://github.com/psf/black + rev: 23.7.0 + hooks: + - id: black + +- repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.4.1 + hooks: + - id: mypy diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.secrets/.gitignore b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.secrets/.gitignore new file mode 100644 index 000000000..33c6acd03 --- /dev/null +++ b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.secrets/.gitignore @@ -0,0 +1,10 @@ +# IMPORTANT! This folder is hidden from git - if you need to store config files or other secrets, +# make sure those are never staged for commit into your git repo. You can store them here or another +# secure location. +# +# Note: This may be redundant with the global .gitignore for, and is provided +# for redundancy. If the `.secrets` folder is not needed, you may delete it +# from the project. + +* +!.gitignore diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/README.md b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/README.md new file mode 100644 index 000000000..ded365fb2 --- /dev/null +++ b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/README.md @@ -0,0 +1,128 @@ +# {{ cookiecutter.mapper_id }} + +`{{ cookiecutter.mapper_id }}` is a Singer mapper for {{ cookiecutter.name }}. + +Built with the [Meltano Mapper SDK](https://sdk.meltano.com) for Singer Mappers. + + + +## Configuration + +### Accepted Config Options + + + +A full list of supported settings and capabilities for this +mapper is available by running: + +```bash +{{ cookiecutter.mapper_id }} --about +``` + +### Configure using environment variables + +This Singer mapper will automatically import any environment variables within the working directory's +`.env` if the `--config=ENV` is provided, such that config values will be considered if a matching +environment variable is set either in the terminal context or in the `.env` file. + +### Source Authentication and Authorization + + + +## Usage + +You can easily run `{{ cookiecutter.mapper_id }}` by itself or in a pipeline using [Meltano](https://meltano.com/). + +### Executing the Mapper Directly + +```bash +{{ cookiecutter.mapper_id }} --version +{{ cookiecutter.mapper_id }} --help +``` + +## Developer Resources + +Follow these instructions to contribute to this project. + +### Initialize your Development Environment + +```bash +pipx install poetry +poetry install +``` + +### Create and Run Tests + +Create tests within the `tests` subfolder and + then run: + +```bash +poetry run pytest +``` + +You can also test the `{{cookiecutter.mapper_id}}` CLI interface directly using `poetry run`: + +```bash +poetry run {{cookiecutter.mapper_id}} --help +``` + +### Testing with [Meltano](https://www.meltano.com) + +_**Note:** This mapper will work in any Singer environment and does not require Meltano. +Examples here are for convenience and to streamline end-to-end orchestration scenarios._ + + + +Next, install Meltano (if you haven't already) and any needed plugins: + +```bash +# Install meltano +pipx install meltano +# Initialize meltano within this directory +cd {{ cookiecutter.mapper_id }} +meltano install +``` + +Now you can test and orchestrate using Meltano: + +```bash +# Run a test `run` pipeline: +meltano run tap-smoke-test {{ cookiecutter.mapper_id }} target-jsonl +``` + +### SDK Dev Guide + +See the [dev guide](https://sdk.meltano.com/en/latest/dev_guide.html) for more instructions on how to use the SDK to +develop your own taps, targets, and mappers. diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/meltano.yml b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/meltano.yml new file mode 100644 index 000000000..019015d06 --- /dev/null +++ b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/meltano.yml @@ -0,0 +1,31 @@ +version: 1 +send_anonymous_usage_stats: true +project_id: "{{cookiecutter.mapper_id}}" +default_environment: test +environments: +- name: test +plugins: + extractors: + - name: tap-smoke-test + variant: meltano + pip_url: git+https://github.com/meltano/tap-smoke-test.git + config: + streams: + - stream_name: animals + input_filename: https://raw.githubusercontent.com/meltano/tap-smoke-test/main/demo-data/animals-data.jsonl + loaders: + - name: target-jsonl + variant: andyh1203 + pip_url: target-jsonl + mappers: + - name: "{{cookiecutter.mapper_id}}" + pip_url: -e . + namespace: "{{cookiecutter.library_name}}" + # TODO: replace these with the actual settings + settings: + - name: example_config + kind: string + mappings: + - name: example + config: + example_config: foo diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/output/.gitignore b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/output/.gitignore new file mode 100644 index 000000000..80ff9d2a6 --- /dev/null +++ b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/output/.gitignore @@ -0,0 +1,4 @@ +# This directory is used as a target by target-jsonl, so ignore all files + +* +!.gitignore diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml new file mode 100644 index 000000000..9947e314e --- /dev/null +++ b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml @@ -0,0 +1,64 @@ +[tool.poetry] +{%- if cookiecutter.variant != "None (Skip)" %} +name = "{{cookiecutter.variant}}-{{cookiecutter.mapper_id}}" +{%- else %} +name = "{{cookiecutter.mapper_id}}" +{%- endif %} +version = "0.0.1" +description = "`{{cookiecutter.mapper_id}}` is a Singer mapper {{cookiecutter.name}}, built with the Meltano Singer SDK." +readme = "README.md" +authors = ["{{ cookiecutter.admin_name }}"] +keywords = [ + "ELT", + "Mapper", + "{{cookiecutter.name}}", +] +license = "Apache-2.0" +{%- if cookiecutter.variant != "None (Skip)" %} +packages = [ + { include = "{{cookiecutter.library_name}}" }, +] +{%- endif %} + +[tool.poetry.dependencies] +python = "<3.12,>=3.7.1" +singer-sdk = { version="^0.30.0" } +fs-s3fs = { version = "^1.1.1", optional = true } + +[tool.poetry.group.dev.dependencies] +pytest = "^7.2.1" +singer-sdk = { version="^0.30.0", extras = ["testing"] } + +[tool.poetry.extras] +s3 = ["fs-s3fs"] + +[tool.mypy] +python_version = "3.9" +warn_unused_configs = true + +[tool.ruff] +ignore = [ + "ANN101", # missing-type-self + "ANN102", # missing-type-cls +] +select = ["ALL"] +src = ["{{cookiecutter.library_name}}"] +target-version = "py37" + + +[tool.ruff.flake8-annotations] +allow-star-arg-any = true + +[tool.ruff.isort] +known-first-party = ["{{cookiecutter.library_name}}"] + +[tool.ruff.pydocstyle] +convention = "google" + +[build-system] +requires = ["poetry-core>=1.0.8"] +build-backend = "poetry.core.masonry.api" + +[tool.poetry.scripts] +# CLI declaration +{{cookiecutter.mapper_id}} = '{{cookiecutter.library_name}}.mapper:{{cookiecutter.name}}Mapper.cli' diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/tests/__init__.py b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/tests/__init__.py new file mode 100644 index 000000000..7caba56f7 --- /dev/null +++ b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/tests/__init__.py @@ -0,0 +1 @@ +"""Test suite for {{ cookiecutter.mapper_id }}.""" diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/tests/conftest.py b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/tests/conftest.py new file mode 100644 index 000000000..6bb3ec2d7 --- /dev/null +++ b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/tests/conftest.py @@ -0,0 +1,3 @@ +"""Test Configuration.""" + +pytest_plugins = ("singer_sdk.testing.pytest_plugin",) diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/tox.ini b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/tox.ini new file mode 100644 index 000000000..70b9e4ac7 --- /dev/null +++ b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/tox.ini @@ -0,0 +1,19 @@ +# This file can be used to customize tox tests as well as other test frameworks like flake8 and mypy + +[tox] +envlist = py37, py38, py39, py310, py311 +isolated_build = true + +[testenv] +allowlist_externals = poetry +commands = + poetry install -v + poetry run pytest + +[testenv:pytest] +# Run the python tests. +# To execute, run `tox -e pytest` +envlist = py37, py38, py39, py310, py311 +commands = + poetry install -v + poetry run pytest diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/{%if 'Apache-2.0' == cookiecutter.license %}LICENSE{%endif%} b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/{%if 'Apache-2.0' == cookiecutter.license %}LICENSE{%endif%} new file mode 100644 index 000000000..62913ff3a --- /dev/null +++ b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/{%if 'Apache-2.0' == cookiecutter.license %}LICENSE{%endif%} @@ -0,0 +1,202 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + + Copyright {% now 'utc', '%Y' %} {{ cookiecutter.admin_name }} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/{{cookiecutter.library_name}}/__init__.py b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/{{cookiecutter.library_name}}/__init__.py new file mode 100644 index 000000000..5781fbbc4 --- /dev/null +++ b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/{{cookiecutter.library_name}}/__init__.py @@ -0,0 +1 @@ +"""{{ cookiecutter.name }} Mapper.""" diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/{{cookiecutter.library_name}}/mapper.py b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/{{cookiecutter.library_name}}/mapper.py new file mode 100644 index 000000000..c8c3d23ec --- /dev/null +++ b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/{{cookiecutter.library_name}}/mapper.py @@ -0,0 +1,96 @@ +"""{{ cookiecutter.name }} mapper class.""" + +from __future__ import annotations + +import typing as t +from typing import TYPE_CHECKING + +import singer_sdk.typing as th +from singer_sdk import _singerlib as singer +from singer_sdk.mapper import PluginMapper +from singer_sdk.mapper_base import InlineMapper + +if TYPE_CHECKING: + from pathlib import PurePath + + +class {{ cookiecutter.name }}Mapper(InlineMapper): + """Sample mapper for {{ cookiecutter.name }}.""" + + name = "{{ cookiecutter.mapper_id }}" + + config_jsonschema = th.PropertiesList( + # TODO: Replace or remove this example config based on your needs + th.Property( + "example_config", + th.StringType, + description="An example config, replace or remove based on your needs.", + ), + ).to_dict() + + def __init__( + self, + *, + config: dict | PurePath | str | list[PurePath | str] | None = None, + parse_env_config: bool = False, + validate_config: bool = True, + ) -> None: + """Create a new inline mapper. + + Args: + config: Mapper configuration. Can be a dictionary, a single path to a + configuration file, or a list of paths to multiple configuration + files. + parse_env_config: Whether to look for configuration values in environment + variables. + validate_config: True to require validation of config settings. + """ + super().__init__( + config=config, + parse_env_config=parse_env_config, + validate_config=validate_config, + ) + + self.mapper = PluginMapper(plugin_config=dict(self.config), logger=self.logger) + + def map_schema_message(self, message_dict: dict) -> t.Iterable[singer.Message]: + """Map a schema message to zero or more new messages. + + Args: + message_dict: A SCHEMA message JSON dictionary. + """ + yield singer.SchemaMessage.from_dict(message_dict) + + def map_record_message( + self, + message_dict: dict, + ) -> t.Iterable[singer.RecordMessage]: + """Map a record message to zero or more new messages. + + Args: + message_dict: A RECORD message JSON dictionary. + """ + yield singer.RecordMessage.from_dict(message_dict) + + def map_state_message(self, message_dict: dict) -> t.Iterable[singer.Message]: + """Map a state message to zero or more new messages. + + Args: + message_dict: A STATE message JSON dictionary. + """ + yield singer.StateMessage.from_dict(message_dict) + + def map_activate_version_message( + self, + message_dict: dict, + ) -> t.Iterable[singer.Message]: + """Map a version message to zero or more new messages. + + Args: + message_dict: An ACTIVATE_VERSION message JSON dictionary. + """ + yield singer.ActivateVersionMessage.from_dict(message_dict) + + +if __name__ == "__main__": + {{ cookiecutter.name }}Mapper.cli() diff --git a/e2e-tests/cookiecutters/mapper-base.json b/e2e-tests/cookiecutters/mapper-base.json new file mode 100644 index 000000000..25d40c608 --- /dev/null +++ b/e2e-tests/cookiecutters/mapper-base.json @@ -0,0 +1,13 @@ +{ + "cookiecutter": { + "name": "MyMapperName", + "admin_name": "Automatic Tester", + "mapper_id": "mapper-base", + "library_name": "mapper_base", + "variant": "None (Skip)", + "include_ci_files": "None (Skip)", + "license": "Apache-2.0", + "_template": "../mapper-template/", + "_output_dir": "." + } +} diff --git a/noxfile.py b/noxfile.py index 2838a4503..cbb331faf 100644 --- a/noxfile.py +++ b/noxfile.py @@ -196,21 +196,22 @@ def test_cookiecutter(session: Session, replay_file_path) -> None: cc_build_path = tempfile.gettempdir() folder_base_path = "./cookiecutter" - target_folder = ( - "tap-template" - if Path(replay_file_path).name.startswith("tap") - else "target-template" - ) - tap_template = Path(folder_base_path + "/" + target_folder).resolve() + if Path(replay_file_path).name.startswith("tap"): + folder = "tap-template" + elif Path(replay_file_path).name.startswith("target"): + folder = "target-template" + else: + folder = "mapper-template" + template = Path(folder_base_path + "/" + folder).resolve() replay_file = Path(replay_file_path).resolve() - if not Path(tap_template).exists(): + if not Path(template).exists(): return if not Path(replay_file).is_file(): return - sdk_dir = Path(Path(tap_template).parent).parent + sdk_dir = Path(Path(template).parent).parent cc_output_dir = Path(replay_file_path).name.replace(".json", "") cc_test_output = cc_build_path + "/" + cc_output_dir @@ -224,7 +225,7 @@ def test_cookiecutter(session: Session, replay_file_path) -> None: "cookiecutter", "--replay-file", str(replay_file), - str(tap_template), + str(template), "-o", cc_build_path, )