From af2e4889c5b2e8fd925507dab5473c52fd1e576e Mon Sep 17 00:00:00 2001 From: "Edgar R. M" Date: Mon, 7 Aug 2023 14:59:47 -0600 Subject: [PATCH 1/3] fix(targets): Correctly serialize `decimal.Decimal` in JSON fields of SQL targets (#1898) --- pyproject.toml | 6 ++--- singer_sdk/connectors/sql.py | 42 +++++++++++++++++++++++++++++++- tests/core/test_connector_sql.py | 25 +++++++++++++++++++ 3 files changed, 69 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index bceced1d8..a795820d9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,8 +2,8 @@ name = "singer-sdk" version = "0.30.0" description = "A framework for building Singer taps" -authors = ["Meltano Team and Contributors"] -maintainers = ["Meltano Team and Contributors"] +authors = ["Meltano Team and Contributors "] +maintainers = ["Meltano Team and Contributors "] readme = "README.md" homepage = "https://sdk.meltano.com/en/latest/" repository = "https://github.com/meltano/sdk" @@ -144,7 +144,7 @@ name = "cz_version_bump" version = "0.30.0" tag_format = "v$major.$minor.$patch$prerelease" version_files = [ - "docs/conf.py", + "docs/conf.py:^release =", "pyproject.toml:^version =", "cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml:singer-sdk", "cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml:singer-sdk", diff --git a/singer_sdk/connectors/sql.py b/singer_sdk/connectors/sql.py index e9a65cf80..e05e359da 100644 --- a/singer_sdk/connectors/sql.py +++ b/singer_sdk/connectors/sql.py @@ -2,6 +2,8 @@ from __future__ import annotations +import decimal +import json import logging import typing as t import warnings @@ -9,6 +11,7 @@ from datetime import datetime from functools import lru_cache +import simplejson import sqlalchemy from sqlalchemy.engine import Engine @@ -316,7 +319,12 @@ def create_engine(self) -> Engine: Returns: A new SQLAlchemy Engine. """ - return sqlalchemy.create_engine(self.sqlalchemy_url, echo=False) + return sqlalchemy.create_engine( + self.sqlalchemy_url, + echo=False, + json_serializer=self.serialize_json, + json_deserializer=self.deserialize_json, + ) def quote(self, name: str) -> str: """Quote a name if it needs quoting, using '.' as a name-part delimiter. @@ -1124,3 +1132,35 @@ def _adapt_column_type( ) with self._connect() as conn: conn.execute(alter_column_ddl) + + def serialize_json(self, obj: object) -> str: + """Serialize an object to a JSON string. + + Target connectors may override this method to provide custom serialization logic + for JSON types. + + Args: + obj: The object to serialize. + + Returns: + The JSON string. + + .. versionadded:: 0.31.0 + """ + return simplejson.dumps(obj, use_decimal=True) + + def deserialize_json(self, json_str: str) -> object: + """Deserialize a JSON string to an object. + + Tap connectors may override this method to provide custom deserialization + logic for JSON types. + + Args: + json_str: The JSON string to deserialize. + + Returns: + The deserialized object. + + .. versionadded:: 0.31.0 + """ + return json.loads(json_str, parse_float=decimal.Decimal) diff --git a/tests/core/test_connector_sql.py b/tests/core/test_connector_sql.py index 1c04dbcdd..58ba59ec7 100644 --- a/tests/core/test_connector_sql.py +++ b/tests/core/test_connector_sql.py @@ -1,5 +1,6 @@ from __future__ import annotations +from decimal import Decimal from unittest import mock import pytest @@ -258,3 +259,27 @@ def test_merge_generic_sql_types( ): merged_type = connector.merge_sql_types(types) assert isinstance(merged_type, expected_type) + + def test_engine_json_serialization(self, connector: SQLConnector): + engine = connector._engine + meta = sqlalchemy.MetaData() + table = sqlalchemy.Table( + "test_table", + meta, + sqlalchemy.Column("id", sqlalchemy.Integer, primary_key=True), + sqlalchemy.Column("attrs", sqlalchemy.JSON), + ) + meta.create_all(engine) + with engine.connect() as conn: + conn.execute( + table.insert(), + [ + {"attrs": {"x": Decimal("1.0")}}, + {"attrs": {"x": Decimal("2.0"), "y": [1, 2, 3]}}, + ], + ) + result = conn.execute(table.select()) + assert result.fetchall() == [ + (1, {"x": Decimal("1.0")}), + (2, {"x": Decimal("2.0"), "y": [1, 2, 3]}), + ] From f4af551e6d1fea84a195beb8c5f787671057b283 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 7 Aug 2023 15:56:03 -0600 Subject: [PATCH 2/3] chore: Release v0.31.0 (#1899) chore: Bump package version Co-authored-by: edgarrmondragon --- .github/ISSUE_TEMPLATE/bug.yml | 2 +- CHANGELOG.md | 30 +++++++++++++++++++ .../{{cookiecutter.tap_id}}/pyproject.toml | 4 +-- .../{{cookiecutter.target_id}}/pyproject.toml | 4 +-- docs/conf.py | 2 +- pyproject.toml | 4 +-- 6 files changed, 38 insertions(+), 8 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml index 7d3b1a897..45d9aabf1 100644 --- a/.github/ISSUE_TEMPLATE/bug.yml +++ b/.github/ISSUE_TEMPLATE/bug.yml @@ -15,7 +15,7 @@ body: attributes: label: Singer SDK Version description: Version of the library you are using - placeholder: "0.30.0" + placeholder: "0.31.0" validations: required: true - type: checkboxes diff --git a/CHANGELOG.md b/CHANGELOG.md index 56198b597..b7d772080 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,36 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## v0.31.0 (2023-08-07) + +### ✨ New + +- [#1892](https://github.com/meltano/sdk/issues/1892) Add a mapper cookiecutter template +- [#1864](https://github.com/meltano/sdk/issues/1864) SQLTarget connector instance shared with sinks -- _**Thanks @BuzzCutNorman!**_ +- [#1878](https://github.com/meltano/sdk/issues/1878) Add `_sdc_sync_started_at` metadata column to indicate the start of the target process +- [#1484](https://github.com/meltano/sdk/issues/1484) Bump latest supported sqlalchemy from `1.*` to `2.*` + +### 🐛 Fixes + +- [#1898](https://github.com/meltano/sdk/issues/1898) Correctly serialize `decimal.Decimal` in JSON fields of SQL targets +- [#1881](https://github.com/meltano/sdk/issues/1881) Expose `add_record_metadata` as a builtin target setting +- [#1880](https://github.com/meltano/sdk/issues/1880) Append batch config if target supports the batch capability +- [#1865](https://github.com/meltano/sdk/issues/1865) Handle missing record properties in SQL sinks +- [#1838](https://github.com/meltano/sdk/issues/1838) Add deprecation warning when importing legacy testing helpers +- [#1842](https://github.com/meltano/sdk/issues/1842) Ensure all expected tap parameters are passed to `SQLTap` initializer +- [#1853](https://github.com/meltano/sdk/issues/1853) Check against the unconformed key properties when validating record keys +- [#1843](https://github.com/meltano/sdk/issues/1843) Target template should not reference `tap_id` +- [#1708](https://github.com/meltano/sdk/issues/1708) Finalize and write last state message with dedupe +- [#1835](https://github.com/meltano/sdk/issues/1835) Avoid setting up mapper in discovery mode + +### ⚙️ Under the Hood + +- [#1877](https://github.com/meltano/sdk/issues/1877) Use `importlib.resources` instead of `__file__` to retrieve sample Singer output files + +### 📚 Documentation Improvements + +- [#1852](https://github.com/meltano/sdk/issues/1852) Fix stale `pip_url` example that uses shell script workaround for editable installation + ## v0.30.0 (2023-07-10) ### ✨ New diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml index 4be063719..6c1e88fbf 100644 --- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml +++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml @@ -21,7 +21,7 @@ packages = [ [tool.poetry.dependencies] python = "<3.12,>=3.7.1" -singer-sdk = { version="^0.30.0" } +singer-sdk = { version="^0.31.0" } fs-s3fs = { version = "^1.1.1", optional = true } {%- if cookiecutter.stream_type in ["REST", "GraphQL"] %} requests = "^2.31.0" @@ -32,7 +32,7 @@ cached-property = "^1" # Remove after Python 3.7 support is dropped [tool.poetry.group.dev.dependencies] pytest = "^7.4.0" -singer-sdk = { version="^0.30.0", extras = ["testing"] } +singer-sdk = { version="^0.31.0", extras = ["testing"] } [tool.poetry.extras] s3 = ["fs-s3fs"] diff --git a/cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml b/cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml index 16f3f80c8..ee8a65192 100644 --- a/cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml +++ b/cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml @@ -21,7 +21,7 @@ packages = [ [tool.poetry.dependencies] python = "<3.12,>=3.7.1" -singer-sdk = { version="^0.30.0" } +singer-sdk = { version="^0.31.0" } fs-s3fs = { version = "^1.1.1", optional = true } {%- if cookiecutter.serialization_method != "SQL" %} requests = "^2.31.0" @@ -29,7 +29,7 @@ requests = "^2.31.0" [tool.poetry.dev-dependencies] pytest = "^7.4.0" -singer-sdk = { version="^0.30.0", extras = ["testing"] } +singer-sdk = { version="^0.31.0", extras = ["testing"] } [tool.poetry.extras] s3 = ["fs-s3fs"] diff --git a/docs/conf.py b/docs/conf.py index 61ac4b071..14562fdc0 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -26,7 +26,7 @@ author = "Meltano Core Team and Contributors" # The full version, including alpha/beta/rc tags -release = "0.30.0" +release = "0.31.0" # -- General configuration --------------------------------------------------- diff --git a/pyproject.toml b/pyproject.toml index a795820d9..eb691dd8c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "singer-sdk" -version = "0.30.0" +version = "0.31.0" description = "A framework for building Singer taps" authors = ["Meltano Team and Contributors "] maintainers = ["Meltano Team and Contributors "] @@ -141,7 +141,7 @@ norecursedirs = "cookiecutter" [tool.commitizen] name = "cz_version_bump" -version = "0.30.0" +version = "0.31.0" tag_format = "v$major.$minor.$patch$prerelease" version_files = [ "docs/conf.py:^release =", From 7f2df99148e28a65a5dd53f915041fe8f652b03f Mon Sep 17 00:00:00 2001 From: "Edgar R. M" Date: Tue, 8 Aug 2023 07:35:14 -0600 Subject: [PATCH 3/3] chore: Bump SDK version in mapper template (#1900) --- .../mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml | 2 +- pyproject.toml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml index 59c484df1..62fd97284 100644 --- a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml +++ b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml @@ -22,7 +22,7 @@ packages = [ [tool.poetry.dependencies] python = "<3.12,>=3.7.1" -singer-sdk = { version="^0.30.0" } +singer-sdk = { version="^0.31.0" } fs-s3fs = { version = "^1.1.1", optional = true } [tool.poetry.group.dev.dependencies] diff --git a/pyproject.toml b/pyproject.toml index eb691dd8c..4a3601198 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -148,6 +148,7 @@ version_files = [ "pyproject.toml:^version =", "cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml:singer-sdk", "cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml:singer-sdk", + "cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml:singer-sdk", ".github/ISSUE_TEMPLATE/bug.yml:^ placeholder:", ]