From ad21a6867804b82901aebd32582c07cad4f94626 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Ram=C3=ADrez=20Mondrag=C3=B3n?= Date: Tue, 19 Nov 2024 19:21:16 -0600 Subject: [PATCH] fix: Respect standard Singer stream metadata for key properties, replication key and replication method - https://hub.meltano.com/singer/spec/#metadata - https://github.com/singer-io/getting-started/blob/master/docs/DISCOVERY_MODE.md#metadata --- singer_sdk/_singerlib/catalog.py | 1 + singer_sdk/streams/core.py | 25 +++++++++++++++++++++---- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/singer_sdk/_singerlib/catalog.py b/singer_sdk/_singerlib/catalog.py index 986272e3c..c1281e091 100644 --- a/singer_sdk/_singerlib/catalog.py +++ b/singer_sdk/_singerlib/catalog.py @@ -87,6 +87,7 @@ class StreamMetadata(Metadata): """Stream metadata.""" table_key_properties: t.Sequence[str] | None = None + replication_key: str | None = None forced_replication_method: str | None = None valid_replication_keys: list[str] | None = None schema_name: str | None = None diff --git a/singer_sdk/streams/core.py b/singer_sdk/streams/core.py index 009726b7f..9d091ae98 100644 --- a/singer_sdk/streams/core.py +++ b/singer_sdk/streams/core.py @@ -56,6 +56,7 @@ if t.TYPE_CHECKING: import logging + from singer_sdk._singerlib.catalog import StreamMetadata from singer_sdk.helpers import types from singer_sdk.helpers._compat import Traversable from singer_sdk.tap_base import Tap @@ -1277,10 +1278,26 @@ def apply_catalog(self, catalog: singer.Catalog) -> None: catalog_entry = catalog.get_stream(self.name) if catalog_entry: - self.primary_keys = catalog_entry.key_properties - self.replication_key = catalog_entry.replication_key - if catalog_entry.replication_method: - self.forced_replication_method = catalog_entry.replication_method + stream_metadata: StreamMetadata | None + if stream_metadata := catalog_entry.metadata.get(()): + table_key_properties = stream_metadata.table_key_properties + table_replication_key = stream_metadata.replication_key + table_replication_method = stream_metadata.forced_replication_method + else: + table_key_properties = None + table_replication_key = None + table_replication_method = None + + self.primary_keys = catalog_entry.key_properties or table_key_properties + self.replication_key = ( + catalog_entry.replication_key or table_replication_key + ) + + replication_method = ( + catalog_entry.replication_method or table_replication_method + ) + if replication_method: + self.forced_replication_method = replication_method def _get_state_partition_context( self,