Merge branch 'main' into edgarrmondragon/feat/jsonschema-to-sql

meltano · Nov 6, 2024 · dea21ae · dea21ae
2 parents de3a207 + 2ad977e
commit dea21ae
Show file tree

Hide file tree

Showing 22 changed files with 104 additions and 75 deletions.
diff --git a/singer_sdk/mapper.py b/singer_sdk/mapper.py
@@ -401,10 +401,12 @@ def _eval_type(  # noqa: PLR0911
         if expr.startswith("bool("):
             return th.BooleanType()
 
-        if expr.startswith("datetime.date") or expr.endswith(".date()"):
+        if expr.startswith(("datetime.date.", "datetime.date(")) or expr.endswith(
+            ".date()"
+        ):
             return th.DateType()
 
-        if expr.startswith("datetime.datetime"):
+        if expr.startswith(("datetime.datetime.", "datetime.datetime(")):
             return th.DateTimeType()
 
         return th.StringType() if expr[0] == "'" and expr[-1] == "'" else default

diff --git a/tests/core/test_mapper.py b/tests/core/test_mapper.py
@@ -23,6 +23,7 @@
     ArrayType,
     BooleanType,
     CustomType,
+    DateTimeType,
     IntegerType,
     NumberType,
     ObjectType,
@@ -626,6 +627,7 @@ class MappedStream(Stream):
                 Property("some_numbers", ArrayType(NumberType())),
             ),
         ),
+        Property("joined_at", DateTimeType),
     ).to_dict()
 
     def get_records(self, context):  # noqa: ARG002
@@ -637,6 +639,7 @@ def get_records(self, context):  # noqa: ARG002
                 "sub": {"num": 1, "custom_obj": CustomObj("hello")},
                 "some_numbers": [Decimal("3.14"), Decimal("2.718")],
             },
+            "joined_at": "2022-01-01T00:00:00Z",
         }
         yield {
             "email": "bob@example.com",
@@ -646,6 +649,7 @@ def get_records(self, context):  # noqa: ARG002
                 "sub": {"num": 2, "custom_obj": CustomObj("world")},
                 "some_numbers": [Decimal("10.32"), Decimal("1.618")],
             },
+            "joined_at": "2022-01-01T00:00:00Z",
         }
         yield {
             "email": "charlie@example.com",
@@ -655,6 +659,7 @@ def get_records(self, context):  # noqa: ARG002
                 "sub": {"num": 3, "custom_obj": CustomObj("hello")},
                 "some_numbers": [Decimal("1.414"), Decimal("1.732")],
             },
+            "joined_at": "2022-01-01T00:00:00Z",
         }
 
     def get_batches(self, batch_config, context):  # noqa: ARG002
@@ -942,6 +947,22 @@ def discover_streams(self):
             "fake_email_seed_instance.jsonl",
             id="fake_email_seed_instance",
         ),
+        pytest.param(
+            {
+                "mystream": {
+                    "joined_date": "datetime.datetime.fromisoformat(joined_at).date()",
+                    "joined_timestamp": "float(datetime.datetime.fromisoformat(joined_at).timestamp())",  # noqa: E501
+                    "some_datetime": "datetime.datetime.fromisoformat(config['some_date_string'])",  # noqa: E501
+                },
+            },
+            {
+                "stream_map_config": {
+                    "some_date_string": "2024-10-10T10:10:10Z",
+                },
+            },
+            "dates.jsonl",
+            id="dates",
+        ),
     ],
 )
 def test_mapped_stream(

diff --git a/tests/snapshots/mapped_stream/aliased_stream.jsonl b/tests/snapshots/mapped_stream/aliased_stream.jsonl
@@ -1,6 +1,6 @@
 {"type":"STATE","value":{}}
-{"type":"SCHEMA","stream":"aliased_stream","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]}},"type":"object","required":["email"]},"key_properties":[]}
-{"type":"RECORD","stream":"aliased_stream","record":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1,"custom_obj":"obj-hello"},"some_numbers":[3.14,2.718]}},"time_extracted":"2022-01-01T00:00:00+00:00"}
-{"type":"RECORD","stream":"aliased_stream","record":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2,"custom_obj":"obj-world"},"some_numbers":[10.32,1.618]}},"time_extracted":"2022-01-01T00:00:00+00:00"}
-{"type":"RECORD","stream":"aliased_stream","record":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3,"custom_obj":"obj-hello"},"some_numbers":[1.414,1.732]}},"time_extracted":"2022-01-01T00:00:00+00:00"}
+{"type":"SCHEMA","stream":"aliased_stream","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]},"joined_at":{"format":"date-time","type":["string","null"]}},"type":"object","required":["email"]},"key_properties":[]}
+{"type":"RECORD","stream":"aliased_stream","record":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1,"custom_obj":"obj-hello"},"some_numbers":[3.14,2.718]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"}
+{"type":"RECORD","stream":"aliased_stream","record":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2,"custom_obj":"obj-world"},"some_numbers":[10.32,1.618]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"}
+{"type":"RECORD","stream":"aliased_stream","record":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3,"custom_obj":"obj-hello"},"some_numbers":[1.414,1.732]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"}
 {"type":"STATE","value":{"bookmarks":{"mystream":{}}}}
diff --git a/tests/snapshots/mapped_stream/aliased_stream_batch.jsonl b/tests/snapshots/mapped_stream/aliased_stream_batch.jsonl
@@ -1,5 +1,5 @@
 {"type":"STATE","value":{}}
-{"type":"SCHEMA","stream":"aliased_stream","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]}},"type":"object","required":["email"]},"key_properties":[]}
+{"type":"SCHEMA","stream":"aliased_stream","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]},"joined_at":{"format":"date-time","type":["string","null"]}},"type":"object","required":["email"]},"key_properties":[]}
 {"type":"BATCH","stream":"aliased_stream","encoding":{"format":"jsonl","compression":"gzip"},"manifest":["file:///tmp/stream.json.gz"]}
 {"type":"STATE","value":{}}
 {"type":"STATE","value":{"bookmarks":{"mystream":{}}}}
diff --git a/tests/snapshots/mapped_stream/aliased_stream_not_expr.jsonl b/tests/snapshots/mapped_stream/aliased_stream_not_expr.jsonl
@@ -1,6 +1,6 @@
 {"type":"STATE","value":{}}
-{"type":"SCHEMA","stream":"aliased.stream","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]}},"type":"object","required":["email"]},"key_properties":[]}
-{"type":"RECORD","stream":"aliased.stream","record":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1,"custom_obj":"obj-hello"},"some_numbers":[3.14,2.718]}},"time_extracted":"2022-01-01T00:00:00+00:00"}
-{"type":"RECORD","stream":"aliased.stream","record":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2,"custom_obj":"obj-world"},"some_numbers":[10.32,1.618]}},"time_extracted":"2022-01-01T00:00:00+00:00"}
-{"type":"RECORD","stream":"aliased.stream","record":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3,"custom_obj":"obj-hello"},"some_numbers":[1.414,1.732]}},"time_extracted":"2022-01-01T00:00:00+00:00"}
+{"type":"SCHEMA","stream":"aliased.stream","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]},"joined_at":{"format":"date-time","type":["string","null"]}},"type":"object","required":["email"]},"key_properties":[]}
+{"type":"RECORD","stream":"aliased.stream","record":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1,"custom_obj":"obj-hello"},"some_numbers":[3.14,2.718]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"}
+{"type":"RECORD","stream":"aliased.stream","record":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2,"custom_obj":"obj-world"},"some_numbers":[10.32,1.618]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"}
+{"type":"RECORD","stream":"aliased.stream","record":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3,"custom_obj":"obj-hello"},"some_numbers":[1.414,1.732]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"}
 {"type":"STATE","value":{"bookmarks":{"mystream":{}}}}
diff --git a/tests/snapshots/mapped_stream/aliased_stream_quoted.jsonl b/tests/snapshots/mapped_stream/aliased_stream_quoted.jsonl
@@ -1,6 +1,6 @@
 {"type":"STATE","value":{}}
-{"type":"SCHEMA","stream":"__stream_name__","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]}},"type":"object","required":["email"]},"key_properties":[]}
-{"type":"RECORD","stream":"__stream_name__","record":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1,"custom_obj":"obj-hello"},"some_numbers":[3.14,2.718]}},"time_extracted":"2022-01-01T00:00:00+00:00"}
-{"type":"RECORD","stream":"__stream_name__","record":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2,"custom_obj":"obj-world"},"some_numbers":[10.32,1.618]}},"time_extracted":"2022-01-01T00:00:00+00:00"}
-{"type":"RECORD","stream":"__stream_name__","record":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3,"custom_obj":"obj-hello"},"some_numbers":[1.414,1.732]}},"time_extracted":"2022-01-01T00:00:00+00:00"}
+{"type":"SCHEMA","stream":"__stream_name__","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]},"joined_at":{"format":"date-time","type":["string","null"]}},"type":"object","required":["email"]},"key_properties":[]}
+{"type":"RECORD","stream":"__stream_name__","record":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1,"custom_obj":"obj-hello"},"some_numbers":[3.14,2.718]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"}
+{"type":"RECORD","stream":"__stream_name__","record":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2,"custom_obj":"obj-world"},"some_numbers":[10.32,1.618]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"}
+{"type":"RECORD","stream":"__stream_name__","record":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3,"custom_obj":"obj-hello"},"some_numbers":[1.414,1.732]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"}
 {"type":"STATE","value":{"bookmarks":{"mystream":{}}}}
diff --git a/tests/snapshots/mapped_stream/builtin_variable_stream_name.jsonl b/tests/snapshots/mapped_stream/builtin_variable_stream_name.jsonl
@@ -1,6 +1,6 @@
 {"type":"STATE","value":{}}
-{"type":"SCHEMA","stream":"mystream","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]},"source_table":{"type":["string","null"]}},"type":"object","required":["email"]},"key_properties":[]}
-{"type":"RECORD","stream":"mystream","record":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1,"custom_obj":"obj-hello"},"some_numbers":[3.14,2.718]},"source_table":"mystream"},"time_extracted":"2022-01-01T00:00:00+00:00"}
-{"type":"RECORD","stream":"mystream","record":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2,"custom_obj":"obj-world"},"some_numbers":[10.32,1.618]},"source_table":"mystream"},"time_extracted":"2022-01-01T00:00:00+00:00"}
-{"type":"RECORD","stream":"mystream","record":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3,"custom_obj":"obj-hello"},"some_numbers":[1.414,1.732]},"source_table":"mystream"},"time_extracted":"2022-01-01T00:00:00+00:00"}
+{"type":"SCHEMA","stream":"mystream","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]},"joined_at":{"format":"date-time","type":["string","null"]},"source_table":{"type":["string","null"]}},"type":"object","required":["email"]},"key_properties":[]}
+{"type":"RECORD","stream":"mystream","record":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1,"custom_obj":"obj-hello"},"some_numbers":[3.14,2.718]},"joined_at":"2022-01-01T00:00:00Z","source_table":"mystream"},"time_extracted":"2022-01-01T00:00:00+00:00"}
+{"type":"RECORD","stream":"mystream","record":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2,"custom_obj":"obj-world"},"some_numbers":[10.32,1.618]},"joined_at":"2022-01-01T00:00:00Z","source_table":"mystream"},"time_extracted":"2022-01-01T00:00:00+00:00"}
+{"type":"RECORD","stream":"mystream","record":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3,"custom_obj":"obj-hello"},"some_numbers":[1.414,1.732]},"joined_at":"2022-01-01T00:00:00Z","source_table":"mystream"},"time_extracted":"2022-01-01T00:00:00+00:00"}
 {"type":"STATE","value":{"bookmarks":{"mystream":{}}}}
diff --git a/tests/snapshots/mapped_stream/builtin_variable_stream_name_alias.jsonl b/tests/snapshots/mapped_stream/builtin_variable_stream_name_alias.jsonl
@@ -1,6 +1,6 @@
 {"type":"STATE","value":{}}
-{"type":"SCHEMA","stream":"aliased_mystream","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]}},"type":"object","required":["email"]},"key_properties":[]}
-{"type":"RECORD","stream":"aliased_mystream","record":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1,"custom_obj":"obj-hello"},"some_numbers":[3.14,2.718]}},"time_extracted":"2022-01-01T00:00:00+00:00"}
-{"type":"RECORD","stream":"aliased_mystream","record":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2,"custom_obj":"obj-world"},"some_numbers":[10.32,1.618]}},"time_extracted":"2022-01-01T00:00:00+00:00"}
-{"type":"RECORD","stream":"aliased_mystream","record":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3,"custom_obj":"obj-hello"},"some_numbers":[1.414,1.732]}},"time_extracted":"2022-01-01T00:00:00+00:00"}
+{"type":"SCHEMA","stream":"aliased_mystream","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]},"joined_at":{"format":"date-time","type":["string","null"]}},"type":"object","required":["email"]},"key_properties":[]}
+{"type":"RECORD","stream":"aliased_mystream","record":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1,"custom_obj":"obj-hello"},"some_numbers":[3.14,2.718]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"}
+{"type":"RECORD","stream":"aliased_mystream","record":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2,"custom_obj":"obj-world"},"some_numbers":[10.32,1.618]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"}
+{"type":"RECORD","stream":"aliased_mystream","record":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3,"custom_obj":"obj-hello"},"some_numbers":[1.414,1.732]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"}
 {"type":"STATE","value":{"bookmarks":{"mystream":{}}}}
diff --git a/tests/snapshots/mapped_stream/builtin_variable_stream_name_alias_expr.jsonl b/tests/snapshots/mapped_stream/builtin_variable_stream_name_alias_expr.jsonl
@@ -1,6 +1,6 @@
 {"type":"STATE","value":{}}
-{"type":"SCHEMA","stream":"MYSTREAM","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]}},"type":"object","required":["email"]},"key_properties":[]}
-{"type":"RECORD","stream":"MYSTREAM","record":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1,"custom_obj":"obj-hello"},"some_numbers":[3.14,2.718]}},"time_extracted":"2022-01-01T00:00:00+00:00"}
-{"type":"RECORD","stream":"MYSTREAM","record":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2,"custom_obj":"obj-world"},"some_numbers":[10.32,1.618]}},"time_extracted":"2022-01-01T00:00:00+00:00"}
-{"type":"RECORD","stream":"MYSTREAM","record":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3,"custom_obj":"obj-hello"},"some_numbers":[1.414,1.732]}},"time_extracted":"2022-01-01T00:00:00+00:00"}
+{"type":"SCHEMA","stream":"MYSTREAM","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]},"joined_at":{"format":"date-time","type":["string","null"]}},"type":"object","required":["email"]},"key_properties":[]}
+{"type":"RECORD","stream":"MYSTREAM","record":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1,"custom_obj":"obj-hello"},"some_numbers":[3.14,2.718]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"}
+{"type":"RECORD","stream":"MYSTREAM","record":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2,"custom_obj":"obj-world"},"some_numbers":[10.32,1.618]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"}
+{"type":"RECORD","stream":"MYSTREAM","record":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3,"custom_obj":"obj-hello"},"some_numbers":[1.414,1.732]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"}
 {"type":"STATE","value":{"bookmarks":{"mystream":{}}}}