fixed bulk csv serialization bug

georgebv · Mar 28, 2024 · 0350122 · 0350122
1 parent 3696f84
commit 0350122
Show file tree

Hide file tree

Showing 3 changed files with 60 additions and 3 deletions.
diff --git a/src/aiosalesforce/__init__.py b/src/aiosalesforce/__init__.py
@@ -1,4 +1,4 @@
-__version__ = "0.5.2"
+__version__ = "0.5.3"
 
 __all__ = [
     "ClientCredentialsFlow",

diff --git a/src/aiosalesforce/bulk/v2/_csv.py b/src/aiosalesforce/bulk/v2/_csv.py
@@ -141,7 +141,7 @@ def serialize_ingest_data(
                 ),
                 UserWarning,
             )
-        data = list(data)
+        data = [_serialize_dict(value) for value in data]
         fieldnames = dict.fromkeys(itertools.chain.from_iterable(data)).keys()
 
     buffer = CsvBuffer()

diff --git a/tests/test_client/test_bulk/test_v2/test_csv.py b/tests/test_client/test_bulk/test_v2/test_csv.py
@@ -87,7 +87,7 @@ class TestCsvSerializer:
     def test_empty(self):
         assert list(serialize_ingest_data([])) == []
 
-    def test_single(self):
+    def test_single_row(self):
         data = [{"FirstName": "Jon", "LastName": "Doe"}]
         csvs = list(serialize_ingest_data(data))
         assert len(csvs) == 1
@@ -131,3 +131,60 @@ def test_carryover(self):
         csvs = list(serialize_ingest_data(data, max_size_bytes=117))
         assert len(csvs) == 9
         assert sum(len(deserialize_ingest_results(csv)) for csv in csvs) == len(data)
+
+    def test_nested(self):
+        data = [
+            {"FirstName": "Jon", "LastName": "Doe", "Employer__r": {"EID__c": "123"}}
+            for _ in range(100)
+        ]
+        csvs = list(serialize_ingest_data(data))
+        assert len(csvs) == 1
+        assert csvs[0] == b"\n".join(
+            [
+                b"FirstName,LastName,Employer__r.EID__c",
+                *[b"Jon,Doe,123" for _ in range(100)],
+                b"",
+            ]
+        )
+
+    def test_everything(self):
+        data = [
+            {
+                "Active": True,
+                "Rehire": False,
+                "Bonus": None,
+                "Hired": datetime.datetime(2024, 5, 6, 12, 13, 14),
+                "Birthdate": datetime.date(2000, 1, 1),
+                "FirstName": "Jon",
+                "LastName": "Doe",
+                "Salary": 100_000,
+                "TaxRate": 0.25,
+                "Employer__r": {"EID__c": "123"},
+            }
+            for _ in range(100)
+        ]
+        csvs = list(serialize_ingest_data(data, max_records=30))
+        assert len(csvs) == 4
+        for i in range(3):
+            assert csvs[i] == b"\n".join(
+                [
+                    b"Active,Rehire,Bonus,Hired,Birthdate,FirstName,LastName,Salary,TaxRate,Employer__r.EID__c",
+                    *[
+                        b"true,false,,2024-05-06T12:13:14.000,2000-01-01,Jon,Doe,100000,0.25,123"
+                        for _ in range(30)
+                    ],
+                    b"",
+                ]
+            )
+            assert len(deserialize_ingest_results(csvs[i])) == 30
+        assert csvs[-1] == b"\n".join(
+            [
+                b"Active,Rehire,Bonus,Hired,Birthdate,FirstName,LastName,Salary,TaxRate,Employer__r.EID__c",
+                *[
+                    b"true,false,,2024-05-06T12:13:14.000,2000-01-01,Jon,Doe,100000,0.25,123"
+                    for _ in range(10)
+                ],
+                b"",
+            ]
+        )
+        assert len(deserialize_ingest_results(csvs[-1])) == 10