bugfix/first-field-quoted (#91)

* done * done * teste with and without parallel * add test for basic cenario
leandromoh · Nov 12, 2023 · dc8f106 · dc8f106
1 parent f5f28bd
commit dc8f106
Show file tree

Hide file tree

Showing 2 changed files with 161 additions and 1 deletion.
diff --git a/RecordParser.Test/QuotedFileReaderTest.cs b/RecordParser.Test/QuotedFileReaderTest.cs
@@ -2,7 +2,9 @@
 using RecordParser.Builders.Reader;
 using RecordParser.Extensions;
 using System;
+using System.Collections.Generic;
 using System.IO;
+using System.Linq;
 using System.Text;
 using Xunit;
 
@@ -18,6 +20,141 @@ public class Quoted
             public Gender Gender;
         }
 
+        public static IEnumerable<object[]> QuotedFieldsAnyColumn()
+        {
+            foreach (var parallel in new[] { true, false })
+            {
+                foreach (var newline in new[] { "\r", "\n", "\r\n" })
+                {
+                    yield return new object[] { parallel, newline };
+                }
+            }
+        }
+
+        [Theory]
+        [MemberData(nameof(QuotedFieldsAnyColumn))]
+        public void Given_quoted_field_in_any_column_should_parse_successfully(bool parallel, string newline)
+        {
+            // Arrange
+
+            var fileContent = """
+                A,B,C,D
+                "x
+                y",2,3,4
+                1,"a,
+                b",3,4
+                7,8,"a
+                z",9
+                98,99,100,101
+                12,13,14,"w
+                s"
+                  "
+                a,1
+                ",3, b,4
+                a,b,c,d
+                """.Replace(Environment.NewLine, newline);
+
+            var expected = new[]
+            {
+                // column quoted = 1
+                ($"x{newline}y","2","3","4"),
+                // column quoted = 2
+                ("1",$"a,{newline}b","3","4"),
+                // column quoted = 3
+                ("7","8",$"a{newline}z","9"),
+                // no quoted column
+                ("98","99","100","101"),
+                // column quoted = 4
+                ("12","13","14",$"w{newline}s"),
+                // column quoted = 1 with whitespace before quote
+                ($"{newline}a,1{newline}", "3", " b", "4"),
+                // no quoted column
+                ("a","b","c","d"),
+            };
+
+            var reader = new StringReader(fileContent);
+            var options = new VariableLengthReaderRawOptions
+            {
+                HasHeader = true,
+                ContainsQuotedFields = true,
+                ColumnCount = 4,
+                Separator = ",",
+                ParallelismOptions = new()
+                {
+                    Enabled = parallel,
+                    MaxDegreeOfParallelism = 2,
+                    EnsureOriginalOrdering = true,
+                },
+            };
+
+            // Act
+
+            var records = reader.ReadRecordsRaw(options, getField =>
+            {
+                var record =
+                (
+                    getField(0),
+                    getField(1),
+                    getField(2),
+                    getField(3)
+                );
+                return record;
+            }).ToList();
+
+            // Assert
+
+            records.Should().BeEquivalentTo(expected, cfg => cfg.WithStrictOrdering());
+        }
+
+        [Theory]
+        [InlineData(true)]
+        [InlineData(false)]
+        public void Given_quoted_field_in_first_column_should_parse_successfully(bool parallel)
+        {
+            // Arrange
+
+            var fileContent = $"""
+                A,B,C,D
+                "x
+                y",2,3,4
+                """;
+
+            var expected = ($"x{Environment.NewLine}y","2","3","4");
+            var reader = new StringReader(fileContent);
+            var options = new VariableLengthReaderRawOptions
+            {
+                HasHeader = true,
+                ContainsQuotedFields = true,
+                ColumnCount = 4,
+                Separator = ",",
+                ParallelismOptions = new()
+                {
+                    Enabled = parallel,
+                    MaxDegreeOfParallelism = 2,
+                },
+            };
+
+            // Act
+
+            var result = reader.ReadRecordsRaw(options, getField =>
+            {
+                var record =
+                (
+                    getField(0),
+                    getField(1),
+                    getField(2),
+                    getField(3)
+                );
+                return record;
+            }).ToList();
+
+            // Assert
+
+            result.Should().HaveCount(1);
+            var row = result[0];
+            row.Should().Be(expected);
+        }
+
         [Theory]
         [InlineData(true)]
         [InlineData(false)]

diff --git a/RecordParser/Extensions/FileReader/RowReaders/RowByQuote.cs b/RecordParser/Extensions/FileReader/RowReaders/RowByQuote.cs
@@ -2,6 +2,7 @@
 using System;
 using System.Collections.Generic;
 using System.IO;
+using System.Runtime.CompilerServices;
 
 namespace RecordParser.Extensions.FileReader.RowReaders
 {
@@ -58,7 +59,7 @@ public override IEnumerable<ReadOnlyMemory<char>> ReadLines()
                 else if (c == quote)
                 {
                     ReadOnlySpan<char> span = buffer.AsSpan().Slice(0, i - 1);
-                    var isQuotedField = span.TrimEnd().EndsWith(separator);
+                    var isQuotedField = IsFirstColumn(span) || span.TrimEnd().EndsWith(separator);
 
                     if (isQuotedField is false)
                         continue;
@@ -102,6 +103,28 @@ public override IEnumerable<ReadOnlyMemory<char>> ReadLines()
                 yield return y;
 
             goto reloop;
+
+
+            [MethodImpl(MethodImplOptions.AggressiveInlining)]
+            static bool IsFirstColumn(ReadOnlySpan<char> span)
+            {
+                var onlyWhiteSpace = true;
+
+                for (var i = span.Length - 1; i >= 0; i--)
+                {
+                    if (char.IsWhiteSpace(span[i]))
+                    {
+                        if (span[i] is '\n' or '\r')
+                            return true;
+                        else
+                            continue;
+                    }
+                    onlyWhiteSpace = false;
+                    break;
+                }
+
+                return onlyWhiteSpace;
+            }
         }
     }
 }