Optimize CSV reader and writer for performance

potatoqualitee · potatoqualitee · commit af724b215bba · 2025-12-04T13:20:09.000+01:00
Introduces fast path optimizations for CsvDataReader and CsvWriter, including ultra-fast inline parsing for string-only columns and direct writing of numeric types without StringBuilder allocation. Updates benchmarks and documentation to reflect ~25% performance improvement for all-columns read scenarios and reduces gap vs Sylvan. Bumps package version to 1.1.1.
diff --git a/project/Dataplat.Dbatools.Csv/CHANGELOG.md b/project/Dataplat.Dbatools.Csv/CHANGELOG.md
@@ -7,6 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [1.1.1] - 2025-12-04
+
+### Changed
+- **~25% performance improvement** for all-columns read scenarios (55ms vs 73ms for 100K rows)
+- Added fast conversion path for simple string-only columns, bypassing unnecessary checks
+- Added ultra-fast inline parsing path (.NET 8+) that writes directly to output, skipping intermediate buffers
+- Gap vs Sylvan reduced from 2.0x to 1.6x for all-columns reads
+- Added fast write path for CsvWriter - writes numeric types directly without StringBuilder allocation
+
 ## [1.1.0] - 2025-12-04
 
 ### Added
diff --git a/project/Dataplat.Dbatools.Csv/Dataplat.Dbatools.Csv.csproj b/project/Dataplat.Dbatools.Csv/Dataplat.Dbatools.Csv.csproj
@@ -7,7 +7,7 @@
 
     <!-- NuGet Package Metadata -->
     <PackageId>Dataplat.Dbatools.Csv</PackageId>
-    <Version>1.1.0</Version>
+    <Version>1.1.1</Version>
     <Authors>Chrissy LeMaire</Authors>
     <Company>Dataplat</Company>
     <Product>Dataplat.Dbatools.Csv</Product>
diff --git a/project/Dataplat.Dbatools.Csv/README.md b/project/Dataplat.Dbatools.Csv/README.md
@@ -49,21 +49,21 @@ Benchmark: 100,000 rows × 10 columns (.NET 8, AVX-512)
 
 | Library | Time (ms) | vs Dataplat |
 |---------|-----------|-------------|
-| Sep | 19 ms | 3.8x faster |
-| Sylvan | 29 ms | 2.5x faster |
-| **Dataplat** | **74 ms** | **baseline** |
-| CsvHelper | 76 ms | ~same |
-| LumenWorks | 433 ms | **5.9x slower** |
+| Sep | 18 ms | 3.7x faster |
+| Sylvan | 27 ms | 2.5x faster |
+| **Dataplat** | **67 ms** | **baseline** |
+| CsvHelper | 76 ms | 1.1x slower |
+| LumenWorks | 395 ms | **5.9x slower** |
 
 **All columns read (full row processing):**
 
 | Library | Time (ms) | vs Dataplat |
 |---------|-----------|-------------|
-| Sep | 35 ms | 2.1x faster |
-| Sylvan | 37 ms | 2.0x faster |
-| **Dataplat** | **73 ms** | **baseline** |
-| CsvHelper | 101 ms | 1.4x slower |
-| LumenWorks | 100 ms | 1.4x slower |
+| Sep | 30 ms | 1.8x faster |
+| Sylvan | 35 ms | 1.6x faster |
+| **Dataplat** | **55 ms** | **baseline** |
+| CsvHelper | 97 ms | 1.8x slower |
+| LumenWorks | 102 ms | 1.9x slower |
 
 ### Understanding the performance tradeoffs
 
diff --git a/project/dbatools/Csv/Reader/CsvDataReader.cs b/project/dbatools/Csv/Reader/CsvDataReader.cs
@@ -118,6 +118,10 @@ public sealed class CsvDataReader : IDataReader
         private System.Buffers.SearchValues<char> _fieldTerminators;
 #endif
 
+        // Fast path optimization flags - determined during initialization
+        private bool _useFastConversion;               // True when simple string-only conversion can be used
+        private bool _useFastParsing;                  // True when ultra-fast inline parsing can be used
+
         #endregion
 
         #region Field Info Structure
@@ -466,6 +470,9 @@ private void Initialize()
             // Cache converters for each column to avoid per-row registry lookups
             CacheColumnConverters();
 
+            // Determine if we can use fast path optimizations
+            InitializeFastPathOptimizations();
+
             // Prepare converted values array
             _convertedValues = new object[_columns.Count + _staticColumns.Count];
 
@@ -489,6 +496,56 @@ private void CacheColumnConverters()
             }
         }
 
+        /// <summary>
+        /// Initializes fast path optimization flags based on options and column configuration.
+        /// </summary>
+        private void InitializeFastPathOptimizations()
+        {
+            // Check if all columns are strings (no type conversion needed)
+            bool hasNonStringColumns = false;
+            for (int i = 0; i < _columns.Count; i++)
+            {
+                if (_columns[i].DataType != typeof(string) || _columns[i].CachedConverter != null)
+                {
+                    hasNonStringColumns = true;
+                    break;
+                }
+            }
+
+            // Static columns always need conversion (they compute values)
+            if (_staticColumns.Count > 0)
+            {
+                hasNonStringColumns = true;
+            }
+
+            // Determine if we can use the fast conversion path:
+            // - No trimming options
+            // - No null value configured
+            // - No DistinguishEmptyFromNull
+            // - No UseColumnDefaults
+            // - No static columns
+            // - All columns are strings
+            _useFastConversion = !hasNonStringColumns
+                && _options.TrimmingOptions == ValueTrimmingOptions.None
+                && _options.NullValue == null
+                && !_options.DistinguishEmptyFromNull
+                && !_options.UseColumnDefaults
+                && _staticColumns.Count == 0;
+
+            // Determine if we can use the ultra-fast inline parsing path:
+            // - Single-character delimiter
+            // - No quote normalization
+            // - No comment character
+            // - No parallel processing
+            // All of the above plus fast conversion conditions
+            _useFastParsing = _useFastConversion
+                && _singleCharDelimiter
+                && !_options.NormalizeQuotes
+                && _options.Comment == '\0'
+                && !_options.EnableParallelProcessing
+                && _options.QuoteMode != QuoteMode.Lenient;
+        }
+
         private void InitializeColumnsFromFirstDataRow()
         {
             // Read the first data row to determine column count
@@ -1507,6 +1564,14 @@ private bool ReadBufferedFirstLine()
         /// </summary>
         private bool ReadSequentialDirect()
         {
+#if NET8_0_OR_GREATER
+            // Ultra-fast path: inline parsing directly to _convertedValues for simple CSV
+            if (_useFastParsing && _isInitialized)
+            {
+                return ReadSequentialUltraFast();
+            }
+#endif
+
             while (true)
             {
                 try
@@ -1553,6 +1618,155 @@ private bool ReadSequentialDirect()
             }
         }
 
+#if NET8_0_OR_GREATER
+        /// <summary>
+        /// Ultra-fast inline parsing for simple CSV files (no quotes, no special options).
+        /// Writes directly to _convertedValues, skipping all intermediate buffers.
+        /// </summary>
+        [MethodImpl(MethodImplOptions.AggressiveOptimization)]
+        private bool ReadSequentialUltraFast()
+        {
+            char delimChar = _delimiterFirstChar;
+            char quoteChar = _options.Quote;
+            int columnCount = _columns.Count;
+            var values = _convertedValues;
+
+            while (true)
+            {
+                // Ensure we have data
+                if (_bufferPosition >= _bufferLength)
+                {
+                    if (!RefillBuffer())
+                    {
+                        _currentRecord = null;
+                        return false;
+                    }
+                }
+
+                // Skip empty lines
+                while (_bufferPosition < _bufferLength)
+                {
+                    char c = _buffer[_bufferPosition];
+                    if (c == '\r')
+                    {
+                        _bufferPosition++;
+                        _currentLineNumber++;
+                        if (_bufferPosition < _bufferLength && _buffer[_bufferPosition] == '\n')
+                            _bufferPosition++;
+                        continue;
+                    }
+                    if (c == '\n')
+                    {
+                        _bufferPosition++;
+                        _currentLineNumber++;
+                        continue;
+                    }
+                    break; // Found start of record
+                }
+
+                if (_bufferPosition >= _bufferLength)
+                    continue; // Need more data
+
+                // Parse the record directly into _convertedValues
+                _currentRecordIndex++;
+                int fieldIndex = 0;
+
+                while (fieldIndex < columnCount)
+                {
+                    if (_bufferPosition >= _bufferLength)
+                    {
+                        // Buffer exhausted mid-record - fall back to standard path
+                        _currentRecordIndex--;
+                        _useFastParsing = false;
+                        return ReadSequentialDirect();
+                    }
+
+                    char c = _buffer[_bufferPosition];
+
+                    // Check for quoted field - fall back to standard path
+                    if (c == quoteChar)
+                    {
+                        _currentRecordIndex--;
+                        _useFastParsing = false;
+                        return ReadSequentialDirect();
+                    }
+
+                    int fieldStart = _bufferPosition;
+
+                    // Use SIMD to find delimiter or newline
+                    ReadOnlySpan<char> remaining = _buffer.AsSpan(_bufferPosition, _bufferLength - _bufferPosition);
+                    int idx = remaining.IndexOfAny(_fieldTerminators);
+
+                    if (idx < 0)
+                    {
+                        // No terminator found - fall back to standard path
+                        _currentRecordIndex--;
+                        _useFastParsing = false;
+                        return ReadSequentialDirect();
+                    }
+
+                    _bufferPosition += idx;
+                    c = _buffer[_bufferPosition];
+
+                    // Create field string
+                    int sourceIndex = _columns[fieldIndex].SourceIndex;
+                    if (sourceIndex == fieldIndex) // Common case: sequential columns
+                    {
+                        int length = _bufferPosition - fieldStart;
+                        if (length == 0)
+                        {
+                            values[fieldIndex] = DBNull.Value;
+                        }
+                        else
+                        {
+                            values[fieldIndex] = new string(_buffer, fieldStart, length);
+                        }
+                    }
+                    else
+                    {
+                        // Column mapping is non-trivial - fall back
+                        _currentRecordIndex--;
+                        _useFastParsing = false;
+                        return ReadSequentialDirect();
+                    }
+
+                    if (c == delimChar)
+                    {
+                        _bufferPosition++; // Skip delimiter
+                        fieldIndex++;
+                    }
+                    else // c == '\r' || c == '\n'
+                    {
+                        // End of record - skip newline
+                        if (c == '\r')
+                        {
+                            _bufferPosition++;
+                            if (_bufferPosition < _bufferLength && _buffer[_bufferPosition] == '\n')
+                                _bufferPosition++;
+                        }
+                        else
+                        {
+                            _bufferPosition++;
+                        }
+                        fieldIndex++;
+                        break;
+                    }
+                }
+
+                // Fill remaining columns with DBNull
+                while (fieldIndex < columnCount)
+                {
+                    values[fieldIndex] = DBNull.Value;
+                    fieldIndex++;
+                }
+
+                _currentRecord = _recordBuffer;
+                _currentLineNumber++;
+                return true;
+            }
+        }
+#endif
+
         /// <summary>
         /// Handles parse errors consistently for both parsing paths.
         /// </summary>
@@ -1681,6 +1895,14 @@ private void HandleFieldCountMismatch(string line, int expectedCount)
 
         private void ConvertCurrentRecord()
         {
+            // Fast path: all columns are strings with no special handling needed
+            if (_useFastConversion)
+            {
+                ConvertCurrentRecordFast();
+                return;
+            }
+
+            // Standard path with all options supported
             for (int i = 0; i < _columns.Count; i++)
             {
                 var column = _columns[i];
@@ -1761,6 +1983,35 @@ private void ConvertCurrentRecord()
             }
         }
 
+        /// <summary>
+        /// Fast conversion path for simple string-only columns with no special handling.
+        /// This avoids all the per-column checks and branching.
+        /// </summary>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private void ConvertCurrentRecordFast()
+        {
+            int columnCount = _columns.Count;
+            var record = _currentRecord;
+            var values = _convertedValues;
+
+            // Direct copy of string values - no conversion, trimming, or null handling
+            for (int i = 0; i < columnCount; i++)
+            {
+                int sourceIndex = _columns[i].SourceIndex;
+                string rawValue = sourceIndex < record.Length ? record[sourceIndex] : null;
+
+                // Empty strings become DBNull for consistency with database behavior
+                if (string.IsNullOrEmpty(rawValue))
+                {
+                    values[i] = DBNull.Value;
+                }
+                else
+                {
+                    values[i] = rawValue;
+                }
+            }
+        }
+
         private object ConvertValue(string value, CsvColumn column)
         {
             if (column.DataType == typeof(string))
@@ -3608,6 +3859,9 @@ public void SetColumnType(string columnName, Type type)
                     if (type != typeof(string))
                     {
                         _columns[i].CachedConverter = _columns[i].Converter ?? _options.TypeConverterRegistry?.GetConverter(type);
+                        // Invalidate fast path when non-string column type is set
+                        _useFastConversion = false;
+                        _useFastParsing = false;
                     }
                     else
                     {
diff --git a/project/dbatools/Csv/Writer/CsvWriter.cs b/project/dbatools/Csv/Writer/CsvWriter.cs
