Assembly Conflict Handling & Documentation + CSV stuff (#38)

Author: potatoqualitee

.github/workflows/test-avoidconflicts.yml

@@ -60,19 +60,22 @@ If you need to use both the SqlServer module and dbatools.library in the same se
 
 ### Usage
 
-Import the SqlServer module first, then import dbatools.library with `-AvoidConflicts`:
+Import the SqlServer module first, then import dbatools.library **directly** with `-AvoidConflicts`:
 
 ```powershell
 # Import SqlServer module first
 Import-Module SqlServer
 
 # Then import dbatools.library with -AvoidConflicts
+# IMPORTANT: Use -ArgumentList $true (NOT a hashtable)
 Import-Module dbatools.library -ArgumentList $true
 ```
 
+> **Note:** You must import `dbatools.library` directly, not via the `dbatools` module. The `-ArgumentList` parameter cannot be passed through module dependencies. If you need to use the full `dbatools` module with SqlServer, import `dbatools.library` first with `-ArgumentList $true`, then import `dbatools`.
+
 When `-AvoidConflicts` is enabled, dbatools.library will:
 - Check if each assembly is already loaded in the current session
-- Skip loading any assemblies that are already present
+- Skip loading any assemblies that are already present (including Microsoft.Data.SqlClient)
 - Load only the assemblies that are missing
 
 ### Examples
@@ -83,6 +86,13 @@ Import-Module SqlServer
 Import-Module dbatools.library -ArgumentList $true
 ```
 
+**Using with the full dbatools module:**
+```powershell
+Import-Module SqlServer
+Import-Module dbatools.library -ArgumentList $true  # Load library first with AvoidConflicts
+Import-Module dbatools                              # Then load dbatools (will use already-loaded library)
+```
+
 **See what's being skipped with -Verbose:**
 ```powershell
 Import-Module SqlServer
@@ -96,6 +106,18 @@ Import-Module dbatools.library -ArgumentList $true -Verbose
 Import-Module dbatools.library
 ```
 
+### Common Mistakes
+
+❌ **Wrong:** Using a hashtable for ArgumentList
+```powershell
+Import-Module dbatools.library -ArgumentList @{AvoidConflicts = $true}  # This will NOT work
+```
+
+✅ **Correct:** Using a boolean value
+```powershell
+Import-Module dbatools.library -ArgumentList $true  # This works correctly
+```
+
 ### ⚠️ Important: PowerShell Core + Credentials Issue
 
 **If you plan to use SQL Server credentials with PowerShell Core (pwsh), you MUST install to AllUsers scope or grant appropriate permissions.**

README.md

@@ -7,7 +7,7 @@
 #
 @{
     # Version number of this module.
-    ModuleVersion          = '2025.12.21'
+    ModuleVersion          = '2025.12.26'
 
     # ID used to uniquely identify this module
     GUID                   = '00b61a37-6c36-40d8-8865-ac0180288c84'

dbatools.library.psd1

@@ -17,55 +17,26 @@ function Get-DbatoolsLibraryPath {
 $script:libraryroot = Get-DbatoolsLibraryPath
 
 if ($PSVersionTable.PSEdition -ne "Core") {
-    $dir = [System.IO.Path]::Combine($script:libraryroot, "lib")
-    $dir = ("$dir\").Replace('\', '\\')
-
     if (-not ("Redirector" -as [type])) {
         $source = @"
             using System;
-            using System.Linq;
+            using System.IO;
             using System.Reflection;
-            using System.Text.RegularExpressions;
 
             public class Redirector
             {
-                public Redirector()
+                private static string _libPath;
+
+                public Redirector(string libPath)
                 {
+                    _libPath = libPath;
                     this.EventHandler = new ResolveEventHandler(AssemblyResolve);
                 }
 
                 public readonly ResolveEventHandler EventHandler;
 
-                protected Assembly AssemblyResolve(object sender, ResolveEventArgs e)
+                protected static Assembly AssemblyResolve(object sender, ResolveEventArgs e)
                 {
-                    string[] dlls = {
-                        "System.Memory",
-                        "System.Runtime",
-                        "System.Management.Automation",
-                        "System.Runtime.CompilerServices.Unsafe",
-                        "Microsoft.Bcl.AsyncInterfaces",
-                        "System.Text.Json",
-                        "System.Resources.Extensions",
-                        "Microsoft.SqlServer.ConnectionInfo",
-                        "Microsoft.SqlServer.Smo",
-                        "Microsoft.Identity.Client",
-                        "System.Diagnostics.DiagnosticSource",
-                        "Microsoft.IdentityModel.Abstractions",
-                        "Microsoft.Data.SqlClient",
-                        "Microsoft.SqlServer.Types",
-                        "System.Configuration.ConfigurationManager",
-                        "Microsoft.SqlServer.Management.Sdk.Sfc",
-                        "Microsoft.SqlServer.Management.IntegrationServices",
-                        "Microsoft.SqlServer.Replication",
-                        "Microsoft.SqlServer.Rmo",
-                        "System.Private.CoreLib",
-                        "Azure.Core",
-                        "Azure.Identity",
-                        "Microsoft.Data.Tools.Utilities",
-                        "Microsoft.Data.Tools.Schema.Sql",
-                        "Microsoft.SqlServer.TransactSql.ScriptDom"
-                    };
-
                     var requestedName = new AssemblyName(e.Name);
                     var assemblyName = requestedName.Name;
 
@@ -86,13 +57,17 @@ if ($PSVersionTable.PSEdition -ne "Core") {
                         }
                     }
 
-                    // Only load from disk if not already loaded and it's in our list
-                    foreach (string dll in dlls)
+                    // Try to load from our lib folder if the file exists
+                    string dllPath = Path.Combine(_libPath, assemblyName + ".dll");
+                    if (File.Exists(dllPath))
                     {
-                        if (assemblyName == dll)
+                        try
                         {
-                            string filelocation = "$dir" + dll + ".dll";
-                            return Assembly.LoadFrom(filelocation);
+                            return Assembly.LoadFrom(dllPath);
+                        }
+                        catch
+                        {
+                            // Failed to load, return null to let default resolution continue
                         }
                     }
 
@@ -105,10 +80,90 @@ if ($PSVersionTable.PSEdition -ne "Core") {
     }
 
     try {
-        $redirector = New-Object Redirector
+        $libPath = [System.IO.Path]::Combine($script:libraryroot, "lib")
+        $redirector = New-Object Redirector($libPath)
         [System.AppDomain]::CurrentDomain.add_AssemblyResolve($redirector.EventHandler)
     } catch {
-        # unsure
+        Write-Verbose "Could not register Redirector: $_"
+    }
+} else {
+    # PowerShell Core: Use AssemblyLoadContext.Resolving event for version redirection
+    # This handles version mismatches when SqlServer module loads different versions of assemblies
+    # IMPORTANT: Must be implemented in C# because the resolver runs on .NET threads without PowerShell runspaces
+    $dir = [System.IO.Path]::Combine($script:libraryroot, "lib")
+    $dir = ("$dir" + [System.IO.Path]::DirectorySeparatorChar).Replace('\', '\\')
+
+    if (-not ("CoreRedirector" -as [type])) {
+        $coreSource = @"
+            using System;
+            using System.IO;
+            using System.Reflection;
+            using System.Runtime.Loader;
+
+            public class CoreRedirector
+            {
+                private static string _libPath;
+                private static bool _registered = false;
+
+                public static void Register(string libPath)
+                {
+                    if (_registered) return;
+                    _libPath = libPath;
+                    AssemblyLoadContext.Default.Resolving += OnResolving;
+                    _registered = true;
+                }
+
+                private static Assembly OnResolving(AssemblyLoadContext context, AssemblyName assemblyName)
+                {
+                    string name = assemblyName.Name;
+
+                    // First, check if any version of this assembly is already loaded
+                    // This handles version mismatches (e.g., dbatools.dll requesting ConnectionInfo 17.100.0.0 when 17.200.0.0 is loaded)
+                    foreach (var assembly in AppDomain.CurrentDomain.GetAssemblies())
+                    {
+                        try
+                        {
+                            if (assembly.GetName().Name == name)
+                            {
+                                return assembly;
+                            }
+                        }
+                        catch
+                        {
+                            // Some assemblies may throw when accessing GetName()
+                        }
+                    }
+
+                    // Try to load from our lib folder if the file exists
+                    string dllPath = _libPath + name + ".dll";
+                    if (File.Exists(dllPath))
+                    {
+                        try
+                        {
+                            return AssemblyLoadContext.Default.LoadFromAssemblyPath(dllPath);
+                        }
+                        catch
+                        {
+                            // Failed to load, return null to let default resolution continue
+                        }
+                    }
+
+                    return null;
+                }
+            }
+"@
+
+        try {
+            $null = Add-Type -TypeDefinition $coreSource -ReferencedAssemblies 'System.Runtime.Loader'
+        } catch {
+            Write-Verbose "Could not compile CoreRedirector: $_"
+        }
+    }
+
+    try {
+        [CoreRedirector]::Register($dir)
+    } catch {
+        Write-Verbose "Could not register CoreRedirector: $_"
     }
 }
 
@@ -118,11 +173,44 @@ $sqlclient = [System.IO.Path]::Combine($script:libraryroot, "lib", "Microsoft.Da
 # Get loaded assemblies once for reuse (used for AvoidConflicts checks and later assembly loading)
 $script:loadedAssemblies = [System.AppDomain]::CurrentDomain.GetAssemblies()
 
+# Check for incompatible System.ClientModel or SqlClient versions
+# Azure.Core 1.44+ requires System.ClientModel 1.1+ with IPersistableModel.Write() method
+# SqlServer module's SqlClient 5.x includes older System.ClientModel that's incompatible
+$script:hasIncompatibleClientModel = $false
+if ($AvoidConflicts) {
+    # Check if System.ClientModel is already loaded with incompatible version
+    $existingClientModel = $script:loadedAssemblies | Where-Object { $_.GetName().Name -eq 'System.ClientModel' }
+    if ($existingClientModel) {
+        $clientModelVersion = $existingClientModel.GetName().Version
+        # System.ClientModel 1.1.0+ has the required IPersistableModel interface changes
+        if ($clientModelVersion -lt [Version]'1.1.0') {
+            $script:hasIncompatibleClientModel = $true
+            Write-Verbose "Detected incompatible System.ClientModel version $clientModelVersion - will skip Azure.Core and Azure.Identity to avoid MissingMethodException"
+        }
+    }
+
+    # Check if SqlServer's older SqlClient is loaded (which bundles incompatible System.ClientModel)
+    # SqlClient 5.x from SqlServer module uses System.ClientModel 1.0.x
+    # Our Azure.Core requires System.ClientModel 1.1+
+    if (-not $script:hasIncompatibleClientModel) {
+        $existingSqlClient = $script:loadedAssemblies | Where-Object { $_.GetName().Name -eq 'Microsoft.Data.SqlClient' }
+        if ($existingSqlClient) {
+            $sqlClientVersion = $existingSqlClient.GetName().Version
+            # SqlClient 5.x bundles older System.ClientModel; 6.x bundles compatible versions
+            if ($sqlClientVersion.Major -lt 6) {
+                $script:hasIncompatibleClientModel = $true
+                Write-Verbose "Detected SqlClient $sqlClientVersion (pre-6.0) which uses incompatible System.ClientModel - will skip Azure.Core and Azure.Identity to avoid MissingMethodException"
+            }
+        }
+    }
+}
+
 # Check if SqlClient is already loaded when AvoidConflicts is set
 $skipSqlClient = $false
 if ($AvoidConflicts) {
-    $skipSqlClient = $script:loadedAssemblies | Where-Object { $_.GetName().Name -eq 'Microsoft.Data.SqlClient' }
-    if ($skipSqlClient) {
+    $existingAssembly = $script:loadedAssemblies | Where-Object { $_.GetName().Name -eq 'Microsoft.Data.SqlClient' }
+    if ($existingAssembly) {
+        $skipSqlClient = $true
         Write-Verbose "Skipping Microsoft.Data.SqlClient.dll - already loaded"
     }
 }
@@ -210,6 +298,13 @@ foreach ($name in $names) {
         continue
     }
 
+    # Skip Azure.Core and Azure.Identity if System.ClientModel is incompatible
+    # These assemblies depend on System.ClientModel 1.1+ which has breaking API changes
+    if ($script:hasIncompatibleClientModel -and $name -in @('Azure.Core', 'Azure.Identity')) {
+        Write-Verbose "Skipping $name.dll - incompatible System.ClientModel already loaded"
+        continue
+    }
+
     # Check if assembly is already loaded (always check to avoid duplicate loads)
     if ($script:loadedAssemblyNames.Contains("$name,")) {
         if ($AvoidConflicts) {

dbatools.library.psm1

@@ -7,6 +7,25 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [1.1.10] - 2025-12-26
+
+### Added
+- **SQL Server schema inference** - New `CsvSchemaInference` class that analyzes CSV data to determine optimal SQL Server column types. Two modes available:
+  - `InferSchemaFromSample()` - Fast inference from first N rows (default 1000)
+  - `InferSchema()` - Full file scan with progress callback for zero-risk type detection
+- `InferredColumn` class containing column name, SQL data type, max length, nullability, unicode flag, and decimal precision/scale
+- Type detection for: `uniqueidentifier`, `bit`, `int`, `bigint`, `decimal(p,s)`, `datetime2`, `varchar(n)`, `nvarchar(n)`
+- `GenerateCreateTableStatement()` utility to produce SQL DDL from inferred schema
+- `ToColumnTypes()` utility to convert inferred schema to `CsvReaderOptions.ColumnTypes` dictionary
+- Early exit optimization: types are eliminated as values fail validation, reducing unnecessary checks
+- Progress callback support for full-scan mode (fires every ~1% or 10K rows)
+- **MoneyConverter** for SQL Server `money`/`smallmoney` types with support for currency symbols, thousands separators, and accounting format
+- **VectorConverter** for SQL Server 2025 `VECTOR` data type with support for JSON array and comma-separated formats
+
+### Fixed
+- **DecimalConverter scientific notation support** - Changed `NumberStyles` from `Number` to `Float | AllowThousands` to properly parse scientific notation (e.g., `1.2345678E5`)
+- **SQL identifier escaping** - Escape closing brackets in schema, table, and column names to prevent SQL injection in generated `CREATE TABLE` statements
+
 ## [1.1.1] - 2025-12-04
 
 ### Changed

project/Dataplat.Dbatools.Csv/CHANGELOG.md

@@ -7,20 +7,20 @@
 
     <!-- NuGet Package Metadata -->
     <PackageId>Dataplat.Dbatools.Csv</PackageId>
-    <Version>1.1.1</Version>
+    <Version>1.1.10</Version>
     <Authors>Chrissy LeMaire</Authors>
     <Company>Dataplat</Company>
     <Product>Dataplat.Dbatools.Csv</Product>
-    <Description>High-performance CSV reader and writer for .NET. Features streaming IDataReader for SqlBulkCopy, automatic compression (GZip, Deflate, Brotli, ZLib), multi-character delimiters, parallel processing, string interning, and robust error handling. 20%+ faster than LumenWorks CsvReader. From the trusted dbatools project.</Description>
+    <Description>High-performance CSV reader with native IDataReader for SqlBulkCopy - 6x faster than legacy solutions for database imports. Database-first design with culture-aware parsing, intelligent null handling, and robust support for messy real-world data (duplicate headers, field mismatches). Features automatic compression (GZip, Deflate, Brotli, ZLib), progress reporting with rows/second metrics, and cancellation support. From the trusted dbatools project.</Description>
     <Copyright>Copyright (c) 2025 Chrissy LeMaire</Copyright>
     <PackageTags>csv;parser;reader;writer;datareader;idatareader;sqlbulkcopy;compression;gzip;brotli;dbatools;high-performance;parallel</PackageTags>
     <PackageLicenseExpression>MIT</PackageLicenseExpression>
-    <PackageProjectUrl>https://github.com/dataplat/dbatools.library</PackageProjectUrl>
+    <PackageProjectUrl>https://dataplat.dbatools.io/csv</PackageProjectUrl>
     <RepositoryUrl>https://github.com/dataplat/dbatools.library</RepositoryUrl>
     <RepositoryType>git</RepositoryType>
     <RepositoryBranch>main</RepositoryBranch>
     <PackageReadmeFile>README.md</PackageReadmeFile>
-    <PackageReleaseNotes>Initial release with high-performance CSV parsing, parallel processing support, and comprehensive edge case handling.</PackageReleaseNotes>
+    <PackageReleaseNotes>v1.1.10: SQL Server schema inference - auto-detect column types (int, bigint, decimal, datetime2, bit, uniqueidentifier, varchar/nvarchar). v1.1.5: Updated package metadata and URL. v1.1.1: ~25% performance improvement for all-columns reads. v1.1.0: Added CancellationToken and progress reporting support.</PackageReleaseNotes>
     <PublishRepositoryUrl>true</PublishRepositoryUrl>
     <EmbedUntrackedSources>true</EmbedUntrackedSources>
     <IncludeSymbols>false</IncludeSymbols>