diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index edce4ab3..18e285ca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,12 +35,14 @@ jobs: echo "BuildBranch=nightly-builds" >> "$GITHUB_ENV" echo "BuildBranch=nightly-builds" >> $env:GITHUB_ENV + # Nightly native libraries: artifacts.duckdb.org//, where 'latest' is DuckDB's main branch. + # Tracks the v2.0 release branch until DuckDB 2.0 ships; switch back to 'latest' after that. - name: Set Env if: startsWith(github.ref_name, 'nightly') || startsWith(env.BuildBranch, 'nightly') run: | - echo "DuckDbArtifactRoot=https://artifacts.duckdb.org/latest" >> "$GITHUB_ENV" + echo "DuckDbArtifactRoot=https://artifacts.duckdb.org/v2.0-cyanoptera" >> "$GITHUB_ENV" echo "NightlyBuild=true" >> "$GITHUB_ENV" - echo "DuckDbArtifactRoot=https://artifacts.duckdb.org/latest" >> $env:GITHUB_ENV + echo "DuckDbArtifactRoot=https://artifacts.duckdb.org/v2.0-cyanoptera" >> $env:GITHUB_ENV echo "NightlyBuild=true" >> $env:GITHUB_ENV - name: Checkout diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000..c038c51f --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,187 @@ +# AGENTS.md + +This file provides guidance to coding agents working with code in this repository. + +DuckDB.NET is an ADO.NET provider and low-level bindings for DuckDB. The solution (`DuckDB.NET.slnx`) has five +projects: `Bindings` (P/Invoke over the DuckDB C API), `Data` (the ADO.NET provider), `Test` (xUnit), +`Samples` and `Benchmarks` (BenchmarkDotNet). The libraries target `net8.0;net10.0`; the test project targets +**net8.0 only**, so `dotnet test -f net10.0` silently runs nothing. The SDK is pinned by `global.json` (10.0.2xx). +User documentation lives in a separate docfx repository (`S:\src\DuckDB.NET-Docs`, published at https://duckdb.net). + +## Commands + +```bash +# Build everything the way CI does (Full = download native libraries for all platforms) +dotnet build --configuration Release /p:BuildType=Full + +# Run all tests (CI also adds /p:CollectCoverage=true /p:CoverletOutputFormat=lcov) +dotnet test DuckDB.NET.Test/Test.csproj --configuration Release /p:BuildType=Full + +# One class, one test, or everything except the slow VARINT parameter tests +dotnet test DuckDB.NET.Test/Test.csproj --filter "FullyQualifiedName~DuckDB.NET.Test.TableFunctionTests" +dotnet test DuckDB.NET.Test/Test.csproj --filter "FullyQualifiedName~DuckDB.NET.Test.TableFunctionTests.TestTableFunction" +dotnet test DuckDB.NET.Test/Test.csproj --filter "FullyQualifiedName!~VarintTest" + +# Packages: the plain packages need a separately installed native library; /p:BuildType=Full produces the .Full variants +dotnet pack DuckDB.NET.Data/Data.csproj --configuration Release /p:BuildType=Full + +# Benchmarks +dotnet run -c Release --project DuckDB.NET.Benchmarks +``` + +### Native libraries + +- The `DownloadNativeLibs` target in `Bindings.csproj` (with `DownloadNativeLibs.targets`) downloads the DuckDB + release zips into `DuckDB.NET.Bindings/obj/runtimes//native`. The test project always triggers this download, + whatever `BuildType` is. +- **The cache is version-blind:** a platform is downloaded only if its folder is missing. After changing the DuckDB + version, delete `DuckDB.NET.Bindings/obj/runtimes`, rebuild, and confirm the version with `SELECT version()`. +- **Nightly builds:** `/p:NightlyBuild=true /p:DuckDbArtifactRoot=https://artifacts.duckdb.org/` downloads + DuckDB's `duckdb-shared-libs-.tar.gz` instead. `` is a DuckDB branch name, and `latest` means + DuckDB's `main`. CI does this on the schedule: it uses the workflow file from `develop`, builds the code of the + `nightly-builds` branch, and currently tracks `v2.0-cyanoptera` until DuckDB 2.0 ships. + +## DuckDB version and releases + +- **Current target: DuckDB v1.5.6.** The version is set in `DuckDB.NET.Bindings/Bindings.csproj` + (`DuckDbArtifactRoot`). The "Updated to DuckDB vX" line in the `PackageReleaseNotes` of both `Bindings.csproj` and + `Data.csproj` also needs updating. Every `LibraryImport` names its `EntryPoint`, so after an update, compare those + names with the library's exports to catch removed functions. +- **Package version:** comes from git tags through GitVersion (`GitVersion.yml`). DuckDB.NET versions follow the + DuckDB version (tag `1.5.6`). +- **Release steps:** + 1. Update the release notes in both project files. + 2. Merge `develop` into `main` and tag. + 3. Wait for the "Build" run on `main` to succeed. + 4. Start the `NuGet` workflow (`.github/workflows/NuGet.yml`, manual) on `main`. It publishes the + `nugetPackages-main` artifact of the latest *successful* main build, so starting it earlier publishes an old + build. Pushes to `develop` publish to GitHub Packages only. + +## Architecture + +### Bindings (`DuckDB.NET.Bindings`) + +- **`NativeMethods`:** a partial class split into one file per C API area under `NativeMethods/` (Appender, Arrow, + DataChunks, PreparedStatements, Query, ScalarFunction, StreamingResult, TableFunction, Value, Vectors, ...). Each + declaration is a `LibraryImport` with an explicit `EntryPoint` and the `DuckDbLibrary` constant; some hot paths + use `[SuppressGCTransition]`. +- **Lifetimes:** native objects are `SafeHandle`s in `DuckDBWrapperObjects.cs` (database, connection, prepared + statement, data chunk, logical type, value, ...). The exception is **`DuckDBResult`**, a plain struct in + `DuckDBNativeObjects.cs` with no finalizer. Every result must be `Close()`d exactly once, or it leaks permanently. +- **Strings:** returned strings use the marshallers in `DuckDBStringMarshallers.cs`. Use + `DuckDBOwnedStringMarshaller` when DuckDB owns the memory, and `DuckDBCallerOwnedStringMarshaller` when the caller + must `duckdb_free` it. + +### Connections (`DuckDB.NET.Data/Connection`) + +- **Pooling:** `DuckDBConnection` gets its native connection from the static `ConnectionManager`. It caches one + `FileReference` (native database plus a reference count) per file. Returning the last connection disposes the + database and removes the cache entry. +- **Data sources:** + - `:memory:` gives every connection its own private database; + - `:memory:?cache=shared` shares one; + - `md:?motherduck_token=...` connects to MotherDuck. +- **Configuration:** `DuckDBConnectionStringBuilder` normalizes `Data Source`/`DataSource` and exposes DuckDB + configuration options as typed properties. +- **Why not DuckDB's own instance cache:** `duckdb_get_or_create_from_cache` fails under concurrent opens of the same + path (duckdb/duckdb#22277), so `ConnectionManager` does the pooling itself. + +### Commands and results + +- **Execution:** `DuckDBCommand` calls `PreparedStatement.PrepareMultiple`, an iterator that extracts the statements + and prepares and executes one statement per `MoveNext()`. Parameters are converted by `DuckDBTypeMap` and + `ClrToDuckDBConverter` (positional and named). +- **Result modes:** `UseStreamingMode` **defaults to `false`**, so results are materialized. `true` uses + `duckdb_execute_prepared_streaming` and `duckdb_stream_fetch_chunk`. +- **Errors:** every native result error goes through `DuckDBResultExtensions.ThrowOnError`. An interrupt becomes + `OperationCanceledException`. An exception thrown by a C# UDF is attached as `InnerException` through + `UdfExceptionStore`, which is keyed by connection ID. Connection IDs repeat across databases, so parallel tests on + different databases can see each other's stored exception. This is a known flaky test source. +- **`DuckDBDataReader`:** reads one chunk at a time and creates readers through `VectorDataReaderFactory`, one per + column. Composite readers (struct, list, map, decimal) hold child readers. After the first chunk, readers are + reused through `VectorDataReaderBase.Reset(IntPtr vector)`. A new `VectorDataReaderBase` subclass must be + registered in the factory, and must override `Reset` if it has child readers. +- **Invariants in `DuckDBDataReader`, each guarded by a regression test:** + - A NULL chunk from a streaming fetch means either the end of the results or an error. Check the result's error + before treating it as the end. + - Reset the row counters *before* fetching, so a failed fetch cannot leave accessors reading the freed chunk. + - `NextResult()` closes the previous result: its chunk first, then the result. + - `Close()` releases the reader's own objects and statements *before* `CommandBehavior.CloseConnection` closes the + connection. Otherwise a still-alive prepared statement keeps the native database alive after `ConnectionManager` + disposes it, and the next open creates a second instance on the same file. That corrupts data on Linux and + macOS, and fails with "file in use" on Windows. +- **Arrow:** `ExecuteArrowStream` and `ExecuteArrowBatchesAsync` stream results as Apache Arrow record batches + (`Arrow/DuckDBArrowArrayStream.cs`), using DuckDB's Arrow C Data Interface. + +### Writing data: appenders and UDF output + +- **Writers:** `VectorDataWriterFactory` (`DataChunk/Writer`) creates the column writers used by the appender and by + scalar and table function output. `CollectionVectorDataWriter` holds the shared collection logic; + `ListVectorDataWriter` (running offsets, growth) and `ArrayVectorDataWriter` (fixed size, row-based positions) + are separate because DuckDB stores the two differently. +- **`DuckDBAppender`:** the low-level appender (`connection.CreateAppender(table)`, then `CreateRow().AppendValue(...)`, + or the allocation-free `AppendRow`). It buffers rows in a data chunk of `VectorSize` (2048) rows. It does **no type + checking** and writes the raw bytes of the .NET type it is given; the docs call this out. A row is written only once + every column has a value. Moving on from an incomplete row discards it and faults the appender. +- **`DuckDBMappedAppender`:** the type-checked alternative (`CreateAppender(table)`). A + `DuckDBAppenderMap` subclass declares the columns with `Map`, `DefaultValue` and `NullValue`, and type + mismatches are reported when the appender is created. See `AppenderMap-Usage.md`. +- **UDFs:** scalar functions (`DuckDBConnection.ScalarFunction*.cs`) and table functions + (`DuckDBConnection.TableFunction*.cs`) are registered through generic overloads for different parameter counts. + Callbacks are pinned with `GCHandle`. A table function creates its data enumerator per scan init, not per bind, + because DuckDB re-initializes scans without re-binding, for example in recursive CTEs. + +## Coding style + +There is no `.editorconfig`, so match the surrounding code: +- **Layout:** file-scoped namespaces (`namespace DuckDB.NET.Data;`) and 4-space indentation. +- **Naming:** `PascalCase` for types, methods and properties; `camelCase` for locals and fields, with no underscore + prefix. +- **Language features:** `LangVersion` is `latest`. Use `var` when the type is obvious. Primary constructors and + collection expressions (`[]`) are already used. +- **Nullable reference types:** enabled in `Bindings`, `Data` and `Benchmarks`, but **disabled in the test + project**. Don't put `?` on reference types in tests (`string?`, `List`); it produces CS8632 warnings. + `int?` and other nullable value types are fine. +- **Files:** files are UTF-8, and about half start with a byte-order mark. The checkout uses CRLF line endings + (`core.autocrlf`). Edits made with scripts must keep the file's byte-order mark as it is and keep CRLF throughout: + no stray LF, no doubled CR, and no lone CR at the end of a file. Otherwise git can treat the file as binary. + +## Tests + +Test classes take the per-class `DuckDBDatabaseFixture` (a shared in-memory connection) through +`DuckDBTestBase`. Test classes run in parallel, each with its own database. `connection.ExecuteNonQuery(sql)` is +an extension in `Extensions/DbConnectionExtension.cs`. + +```csharp +public class MyTests(DuckDBDatabaseFixture db) : DuckDBTestBase(db) +{ + [Fact] + public void TestSomething() + { + Connection.ExecuteNonQuery("CREATE TABLE ..."); + } +} +``` + +Tests that close or reconfigure a connection, or need a file database, should create their own `DuckDBConnection` +instead of using the shared fixture connection. + +Rules for every new or changed test (both have caused tests that passed locally and failed CI): +- **Match `AppendValue` argument types to the column types exactly.** A bare integer literal such as + `AppendValue(1)` binds to a 1-byte overload, and so does `AppendValue((int)1)`, because it is still a constant. + Written into an INTEGER column, that stores garbage. Use `(int?)1`, a typed variable, or the exact CLR type of + the column. +- **To check row order, use `ORDER BY rowid`** instead of adding an ordering column. +- **Run a new test at least 5 times before committing.** One pass proves nothing. + +**Known flaky failure:** with DuckDB 1.5.x, a later-chunk error test can fail with `OperationCanceledException`. +A worker-thread error sets the interrupt flag, and a streaming fetch can report that flag instead of the real error. +DuckDB fixes this in 2.0. Re-run the job; don't work around it. + +## Known debugging issue + +While debugging code that uses DuckDB.NET, `System.AccessViolationException` can appear because the debugger +interacts with native memory during marshalling. For the workaround, see +https://youtrack.jetbrains.com/issue/RIDER-114126. + +Building DuckDB extensions in C# is a separate project: https://github.com/Giorgi/DuckDB.ExtensionKit diff --git a/DuckDB.NET.Bindings/DuckDBHugeInt.cs b/DuckDB.NET.Bindings/DuckDBHugeInt.cs index 2215f5ca..f097910d 100644 --- a/DuckDB.NET.Bindings/DuckDBHugeInt.cs +++ b/DuckDB.NET.Bindings/DuckDBHugeInt.cs @@ -7,7 +7,8 @@ public readonly struct DuckDBHugeInt { private static readonly BigInteger Base = BigInteger.Pow(2, 64); - public static BigInteger HugeIntMinValue { get; } = BigInteger.Parse("-170141183460469231731687303715884105727"); + // DuckDB's HUGEINT is a 128-bit two's complement integer: -2^127 to 2^127 - 1. + public static BigInteger HugeIntMinValue { get; } = BigInteger.Parse("-170141183460469231731687303715884105728"); public static BigInteger HugeIntMaxValue { get; } = BigInteger.Parse("170141183460469231731687303715884105727"); public DuckDBHugeInt(BigInteger value) diff --git a/DuckDB.NET.Bindings/DuckDBInterval.cs b/DuckDB.NET.Bindings/DuckDBInterval.cs index ec832c2d..ebb48c94 100644 --- a/DuckDB.NET.Bindings/DuckDBInterval.cs +++ b/DuckDB.NET.Bindings/DuckDBInterval.cs @@ -5,7 +5,6 @@ namespace DuckDB.NET.Native; [StructLayout(LayoutKind.Sequential)] public readonly struct DuckDBInterval(int months, int days, ulong micros) { - private const ulong MillisecondsByDay = (ulong)(24 * 60 * 60 * 1e6); public int Months { get; } = months; public int Days { get; } = days; @@ -24,48 +23,24 @@ public bool TryConvert([NotNullWhen(true)] out TimeSpan? timeSpan) return exception is null; } + // DuckDB's micros is signed (duckdb_interval.micros is int64_t), and Days can be negative too. Micros + // exposes the value as ulong, so it is read and written as its two's complement. private static (TimeSpan?, Exception?) ToTimeSpan(DuckDBInterval interval) { - if (interval.Months > 0) + if (interval.Months != 0) { - return (null, new ArgumentOutOfRangeException(nameof(interval), $"Cannot convert a value of type {nameof(DuckDBInterval)} to type {nameof(TimeSpan)} when the attribute 'Months' is greater or equal to 1")); + return (null, new ArgumentOutOfRangeException(nameof(interval), $"Cannot convert a value of type {nameof(DuckDBInterval)} to type {nameof(TimeSpan)} when the attribute 'Months' is not 0")); } - var days = 0; - var micros = interval.Micros; - - if (interval.Micros >= MillisecondsByDay) - { - days = Convert.ToInt32(Math.Floor((double)(interval.Micros / MillisecondsByDay))); - if (days > int.MaxValue - interval.Days) - { - return (null, new ArgumentOutOfRangeException(nameof(interval), $"Cannot convert a value of type {nameof(DuckDBInterval)} to type {nameof(TimeSpan)} when the total days value is larger than {int.MaxValue}")); - } - - if (days > 0) - { - micros = interval.Micros - ((ulong)days * MillisecondsByDay); - } - days = interval.Days + days; - } - else - { - days = interval.Days; - } - - if (micros * 10 > long.MaxValue) - { - return (null, new ArgumentOutOfRangeException(nameof(interval), $"Cannot convert a value of type {nameof(DuckDBInterval)} to type {nameof(TimeSpan)} when the value of microseconds is larger than {long.MaxValue / 10}")); - } - - if ((ulong)days * MillisecondsByDay * 100 + micros * 10 > long.MaxValue) + var ticks = (Int128)interval.Days * TimeSpan.TicksPerDay + (Int128)unchecked((long)interval.Micros) * TimeSpan.TicksPerMicrosecond; + if (ticks > long.MaxValue || ticks < long.MinValue) { - return (null, new ArgumentOutOfRangeException(nameof(interval), $"Cannot convert a value of type {nameof(DuckDBInterval)} to type {nameof(TimeSpan)} when the value of total microseconds is larger than {long.MaxValue}")); + return (null, new ArgumentOutOfRangeException(nameof(interval), $"Cannot convert a value of type {nameof(DuckDBInterval)} to type {nameof(TimeSpan)} when the total value is outside the range of {nameof(TimeSpan)}")); } - return (new TimeSpan(days, 0, 0, 0) + new TimeSpan((long)micros * 10), null); + return (new TimeSpan((long)ticks), null); } private static DuckDBInterval FromTimeSpan(TimeSpan timeSpan) - => new(0, timeSpan.Days, Convert.ToUInt64(timeSpan.Ticks / 10 - new TimeSpan(timeSpan.Days, 0, 0, 0).Ticks / 10)); + => new(0, timeSpan.Days, unchecked((ulong)(timeSpan.Ticks % TimeSpan.TicksPerDay / TimeSpan.TicksPerMicrosecond))); } \ No newline at end of file diff --git a/DuckDB.NET.Bindings/DuckDBWrapperObjects.cs b/DuckDB.NET.Bindings/DuckDBWrapperObjects.cs index 343074cf..50736017 100644 --- a/DuckDB.NET.Bindings/DuckDBWrapperObjects.cs +++ b/DuckDB.NET.Bindings/DuckDBWrapperObjects.cs @@ -167,7 +167,11 @@ public T GetValue() DuckDBType.Varchar => Cast(NativeMethods.Value.DuckDBGetVarchar(this)), + // DATE and TIME can be read as the .NET types or as the provider's own types. Cast reinterprets the value + // without a check, so the value has to be produced as exactly the requested type. + DuckDBType.Date when typeof(T) == typeof(DuckDBDateOnly) => Cast(DuckDBDateOnly.FromDuckDBDate(NativeMethods.Value.DuckDBGetDate(this))), DuckDBType.Date => Cast((DateOnly)DuckDBDateOnly.FromDuckDBDate(NativeMethods.Value.DuckDBGetDate(this))), + DuckDBType.Time when typeof(T) == typeof(DuckDBTimeOnly) => Cast(NativeMethods.DateTimeHelpers.DuckDBFromTime(NativeMethods.Value.DuckDBGetTime(this))), DuckDBType.Time => Cast((TimeOnly)NativeMethods.DateTimeHelpers.DuckDBFromTime(NativeMethods.Value.DuckDBGetTime(this))), DuckDBType.TimeTz => Cast(GetTimeTzValue()), DuckDBType.Interval => Cast((TimeSpan)NativeMethods.Value.DuckDBGetInterval(this)), diff --git a/DuckDB.NET.Data/Arrow/DuckDBArrowArrayStream.cs b/DuckDB.NET.Data/Arrow/DuckDBArrowArrayStream.cs index 39d71184..afbe72e1 100644 --- a/DuckDB.NET.Data/Arrow/DuckDBArrowArrayStream.cs +++ b/DuckDB.NET.Data/Arrow/DuckDBArrowArrayStream.cs @@ -19,6 +19,7 @@ internal sealed class DuckDBArrowArrayStream : IArrowArrayStream private readonly DuckDBArrowOptions arrowOptions; private readonly bool streaming; private bool disposed; + private bool ended; public Schema Schema { get; } @@ -112,6 +113,13 @@ private unsafe Schema BuildSchema() return new ValueTask(Task.FromCanceled(cancellationToken)); } + // DuckDB closes a streaming result when a fetch reaches its end, and every later fetch then fails with a + // "closed pending query result" error. Stop fetching after a clean end, so that error is not reported. + if (ended) + { + return new ValueTask((RecordBatch?)null); + } + var chunk = streaming ? NativeMethods.StreamingResult.DuckDBStreamFetchChunk(result) : NativeMethods.Query.DuckDBFetchChunk(result); @@ -125,6 +133,7 @@ private unsafe Schema BuildSchema() result.ThrowOnError(connection); } + ended = true; return new ValueTask((RecordBatch?)null); } diff --git a/DuckDB.NET.Data/DataChunk/Writer/NumericVectorDataWriter.cs b/DuckDB.NET.Data/DataChunk/Writer/NumericVectorDataWriter.cs index eb028dca..d90370fd 100644 --- a/DuckDB.NET.Data/DataChunk/Writer/NumericVectorDataWriter.cs +++ b/DuckDB.NET.Data/DataChunk/Writer/NumericVectorDataWriter.cs @@ -4,5 +4,7 @@ internal sealed unsafe class NumericVectorDataWriter(IntPtr vector, void* vector { internal override bool AppendNumeric(T value, ulong rowIndex) => AppendValueInternal(value, rowIndex); - internal override bool AppendBigInteger(BigInteger value, ulong rowIndex) => AppendValueInternal(new DuckDBHugeInt(value), rowIndex); + internal override bool AppendBigInteger(BigInteger value, ulong rowIndex) => ColumnType == DuckDBType.UnsignedHugeInt + ? AppendValueInternal(new DuckDBUHugeInt(value), rowIndex) + : AppendValueInternal(new DuckDBHugeInt(value), rowIndex); } diff --git a/DuckDB.NET.Data/DuckDBAppenderRow.cs b/DuckDB.NET.Data/DuckDBAppenderRow.cs index 2d8cfedb..84b20f24 100644 --- a/DuckDB.NET.Data/DuckDBAppenderRow.cs +++ b/DuckDB.NET.Data/DuckDBAppenderRow.cs @@ -1,4 +1,6 @@ using DuckDB.NET.Data.DataChunk.Writer; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; namespace DuckDB.NET.Data; @@ -194,7 +196,9 @@ private DuckDBAppenderRow AppendValueInternalClass(T? value) where T : class private unsafe IDuckDBAppenderRow AppendSpan(Span val) { - if (val == null) + // A null byte[] or a default span has no memory at all and writes NULL. An empty array still has memory and is + // an empty BLOB, so IsEmpty would be wrong here. + if (Unsafe.IsNullRef(ref MemoryMarshal.GetReference(val))) { return AppendNullValue(); } diff --git a/DuckDB.NET.Data/DuckDBClientFactory.cs b/DuckDB.NET.Data/DuckDBClientFactory.cs index bb8c1f71..7dad7438 100644 --- a/DuckDB.NET.Data/DuckDBClientFactory.cs +++ b/DuckDB.NET.Data/DuckDBClientFactory.cs @@ -24,16 +24,15 @@ private DuckDBClientFactory() { } #region Methods - public override DbCommand CreateCommand() => new DuckDBCommand(); + // DuckDB.NET has no command builder or data adapter. CreateCommandBuilder and CreateDataAdapter are not + // overridden, so they return null and CanCreateCommandBuilder and CanCreateDataAdapter report false. - public override DbCommandBuilder CreateCommandBuilder() => throw new NotImplementedException(); + public override DbCommand CreateCommand() => new DuckDBCommand(); public override DbConnection CreateConnection() => new DuckDBConnection(); public override DbConnectionStringBuilder CreateConnectionStringBuilder() => new DuckDBConnectionStringBuilder(); - public override DbDataAdapter CreateDataAdapter() => throw new NotImplementedException(); - public override DbParameter CreateParameter() => new DuckDBParameter(); #endregion diff --git a/DuckDB.NET.Data/DuckDBCommand.cs b/DuckDB.NET.Data/DuckDBCommand.cs index 6d602614..498727e9 100644 --- a/DuckDB.NET.Data/DuckDBCommand.cs +++ b/DuckDB.NET.Data/DuckDBCommand.cs @@ -12,6 +12,7 @@ public class DuckDBCommand : DbCommand { private DuckDBConnection? connection; private readonly DuckDBParameterCollection parameters = new(); + private bool readerOpen; protected override DbTransaction? DbTransaction { get; set; } protected override DbParameterCollection DbParameterCollection => parameters; @@ -19,7 +20,7 @@ public class DuckDBCommand : DbCommand public new virtual DuckDBParameterCollection Parameters => parameters; public override int CommandTimeout { get; set; } - public override CommandType CommandType { get; set; } + public override CommandType CommandType { get; set; } = CommandType.Text; public override bool DesignTimeVisible { get; set; } public override UpdateRowSource UpdatedRowSource { get; set; } @@ -39,7 +40,8 @@ public override string CommandText get; set { - // TODO: We shouldn't be able to change the CommandText when the command is in execution (requires CommandState implementation) + if (readerOpen) + throw new InvalidOperationException("CommandText cannot be changed while a reader is open."); field = value ?? string.Empty; } } = string.Empty; @@ -47,7 +49,12 @@ public override string CommandText protected override DbConnection? DbConnection { get => connection; - set => connection = (DuckDBConnection?)value; + set + { + if (readerOpen) + throw new InvalidOperationException("Connection cannot be changed while a reader is open."); + connection = (DuckDBConnection?)value; + } } public DuckDBCommand() @@ -64,7 +71,11 @@ public DuckDBCommand(string commandText, DuckDBConnection connection) Connection = connection; } - public override void Cancel() => connection?.NativeConnection.Interrupt(); + public override void Cancel() + { + if (connection?.State == ConnectionState.Open) + connection.NativeConnection.Interrupt(); + } public override int ExecuteNonQuery() { @@ -73,15 +84,19 @@ public override int ExecuteNonQuery() var results = PreparedStatement.PreparedStatement.PrepareMultiple(connection!.NativeConnection, CommandText, parameters, UseStreamingMode); var count = 0; + var returnedRows = false; foreach (var result in results) { var current = result; - count += (int)NativeMethods.Query.DuckDBRowsChanged(ref current); + if (NativeMethods.Query.DuckDBResultReturnType(current) == DuckDBResultType.QueryResult) + returnedRows = true; + else + count += (int)NativeMethods.Query.DuckDBRowsChanged(ref current); result.Close(); } - return count; + return returnedRows ? -1 : count; } public override object? ExecuteScalar() @@ -105,12 +120,20 @@ public override int ExecuteNonQuery() protected override DbDataReader ExecuteDbDataReader(CommandBehavior behavior) { EnsureConnectionOpen(); + if (readerOpen) + throw new InvalidOperationException("A reader is already open for this command."); var results = PreparedStatement.PreparedStatement.PrepareMultiple(connection!.NativeConnection, CommandText, parameters, UseStreamingMode); - - var reader = new DuckDBDataReader(this, results, behavior); - - return reader; + readerOpen = true; + try + { + return new DuckDBDataReader(this, results, behavior); + } + catch + { + readerOpen = false; + throw; + } } /// @@ -164,17 +187,31 @@ public async IAsyncEnumerable ExecuteArrowBatchesAsync([EnumeratorC } } - public override void Prepare() { } + public override void Prepare() + { + EnsureConnectionOpen(); + if (string.IsNullOrWhiteSpace(CommandText)) + throw new InvalidOperationException("CommandText is required."); + } protected override DbParameter CreateDbParameter() => new DuckDBParameter(); internal void CloseConnection() => Connection!.Close(); + internal void ReaderClosed() => readerOpen = false; + private void EnsureConnectionOpen([CallerMemberName] string operation = "") { if (Connection is null || Connection.State != ConnectionState.Open) { throw new InvalidOperationException($"{operation} requires an open connection"); } + if (string.IsNullOrWhiteSpace(CommandText)) + throw new InvalidOperationException("CommandText is required."); + var openConnection = connection!; + if (DbTransaction is not null && !ReferenceEquals(DbTransaction.Connection, openConnection)) + throw new InvalidOperationException("The command transaction belongs to a different connection."); + if (openConnection.Transaction is not null && !ReferenceEquals(DbTransaction, openConnection.Transaction)) + throw new InvalidOperationException("The command transaction does not match the active connection transaction."); } -} \ No newline at end of file +} diff --git a/DuckDB.NET.Data/DuckDBConnection.cs b/DuckDB.NET.Data/DuckDBConnection.cs index 9f20e8ad..48041077 100644 --- a/DuckDB.NET.Data/DuckDBConnection.cs +++ b/DuckDB.NET.Data/DuckDBConnection.cs @@ -38,7 +38,31 @@ public DuckDBConnection(string connectionString) [AllowNull] [DefaultValue("")] - public override string ConnectionString { get; set; } + public override string ConnectionString + { + get; + set + { + if (connectionState != ConnectionState.Closed) + throw new InvalidOperationException("The connection string cannot be changed while the connection is open."); + + var normalized = value ?? string.Empty; + if (!string.IsNullOrWhiteSpace(normalized)) + { + try + { + _ = DuckDBConnectionStringBuilder.Parse(normalized); + } + catch (Exception ex) when (ex is InvalidOperationException or ArgumentException) + { + throw new ArgumentException("The connection string is invalid.", nameof(value), ex); + } + } + + field = normalized; + parsedConnection = null; + } + } public override string Database { @@ -49,7 +73,7 @@ public override string Database return ParsedConnection.DataSource; } - throw new InvalidOperationException("Connection string must be specified."); + return string.Empty; } } @@ -62,7 +86,7 @@ public override string DataSource return ParsedConnection!.DataSource; } - throw new InvalidOperationException("Connection string must be specified."); + return string.Empty; } } @@ -72,7 +96,9 @@ public override string DataSource public DuckDBNativeConnection NativeConnection => connectionReference?.NativeConnection ?? throw new InvalidOperationException("The DuckDBConnection must be open to access the native connection."); - public override string ServerVersion => NativeMethods.Startup.DuckDBLibraryVersion(); + public override string ServerVersion => connectionState == ConnectionState.Open + ? NativeMethods.Startup.DuckDBLibraryVersion() + : throw new InvalidOperationException("The connection must be open."); public override ConnectionState State => connectionState; @@ -85,7 +111,7 @@ public override void Close() { if (connectionState == ConnectionState.Closed) { - throw new InvalidOperationException("Connection is already closed."); + return; } if (connectionReference is not null) //Should always be the case @@ -142,8 +168,7 @@ protected override DbCommand CreateDbCommand() { return new DuckDBCommand { - Connection = this, - Transaction = Transaction + Connection = this }; } @@ -217,7 +242,17 @@ public DuckDBMappedAppender CreateAppender(string? catalog, st where TMap : Mapping.DuckDBAppenderMap, new() { var appender = CreateAppender(catalog, schema, table); - return new DuckDBMappedAppender(appender); + + try + { + return new DuckDBMappedAppender(appender); + } + catch + { + // The mapping did not pass validation, so nobody else gets to dispose the appender. + appender.Dispose(); + throw; + } } protected override void Dispose(bool disposing) diff --git a/DuckDB.NET.Data/DuckDBConnectionStringBuilder.cs b/DuckDB.NET.Data/DuckDBConnectionStringBuilder.cs index 2f603e48..aa3e8aa8 100644 --- a/DuckDB.NET.Data/DuckDBConnectionStringBuilder.cs +++ b/DuckDB.NET.Data/DuckDBConnectionStringBuilder.cs @@ -180,7 +180,7 @@ public string DataSource public override bool ContainsKey(string keyword) => base.ContainsKey(Normalize(keyword)); /// - public override bool TryGetValue(string keyword, out object value) => base.TryGetValue(Normalize(keyword), out value); + public override bool TryGetValue(string keyword, [NotNullWhen(true)] out object? value) => base.TryGetValue(Normalize(keyword), out value); /// public override bool Remove(string keyword) => base.Remove(Normalize(keyword)); diff --git a/DuckDB.NET.Data/DuckDBDataReader.cs b/DuckDB.NET.Data/DuckDBDataReader.cs index 02023e4e..42b12093 100644 --- a/DuckDB.NET.Data/DuckDBDataReader.cs +++ b/DuckDB.NET.Data/DuckDBDataReader.cs @@ -21,6 +21,7 @@ public class DuckDBDataReader : DbDataReader private bool closed; private bool hasRows; private bool streamingResult; + private bool streamEnded; private long currentChunkIndex; private readonly IEnumerator resultEnumerator; @@ -67,6 +68,7 @@ private bool InitNextReader() currentResult.Close(); currentChunkIndex = 0; + streamEnded = false; currentResult = result; columnMapping = []; @@ -95,6 +97,13 @@ private bool InitNextReader() private bool InitChunkData() { + // DuckDB closes a streaming result when a fetch reaches its end, and every later fetch then fails with a + // "closed pending query result" error. Stop fetching after a clean end, so that error is not reported. + if (streamEnded) + { + return false; + } + var canReuse = vectorReaders.Length > 0; // Reset before fetching: if the fetch throws, the readers still point into the chunk freed here, @@ -107,7 +116,9 @@ private bool InitChunkData() if (streamingResult && currentChunk.IsInvalid) { + // No chunk and no error is the clean end of the stream; an error from producing the chunk still throws. currentResult.ThrowOnError(nativeConnection); + streamEnded = true; } currentChunkRowCount = NativeMethods.DataChunks.DuckDBDataChunkGetSize(currentChunk); @@ -408,6 +419,7 @@ public override void Close() // Finish the reader's own cleanup first, so an exception from closing the connection cannot skip it. closed = true; resultEnumerator.Dispose(); + command.ReaderClosed(); if (behavior == CommandBehavior.CloseConnection) { @@ -429,4 +441,4 @@ private static void ThrowNoRowRead() { throw new InvalidOperationException("No row has been read"); } -} \ No newline at end of file +} diff --git a/DuckDB.NET.Data/DuckDBMappedAppender.cs b/DuckDB.NET.Data/DuckDBMappedAppender.cs index 25a3444f..14e0af86 100644 --- a/DuckDB.NET.Data/DuckDBMappedAppender.cs +++ b/DuckDB.NET.Data/DuckDBMappedAppender.cs @@ -42,12 +42,12 @@ internal DuckDBMappedAppender(DuckDBAppender appender) } var columnType = NativeMethods.LogicalType.DuckDBGetTypeId(columnTypes[index]); - var expectedType = GetExpectedDuckDBType(mapping.PropertyType); + var acceptedTypes = GetAcceptedDuckDBTypes(mapping.PropertyType); - if (expectedType != columnType) + if (Array.IndexOf(acceptedTypes, columnType) < 0) { throw new InvalidOperationException( - $"Type mismatch at column index {index}: Mapped type is {mapping.PropertyType.Name} (expected DuckDB type: {expectedType}) but actual column type is {columnType}"); + $"Type mismatch at column index {index}: Mapped type is {mapping.PropertyType.UnderlyingTypeOrSelf().Name} (expected DuckDB type: {string.Join(" or ", acceptedTypes)}) but actual column type is {columnType}"); } } } @@ -86,14 +86,24 @@ private void AppendRecord(T record) }); } - private static DuckDBType GetExpectedDuckDBType(Type type) + // The column types a property of the given type can be appended to. + private static DuckDBType[] GetAcceptedDuckDBTypes(Type type) { - var duckDBType = type.UnderlyingTypeOrSelf().GetDuckDBType(); + var underlyingType = type.UnderlyingTypeOrSelf(); + + // BigInteger is the .NET type for both 128-bit integer columns. The writer stores the value signed or + // unsigned to match the column, and rejects a value outside the column's range when it is appended. + if (underlyingType == typeof(BigInteger)) + { + return [DuckDBType.HugeInt, DuckDBType.UnsignedHugeInt]; + } + + var duckDBType = underlyingType.GetDuckDBType(); return duckDBType switch { DuckDBType.Invalid => throw new NotSupportedException($"Type {type.Name} is not supported for mapping"), - _ => duckDBType + _ => [duckDBType] }; } diff --git a/DuckDB.NET.Data/DuckDBParameter.cs b/DuckDB.NET.Data/DuckDBParameter.cs index 7ace2d50..8619e955 100644 --- a/DuckDB.NET.Data/DuckDBParameter.cs +++ b/DuckDB.NET.Data/DuckDBParameter.cs @@ -6,15 +6,21 @@ namespace DuckDB.NET.Data; public class DuckDBParameter : DbParameter { - private const DbType DefaultDbType = DbType.String; + private const DbType DefaultDbType = DbType.Object; private object? value; + private string parameterName = string.Empty; + private string sourceColumn = string.Empty; public override DbType DbType { get; set; } [AllowNull] [DefaultValue("")] - public override string ParameterName { get; set; } + public override string ParameterName + { + get => parameterName; + set => parameterName = value ?? string.Empty; + } public override object? Value { @@ -29,12 +35,17 @@ public override object? Value } } - public override ParameterDirection Direction { get; set; } + public override ParameterDirection Direction { get; set; } = ParameterDirection.Input; public override bool IsNullable { get; set; } [AllowNull] [DefaultValue("")] - public override string SourceColumn { get; set; } + public override string SourceColumn + { + get => sourceColumn; + set => sourceColumn = value ?? string.Empty; + } + public override DataRowVersion SourceVersion { get; set; } = DataRowVersion.Current; public override bool SourceColumnNullMapping { get; set; } public override int Size { get; set; } @@ -66,4 +77,4 @@ public DuckDBParameter(string name, DbType type, object? value) } public override void ResetDbType() => DbType = DefaultDbType; -} \ No newline at end of file +} diff --git a/DuckDB.NET.Data/DuckDBParameterCollection.cs b/DuckDB.NET.Data/DuckDBParameterCollection.cs index 86204812..ae0e727e 100644 --- a/DuckDB.NET.Data/DuckDBParameterCollection.cs +++ b/DuckDB.NET.Data/DuckDBParameterCollection.cs @@ -23,7 +23,7 @@ public class DuckDBParameterCollection : DbParameterCollection public override int Add(object value) { - parameters.Add((DuckDBParameter)value); + parameters.Add(RequireParameter(value)); return parameters.Count - 1; } @@ -33,9 +33,9 @@ public override int Add(object value) public override int IndexOf(object value) => parameters.IndexOf((DuckDBParameter) value); - public override void Insert(int index, object value) => parameters.Insert(index, (DuckDBParameter) value); + public override void Insert(int index, object value) => parameters.Insert(index, RequireParameter(value)); - public override void Remove(object value) => parameters.Remove((DuckDBParameter) value); + public override void Remove(object value) => parameters.Remove(RequireParameter(value)); public int Add(DuckDBParameter value) { @@ -61,12 +61,12 @@ public override void RemoveAt(string parameterName) } protected override void SetParameter(int index, DbParameter value) - => parameters[index] = (DuckDBParameter)value; + => parameters[index] = RequireParameter(value); protected override void SetParameter(string parameterName, DbParameter value) { var index = IndexOfSafe(parameterName); - parameters[index] = (DuckDBParameter)value; + parameters[index] = RequireParameter(value); } public override int IndexOf(string parameterName) @@ -85,11 +85,7 @@ public void CopyTo(DuckDBParameter[] array, int index) protected override DbParameter GetParameter(int index) => parameters[index]; - protected override DbParameter GetParameter(string parameterName) - { - var index = IndexOf(parameterName); - return parameters[index]; - } + protected override DbParameter GetParameter(string parameterName) => parameters[IndexOfSafe(parameterName)]; public override void AddRange(Array values) => AddRange(values.Cast()); @@ -104,4 +100,7 @@ private int IndexOfSafe(string parameterName) throw new IndexOutOfRangeException($"Parameter '{parameterName}' not found"); return index; } -} \ No newline at end of file + + private static DuckDBParameter RequireParameter(object? value) => + value as DuckDBParameter ?? throw new ArgumentNullException(nameof(value)); +} diff --git a/DuckDB.NET.Data/DuckDBSchema.cs b/DuckDB.NET.Data/DuckDBSchema.cs index 63e0e98b..f0395106 100644 --- a/DuckDB.NET.Data/DuckDBSchema.cs +++ b/DuckDB.NET.Data/DuckDBSchema.cs @@ -1,5 +1,6 @@ using System.Linq; using System.Text; +using DuckDB.NET.Native; namespace DuckDB.NET.Data; @@ -13,10 +14,14 @@ internal static class DuckDBSchema private static readonly string[] IndexesRestrictions = ["index_catalog", "index_schema", "table_name", "index_name"]; - public static DataTable GetSchema(DuckDBConnection connection, string collectionName, string?[]? restrictionValues) => - collectionName.ToUpperInvariant() switch + public static DataTable GetSchema(DuckDBConnection connection, string collectionName, string?[]? restrictionValues) + { + ValidateRestrictions(collectionName, restrictionValues); + return collectionName.ToUpperInvariant() switch { "METADATACOLLECTIONS" => GetMetaDataCollections(), + "DATASOURCEINFORMATION" => GetDataSourceInformation(connection), + "DATATYPES" => GetDataTypes(), "RESTRICTIONS" => GetRestrictions(), "RESERVEDWORDS" => GetReservedWords(connection), "TABLES" => GetTables(connection, restrictionValues), @@ -25,6 +30,17 @@ public static DataTable GetSchema(DuckDBConnection connection, string collection "INDEXES" => GetIndexes(connection, restrictionValues), _ => throw new ArgumentOutOfRangeException(nameof(collectionName), collectionName, "Invalid collection name.") }; + } + + private static void ValidateRestrictions(string collectionName, string?[]? restrictionValues) + { + if (restrictionValues is { Length: > 0 } + && collectionName.ToUpperInvariant() is "METADATACOLLECTIONS" or "DATASOURCEINFORMATION" or "DATATYPES" or "RESTRICTIONS" or "RESERVEDWORDS") + { + throw new ArgumentException("This collection does not support restrictions.", nameof(restrictionValues)); + } + + } private static DataTable GetMetaDataCollections() => new(DbMetaDataCollectionNames.MetaDataCollections) @@ -37,9 +53,11 @@ private static DataTable GetMetaDataCollections() => }, Rows = { + { DbMetaDataCollectionNames.DataSourceInformation, 0, 0 }, + { DbMetaDataCollectionNames.DataTypes, 0, 0 }, { DbMetaDataCollectionNames.MetaDataCollections, 0, 0 }, - { DbMetaDataCollectionNames.Restrictions, 0, 0 }, { DbMetaDataCollectionNames.ReservedWords, 0, 0 }, + { DbMetaDataCollectionNames.Restrictions, 0, 0 }, { DuckDbMetaDataCollectionNames.Tables, TableRestrictions.Length, 3 }, { DuckDbMetaDataCollectionNames.Columns, ColumnRestrictions.Length, 4 }, { DuckDbMetaDataCollectionNames.ForeignKeys, ForeignKeyRestrictions.Length, 3 }, @@ -47,6 +65,130 @@ private static DataTable GetMetaDataCollections() => } }; + private static DataTable GetDataSourceInformation(DuckDBConnection connection) => + new(DbMetaDataCollectionNames.DataSourceInformation) + { + Columns = + { + { DbMetaDataColumnNames.CompositeIdentifierSeparatorPattern, typeof(string) }, + { DbMetaDataColumnNames.DataSourceProductName, typeof(string) }, + { DbMetaDataColumnNames.DataSourceProductVersion, typeof(string) }, + { DbMetaDataColumnNames.DataSourceProductVersionNormalized, typeof(string) }, + { DbMetaDataColumnNames.GroupByBehavior, typeof(int) }, + { DbMetaDataColumnNames.IdentifierCase, typeof(int) }, + { DbMetaDataColumnNames.IdentifierPattern, typeof(string) }, + { DbMetaDataColumnNames.OrderByColumnsInSelect, typeof(bool) }, + { DbMetaDataColumnNames.ParameterMarkerFormat, typeof(string) }, + { DbMetaDataColumnNames.ParameterMarkerPattern, typeof(string) }, + { DbMetaDataColumnNames.ParameterNameMaxLength, typeof(int) }, + { DbMetaDataColumnNames.ParameterNamePattern, typeof(string) }, + { DbMetaDataColumnNames.QuotedIdentifierCase, typeof(int) }, + { DbMetaDataColumnNames.QuotedIdentifierPattern, typeof(string) }, + { DbMetaDataColumnNames.StatementSeparatorPattern, typeof(string) }, + { DbMetaDataColumnNames.StringLiteralPattern, typeof(string) }, + { DbMetaDataColumnNames.SupportedJoinOperators, typeof(int) } + }, + Rows = + { + { + @"\.", "DuckDB", connection.ServerVersion, NormalizeVersion(connection.ServerVersion), + (int)GroupByBehavior.MustContainAll, + (int)IdentifierCase.Insensitive, + @"^[\p{L}_][\p{L}\p{N}_$]*$", false, + "${0}", @"(\$[\p{L}_][\p{L}\p{N}_]*)", DBNull.Value, + @"^[\p{L}_][\p{L}\p{N}_$]*$", (int)IdentifierCase.Insensitive, + "^(([^\\\"]|\\\"\\\")*)$", ";", @"'(([^']|'')*)'", (int)(SupportedJoinOperators.Inner | SupportedJoinOperators.LeftOuter + | SupportedJoinOperators.RightOuter | SupportedJoinOperators.FullOuter) + } + } + }; + + private static DataTable GetDataTypes() + { + var table = new DataTable(DbMetaDataCollectionNames.DataTypes); + foreach (var column in new (string Name, Type Type)[] + { + (DbMetaDataColumnNames.TypeName, typeof(string)), (DbMetaDataColumnNames.ProviderDbType, typeof(int)), + (DbMetaDataColumnNames.DataType, typeof(string)), (DbMetaDataColumnNames.ColumnSize, typeof(long)), + (DbMetaDataColumnNames.CreateFormat, typeof(string)), (DbMetaDataColumnNames.CreateParameters, typeof(string)), + (DbMetaDataColumnNames.IsAutoIncrementable, typeof(bool)), (DbMetaDataColumnNames.IsBestMatch, typeof(bool)), + (DbMetaDataColumnNames.IsCaseSensitive, typeof(bool)), (DbMetaDataColumnNames.IsConcurrencyType, typeof(bool)), + (DbMetaDataColumnNames.IsFixedLength, typeof(bool)), (DbMetaDataColumnNames.IsFixedPrecisionScale, typeof(bool)), + (DbMetaDataColumnNames.IsLiteralSupported, typeof(bool)), (DbMetaDataColumnNames.IsLong, typeof(bool)), + (DbMetaDataColumnNames.IsNullable, typeof(bool)), (DbMetaDataColumnNames.IsSearchable, typeof(bool)), + (DbMetaDataColumnNames.IsSearchableWithLike, typeof(bool)), (DbMetaDataColumnNames.IsUnsigned, typeof(bool)), + (DbMetaDataColumnNames.LiteralPrefix, typeof(string)), (DbMetaDataColumnNames.LiteralSuffix, typeof(string)), + (DbMetaDataColumnNames.MaximumScale, typeof(short)), (DbMetaDataColumnNames.MinimumScale, typeof(short)), + ("NativeDataType", typeof(string)) + }) + { + table.Columns.Add(column.Name, column.Type); + } + + AddType(table, "BOOLEAN", DuckDBType.Boolean, typeof(bool), isBestMatch: true); + AddType(table, "TINYINT", DuckDBType.TinyInt, typeof(sbyte), isBestMatch: true); + AddType(table, "SMALLINT", DuckDBType.SmallInt, typeof(short), isBestMatch: true); + AddType(table, "INTEGER", DuckDBType.Integer, typeof(int), isBestMatch: true); + AddType(table, "BIGINT", DuckDBType.BigInt, typeof(long), isBestMatch: true); + AddType(table, "UTINYINT", DuckDBType.UnsignedTinyInt, typeof(byte), isUnsigned: true, isBestMatch: true); + AddType(table, "USMALLINT", DuckDBType.UnsignedSmallInt, typeof(ushort), isUnsigned: true, isBestMatch: true); + AddType(table, "UINTEGER", DuckDBType.UnsignedInteger, typeof(uint), isUnsigned: true, isBestMatch: true); + AddType(table, "UBIGINT", DuckDBType.UnsignedBigInt, typeof(ulong), isUnsigned: true, isBestMatch: true); + AddType(table, "FLOAT", DuckDBType.Float, typeof(float), isBestMatch: true); + AddType(table, "DOUBLE", DuckDBType.Double, typeof(double), isBestMatch: true); + AddType(table, "DECIMAL", DuckDBType.Decimal, typeof(decimal), createFormat: "DECIMAL({0},{1})", createParameters: "precision,scale", isBestMatch: true); + AddType(table, "VARCHAR", DuckDBType.Varchar, typeof(string), long.MaxValue, isCaseSensitive: true, isBestMatch: true); + AddType(table, "BLOB", DuckDBType.Blob, typeof(byte[]), long.MaxValue, isLong: true, isBestMatch: true); + AddType(table, "UUID", DuckDBType.Uuid, typeof(Guid), isBestMatch: true); + AddType(table, "DATE", DuckDBType.Date, typeof(DateOnly), isBestMatch: true); + AddType(table, "TIME", DuckDBType.Time, typeof(TimeOnly), isBestMatch: true); + AddType(table, "TIMETZ", DuckDBType.TimeTz, typeof(DateTimeOffset), isBestMatch: true); + AddType(table, "TIMESTAMP", DuckDBType.Timestamp, typeof(DateTime), isBestMatch: true); + AddType(table, "TIMESTAMPTZ", DuckDBType.TimestampTz, typeof(DateTimeOffset), isBestMatch: true); + AddType(table, "INTERVAL", DuckDBType.Interval, typeof(TimeSpan), isBestMatch: true); + AddType(table, "HUGEINT", DuckDBType.HugeInt, typeof(System.Numerics.BigInteger), isBestMatch: true); + AddType(table, "UHUGEINT", DuckDBType.UnsignedHugeInt, typeof(System.Numerics.BigInteger), isUnsigned: true); + AddType(table, "VARINT", DuckDBType.VarInt, typeof(System.Numerics.BigInteger)); + AddType(table, "LIST", DuckDBType.List, typeof(object)); + AddType(table, "ARRAY", DuckDBType.Array, typeof(object)); + AddType(table, "STRUCT", DuckDBType.Struct, typeof(object)); + AddType(table, "MAP", DuckDBType.Map, typeof(object)); + AddType(table, "UNION", DuckDBType.Union, typeof(object)); + AddType(table, "ENUM", DuckDBType.Enum, typeof(string)); + AddType(table, "BIT", DuckDBType.Bit, typeof(string)); + + return table; + } + + private static void AddType( + DataTable table, + string typeName, + DuckDBType providerType, + Type dataType, + long columnSize = -1, + string? createFormat = null, + string? createParameters = null, + bool isUnsigned = false, + bool isCaseSensitive = false, + bool isLong = false, + bool isBestMatch = false) + { + table.Rows.Add( + typeName, (int)providerType, dataType.FullName!, columnSize, + createFormat ?? (object)DBNull.Value, createParameters ?? (object)DBNull.Value, + false, isBestMatch, isCaseSensitive, false, false, false, true, isLong, + true, true, isCaseSensitive, isUnsigned ? (object)true : DBNull.Value, + DBNull.Value, DBNull.Value, DBNull.Value, DBNull.Value, typeName); + } + + private static string NormalizeVersion(string serverVersion) + { + var value = serverVersion.TrimStart('v', 'V'); + return Version.TryParse(value, out var version) + ? $"{version.Major:00}.{version.Minor:000}.{(version.Build < 0 ? 0 : version.Build):0000}.{(version.Revision < 0 ? 0 : version.Revision):0000}" + : serverVersion; + } + private static DataTable GetRestrictions() => new(DbMetaDataCollectionNames.Restrictions) { @@ -74,10 +216,10 @@ private static DataTable GetRestrictions() => { DuckDbMetaDataCollectionNames.ForeignKeys, "Table", "table_name", 3 }, { DuckDbMetaDataCollectionNames.ForeignKeys, "Constraint", "constraint_name", 4 }, - { DuckDbMetaDataCollectionNames.Indexes, "Catalog", "constraint_catalog", 1 }, - { DuckDbMetaDataCollectionNames.Indexes, "Schema", "constraint_schema", 2 }, + { DuckDbMetaDataCollectionNames.Indexes, "Catalog", "index_catalog", 1 }, + { DuckDbMetaDataCollectionNames.Indexes, "Schema", "index_schema", 2 }, { DuckDbMetaDataCollectionNames.Indexes, "Table", "table_name", 3 }, - { DuckDbMetaDataCollectionNames.Indexes, "Constraint", "constraint_name", 4 }, + { DuckDbMetaDataCollectionNames.Indexes, "Index", "index_name", 4 }, }, }; @@ -218,4 +360,4 @@ private static DataTable GetDataTable(string tableName, DuckDBCommand command) return table; } -} \ No newline at end of file +} diff --git a/DuckDB.NET.Data/DuckDBTransaction.cs b/DuckDB.NET.Data/DuckDBTransaction.cs index 94a95cd9..aab103c9 100644 --- a/DuckDB.NET.Data/DuckDBTransaction.cs +++ b/DuckDB.NET.Data/DuckDBTransaction.cs @@ -35,7 +35,10 @@ private void FinishTransaction(string finalizer) try { - connection.ExecuteNonQuery(finalizer); + using var command = connection.CreateCommand(); + command.Transaction = this; + command.CommandText = finalizer; + command.ExecuteNonQuery(); Cleanup(); } // If something goes wrong with the transaction, to match the diff --git a/DuckDB.NET.Data/Extensions/TypeExtensions.cs b/DuckDB.NET.Data/Extensions/TypeExtensions.cs index a95c6a46..2ff375c5 100644 --- a/DuckDB.NET.Data/Extensions/TypeExtensions.cs +++ b/DuckDB.NET.Data/Extensions/TypeExtensions.cs @@ -34,6 +34,8 @@ internal static class TypeExtensions { typeof(TimeSpan), DuckDBType.Interval}, { typeof(DateOnly), DuckDBType.Date}, { typeof(TimeOnly), DuckDBType.Time}, + { typeof(DuckDBDateOnly), DuckDBType.Date}, + { typeof(DuckDBTimeOnly), DuckDBType.Time}, { typeof(DateTimeOffset), DuckDBType.TimestampTz}, { typeof(BigInteger), DuckDBType.HugeInt}, { typeof(string), DuckDBType.Varchar}, @@ -105,5 +107,5 @@ public static DuckDBLogicalType GetLogicalType(this Type type) throw new InvalidOperationException($"Cannot map type {type.FullName} to DuckDBType."); } - public static DuckDBType GetDuckDBType(this Type type) => ClrToDuckDBTypeMap.TryGetValue(type, out var duckDBType) ? duckDBType : DuckDBType.Invalid; + public static DuckDBType GetDuckDBType(this Type type) => ClrToDuckDBTypeMap.GetValueOrDefault(type, DuckDBType.Invalid); } \ No newline at end of file diff --git a/DuckDB.NET.Data/PreparedStatement/ClrToDuckDBConverter.cs b/DuckDB.NET.Data/PreparedStatement/ClrToDuckDBConverter.cs index a2ada158..d322521f 100644 --- a/DuckDB.NET.Data/PreparedStatement/ClrToDuckDBConverter.cs +++ b/DuckDB.NET.Data/PreparedStatement/ClrToDuckDBConverter.cs @@ -18,7 +18,7 @@ internal static class ClrToDuckDBConverter { DbType.Single, value => NativeMethods.Value.DuckDBCreateFloat((float)value) }, { DbType.Double, value => NativeMethods.Value.DuckDBCreateDouble((double)value) }, { DbType.String, value => NativeMethods.Value.DuckDBCreateVarchar((string?)value) }, - { DbType.VarNumeric, value => NativeMethods.Value.DuckDBCreateHugeInt(new((BigInteger)value)) }, + { DbType.VarNumeric, value => BigIntegerWithoutTargetType((BigInteger)value) }, { DbType.Binary, value => { var bytes = (byte[])value; @@ -33,7 +33,13 @@ internal static class ClrToDuckDBConverter }, { DbType.Time, value => { - var time = NativeMethods.DateTimeHelpers.DuckDBToTime(value is TimeOnly timeOnly ? (DuckDBTimeOnly)timeOnly : (DuckDBTimeOnly)value); + // A TimeSpan is a time of day here: TimeOnly.FromTimeSpan refuses a negative one or one of a day or more. + var time = NativeMethods.DateTimeHelpers.DuckDBToTime(value switch + { + TimeOnly timeOnly => (DuckDBTimeOnly)timeOnly, + TimeSpan timeSpan => (DuckDBTimeOnly)TimeOnly.FromTimeSpan(timeSpan), + _ => (DuckDBTimeOnly)value + }); return NativeMethods.Value.DuckDBCreateTime(time); } }, @@ -72,6 +78,7 @@ public static DuckDBValue ToDuckDBValue(this object? item, DuckDBLogicalType log (DuckDBType.Decimal, decimal value) => DecimalToDuckDBValue(value), (DuckDBType.HugeInt, BigInteger value) => NativeMethods.Value.DuckDBCreateHugeInt(new DuckDBHugeInt(value)), + (DuckDBType.UnsignedHugeInt, BigInteger value) => NativeMethods.Value.DuckDBCreateUHugeInt(new DuckDBUHugeInt(value)), (DuckDBType.VarInt, BigInteger value) => BigIntegerToDuckDBValue(value), (DuckDBType.Varchar, string value) => NativeMethods.Value.DuckDBCreateVarchar(value), @@ -84,6 +91,9 @@ public static DuckDBValue ToDuckDBValue(this object? item, DuckDBLogicalType log (DuckDBType.TimestampTz, DateTime value) => NativeMethods.Value.DuckDBCreateTimestampTz(value.ToTimestampStruct(duckDBType)), (DuckDBType.TimestampTz, DateTimeOffset value) => NativeMethods.Value.DuckDBCreateTimestampTz(value.ToTimestampStruct()), (DuckDBType.Interval, TimeSpan value) => NativeMethods.Value.DuckDBCreateInterval(value), + // No inferred type (a bare "?"): TimeSpan has no DbType, and its ToString() isn't interval text + // DuckDB can parse once there are days ("1.01:01:01"). DbType.Time sends it as a time instead. + (DuckDBType.Invalid, TimeSpan value) when dbType != DbType.Time => NativeMethods.Value.DuckDBCreateInterval(value), (DuckDBType.Date, DateTime value) => NativeMethods.Value.DuckDBCreateDate(((DuckDBDateOnly)value).ToDuckDBDate()), (DuckDBType.Date, DuckDBDateOnly value) => NativeMethods.Value.DuckDBCreateDate(value.ToDuckDBDate()), (DuckDBType.Time, DateTime value) => NativeMethods.Value.DuckDBCreateTime(NativeMethods.DateTimeHelpers.DuckDBToTime((DuckDBTimeOnly)value)), @@ -160,6 +170,23 @@ private static DuckDBValue[] BuildValues(DuckDBLogicalType childType, ICollectio return values; } + // DuckDB reported no type for the parameter, for example in "SELECT $1" or "$1::UHUGEINT", so pick the smallest + // type that holds the value exactly: HUGEINT, then UHUGEINT, then BIGNUM. + private static DuckDBValue BigIntegerWithoutTargetType(BigInteger value) + { + if (value >= DuckDBHugeInt.HugeIntMinValue && value <= DuckDBHugeInt.HugeIntMaxValue) + { + return NativeMethods.Value.DuckDBCreateHugeInt(new DuckDBHugeInt(value)); + } + + if (value >= DuckDBUHugeInt.HugeIntMinValue && value <= DuckDBUHugeInt.HugeIntMaxValue) + { + return NativeMethods.Value.DuckDBCreateUHugeInt(new DuckDBUHugeInt(value)); + } + + return BigIntegerToDuckDBValue(value); + } + private static unsafe DuckDBValue BigIntegerToDuckDBValue(BigInteger value) { // A BIGNUM is built from the magnitude bytes and a sign, which keeps values of any size exact. diff --git a/DuckDB.NET.Data/PreparedStatement/PreparedStatement.cs b/DuckDB.NET.Data/PreparedStatement/PreparedStatement.cs index 682dcb1e..54f977ce 100644 --- a/DuckDB.NET.Data/PreparedStatement/PreparedStatement.cs +++ b/DuckDB.NET.Data/PreparedStatement/PreparedStatement.cs @@ -98,7 +98,8 @@ private static void BindParameters(DuckDBPreparedStatement preparedStatement, Du for (var i = 0; i < count; i++) { var param = parameterCollection[i]; - var state = NativeMethods.PreparedStatements.DuckDBBindParameterIndex(preparedStatement, out var index, param.ParameterName); + var state = NativeMethods.PreparedStatements.DuckDBBindParameterIndex( + preparedStatement, out var index, param.ParameterName.TrimStart('$')); if (state.IsSuccess()) { BindParameter(preparedStatement, index, param); @@ -135,4 +136,4 @@ public void Dispose() { statement.Dispose(); } -} \ No newline at end of file +} diff --git a/DuckDB.NET.Test/Arrow/ArrowResultTests.cs b/DuckDB.NET.Test/Arrow/ArrowResultTests.cs index e797004f..67f32458 100644 --- a/DuckDB.NET.Test/Arrow/ArrowResultTests.cs +++ b/DuckDB.NET.Test/Arrow/ArrowResultTests.cs @@ -82,10 +82,12 @@ public async Task ExecuteArrowBatches_StreamsMultipleChunks(bool useStreamingMod public async Task ExecuteArrowStream_ThrowsWhenLaterStreamingChunkFails() { // The first chunks convert fine; the error is raised only while producing a later chunk. - Command.UseStreamingMode = true; - Command.CommandText = "SELECT CAST(CASE WHEN i < 500000 THEN CAST(i AS VARCHAR) ELSE 'not a number' END AS INTEGER) FROM range(1000000) t(i)"; + using var connection = OpenConnectionForLateStreamingErrors(); + using var command = connection.CreateCommand(); + command.UseStreamingMode = true; + command.CommandText = "SELECT CAST(CASE WHEN i < 500000 THEN CAST(i AS VARCHAR) ELSE 'not a number' END AS INTEGER) FROM range(1000000) t(i)"; - using var stream = Command.ExecuteArrowStream(); + using var stream = command.ExecuteArrowStream(); var rows = 0; var act = async () => @@ -130,6 +132,29 @@ public async Task ExecuteArrowStream_ReadsBatchesUntilNull() rows.Should().Be(10); } + // Regression test for https://github.com/Giorgi/DuckDB.NET/issues/358 + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ExecuteArrowStream_ReadAfterEndReturnsNull(bool useStreamingMode) + { + Command.UseStreamingMode = useStreamingMode; + Command.CommandText = "select i from range(5000) t(i)"; + + using var stream = Command.ExecuteArrowStream(); + + var rows = 0; + while (await stream.ReadNextRecordBatchAsync(CancellationToken.None) is { } batch) + { + rows += batch.Length; + batch.Dispose(); + } + + rows.Should().Be(5000); + (await stream.ReadNextRecordBatchAsync(CancellationToken.None)).Should().BeNull(); + (await stream.ReadNextRecordBatchAsync(CancellationToken.None)).Should().BeNull(); + } + [Fact] public void ExecuteArrowStream_ThrowsWhenNoResultSet() { diff --git a/DuckDB.NET.Test/ConnectionStringTests.cs b/DuckDB.NET.Test/ConnectionStringTests.cs index 702b6ef2..488bb6fd 100644 --- a/DuckDB.NET.Test/ConnectionStringTests.cs +++ b/DuckDB.NET.Test/ConnectionStringTests.cs @@ -1,4 +1,6 @@ using System.ComponentModel; +using System.Globalization; +using System.Reflection; namespace DuckDB.NET.Test; @@ -349,6 +351,103 @@ public void TypedPropertiesThrowOnMalformedValues() builder.Invoking(b => b.AccessMode).Should().Throw(); } + // Every typed property, found by reflection, so a property added later is covered without a change here. + public static IEnumerable TypedPropertyNames() => + typeof(DuckDBConnectionStringBuilder) + .GetProperties(BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly) + .Where(property => property.GetCustomAttribute() != null) + .Select(property => new object[] { property.Name }); + + [Theory] + [MemberData(nameof(TypedPropertyNames))] + public void EveryTypedPropertyRoundTripsAndClears(string propertyName) + { + var property = typeof(DuckDBConnectionStringBuilder).GetProperty(propertyName)!; + var keyword = property.GetCustomAttribute()!.DisplayName; + var type = Nullable.GetUnderlyingType(property.PropertyType) ?? property.PropertyType; + + object[] samples; + + if (type == typeof(string)) + { + samples = ["sample"]; + } + else if (type == typeof(bool)) + { + samples = [true, false]; + } + else if (type == typeof(int)) + { + samples = [3]; + } + else if (type.IsEnum) + { + samples = Enum.GetValues(type).Cast().ToArray(); + } + else + { + throw new NotSupportedException($"Add sample values for {type.Name}, the type of {propertyName}."); + } + + foreach (var sample in samples) + { + var builder = new DuckDBConnectionStringBuilder(); + + property.SetValue(builder, sample); + + builder.ContainsKey(keyword).Should().BeTrue($"setting {propertyName} should write '{keyword}'"); + property.GetValue(builder).Should().Be(sample); + + property.SetValue(builder, null); + + builder.ContainsKey(keyword).Should().BeFalse($"setting {propertyName} to null should remove '{keyword}'"); + property.GetValue(builder).Should().Be(type == typeof(string) ? "" : null); + } + } + + [Theory] + [InlineData("asc", DuckDBSortOrder.Ascending)] + [InlineData("ASCENDING", DuckDBSortOrder.Ascending)] + [InlineData("desc", DuckDBSortOrder.Descending)] + [InlineData("Descending", DuckDBSortOrder.Descending)] + public void DefaultOrderAcceptsEverySpellingDuckDBAccepts(string value, DuckDBSortOrder expected) + { + var builder = new DuckDBConnectionStringBuilder { ConnectionString = $"DataSource=:memory:;default_order={value}" }; + + builder.DefaultOrder.Should().Be(expected); + } + + [Fact] + public void SortOrderPropertiesThrowOnMalformedValues() + { + var builder = new DuckDBConnectionStringBuilder { ConnectionString = "DataSource=:memory:;default_order=sideways;default_null_order=middle" }; + + builder.Invoking(b => b.DefaultOrder).Should().Throw(); + builder.Invoking(b => b.DefaultNullOrder).Should().Throw(); + } + + [Fact] + public void EnumPropertiesRejectUndefinedValues() + { + var builder = new DuckDBConnectionStringBuilder(); + + builder.Invoking(b => b.AccessMode = (DuckDBAccessMode)999).Should().Throw(); + builder.Invoking(b => b.DefaultOrder = (DuckDBSortOrder)999).Should().Throw(); + builder.Invoking(b => b.DefaultNullOrder = (DuckDBNullOrder)999).Should().Throw(); + } + + [Fact] + public void EmptyNumberAndBooleanOptionsReadAsNotSet() + { + var builder = new DuckDBConnectionStringBuilder { DataSource = DuckDBConnectionStringBuilder.InMemoryDataSource }; + + builder["threads"] = ""; + builder["enable_external_access"] = ""; + + builder.Threads.Should().BeNull(); + builder.EnableExternalAccess.Should().BeNull(); + } + [Fact] public void TypedPropertiesProduceAConnectionStringDuckDBAccepts() { @@ -542,6 +641,95 @@ public void BooleanOptionDescriptorsWriteValuesDuckDBAccepts() connection.State.Should().Be(ConnectionState.Open); } + [Fact] + public void ConfigurationOptionDescriptorsDescribeTheBuilder() + { + var builder = new DuckDBConnectionStringBuilder { DataSource = DuckDBConnectionStringBuilder.InMemoryDataSource }; + var option = TypeDescriptor.GetProperties(builder).Find("zstd_min_string_length", false)!; + + option.ComponentType.Should().Be(typeof(DuckDBConnectionStringBuilder)); + option.IsReadOnly.Should().BeFalse(); + + option.CanResetValue(builder).Should().BeFalse(); + option.SetValue(builder, 8192UL); + option.CanResetValue(builder).Should().BeTrue(); + + // A component that is not a connection string builder has no value to read. + option.GetValue(new object()).Should().BeNull(); + } + + // The connection string stores every value as text, so reading converts it back to the option's type. Text that + // does not fit the type reads as null instead of throwing out of a property grid. + [Theory] + [InlineData("arrow_large_buffer_size", "yes", true)] + [InlineData("arrow_large_buffer_size", "0", false)] + [InlineData("arrow_large_buffer_size", "maybe", null)] + [InlineData("geometry_minimum_shredding_size", "-123", -123L)] + [InlineData("geometry_minimum_shredding_size", "abc", null)] + [InlineData("zstd_min_string_length", "8192", 8192UL)] + [InlineData("zstd_min_string_length", "-1", null)] + [InlineData("index_scan_percentage", "0.001", 0.001)] + [InlineData("index_scan_percentage", "abc", null)] + [InlineData("default_secret_storage", "local_file", "local_file")] + public void ConfigurationOptionDescriptorsConvertTextToTheOptionType(string optionName, string text, object expected) + { + var builder = new DuckDBConnectionStringBuilder { DataSource = DuckDBConnectionStringBuilder.InMemoryDataSource }; + var option = TypeDescriptor.GetProperties(builder).Find(optionName, false)!; + + builder[optionName] = text; + + option.GetValue(builder).Should().Be(expected); + } + + [Theory] + [InlineData("default_secret_storage", "")] + [InlineData("zstd_min_string_length", null)] + public void ConfigurationOptionDescriptorsReadEmptyTextAsEmptyOrNull(string optionName, string expected) + { + var builder = new DuckDBConnectionStringBuilder { DataSource = DuckDBConnectionStringBuilder.InMemoryDataSource }; + var option = TypeDescriptor.GetProperties(builder).Find(optionName, false)!; + + builder[optionName] = ""; + + // Only a text option can hold an empty value; for any other type it means "not set". + option.GetValue(builder).Should().Be(expected); + } + + [Fact] + public void ConfigurationOptionDescriptorsWriteInvariantText() + { + var builder = new DuckDBConnectionStringBuilder { DataSource = DuckDBConnectionStringBuilder.InMemoryDataSource }; + var properties = TypeDescriptor.GetProperties(builder); + var previousCulture = CultureInfo.CurrentCulture; + + try + { + // German formats 0.001 as "0,001", which DuckDB would reject. + CultureInfo.CurrentCulture = CultureInfo.GetCultureInfo("de-DE"); + + properties.Find("index_scan_percentage", false)!.SetValue(builder, 0.001); + properties.Find("arrow_large_buffer_size", false)!.SetValue(builder, false); + properties.Find("default_secret_storage", false)!.SetValue(builder, "local_file"); + + // Anything else is written with its ToString(). + properties.Find("zstd_min_string_length", false)!.SetValue(builder, new StringBuilder("8192")); + } + finally + { + CultureInfo.CurrentCulture = previousCulture; + } + + builder["index_scan_percentage"].Should().Be("0.001"); + builder["arrow_large_buffer_size"].Should().Be("false"); + builder["default_secret_storage"].Should().Be("local_file"); + builder["zstd_min_string_length"].Should().Be("8192"); + + // Writing null removes the option. + properties.Find("default_secret_storage", false)!.SetValue(builder, null); + + builder.ContainsKey("default_secret_storage").Should().BeFalse(); + } + [Fact] public void OptionsThatCannotBeAppliedAtOpenAreNotOffered() { diff --git a/DuckDB.NET.Test/DuckDBClientFactoryTests.cs b/DuckDB.NET.Test/DuckDBClientFactoryTests.cs index 4d27ee2b..48c8ed91 100644 --- a/DuckDB.NET.Test/DuckDBClientFactoryTests.cs +++ b/DuckDB.NET.Test/DuckDBClientFactoryTests.cs @@ -47,4 +47,18 @@ public void UseFactory() Assert.Equal(42, value); } + + [Fact] + public void FactoryReportsWhatItCannotCreate() + { + DbProviderFactory factory = DuckDBClientFactory.Instance; + + // Generic data code asks these before creating anything, so they must answer, not throw. + Assert.False(factory.CanCreateDataAdapter); + Assert.False(factory.CanCreateCommandBuilder); + Assert.False(factory.CanCreateDataSourceEnumerator); + + Assert.Null(factory.CreateDataAdapter()); + Assert.Null(factory.CreateCommandBuilder()); + } } \ No newline at end of file diff --git a/DuckDB.NET.Test/DuckDBDataReaderTests.cs b/DuckDB.NET.Test/DuckDBDataReaderTests.cs index 370d32a9..797feba4 100644 --- a/DuckDB.NET.Test/DuckDBDataReaderTests.cs +++ b/DuckDB.NET.Test/DuckDBDataReaderTests.cs @@ -182,8 +182,7 @@ public void FailedExecuteReaderClosesConnectionOnlyWithCloseConnection(string sq public void StreamingReadFailureClosesConnectionOnlyWhenReaderIsDisposed() { // Matches Npgsql: a failed Read() leaves the connection open, and disposing the reader closes it. - using var connection = new DuckDBConnection("DataSource=:memory:"); - connection.Open(); + using var connection = OpenConnectionForLateStreamingErrors(); using var command = connection.CreateCommand(); command.UseStreamingMode = true; command.CommandText = "SELECT CAST(CASE WHEN i < 500000 THEN CAST(i AS VARCHAR) ELSE 'not a number' END AS INTEGER) FROM range(1000000) t(i)"; @@ -454,14 +453,73 @@ public void ReadPivotStatementResult() reader.NextResult().Should().BeFalse(); } + // Regression tests for https://github.com/Giorgi/DuckDB.NET/issues/358 + [Theory] + [InlineData(false)] + [InlineData(true)] + public void ReadAfterEndOfResultReturnsFalse(bool useStreamingMode) + { + Command.UseStreamingMode = useStreamingMode; + Command.CommandText = "SELECT i FROM range(5000) t(i)"; + + using var reader = Command.ExecuteReader(); + + var rows = 0; + while (reader.Read()) + { + rows++; + } + + rows.Should().Be(5000); + reader.Read().Should().BeFalse(); + reader.Read().Should().BeFalse(); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void ReadOnEmptyResultReturnsFalse(bool useStreamingMode) + { + Command.UseStreamingMode = useStreamingMode; + Command.CommandText = "SELECT i FROM range(0) t(i)"; + + using var reader = Command.ExecuteReader(); + + reader.HasRows.Should().BeFalse(); + reader.Read().Should().BeFalse(); + reader.Read().Should().BeFalse(); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void NextResultAfterReadingPastEndReturnsNextResultSet(bool useStreamingMode) + { + Command.UseStreamingMode = useStreamingMode; + Command.CommandText = "SELECT i FROM range(0) t(i); SELECT 42"; + + using var reader = Command.ExecuteReader(); + + reader.Read().Should().BeFalse(); + reader.Read().Should().BeFalse(); + + reader.NextResult().Should().BeTrue(); + reader.Read().Should().BeTrue(); + reader.GetInt32(0).Should().Be(42); + reader.Read().Should().BeFalse(); + reader.Read().Should().BeFalse(); + } + [Fact] public void StreamingReadThrowsWhenLaterChunkFails() { // The first chunks convert fine; the error is raised only while producing a later chunk. - Command.UseStreamingMode = true; - Command.CommandText = "SELECT CAST(CASE WHEN i < 500000 THEN CAST(i AS VARCHAR) ELSE 'not a number' END AS INTEGER) FROM range(1000000) t(i)"; + using var connection = OpenConnectionForLateStreamingErrors(); + using var command = connection.CreateCommand(); + command.UseStreamingMode = true; + command.CommandText = "SELECT CAST(CASE WHEN i < 500000 THEN CAST(i AS VARCHAR) ELSE 'not a number' END AS INTEGER) FROM range(1000000) t(i)"; - using var reader = Command.ExecuteReader(); + using var reader = command.ExecuteReader(); var rows = 0; var act = () => diff --git a/DuckDB.NET.Test/DuckDBDatabaseFixture.cs b/DuckDB.NET.Test/DuckDBDatabaseFixture.cs index c9cf7dbb..c5e33f75 100644 --- a/DuckDB.NET.Test/DuckDBDatabaseFixture.cs +++ b/DuckDB.NET.Test/DuckDBDatabaseFixture.cs @@ -44,4 +44,24 @@ public virtual void Dispose() { Command?.Dispose(); } + + /// + /// Opens a private in-memory database for tests where a streaming query must fail only after some rows were read. + /// One thread produces the chunks in order, and a small streaming buffer lets it get only a few chunks ahead of + /// the reader, so the rows before the failing one are read first. With the default settings DuckDB 2.0 buffers + /// past the failing row and reports the error on the first fetch, and DuckDB 1.5 can report such an error as an + /// interrupt when several threads are running. + /// + protected static DuckDBConnection OpenConnectionForLateStreamingErrors() + { + var connection = new DuckDBConnection("DataSource=:memory:;threads=1"); + connection.Open(); + + // A per-connection setting, so it cannot go into the connection string. + using var command = connection.CreateCommand(); + command.CommandText = "SET streaming_buffer_size = '100KB'"; + command.ExecuteNonQuery(); + + return connection; + } } \ No newline at end of file diff --git a/DuckDB.NET.Test/DuckDBIntervalTests.cs b/DuckDB.NET.Test/DuckDBIntervalTests.cs index cd0cbc0e..6f80ca4d 100644 --- a/DuckDB.NET.Test/DuckDBIntervalTests.cs +++ b/DuckDB.NET.Test/DuckDBIntervalTests.cs @@ -47,6 +47,50 @@ public void ToTimeSpan_TooBigDays_Exception() Assert.Throws(() => (TimeSpan)interval); } + // DuckDB stores an interval's micros as a signed 64-bit value (duckdb_interval.micros is int64_t). + [Theory] + [InlineData(0, -5_400_000_000L, "-01:30:00")] + [InlineData(0, -17L, "-00:00:00.000017")] + [InlineData(-1, -3_600_000_000L, "-1.01:00:00")] + [InlineData(1, -3_600_000_000L, "23:00:00")] + [InlineData(-2, 0L, "-2.00:00:00")] + [InlineData(-1, 25 * 3_600_000_000L, "01:00:00")] + public void ToTimeSpan_NegativeValue_ExpectedResult(int days, long micros, string ts) + { + var interval = new DuckDBInterval(0, days, unchecked((ulong)micros)); + Assert.True(interval.TryConvert(out var timeSpan)); + Assert.Equal(TimeSpan.Parse(ts), timeSpan); + Assert.Equal(TimeSpan.Parse(ts), (TimeSpan)interval); + } + + [Fact] + public void ToTimeSpan_NegativeMonthInterval_Exception() + { + var interval = new DuckDBInterval(-1, 0, 0); + Assert.False(interval.TryConvert(out var ts)); + Assert.Throws(() => (TimeSpan)interval); + } + + [Fact] + public void ToTimeSpan_TooSmall_Exception() + { + var interval = new DuckDBInterval(0, int.MinValue, unchecked((ulong)long.MinValue)); + Assert.False(interval.TryConvert(out var ts)); + Assert.Throws(() => (TimeSpan)interval); + } + + [Theory] + [InlineData("-01:30:00", 0, -5_400_000_000L)] + [InlineData("-1.01:00:00", -1, -3_600_000_000L)] + [InlineData("-00:00:00.0000017", 0, -1L)] + public void ToDuckDBInterval_NegativeValue_ExpectedResult(string ts, int days, long micros) + { + DuckDBInterval interval = TimeSpan.Parse(ts); + Assert.Equal(0, interval.Months); + Assert.Equal(days, interval.Days); + Assert.Equal(micros, unchecked((long)interval.Micros)); + } + [Fact] public void ToDuckDBInterval_ValidValue_ExpectedResult() { diff --git a/DuckDB.NET.Test/DuckDBManagedAppenderListTests.cs b/DuckDB.NET.Test/DuckDBManagedAppenderListTests.cs index d014ed5b..49b01074 100644 --- a/DuckDB.NET.Test/DuckDBManagedAppenderListTests.cs +++ b/DuckDB.NET.Test/DuckDBManagedAppenderListTests.cs @@ -222,7 +222,7 @@ public void VarcharArrayValuesAfterNullArrayStayAligned() [Fact] public void ListOfArraysAfterNullArrayStayAligned() { - VerifyArrayRowsAfterNull?>("managedAppenderListOfArraysAfterNull", "INTEGER[2][]", + VerifyArrayRowsAfterNull>("managedAppenderListOfArraysAfterNull", "INTEGER[2][]", [[[1, 2], [3, 4]], [[5, 6], null, [7, 8]], [[9, 10]]]); } @@ -232,7 +232,7 @@ public void ListOfArraysAfterNullArrayStayAligned() [Fact] public void ArrayOfListsAfterNullArrayStayAligned() { - VerifyArrayRowsAfterNull?>("managedAppenderArrayOfListsAfterNull", "INTEGER[][2]", + VerifyArrayRowsAfterNull>("managedAppenderArrayOfListsAfterNull", "INTEGER[][2]", [[[1], [2, 3]], null, [[4], [5]], [[6, 7], [8]], [[9], null]]); } @@ -253,7 +253,7 @@ public void ArrayValuesWithNullItemsAfterNullArrayStayAligned() [Fact] public void VarcharArrayValuesWithNullItemsAfterNullArrayStayAligned() { - VerifyArrayRowsAfterNull("managedAppenderVarcharArrayNullItemsAfterNull", "VARCHAR[2]", + VerifyArrayRowsAfterNull("managedAppenderVarcharArrayNullItemsAfterNull", "VARCHAR[2]", [["a", null], null, [null, "d"], ["e", "f"]]); } @@ -283,7 +283,7 @@ public void ListOfArraysKeepValuesAfterListGrows() var batchSize = (int)DuckDBGlobalData.VectorSize; // Three arrays per row take the list past its initial VectorSize items a third of the way in. - var rows = Enumerable.Range(0, batchSize).Select(List?>? (i) => i switch + var rows = Enumerable.Range(0, batchSize).Select(List> (i) => i switch { 1000 => null, 1500 => [[i * 10, null], null, [i * 10 + 2, -(i * 10 + 2)]], @@ -301,7 +301,7 @@ public void ListOfNestedArraysKeepValuesAfterListGrows() var batchSize = (int)DuckDBGlobalData.VectorSize; // Three items per row take the list past its initial VectorSize items a third of the way in. - var rows = Enumerable.Range(0, batchSize).Select(List?>?>? (i) => + var rows = Enumerable.Range(0, batchSize).Select(List>> (i) => { var x = i * 10; @@ -324,7 +324,7 @@ public void ListOfArraysOfListsKeepValuesAfterListGrows() var batchSize = (int)DuckDBGlobalData.VectorSize; // Three items per row take the outer list past its initial VectorSize items a third of the way in. - var rows = Enumerable.Range(0, batchSize).Select(List?>?>? (i) => + var rows = Enumerable.Range(0, batchSize).Select(List>> (i) => { var x = i * 10; @@ -339,7 +339,7 @@ public void ListOfArraysOfListsKeepValuesAfterListGrows() VerifyArrayRowsAfterNull("managedAppenderListOfArraysOfListsAfterGrowth", "INTEGER[][2][]", rows); } - private void VerifyArrayRowsAfterNull(string table, string columnType, List?[] rows) + private void VerifyArrayRowsAfterNull(string table, string columnType, List[] rows) { Command.CommandText = $"CREATE TABLE {table} (a INTEGER, b {columnType});"; Command.ExecuteNonQuery(); diff --git a/DuckDB.NET.Test/DuckDBManagedAppenderTests.cs b/DuckDB.NET.Test/DuckDBManagedAppenderTests.cs index 4def6ec7..3d781079 100644 --- a/DuckDB.NET.Test/DuckDBManagedAppenderTests.cs +++ b/DuckDB.NET.Test/DuckDBManagedAppenderTests.cs @@ -146,6 +146,92 @@ public void ByteArray() dataReader.IsDBNull(0).Should().BeTrue(); } + [Fact] + public void EmptyByteArrayIsEmptyBlobAndDefaultSpanIsNull() + { + // Only a BLOB column, ordered by rowid: an integer literal such as AppendValue(1) binds to the byte? overload, + // which the low-level appender would write into one byte of an INTEGER slot. + Command.CommandText = "CREATE TABLE blobAppenderEmptyTest(b blob)"; + Command.ExecuteNonQuery(); + + using (var appender = Connection.CreateAppender("blobAppenderEmptyTest")) + { + appender.CreateRow().AppendValue(Array.Empty()).EndRow(); + appender.CreateRow().AppendValue(new Span(Array.Empty())).EndRow(); + appender.CreateRow().AppendValue(default(Span)).EndRow(); + } + + var results = Connection.Query<(bool IsNull, long? Length)>( + "SELECT b IS NULL, octet_length(b) FROM blobAppenderEmptyTest ORDER BY rowid").ToList(); + + results.Should().Equal((false, 0L), (false, 0L), (true, null)); + } + + [Fact] + public void UnsignedHugeIntAboveSignedHugeIntRange() + { + // Only UHUGEINT columns, ordered by rowid: the low-level appender does no type checking. + Command.CommandText = "CREATE TABLE uhugeintAppenderTest(a UHUGEINT, b UHUGEINT[])"; + Command.ExecuteNonQuery(); + + BigInteger?[] values = + [ + BigInteger.Zero, + DuckDBHugeInt.HugeIntMaxValue, + BigInteger.Pow(2, 127) + 42, + DuckDBUHugeInt.HugeIntMaxValue, + ]; + + using (var appender = Connection.CreateAppender("uhugeintAppenderTest")) + { + foreach (var value in values) + { + appender.CreateRow().AppendValue(value).AppendValue(new List { value.Value, value.Value }).EndRow(); + } + } + + Command.CommandText = "SELECT a, b, a::VARCHAR FROM uhugeintAppenderTest ORDER BY rowid"; + using var reader = Command.ExecuteReader(); + + foreach (var value in values) + { + reader.Read().Should().BeTrue(); + reader.GetFieldValue(0).Should().Be(value.Value); + reader.GetFieldValue>(1).Should().Equal(value.Value, value.Value); + reader.GetString(2).Should().Be(value.Value.ToString()); + } + + reader.Read().Should().BeFalse(); + } + + [Fact] + public void UnsignedHugeIntRejectsValuesOutOfRange() + { + Command.CommandText = "CREATE TABLE uhugeintAppenderRangeTest(a UHUGEINT)"; + Command.ExecuteNonQuery(); + + BigInteger? negative = BigInteger.MinusOne; + BigInteger? tooLarge = DuckDBUHugeInt.HugeIntMaxValue + 1; + BigInteger? valid = BigInteger.Pow(2, 127); + + using (var appender = Connection.CreateAppender("uhugeintAppenderRangeTest")) + { + var row = appender.CreateRow(); + + row.Invoking(r => r.AppendValue(negative)).Should().Throw(); + row.Invoking(r => r.AppendValue(tooLarge)).Should().Throw(); + + row.AppendValue(valid).EndRow(); + } + + Command.CommandText = "SELECT a::VARCHAR FROM uhugeintAppenderRangeTest"; + using var reader = Command.ExecuteReader(); + + reader.Read().Should().BeTrue(); + reader.GetString(0).Should().Be(valid.Value.ToString()); + reader.Read().Should().BeFalse(); + } + [Fact] public void Decimals() { @@ -935,7 +1021,7 @@ public void CollectionFinishedAfterFailedWriteHasNoNullItems(string columnType) using (var appender = Connection.CreateAppender(table)) { var row = appender.CreateRow(); - row.Invoking(r => r.AppendValue(new List { null, "bad" })).Should().Throw(); + row.Invoking(r => r.AppendValue(new List { null, "bad" })).Should().Throw(); row.AppendValue(new List { 1, 2 }).AppendValue("retried").EndRow(); } @@ -961,7 +1047,7 @@ public void NestedCollectionFinishedAfterFailedWriteHasNoNullItems(string column using (var appender = Connection.CreateAppender(table)) { var row = appender.CreateRow(); - row.Invoking(r => r.AppendValue(new List { null, "bad" })).Should().Throw(); + row.Invoking(r => r.AppendValue(new List { null, "bad" })).Should().Throw(); row.AppendValue(new List> { new() { 1, 2 }, new() { 3, 4 } }).AppendValue("retried").EndRow(); } @@ -988,7 +1074,7 @@ public void ListFinishedAfterFailedWriteHasNoNullItemsAfterListGrows() using (var appender = Connection.CreateAppender("managedAppenderListRetryGrows")) { var row = appender.CreateRow(); - row.Invoking(r => r.AppendValue(new List { null, "bad" })).Should().Throw(); + row.Invoking(r => r.AppendValue(new List { null, "bad" })).Should().Throw(); row.AppendValue(items).AppendValue("retried").EndRow(); } @@ -1013,7 +1099,7 @@ public void ListOfArraysFinishedAfterFailedWriteHasNoNullItemsAfterListGrows() using (var appender = Connection.CreateAppender("managedAppenderListOfArraysRetryGrows")) { var row = appender.CreateRow(); - row.Invoking(r => r.AppendValue(new List { null, "bad" })).Should().Throw(); + row.Invoking(r => r.AppendValue(new List { null, "bad" })).Should().Throw(); row.AppendValue(items).AppendValue("retried").EndRow(); } @@ -1188,7 +1274,7 @@ public void IncompleteRowWithPartialListIsNotWrittenOnDispose() appender.CreateRow().AppendValue(new List { 1 }).AppendValue("completed").EndRow(); var row = appender.CreateRow(); - row.Invoking(r => r.AppendValue(new List { 5, "bad" })).Should().Throw(); + row.Invoking(r => r.AppendValue(new List { 5, "bad" })).Should().Throw(); } Command.CommandText = "SELECT a, b FROM managedAppenderIncompleteList"; @@ -1210,7 +1296,7 @@ public void IncompleteRowWithPartialArrayIsNotWrittenOnDispose() appender.CreateRow().AppendValue(new List { 1, 2 }).AppendValue("completed").EndRow(); var row = appender.CreateRow(); - row.Invoking(r => r.AppendValue(new List { null, "bad" })).Should().Throw(); + row.Invoking(r => r.AppendValue(new List { null, "bad" })).Should().Throw(); } Command.CommandText = "SELECT a, b FROM managedAppenderIncompleteArray"; diff --git a/DuckDB.NET.Test/DuckDBMappedAppenderTests.cs b/DuckDB.NET.Test/DuckDBMappedAppenderTests.cs index 9186bd58..7dd93e4f 100644 --- a/DuckDB.NET.Test/DuckDBMappedAppenderTests.cs +++ b/DuckDB.NET.Test/DuckDBMappedAppenderTests.cs @@ -185,4 +185,352 @@ public void MappedAppender_SupportsDefaultAndNull() reader.GetInt32(2).Should().Be(18); reader.IsDBNull(3).Should().BeTrue(); } + + // One property for every type the mapped appender supports that the other tests do not use. + public class AllTypes + { + public bool Bool { get; set; } + public sbyte SByte { get; set; } + public short Short { get; set; } + public long Long { get; set; } + public byte Byte { get; set; } + public ushort UShort { get; set; } + public uint UInt { get; set; } + public ulong ULong { get; set; } + public double Double { get; set; } + public decimal Decimal { get; set; } + public DateTimeOffset DateTimeOffset { get; set; } + public TimeSpan TimeSpan { get; set; } + public Guid Guid { get; set; } + public BigInteger BigInteger { get; set; } + public DateOnly DateOnly { get; set; } + public TimeOnly TimeOnly { get; set; } + public int? NullableInt { get; set; } + public string NullableString { get; set; } + } + + public class AllTypesMap : DuckDBAppenderMap + { + public AllTypesMap() + { + Map(x => x.Bool); + Map(x => x.SByte); + Map(x => x.Short); + Map(x => x.Long); + Map(x => x.Byte); + Map(x => x.UShort); + Map(x => x.UInt); + Map(x => x.ULong); + Map(x => x.Double); + Map(x => x.Decimal); + Map(x => x.DateTimeOffset); + Map(x => x.TimeSpan); + Map(x => x.Guid); + Map(x => x.BigInteger); + Map(x => x.DateOnly); + Map(x => x.TimeOnly); + Map(x => x.NullableInt); + Map(x => x.NullableString); + } + } + + [Fact] + public void MappedAppender_SupportsAllMappedTypes() + { + Command.CommandText = """ + CREATE TABLE mapped_all_types( + c_bool BOOLEAN, c_tinyint TINYINT, c_smallint SMALLINT, c_bigint BIGINT, + c_utinyint UTINYINT, c_usmallint USMALLINT, c_uinteger UINTEGER, c_ubigint UBIGINT, + c_double DOUBLE, c_decimal DECIMAL(18, 4), c_timestamptz TIMESTAMPTZ, c_interval INTERVAL, + c_uuid UUID, c_hugeint HUGEINT, c_date DATE, c_time TIME, + c_nullable_int INTEGER, c_nullable_string VARCHAR); + """; + Command.ExecuteNonQuery(); + + var record = new AllTypes + { + Bool = true, + SByte = -8, + Short = -1600, + Long = -6_400_000_000_000, + Byte = 200, + UShort = 60_000, + UInt = 4_000_000_000, + ULong = 18_000_000_000_000_000_000, + Double = 1234.5678, + Decimal = 12345.6789m, + DateTimeOffset = new DateTimeOffset(2024, 5, 17, 13, 45, 30, 123, TimeSpan.Zero), + TimeSpan = new TimeSpan(3, 4, 5, 6, 789), + Guid = Guid.Parse("0f8fad5b-d9cb-469f-a165-70867728950e"), + BigInteger = BigInteger.Parse("123456789012345678901234567890"), + DateOnly = new DateOnly(2024, 5, 17), + TimeOnly = new TimeOnly(13, 45, 30, 123), + NullableInt = 42, + NullableString = "text", + }; + + // The same values again, but with the two nullable properties left null. + var recordWithNulls = new AllTypes + { + Bool = record.Bool, SByte = record.SByte, Short = record.Short, Long = record.Long, + Byte = record.Byte, UShort = record.UShort, UInt = record.UInt, ULong = record.ULong, + Double = record.Double, Decimal = record.Decimal, DateTimeOffset = record.DateTimeOffset, + TimeSpan = record.TimeSpan, Guid = record.Guid, BigInteger = record.BigInteger, + DateOnly = record.DateOnly, TimeOnly = record.TimeOnly, + }; + + using (var appender = Connection.CreateAppender("mapped_all_types")) + { + appender.AppendRecords(new[] { record, recordWithNulls }); + } + + Command.CommandText = "SELECT * FROM mapped_all_types ORDER BY rowid"; + using var reader = Command.ExecuteReader(); + + for (var row = 0; row < 2; row++) + { + reader.Read().Should().BeTrue(); + + reader.GetBoolean(0).Should().Be(record.Bool); + reader.GetFieldValue(1).Should().Be(record.SByte); + reader.GetInt16(2).Should().Be(record.Short); + reader.GetInt64(3).Should().Be(record.Long); + reader.GetByte(4).Should().Be(record.Byte); + reader.GetFieldValue(5).Should().Be(record.UShort); + reader.GetFieldValue(6).Should().Be(record.UInt); + reader.GetFieldValue(7).Should().Be(record.ULong); + reader.GetDouble(8).Should().Be(record.Double); + reader.GetDecimal(9).Should().Be(record.Decimal); + reader.GetFieldValue(10).UtcDateTime.Should().Be(record.DateTimeOffset.UtcDateTime); + reader.GetFieldValue(11).Should().Be(record.TimeSpan); + reader.GetGuid(12).Should().Be(record.Guid); + reader.GetFieldValue(13).Should().Be(record.BigInteger); + reader.GetFieldValue(14).Should().Be(record.DateOnly); + reader.GetFieldValue(15).Should().Be(record.TimeOnly); + } + + Command.CommandText = "SELECT c_nullable_int, c_nullable_string FROM mapped_all_types ORDER BY rowid"; + using var nullableReader = Command.ExecuteReader(); + + nullableReader.Read().Should().BeTrue(); + nullableReader.GetInt32(0).Should().Be(42); + nullableReader.GetString(1).Should().Be("text"); + + nullableReader.Read().Should().BeTrue(); + nullableReader.IsDBNull(0).Should().BeTrue(); + nullableReader.IsDBNull(1).Should().BeTrue(); + } + // The provider's own date and time types. DuckDBDateOnly can hold the infinite dates that DateOnly cannot. + public class DuckDBDateAndTime + { + public DuckDBDateOnly Date { get; set; } + public DuckDBTimeOnly Time { get; set; } + public DuckDBDateOnly? NullableDate { get; set; } + public DuckDBTimeOnly? NullableTime { get; set; } + } + + public class DuckDBDateAndTimeMap : DuckDBAppenderMap + { + public DuckDBDateAndTimeMap() + { + Map(x => x.Date); + Map(x => x.Time); + Map(x => x.NullableDate); + Map(x => x.NullableTime); + } + } + + [Fact] + public void MappedAppender_SupportsDuckDBDateOnlyAndTimeOnly() + { + Command.CommandText = "CREATE TABLE mapped_duckdb_date_time(d DATE, t TIME, nullable_d DATE, nullable_t TIME);"; + Command.ExecuteNonQuery(); + + var records = new[] + { + new DuckDBDateAndTime + { + Date = new DuckDBDateOnly(2024, 5, 17), + Time = new DuckDBTimeOnly(13, 45, 30, 123456), + NullableDate = new DuckDBDateOnly(2000, 1, 2), + NullableTime = new DuckDBTimeOnly(1, 2, 3), + }, + new DuckDBDateAndTime { Date = DuckDBDateOnly.PositiveInfinity, Time = new DuckDBTimeOnly(0, 0, 0) }, + new DuckDBDateAndTime { Date = DuckDBDateOnly.NegativeInfinity, Time = new DuckDBTimeOnly(23, 59, 59, 999999) }, + }; + + using (var appender = Connection.CreateAppender("mapped_duckdb_date_time")) + { + appender.AppendRecords(records); + } + + Command.CommandText = "SELECT d, t, nullable_d, nullable_t, d::VARCHAR FROM mapped_duckdb_date_time ORDER BY rowid"; + using var reader = Command.ExecuteReader(); + + reader.Read().Should().BeTrue(); + reader.GetFieldValue(0).Should().Be(records[0].Date); + reader.GetFieldValue(1).Should().Be(records[0].Time); + reader.GetFieldValue(2).Should().Be(records[0].NullableDate.Value); + reader.GetFieldValue(3).Should().Be(records[0].NullableTime.Value); + reader.GetString(4).Should().Be("2024-05-17"); + + reader.Read().Should().BeTrue(); + reader.GetFieldValue(0).IsPositiveInfinity.Should().BeTrue(); + reader.GetFieldValue(1).Should().Be(records[1].Time); + reader.IsDBNull(2).Should().BeTrue(); + reader.IsDBNull(3).Should().BeTrue(); + reader.GetString(4).Should().Be("infinity"); + + reader.Read().Should().BeTrue(); + reader.GetFieldValue(0).IsNegativeInfinity.Should().BeTrue(); + reader.GetFieldValue(1).Should().Be(records[2].Time); + reader.GetString(4).Should().Be("-infinity"); + } + + public class BigNumbers + { + public BigInteger Signed { get; set; } + public BigInteger Unsigned { get; set; } + } + + public class BigNumbersMap : DuckDBAppenderMap + { + public BigNumbersMap() + { + Map(x => x.Signed); + Map(x => x.Unsigned); + } + } + + public class NullableBigNumbers + { + public BigInteger? Signed { get; set; } + public BigInteger? Unsigned { get; set; } + } + + public class NullableBigNumbersMap : DuckDBAppenderMap + { + public NullableBigNumbersMap() + { + Map(x => x.Signed); + Map(x => x.Unsigned); + } + } + + [Fact] + public void MappedAppender_SupportsBigIntegerForUnsignedHugeInt() + { + Command.CommandText = "CREATE TABLE mapped_big_numbers(signed_value HUGEINT, unsigned_value UHUGEINT);"; + Command.ExecuteNonQuery(); + + // The second and third unsigned values do not fit a signed HUGEINT. + var records = new[] + { + new BigNumbers { Signed = -5, Unsigned = 5 }, + new BigNumbers { Signed = BigInteger.Pow(2, 126), Unsigned = BigInteger.Pow(2, 127) + 42 }, + new BigNumbers { Signed = DuckDBHugeInt.HugeIntMinValue, Unsigned = BigInteger.Pow(2, 128) - 1 }, + new BigNumbers { Signed = DuckDBHugeInt.HugeIntMaxValue, Unsigned = 0 }, + }; + + using (var appender = Connection.CreateAppender("mapped_big_numbers")) + { + appender.AppendRecords(records); + } + + Command.CommandText = "SELECT signed_value::VARCHAR, unsigned_value::VARCHAR FROM mapped_big_numbers ORDER BY rowid"; + using var reader = Command.ExecuteReader(); + + foreach (var record in records) + { + reader.Read().Should().BeTrue(); + reader.GetString(0).Should().Be(record.Signed.ToString()); + reader.GetString(1).Should().Be(record.Unsigned.ToString()); + } + } + + [Fact] + public void MappedAppender_RejectsNegativeBigIntegerForUnsignedHugeInt() + { + Command.CommandText = "CREATE TABLE mapped_big_numbers_negative(signed_value HUGEINT, unsigned_value UHUGEINT);"; + Command.ExecuteNonQuery(); + + using var appender = Connection.CreateAppender("mapped_big_numbers_negative"); + + appender.Invoking(a => a.AppendRecords(new[] { new BigNumbers { Signed = 1, Unsigned = -1 } })) + .Should().Throw(); + } + + [Fact] + public void MappedAppender_RejectsBigIntegerAboveUnsignedHugeIntRange() + { + Command.CommandText = "CREATE TABLE mapped_big_numbers_too_big(signed_value HUGEINT, unsigned_value UHUGEINT);"; + Command.ExecuteNonQuery(); + + using var appender = Connection.CreateAppender("mapped_big_numbers_too_big"); + + appender.Invoking(a => a.AppendRecords(new[] { new BigNumbers { Signed = 1, Unsigned = BigInteger.Pow(2, 128) } })) + .Should().Throw(); + } + + [Fact] + public void MappedAppender_SupportsNullableBigIntegerForUnsignedHugeInt() + { + Command.CommandText = "CREATE TABLE mapped_nullable_big_numbers(signed_value HUGEINT, unsigned_value UHUGEINT);"; + Command.ExecuteNonQuery(); + + var records = new[] + { + new NullableBigNumbers { Signed = -5, Unsigned = BigInteger.Pow(2, 127) + 42 }, + new NullableBigNumbers { Signed = null, Unsigned = null }, + new NullableBigNumbers { Signed = null, Unsigned = BigInteger.Pow(2, 128) - 1 }, + }; + + using (var appender = Connection.CreateAppender("mapped_nullable_big_numbers")) + { + appender.AppendRecords(records); + } + + Command.CommandText = "SELECT signed_value::VARCHAR, unsigned_value::VARCHAR FROM mapped_nullable_big_numbers ORDER BY rowid"; + using var reader = Command.ExecuteReader(); + + foreach (var record in records) + { + reader.Read().Should().BeTrue(); + reader.IsDBNull(0).Should().Be(record.Signed == null); + reader.IsDBNull(1).Should().Be(record.Unsigned == null); + + if (record.Signed != null) + { + reader.GetString(0).Should().Be(record.Signed.ToString()); + } + + if (record.Unsigned != null) + { + reader.GetString(1).Should().Be(record.Unsigned.ToString()); + } + } + } + + [Fact] + public void MappedAppender_NamesBothColumnTypesAcceptedForBigInteger() + { + Command.CommandText = "CREATE TABLE mapped_big_numbers_mismatch(signed_value HUGEINT, unsigned_value VARCHAR);"; + Command.ExecuteNonQuery(); + + Connection.Invoking(c => c.CreateAppender("mapped_big_numbers_mismatch")) + .Should().Throw() + .WithMessage("*column index 1*BigInteger*HugeInt or UnsignedHugeInt*Varchar*"); + } + + // The appender has no writer for BIGNUM, so the column is rejected instead of being written with the wrong layout. + [Fact] + public void MappedAppender_RejectsBigIntegerForBigNumColumn() + { + Command.CommandText = "CREATE TABLE mapped_big_numbers_bignum(signed_value HUGEINT, unsigned_value BIGNUM);"; + Command.ExecuteNonQuery(); + + Connection.Invoking(c => c.CreateAppender("mapped_big_numbers_bignum")) + .Should().Throw() + .WithMessage("*column index 1*Mapped type is BigInteger *HugeInt or UnsignedHugeInt*"); + } } diff --git a/DuckDB.NET.Test/Parameters/HugeIntParameterTests.cs b/DuckDB.NET.Test/Parameters/HugeIntParameterTests.cs index 60ee0f16..ff583bff 100644 --- a/DuckDB.NET.Test/Parameters/HugeIntParameterTests.cs +++ b/DuckDB.NET.Test/Parameters/HugeIntParameterTests.cs @@ -50,7 +50,8 @@ public void BindValueTest() [Fact] public void SimpleNegativeHugeIntTest() { - Command.CommandText = $"SELECT {DuckDBHugeInt.HugeIntMinValue}::HUGEINT;"; + // The parentheses matter: a cast binds tighter than the minus sign, and 2^127 itself is not a HUGEINT. + Command.CommandText = $"SELECT ({DuckDBHugeInt.HugeIntMinValue})::HUGEINT;"; Command.ExecuteNonQuery(); var scalar = Command.ExecuteScalar(); @@ -83,6 +84,46 @@ public void BindNegativeHugeIntValueTest() receivedValue.Should().Be(value); } + [Fact] + public void HugeIntMinimumAndMaximumRoundTrip() + { + // HUGEINT is a 128-bit two's complement integer: -2^127 to 2^127 - 1. + DuckDBHugeInt.HugeIntMinValue.Should().Be(-BigInteger.Pow(2, 127)); + DuckDBHugeInt.HugeIntMaxValue.Should().Be(BigInteger.Pow(2, 127) - 1); + + Command.CommandText = "CREATE OR REPLACE TABLE HugeIntRangeTests (value HUGEINT);"; + Command.ExecuteNonQuery(); + + BigInteger? minimum = DuckDBHugeInt.HugeIntMinValue; + BigInteger? maximum = DuckDBHugeInt.HugeIntMaxValue; + + // Once as parameters and once through the appender. + Command.CommandText = "INSERT INTO HugeIntRangeTests VALUES ($1), ($2);"; + Command.Parameters.Add(new DuckDBParameter(minimum.Value)); + Command.Parameters.Add(new DuckDBParameter(maximum.Value)); + Command.ExecuteNonQuery(); + Command.Parameters.Clear(); + + using (var appender = Connection.CreateAppender("HugeIntRangeTests")) + { + appender.CreateRow().AppendValue(minimum).EndRow(); + appender.CreateRow().AppendValue(maximum).EndRow(); + } + + Command.CommandText = "SELECT value::VARCHAR FROM HugeIntRangeTests ORDER BY rowid;"; + using var reader = Command.ExecuteReader(); + + foreach (var expected in new[] { minimum.Value, maximum.Value, minimum.Value, maximum.Value }) + { + reader.Read().Should().BeTrue(); + reader.GetString(0).Should().Be(expected.ToString()); + } + + // One step outside the range on either side is rejected. + FluentActions.Invoking(() => new DuckDBHugeInt(minimum.Value - 1)).Should().Throw(); + FluentActions.Invoking(() => new DuckDBHugeInt(maximum.Value + 1)).Should().Throw(); + } + [Fact] public void BindParameterWithoutTable_HugeInt() { @@ -97,4 +138,80 @@ public void BindParameterWithoutTable_HugeInt() result.Should().BeOfType().Subject .Should().Be(value); } + // 2^127 + 42 and 2^128 - 1 fit UHUGEINT but not HUGEINT. + public static IEnumerable UnsignedHugeIntValues() => + [ + [BigInteger.Zero], + [BigInteger.Pow(2, 127) + 42], + [BigInteger.Pow(2, 128) - 1], + ]; + + [Theory] + [MemberData(nameof(UnsignedHugeIntValues))] + public void BindBigIntegerToUnsignedHugeIntColumn(BigInteger value) + { + Command.CommandText = "CREATE OR REPLACE TABLE UHugeIntParameterTests (value UHUGEINT);"; + Command.ExecuteNonQuery(); + + Command.CommandText = "INSERT INTO UHugeIntParameterTests VALUES ($1);"; + Command.Parameters.Add(new DuckDBParameter(value)); + Command.ExecuteNonQuery(); + + // The parameter is compared with the UHUGEINT column, so DuckDB reports its type as UHUGEINT here too. + Command.CommandText = "SELECT value::VARCHAR FROM UHugeIntParameterTests WHERE value = $1;"; + + Command.ExecuteScalar().Should().Be(value.ToString()); + } + + [Fact] + public void BindOutOfRangeBigIntegerToUnsignedHugeIntColumnThrows() + { + Command.CommandText = "CREATE OR REPLACE TABLE UHugeIntParameterRangeTests (value UHUGEINT);"; + Command.ExecuteNonQuery(); + + Command.CommandText = "INSERT INTO UHugeIntParameterRangeTests VALUES ($1);"; + + foreach (var value in new[] { BigInteger.MinusOne, BigInteger.Pow(2, 128) }) + { + Command.Parameters.Clear(); + Command.Parameters.Add(new DuckDBParameter(value)); + + Command.Invoking(command => command.ExecuteNonQuery()).Should().Throw(); + } + } + + // With a cast or no context, DuckDB reports no parameter type and the value picks its own type. + [Theory] + [InlineData("SELECT ($1::UHUGEINT)::VARCHAR;")] + [InlineData("SELECT $1::VARCHAR;")] + public void BindBigIntegerAboveHugeIntRangeWithoutColumn(string query) + { + var value = BigInteger.Pow(2, 127) + 42; + + Command.CommandText = query; + Command.Parameters.Add(new DuckDBParameter(value)); + + Command.ExecuteScalar().Should().Be(value.ToString()); + } + + [Theory] + [InlineData("170141183460469231731687303715884105727", "HUGEINT")] + [InlineData("-170141183460469231731687303715884105728", "HUGEINT")] + [InlineData("170141183460469231731687303715884105728", "UHUGEINT")] + [InlineData("340282366920938463463374607431768211455", "UHUGEINT")] + [InlineData("340282366920938463463374607431768211456", "BIGNUM")] + [InlineData("-170141183460469231731687303715884105729", "BIGNUM")] + public void BigIntegerWithoutColumnPicksTheSmallestTypeThatFits(string text, string expectedType) + { + var value = BigInteger.Parse(text); + + Command.CommandText = "SELECT typeof($1), $1::VARCHAR;"; + Command.Parameters.Add(new DuckDBParameter(value)); + + using var reader = Command.ExecuteReader(); + reader.Read().Should().BeTrue(); + + reader.GetString(0).Should().Be(expectedType); + reader.GetString(1).Should().Be(text); + } } \ No newline at end of file diff --git a/DuckDB.NET.Test/Parameters/IntervalTests.cs b/DuckDB.NET.Test/Parameters/IntervalTests.cs new file mode 100644 index 00000000..08e8c8f4 --- /dev/null +++ b/DuckDB.NET.Test/Parameters/IntervalTests.cs @@ -0,0 +1,171 @@ +namespace DuckDB.NET.Test.Parameters; + +// DuckDB intervals can be negative: duckdb_interval.micros is a signed 64-bit value. +public class IntervalTests(DuckDBDatabaseFixture db) : DuckDBTestBase(db) +{ + [Theory] + [InlineData("-90 minutes", -90 * 60)] + [InlineData("-1 day", -86_400)] + [InlineData("-1 day 2 hours", -79_200)] + [InlineData("1 day -2 hours", 79_200)] + public void QueryNegativeInterval(string interval, int seconds) + { + Command.CommandText = $"SELECT INTERVAL '{interval}';"; + + using var reader = Command.ExecuteReader(); + reader.Read(); + + reader.GetValue(0).Should().Be(TimeSpan.FromSeconds(seconds)); + reader.GetFieldValue(0).Should().Be(TimeSpan.FromSeconds(seconds)); + } + + [Theory] + [InlineData("-1 month")] + [InlineData("1 month")] + public void QueryMonthInterval_AsTimeSpan_Throws(string interval) + { + Command.CommandText = $"SELECT INTERVAL '{interval}';"; + + using var reader = Command.ExecuteReader(); + reader.Read(); + + reader.Invoking(r => r.GetFieldValue(0)).Should().Throw(); + reader.GetFieldValue(0).Months.Should().Be(interval.StartsWith('-') ? -1 : 1); + } + + [Theory] + [InlineData(-5_400_000_000L)] + [InlineData(-1L)] + [InlineData(-90_061_000_001L)] + public void InsertNegativeTimeSpanParameter(long micros) + { + var expected = TimeSpan.FromTicks(micros * 10); + + Command.CommandText = "CREATE OR REPLACE TABLE NegativeIntervalParameterTable (v INTERVAL);"; + Command.ExecuteNonQuery(); + + Command.CommandText = "INSERT INTO NegativeIntervalParameterTable (v) VALUES (?);"; + Command.Parameters.Add(new DuckDBParameter(expected)); + Command.ExecuteNonQuery(); + Command.Parameters.Clear(); + + Command.CommandText = $"SELECT v, v = CAST('{micros} microseconds' AS INTERVAL) FROM NegativeIntervalParameterTable;"; + using var reader = Command.ExecuteReader(); + reader.Read(); + + reader.GetFieldValue(0).Should().Be(expected); + reader.GetBoolean(1).Should().BeTrue(); + } + + [Fact] + public void AppendNegativeTimeSpan() + { + Command.CommandText = "CREATE TABLE NegativeIntervalTable (v INTERVAL);"; + Command.ExecuteNonQuery(); + var expected = TimeSpan.FromMinutes(-90).Add(TimeSpan.FromTicks(-10)); + + using (var appender = Connection.CreateAppender("NegativeIntervalTable")) + { + appender.CreateRow().AppendValue((TimeSpan?)expected).EndRow(); + } + + Command.CommandText = "SELECT v, v = INTERVAL '-5400000001 microseconds' FROM NegativeIntervalTable;"; + using var reader = Command.ExecuteReader(); + reader.Read(); + + reader.GetFieldValue(0).Should().Be(expected); + reader.GetBoolean(1).Should().BeTrue(); + } + + // A TimeSpan has no DbType, so where DuckDB can't infer the parameter's type (a bare "?") it was + // sent as TimeSpan.ToString(), which DuckDB can't parse as an interval once there are days + // ("1.01:01:01"), and which came back as a string. + [Theory] + [InlineData(90_061_000_001L)] + [InlineData(-90_061_000_001L)] + [InlineData(5_400_000_000L)] + public void BindTimeSpanWithoutTargetType(long micros) + { + var expected = TimeSpan.FromTicks(micros * 10); + + Command.CommandText = "SELECT ?::INTERVAL, ?;"; + Command.Parameters.Add(new DuckDBParameter(expected)); + Command.Parameters.Add(new DuckDBParameter(expected)); + + using var reader = Command.ExecuteReader(); + reader.Read(); + + reader.GetFieldValue(0).Should().Be(expected); + reader.GetValue(1).Should().Be(expected); + } + + // A TimeSpan bound where DuckDB knows the parameter's type keeps its existing behavior. + [Fact] + public void BindTimeSpanIntoTimeAndVarcharColumns() + { + Command.CommandText = "CREATE OR REPLACE TABLE TimeSpanTargets (t TIME, s VARCHAR);"; + Command.ExecuteNonQuery(); + + Command.CommandText = "INSERT INTO TimeSpanTargets (t, s) VALUES (?, ?);"; + Command.Parameters.Add(new DuckDBParameter(new TimeSpan(0, 13, 45, 30, 125))); + Command.Parameters.Add(new DuckDBParameter(new TimeSpan(0, 1, 30, 0))); + Command.ExecuteNonQuery(); + Command.Parameters.Clear(); + + Command.CommandText = "SELECT t, s FROM TimeSpanTargets;"; + using var reader = Command.ExecuteReader(); + reader.Read(); + + reader.GetFieldValue(0).Should().Be(new TimeOnly(13, 45, 30, 125)); + reader.GetString(1).Should().Be("01:30:00"); + } + + // As in Npgsql, a caller who wants a TimeSpan sent as a time declares DbType.Time: where DuckDB can't + // infer the type, an interval has no cast to TIME. + [Theory] + [InlineData(0, 0, 0, 0)] + [InlineData(13, 45, 30, 125)] + [InlineData(23, 59, 59, 999)] + public void InsertSelectTimeSpanDeclaredTime(int hours, int minutes, int seconds, int milliseconds) + { + var value = new TimeSpan(0, hours, minutes, seconds, milliseconds); + + Command.CommandText = "CREATE OR REPLACE TABLE TimeSpanDeclaredTime (t TIME);"; + Command.ExecuteNonQuery(); + + Command.CommandText = "INSERT INTO TimeSpanDeclaredTime (t) SELECT ?;"; + Command.Parameters.Add(new DuckDBParameter(DbType.Time, value)); + Command.ExecuteNonQuery(); + Command.Parameters.Clear(); + + Command.CommandText = "SELECT t FROM TimeSpanDeclaredTime;"; + Command.ExecuteScalar().Should().Be(TimeOnly.FromTimeSpan(value)); + } + + [Fact] + public void SelectTimeSpanDeclaredTime() + { + Command.CommandText = "SELECT ?::TIME::VARCHAR, ? UNION ALL SELECT '', TIME '01:00';"; + Command.Parameters.Add(new DuckDBParameter(DbType.Time, new TimeSpan(0, 1, 30, 0))); + Command.Parameters.Add(new DuckDBParameter(DbType.Time, new TimeSpan(0, 2, 0, 0))); + + using var reader = Command.ExecuteReader(); + reader.Read(); + + reader.GetString(0).Should().Be("01:30:00"); + reader.GetFieldValue(1).Should().Be(new TimeOnly(2, 0)); + } + + [Theory] + [InlineData(1, 0)] + [InlineData(0, -1)] + public void TimeSpanDeclaredTime_OutOfRange_Throws(int days, int seconds) + { + var value = new TimeSpan(days, 0, 0, seconds); + + Command.CommandText = "SELECT ?;"; + Command.Parameters.Add(new DuckDBParameter(DbType.Time, value)); + + Command.Invoking(c => c.ExecuteScalar()).Should().Throw(); + } +} diff --git a/DuckDB.NET.Test/Parameters/ParameterCollectionTests.cs b/DuckDB.NET.Test/Parameters/ParameterCollectionTests.cs index c4c2f896..786c93be 100644 --- a/DuckDB.NET.Test/Parameters/ParameterCollectionTests.cs +++ b/DuckDB.NET.Test/Parameters/ParameterCollectionTests.cs @@ -1,4 +1,5 @@ -using DuckDB.NET.Test.Helpers; +using System.Data.Common; +using DuckDB.NET.Test.Helpers; namespace DuckDB.NET.Test.Parameters; @@ -109,6 +110,61 @@ public void DuckDBParameterCollectionTests() parameters.Count.Should().Be(0); } + // Generic data code holds the collection as DbParameterCollection, which routes to the object-typed members. + [Fact] + public void ParameterCollectionWorksThroughTheBaseType() + { + DbParameterCollection parameters = new DuckDBParameterCollection(); + var first = new DuckDBParameter("first", 1); + var second = new DuckDBParameter("second", 2); + var third = new DuckDBParameter("third", 3); + + parameters.AddRange(new DbParameter[] { first, third }); + parameters.Insert(1, second); + + parameters.Count.Should().Be(3); + parameters.SyncRoot.Should().NotBeNull(); + parameters.Cast().Should().Equal(first, second, third); + + parameters.Contains((object)second).Should().BeTrue(); + parameters.Contains("second").Should().BeTrue(); + parameters.IndexOf((object)third).Should().Be(2); + parameters[1].Should().BeSameAs(second); + parameters["third"].Should().BeSameAs(third); + + var copy = new DuckDBParameter[4]; + parameters.CopyTo(copy, 1); + copy.Should().Equal(new DuckDBParameter[] { null, first, second, third }); + + var typedCopy = new DuckDBParameter[3]; + ((DuckDBParameterCollection)parameters).CopyTo(typedCopy, 0); + typedCopy.Should().Equal(first, second, third); + + var replacement = new DuckDBParameter("replacement", 4); + parameters[0] = replacement; + parameters["second"] = first; + parameters.Cast().Should().Equal(replacement, first, third); + + parameters.Remove((object)third); + parameters.Contains((object)third).Should().BeFalse(); + parameters.Count.Should().Be(2); + } + + [Fact] + public void MissingParameterNameIsReportedTheSameWayOnEveryPath() + { + var typedParameters = new DuckDBParameterCollection { new DuckDBParameter("present", 1) }; + DbParameterCollection parameters = typedParameters; + + parameters.Invoking(p => p["missing"]).Should().Throw().WithMessage("*'missing'*"); + parameters.Invoking(p => p["missing"] = new DuckDBParameter("other", 2)).Should().Throw().WithMessage("*'missing'*"); + parameters.Invoking(p => p.RemoveAt("missing")).Should().Throw().WithMessage("*'missing'*"); + typedParameters.Invoking(p => p["missing"]).Should().Throw().WithMessage("*'missing'*"); + + parameters.Contains("missing").Should().BeFalse(); + parameters.IndexOf("missing").Should().Be(-1); + } + [Theory] [InlineData("INSERT INTO ParametersTestKeyValue (KEY, VALUE) VALUES (?, ?)")] [InlineData("INSERT INTO ParametersTestKeyValue (KEY, VALUE) VALUES (?1, ?2)")] diff --git a/DuckDB.NET.Test/ScalarFunctionTests.cs b/DuckDB.NET.Test/ScalarFunctionTests.cs index 336cf8e2..f4bf788b 100644 --- a/DuckDB.NET.Test/ScalarFunctionTests.cs +++ b/DuckDB.NET.Test/ScalarFunctionTests.cs @@ -113,13 +113,15 @@ public void RegisterScalarFunctionCallbackThrows() [Fact] public void RegisterScalarFunctionCallbackThrowsInLaterStreamingChunk() { - Connection.RegisterScalarFunction("throwing_late_scalar", + using var connection = OpenConnectionForLateStreamingErrors(); + connection.RegisterScalarFunction("throwing_late_scalar", x => x < 500_000 ? x : throw new InvalidOperationException("Scalar callback failed late")); - Command.UseStreamingMode = true; - Command.CommandText = "SELECT throwing_late_scalar(i) FROM range(1000000) t(i)"; + using var command = connection.CreateCommand(); + command.UseStreamingMode = true; + command.CommandText = "SELECT throwing_late_scalar(i) FROM range(1000000) t(i)"; - using (var reader = Command.ExecuteReader()) + using (var reader = command.ExecuteReader()) { var rows = 0; var act = () => @@ -137,14 +139,35 @@ public void RegisterScalarFunctionCallbackThrowsInLaterStreamingChunk() } // The stored callback exception was consumed, so it must not leak into the next error on this connection. - Command.UseStreamingMode = false; - Command.CommandText = "SELECT CAST('not a number' AS INTEGER)"; + command.UseStreamingMode = false; + command.CommandText = "SELECT CAST('not a number' AS INTEGER)"; - Command.Invoking(command => command.ExecuteScalar()) + command.Invoking(c => c.ExecuteScalar()) .Should().Throw() .Where(e => e.InnerException == null); } + [Fact] + public void RegisterScalarFunctionWithDuckDBDateOnlyAndTimeOnly() + { + Connection.RegisterScalarFunction("same_duckdb_date", date => date); + Connection.RegisterScalarFunction("same_duckdb_time", time => time); + + Command.CommandText = """ + SELECT same_duckdb_date(DATE '2024-05-17')::VARCHAR, same_duckdb_date('infinity'::DATE)::VARCHAR, + same_duckdb_time(TIME '13:45:30.123456')::VARCHAR, + typeof(same_duckdb_date(DATE '2024-05-17')), typeof(same_duckdb_time(TIME '13:45:30')) + """; + using var reader = Command.ExecuteReader(); + + reader.Read().Should().BeTrue(); + reader.GetString(0).Should().Be("2024-05-17"); + reader.GetString(1).Should().Be("infinity"); + reader.GetString(2).Should().Be("13:45:30.123456"); + reader.GetString(3).Should().Be("DATE"); + reader.GetString(4).Should().Be("TIME"); + } + [Fact] public void RegisterScalarFunctionIsPrime() { diff --git a/DuckDB.NET.Test/SchemaTests.cs b/DuckDB.NET.Test/SchemaTests.cs index f9c2a4f4..e34885dd 100644 --- a/DuckDB.NET.Test/SchemaTests.cs +++ b/DuckDB.NET.Test/SchemaTests.cs @@ -20,6 +20,17 @@ public void InvalidCollection() { Assert.Throws(() => Connection.GetSchema("Invalid")); } + + [Theory] + [InlineData("MetaDataCollections")] + [InlineData("DataSourceInformation")] + [InlineData("DataTypes")] + [InlineData("Restrictions")] + [InlineData("ReservedWords")] + public void CollectionsWithoutRestrictionsRejectRestrictionValues(string collectionName) + { + Assert.Throws(() => Connection.GetSchema(collectionName, ["unexpected"])); + } [Fact] public void MetaDataCollections() @@ -119,6 +130,35 @@ public void ColumnsWithRestrictions() Assert.Equal("foo_id", schema.Rows[0]["column_name"]); } + [Fact] + public void ColumnsExposeDataType() + { + var schema = Connection.GetSchema("Columns", [null, null, "foo", "foo_id"]); + Assert.Equal("INTEGER", schema.Rows[0]["data_type"]); + } + + [Fact] + public void DataSourceInformationSupportsProbing() + { + var schema = Connection.GetSchema(DbMetaDataCollectionNames.DataSourceInformation); + var row = Assert.IsType(Assert.Single(schema.Rows)); + Assert.Equal("DuckDB", row["DataSourceProductName"]); + Assert.Equal(Connection.ServerVersion, row["DataSourceProductVersion"]); + Assert.Equal("${0}", row["ParameterMarkerFormat"]); + } + + [Fact] + public void DataTypesExposeStandardAdoNetMetadata() + { + var schema = Connection.GetSchema("DataTypes"); + var integer = schema.Rows.Cast().Single(row => (string)row["TypeName"] == "INTEGER"); + + Assert.Equal(typeof(int).FullName, integer["DataType"]); + Assert.Equal((int)DuckDBType.Integer, integer["ProviderDbType"]); + Assert.True((bool)integer["IsBestMatch"]); + Assert.True((bool)integer["IsSearchable"]); + } + [Fact] public void ColumnsTooManyRestrictions() { @@ -173,4 +213,4 @@ public void IndexesTooManyRestrictions() private static IEnumerable GetValues(DataTable schema, string columnName) => schema.Rows.Cast().Select(r => (string)r[columnName]); -} \ No newline at end of file +} diff --git a/DuckDB.NET.Test/TableFunctionTests.cs b/DuckDB.NET.Test/TableFunctionTests.cs index 0120e8e4..91c5ccf7 100644 --- a/DuckDB.NET.Test/TableFunctionTests.cs +++ b/DuckDB.NET.Test/TableFunctionTests.cs @@ -305,6 +305,41 @@ public void RegisterFunctionWithDateOnlyTimeOnlyParameters() data.Should().BeEquivalentTo([new DateTime(2024, 11, 6, 10, 30, 24)]); } + [Fact] + public void RegisterFunctionWithDuckDBDateOnlyTimeOnlyParametersAndColumns() + { + Connection.RegisterTableFunction("duckdb_date_time_rows", parameters => + { + var date = parameters[0].GetValue(); + var time = parameters[1].GetValue(); + + return new TableFunction(new List + { + new("d", typeof(DuckDBDateOnly)), + new("t", typeof(DuckDBTimeOnly)), + }, new[] { (date, time), (DuckDBDateOnly.PositiveInfinity, time) }); + }, (item, writers, rowIndex) => + { + var (date, time) = ((DuckDBDateOnly, DuckDBTimeOnly))item; + + writers[0].WriteValue(date, rowIndex); + writers[1].WriteValue(time, rowIndex); + }); + + Command.CommandText = "SELECT d::VARCHAR, t::VARCHAR, typeof(d), typeof(t) FROM duckdb_date_time_rows(DATE '2024-11-06', TIME '10:30:24.123456')"; + using var reader = Command.ExecuteReader(); + + reader.Read().Should().BeTrue(); + reader.GetString(0).Should().Be("2024-11-06"); + reader.GetString(1).Should().Be("10:30:24.123456"); + reader.GetString(2).Should().Be("DATE"); + reader.GetString(3).Should().Be("TIME"); + + reader.Read().Should().BeTrue(); + reader.GetString(0).Should().Be("infinity"); + reader.GetString(1).Should().Be("10:30:24.123456"); + } + [Fact] public void RegisterTableFunctionWithExplicitTimestampTypes() { diff --git a/DuckDB.NET.Test/Test.csproj b/DuckDB.NET.Test/Test.csproj index 37ced068..0e01f370 100644 --- a/DuckDB.NET.Test/Test.csproj +++ b/DuckDB.NET.Test/Test.csproj @@ -11,10 +11,10 @@ - + - +