From 3f8776b1ec9fde1933179abed5d37da9555ada7b Mon Sep 17 00:00:00 2001 From: XperiAndri Date: Sat, 10 Oct 2026 03:33:46 +0200 Subject: [PATCH] test: compare SQL with golden baselines that can be rewritten ADR 0001 plans golden baselines for the SQL the translator produces: files checked into the test project, compared regardless of whitespace, and rewritten on request instead of edited by hand. Baseline in the shared test infrastructure provides them before their first users arrive, the printer tests of the SQL model and later the translator tests. Baseline.assertMatches compares the SQL with Baselines//.sql after collapsing whitespace, and the parameters with .params.json as JSON; a query without parameters has no parameter file. With COSMOS_REWRITE_BASELINES=1 it writes both files instead, keeping the layout of the SQL and writing the JSON indented with LF line breaks, and reports the test inconclusive, so that a rewriting run never looks like a passing one. A run on CI, where CI is true, refuses to rewrite. Baseline.directoryOf finds the Baselines directory next to the project file of a test assembly by walking up from where the assembly runs, which does not depend on how the compiler records source paths. BaselineTests covers matching, both kinds of difference, a missing baseline, rewriting with and without parameters, invalid case names and finding the directory. The agent instructions list the new environment variable. Co-Authored-By: Claude Opus 5.5 --- .github/copilot-instructions.md | 1 + tests/Cosmos.Tests.Infrastructure/Baseline.fs | 195 +++++++++++++++++ ...p.Azure.Cosmos.Tests.Infrastructure.fsproj | 1 + tests/Cosmos.Tests/BaselineTests.fs | 201 ++++++++++++++++++ .../FSharp.Azure.Cosmos.Tests.fsproj | 1 + 5 files changed, 399 insertions(+) create mode 100644 tests/Cosmos.Tests.Infrastructure/Baseline.fs create mode 100644 tests/Cosmos.Tests/BaselineTests.fs diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 15f3518e..46978db5 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -237,6 +237,7 @@ module MyTypeExtensions = * `COSMOS_EMULATOR_KEY` overrides the emulator key the tests authenticate with. * `COSMOS_EMULATOR_PARTITION_COUNT` is the number of partitions the emulator offers (default `25`, the Windows emulator's default `/PartitionCount`); the fixtures keep at most that many test containers at a time, one partition each. * `COSMOS_TEST_LEFTOVER_AGE_MINUTES` is how many minutes a `fsac-test-` database must stay unmodified before the leftover sweep of `[]` and `[]` deletes it (default `60`), so that the live databases of another test process on the same emulator are kept; `0` deletes every test database and is safe only while no other test process uses the emulator. +* `COSMOS_REWRITE_BASELINES` set to `1` or `true` makes a test that compares SQL with its golden baseline (`Baseline.assertMatches`, files under `Baselines//` of the test project) write what it produced into the baseline instead of comparing; the test then reports itself inconclusive, the rewritten files are reviewed in the diff, and CI refuses to rewrite. * Every test project under `tests/` references `tests/Cosmos.Tests.Infrastructure` through `tests/Directory.Build.props`; a new test project gets it without a `ProjectReference` of its own. * `CollectionAssert` cannot work with F# lists – use F# array syntax (`[| ... |]`) instead. * `StringAssert` has overloads with `StringComparison`. diff --git a/tests/Cosmos.Tests.Infrastructure/Baseline.fs b/tests/Cosmos.Tests.Infrastructure/Baseline.fs new file mode 100644 index 00000000..e485c642 --- /dev/null +++ b/tests/Cosmos.Tests.Infrastructure/Baseline.fs @@ -0,0 +1,195 @@ +/// +/// Golden baselines: the SQL text and the parameters a test produces, compared with files checked into the test project, +/// Baselines/<suite>/<case>.sql and Baselines/<suite>/<case>.params.json. +/// +/// +/// +/// The SQL is compared after has collapsed its whitespace, so that a baseline may be laid out +/// for reading, and the parameters are compared as JSON, so that their layout does not matter either. A query without +/// parameters has no parameter file. +/// +/// +/// With the environment variable set to 1 or true, a test writes what it +/// produced into its baseline files instead of comparing them, and reports itself inconclusive, so that a rewriting run +/// is never mistaken for a passing one; the rewritten files are then reviewed in the diff. A run on CI, where the +/// environment variable CI is true, refuses to rewrite. +/// +/// +[] +module FSharp.Azure.Cosmos.Tests.Baseline + +open System +open System.IO +open System.Reflection +open System.Text +open System.Text.Json +open System.Text.Json.Nodes + +open Microsoft.VisualStudio.TestTools.UnitTesting + +/// The environment variable that makes the tests rewrite their baselines instead of comparing them. +[] +let RewriteVariable = "COSMOS_REWRITE_BASELINES" + +/// The result of comparing what a test produced with its baseline. +[] +type Comparison = + /// The produced SQL and parameters match the baseline. + | Matches + /// The baseline does not exist yet; the path is that of its SQL file. + | Missing of sqlPath : string + /// The produced SQL or parameters differ from the baseline, both given in the form they were compared in. + | Differs of expected : string * actual : string + /// The baseline was written from what the test produced; the path is that of its SQL file. + | Rewritten of sqlPath : string + +// Characters that make a suite or case name unusable as part of a file path on any platform +let private invalidNameCharacters = + seq { + yield! Path.GetInvalidFileNameChars () + yield! [| '/'; '\\'; ':'; '*'; '?'; '"'; '<'; '>'; '|' |] + } + |> Seq.distinct + |> Seq.toArray + +let private ensureValidName (parameterName : string) (name : string) = + if + String.IsNullOrWhiteSpace name + || name.IndexOfAny invalidNameCharacters >= 0 + then + invalidArg parameterName $"'{name}' cannot be part of the name of a baseline file." + +// LF on every platform, so that a baseline does not change with the system that rewrote it +let private indented = JsonSerializerOptions (WriteIndented = true, NewLine = "\n") + +/// +/// Collapses every run of whitespace in into one space and removes the whitespace at both ends, +/// so that line breaks and indentation do not count when SQL is compared with its baseline. +/// +let normalizeSql (sql : string) : string = + let builder = StringBuilder sql.Length + let mutable pendingSpace = false + let trimmed = sql.AsSpan().Trim() + + for index in 0 .. trimmed.Length - 1 do + let character = trimmed[index] + + if Char.IsWhiteSpace character then + pendingSpace <- true + else + if pendingSpace then + builder.Append ' ' |> ignore + pendingSpace <- false + + builder.Append character |> ignore + + builder.ToString () + +/// +/// Finds the Baselines directory of the test project that built : the directory next +/// to the project file, found by walking up from the directory the assembly runs from. +/// +/// No directory above the assembly holds its project file. +let directoryOf (assembly : Assembly) : string = + let projectFileName = $"{assembly.GetName().Name}.fsproj" + + let rec find (directory : DirectoryInfo | null) = + match directory with + | null -> + invalidOp + $"No directory above '{assembly.Location}' holds '{projectFileName}', so the baselines of the test project cannot be found." + | directory when File.Exists (Path.Combine (directory.FullName, projectFileName)) -> + Path.Combine (directory.FullName, "Baselines") + | directory -> find directory.Parent + + match Path.GetDirectoryName assembly.Location with + | null -> find null + | assemblyDirectory -> find (DirectoryInfo assemblyDirectory) + +/// +/// Compares and , the parameter names mapped to their values, with +/// the baseline of under , or, when +/// is , writes them as that baseline. +/// +/// The suite or the case name cannot be part of a file name. +let compare + (rewrite : bool) + (directory : string) + (suite : string) + (caseName : string) + (sql : string) + (parameters : JsonObject) + : Comparison = + ensureValidName (nameof suite) suite + ensureValidName (nameof caseName) caseName + let suiteDirectory = Path.Combine (directory, suite) + let sqlPath = Path.Combine (suiteDirectory, $"{caseName}.sql") + let parametersPath = Path.Combine (suiteDirectory, $"{caseName}.params.json") + + if rewrite then + Directory.CreateDirectory suiteDirectory |> ignore + File.WriteAllText (sqlPath, sql.Trim () + "\n") + + if parameters.Count = 0 then + // A query that lost its parameters must not keep a parameter file from before + File.Delete parametersPath + else + File.WriteAllText (parametersPath, parameters.ToJsonString indented + "\n") + + Comparison.Rewritten sqlPath + elif not (File.Exists sqlPath) then + Comparison.Missing sqlPath + else + let expectedSql = normalizeSql (File.ReadAllText sqlPath) + + let expectedParameters = + if File.Exists parametersPath then + nonNull (JsonNode.Parse (File.ReadAllText parametersPath)) + else + JsonObject () + + let actualSql = normalizeSql sql + + if + String.Equals (expectedSql, actualSql, StringComparison.Ordinal) + && JsonNode.DeepEquals (expectedParameters, parameters) + then + Comparison.Matches + else + Comparison.Differs ( + $"{expectedSql}\n{expectedParameters.ToJsonString ()}", + $"{actualSql}\n{parameters.ToJsonString ()}" + ) + +let private isSet (variable : string) = + match Environment.GetEnvironmentVariable variable with + | null -> false + | value -> + let value = value.AsSpan().Trim() + value.Equals ("1", StringComparison.Ordinal) + || value.Equals ("true", StringComparison.OrdinalIgnoreCase) + +/// +/// Fails the test unless and match the baseline +/// of under , usually +/// the test assembly. With set, writes the baseline instead and +/// reports the test inconclusive. +/// +let assertMatches (directory : string) (suite : string) (caseName : string) (sql : string) (parameters : JsonObject) = + let rewrite = isSet RewriteVariable + + if rewrite && isSet "CI" then + Assert.Fail $"{RewriteVariable} is set on CI, where baselines are never rewritten." + + match compare rewrite directory suite caseName sql parameters with + | Comparison.Matches -> () + | Comparison.Missing sqlPath -> + Assert.Fail $"There is no baseline at {sqlPath}. Run the test with {RewriteVariable}=1 to write it, then review it." + | Comparison.Differs (expected, actual) -> + Assert.AreEqual ( + expected, + actual, + $"The SQL or the parameters of {suite}/{caseName} differ from the baseline. Run the test with {RewriteVariable}=1 to accept them, then review the diff." + ) + | Comparison.Rewritten sqlPath -> + Assert.Inconclusive $"Rewrote the baseline at {sqlPath}. Review it and run the test again without {RewriteVariable}." diff --git a/tests/Cosmos.Tests.Infrastructure/FSharp.Azure.Cosmos.Tests.Infrastructure.fsproj b/tests/Cosmos.Tests.Infrastructure/FSharp.Azure.Cosmos.Tests.Infrastructure.fsproj index a415dcf0..8ba1e5c8 100644 --- a/tests/Cosmos.Tests.Infrastructure/FSharp.Azure.Cosmos.Tests.Infrastructure.fsproj +++ b/tests/Cosmos.Tests.Infrastructure/FSharp.Azure.Cosmos.Tests.Infrastructure.fsproj @@ -18,6 +18,7 @@ + diff --git a/tests/Cosmos.Tests/BaselineTests.fs b/tests/Cosmos.Tests/BaselineTests.fs new file mode 100644 index 00000000..fb1043da --- /dev/null +++ b/tests/Cosmos.Tests/BaselineTests.fs @@ -0,0 +1,201 @@ +namespace FSharp.Azure.Cosmos.Tests + +open System +open System.IO +open System.Text.Json.Nodes +open Microsoft.VisualStudio.TestTools.UnitTesting + +/// +/// Emulator-free coverage of the golden baselines: how matches, reports and rewrites a +/// baseline, how ignores layout, and how finds +/// the baselines of a test project. +/// +[] +type BaselineTests () = + + // Gives a test a directory of its own, deleted afterwards + static let inTemporaryDirectory (test : string -> unit) = + let directory = Path.Combine (Path.GetTempPath (), "fsac-baseline-tests", Guid.NewGuid().ToString "N") + Directory.CreateDirectory directory |> ignore + + try + test directory + finally + Directory.Delete (directory, true) + + static let write (path : string) (content : string) = + Directory.CreateDirectory (nonNull (Path.GetDirectoryName path)) + |> ignore + File.WriteAllText (path, content) + + static let parameters (json : string) = (nonNull (JsonNode.Parse json)).AsObject() + + [] + member _.``normalizeSql collapses every run of whitespace into one space and trims both ends`` () = + Assert.AreEqual ( + "SELECT VALUE c FROM c WHERE c.n > @n", + Baseline.normalizeSql "\n SELECT VALUE c\r\n\tFROM c\n WHERE c.n > @n \n", + "normalizeSql should turn line breaks, tabs and repeated spaces into single spaces and drop them at both ends." + ) + + [] + member _.``compare matches a baseline that differs only in layout`` () = + inTemporaryDirectory (fun directory -> + write (Path.Combine (directory, "Suite", "case.sql")) "SELECT VALUE c\nFROM c\nWHERE c.n > @n\n" + write (Path.Combine (directory, "Suite", "case.params.json")) "{\n \"@n\": 1\n}\n" + + let comparison = + Baseline.compare + false + directory + "Suite" + "case" + "SELECT VALUE c FROM c WHERE c.n > @n" + (parameters """{"@n": 1}""") + + Assert.AreEqual ( + Baseline.Comparison.Matches, + comparison, + "compare should match SQL and parameters that differ from the baseline only in layout." + ) + ) + + [] + member _.``compare reports different SQL in its normalised form`` () = + inTemporaryDirectory (fun directory -> + write (Path.Combine (directory, "Suite", "case.sql")) "SELECT VALUE c\nFROM c\nWHERE c.n > @n\n" + write (Path.Combine (directory, "Suite", "case.params.json")) """{"@n": 1}""" + + let comparison = + Baseline.compare + false + directory + "Suite" + "case" + "SELECT VALUE c FROM c WHERE c.n >= @n" + (parameters """{"@n": 1}""") + + Assert.AreEqual ( + Baseline.Comparison.Differs ( + "SELECT VALUE c FROM c WHERE c.n > @n\n{\"@n\":1}", + "SELECT VALUE c FROM c WHERE c.n >= @n\n{\"@n\":1}" + ), + comparison, + "compare should report SQL that differs from the baseline, both sides normalised." + ) + ) + + [] + member _.``compare reports different parameters`` () = + inTemporaryDirectory (fun directory -> + write (Path.Combine (directory, "Suite", "case.sql")) "SELECT VALUE c FROM c WHERE c.n > @n" + write (Path.Combine (directory, "Suite", "case.params.json")) """{"@n": 1}""" + + let comparison = + Baseline.compare + false + directory + "Suite" + "case" + "SELECT VALUE c FROM c WHERE c.n > @n" + (parameters """{"@n": 2}""") + + Assert.AreEqual ( + Baseline.Comparison.Differs ( + "SELECT VALUE c FROM c WHERE c.n > @n\n{\"@n\":1}", + "SELECT VALUE c FROM c WHERE c.n > @n\n{\"@n\":2}" + ), + comparison, + "compare should report parameters that differ from the baseline." + ) + ) + + [] + member _.``compare matches a query without parameters against a baseline without a parameter file`` () = + inTemporaryDirectory (fun directory -> + write (Path.Combine (directory, "Suite", "case.sql")) "SELECT VALUE 1" + + Assert.AreEqual ( + Baseline.Comparison.Matches, + Baseline.compare false directory "Suite" "case" "SELECT VALUE 1" (JsonObject ()), + "compare should match a query without parameters when the baseline has no parameter file." + ) + ) + + [] + member _.``compare reports a missing baseline with the path of its SQL file`` () = + inTemporaryDirectory (fun directory -> + Assert.AreEqual ( + Baseline.Comparison.Missing (Path.Combine (directory, "Suite", "case.sql")), + Baseline.compare false directory "Suite" "case" "SELECT VALUE 1" (JsonObject ()), + "compare should report a baseline that does not exist yet." + ) + ) + + [] + member _.``compare rewrites a baseline that the next comparison matches`` () = + inTemporaryDirectory (fun directory -> + let sqlPath = Path.Combine (directory, "Suite", "case.sql") + let parametersPath = Path.Combine (directory, "Suite", "case.params.json") + let sql = "SELECT VALUE c\nFROM c\nWHERE c.n > @n" + + Assert.AreEqual ( + Baseline.Comparison.Rewritten sqlPath, + Baseline.compare true directory "Suite" "case" sql (parameters """{"@n": 1}"""), + "compare should report that it rewrote the baseline." + ) + + Assert.AreEqual ($"{sql}\n", File.ReadAllText sqlPath, "The rewritten SQL file should keep the layout of the SQL.") + + Assert.AreEqual ( + "{\n \"@n\": 1\n}\n", + File.ReadAllText parametersPath, + "The rewritten parameter file should hold the parameters as indented JSON with LF line breaks." + ) + + Assert.AreEqual ( + Baseline.Comparison.Matches, + Baseline.compare false directory "Suite" "case" sql (parameters """{"@n": 1}"""), + "compare should match the baseline it has just rewritten." + ) + ) + + [] + member _.``compare removes the parameter file of a baseline it rewrites for a query without parameters`` () = + inTemporaryDirectory (fun directory -> + let parametersPath = Path.Combine (directory, "Suite", "case.params.json") + write parametersPath """{"@old": 1}""" + + Baseline.compare true directory "Suite" "case" "SELECT VALUE 1" (JsonObject ()) + |> ignore + + Assert.IsFalse ( + File.Exists parametersPath, + "Rewriting the baseline of a query without parameters should remove its old parameter file." + ) + ) + + [] + [] + [] + [] + member _.``compare rejects a case name that cannot be part of a file name`` (caseName : string) = + Assert.ThrowsExactly( + Action (fun () -> + Baseline.compare false (Path.GetTempPath ()) "Suite" caseName "SELECT VALUE 1" (JsonObject ()) + |> ignore + ), + "compare should reject a case name that cannot be part of a file name on every platform." + ) + |> ignore + + [] + member _.``directoryOf finds the Baselines directory next to the project file of the test assembly`` () = + let directory = Baseline.directoryOf typeof.Assembly + + Assert.AreEqual ("Baselines", Path.GetFileName directory, "directoryOf should return a directory named Baselines.") + + Assert.IsTrue ( + File.Exists (Path.Combine (nonNull (Path.GetDirectoryName directory), "FSharp.Azure.Cosmos.Tests.fsproj")), + "directoryOf should return the Baselines directory next to the project file of the test assembly." + ) diff --git a/tests/Cosmos.Tests/FSharp.Azure.Cosmos.Tests.fsproj b/tests/Cosmos.Tests/FSharp.Azure.Cosmos.Tests.fsproj index fd2715c9..37cbaf0c 100644 --- a/tests/Cosmos.Tests/FSharp.Azure.Cosmos.Tests.fsproj +++ b/tests/Cosmos.Tests/FSharp.Azure.Cosmos.Tests.fsproj @@ -27,6 +27,7 @@ +