diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 15f3518e..46978db5 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -237,6 +237,7 @@ module MyTypeExtensions = * `COSMOS_EMULATOR_KEY` overrides the emulator key the tests authenticate with. * `COSMOS_EMULATOR_PARTITION_COUNT` is the number of partitions the emulator offers (default `25`, the Windows emulator's default `/PartitionCount`); the fixtures keep at most that many test containers at a time, one partition each. * `COSMOS_TEST_LEFTOVER_AGE_MINUTES` is how many minutes a `fsac-test-` database must stay unmodified before the leftover sweep of `[]` and `[]` deletes it (default `60`), so that the live databases of another test process on the same emulator are kept; `0` deletes every test database and is safe only while no other test process uses the emulator. +* `COSMOS_REWRITE_BASELINES` set to `1` or `true` makes a test that compares SQL with its golden baseline (`Baseline.assertMatches`, files under `Baselines//` of the test project) write what it produced into the baseline instead of comparing; the test then reports itself inconclusive, the rewritten files are reviewed in the diff, and CI refuses to rewrite. * Every test project under `tests/` references `tests/Cosmos.Tests.Infrastructure` through `tests/Directory.Build.props`; a new test project gets it without a `ProjectReference` of its own. * `CollectionAssert` cannot work with F# lists – use F# array syntax (`[| ... |]`) instead. * `StringAssert` has overloads with `StringComparison`. diff --git a/tests/Cosmos.Tests.Infrastructure/Baseline.fs b/tests/Cosmos.Tests.Infrastructure/Baseline.fs new file mode 100644 index 00000000..e485c642 --- /dev/null +++ b/tests/Cosmos.Tests.Infrastructure/Baseline.fs @@ -0,0 +1,195 @@ +/// +/// Golden baselines: the SQL text and the parameters a test produces, compared with files checked into the test project, +/// Baselines/<suite>/<case>.sql and Baselines/<suite>/<case>.params.json. +/// +/// +/// +/// The SQL is compared after has collapsed its whitespace, so that a baseline may be laid out +/// for reading, and the parameters are compared as JSON, so that their layout does not matter either. A query without +/// parameters has no parameter file. +/// +/// +/// With the environment variable set to 1 or true, a test writes what it +/// produced into its baseline files instead of comparing them, and reports itself inconclusive, so that a rewriting run +/// is never mistaken for a passing one; the rewritten files are then reviewed in the diff. A run on CI, where the +/// environment variable CI is true, refuses to rewrite. +/// +/// +[] +module FSharp.Azure.Cosmos.Tests.Baseline + +open System +open System.IO +open System.Reflection +open System.Text +open System.Text.Json +open System.Text.Json.Nodes + +open Microsoft.VisualStudio.TestTools.UnitTesting + +/// The environment variable that makes the tests rewrite their baselines instead of comparing them. +[] +let RewriteVariable = "COSMOS_REWRITE_BASELINES" + +/// The result of comparing what a test produced with its baseline. +[] +type Comparison = + /// The produced SQL and parameters match the baseline. + | Matches + /// The baseline does not exist yet; the path is that of its SQL file. + | Missing of sqlPath : string + /// The produced SQL or parameters differ from the baseline, both given in the form they were compared in. + | Differs of expected : string * actual : string + /// The baseline was written from what the test produced; the path is that of its SQL file. + | Rewritten of sqlPath : string + +// Characters that make a suite or case name unusable as part of a file path on any platform +let private invalidNameCharacters = + seq { + yield! Path.GetInvalidFileNameChars () + yield! [| '/'; '\\'; ':'; '*'; '?'; '"'; '<'; '>'; '|' |] + } + |> Seq.distinct + |> Seq.toArray + +let private ensureValidName (parameterName : string) (name : string) = + if + String.IsNullOrWhiteSpace name + || name.IndexOfAny invalidNameCharacters >= 0 + then + invalidArg parameterName $"'{name}' cannot be part of the name of a baseline file." + +// LF on every platform, so that a baseline does not change with the system that rewrote it +let private indented = JsonSerializerOptions (WriteIndented = true, NewLine = "\n") + +/// +/// Collapses every run of whitespace in into one space and removes the whitespace at both ends, +/// so that line breaks and indentation do not count when SQL is compared with its baseline. +/// +let normalizeSql (sql : string) : string = + let builder = StringBuilder sql.Length + let mutable pendingSpace = false + let trimmed = sql.AsSpan().Trim() + + for index in 0 .. trimmed.Length - 1 do + let character = trimmed[index] + + if Char.IsWhiteSpace character then + pendingSpace <- true + else + if pendingSpace then + builder.Append ' ' |> ignore + pendingSpace <- false + + builder.Append character |> ignore + + builder.ToString () + +/// +/// Finds the Baselines directory of the test project that built : the directory next +/// to the project file, found by walking up from the directory the assembly runs from. +/// +/// No directory above the assembly holds its project file. +let directoryOf (assembly : Assembly) : string = + let projectFileName = $"{assembly.GetName().Name}.fsproj" + + let rec find (directory : DirectoryInfo | null) = + match directory with + | null -> + invalidOp + $"No directory above '{assembly.Location}' holds '{projectFileName}', so the baselines of the test project cannot be found." + | directory when File.Exists (Path.Combine (directory.FullName, projectFileName)) -> + Path.Combine (directory.FullName, "Baselines") + | directory -> find directory.Parent + + match Path.GetDirectoryName assembly.Location with + | null -> find null + | assemblyDirectory -> find (DirectoryInfo assemblyDirectory) + +/// +/// Compares and , the parameter names mapped to their values, with +/// the baseline of under , or, when +/// is , writes them as that baseline. +/// +/// The suite or the case name cannot be part of a file name. +let compare + (rewrite : bool) + (directory : string) + (suite : string) + (caseName : string) + (sql : string) + (parameters : JsonObject) + : Comparison = + ensureValidName (nameof suite) suite + ensureValidName (nameof caseName) caseName + let suiteDirectory = Path.Combine (directory, suite) + let sqlPath = Path.Combine (suiteDirectory, $"{caseName}.sql") + let parametersPath = Path.Combine (suiteDirectory, $"{caseName}.params.json") + + if rewrite then + Directory.CreateDirectory suiteDirectory |> ignore + File.WriteAllText (sqlPath, sql.Trim () + "\n") + + if parameters.Count = 0 then + // A query that lost its parameters must not keep a parameter file from before + File.Delete parametersPath + else + File.WriteAllText (parametersPath, parameters.ToJsonString indented + "\n") + + Comparison.Rewritten sqlPath + elif not (File.Exists sqlPath) then + Comparison.Missing sqlPath + else + let expectedSql = normalizeSql (File.ReadAllText sqlPath) + + let expectedParameters = + if File.Exists parametersPath then + nonNull (JsonNode.Parse (File.ReadAllText parametersPath)) + else + JsonObject () + + let actualSql = normalizeSql sql + + if + String.Equals (expectedSql, actualSql, StringComparison.Ordinal) + && JsonNode.DeepEquals (expectedParameters, parameters) + then + Comparison.Matches + else + Comparison.Differs ( + $"{expectedSql}\n{expectedParameters.ToJsonString ()}", + $"{actualSql}\n{parameters.ToJsonString ()}" + ) + +let private isSet (variable : string) = + match Environment.GetEnvironmentVariable variable with + | null -> false + | value -> + let value = value.AsSpan().Trim() + value.Equals ("1", StringComparison.Ordinal) + || value.Equals ("true", StringComparison.OrdinalIgnoreCase) + +/// +/// Fails the test unless and match the baseline +/// of under , usually +/// the test assembly. With set, writes the baseline instead and +/// reports the test inconclusive. +/// +let assertMatches (directory : string) (suite : string) (caseName : string) (sql : string) (parameters : JsonObject) = + let rewrite = isSet RewriteVariable + + if rewrite && isSet "CI" then + Assert.Fail $"{RewriteVariable} is set on CI, where baselines are never rewritten." + + match compare rewrite directory suite caseName sql parameters with + | Comparison.Matches -> () + | Comparison.Missing sqlPath -> + Assert.Fail $"There is no baseline at {sqlPath}. Run the test with {RewriteVariable}=1 to write it, then review it." + | Comparison.Differs (expected, actual) -> + Assert.AreEqual ( + expected, + actual, + $"The SQL or the parameters of {suite}/{caseName} differ from the baseline. Run the test with {RewriteVariable}=1 to accept them, then review the diff." + ) + | Comparison.Rewritten sqlPath -> + Assert.Inconclusive $"Rewrote the baseline at {sqlPath}. Review it and run the test again without {RewriteVariable}." diff --git a/tests/Cosmos.Tests.Infrastructure/FSharp.Azure.Cosmos.Tests.Infrastructure.fsproj b/tests/Cosmos.Tests.Infrastructure/FSharp.Azure.Cosmos.Tests.Infrastructure.fsproj index a415dcf0..8ba1e5c8 100644 --- a/tests/Cosmos.Tests.Infrastructure/FSharp.Azure.Cosmos.Tests.Infrastructure.fsproj +++ b/tests/Cosmos.Tests.Infrastructure/FSharp.Azure.Cosmos.Tests.Infrastructure.fsproj @@ -18,6 +18,7 @@ + diff --git a/tests/Cosmos.Tests/BaselineTests.fs b/tests/Cosmos.Tests/BaselineTests.fs new file mode 100644 index 00000000..fb1043da --- /dev/null +++ b/tests/Cosmos.Tests/BaselineTests.fs @@ -0,0 +1,201 @@ +namespace FSharp.Azure.Cosmos.Tests + +open System +open System.IO +open System.Text.Json.Nodes +open Microsoft.VisualStudio.TestTools.UnitTesting + +/// +/// Emulator-free coverage of the golden baselines: how matches, reports and rewrites a +/// baseline, how ignores layout, and how finds +/// the baselines of a test project. +/// +[] +type BaselineTests () = + + // Gives a test a directory of its own, deleted afterwards + static let inTemporaryDirectory (test : string -> unit) = + let directory = Path.Combine (Path.GetTempPath (), "fsac-baseline-tests", Guid.NewGuid().ToString "N") + Directory.CreateDirectory directory |> ignore + + try + test directory + finally + Directory.Delete (directory, true) + + static let write (path : string) (content : string) = + Directory.CreateDirectory (nonNull (Path.GetDirectoryName path)) + |> ignore + File.WriteAllText (path, content) + + static let parameters (json : string) = (nonNull (JsonNode.Parse json)).AsObject() + + [] + member _.``normalizeSql collapses every run of whitespace into one space and trims both ends`` () = + Assert.AreEqual ( + "SELECT VALUE c FROM c WHERE c.n > @n", + Baseline.normalizeSql "\n SELECT VALUE c\r\n\tFROM c\n WHERE c.n > @n \n", + "normalizeSql should turn line breaks, tabs and repeated spaces into single spaces and drop them at both ends." + ) + + [] + member _.``compare matches a baseline that differs only in layout`` () = + inTemporaryDirectory (fun directory -> + write (Path.Combine (directory, "Suite", "case.sql")) "SELECT VALUE c\nFROM c\nWHERE c.n > @n\n" + write (Path.Combine (directory, "Suite", "case.params.json")) "{\n \"@n\": 1\n}\n" + + let comparison = + Baseline.compare + false + directory + "Suite" + "case" + "SELECT VALUE c FROM c WHERE c.n > @n" + (parameters """{"@n": 1}""") + + Assert.AreEqual ( + Baseline.Comparison.Matches, + comparison, + "compare should match SQL and parameters that differ from the baseline only in layout." + ) + ) + + [] + member _.``compare reports different SQL in its normalised form`` () = + inTemporaryDirectory (fun directory -> + write (Path.Combine (directory, "Suite", "case.sql")) "SELECT VALUE c\nFROM c\nWHERE c.n > @n\n" + write (Path.Combine (directory, "Suite", "case.params.json")) """{"@n": 1}""" + + let comparison = + Baseline.compare + false + directory + "Suite" + "case" + "SELECT VALUE c FROM c WHERE c.n >= @n" + (parameters """{"@n": 1}""") + + Assert.AreEqual ( + Baseline.Comparison.Differs ( + "SELECT VALUE c FROM c WHERE c.n > @n\n{\"@n\":1}", + "SELECT VALUE c FROM c WHERE c.n >= @n\n{\"@n\":1}" + ), + comparison, + "compare should report SQL that differs from the baseline, both sides normalised." + ) + ) + + [] + member _.``compare reports different parameters`` () = + inTemporaryDirectory (fun directory -> + write (Path.Combine (directory, "Suite", "case.sql")) "SELECT VALUE c FROM c WHERE c.n > @n" + write (Path.Combine (directory, "Suite", "case.params.json")) """{"@n": 1}""" + + let comparison = + Baseline.compare + false + directory + "Suite" + "case" + "SELECT VALUE c FROM c WHERE c.n > @n" + (parameters """{"@n": 2}""") + + Assert.AreEqual ( + Baseline.Comparison.Differs ( + "SELECT VALUE c FROM c WHERE c.n > @n\n{\"@n\":1}", + "SELECT VALUE c FROM c WHERE c.n > @n\n{\"@n\":2}" + ), + comparison, + "compare should report parameters that differ from the baseline." + ) + ) + + [] + member _.``compare matches a query without parameters against a baseline without a parameter file`` () = + inTemporaryDirectory (fun directory -> + write (Path.Combine (directory, "Suite", "case.sql")) "SELECT VALUE 1" + + Assert.AreEqual ( + Baseline.Comparison.Matches, + Baseline.compare false directory "Suite" "case" "SELECT VALUE 1" (JsonObject ()), + "compare should match a query without parameters when the baseline has no parameter file." + ) + ) + + [] + member _.``compare reports a missing baseline with the path of its SQL file`` () = + inTemporaryDirectory (fun directory -> + Assert.AreEqual ( + Baseline.Comparison.Missing (Path.Combine (directory, "Suite", "case.sql")), + Baseline.compare false directory "Suite" "case" "SELECT VALUE 1" (JsonObject ()), + "compare should report a baseline that does not exist yet." + ) + ) + + [] + member _.``compare rewrites a baseline that the next comparison matches`` () = + inTemporaryDirectory (fun directory -> + let sqlPath = Path.Combine (directory, "Suite", "case.sql") + let parametersPath = Path.Combine (directory, "Suite", "case.params.json") + let sql = "SELECT VALUE c\nFROM c\nWHERE c.n > @n" + + Assert.AreEqual ( + Baseline.Comparison.Rewritten sqlPath, + Baseline.compare true directory "Suite" "case" sql (parameters """{"@n": 1}"""), + "compare should report that it rewrote the baseline." + ) + + Assert.AreEqual ($"{sql}\n", File.ReadAllText sqlPath, "The rewritten SQL file should keep the layout of the SQL.") + + Assert.AreEqual ( + "{\n \"@n\": 1\n}\n", + File.ReadAllText parametersPath, + "The rewritten parameter file should hold the parameters as indented JSON with LF line breaks." + ) + + Assert.AreEqual ( + Baseline.Comparison.Matches, + Baseline.compare false directory "Suite" "case" sql (parameters """{"@n": 1}"""), + "compare should match the baseline it has just rewritten." + ) + ) + + [] + member _.``compare removes the parameter file of a baseline it rewrites for a query without parameters`` () = + inTemporaryDirectory (fun directory -> + let parametersPath = Path.Combine (directory, "Suite", "case.params.json") + write parametersPath """{"@old": 1}""" + + Baseline.compare true directory "Suite" "case" "SELECT VALUE 1" (JsonObject ()) + |> ignore + + Assert.IsFalse ( + File.Exists parametersPath, + "Rewriting the baseline of a query without parameters should remove its old parameter file." + ) + ) + + [] + [] + [] + [] + member _.``compare rejects a case name that cannot be part of a file name`` (caseName : string) = + Assert.ThrowsExactly( + Action (fun () -> + Baseline.compare false (Path.GetTempPath ()) "Suite" caseName "SELECT VALUE 1" (JsonObject ()) + |> ignore + ), + "compare should reject a case name that cannot be part of a file name on every platform." + ) + |> ignore + + [] + member _.``directoryOf finds the Baselines directory next to the project file of the test assembly`` () = + let directory = Baseline.directoryOf typeof.Assembly + + Assert.AreEqual ("Baselines", Path.GetFileName directory, "directoryOf should return a directory named Baselines.") + + Assert.IsTrue ( + File.Exists (Path.Combine (nonNull (Path.GetDirectoryName directory), "FSharp.Azure.Cosmos.Tests.fsproj")), + "directoryOf should return the Baselines directory next to the project file of the test assembly." + ) diff --git a/tests/Cosmos.Tests/FSharp.Azure.Cosmos.Tests.fsproj b/tests/Cosmos.Tests/FSharp.Azure.Cosmos.Tests.fsproj index fd2715c9..37cbaf0c 100644 --- a/tests/Cosmos.Tests/FSharp.Azure.Cosmos.Tests.fsproj +++ b/tests/Cosmos.Tests/FSharp.Azure.Cosmos.Tests.fsproj @@ -27,6 +27,7 @@ +