Extract the benchmark comparator into a reusable library; benchmark-compare becomes a thin wrapper (SAM#211)

.github/workflows/build.yml

@@ -40,30 +40,37 @@ jobs:
         shell: pwsh
         run: |
           $ErrorActionPreference = 'Stop'
-          $proj = "${{ env.REPO_NAME }}/SAM.Analytical.Benchmark.Compare/SAM.Analytical.Benchmark.Compare.csproj"
-          [xml]$xml = Get-Content $proj
-          $allowedProjectReference = 'SAM.Analytical.Benchmark.csproj'
+          # The comparator library and its benchmark-compare wrapper (SAM#211) are both engine-independent. Each may
+          # reference only the projects listed for it - the neutral schema, and for the wrapper the comparator
+          # library - and neither carries a binary <Reference> or a NuGet <PackageReference> of its own. Anything
+          # else (an engine DLL HintPath, the OpenStudio/TAS interop packages, ...) would couple the comparator to
+          # an engine and must fail here.
+          $projects = @(
+            @{ Path = "${{ env.REPO_NAME }}/SAM.Analytical.Benchmark.Compare/SAM.Analytical.Benchmark.Compare.csproj"; Allowed = @('SAM.Analytical.Benchmark.csproj') },
+            @{ Path = "${{ env.REPO_NAME }}/SAM.Analytical.Benchmark.Compare.Cli/SAM.Analytical.Benchmark.Compare.Cli.csproj"; Allowed = @('SAM.Analytical.Benchmark.csproj', 'SAM.Analytical.Benchmark.Compare.csproj') }
+          )
           $violations = @()
 
-          foreach ($node in $xml.SelectNodes('//ProjectReference')) {
-            $include = [string]$node.Include
-            if ($include -notlike "*$allowedProjectReference") { $violations += "ProjectReference '$include'" }
-          }
-          # The comparator carries NO binary <Reference> and NO NuGet <PackageReference> of its own; the only
-          # dependency permitted is the neutral schema project. Anything else (an engine DLL HintPath, the
-          # OpenStudio/TAS interop packages, ...) would couple the comparator to an engine and must fail here.
-          foreach ($node in $xml.SelectNodes('//Reference')) {
-            $violations += "binary Reference '$([string]$node.Include)'"
-          }
-          foreach ($node in $xml.SelectNodes('//PackageReference')) {
-            $violations += "PackageReference '$([string]$node.Include)'"
+          foreach ($project in $projects) {
+            [xml]$xml = Get-Content $project.Path
+            foreach ($node in $xml.SelectNodes('//ProjectReference')) {
+              $include = [string]$node.Include
+              $leaf = Split-Path $include -Leaf
+              if ($project.Allowed -notcontains $leaf) { $violations += "$($project.Path): ProjectReference '$include'" }
+            }
+            foreach ($node in $xml.SelectNodes('//Reference')) {
+              $violations += "$($project.Path): binary Reference '$([string]$node.Include)'"
+            }
+            foreach ($node in $xml.SelectNodes('//PackageReference')) {
+              $violations += "$($project.Path): PackageReference '$([string]$node.Include)'"
+            }
           }
 
           if ($violations.Count -gt 0) {
-            Write-Error ("The benchmark comparator must reference ONLY $allowedProjectReference. Offending references: " + ($violations -join '; '))
+            Write-Error ("The benchmark comparator must reference ONLY the neutral schema (and, for the CLI, the comparator library). Offending references: " + ($violations -join '; '))
             exit 1
           }
-          Write-Host "Comparator references only the neutral schema project."
+          Write-Host "Comparator library and CLI reference only the neutral schema / comparator projects."
 
       - name: Compute SAMVersion
         id: ver
@@ -207,6 +214,13 @@ jobs:
           dotnet test "${{ env.REPO_NAME }}/SAM.Analytical.Benchmark.Compare.Tests/SAM.Analytical.Benchmark.Compare.Tests.csproj" `
             -c Release `
             "/p:SAMVersion=${{ steps.ver.outputs.samversion }}"
+          if ($LASTEXITCODE -ne 0) { throw "Comparator tests failed" }
+
+          # SAM#211 public-API contract: compiles against the library's public surface only (no InternalsVisibleTo).
+          dotnet test "${{ env.REPO_NAME }}/SAM.Analytical.Benchmark.Compare.ApiTests/SAM.Analytical.Benchmark.Compare.ApiTests.csproj" `
+            -c Release `
+            "/p:SAMVersion=${{ steps.ver.outputs.samversion }}"
+          if ($LASTEXITCODE -ne 0) { throw "Comparator public-API tests failed" }
 
       - name: Upload build folders (optional)
         uses: actions/upload-artifact@v4

SAM.Analytical.Benchmark.Compare.ApiTests/PublicApiTests.cs

@@ -0,0 +1,124 @@
+// SPDX-License-Identifier: LGPL-3.0-or-later
+// Copyright (c) 2020-2026 Michal Dengusiak & Jakub Ziolkowski and contributors
+
+using System.Runtime.CompilerServices;
+using SAM.Analytical.Benchmark;
+using SAM.Analytical.Benchmark.Compare;
+
+namespace SAM.Analytical.Benchmark.Compare.ApiTests
+{
+    /// <summary>
+    /// The supported public contract of the comparator library (SAM#211), exercised the way SAM_UI uses it: read two
+    /// B1a documents, compare them, read every row, status and diagnostic, render the cells with the reports' own
+    /// formatting, and write the reports. This project has no access to internals, so a member made internal again
+    /// breaks the build here.
+    /// </summary>
+    [TestClass]
+    public class PublicApiTests
+    {
+        [TestMethod]
+        public void ConsumerCanCompareReadEveryRowAndWriteTheReports()
+        {
+            string caseDirectory = Path.Combine(SourceDirectory(), "..", "SAM.Analytical.Benchmark.Compare.Tests", "Fixtures", "Regression", "weather-mismatch");
+
+            BenchmarkDocument tas = BenchmarkSerializer.Read(Path.Combine(caseDirectory, "benchmark-TAS.json"));
+            BenchmarkDocument openStudio = BenchmarkSerializer.Read(Path.Combine(caseDirectory, "benchmark-OpenStudio.json"));
+
+            ComparisonResult result = Query.Compare(tas, openStudio, ToleranceProfile.Default);
+
+            // Statuses and counts.
+            Assert.AreEqual(ToleranceProfile.DefaultName, result.ToleranceProfileName);
+            GateStatus[] statuses = { result.Gate, result.NumericalStatus, result.CoverageStatus, result.ProvenanceStatus, result.ReconciliationStatus };
+            Assert.AreEqual(5, statuses.Length);
+            Assert.AreEqual(result.MatchCount + result.WarnCount + result.FailCount + result.NotApplicableCount, result.AllMetrics.Count());
+            Assert.IsTrue(result.RequiredMetricCount >= result.ComparableMetricCount);
+
+            // Provenance: the weather mismatch of this case is reported field by field.
+            Assert.IsFalse(result.ProvenanceCompatibility.IsCompatible);
+            ProvenanceMismatch mismatch = result.ProvenanceCompatibility.Mismatches.First(x => x.Field == "weatherHash");
+            Assert.AreNotEqual(mismatch.Tas, mismatch.OpenStudio);
+            Assert.IsNotNull(result.TasProvenance);
+            Assert.IsNotNull(result.OpenStudioProvenance);
+
+            // Every metric row, rendered with the reports' own formatting.
+            foreach (MetricComparison metric in result.AllMetrics)
+            {
+                string[] cells =
+                {
+                    metric.Scope, metric.Key, Format.Unit(metric.Unit), Format.Availability(metric.Availability),
+                    Format.Number(metric.TasValue), Format.Number(metric.OpenStudioValue), Format.Number(metric.AbsoluteDifference),
+                    Format.Percent(metric.RelativeDifference), Format.Number(metric.SignedDifference), Format.Band(metric.Band), Format.MetricNote(metric),
+                };
+                Assert.AreEqual(11, cells.Length);
+
+                bool informational = Query.IsInformationalForNumericalGate(metric);
+                _ = informational;
+                _ = (metric.TasAvailable, metric.OpenStudioAvailable, metric.Circular, metric.NotApplicableReason, metric.TasUnit, metric.OpenStudioUnit);
+
+                if (metric.Availability != ComparisonAvailability.Both)
+                {
+                    Assert.AreEqual(ComparisonBand.NotApplicable, metric.Band);
+                    Assert.IsNull(metric.AbsoluteDifference);
+                }
+            }
+
+            // Spaces, alignment and reconciliation.
+            foreach (SpaceComparison space in result.Spaces)
+            {
+                _ = (space.Guid, space.TasName, space.OpenStudioName, space.MatchKind, space.Scope, space.Metrics.Count, Format.MatchKind(space.MatchKind));
+            }
+
+            SpaceMatchDiagnostics diagnostics = result.SpaceDiagnostics;
+            _ = (diagnostics.TasCount, diagnostics.OpenStudioCount, diagnostics.MatchedByGuid, diagnostics.MatchedByName, diagnostics.OnlyInTas, diagnostics.OnlyInOpenStudio, diagnostics.AmbiguousNameMatches);
+            foreach (ReconciliationResult reconciliation in result.Reconciliations)
+            {
+                _ = (reconciliation.Engine, reconciliation.Key, reconciliation.ModelTotal, reconciliation.SumOfSpaces, reconciliation.Complete, reconciliation.Band);
+            }
+
+            Assert.AreEqual("Fail", Format.Gate(GateStatus.Fail));
+
+            // Reports.
+            string outputDirectory = Path.Combine(Path.GetTempPath(), "sam-compare-api-" + Guid.NewGuid().ToString("N"));
+            try
+            {
+                IReadOnlyList<string> written = Modify.WriteReports(result, outputDirectory);
+                CollectionAssert.AreEquivalent(
+                    new[] { Modify.MarkdownFileName, Modify.CsvFileName, Modify.SummaryFileName },
+                    written.Select(Path.GetFileName).ToArray());
+                Assert.AreEqual(File.ReadAllText(Path.Combine(caseDirectory, Modify.CsvFileName)), Modify.WriteCsv(result));
+            }
+            finally
+            {
+                if (Directory.Exists(outputDirectory))
+                {
+                    Directory.Delete(outputDirectory, true);
+                }
+            }
+        }
+
+        [TestMethod]
+        public void ConsumerCanListOneEnginesMetricsWithTheComparatorsKeys()
+        {
+            string caseDirectory = Path.Combine(SourceDirectory(), "..", "SAM.Analytical.Benchmark.Compare.Tests", "Fixtures", "Regression", "weather-mismatch");
+            BenchmarkDocument tas = BenchmarkSerializer.Read(Path.Combine(caseDirectory, "benchmark-TAS.json"));
+            BenchmarkDocument openStudio = BenchmarkSerializer.Read(Path.Combine(caseDirectory, "benchmark-OpenStudio.json"));
+            ComparisonResult result = Query.Compare(tas, openStudio, ToleranceProfile.Default);
+
+            // Same keys, same order as the comparator's model rows; a missing result lists every key with no value.
+            IReadOnlyList<KeyValuePair<string, MetricValue?>> model = Query.ModelMetricValues(tas.Model);
+            CollectionAssert.AreEqual(result.ModelMetrics.Select(x => x.Key).ToArray(), model.Select(x => x.Key).ToArray());
+            Assert.IsTrue(Query.ModelMetricValues(null).All(x => x.Value == null));
+
+            SpaceComparison space = result.Spaces.First();
+            BenchmarkSpaceResult tasSpace = tas.Spaces!.First(x => x.Name == space.TasName);
+            IReadOnlyList<KeyValuePair<string, MetricValue?>> spaceValues = Query.SpaceMetricValues(tasSpace);
+            CollectionAssert.AreEqual(space.Metrics.Select(x => x.Key).ToArray(), spaceValues.Select(x => x.Key).ToArray());
+            Assert.IsTrue(Query.SpaceMetricValues(null).All(x => x.Value == null));
+        }
+
+        private static string SourceDirectory([CallerFilePath] string? sourceFilePath = null)
+        {
+            return Path.GetDirectoryName(sourceFilePath!)!;
+        }
+    }
+}

SAM.Analytical.Benchmark.Compare.ApiTests/SAM.Analytical.Benchmark.Compare.ApiTests.csproj

@@ -0,0 +1,32 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <!--
+    SAM#211 public-API contract: this project is deliberately NOT named in any InternalsVisibleTo, so it compiles
+    only against the comparator library's public surface - what SAM_UI and later SAM MCP consume.
+  -->
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <LangVersion>latest</LangVersion>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+    <EnableMSTestRunner>true</EnableMSTestRunner>
+    <OutputType>Exe</OutputType>
+    <TestingPlatformDotnetTestSupport>true</TestingPlatformDotnetTestSupport>
+    <TestingPlatformShowTestsFailure>true</TestingPlatformShowTestsFailure>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.12.0" />
+    <PackageReference Include="MSTest" Version="3.6.4" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\SAM.Analytical.Benchmark.Compare\SAM.Analytical.Benchmark.Compare.csproj" />
+    <ProjectReference Include="..\SAM.Analytical.Benchmark\SAM.Analytical.Benchmark.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <Using Include="Microsoft.VisualStudio.TestTools.UnitTesting" />
+  </ItemGroup>
+
+</Project>

SAM.Analytical.Benchmark.Compare.Cli/SAM.Analytical.Benchmark.Compare.Cli.csproj

@@ -0,0 +1,30 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net8.0</TargetFramework>
+    <AssemblyName>benchmark-compare</AssemblyName>
+    <RootNamespace>SAM.Analytical.Benchmark.Compare</RootNamespace>
+    <LangVersion>latest</LangVersion>
+    <Nullable>enable</Nullable>
+    <ImplicitUsings>disable</ImplicitUsings>
+  </PropertyGroup>
+  <!--
+    benchmark-compare: the command-line wrapper over the comparator library (SAM#211). It holds only argument
+    parsing, file IO and the exit-code contract; every comparison, band, gate and report comes from
+    SAM.Analytical.Benchmark.Compare. Shipped flat in the repository build\ folder as before, so the executable
+    name, location, options, outputs and exit codes are unchanged.
+  -->
+  <PropertyGroup>
+    <AppendTargetFrameworkToOutputPath>false</AppendTargetFrameworkToOutputPath>
+    <AppendRuntimeIdentifierToOutputPath>false</AppendRuntimeIdentifierToOutputPath>
+    <OutputPath>..\build\</OutputPath>
+  </PropertyGroup>
+  <!-- Engine-independent like the library: only the comparator and the neutral schema (CI reference check). -->
+  <ItemGroup>
+    <ProjectReference Include="..\SAM.Analytical.Benchmark.Compare\SAM.Analytical.Benchmark.Compare.csproj" />
+    <ProjectReference Include="..\SAM.Analytical.Benchmark\SAM.Analytical.Benchmark.csproj" />
+  </ItemGroup>
+  <ItemGroup>
+    <InternalsVisibleTo Include="SAM.Analytical.Benchmark.Compare.Tests" />
+  </ItemGroup>
+</Project>

SAM.Analytical.Benchmark.Compare.Tests/ExtractionRegressionTests.cs

@@ -0,0 +1,109 @@
+// SPDX-License-Identifier: LGPL-3.0-or-later
+// Copyright (c) 2020-2026 Michal Dengusiak & Jakub Ziolkowski and contributors
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Runtime.CompilerServices;
+using SAM.Analytical.Benchmark;
+
+namespace SAM.Analytical.Benchmark.Compare.Tests
+{
+    /// <summary>
+    /// SAM#211: the comparator moved from the benchmark-compare executable into a reusable library, with the CLI
+    /// kept as a thin wrapper. Fixtures/Regression/&lt;case&gt; holds what the PRE-extraction CLI produced for each
+    /// case - the two input documents, comparison.md/.csv/comparison-summary.json, and, per --fail-on-gate mode,
+    /// the exit code and console output (cli-&lt;mode&gt;.txt, output directory written as &lt;out&gt;). Both routes
+    /// after the extraction must reproduce them byte for byte: the wrapper CLI, and a direct library call
+    /// (BenchmarkSerializer.Read + Query.Compare + Modify.WriteReports), which is how SAM_UI consumes it.
+    /// </summary>
+    [TestClass]
+    public class ExtractionRegressionTests
+    {
+        private static readonly string[] ReportFiles = { Modify.MarkdownFileName, Modify.CsvFileName, Modify.SummaryFileName };
+
+        private static readonly string[] FailOnGateModes = { "never", "warn", "fail" };
+
+        public static IEnumerable<object[]> Cases()
+        {
+            yield return new object[] { "fixtures" };
+            yield return new object[] { "scenario" };
+            yield return new object[] { "weather-mismatch" };
+        }
+
+        [TestMethod]
+        [DynamicData(nameof(Cases), DynamicDataSourceType.Method)]
+        public void LibraryReproducesThePreExtractionReportsByteForByte(string name)
+        {
+            string caseDirectory = CaseDirectory(name);
+            string outputDirectory = NewDirectory();
+            try
+            {
+                BenchmarkDocument tas = BenchmarkSerializer.Read(Path.Combine(caseDirectory, "benchmark-TAS.json"));
+                BenchmarkDocument openStudio = BenchmarkSerializer.Read(Path.Combine(caseDirectory, "benchmark-OpenStudio.json"));
+
+                ComparisonResult result = Query.Compare(tas, openStudio, ToleranceProfile.Resolve(null));
+                Modify.WriteReports(result, outputDirectory);
+
+                AssertReportsMatch(caseDirectory, outputDirectory);
+            }
+            finally
+            {
+                Directory.Delete(outputDirectory, true);
+            }
+        }
+
+        [TestMethod]
+        [DynamicData(nameof(Cases), DynamicDataSourceType.Method)]
+        public void CliReproducesThePreExtractionExitCodesConsoleOutputAndReports(string name)
+        {
+            string caseDirectory = CaseDirectory(name);
+            foreach (string mode in FailOnGateModes)
+            {
+                string outputDirectory = NewDirectory();
+                try
+                {
+                    var standardOutput = new StringWriter();
+                    int exitCode = Program.Run(
+                        new[] { "--tas", Path.Combine(caseDirectory, "benchmark-TAS.json"), "--openstudio", Path.Combine(caseDirectory, "benchmark-OpenStudio.json"), "--out", outputDirectory, "--fail-on-gate", mode },
+                        standardOutput,
+                        new StringWriter());
+
+                    // The console output names the output directory; the golden holds it as <out>, with '/' separators.
+                    string actual = "exit=" + exitCode + "\n" + standardOutput.ToString().Replace(outputDirectory, "<out>").Replace("\r\n", "\n").Replace('\\', '/');
+                    string expected = File.ReadAllText(Path.Combine(caseDirectory, "cli-" + mode + ".txt"));
+                    Assert.AreEqual(expected, actual, name + " --fail-on-gate " + mode);
+
+                    AssertReportsMatch(caseDirectory, outputDirectory);
+                }
+                finally
+                {
+                    Directory.Delete(outputDirectory, true);
+                }
+            }
+        }
+
+        private static void AssertReportsMatch(string expectedDirectory, string actualDirectory)
+        {
+            foreach (string file in ReportFiles)
+            {
+                CollectionAssert.AreEqual(
+                    File.ReadAllBytes(Path.Combine(expectedDirectory, file)),
+                    File.ReadAllBytes(Path.Combine(actualDirectory, file)),
+                    expectedDirectory + ": " + file + " differs from the pre-extraction output");
+            }
+        }
+
+        private static string NewDirectory()
+        {
+            string path = Path.Combine(Path.GetTempPath(), "sam-compare-regression-" + Guid.NewGuid().ToString("N"));
+            Directory.CreateDirectory(path);
+            return path;
+        }
+
+        private static string CaseDirectory(string name, [CallerFilePath] string? sourceFilePath = null)
+        {
+            return Path.Combine(Path.GetDirectoryName(sourceFilePath!)!, "Fixtures", "Regression", name);
+        }
+    }
+}