diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d876d08..d223d13 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -295,6 +295,29 @@ jobs: if ($LASTEXITCODE -ne 0) { throw "Rebuild failed: $($it.Repo)" } } + - name: Check S11 results read-back projects are built + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + + $repo = '${{ env.REPO_NAME }}' + + # S11: the reference library and its CLI are members of SAM_OpenStudio.sln and were rebuilt with it above. + # Fail if either is missing from the solution or produced no output. + $listed = @(& dotnet sln (Join-Path $repo 'SAM_OpenStudio.sln') list) + if ($LASTEXITCODE -ne 0) { throw "dotnet sln list failed" } + foreach ($p in @('SAM.Analytical.OpenStudio.ResultsReadBack', 'SAM.Analytical.OpenStudio.ResultsReadBack.Cli')) { + if (-not ($listed | Where-Object { $_.Trim() -eq "benchmark\$p\$p.csproj" })) { throw "$p is not in SAM_OpenStudio.sln" } + } + foreach ($f in @( + 'benchmark/SAM.Analytical.OpenStudio.ResultsReadBack/bin/x64/Release/net8.0/SAM.Analytical.OpenStudio.ResultsReadBack.dll', + 'benchmark/SAM.Analytical.OpenStudio.ResultsReadBack.Cli/bin/x64/Release/net8.0/os-results-readback.dll' + )) { + $path = Join-Path $repo $f + if (-not (Test-Path $path)) { throw "Expected S11 build output missing: $path" } + Write-Host "S11 build output present: $path" + } + - name: Run benchmark producer tests (offline) shell: pwsh run: | @@ -316,7 +339,7 @@ jobs: --logger 'console;verbosity=normal' if ($LASTEXITCODE -ne 0) { throw "Benchmark producer tests failed" } - - name: Run S09 OpenStudio native-input read-back tests (offline) + - name: Run S09 native-input and S11 results read-back tests (offline) shell: pwsh run: | $ErrorActionPreference = 'Stop' @@ -325,6 +348,7 @@ jobs: # S09: the writer's ventilation policy and the fresh-process OSM/IDF read-back on the S08 signed-off fixtures with # the SAM#212 Budapest EPW (from the SAM sibling). OpenStudio SDK NuGet only: no CLI, no EnergyPlus run. + # S11: the results round-trip reference on synthetic SQL/ESO files keyed by the S09 exports (no EnergyPlus run). $testProject = Join-Path $repo 'benchmark/SAM.Analytical.OpenStudio.InputReadBack.Tests/SAM.Analytical.OpenStudio.InputReadBack.Tests.csproj' Write-Host "==> Testing: $testProject" & dotnet test $testProject ` @@ -332,7 +356,7 @@ jobs: '/p:Platform=x64' ` '/p:SAMVersion=${{ steps.ver.outputs.samversion }}' ` --logger 'console;verbosity=normal' - if ($LASTEXITCODE -ne 0) { throw "S09 read-back tests failed" } + if ($LASTEXITCODE -ne 0) { throw "S09/S11 read-back tests failed" } - name: Upload S09 failure diagnostics # Only when a step failed: the bounded text evidence (exit code, stdout/stderr, process identities, file list) the diff --git a/SAM_OpenStudio.sln b/SAM_OpenStudio.sln index 6091aa3..9bccb8b 100644 --- a/SAM_OpenStudio.sln +++ b/SAM_OpenStudio.sln @@ -38,6 +38,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SAM.Analytical.OpenStudio.I EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SAM.Analytical.OpenStudio.InputReadBack.Tests", "benchmark\SAM.Analytical.OpenStudio.InputReadBack.Tests\SAM.Analytical.OpenStudio.InputReadBack.Tests.csproj", "{3F2CC081-C850-4FE7-BEF7-E7BFC5699603}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SAM.Analytical.OpenStudio.ResultsReadBack", "benchmark\SAM.Analytical.OpenStudio.ResultsReadBack\SAM.Analytical.OpenStudio.ResultsReadBack.csproj", "{7A1F3C52-0B6E-4D8A-9C21-5E11A0B7C301}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SAM.Analytical.OpenStudio.ResultsReadBack.Cli", "benchmark\SAM.Analytical.OpenStudio.ResultsReadBack.Cli\SAM.Analytical.OpenStudio.ResultsReadBack.Cli.csproj", "{7A1F3C52-0B6E-4D8A-9C21-5E11A0B7C302}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -180,6 +184,30 @@ Global {3F2CC081-C850-4FE7-BEF7-E7BFC5699603}.Release|x64.Build.0 = Release|x64 {3F2CC081-C850-4FE7-BEF7-E7BFC5699603}.Release|x86.ActiveCfg = Release|x64 {3F2CC081-C850-4FE7-BEF7-E7BFC5699603}.Release|x86.Build.0 = Release|x64 + {7A1F3C52-0B6E-4D8A-9C21-5E11A0B7C301}.Debug|Any CPU.ActiveCfg = Debug|x64 + {7A1F3C52-0B6E-4D8A-9C21-5E11A0B7C301}.Debug|Any CPU.Build.0 = Debug|x64 + {7A1F3C52-0B6E-4D8A-9C21-5E11A0B7C301}.Debug|x64.ActiveCfg = Debug|x64 + {7A1F3C52-0B6E-4D8A-9C21-5E11A0B7C301}.Debug|x64.Build.0 = Debug|x64 + {7A1F3C52-0B6E-4D8A-9C21-5E11A0B7C301}.Debug|x86.ActiveCfg = Debug|x64 + {7A1F3C52-0B6E-4D8A-9C21-5E11A0B7C301}.Debug|x86.Build.0 = Debug|x64 + {7A1F3C52-0B6E-4D8A-9C21-5E11A0B7C301}.Release|Any CPU.ActiveCfg = Release|x64 + {7A1F3C52-0B6E-4D8A-9C21-5E11A0B7C301}.Release|Any CPU.Build.0 = Release|x64 + {7A1F3C52-0B6E-4D8A-9C21-5E11A0B7C301}.Release|x64.ActiveCfg = Release|x64 + {7A1F3C52-0B6E-4D8A-9C21-5E11A0B7C301}.Release|x64.Build.0 = Release|x64 + {7A1F3C52-0B6E-4D8A-9C21-5E11A0B7C301}.Release|x86.ActiveCfg = Release|x64 + {7A1F3C52-0B6E-4D8A-9C21-5E11A0B7C301}.Release|x86.Build.0 = Release|x64 + {7A1F3C52-0B6E-4D8A-9C21-5E11A0B7C302}.Debug|Any CPU.ActiveCfg = Debug|x64 + {7A1F3C52-0B6E-4D8A-9C21-5E11A0B7C302}.Debug|Any CPU.Build.0 = Debug|x64 + {7A1F3C52-0B6E-4D8A-9C21-5E11A0B7C302}.Debug|x64.ActiveCfg = Debug|x64 + {7A1F3C52-0B6E-4D8A-9C21-5E11A0B7C302}.Debug|x64.Build.0 = Debug|x64 + {7A1F3C52-0B6E-4D8A-9C21-5E11A0B7C302}.Debug|x86.ActiveCfg = Debug|x64 + {7A1F3C52-0B6E-4D8A-9C21-5E11A0B7C302}.Debug|x86.Build.0 = Debug|x64 + {7A1F3C52-0B6E-4D8A-9C21-5E11A0B7C302}.Release|Any CPU.ActiveCfg = Release|x64 + {7A1F3C52-0B6E-4D8A-9C21-5E11A0B7C302}.Release|Any CPU.Build.0 = Release|x64 + {7A1F3C52-0B6E-4D8A-9C21-5E11A0B7C302}.Release|x64.ActiveCfg = Release|x64 + {7A1F3C52-0B6E-4D8A-9C21-5E11A0B7C302}.Release|x64.Build.0 = Release|x64 + {7A1F3C52-0B6E-4D8A-9C21-5E11A0B7C302}.Release|x86.ActiveCfg = Release|x64 + {7A1F3C52-0B6E-4D8A-9C21-5E11A0B7C302}.Release|x86.Build.0 = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -196,6 +224,8 @@ Global {EBF33B94-81C3-4D70-BD25-F10C0E119B6F} = {09EED85C-BE3C-7566-DC0E-2E8E43466740} {850D2D75-52C9-4322-B233-234F72981C39} = {09EED85C-BE3C-7566-DC0E-2E8E43466740} {3F2CC081-C850-4FE7-BEF7-E7BFC5699603} = {09EED85C-BE3C-7566-DC0E-2E8E43466740} + {7A1F3C52-0B6E-4D8A-9C21-5E11A0B7C301} = {09EED85C-BE3C-7566-DC0E-2E8E43466740} + {7A1F3C52-0B6E-4D8A-9C21-5E11A0B7C302} = {09EED85C-BE3C-7566-DC0E-2E8E43466740} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {4E0308FF-C8D5-4700-951A-859CB9D4FD09} diff --git a/benchmark/SAM.Analytical.OpenStudio.Benchmark.Tests/DiagnosticPropagationTests.cs b/benchmark/SAM.Analytical.OpenStudio.Benchmark.Tests/DiagnosticPropagationTests.cs index c56a3e3..975fad2 100644 --- a/benchmark/SAM.Analytical.OpenStudio.Benchmark.Tests/DiagnosticPropagationTests.cs +++ b/benchmark/SAM.Analytical.OpenStudio.Benchmark.Tests/DiagnosticPropagationTests.cs @@ -234,7 +234,8 @@ public void NullAndEmptyDiagnostics_LeaveValidEmptyArrays() BenchmarkDocument document = model.ToBenchmark(emptyDiagnostics); Assert.That(document.Provenance.Warnings, Is.Not.Null.And.Empty, "Empty arrays still serialize as valid empty arrays"); - Assert.That(document.Provenance.Notes, Is.Not.Null.And.Empty); + // S11 F3: the only note on a successful document with no diagnostics is the Total-basis label. + Assert.That(document.Provenance.Notes, Is.EqualTo(new[] { Modify.TotalBasisNote })); Assert.That(BenchmarkValidator.Validate(document).IsValid, Is.True, "An empty-diagnostic document stays schema-valid"); } diff --git a/benchmark/SAM.Analytical.OpenStudio.Benchmark.Tests/Fixtures/golden-openstudio-benchmark.json b/benchmark/SAM.Analytical.OpenStudio.Benchmark.Tests/Fixtures/golden-openstudio-benchmark.json index 2140dd8..594d770 100644 --- a/benchmark/SAM.Analytical.OpenStudio.Benchmark.Tests/Fixtures/golden-openstudio-benchmark.json +++ b/benchmark/SAM.Analytical.OpenStudio.Benchmark.Tests/Fixtures/golden-openstudio-benchmark.json @@ -27,7 +27,9 @@ "SAM.Analytical.OpenStudio" ], "warnings": [], - "notes": [] + "notes": [ + "Cooling metrics are EnergyPlus Ideal Loads Supply Air Total cooling (sensible \u002B latent); heating is Total heating." + ] }, "model": { "consumptionHeating": { diff --git a/benchmark/SAM.Analytical.OpenStudio.Benchmark.Tests/S11ProducerTests.cs b/benchmark/SAM.Analytical.OpenStudio.Benchmark.Tests/S11ProducerTests.cs new file mode 100644 index 0000000..76df9b5 --- /dev/null +++ b/benchmark/SAM.Analytical.OpenStudio.Benchmark.Tests/S11ProducerTests.cs @@ -0,0 +1,161 @@ +// SPDX-License-Identifier: LGPL-3.0-or-later +// Copyright (c) 2020-2026 Michal Dengusiak & Jakub Ziolkowski and contributors + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.Json.Nodes; +using NUnit.Framework; +using SAM.Analytical.Benchmark; + +namespace SAM.Analytical.OpenStudio.Benchmark.Tests +{ + /// + /// S11 producer rules: the all-zero peak rule, unavailable consumption without zone keys, the F3 Total-basis note, the + /// three-argument mapping of exactly the captured instances, the closure of the --sam-results file, and its CLI guard. + /// + [TestFixture] + public class S11ProducerTests + { + private static OpenStudioSimulationResultSet ResultSet(double coolingPeakKw, int coolingHour, double coolingKwh, bool coolingKeyed = true, bool heatingKeyed = true) + { + string key = BenchmarkFixture.IdealLoadsKey(); + Dictionary empty = new Dictionary(); + return new OpenStudioSimulationResultSet( + heatingKeyed ? new Dictionary { { key, 10.5 } } : empty, + coolingKeyed ? new Dictionary { { key, coolingKwh } } : empty, + new Dictionary { { key, 3.25 } }, + new Dictionary { { key, coolingPeakKw } }, + new Dictionary { { key, 100 } }, + new Dictionary { { key, coolingHour } }, + 3.25, 100, coolingPeakKw, coolingHour, + new Dictionary { { key, 2.0 } }, + new Dictionary { { key, 0.0 } }, + empty, empty, empty, empty, empty, empty, empty, + null, null, null, 1.0, 0, 0, 0); + } + + private static BenchmarkDocument Document(OpenStudioSimulationResultSet resultSet, RunState state = RunState.Success) + { + AnalyticalModel model = BenchmarkFixture.SingleSpaceModel(); + OpenStudioBenchmarkContext context = BenchmarkFixture.Context(model, resultSet); + context.State = state; + return model.ToBenchmark(context); + } + + [Test] + public void AllZeroPeak_LoadIsAMeasuredZero_HourIsUnavailable() + { + BenchmarkDocument document = Document(ResultSet(0, 0, 0)); + BenchmarkConditionResult cooling = document.Spaces.Single().Cooling; + + Assert.That(cooling.PeakLoad.Available, Is.True); + Assert.That(cooling.PeakLoad.Value, Is.EqualTo(0)); + Assert.That(cooling.PeakHour.Available, Is.False, "an all-zero year ties at every hour: no peak hour"); + Assert.That(document.Model.PeakCoolingLoad.Value, Is.EqualTo(0)); + Assert.That(document.Model.PeakCoolingHour.Available, Is.False); + Assert.That(document.Model.ConsumptionCooling.Available, Is.True, "a keyed zero consumption is a measured 0 kWh"); + Assert.That(document.Model.ConsumptionCooling.Value, Is.EqualTo(0)); + Assert.That(document.Spaces.Single().Heating.PeakHour.Value, Is.EqualTo(100), "a non-zero peak keeps its hour"); + Assert.That(BenchmarkValidator.Validate(document).IsValid, Is.True); + } + + [Test] + public void NoZoneKey_ConsumptionIsUnavailable_NotZero() + { + BenchmarkDocument document = Document(ResultSet(2.4, 4602, 0, coolingKeyed: false, heatingKeyed: false)); + + Assert.That(document.Model.ConsumptionCooling.Available, Is.False); + Assert.That(document.Model.ConsumptionCooling.Value, Is.Null); + Assert.That(document.Model.ConsumptionHeating.Available, Is.False); + Assert.That(BenchmarkValidator.Validate(document).IsValid, Is.True); + } + + [Test] + public void TotalBasisNote_OnSuccessfulDocumentsOnly_ExactlyOnce() + { + BenchmarkDocument success = Document(BenchmarkFixture.ResultSet()); + Assert.That(success.Provenance.Notes.Count(x => x == Modify.TotalBasisNote), Is.EqualTo(1)); + Assert.That(Modify.TotalBasisNote, Is.EqualTo("Cooling metrics are EnergyPlus Ideal Loads Supply Air Total cooling (sensible + latent); heating is Total heating.")); + + AnalyticalModel model = BenchmarkFixture.SingleSpaceModel(); + OpenStudioBenchmarkContext again = BenchmarkFixture.Context(model, BenchmarkFixture.ResultSet()); + again.Notes.Add(Modify.TotalBasisNote); + Assert.That(model.ToBenchmark(again).Provenance.Notes.Count(x => x == Modify.TotalBasisNote), Is.EqualTo(1), "a context already carrying the note keeps one"); + + BenchmarkDocument failure = Document(null, RunState.Failure); + Assert.That(failure.Provenance.Notes, Has.None.EqualTo(Modify.TotalBasisNote)); + } + + [Test] + public void ThreeArgumentToBenchmark_MapsExactlyTheGivenInstances() + { + AnalyticalModel model = BenchmarkFixture.SingleSpaceModel(); + OpenStudioBenchmarkContext context = BenchmarkFixture.Context(model, BenchmarkFixture.ResultSet()); + OpenStudioBenchmarkInputs inputs = model.BenchmarkInputs(context); + + Assert.That(BenchmarkSerializer.Serialize(model.ToBenchmark(context, inputs)), Is.EqualTo(BenchmarkSerializer.Serialize(model.ToBenchmark(context))), "the two-argument form is the three-argument form on fresh inputs"); + + inputs.SpaceResults.First(x => x.TryGetValue(Analytical.SpaceSimulationResultParameter.LoadType, out string t) && t == "Cooling").SetValue(Analytical.SpaceSimulationResultParameter.Load, 1111.0); + inputs.ModelResult.SetValue(AnalyticalModelSimulationResultParameter.ConsumptionHeating, 77.0); + BenchmarkDocument document = model.ToBenchmark(context, inputs); + Assert.That(document.Spaces.Single().Cooling.PeakLoad.Value, Is.EqualTo(1111.0), "the document follows the instances it is given, not the context's result set"); + Assert.That(document.Model.ConsumptionHeating.Value, Is.EqualTo(77.0)); + } + + [Test] + public void SamResultsFile_RoundTrips_AndReMapsToByteIdenticalModelAndSpaces() + { + AnalyticalModel model = BenchmarkFixture.SingleSpaceModel(); + OpenStudioBenchmarkContext context = BenchmarkFixture.Context(model, BenchmarkFixture.ResultSet()); + OpenStudioBenchmarkInputs inputs = model.BenchmarkInputs(context); + BenchmarkDocument produced = model.ToBenchmark(context, inputs); + + string path = Path.Combine(TestContext.CurrentContext.WorkDirectory, "s11-sam-results-" + Guid.NewGuid().ToString("N").Substring(0, 6) + ".json"); + StringWriter output = new StringWriter(); + Program.WriteSamResults(inputs, path, output); + Assert.That(output.ToString(), Does.Contain(BenchmarkHash.ComputeSha256(File.ReadAllBytes(path)))); + byte[] bytes = File.ReadAllBytes(path); + Assert.That(bytes.Take(3), Is.Not.EqualTo(new byte[] { 0xEF, 0xBB, 0xBF }), "UTF-8 without BOM"); + Assert.That(System.Text.Encoding.UTF8.GetString(bytes), Does.Not.Contain("\r\n")); + + OpenStudioBenchmarkInputs read = OpenStudioBenchmarkInputs.Read(path); + BenchmarkDocument remapped = model.ToBenchmark(context, read); + JsonObject a = JsonNode.Parse(BenchmarkSerializer.Serialize(produced)) as JsonObject; + JsonObject b = JsonNode.Parse(BenchmarkSerializer.Serialize(remapped)) as JsonObject; + Assert.That(b["model"].ToJsonString(), Is.EqualTo(a["model"].ToJsonString())); + Assert.That(b["spaces"].ToJsonString(), Is.EqualTo(a["spaces"].ToJsonString())); + Assert.That(read.HeatingConsumptionKeyed && read.CoolingConsumptionKeyed, Is.True); + + read.SpaceResults[0].SetValue(Analytical.SpaceSimulationResultParameter.Load, 1.0); + JsonObject altered = JsonNode.Parse(BenchmarkSerializer.Serialize(model.ToBenchmark(context, read))) as JsonObject; + Assert.That(altered["spaces"].ToJsonString(), Is.Not.EqualTo(a["spaces"].ToJsonString()), "an altered object no longer closes"); + } + + [Test] + public void SamResultsFile_ForeignSchema_IsRejected() + { + Assert.Throws(() => OpenStudioBenchmarkInputs.FromJsonObject(new JsonObject { ["schema"] = "other/1" })); + } + + [Test] + public void SamResultsSameAsOut_IsAUsageError_BeforeAnyRun() + { + string directory = Path.Combine(TestContext.CurrentContext.WorkDirectory, "s11-cli-" + Guid.NewGuid().ToString("N").Substring(0, 6)); + Directory.CreateDirectory(directory); + string model = Path.Combine(directory, "model.json"); + string weather = Path.Combine(directory, "weather.epw"); + File.WriteAllText(model, "{}"); + File.WriteAllText(weather, "x"); + string output = Path.Combine(directory, "b1a.json"); + + StringWriter standardOutput = new StringWriter(); + StringWriter standardError = new StringWriter(); + int exit = Program.Run(new[] { "--model", model, "--weather", weather, "--out", output, "--sam-results", output }, standardOutput, standardError); + + Assert.That(exit, Is.EqualTo((int)BenchmarkExitCode.InvalidUsage), standardError.ToString()); + Assert.That(File.Exists(output), Is.False); + } + } +} diff --git a/benchmark/SAM.Analytical.OpenStudio.Benchmark/Classes/OpenStudioBenchmarkInputs.cs b/benchmark/SAM.Analytical.OpenStudio.Benchmark/Classes/OpenStudioBenchmarkInputs.cs new file mode 100644 index 0000000..41dbad7 --- /dev/null +++ b/benchmark/SAM.Analytical.OpenStudio.Benchmark/Classes/OpenStudioBenchmarkInputs.cs @@ -0,0 +1,107 @@ +// SPDX-License-Identifier: LGPL-3.0-or-later +// Copyright (c) 2020-2026 Michal Dengusiak & Jakub Ziolkowski and contributors + +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; + +namespace SAM.Analytical.OpenStudio.Benchmark +{ + /// + /// The SAM result objects a B1a document is mapped from: the model result and the per-space results the established + /// SAM mappings produced from the run's result set, the per-space design-load results, and whether the result set + /// carried any zone key for each annual consumption. builds them once and + /// maps exactly + /// these instances, so the objects written with --sam-results are the ones that produced the document (S11). + /// + public sealed class OpenStudioBenchmarkInputs + { + /// Schema of the --sam-results file. + public const string Schema = "s11-sam-results/1"; + + /// The model result (consumption, coincident peaks, floor area, volume); null for a failed run. + public AnalyticalModelSimulationResult ModelResult { get; set; } + + /// Per-space, per-load-type annual results (peak load, its hour, unmet hours). + public List SpaceResults { get; set; } = new List(); + + /// Per-space design-day (ZoneSizes) results; empty when no sizing ran. + public List DesignLoadResults { get; set; } = new List(); + + /// + /// True when the result set carried at least one zone key for annual heating. Without one the model's heating + /// consumption is missing, never a measured 0 kWh (the result set's empty sum). + /// + public bool HeatingConsumptionKeyed { get; set; } + + /// As , for annual cooling. + public bool CoolingConsumptionKeyed { get; set; } + + /// The objects as SAM JSON inside a small envelope (the --sam-results file). + public JsonObject ToJsonObject() + { + JsonArray spaceResults = new JsonArray(); + foreach (SpaceSimulationResult spaceResult in SpaceResults ?? new List()) + { + spaceResults.Add(spaceResult?.ToJsonObject()); + } + + JsonArray designLoadResults = new JsonArray(); + foreach (SpaceSimulationResult designLoadResult in DesignLoadResults ?? new List()) + { + designLoadResults.Add(designLoadResult?.ToJsonObject()); + } + + return new JsonObject + { + ["schema"] = Schema, + ["heatingConsumptionKeyed"] = HeatingConsumptionKeyed, + ["coolingConsumptionKeyed"] = CoolingConsumptionKeyed, + ["modelResult"] = ModelResult?.ToJsonObject(), + ["spaceResults"] = spaceResults, + ["designLoadResults"] = designLoadResults, + }; + } + + /// Reads an envelope written by ; throws for another schema. + public static OpenStudioBenchmarkInputs FromJsonObject(JsonObject jsonObject) + { + if (jsonObject == null || (string)jsonObject["schema"] != Schema) + { + throw new JsonException("Not a " + Schema + " document."); + } + + return new OpenStudioBenchmarkInputs + { + HeatingConsumptionKeyed = (bool)jsonObject["heatingConsumptionKeyed"], + CoolingConsumptionKeyed = (bool)jsonObject["coolingConsumptionKeyed"], + ModelResult = jsonObject["modelResult"] is JsonObject model ? Core.Create.IJSAMObject(model) : null, + SpaceResults = Results(jsonObject["spaceResults"] as JsonArray), + DesignLoadResults = Results(jsonObject["designLoadResults"] as JsonArray), + }; + } + + /// Writes the envelope as indented UTF-8 (no BOM) with LF line ends. + public void Write(string path) + { + string json = ToJsonObject().ToJsonString(new JsonSerializerOptions { WriteIndented = true }).Replace("\r\n", "\n") + "\n"; + File.WriteAllBytes(path, new UTF8Encoding(false).GetBytes(json)); + } + + /// Reads a file written by . + public static OpenStudioBenchmarkInputs Read(string path) + { + return FromJsonObject(JsonNode.Parse(File.ReadAllText(path)) as JsonObject); + } + + private static List Results(JsonArray jsonArray) + { + return jsonArray == null + ? new List() + : jsonArray.OfType().Select(x => Core.Create.IJSAMObject(x)).ToList(); + } + } +} diff --git a/benchmark/SAM.Analytical.OpenStudio.Benchmark/Modify/ToBenchmark.cs b/benchmark/SAM.Analytical.OpenStudio.Benchmark/Modify/ToBenchmark.cs index 31a97c2..99016a6 100644 --- a/benchmark/SAM.Analytical.OpenStudio.Benchmark/Modify/ToBenchmark.cs +++ b/benchmark/SAM.Analytical.OpenStudio.Benchmark/Modify/ToBenchmark.cs @@ -26,11 +26,34 @@ public static partial class Modify /// missing or out of range is available:false with a null value. A measured zero /// stays a value of 0. Peak load and its hour are coupled: a peak with no valid hour-of-year /// is emitted as unavailable rather than a misleading zero. + /// + /// S11 rules: an all-zero peak (load exactly 0) is a measured load of 0 with NO hour — every + /// hour of the year ties, so no hour is the peak hour (the SAM result's hour 0 is the earliest + /// of that full-year tie and is not published). An annual consumption whose result set carried + /// no zone key is unavailable, not 0 kWh. A successful document carries the + /// (F3): cooling and heating are the EnergyPlus Ideal Loads + /// supply-air Total energies, cooling including latent. + /// /// /// The neutral source model (provides GUIDs, names, areas and volumes). /// Run context: result set, hashes and provenance. /// The benchmark document, or null when either argument is null. public static BenchmarkDocument ToBenchmark(this AnalyticalModel analyticalModel, OpenStudioBenchmarkContext openStudioBenchmarkContext) + { + return ToBenchmark(analyticalModel, openStudioBenchmarkContext, BenchmarkInputs(analyticalModel, openStudioBenchmarkContext)); + } + + /// + /// The F3 basis label on every successful OpenStudio B1a document. The values are not changed: + /// the route always reported the Ideal Loads Total energies; this states it. + /// + public const string TotalBasisNote = "Cooling metrics are EnergyPlus Ideal Loads Supply Air Total cooling (sensible + latent); heating is Total heating."; + + /// + /// The SAM result objects the document is mapped from, built once from the context's result set + /// through the established SAM mappings. Null when either argument is null. + /// + public static OpenStudioBenchmarkInputs BenchmarkInputs(this AnalyticalModel analyticalModel, OpenStudioBenchmarkContext openStudioBenchmarkContext) { if (analyticalModel == null || openStudioBenchmarkContext == null) { @@ -38,20 +61,49 @@ public static BenchmarkDocument ToBenchmark(this AnalyticalModel analyticalModel } OpenStudioSimulationResultSet resultSet = openStudioBenchmarkContext.ResultSet; - AnalyticalModelSimulationResult modelResult = resultSet?.ToSAM(analyticalModel); - List spaceResults = resultSet?.ToSAM_SpaceSimulationResults(analyticalModel) ?? new List(); + return new OpenStudioBenchmarkInputs + { + ModelResult = resultSet?.ToSAM(analyticalModel), + SpaceResults = resultSet?.ToSAM_SpaceSimulationResults(analyticalModel) ?? new List(), + DesignLoadResults = openStudioBenchmarkContext.SpaceDesignLoadResults?.ToList() ?? new List(), + HeatingConsumptionKeyed = resultSet?.AnnualHeatingEnergy.Count > 0, + CoolingConsumptionKeyed = resultSet?.AnnualCoolingEnergy.Count > 0, + }; + } + + /// + /// Maps exactly the given instances (S11: the objects + /// that produce the document are the ones captured beside it). The context supplies provenance only. + /// + /// The benchmark document, or null when any argument is null. + public static BenchmarkDocument ToBenchmark(this AnalyticalModel analyticalModel, OpenStudioBenchmarkContext openStudioBenchmarkContext, OpenStudioBenchmarkInputs openStudioBenchmarkInputs) + { + if (analyticalModel == null || openStudioBenchmarkContext == null || openStudioBenchmarkInputs == null) + { + return null; + } + + AnalyticalModelSimulationResult modelResult = openStudioBenchmarkInputs.ModelResult; + List spaceResults = openStudioBenchmarkInputs.SpaceResults ?? new List(); return new BenchmarkDocument { SchemaVersion = BenchmarkSchema.CurrentVersion, Provenance = Provenance(openStudioBenchmarkContext, modelResult, spaceResults), - Model = Model(modelResult), - Spaces = Spaces(analyticalModel, spaceResults, openStudioBenchmarkContext.SpaceDesignLoadResults), + Model = Model(modelResult, openStudioBenchmarkInputs), + Spaces = Spaces(analyticalModel, spaceResults, openStudioBenchmarkInputs.DesignLoadResults), }; } private static BenchmarkProvenance Provenance(OpenStudioBenchmarkContext context, AnalyticalModelSimulationResult modelResult, IEnumerable spaceResults) { + List notes = new List(context.Notes ?? new List()); + if (context.State == RunState.Success && !notes.Contains(TotalBasisNote)) + { + notes.Add(TotalBasisNote); + notes.Sort(StringComparer.Ordinal); + } + return new BenchmarkProvenance { SourceModelName = context.SourceModelName, @@ -76,7 +128,7 @@ private static BenchmarkProvenance Provenance(OpenStudioBenchmarkContext context State = context.State, ResultSources = ResultSources(modelResult, spaceResults), Warnings = new List(context.Warnings ?? new List()), - Notes = new List(context.Notes ?? new List()), + Notes = notes, }; } @@ -103,7 +155,7 @@ private static List ResultSources(AnalyticalModelSimulationResult modelR return sources.ToList(); } - private static BenchmarkModelResult Model(AnalyticalModelSimulationResult modelResult) + private static BenchmarkModelResult Model(AnalyticalModelSimulationResult modelResult, OpenStudioBenchmarkInputs inputs) { (MetricValue heatingLoad, MetricValue heatingHour) = Peak( Double(modelResult, AnalyticalModelSimulationResultParameter.PeakHeatingLoad), @@ -117,8 +169,13 @@ private static BenchmarkModelResult Model(AnalyticalModelSimulationResult modelR return new BenchmarkModelResult { - ConsumptionHeating = NonNegative(Double(modelResult, AnalyticalModelSimulationResultParameter.ConsumptionHeating), MetricUnit.KilowattHour), - ConsumptionCooling = NonNegative(Double(modelResult, AnalyticalModelSimulationResultParameter.ConsumptionCooling), MetricUnit.KilowattHour), + // No zone key: the result set's empty sum is not a measured 0 kWh. + ConsumptionHeating = inputs.HeatingConsumptionKeyed + ? NonNegative(Double(modelResult, AnalyticalModelSimulationResultParameter.ConsumptionHeating), MetricUnit.KilowattHour) + : MetricValue.Unavailable(MetricUnit.KilowattHour), + ConsumptionCooling = inputs.CoolingConsumptionKeyed + ? NonNegative(Double(modelResult, AnalyticalModelSimulationResultParameter.ConsumptionCooling), MetricUnit.KilowattHour) + : MetricValue.Unavailable(MetricUnit.KilowattHour), PeakHeatingLoad = heatingLoad, PeakHeatingHour = heatingHour, PeakCoolingLoad = coolingLoad, @@ -203,12 +260,19 @@ private static MetricValue NonNegative(double? value, MetricUnit unit) /// /// A peak load and its hour-of-year, coupled: both are available only when the load is a /// finite non-negative number and the hour is an integer in 0..8759; otherwise both are - /// unavailable (a peak without a valid hour is missing, never a measured zero). + /// unavailable (a peak without a valid hour is missing, never a measured zero). A load of + /// exactly 0 is available with its hour unavailable: an all-zero series ties at every hour, + /// so it has no peak hour (S11 zero rule, symmetric with S10). /// private static (MetricValue load, MetricValue hour) Peak(double? loadValue, int? hourValue, MetricUnit loadUnit) { bool hourValid = hourValue.HasValue && hourValue.Value >= 0 && hourValue.Value <= 8759; bool loadValid = loadValue.HasValue && IsFinite(loadValue.Value) && loadValue.Value >= 0; + if (hourValid && loadValid && loadValue.Value == 0) + { + return (MetricValue.AvailableValue(0, loadUnit), MetricValue.Unavailable(MetricUnit.HourOfYear)); + } + if (hourValid && loadValid) { return (MetricValue.AvailableValue(loadValue.Value, loadUnit), MetricValue.AvailableValue(hourValue.Value, MetricUnit.HourOfYear)); diff --git a/benchmark/SAM.Analytical.OpenStudio.Benchmark/Program.cs b/benchmark/SAM.Analytical.OpenStudio.Benchmark/Program.cs index 5873efc..82297ee 100644 --- a/benchmark/SAM.Analytical.OpenStudio.Benchmark/Program.cs +++ b/benchmark/SAM.Analytical.OpenStudio.Benchmark/Program.cs @@ -59,6 +59,11 @@ private static int Execute(BenchmarkArguments arguments, TextWriter standardOutp string modelPath = BenchmarkCliPaths.ValidateInputFile(arguments.RequireOption("model")); string weatherPath = BenchmarkCliPaths.ValidateInputFile(arguments.RequireOption("weather")); string outputPath = BenchmarkCliPaths.ValidateOutputFile(arguments.RequireOption("out")); + string samResultsPath = arguments.GetOption("sam-results") == null ? null : BenchmarkCliPaths.ValidateOutputFile(arguments.GetOption("sam-results")); + if (samResultsPath != null && string.Equals(Path.GetFullPath(samResultsPath), Path.GetFullPath(outputPath), StringComparison.OrdinalIgnoreCase)) + { + throw new ArgumentException("--sam-results must not be the --out benchmark file."); + } // Provenance hashes (B1a helpers, exactly per SCHEMA.md "Canonical model hashing"). string sourceFileHash = BenchmarkHash.ComputeSha256(File.ReadAllBytes(modelPath)); @@ -152,11 +157,30 @@ private static int Execute(BenchmarkArguments arguments, TextWriter standardOutp // producer-generated notes added above, so both arrays are deterministic. Modify.ApplyConversionDiagnostics(context, Concat(conversionResult?.Diagnostics, designLoadDiagnostics)); - BenchmarkDocument document = model.ToBenchmark(context); - return Emit(document, outputPath, success, standardOutput); + // The SAM result objects are built once and the document is mapped from exactly these + // instances; --sam-results writes the same instances after the document (S11 subject capture). + OpenStudioBenchmarkInputs inputs = model.BenchmarkInputs(context); + BenchmarkDocument document = model.ToBenchmark(context, inputs); + int exitCode = Emit(document, outputPath, success, standardOutput); + if (samResultsPath != null) + { + WriteSamResults(inputs, samResultsPath, standardOutput); + } + + return exitCode; } } + /// + /// Writes the SAM result objects that produced the document (, + /// SAM JSON) and reports their SHA-256, which the S11 run manifest records. + /// + internal static void WriteSamResults(OpenStudioBenchmarkInputs inputs, string path, TextWriter standardOutput) + { + inputs.Write(path); + standardOutput.WriteLine("Wrote " + path + " (" + BenchmarkHash.ComputeSha256(File.ReadAllBytes(path)) + ", the SAM result objects of the document)."); + } + /// /// Validates, writes and reports a produced document, returning the shared exit code: a /// document that fails schema validation throws @@ -385,6 +409,7 @@ private static string GitHead(string directory) private const string Usage = "Usage: benchmark-openstudio --model --weather --out \n" + " [--work ] [--sam-commit ] [--runner-commit ]\n" + + " [--sam-results ]\n" + "\n" + "Exit codes: 0 success, 2 usage error, 3 input/IO/serialization error, 4 validation failure, 5 producer failure."; } diff --git a/benchmark/SAM.Analytical.OpenStudio.InputReadBack.Tests/S11CompletenessTests.cs b/benchmark/SAM.Analytical.OpenStudio.InputReadBack.Tests/S11CompletenessTests.cs new file mode 100644 index 0000000..0a46664 --- /dev/null +++ b/benchmark/SAM.Analytical.OpenStudio.InputReadBack.Tests/S11CompletenessTests.cs @@ -0,0 +1,265 @@ +// SPDX-License-Identifier: LGPL-3.0-or-later +// Copyright (c) 2020–2026 Michal Dengusiak & Jakub Ziolkowski and contributors + +using NUnit.Framework; +using SAM.Analytical.OpenStudio.ResultsReadBack; +using System; +using System.Collections.Generic; +using System.Data.SQLite; +using System.Linq; +using System.Text.RegularExpressions; + +namespace SAM.Analytical.OpenStudio.InputReadBack.Tests +{ + /// + /// S11-C1/C1b (revision-2 change 1): every (zone x required quantity x file) must hold each of the 8760 hours of the + /// in.idf RunPeriod calendar exactly once, judged per file. An error shared by SQL and ESO, a one-sided error and a + /// count-preserving shift all BLOCK. For the shared errors the production reader still returns values for the same + /// files, so the detection comes from the reference, not from the importer. + /// + [TestFixture] + public class S11CompletenessTests + { + public enum Defect + { + MissingHourBoth, + DuplicatedHourBothSameValue, + DuplicatedHourBothDifferentValue, + CountPreservingShiftBoth, + MissingHourSqlOnly, + MissingHourEsoOnly, + DuplicatedHourSqlOnly, + DuplicatedHourEsoOnly, + SeriesInSqlAbsentFromEso, + SeriesInEsoAbsentFromSql, + ZoneMissingFromBoth, + EsoIdRepeatedInsideOneHourlyBlock, + SqlSecondHourlyDictionaryEntry, + EsoSecondHourlyDictionaryEntry, + ExtraHourBoth, + February29Both, + WarmupRowsInRunPeriodSql, + DesignDayRowsMergedIntoRunPeriodBoth, + NonAnnualRunPeriod, + LeapRunPeriod, + SecondRunPeriodEnvironmentEso, + DaylightSavingStampBoth, + SqlExtraTimestepFrequencyZeroValue, + EsoExtraTimestepFrequencyZeroValue, + SqlSecondWeatherRunEnvironmentOtherName, + } + + /// The defects present identically in both files (the paired production-reader check applies to them). + private static readonly Defect[] Shared = + { + Defect.MissingHourBoth, Defect.DuplicatedHourBothSameValue, Defect.DuplicatedHourBothDifferentValue, Defect.CountPreservingShiftBoth, + Defect.ExtraHourBoth, Defect.February29Both, Defect.DesignDayRowsMergedIntoRunPeriodBoth, Defect.SqlSecondWeatherRunEnvironmentOtherName, + }; + + /// A second weather-run (EnvironmentType 3) environment, named otherwise; the production reader reads every type-3 environment. + public const string OtherWeatherRun = "WEATHER FILE RUN PERIOD 2"; + + private static void Break(SyntheticRun run, Defect defect) + { + string firstKey = SyntheticRun.Key(S11Names.TotalCooling, run.Zones[0].IdealLoads); + Func one = x => x == firstKey; + switch (defect) + { + case Defect.MissingHourBoth: + run.EditSeries(true, true, s => s.RemoveAll(p => p.Month == 7 && p.Day == 7 && p.Hour == 13)); + break; + case Defect.DuplicatedHourBothSameValue: + run.EditSeries(true, true, s => s.Add(s.First(p => p.Month == 7 && p.Day == 7 && p.Hour == 13).Copy())); + break; + case Defect.DuplicatedHourBothDifferentValue: + run.EditSeries(true, true, s => { SyntheticPoint copy = s.First(p => p.Month == 7 && p.Day == 7 && p.Hour == 13).Copy(); copy.Value += 1; s.Add(copy); }); + break; + case Defect.CountPreservingShiftBoth: + run.EditSeries(true, true, s => + { + SyntheticPoint next = s.First(p => p.Month == 7 && p.Day == 7 && p.Hour == 14); + next.Hour = 13; + Assert.That(s.Count, Is.EqualTo(8760), "the shift keeps the count"); + }); + break; + case Defect.MissingHourSqlOnly: + run.EditSeries(true, false, s => s.RemoveAll(p => p.Month == 7 && p.Day == 7 && p.Hour == 13), one); + break; + case Defect.MissingHourEsoOnly: + run.EditSeries(false, true, s => s.RemoveAll(p => p.Month == 7 && p.Day == 7 && p.Hour == 13), one); + break; + case Defect.DuplicatedHourSqlOnly: + run.EditSeries(true, false, s => s.Add(s.First(p => p.Month == 7 && p.Day == 7 && p.Hour == 13).Copy()), one); + break; + case Defect.DuplicatedHourEsoOnly: + run.EditSeries(false, true, s => s.Add(s.First(p => p.Month == 7 && p.Day == 7 && p.Hour == 13).Copy()), one); + break; + case Defect.SeriesInSqlAbsentFromEso: + run.Eso.Remove(firstKey); + break; + case Defect.SeriesInEsoAbsentFromSql: + run.Sql.Remove(firstKey); + break; + case Defect.ZoneMissingFromBoth: + foreach (string key in run.Sql.Keys.Where(x => x.EndsWith("|" + run.Zones[1].IdealLoads, StringComparison.Ordinal) || x.EndsWith("|" + run.Zones[1].Zone, StringComparison.Ordinal)).ToList()) + { + run.Sql.Remove(key); + run.Eso.Remove(key); + } + + break; + case Defect.EsoIdRepeatedInsideOneHourlyBlock: + run.EsoEdit = lines => + { + int stamp = lines.FindIndex(x => Regex.IsMatch(x, @"^2,\d+, 7, 7, 0,13, 0\.00,60\.00")); + lines.Insert(stamp + 2, lines[stamp + 1]); + return lines; + }; + break; + case Defect.SqlSecondHourlyDictionaryEntry: + run.SqlEdit = connection => + { + using (SQLiteCommand command = connection.CreateCommand()) + { + command.CommandText = "INSERT INTO ReportDataDictionary (IsMeter, Type, IndexGroup, TimestepType, KeyValue, Name, ReportingFrequency, Units) VALUES (0, 'Sum', 'System', 'HVAC System', @k, @n, 'Hourly', 'J')"; + command.Parameters.AddWithValue("@k", run.Zones[0].IdealLoads); + command.Parameters.AddWithValue("@n", S11Names.TotalCooling); + command.ExecuteNonQuery(); + } + }; + break; + case Defect.EsoSecondHourlyDictionaryEntry: + run.EsoEdit = lines => + { + lines.Insert(lines.IndexOf("End of Data Dictionary"), "900,1," + run.Zones[0].IdealLoads + "," + S11Names.TotalCooling + " [J] !Hourly"); + return lines; + }; + break; + case Defect.ExtraHourBoth: + run.EditSeries(true, true, s => s.Add(new SyntheticPoint { Month = 12, Day = 31, Hour = 25, Value = 1, Environment = run.RunPeriodName })); + break; + case Defect.February29Both: + run.EditSeries(true, true, s => s.AddRange(Enumerable.Range(1, 24).Select(h => new SyntheticPoint { Month = 2, Day = 29, Hour = h, Value = 2, Environment = run.RunPeriodName }))); + break; + case Defect.WarmupRowsInRunPeriodSql: + run.EditSeries(true, false, s => s.AddRange(Enumerable.Range(1, 24).Select(h => new SyntheticPoint { Month = 1, Day = 1, Hour = h, Value = 3, Environment = run.RunPeriodName, Warmup = true }))); + break; + case Defect.DesignDayRowsMergedIntoRunPeriodBoth: + run.EditSeries(true, true, s => s.AddRange(Enumerable.Range(1, 24).Select(h => new SyntheticPoint { Month = 7, Day = 21, Hour = h, Value = 4, Environment = run.RunPeriodName }))); + break; + case Defect.NonAnnualRunPeriod: + run.Idf = EditRunPeriod(run.Idf, "End Month", "6"); + break; + case Defect.LeapRunPeriod: + run.Idf = EditRunPeriod(EditRunPeriod(run.Idf, "Begin Year", "2024"), "End Year", "2024"); + break; + case Defect.SecondRunPeriodEnvironmentEso: + run.EsoEnvironments.Add(run.RunPeriodName); + break; + case Defect.DaylightSavingStampBoth: + run.EditSeries(true, true, s => s.First(p => p.Month == 7 && p.Day == 7 && p.Hour == 13).Dst = 1); + break; + case Defect.SqlExtraTimestepFrequencyZeroValue: + // The hourly series stays complete; the additional frequency alone (with a zero value) must block it. + run.SqlEdit = connection => + { + using (SQLiteCommand command = connection.CreateCommand()) + { + command.CommandText = "INSERT INTO ReportDataDictionary (IsMeter, Type, IndexGroup, TimestepType, KeyValue, Name, ReportingFrequency, Units) VALUES (0, 'Sum', 'System', 'HVAC System', @k, @n, 'HVAC System Timestep', 'J')"; + command.Parameters.AddWithValue("@k", run.Zones[0].IdealLoads); + command.Parameters.AddWithValue("@n", S11Names.TotalCooling); + command.ExecuteNonQuery(); + command.CommandText = "INSERT INTO ReportData (TimeIndex, ReportDataDictionaryIndex, Value) VALUES ((SELECT MIN(TimeIndex) FROM Time), (SELECT MAX(ReportDataDictionaryIndex) FROM ReportDataDictionary), 0)"; + command.ExecuteNonQuery(); + } + }; + break; + case Defect.EsoExtraTimestepFrequencyZeroValue: + // As the review reproduction: a second dictionary entry at TimeStep frequency and one zero record. + run.EsoEdit = lines => + { + lines.Insert(lines.IndexOf("End of Data Dictionary"), "999999,1," + run.Zones[0].IdealLoads + "," + S11Names.TotalCooling + " [J] !TimeStep"); + int stamp = lines.FindIndex(x => Regex.IsMatch(x, @"^2,\d+, 7, 7, 0,13, 0\.00,60\.00")); + lines.Insert(stamp + 1, "999999,0"); + return lines; + }; + break; + case Defect.SqlSecondWeatherRunEnvironmentOtherName: + run.SqlEnvironments.Add(new KeyValuePair(OtherWeatherRun, 3)); + run.EsoEnvironments.Add(OtherWeatherRun); + run.EditSeries(true, true, s => s.AddRange(Enumerable.Range(1, 24).Select(h => new SyntheticPoint { Month = 1, Day = 1, Hour = h, Value = 5, Environment = OtherWeatherRun }))); + break; + } + } + + /// Replaces the value of one labelled RunPeriod field in the IDF text. + public static string EditRunPeriod(string idf, string label, string value) + { + int start = idf.IndexOf("RunPeriod,", StringComparison.Ordinal); + Assert.That(start, Is.GreaterThanOrEqualTo(0)); + int end = idf.IndexOf(';', start); + string block = idf.Substring(start, end - start); + string edited = Regex.Replace(block, @"(\n\s*)[^,\n]*(,\s*!- " + Regex.Escape(label) + @"\b)", "${1}" + value + "${2}"); + Assert.That(edited, Is.Not.EqualTo(block), "RunPeriod field " + label + " not found"); + return idf.Substring(0, start) + edited + idf.Substring(end); + } + + [Test] + public void EveryCompletenessDefect_Blocks([Values] Defect defect) + { + string model = S11Support.Two(); + SyntheticRun synthetic = SyntheticRun.For(model, "two"); + Break(synthetic, defect); + S11Case run = S11Support.Evaluate(synthetic, model, "c1-" + defect); + + Assert.That(run.Compare.Verdict, Is.EqualTo("BLOCKED"), run.Summary()); + Assert.That(run.Compare.Failed, Is.Zero, "a completeness defect blocks, it is not a value failure: " + run.Summary()); + string reasons = string.Join(" ", run.Compare.Rows.Where(x => x.Status == "BLOCKED").Select(x => x.Reason)) + " " + string.Join(" ", run.Reference.GlobalBlocks); + Assert.That(reasons, Does.Contain("S11-C1").Or.Contain("calendar").Or.Contain("eso:").Or.Contain("sql:"), reasons); + if (defect == Defect.SqlExtraTimestepFrequencyZeroValue || defect == Defect.EsoExtraTimestepFrequencyZeroValue) + { + Assert.That(reasons, Does.Contain("unsupported reporting frequency"), reasons); + Assert.That(run.Row("b1a|" + run.Space(1) + "|Cooling|peakLoad").Status, Is.EqualTo("PASS"), "only the key with the extra frequency blocks"); + } + + if (defect == Defect.SqlSecondWeatherRunEnvironmentOtherName) + { + Assert.That(string.Join(" ", run.Reference.GlobalBlocks), Does.Contain("2 SQL environment(s) of type 3")); + } + + // Rows that do not depend on the broken series keep their verdict (each check blocks only dependent rows). + if (defect == Defect.MissingHourSqlOnly || defect == Defect.SeriesInSqlAbsentFromEso) + { + Assert.That(run.Row("b1a|" + run.Space(0) + "|Heating|peakLoad").Status, Is.EqualTo("PASS")); + Assert.That(run.Row("b1a|" + run.Space(1) + "|Cooling|peakLoad").Status, Is.EqualTo("PASS")); + Assert.That(run.Row("b1a|" + run.Space(0) + "|Cooling|peakLoad").Status, Is.EqualTo("BLOCKED")); + Assert.That(run.Row("b1a|model|Cooling|consumption").Status, Is.EqualTo("BLOCKED")); + } + + if (Shared.Contains(defect)) + { + OpenStudioSimulationResultSet production = SyntheticProducer.ResultSet(System.IO.Path.Combine(run.Run, "work", "run", "eplusout.sql")); + Assert.That(production.AnnualCoolingEnergy.Count, Is.EqualTo(2), "the production reader still returns values for the defective files"); + Assert.That(production.PeakCoolingLoad.Values.All(x => x > 0), Is.True); + } + } + + [Test] + public void CountPreservingShift_IsReportedAsOneDuplicateAndOneMissingHourPerSeriesAndFile() + { + string model = S11Support.Model(TestSupport.Box); + SyntheticRun synthetic = SyntheticRun.For(model, "box"); + Break(synthetic, Defect.CountPreservingShiftBoth); + S11Case run = S11Support.Evaluate(synthetic, model, "c1-shift-detail"); + + Assert.That(run.Reference.Completeness, Has.Count.EqualTo(12), "6 quantities x 2 files"); + foreach (CompletenessResult result in run.Reference.Completeness) + { + Assert.That(result.Found, Is.EqualTo(8760)); + Assert.That(result.Missing, Is.EqualTo(1), result.Channel + " " + result.Variable); + Assert.That(result.Duplicated, Is.EqualTo(1)); + Assert.That(result.Complete, Is.False); + } + } + } +} diff --git a/benchmark/SAM.Analytical.OpenStudio.InputReadBack.Tests/S11DriverStandIn.ps1 b/benchmark/SAM.Analytical.OpenStudio.InputReadBack.Tests/S11DriverStandIn.ps1 new file mode 100644 index 0000000..b23fbe0 --- /dev/null +++ b/benchmark/SAM.Analytical.OpenStudio.InputReadBack.Tests/S11DriverStandIn.ps1 @@ -0,0 +1,99 @@ +# SPDX-License-Identifier: LGPL-3.0-or-later +# Copyright (c) 2020-2026 Michal Dengusiak & Jakub Ziolkowski and contributors +# +# Offline stand-in for os-results-readback, used only by S11DriverTests: Run-S11-Results.ps1 starts it in place of the +# harness. It starts no engine, loads no model and reads no SQL/ESO; it writes the files the real steps write, with the +# shapes the driver checks. scenario.json beside it sets, per "|", what a step does: +# exit (int) the exit code (default: 0; compare: the verdict the case requires, 0 for a run) +# write (bool) false: write nothing (default true) +# verdict (string) the report verdict (default: from the exit) +# wrongInputs (bool) the report records another SQL hash +# od7 (bool) F3.MeetsOd7 (default true) +# touch (string) a file to append to (e.g. an original the step must not change) +# mutate (bool) negative: false copies without mutating (default true) +# expectedExit (int) negative: the exit negative.json records (default: the case's) +# Every call is appended to calls.txt beside it. +param([Parameter(ValueFromRemainingArguments = $true)][string[]]$Items) + +$step = $Items[0] +$options = @{} +for ($i = 1; $i + 1 -lt $Items.Count; $i += 2) { $options[$Items[$i].TrimStart('-')] = $Items[$i + 1] } +$directory = $(if ($step -eq 'negative') { $options['out'] } else { $options['run'] }) +$name = Split-Path $directory -Leaf +Add-Content -LiteralPath (Join-Path $PSScriptRoot 'calls.txt') -Value "$name|$step" +$scenario = Get-Content -LiteralPath (Join-Path $PSScriptRoot 'scenario.json') -Raw | ConvertFrom-Json +$entry = $scenario."$name|$step" + +function Sha([string]$Path) { + if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) { return $null } + $bytes = [IO.File]::ReadAllBytes($Path) + $sha = [Security.Cryptography.SHA256]::Create() + try { return ([BitConverter]::ToString($sha.ComputeHash($bytes)) -replace '-', '').ToLowerInvariant() } finally { $sha.Dispose() } +} + +function Option($Name, $Default) { + if ($entry -and $null -ne $entry.$Name) { return $entry.$Name } + return $Default +} + +$cases = @{ 'ctl-unmutated-copy' = 0; 'neg-delete-hour-both' = 11; 'neg-duplicate-hour-both' = 11; 'neg-sensible-cooling-substitution' = 10 } +$case = @($cases.Keys | Where-Object { $name.EndsWith('-' + $_) })[0] +$defaultExit = 0 +if ($step -eq 'compare' -and $case) { $defaultExit = $cases[$case] } +$exit = [int](Option 'exit' $defaultExit) +$touch = Option 'touch' $null +if ($touch) { Add-Content -LiteralPath $touch -Value 'changed by a step' } + +if (Option 'write' $true) { + $stamp = [Guid]::NewGuid().ToString('N') + switch ($step) { + 'read' { [IO.File]::WriteAllText((Join-Path $directory 'reference.json'), "{ ""stand-in"": ""read $stamp"" }") } + 'subject' { [IO.File]::WriteAllText((Join-Path $directory 'subject.json'), "{ ""stand-in"": ""subject $stamp"" }") } + 'negative' { + $source = $options['run'] + New-Item -ItemType Directory -Force -Path (Join-Path $directory 'work\run') | Out-Null + foreach ($file in @('b1a.json', 'sam.json', 'work\run\in.idf', 'work\run\eplusout.sql', 'work\run\eplusout.eso')) { + Copy-Item -LiteralPath (Join-Path $source $file) -Destination (Join-Path $directory $file) + } + $mutation = 'none (control)' + if (Option 'mutate' $true) { + if ($case -eq 'neg-delete-hour-both' -or $case -eq 'neg-duplicate-hour-both') { + Add-Content -LiteralPath (Join-Path $directory 'work\run\eplusout.sql') -Value $case + Add-Content -LiteralPath (Join-Path $directory 'work\run\eplusout.eso') -Value $case + $mutation = "$case on the SQL and ESO copies" + } + elseif ($case -eq 'neg-sensible-cooling-substitution') { + Add-Content -LiteralPath (Join-Path $directory 'b1a.json') -Value $case + Add-Content -LiteralPath (Join-Path $directory 'sam.json') -Value $case + [IO.File]::WriteAllText((Join-Path $directory 'subject.json'), "{ ""stand-in"": ""captured subject $stamp"" }") + $mutation = "$case on the b1a.json and sam.json copies" + } + } + $record = [ordered]@{ Case = $case; Source = $source; ExpectedExit = [int](Option 'expectedExit' $cases[$case]); Mutation = $mutation } + [IO.File]::WriteAllText((Join-Path $directory 'negative.json'), ($record | ConvertTo-Json)) + } + 'compare' { + $verdict = Option 'verdict' $(switch ($exit) { 0 { 'PASS' } 10 { 'FAIL' } 11 { 'BLOCKED' } default { 'PASS' } }) + $inputs = [ordered]@{ + 'model' = Sha $options['model'] + 'in.idf' = Sha (Join-Path $directory 'work\run\in.idf') + 'eplusout.sql' = $(if (Option 'wrongInputs' $false) { 'feedface' } else { Sha (Join-Path $directory 'work\run\eplusout.sql') }) + 'eplusout.eso' = Sha (Join-Path $directory 'work\run\eplusout.eso') + 'b1a.json' = Sha (Join-Path $directory 'b1a.json') + 'sam.json' = Sha (Join-Path $directory 'sam.json') + 'reference.json' = Sha (Join-Path $directory 'reference.json') + 'subject.json' = Sha (Join-Path $directory 'subject.json') + } + $failed = $(if ($verdict -eq 'FAIL') { 1 } else { 0 }) + $blocked = $(if ($verdict -eq 'BLOCKED') { 1 } else { 0 }) + $report = [ordered]@{ + Schema = 's11-compare/1'; Verdict = $verdict; ExitCode = $exit; Required = 85; Passed = 85 - $failed - $blocked; Failed = $failed; Blocked = $blocked + Inputs = $inputs; F3 = [ordered]@{ MeetsOd7 = [bool](Option 'od7' $true) } + } + New-Item -ItemType Directory -Force -Path (Join-Path $directory 'compare') | Out-Null + [IO.File]::WriteAllText((Join-Path $directory 'compare\compare.json'), ($report | ConvertTo-Json -Depth 4)) + } + } +} + +exit $exit diff --git a/benchmark/SAM.Analytical.OpenStudio.InputReadBack.Tests/S11DriverTests.cs b/benchmark/SAM.Analytical.OpenStudio.InputReadBack.Tests/S11DriverTests.cs new file mode 100644 index 0000000..2452c5c --- /dev/null +++ b/benchmark/SAM.Analytical.OpenStudio.InputReadBack.Tests/S11DriverTests.cs @@ -0,0 +1,433 @@ +// SPDX-License-Identifier: LGPL-3.0-or-later +// Copyright (c) 2020–2026 Michal Dengusiak & Jakub Ziolkowski and contributors + +using NUnit.Framework; +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +using System.Text.Json; +using System.Text.Json.Nodes; + +namespace SAM.Analytical.OpenStudio.InputReadBack.Tests +{ + /// + /// The acceptance driver Run-S11-Results.ps1 (readback and negative phases) under Windows PowerShell 5.1 with + /// stand-in commands (S11DriverStandIn.ps1 for the harness, an inert file for the producer): no engine starts, no + /// model is loaded, nothing is built or deployed. The run directories are stand-in files described by their own + /// run-manifest.json and a stand-in build receipt. The driver must stop (70) on any infrastructure or integrity + /// failure without reading an earlier report, collect numerical FAIL/BLOCKED for both fixtures and finish nonzero, and + /// fail on an unexpected negative outcome or a changed original. + /// + [TestFixture] + public class S11DriverTests + { + private static readonly (string Fixture, string Label)[] Runs = { ("s08-fix-box", "box"), ("s08-fix-two", "two") }; + + private sealed class Driver + { + public string Directory; + public string Root; + public string Bin; + public string Receipt; + public string Harness; + public string Producer; + public JsonObject Scenario = new JsonObject(); + public int Exit; + public string Output; + + public string Run(string label) => Path.Combine(Root, label); + + public string RunFile(string label, string file) => Path.Combine(Run(label), file); + + public List Calls => File.Exists(Path.Combine(Bin, "calls.txt")) ? File.ReadAllLines(Path.Combine(Bin, "calls.txt")).ToList() : new List(); + + public JsonNode Summary(string phase) => JsonNode.Parse(File.ReadAllText(Path.Combine(Root, "summary-" + phase + ".json"))); + + public void Set(string key, object values) + { + Scenario[key] = JsonSerializer.SerializeToNode(values); + } + + public string Describe() => "exit " + Exit + Environment.NewLine + Output + Environment.NewLine + "calls: " + string.Join(", ", Calls); + } + + private static string Script() + { + DirectoryInfo directory = new DirectoryInfo(TestContext.CurrentContext.TestDirectory); + while (directory != null) + { + string candidate = Path.Combine(directory.FullName, "benchmark", "SAM.Analytical.OpenStudio.ResultsReadBack", "Run-S11-Results.ps1"); + if (File.Exists(candidate)) return candidate; + directory = directory.Parent; + } + + throw new FileNotFoundException("Run-S11-Results.ps1 not found above " + TestContext.CurrentContext.TestDirectory); + } + + private static string Fixture(string fixtureId) => Path.Combine(Path.GetDirectoryName(Script()), "..", "SAM.Analytical.OpenStudio.InputReadBack", "Fixtures", fixtureId + ".json"); + + private static string Sha(string path) + { + using (FileStream stream = File.OpenRead(path)) + using (SHA256 sha = SHA256.Create()) + { + return string.Concat(sha.ComputeHash(stream).Select(x => x.ToString("x2"))); + } + } + + /// Two produced runs (stand-in files), their manifests, the stand-in harness and producer, and a receipt listing them. + private static Driver Arrange(string label) + { + Driver driver = new Driver { Directory = S11Support.NewRun("driver-" + label) }; + driver.Root = Path.Combine(driver.Directory, "acc"); + driver.Bin = Path.Combine(driver.Directory, "bin"); + System.IO.Directory.CreateDirectory(driver.Bin); + driver.Harness = Path.Combine(driver.Bin, "os-results-readback.ps1"); + File.Copy(Path.Combine(TestContext.CurrentContext.TestDirectory, "S11DriverStandIn.ps1"), driver.Harness); + driver.Producer = Path.Combine(driver.Bin, "benchmark-openstudio.ps1"); + File.WriteAllText(driver.Producer, "Add-Content -LiteralPath (Join-Path $PSScriptRoot 'calls.txt') -Value 'PRODUCER STARTED'\r\nexit 99\r\n"); + File.WriteAllText(Path.Combine(driver.Bin, "os-results-readback.dll"), "stand-in harness assembly"); + File.WriteAllText(Path.Combine(driver.Bin, "benchmark-openstudio.dll"), "stand-in producer assembly"); + + driver.Receipt = Path.Combine(driver.Bin, "s11-build-receipt.json"); + JsonArray files = new JsonArray(); + foreach (string file in new[] { "benchmark-openstudio.ps1", "benchmark-openstudio.dll", "os-results-readback.ps1", "os-results-readback.dll" }) + { + files.Add(new JsonObject { ["path"] = Path.Combine(driver.Bin, file), ["sha256"] = Sha(Path.Combine(driver.Bin, file)) }); + } + + File.WriteAllText(driver.Receipt, new JsonObject { ["schema"] = "s11-build-receipt/1", ["commits"] = new JsonArray(), ["files"] = files }.ToJsonString()); + + foreach ((string fixture, string run) in Runs) + { + string folder = Path.Combine(driver.Run(run), "work", "run"); + System.IO.Directory.CreateDirectory(folder); + File.WriteAllText(Path.Combine(folder, "in.idf"), "stand-in in.idf " + run); + File.WriteAllText(Path.Combine(folder, "eplusout.sql"), "stand-in eplusout.sql " + run); + File.WriteAllText(Path.Combine(folder, "eplusout.eso"), "stand-in eplusout.eso " + run); + File.WriteAllText(driver.RunFile(run, "b1a.json"), "{ \"stand-in\": \"b1a " + run + "\" }"); + File.WriteAllText(driver.RunFile(run, "sam.json"), "{ \"stand-in\": \"sam " + run + "\" }"); + JsonObject manifest = new JsonObject + { + ["schema"] = "s11-run-manifest/1", ["label"] = run, ["fixture"] = fixture, ["fixtureSha256"] = Sha(Fixture(fixture)), + ["receiptSha256"] = Sha(driver.Receipt), ["benchmarkDll"] = Sha(Path.Combine(driver.Bin, "benchmark-openstudio.dll")), ["harnessDll"] = Sha(Path.Combine(driver.Bin, "os-results-readback.dll")), + ["idf"] = Path.Combine(folder, "in.idf"), ["idfSha256"] = Sha(Path.Combine(folder, "in.idf")), + ["sql"] = Path.Combine(folder, "eplusout.sql"), ["sqlSha256"] = Sha(Path.Combine(folder, "eplusout.sql")), + ["eso"] = Path.Combine(folder, "eplusout.eso"), ["esoSha256"] = Sha(Path.Combine(folder, "eplusout.eso")), + ["b1aSha256"] = Sha(driver.RunFile(run, "b1a.json")), ["samResultsSha256"] = Sha(driver.RunFile(run, "sam.json")), + }; + File.WriteAllText(driver.RunFile(run, "run-manifest.json"), manifest.ToJsonString()); + + // An earlier attempt's PASS report: never to be read as the result of a later step. + System.IO.Directory.CreateDirectory(Path.Combine(driver.Run(run), "compare")); + string old = Path.Combine(driver.Run(run), "compare", "compare.json"); + File.WriteAllText(old, new JsonObject { ["Schema"] = "s11-compare/1", ["Verdict"] = "PASS", ["ExitCode"] = 0, ["Required"] = 85, ["Passed"] = 85, ["Failed"] = 0, ["Blocked"] = 0, ["F3"] = new JsonObject { ["MeetsOd7"] = true } }.ToJsonString()); + File.SetLastWriteTimeUtc(old, DateTime.UtcNow.AddDays(-1)); + } + + return driver; + } + + private static Driver Act(Driver driver, string phase, params string[] extra) + { + File.WriteAllText(Path.Combine(driver.Bin, "scenario.json"), driver.Scenario.ToJsonString()); + string powershell = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.System), "WindowsPowerShell", "v1.0", "powershell.exe"); + ProcessStartInfo info = new ProcessStartInfo(powershell) { UseShellExecute = false, RedirectStandardOutput = true, RedirectStandardError = true }; + foreach (string argument in new[] { "-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass", "-File", Script(), "-Phase", phase, "-Root", driver.Root, "-BuildReceipt", driver.Receipt, "-HarnessExe", driver.Harness, "-BenchmarkExe", driver.Producer }.Concat(extra)) + { + info.ArgumentList.Add(argument); + } + + using (Process process = Process.Start(info)) + { + System.Threading.Tasks.Task error = process.StandardError.ReadToEndAsync(); + string output = process.StandardOutput.ReadToEnd(); + Assert.That(process.WaitForExit(120000), Is.True, "the driver did not finish"); + driver.Exit = process.ExitCode; + driver.Output = output + error.Result; + } + + Assert.That(driver.Calls, Has.None.EqualTo("PRODUCER STARTED"), "the producer (engine) must never start in these phases"); + return driver; + } + + // ---- readback ---------------------------------------------------------------------------------------------------- + + [Test] + public void Readback_FullyPassingControl_Exits0() + { + Driver driver = Act(Arrange("rb-pass"), "readback"); + + Assert.That(driver.Exit, Is.EqualTo(0), driver.Describe()); + Assert.That(driver.Calls, Is.EqualTo(new[] { "box|read", "box|subject", "box|compare", "two|read", "two|subject", "two|compare" })); + JsonNode summary = driver.Summary("readback"); + Assert.That((string)summary["result"], Is.EqualTo("PASS")); + Assert.That(summary["runs"].AsArray().Select(x => (string)x["verdict"]), Is.EqualTo(new[] { "PASS", "PASS" })); + Assert.That((bool)summary["original-unchanged:box"] && (bool)summary["original-unchanged:two"], Is.True); + } + + /// The review reproduction: both comparator calls return 70 while old PASS reports exist. + [Test] + public void Readback_InfrastructureErrorWithOldPassReport_StopsAt70AndReadsNoReport() + { + Driver driver = Arrange("rb-infra"); + driver.Set("box|compare", new { exit = 70, write = false }); + driver.Set("two|compare", new { exit = 70, write = false }); + Act(driver, "readback"); + + Assert.That(driver.Exit, Is.EqualTo(70), driver.Describe()); + Assert.That(driver.Output, Does.Contain("STOPPED: compare box exit 70")); + Assert.That(driver.Output, Does.Not.Contain("PASS"), "no old report may be presented as a result"); + Assert.That(driver.Calls, Has.None.StartsWith("two|"), "the phase stops at the first infrastructure failure"); + Assert.That((string)driver.Summary("readback")["result"], Is.EqualTo("STOPPED")); + Assert.That(driver.Summary("readback")["runs"].AsArray(), Is.Empty); + } + + public static IEnumerable StaleOrInconsistentReports() + { + yield return new TestCaseData("box|read", new { exit = 70 }, "read box exit 70").SetName("Readback_ReadInfrastructureError_Stops"); + yield return new TestCaseData("box|subject", new { exit = 64 }, "subject box exit 64").SetName("Readback_SubjectUsageError_Stops"); + yield return new TestCaseData("box|subject", new { write = false }, "subject box wrote no").SetName("Readback_SubjectExit0WithoutOutput_Stops"); + yield return new TestCaseData("box|compare", new { write = false }, "compare box did not write").SetName("Readback_CompareExit0WithOnlyAnOldReport_Stops"); + yield return new TestCaseData("box|compare", new { exit = 0, verdict = "FAIL" }, "compare box exit 0 but its report says FAIL").SetName("Readback_ExitDisagreesWithVerdict_Stops"); + yield return new TestCaseData("box|compare", new { exit = 10, verdict = "PASS" }, "compare box exit 10 but its report says PASS").SetName("Readback_FailExitWithPassReport_Stops"); + yield return new TestCaseData("box|compare", new { wrongInputs = true }, "compare box judged a eplusout.sql other than").SetName("Readback_ReportJudgedOtherFiles_Stops"); + } + + [TestCaseSource(nameof(StaleOrInconsistentReports))] + public void Readback_StepThatDidNotCompleteOrReportFreshly_StopsAt70(string key, object scenario, string reason) + { + Driver driver = Arrange("rb-stale"); + driver.Set(key, scenario); + Act(driver, "readback"); + + Assert.That(driver.Exit, Is.EqualTo(70), driver.Describe()); + Assert.That(driver.Output, Does.Contain("STOPPED: " + reason)); + Assert.That(driver.Calls, Has.None.StartsWith("two|")); + } + + [Test] + public void Readback_NumericFail_CollectsBothFixturesAndExits10() + { + Driver driver = Arrange("rb-fail"); + driver.Set("box|compare", new { exit = 10 }); + Act(driver, "readback"); + + Assert.That(driver.Exit, Is.EqualTo(10), driver.Describe()); + JsonNode summary = driver.Summary("readback"); + Assert.That(summary["runs"].AsArray().Select(x => (string)x["label"] + " " + (string)x["verdict"]), Is.EqualTo(new[] { "box FAIL", "two PASS" })); + Assert.That((string)summary["result"], Is.EqualTo("FAIL")); + } + + [Test] + public void Readback_Blocked_CollectsBothFixturesAndExits11() + { + Driver driver = Arrange("rb-blocked"); + driver.Set("two|compare", new { exit = 11 }); + Act(driver, "readback"); + + Assert.That(driver.Exit, Is.EqualTo(11), driver.Describe()); + Assert.That(driver.Summary("readback")["runs"].AsArray().Select(x => (string)x["verdict"]), Is.EqualTo(new[] { "PASS", "BLOCKED" })); + } + + [Test] + public void Readback_FailAndBlocked_Exits10() + { + Driver driver = Arrange("rb-fail-blocked"); + driver.Set("box|compare", new { exit = 11 }); + driver.Set("two|compare", new { exit = 10 }); + Act(driver, "readback"); + + Assert.That(driver.Exit, Is.EqualTo(10), driver.Describe()); + } + + [Test] + public void Readback_Od7MetByNeitherFixture_IsNotPass() + { + Driver driver = Arrange("rb-od7"); + driver.Set("box|compare", new { od7 = false }); + driver.Set("two|compare", new { od7 = false }); + Act(driver, "readback"); + + Assert.That(driver.Exit, Is.EqualTo(11), driver.Describe()); + Assert.That(driver.Output, Does.Contain("OD-7 not met")); + } + + // ---- changed inputs and build ------------------------------------------------------------------------------------ + + public enum Change + { + Idf, + Sql, + Eso, + B1a, + SamResults, + ModelHashRecorded, + ReceiptOfAnotherBuild, + HarnessAssembly, + HarnessNotInReceipt, + ManifestPointsOutsideTheRun, + } + + [Test] + public void ChangedInputOrBuild_BeforeProcessing_StopsAt70BeforeAnyStep([Values] Change change, [Values("readback", "negative")] string phase) + { + Driver driver = Arrange("changed"); + string run = driver.Run("two"); + string folder = Path.Combine(run, "work", "run"); + JsonObject manifest = JsonNode.Parse(File.ReadAllText(Path.Combine(run, "run-manifest.json"))).AsObject(); + switch (change) + { + case Change.Idf: File.AppendAllText(Path.Combine(folder, "in.idf"), "x"); break; + case Change.Sql: File.AppendAllText(Path.Combine(folder, "eplusout.sql"), "x"); break; + case Change.Eso: File.AppendAllText(Path.Combine(folder, "eplusout.eso"), "x"); break; + case Change.B1a: File.AppendAllText(Path.Combine(run, "b1a.json"), "x"); break; + case Change.SamResults: File.AppendAllText(Path.Combine(run, "sam.json"), "x"); break; + case Change.ModelHashRecorded: manifest["fixtureSha256"] = new string('0', 64); break; + case Change.ReceiptOfAnotherBuild: manifest["receiptSha256"] = new string('1', 64); break; + case Change.HarnessAssembly: File.AppendAllText(Path.Combine(driver.Bin, "os-results-readback.dll"), "x"); break; + case Change.HarnessNotInReceipt: + JsonObject receipt = JsonNode.Parse(File.ReadAllText(driver.Receipt)).AsObject(); + JsonArray files = receipt["files"].AsArray(); + files.Remove(files.First(x => ((string)x["path"]).EndsWith("os-results-readback.ps1", StringComparison.Ordinal))); + File.WriteAllText(driver.Receipt, receipt.ToJsonString()); + break; + case Change.ManifestPointsOutsideTheRun: + string elsewhere = Path.Combine(driver.Directory, "eplusout.sql"); + File.Copy(Path.Combine(folder, "eplusout.sql"), elsewhere); + manifest["sql"] = elsewhere; + break; + } + + File.WriteAllText(Path.Combine(run, "run-manifest.json"), manifest.ToJsonString()); + Act(driver, phase); + + Assert.That(driver.Exit, Is.EqualTo(70), driver.Describe()); + Assert.That(driver.Output, Does.Contain("STOPPED")); + Assert.That(driver.Calls, Has.None.StartsWith("two"), "nothing of the changed run is processed"); + if (change == Change.HarnessAssembly || change == Change.HarnessNotInReceipt) + { + Assert.That(driver.Calls, Is.Empty, "a receipt that does not hold stops before any step"); + } + } + + [Test] + public void Readback_OriginalChangedByAStep_StopsAt70() + { + Driver driver = Arrange("rb-touch"); + driver.Set("box|subject", new { touch = driver.RunFile("box", "sam.json") }); + Act(driver, "readback"); + + Assert.That(driver.Exit, Is.EqualTo(70), driver.Describe()); + Assert.That(driver.Output, Does.Contain("STOPPED: box after readback: sam.json is not the file the run recorded")); + Assert.That((bool)driver.Summary("readback")["original-unchanged:box"], Is.False); + } + + // ---- negative ---------------------------------------------------------------------------------------------------- + + [Test] + public void Negative_EveryCaseAsRequired_Exits0() + { + Driver driver = Act(Arrange("neg-pass"), "negative"); + + Assert.That(driver.Exit, Is.EqualTo(0), driver.Describe()); + JsonNode summary = driver.Summary("negative"); + Assert.That(summary["runs"].AsArray(), Has.Count.EqualTo(8)); + Assert.That(summary["runs"].AsArray().All(x => (bool)x["asRequired"] && (bool)x["evidence"]), Is.True, summary.ToJsonString()); + Assert.That(driver.Calls.Count(x => x.EndsWith("|subject", StringComparison.Ordinal)), Is.EqualTo(6), "the substitution cases use their captured subject"); + } + + [Test] + public void Negative_MutationNotDetected_Exits10() + { + Driver driver = Arrange("neg-undetected"); + driver.Set("box-neg-duplicate-hour-both|compare", new { exit = 0 }); + Act(driver, "negative"); + + Assert.That(driver.Exit, Is.EqualTo(10), driver.Describe()); + Assert.That(driver.Output, Does.Contain("box-neg-duplicate-hour-both").And.Contain("NOT AS REQUIRED")); + Assert.That(driver.Calls, Does.Contain("two-neg-sensible-cooling-substitution|compare"), "an unexpected outcome is collected, not a stop"); + } + + [Test] + public void Negative_ControlFails_Exits10AndNoMutationOutcomeIsEvidence() + { + Driver driver = Arrange("neg-control"); + driver.Set("box-ctl-unmutated-copy|compare", new { exit = 10 }); + Act(driver, "negative"); + + Assert.That(driver.Exit, Is.EqualTo(10), driver.Describe()); + JsonArray runs = driver.Summary("negative")["runs"].AsArray(); + JsonNode[] box = runs.Where(x => ((string)x["label"]).StartsWith("box-", StringComparison.Ordinal)).ToArray(); + Assert.That(box.Skip(1).All(x => (bool)x["asRequired"]), Is.True, "the mutations still gave their exits"); + Assert.That(box.All(x => !(bool)x["evidence"]), Is.True, "but without a passing control none of them is evidence"); + Assert.That(driver.Output, Does.Contain("box control did not PASS")); + } + + [Test] + public void Negative_CopyNotMutated_IsNotAsRequired() + { + Driver driver = Arrange("neg-unmutated"); + driver.Set("box-neg-delete-hour-both|negative", new { mutate = false }); + Act(driver, "negative"); + + Assert.That(driver.Exit, Is.EqualTo(10), driver.Describe()); + JsonNode record = driver.Summary("negative")["runs"].AsArray().Single(x => (string)x["label"] == "box-neg-delete-hour-both"); + Assert.That((int)record["exit"], Is.EqualTo(11), "the stand-in compare still reports BLOCKED"); + Assert.That((bool)record["asRequired"], Is.False); + Assert.That((bool)record["copyAsDesigned"], Is.False); + } + + [Test] + public void Negative_RecordDisagreesWithTheDriver_StopsAt70() + { + Driver driver = Arrange("neg-record"); + driver.Set("box-neg-sensible-cooling-substitution|negative", new { expectedExit = 0 }); + Act(driver, "negative"); + + Assert.That(driver.Exit, Is.EqualTo(70), driver.Describe()); + Assert.That(driver.Output, Does.Contain("not 'neg-sensible-cooling-substitution' with exit 10")); + } + + [Test] + public void Negative_CompareInfrastructureError_StopsAt70() + { + Driver driver = Arrange("neg-infra"); + driver.Set("box-neg-delete-hour-both|compare", new { exit = 70 }); + Act(driver, "negative"); + + Assert.That(driver.Exit, Is.EqualTo(70), driver.Describe()); + Assert.That(driver.Output, Does.Contain("STOPPED: compare box-neg-delete-hour-both exit 70")); + Assert.That(driver.Calls, Has.None.StartsWith("box-neg-duplicate")); + } + + [Test] + public void Negative_OriginalChangedByACase_StopsAt70() + { + Driver driver = Arrange("neg-touch"); + driver.Set("box-neg-delete-hour-both|negative", new { touch = Path.Combine(driver.Run("box"), "work", "run", "eplusout.sql") }); + Act(driver, "negative"); + + Assert.That(driver.Exit, Is.EqualTo(70), driver.Describe()); + Assert.That(driver.Output, Does.Contain("STOPPED: box after neg-delete-hour-both: eplusout.sql is not the file the run recorded")); + Assert.That((bool)driver.Summary("negative")["original-unchanged:box"], Is.False); + } + + // ---- run phase guard --------------------------------------------------------------------------------------------- + + [Test] + public void Run_WithoutEngineConfirmation_RefusesBeforeCreatingAnything() + { + Driver driver = Arrange("run-guard"); + System.IO.Directory.Delete(driver.Root, true); + Act(driver, "run"); + + Assert.That(driver.Exit, Is.EqualTo(70), driver.Describe()); + Assert.That(driver.Output, Does.Contain("-ConfirmEngineRuns")); + Assert.That(System.IO.Directory.Exists(driver.Root), Is.False); + } + } +} diff --git a/benchmark/SAM.Analytical.OpenStudio.InputReadBack.Tests/S11F3Tests.cs b/benchmark/SAM.Analytical.OpenStudio.InputReadBack.Tests/S11F3Tests.cs new file mode 100644 index 0000000..3dff1e6 --- /dev/null +++ b/benchmark/SAM.Analytical.OpenStudio.InputReadBack.Tests/S11F3Tests.cs @@ -0,0 +1,169 @@ +// SPDX-License-Identifier: LGPL-3.0-or-later +// Copyright (c) 2020–2026 Michal Dengusiak & Jakub Ziolkowski and contributors + +using NUnit.Framework; +using SAM.Analytical.OpenStudio.Benchmark; +using SAM.Analytical.OpenStudio.ResultsReadBack; +using System; +using System.Collections.Generic; +using System.Linq; + +namespace SAM.Analytical.OpenStudio.InputReadBack.Tests +{ + /// + /// F3 (revision-2 change 2), the mandatory offline proof: Sensible substituted for Total is detected. The synthetic + /// two-zone data cool for 1530 hours with a latent part of at least 10 % of Total, and the Sensible peak is at another + /// hour. A precondition asserts that every targeted row discriminates Total from Sensible by at least 10^6 x its + /// tolerance (hours: a different hour), so the test cannot silently stop discriminating. Each substitution, made in the + /// producer objects (the B1a is mapped from them, so the closure holds) and in the import, must FAIL, never BLOCK. + /// + [TestFixture] + public class S11F3Tests + { + public enum Substitution + { + Consumption, + SpacePeak, + ModelPeak, + All, + } + + private static string Model() => S11Support.Two(); + + /// The rows each substitution targets (load type filled in by the test). + private static IEnumerable Targets(S11Case run, Substitution substitution, LoadKind kind) + { + List result = new List(); + if (substitution == Substitution.Consumption || substitution == Substitution.All) + { + result.Add("samP|model|" + kind + "|consumption"); + result.Add("b1a|model|" + kind + "|consumption"); + } + + if (substitution == Substitution.ModelPeak || substitution == Substitution.All) + { + foreach (string layer in new[] { "samP", "b1a" }) + { + result.Add(layer + "|model|" + kind + "|peakLoad"); + result.Add(layer + "|model|" + kind + "|peakHour"); + } + } + + if (substitution == Substitution.SpacePeak || substitution == Substitution.All) + { + for (int i = 0; i < run.Synthetic.Zones.Count; i++) + { + foreach (string layer in new[] { "samP", "samI", "b1a" }) + { + result.Add(layer + "|" + run.Space(i) + "|" + kind + "|peakLoad"); + result.Add(layer + "|" + run.Space(i) + "|" + kind + "|peakHour"); + } + } + } + + return result; + } + + [Test] + public void Precondition_EveryTargetedRowDiscriminatesTotalFromSensibleByAMillionTolerances([Values] LoadKind kind) + { + S11Case run = S11Support.Evaluate(SyntheticRun.For(Model(), "two"), Model(), "f3-precondition-" + kind); + Assert.That(run.Compare.Verdict, Is.EqualTo("PASS"), run.Summary()); + + foreach (string id in Targets(run, Substitution.All, kind)) + { + ResultRow row = run.Row(id); + if (id.EndsWith("peakHour", StringComparison.Ordinal)) + { + Assert.That(row.SensibleHourDiffers, Is.True, id + ": the Sensible peak must be at another hour (invalid test data otherwise)"); + } + else + { + Assert.That(row.Margin, Is.GreaterThanOrEqualTo(1e6), id + ": invalid test data, Total and Sensible are not 10^6 tolerances apart"); + } + } + + int hours = run.Synthetic.Sql[SyntheticRun.Key(S11Names.Total(kind), run.Synthetic.Zones[0].IdealLoads)].Count(x => x.Value > 0); + Assert.That(hours, Is.GreaterThanOrEqualTo(1000), "at least 1000 hours with load"); + foreach (SpaceReference space in run.Reference.Spaces) + { + double latent = space.LatentJoules[kind.ToString()]; + double total = space.Quantities[S11Names.Total(kind)].Sum; + Assert.That(latent / total, Is.GreaterThanOrEqualTo(0.10), "latent at least 10 % of Total"); + } + } + + [Test] + public void SensibleSubstitutedForTotal_Fails([Values] Substitution substitution, [Values] LoadKind kind) + { + string model = Model(); + SyntheticRun synthetic = SyntheticRun.For(model, "two"); + S11Case reference = S11Support.Evaluate(synthetic, model, "f3-reference"); + Assert.That(reference.Compare.Verdict, Is.EqualTo("PASS"), "unmutated control: " + reference.Summary()); + ModelLoadReference sensible = reference.Reference.ModelOf(kind); + + void Mutate(OpenStudioBenchmarkInputs inputs) + { + bool heating = kind == LoadKind.Heating; + if (substitution == Substitution.Consumption || substitution == Substitution.All) + { + inputs.ModelResult.SetValue(heating ? AnalyticalModelSimulationResultParameter.ConsumptionHeating : AnalyticalModelSimulationResultParameter.ConsumptionCooling, sensible.Sensible.ConsumptionKwh); + } + + if (substitution == Substitution.ModelPeak || substitution == Substitution.All) + { + inputs.ModelResult.SetValue(heating ? AnalyticalModelSimulationResultParameter.PeakHeatingLoad : AnalyticalModelSimulationResultParameter.PeakCoolingLoad, sensible.Sensible.CoincidentPeakKw); + inputs.ModelResult.SetValue(heating ? AnalyticalModelSimulationResultParameter.PeakHeatingHour : AnalyticalModelSimulationResultParameter.PeakCoolingHour, sensible.Sensible.PeakHour); + } + + if (substitution == Substitution.SpacePeak || substitution == Substitution.All) + { + foreach (SyntheticZone zone in synthetic.Zones) + { + QuantityReference space = S11Support.Sensible(reference.Reference, zone.SpaceGuid, kind); + foreach (SpaceSimulationResult result in S11Support.Annual(inputs, zone.SpaceGuid, kind)) + { + result.SetValue(Analytical.SpaceSimulationResultParameter.Load, space.PeakWatts); + result.SetValue(Analytical.SpaceSimulationResultParameter.LoadIndex, space.PeakHour); + } + } + } + } + + S11Case run = S11Support.Evaluate(synthetic, model, "f3-" + substitution + "-" + kind, mutateInputs: Mutate, beforeCompare: c => + { + if (substitution == Substitution.SpacePeak || substitution == Substitution.All) + { + foreach (SubjectResult result in c.Subject.SamI.Where(x => x.IsAnnual && x.LoadType == kind.ToString())) + { + QuantityReference space = S11Support.Sensible(c.Reference, Guid.ParseExact(result.Reference, "N"), kind); + result.Load = space.PeakWatts; + result.LoadIndex = space.PeakHour; + } + } + }); + + Assert.That(run.Subject.ClosureIdentical, Is.True, "the B1a is mapped from the substituted objects: the closure holds"); + Assert.That(run.Compare.Verdict, Is.EqualTo("FAIL"), run.Summary()); + Assert.That(run.Compare.Blocked, Is.Zero, "a substitution must FAIL, never BLOCK: " + run.Summary()); + foreach (string id in Targets(run, substitution, kind)) + { + Assert.That(run.Row(id).Status, Is.EqualTo("FAIL"), id); + } + + Assert.That(run.Row("b1a|model|-|totalBasisNote").Status, Is.EqualTo("PASS"), "the note still says Total while the values are Sensible: the value rows catch it"); + LoadKind other = kind == LoadKind.Heating ? LoadKind.Cooling : LoadKind.Heating; + Assert.That(run.Row("b1a|model|" + other + "|consumption").Status, Is.EqualTo("PASS"), "the other load type is untouched"); + } + + [Test] + public void MissingTotalBasisNote_Fails() + { + string model = Model(); + S11Case run = S11Support.Evaluate(SyntheticRun.For(model, "two"), model, "f3-no-note", afterProduce: c => S11Support.EditB1a(c, d => d.Provenance.Notes.Remove(S11Names.TotalBasisNote))); + + Assert.That(run.Row("b1a|model|-|totalBasisNote").Status, Is.EqualTo("FAIL")); + Assert.That(run.Compare.Verdict, Is.EqualTo("FAIL")); + } + } +} diff --git a/benchmark/SAM.Analytical.OpenStudio.InputReadBack.Tests/S11IdentityTests.cs b/benchmark/SAM.Analytical.OpenStudio.InputReadBack.Tests/S11IdentityTests.cs new file mode 100644 index 0000000..b9b5616 --- /dev/null +++ b/benchmark/SAM.Analytical.OpenStudio.InputReadBack.Tests/S11IdentityTests.cs @@ -0,0 +1,159 @@ +// SPDX-License-Identifier: LGPL-3.0-or-later +// Copyright (c) 2020–2026 Michal Dengusiak & Jakub Ziolkowski and contributors + +using NUnit.Framework; +using SAM.Analytical.OpenStudio.ResultsReadBack; +using SAM.Analytical.Tas.DirectT3D.Validation.ReadBack; +using System; +using System.Collections.Generic; +using System.Linq; + +namespace SAM.Analytical.OpenStudio.InputReadBack.Tests +{ + /// + /// The S11 identity is the S09 native identity (geometry bijection + IDF equipment chain). Owner direction at approval: + /// duplicate room names are handled through geometry and identity and are never rejected only because the names match. + /// + [TestFixture] + public class S11IdentityTests + { + private const string SpaceAName = "\"Name\": \"FixA\""; + private const string SpaceBName = "\"Name\": \"FixB\""; + + [Test] + public void DuplicateSpaceNames_DistinctGeometry_ResolvedByGeometryAndPass() + { + // Same display name: space B gets a distinct GUID prefix, because the same name AND the same prefix give + // identical generated names (the separately tracked generated-name identity residual, next test). + string model = S11Support.Variant(TestSupport.Two, "two-same-name-distinct-prefix", (S11Support.SpaceBGuid, "6181d111-0000-0000-0000-00000000c0b1"), (SpaceBName, SpaceAName)); + AnalyticalModel loaded = Core.Convert.ToSAM(model).Single(); + List spaces = loaded.AdjacencyCluster.GetSpaces(); + Assert.That(spaces.Select(x => x.Name).Distinct().Count(), Is.EqualTo(1), "both spaces carry the same name"); + + S11Case run = S11Support.Evaluate(SyntheticRun.For(model, "two-same-name"), model, "same-name"); + + Assert.That(run.Compare.Verdict, Is.EqualTo("PASS"), run.Summary()); + Assert.That(run.Reference.Spaces.Select(x => x.NativeSpace).Distinct(StringComparer.OrdinalIgnoreCase).Count(), Is.EqualTo(2), "two native spaces, told apart by geometry"); + Assert.That(run.Reference.Spaces.All(x => x.IdentityProblem == null && x.SurfacePairs > 0), Is.True); + Assert.That(run.Compare.Rows.Where(x => x.Layer == "identity" && x.Quantity == "native").All(x => x.Status == "PASS"), Is.True); + } + + /// + /// Same names AND the shared GUID prefix of the signed-off fixture: the writer's names collide, so OpenStudio makes + /// one unique. The reference still identifies both spaces by geometry; the production key cross-check then fails + /// for the renamed one, which is a production naming collision, not a rejection of duplicate names. This is the + /// separately tracked generated-name identity residual (exact-name lookup for identical generated names); F-S11-1 + /// (#35) deliberately leaves such keys unmapped rather than guessing. + /// + [Test] + public void DuplicateSpaceNames_SharedGuidPrefix_IdentityStillByGeometry_ProductionKeyCollisionReported() + { + string model = S11Support.Variant(TestSupport.Two, "two-same-name-shared-prefix", (SpaceBName, SpaceAName)); + S11Case run = S11Support.Evaluate(SyntheticRun.For(model, "two-same-name-shared"), model, "same-name-shared"); + + Assert.That(run.Reference.Spaces.All(x => x.IdentityProblem == null), Is.True, string.Join("; ", run.Reference.Spaces.Select(x => x.IdentityProblem))); + Assert.That(run.Reference.Spaces.Select(x => x.IdealLoads).Distinct(StringComparer.OrdinalIgnoreCase).Count(), Is.EqualTo(2), "the native Ideal Loads are distinct"); + List keyRows = run.Compare.Rows.Where(x => x.Layer == "identity" && x.Quantity == "idealLoadsKey").ToList(); + Assert.That(keyRows.Count(x => x.Status == "FAIL"), Is.GreaterThanOrEqualTo(1), "the colliding production key differs from one native key: " + run.Summary()); + Assert.That(run.Compare.Rows.Where(x => x.Layer == "identity" && x.Quantity == "native").All(x => x.Status == "PASS"), Is.True, "no space is rejected for its name"); + } + + [Test] + public void NativeIdentity_OnTheExportedTwoZoneIdf_IsBijectiveAndIndependentOfNames() + { + string model = S11Support.Model(TestSupport.Two); + AnalyticalModel loaded = Core.Convert.ToSAM(model).Single(); + List idf = IdfText.Parse(SyntheticRun.ExportIdf(model, "two")).Where(x => NativeReader.IdfTypes.Contains(x.Type, StringComparer.OrdinalIgnoreCase)).ToList(); + ExpectedModel expected = ReadBackExpected.Build(loaded, null, TestSupport.Two); + + List identity = NativeZoneIdentity.Resolve(new NativeModel(new NativeSnapshot { Idf = idf }), expected); + Assert.That(identity, Has.Count.EqualTo(2)); + Assert.That(identity.All(x => x.Problem == null && x.NameCarrierAgrees), Is.True); + Assert.That(identity.Select(x => x.Zone).Distinct().Count(), Is.EqualTo(2)); + + // Swapping the names of the two native spaces does not change which geometry each SAM space matches. + string a = identity[0].NativeSpace; + string b = identity[1].NativeSpace; + string swapped = SyntheticRun.ExportIdf(model, "two").Replace(a, "@@A@@").Replace(b, a).Replace("@@A@@", b); + List swappedIdf = IdfText.Parse(swapped).Where(x => NativeReader.IdfTypes.Contains(x.Type, StringComparer.OrdinalIgnoreCase)).ToList(); + List after = NativeZoneIdentity.Resolve(new NativeModel(new NativeSnapshot { Idf = swappedIdf }), expected); + Assert.That(after[0].NativeSpace, Is.EqualTo(b), "geometry, not the name, makes the match"); + Assert.That(after[0].NameCarrierAgrees, Is.True, "both names carry the shared GUID prefix, so the carrier cannot tell them apart"); + } + } + + /// Unit checks of the S11 calendar, ESO dictionary parsing and the completeness judgement. + [TestFixture] + public class S11UnitTests + { + private static RunCalendar Calendar(string beginYear, string endMonth = "12", string endDay = "31", string endYear = null) + { + string idf = "RunPeriod,\n Run 1, !- Name\n 1, !- Begin Month\n 1, !- Begin Day of Month\n " + beginYear + ", !- Begin Year\n " + endMonth + ", !- End Month\n " + endDay + ", !- End Day of Month\n " + (endYear ?? beginYear) + ", !- End Year\n Sunday; !- Day of Week for Start Day\n"; + return RunCalendar.FromIdf(IdfText.Parse(idf)); + } + + [Test] + public void Calendar_FullNonLeapYear_Has8760OrderedHours() + { + RunCalendar calendar = Calendar("2006"); + Assert.That(calendar.Problem, Is.Null); + Assert.That(calendar.Count, Is.EqualTo(8760)); + Assert.That(calendar.IndexOf(new HourKey(1, 1, 1)), Is.EqualTo(0)); + Assert.That(calendar.IndexOf(new HourKey(12, 31, 24)), Is.EqualTo(8759)); + Assert.That(calendar.IndexOf(new HourKey(7, 7, 13)), Is.EqualTo(4500)); + Assert.That(calendar.IndexOf(new HourKey(2, 29, 1)), Is.EqualTo(-1)); + Assert.That(Calendar(string.Empty).Problem, Is.Null, "a blank year is EnergyPlus's non-leap default"); + } + + [TestCase("2024", "12", "31", null, "leap")] + [TestCase("2006", "6", "30", null, "full year")] + [TestCase("2006", "12", "31", "2007", "more than one year")] + public void Calendar_OtherRunPeriods_AreProblems(string beginYear, string endMonth, string endDay, string endYear, string expected) + { + RunCalendar calendar = Calendar(beginYear, endMonth, endDay, endYear); + Assert.That(calendar.Problem, Does.Contain(expected)); + Assert.That(calendar.Count, Is.Zero); + } + + [Test] + public void Calendar_TwoRunPeriods_IsAProblem() + { + string one = "RunPeriod, A, 1, 1, 2006, 12, 31, 2006;\n"; + Assert.That(RunCalendar.FromIdf(IdfText.Parse(one + one.Replace(" A,", " B,"))).Problem, Does.Contain("2 RunPeriod")); + } + + [Test] + public void EsoDictionaryLine_ParsesVariablesAndMeters() + { + EsoVariable variable = EsoResultRead.DictionaryLine("395,1,SAM_IDEALLOADS_OFFICE_CB68BED1,Zone Ideal Loads Supply Air Total Cooling Energy [J] !Hourly"); + Assert.That(variable.Id, Is.EqualTo(395)); + Assert.That(variable.Key, Is.EqualTo("SAM_IDEALLOADS_OFFICE_CB68BED1")); + Assert.That(variable.Name, Is.EqualTo(S11Names.TotalCooling)); + Assert.That(variable.Units, Is.EqualTo("J")); + Assert.That(variable.Frequency, Is.EqualTo("Hourly")); + Assert.That(variable.IsMeter, Is.False); + + EsoVariable meter = EsoResultRead.DictionaryLine("396,1,DistrictCooling:Facility [J] !TimeStep"); + Assert.That(meter.IsMeter, Is.True); + Assert.That(meter.Name, Is.EqualTo("DistrictCooling:Facility")); + Assert.That(meter.Frequency, Is.EqualTo("TimeStep")); + } + + [Test] + public void Completeness_CountAloneIsNeverAccepted() + { + RunCalendar calendar = Calendar("2006"); + ChannelSeries series = new ChannelSeries { Channel = "sql", Variable = S11Names.TotalCooling, Key = "K", DictionaryEntries = 1 }; + series.Points.AddRange(calendar.Hours.Select(x => new HourlyPoint { Month = x.Month, Day = x.Day, Hour = x.Hour, Value = 1 })); + Assert.That(Completeness.Check(calendar, series).Complete, Is.True); + + series.Points[101].Hour = series.Points[100].Hour; + CompletenessResult shifted = Completeness.Check(calendar, series); + Assert.That(shifted.Found, Is.EqualTo(8760)); + Assert.That(shifted.Complete, Is.False); + Assert.That(shifted.Missing, Is.EqualTo(1)); + Assert.That(shifted.Duplicated, Is.EqualTo(1)); + Assert.That(shifted.Values, Is.Null); + } + } +} diff --git a/benchmark/SAM.Analytical.OpenStudio.InputReadBack.Tests/S11NegativeTests.cs b/benchmark/SAM.Analytical.OpenStudio.InputReadBack.Tests/S11NegativeTests.cs new file mode 100644 index 0000000..02fd795 --- /dev/null +++ b/benchmark/SAM.Analytical.OpenStudio.InputReadBack.Tests/S11NegativeTests.cs @@ -0,0 +1,341 @@ +// SPDX-License-Identifier: LGPL-3.0-or-later +// Copyright (c) 2020–2026 Michal Dengusiak & Jakub Ziolkowski and contributors + +using NUnit.Framework; +using SAM.Analytical.Benchmark; +using SAM.Analytical.OpenStudio.Benchmark; +using SAM.Analytical.OpenStudio.ResultsReadBack; +using System; +using System.Data.SQLite; +using System.IO; +using System.Linq; + +namespace SAM.Analytical.OpenStudio.InputReadBack.Tests +{ + /// + /// S11 negatives other than completeness and F3 (plan section 7): each must FAIL or BLOCK by its intended mechanism, + /// and the zero rule, ties and design loads are checked positively as well. + /// + [TestFixture] + public class S11NegativeTests + { + private static string Two() => S11Support.Two(); + + private static SyntheticRun Synthetic() => SyntheticRun.For(Two(), "two"); + + private static void AssertFails(S11Case run, params string[] rows) + { + Assert.That(run.Compare.Verdict, Is.EqualTo("FAIL"), run.Summary()); + foreach (string row in rows) + { + Assert.That(run.Row(row).Status, Is.EqualTo("FAIL"), row + ": " + run.Row(row).Reason); + } + } + + private static void AssertBlocks(S11Case run, string reason) + { + Assert.That(run.Compare.Verdict, Is.EqualTo("BLOCKED"), run.Summary()); + Assert.That(run.Compare.Failed, Is.Zero, run.Summary()); + Assert.That(string.Join(" ", run.Compare.Rows.Select(x => x.Reason)), Does.Contain(reason)); + } + + // ---- zero rule and ties ---------------------------------------------------------------------------------------- + + private static SyntheticRun ZeroCoolingInSpace1() + { + SyntheticRun synthetic = Synthetic(); + synthetic.EditSeries(true, true, s => s.ForEach(p => p.Value = 0), k => k == SyntheticRun.Key(S11Names.TotalCooling, synthetic.Zones[1].IdealLoads) || k == SyntheticRun.Key(S11Names.SensibleCooling, synthetic.Zones[1].IdealLoads)); + return synthetic; + } + + [Test] + public void AllZeroCooling_B1aLoadZeroHourUnavailable_SamHourZeroAccepted_Passes() + { + S11Case run = S11Support.Evaluate(ZeroCoolingInSpace1(), Two(), "zero-rule"); + + Assert.That(run.Compare.Verdict, Is.EqualTo("PASS"), run.Summary()); + Assert.That(run.Reference.Space(run.Synthetic.Zones[1].SpaceGuid).Quantities[S11Names.TotalCooling].AllZero, Is.True); + Assert.That(run.Row("b1a|" + run.Space(1) + "|Cooling|peakHour").Reason, Does.Contain("zero rule")); + Assert.That(run.Row("samP|" + run.Space(1) + "|Cooling|peakHour").Subject, Is.EqualTo(0), "SAM's own earliest-of-a-full-year-tie hour 0"); + } + + [Test] + public void ZeroReplay_B1aPublishesHourZeroForAnAllZeroYear_Fails() + { + S11Case run = S11Support.Evaluate(ZeroCoolingInSpace1(), Two(), "zero-replay", afterProduce: c => S11Support.EditB1a(c, d => + { + d.Spaces.Single(x => x.Guid == c.Synthetic.Zones[1].SpaceGuid.ToString("N")).Cooling.PeakHour = MetricValue.AvailableValue(0, MetricUnit.HourOfYear); + })); + + AssertFails(run, "b1a|" + run.Space(1) + "|Cooling|peakHour"); + } + + [Test] + public void MissingZoneReportedAsZero_Blocks() + { + SyntheticRun synthetic = Synthetic(); + foreach (string key in synthetic.Sql.Keys.Where(x => x.EndsWith("|" + synthetic.Zones[1].IdealLoads, StringComparison.Ordinal)).ToList()) + { + synthetic.Sql.Remove(key); + synthetic.Eso.Remove(key); + } + + S11Case run = S11Support.Evaluate(synthetic, Two(), "missing-zone-zero", afterProduce: c => S11Support.EditB1a(c, d => + { + BenchmarkSpaceResult space = d.Spaces.Single(x => x.Guid == c.Synthetic.Zones[1].SpaceGuid.ToString("N")); + space.Cooling.PeakLoad = MetricValue.AvailableValue(0, MetricUnit.Watt); + })); + + Assert.That(run.Compare.Verdict, Is.Not.EqualTo("PASS")); + Assert.That(run.Row("b1a|" + run.Space(1) + "|Cooling|peakLoad").Status, Is.EqualTo("BLOCKED"), "a missing zone is unavailable in the reference, never 0"); + } + + [Test] + public void Tie_EarliestHourRequired_LaterTieHourFails() + { + SyntheticRun synthetic = Synthetic(); + string total = SyntheticRun.Key(S11Names.TotalCooling, synthetic.Zones[0].IdealLoads); + double peak = synthetic.Sql[total].Max(x => x.Value); + synthetic.EditSeries(true, true, s => s.Single(p => p.Month == 9 && p.Day == 1 && p.Hour == 12).Value = peak, k => k == total); + S11Case control = S11Support.Evaluate(synthetic, Two(), "tie-control"); + Assert.That(control.Reference.Space(synthetic.Zones[0].SpaceGuid).Quantities[S11Names.TotalCooling].TieCount, Is.EqualTo(2)); + Assert.That(control.Compare.Verdict, Is.EqualTo("PASS"), control.Summary()); + + int later = control.Reference.Space(synthetic.Zones[0].SpaceGuid).Quantities[S11Names.TotalCooling].PeakHour; + int septemberFirst = (31 + 28 + 31 + 30 + 31 + 30 + 31 + 31) * 24 + 11; + Assert.That(later, Is.LessThan(septemberFirst), "the earliest tie is the August peak"); + S11Case run = S11Support.Evaluate(synthetic, Two(), "tie-later", mutateInputs: inputs => + { + foreach (SpaceSimulationResult result in S11Support.Annual(inputs, synthetic.Zones[0].SpaceGuid, LoadKind.Cooling)) + { + result.SetValue(Analytical.SpaceSimulationResultParameter.LoadIndex, septemberFirst); + } + }); + + AssertFails(run, "samP|" + run.Space(0) + "|Cooling|peakHour", "b1a|" + run.Space(0) + "|Cooling|peakHour"); + } + + // ---- value slips --------------------------------------------------------------------------------------------- + + [TestCase("hourPlusOne")] + [TestCase("hourMinusOne")] + [TestCase("heatingCoolingSwapped")] + [TestCase("times1000")] + [TestCase("dividedBy3600")] + [TestCase("consumptionOff1e-6")] + public void ValueSlip_Fails(string slip) + { + Guid space = Synthetic().Zones[0].SpaceGuid; + S11Case run = S11Support.Evaluate(Synthetic(), Two(), "slip-" + slip, mutateInputs: inputs => + { + switch (slip) + { + case "hourPlusOne": + case "hourMinusOne": + foreach (SpaceSimulationResult result in S11Support.Annual(inputs, space, LoadKind.Cooling)) + { + result.TryGetValue(Analytical.SpaceSimulationResultParameter.LoadIndex, out int hour); + result.SetValue(Analytical.SpaceSimulationResultParameter.LoadIndex, hour + (slip == "hourPlusOne" ? 1 : -1)); + } + + break; + case "heatingCoolingSwapped": + foreach (SpaceSimulationResult result in inputs.SpaceResults.Where(x => x.Reference == space.ToString("N"))) + { + result.TryGetValue(Analytical.SpaceSimulationResultParameter.LoadType, out string loadType); + result.SetValue(Analytical.SpaceSimulationResultParameter.LoadType, loadType == "Heating" ? "Cooling" : "Heating"); + } + + break; + case "times1000": + foreach (SpaceSimulationResult result in S11Support.Annual(inputs, space, LoadKind.Cooling)) + { + result.TryGetValue(Analytical.SpaceSimulationResultParameter.Load, out double load); + result.SetValue(Analytical.SpaceSimulationResultParameter.Load, load * 1000); + } + + break; + case "dividedBy3600": + inputs.ModelResult.TryGetValue(AnalyticalModelSimulationResultParameter.ConsumptionCooling, out double kwh); + inputs.ModelResult.SetValue(AnalyticalModelSimulationResultParameter.ConsumptionCooling, kwh / 3600); + break; + case "consumptionOff1e-6": + inputs.ModelResult.TryGetValue(AnalyticalModelSimulationResultParameter.ConsumptionCooling, out double value); + inputs.ModelResult.SetValue(AnalyticalModelSimulationResultParameter.ConsumptionCooling, value * (1 + 1e-6)); + break; + } + }); + + string target = slip.StartsWith("hour", StringComparison.Ordinal) ? "b1a|" + run.Space(0) + "|Cooling|peakHour" + : slip == "times1000" || slip == "heatingCoolingSwapped" ? "b1a|" + run.Space(0) + "|Cooling|peakLoad" + : "b1a|model|Cooling|consumption"; + AssertFails(run, target, target.Replace("b1a|", "samP|")); + } + + // ---- channels, keys, identity ---------------------------------------------------------------------------------- + + [Test] + public void SqlAndEsoDisagreeInOneHour_Blocks() + { + SyntheticRun synthetic = Synthetic(); + synthetic.EditSeries(false, true, s => s.Single(p => p.Month == 8 && p.Day == 1 && p.Hour == 12).Value += 1, k => k == SyntheticRun.Key(S11Names.TotalCooling, synthetic.Zones[0].IdealLoads)); + AssertBlocks(S11Support.Evaluate(synthetic, Two(), "c4-disagree"), "S11-C4"); + } + + [Test] + public void UnknownReportKey_Blocks() + { + SyntheticRun synthetic = Synthetic(); + string ghost = SyntheticRun.Key(S11Names.TotalCooling, "SAM_IDEALLOADS_GHOST_0badf00d"); + synthetic.Sql[ghost] = synthetic.Sql[SyntheticRun.Key(S11Names.TotalCooling, synthetic.Zones[0].IdealLoads)].Select(x => x.Copy()).ToList(); + synthetic.Eso[ghost] = synthetic.Sql[ghost].Select(x => x.Copy()).ToList(); + AssertBlocks(S11Support.Evaluate(synthetic, Two(), "c5-unknown-key"), "S11-C5"); + } + + [Test] + public void NativeKeyIsNotTheProductionKey_Fails() + { + S11Case run = S11Support.Evaluate(Synthetic(), Two(), "key-mismatch", beforeCompare: c => c.Subject.ProductionKeys[0].IdealLoads = "SAM_IdealLoads_Other_00000000"); + AssertFails(run, "identity|space:" + run.Subject.ProductionKeys[0].SpaceGuid.ToString("N") + "|-|idealLoadsKey"); + } + + [Test] + public void ZoneWithTwoIdealLoadsSystems_Blocks() + { + SyntheticRun synthetic = Synthetic(); + int at =synthetic.Idf.IndexOf("ZoneHVAC:EquipmentList,", StringComparison.Ordinal); + Assert.That(at, Is.GreaterThanOrEqualTo(0)); + int end = synthetic.Idf.IndexOf(';', at); + string block = synthetic.Idf.Substring(at, end - at); + synthetic.Idf = synthetic.Idf.Substring(0, at) + block + ",\n ZoneHVAC:IdealLoadsAirSystem, !- Zone Equipment Object Type 2\n " + synthetic.Zones[1].IdealLoads + ", !- Zone Equipment Name 2\n 2, !- Zone Equipment Cooling Sequence 2\n 2" + synthetic.Idf.Substring(end); + + S11Case run = S11Support.Evaluate(synthetic, Two(), "two-ideal-loads"); + Assert.That(run.Compare.Verdict, Is.Not.EqualTo("PASS"), run.Summary()); + Assert.That(run.Reference.Spaces.Any(x => x.IdentityProblem != null && (x.IdentityProblem.Contains("Ideal Loads") || x.IdentityProblem.Contains("bijective"))), Is.True, string.Join("; ", run.Reference.Spaces.Select(x => x.IdentityProblem))); + } + + [Test] + public void NonOpenStudioSource_Fails() + { + S11Case run = S11Support.Evaluate(Synthetic(), Two(), "source", beforeCompare: c => c.Subject.SamI.First(x => x.IsAnnual).Source = "SAM.Analytical.Tas"); + Assert.That(run.Compare.Verdict, Is.EqualTo("FAIL"), run.Summary()); + Assert.That(run.Compare.Rows.Any(x => x.Layer == "samI" && x.Quantity == "result" && x.Status == "FAIL" && x.Reason.Contains("source")), Is.True); + } + + [Test] + public void MissingResult_Fails() + { + Guid space = Synthetic().Zones[1].SpaceGuid; + S11Case run = S11Support.Evaluate(Synthetic(), Two(), "missing-result", mutateInputs: inputs => inputs.SpaceResults.RemoveAll(x => x.Reference == space.ToString("N") && x.TryGetValue(Analytical.SpaceSimulationResultParameter.LoadType, out string t) && t == "Heating")); + AssertFails(run, "samP|" + run.Space(1) + "|Heating|result", "b1a|" + run.Space(1) + "|Heating|peakLoad"); + } + + [Test] + public void CapturedObjectsAlteredAfterTheB1a_ClosureFails_AndSamPRowsBlock() + { + S11Case run = S11Support.Evaluate(Synthetic(), Two(), "closure", afterProduce: c => + { + OpenStudioBenchmarkInputs inputs = OpenStudioBenchmarkInputs.Read(Path.Combine(c.Run, RunLayout.SamResultsFile)); + inputs.SpaceResults[0].TryGetValue(Analytical.SpaceSimulationResultParameter.Load, out double load); + inputs.SpaceResults[0].SetValue(Analytical.SpaceSimulationResultParameter.Load, load + 1); + inputs.Write(Path.Combine(c.Run, RunLayout.SamResultsFile)); + }); + + Assert.That(run.Subject.ClosureIdentical, Is.False, run.Subject.ClosureDetail); + AssertFails(run, "closure|model|-|samPToB1a"); + Assert.That(run.Compare.Rows.Where(x => x.Layer == "samP").All(x => x.Status == "BLOCKED"), Is.True, "every SAM-P row blocks without the closure"); + } + + // ---- sizing, design load, versions, meters, basis ----------------------------------------------------------- + + [Test] + public void ZoneSizesWithDesignDaySourceNone_Blocks() + { + SyntheticRun synthetic = Synthetic(); + synthetic.ZoneSizes.Add(new ZoneSizeRow { ZoneName = synthetic.Zones[0].Zone, LoadType = "Heating", CalcDesLoad = 1234.5 }); + AssertBlocks(S11Support.Evaluate(synthetic, Two(), "c8-sizes-none"), "S11-C8"); + } + + [Test] + public void DesignLoadMappedFromCalcDesLoad_WhenSizingRan_Passes() + { + SyntheticRun synthetic = Synthetic(); + foreach (SyntheticZone zone in synthetic.Zones) + { + synthetic.ZoneSizes.Add(new ZoneSizeRow { ZoneName = zone.Zone, LoadType = "Heating", CalcDesLoad = 2345.25 }); + synthetic.ZoneSizes.Add(new ZoneSizeRow { ZoneName = zone.Zone, LoadType = "Cooling", CalcDesLoad = 1875.125 }); + } + + S11Case run = S11Support.Evaluate(synthetic, Two(), "design-load", DesignDaySource.EmbeddedModel); + Assert.That(run.Compare.Verdict, Is.EqualTo("PASS"), run.Summary()); + Assert.That(run.Row("b1a|" + run.Space(0) + "|Heating|designLoad").Reference, Is.EqualTo(2345.25)); + Assert.That(run.Row("samP|" + run.Space(1) + "|Cooling|designLoad").Status, Is.EqualTo("PASS")); + Assert.That(run.Row("samI|" + run.Space(1) + "|Cooling|designLoad").Status, Is.EqualTo("INFO"), "the legacy SAM-I design-day path is informational"); + } + + [Test] + public void DesignLoadPublishedAsZeroWithoutSizing_Fails() + { + S11Case run = S11Support.Evaluate(Synthetic(), Two(), "design-zero", afterProduce: c => S11Support.EditB1a(c, d => d.Spaces[0].Heating.DesignLoad = MetricValue.AvailableValue(0, MetricUnit.Watt))); + Assert.That(run.Compare.Verdict, Is.EqualTo("FAIL")); + Assert.That(run.Compare.Rows.Any(x => x.Layer == "b1a" && x.Quantity == "designLoad" && x.Status == "FAIL"), Is.True); + } + + [Test] + public void WrongEnergyPlusVersion_Blocks() + { + SyntheticRun synthetic = Synthetic(); + synthetic.SqlVersion = "EnergyPlus, Version 24.2.0-94a887817b, YMD=2026.10.24 08:00"; + AssertBlocks(S11Support.Evaluate(synthetic, Two(), "c9-version"), "S11-C9"); + } + + [Test] + public void FacilityMeterDisagreesWithTheZones_BlocksConsumption() + { + SyntheticRun synthetic = Synthetic(); + synthetic.SqlEdit = connection => + { + using (SQLiteCommand command = connection.CreateCommand()) + { + command.CommandText = "UPDATE ReportData SET Value = Value + 1000 WHERE ReportDataIndex = (SELECT MIN(rd.ReportDataIndex) FROM ReportData rd JOIN ReportDataDictionary d ON d.ReportDataDictionaryIndex = rd.ReportDataDictionaryIndex WHERE d.Name = @n)"; + command.Parameters.AddWithValue("@n", S11Names.CoolingMeter); + command.ExecuteNonQuery(); + } + }; + + S11Case run = S11Support.Evaluate(synthetic, Two(), "c7-meter"); + Assert.That(run.Row("b1a|model|Cooling|consumption").Status, Is.EqualTo("BLOCKED")); + Assert.That(run.Row("b1a|model|Cooling|consumption").Reason, Does.Contain("S11-C7")); + Assert.That(run.Row("b1a|model|Cooling|peakLoad").Status, Is.EqualTo("PASS"), "the meter only qualifies the consumption"); + } + + [Test] + public void NegativeLatent_BlocksThatLoadType() + { + SyntheticRun synthetic = Synthetic(); + synthetic.EditSeries(true, true, s => s.Single(p => p.Month == 8 && p.Day == 2 && p.Hour == 12).Value += 5_000_000, k => k == SyntheticRun.Key(S11Names.SensibleCooling, synthetic.Zones[0].IdealLoads)); + S11Case run = S11Support.Evaluate(synthetic, Two(), "c6-negative-latent"); + + Assert.That(run.Row("b1a|" + run.Space(0) + "|Cooling|peakLoad").Status, Is.EqualTo("BLOCKED")); + Assert.That(run.Row("b1a|" + run.Space(0) + "|Cooling|peakLoad").Reason, Does.Contain("S11-C6")); + Assert.That(run.Row("b1a|" + run.Space(0) + "|Heating|peakLoad").Status, Is.EqualTo("PASS")); + } + + [Test] + public void WrongUnit_Blocks() + { + SyntheticRun synthetic = Synthetic(); + synthetic.SqlEdit = connection => + { + using (SQLiteCommand command = connection.CreateCommand()) + { + command.CommandText = "UPDATE ReportDataDictionary SET Units = 'kWh' WHERE Name = @n"; + command.Parameters.AddWithValue("@n", S11Names.TotalHeating); + command.ExecuteNonQuery(); + } + }; + + AssertBlocks(S11Support.Evaluate(synthetic, Two(), "c3-unit"), "S11-C3"); + } + } +} diff --git a/benchmark/SAM.Analytical.OpenStudio.InputReadBack.Tests/S11ResultsTests.cs b/benchmark/SAM.Analytical.OpenStudio.InputReadBack.Tests/S11ResultsTests.cs new file mode 100644 index 0000000..5998a08 --- /dev/null +++ b/benchmark/SAM.Analytical.OpenStudio.InputReadBack.Tests/S11ResultsTests.cs @@ -0,0 +1,229 @@ +// SPDX-License-Identifier: LGPL-3.0-or-later +// Copyright (c) 2020–2026 Michal Dengusiak & Jakub Ziolkowski and contributors + +using NUnit.Framework; +using SAM.Analytical.Benchmark; +using SAM.Analytical.OpenStudio.Benchmark; +using SAM.Analytical.OpenStudio.ResultsReadBack; +using SAM.Analytical.OpenStudio.ResultsReadBack.Cli; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; + +namespace SAM.Analytical.OpenStudio.InputReadBack.Tests +{ + /// + /// S11 positive path and the separate-process harness: a consistent synthetic run PASSes every required row with read, + /// subject and compare each in its own process, and the CLI negatives on copies give their required exits. + /// + [TestFixture] + public class S11ResultsTests + { + [Test] + public void ConsistentTwoZoneRun_PassesEveryRow_InSeparateProcesses_AndTheNegativesRefuseCopies() + { + string model = S11Support.Two(); + S11Case run = S11Support.Prepare(SyntheticRun.For(model, "two"), model, "two-cli"); + + Assert.That(S11Support.Cli(out string read, "read", "--run", run.Run, "--model", model, "--fixture-id", TestSupport.Two), Is.EqualTo(0), read); + Assert.That(S11Support.Cli(out string subject, "subject", "--run", run.Run, "--model", model), Is.EqualTo(0), subject); + int exit = S11Support.Cli(out string compare, "compare", "--run", run.Run, "--model", model); + CompareDocument document = S11Json.Load(Path.Combine(run.Run, "compare", "compare.json")); + Assert.That(exit, Is.EqualTo(0), compare + " " + string.Join(" | ", document.Rows.Where(x => x.Status != "PASS").Take(10).Select(x => x.Id + " " + x.Status + " " + x.Reason))); + Assert.That(document.Verdict, Is.EqualTo("PASS")); + Assert.That(document.Failed + document.Blocked, Is.Zero); + Assert.That(document.Required, Is.GreaterThan(60), "every layer, space, load type and quantity is a row"); + Assert.That(document.Notes, Does.Contain(CompareDocument.S10Limitation)); + Assert.That(document.F3.MeetsOd7, Is.True, document.F3.Detail); + + ReferenceDocument reference = S11Json.Load(Path.Combine(run.Run, "reference.json")); + SubjectDocument captured = S11Json.Load(Path.Combine(run.Run, "subject.json")); + Assert.That(new[] { reference.ProcessId, captured.ProcessId, document.ProcessId }.Distinct().Count(), Is.EqualTo(3), "read, subject and compare ran in three processes"); + Assert.That(reference.SqlSha256Before, Is.EqualTo(reference.SqlSha256After)); + Assert.That(reference.EsoSha256Before, Is.EqualTo(reference.EsoSha256After)); + Assert.That(captured.ClosureIdentical, Is.True, captured.ClosureDetail); + Assert.That(File.ReadAllText(Path.Combine(run.Run, "compare", "compare.md")), Does.Contain("Verdict: **PASS**")); + + // Negatives on copies, each read/subject/compared fresh; the original stays byte-identical. + string before = Sha256.Of(Path.Combine(run.Run, "work", "run", "eplusout.sql")) + Sha256.Of(Path.Combine(run.Run, "work", "run", "eplusout.eso")) + Sha256.Of(Path.Combine(run.Run, "b1a.json")) + Sha256.Of(Path.Combine(run.Run, "sam.json")); + foreach (string caseId in NegativeStep.Cases.Keys) + { + string copy = run.Run + "-" + caseId; + Assert.That(S11Support.Cli(out string negative, "negative", "--run", run.Run, "--model", model, "--case", caseId, "--out", copy), Is.EqualTo(0), negative); + Assert.That(S11Support.Cli(out string copyRead, "read", "--run", copy, "--model", model, "--fixture-id", TestSupport.Two), Is.EqualTo(0), copyRead); + if (!File.Exists(Path.Combine(copy, "subject.json"))) + { + Assert.That(S11Support.Cli(out string copySubject, "subject", "--run", copy, "--model", model), Is.EqualTo(0), copySubject); + } + + int copyExit = S11Support.Cli(out string copyCompare, "compare", "--run", copy, "--model", model); + CompareDocument copyDocument = S11Json.Load(Path.Combine(copy, "compare", "compare.json")); + Assert.That(copyExit, Is.EqualTo(NegativeStep.Cases[caseId]), caseId + ": " + copyCompare + " " + string.Join(" | ", copyDocument.Rows.Where(x => x.Status != "PASS").Take(6).Select(x => x.Id + " " + x.Status + " " + x.Reason))); + if (caseId == "neg-sensible-cooling-substitution") + { + Assert.That(copyDocument.Blocked, Is.Zero, "a substitution must FAIL, never BLOCK"); + Assert.That(copyDocument.Rows.Where(x => x.LoadType == "Cooling" && (x.Quantity == "peakLoad" || x.Quantity == "consumption") && x.Layer != "identity").All(x => x.Status == "FAIL"), Is.True, "every cooling value row fails"); + Assert.That(copyDocument.Inputs["mutation"], Does.Contain("Sensible")); + } + } + + string after = Sha256.Of(Path.Combine(run.Run, "work", "run", "eplusout.sql")) + Sha256.Of(Path.Combine(run.Run, "work", "run", "eplusout.eso")) + Sha256.Of(Path.Combine(run.Run, "b1a.json")) + Sha256.Of(Path.Combine(run.Run, "sam.json")); + Assert.That(after, Is.EqualTo(before), "the negatives never touch the original run"); + } + + [Test] + public void ConsistentBoxRun_PassesEveryRow() + { + string model = S11Support.Model(TestSupport.Box); + S11Case run = S11Support.Evaluate(SyntheticRun.For(model, "box"), model, "box"); + + Assert.That(run.Compare.Verdict, Is.EqualTo("PASS"), run.Summary()); + Assert.That(run.Reference.Spaces, Has.Count.EqualTo(1)); + Assert.That(run.Reference.Checks.Where(x => x.Status == "BLOCKED"), Is.Empty); + Assert.That(run.Reference.Completeness.All(x => x.Complete && x.Found == 8760), Is.True); + Assert.That(run.Reference.Checks.Count(x => x.Id == "S11-C7" && x.Status == "PASS"), Is.EqualTo(4), "both meters in both files equal the zones' Total energy"); + } + + [Test] + public void CompareStep_RefusesInputsTheReadAndSubjectDidNotSee() + { + string model = S11Support.Model(TestSupport.Box); + S11Case run = S11Support.Evaluate(SyntheticRun.For(model, "box"), model, "box-consistency"); + S11Json.Save(run.Reference, Path.Combine(run.Run, RunLayout.ReferenceFile)); + S11Json.Save(run.Subject, Path.Combine(run.Run, RunLayout.SubjectFile)); + File.AppendAllText(Path.Combine(run.Run, "work", "run", "eplusout.eso"), "\n"); + + int exit = S11Support.Cli(out string output, "compare", "--run", run.Run, "--model", model); + + Assert.That(exit, Is.EqualTo(ReadBackExit.Infrastructure), output); + Assert.That(output, Does.Contain("eplusout.eso is not the file the read step saw")); + } + + // ---- F-S11-1 / F-S11-2 integrated: positive regressions on the signed-off fixtures ------------------------------ + + /// + /// F-S11-1 (SAM_OpenStudio#35): the signed-off s08-fix-two spaces FixA and FixB share the GUID prefix 5080c000. + /// Through the production import (SAM-P producer objects, SAM-I AddResults) and B1a each space keeps its own + /// zone-specific fractional unmet hours, and the native and production identities stay distinct and agree. Before + /// #35 one zone's not-met time overwrote the other's and these rows FAILed; a swap between the zones FAILs too. + /// + [Test] + public void SignedOffTwoZone_SharedGuidPrefix_EachSpaceKeepsItsOwnResults() + { + string model = S11Support.Two(); + S11Case run = S11Support.Evaluate(SyntheticRun.For(model, "two", fractionalUnmet: true), model, "two-shared-prefix"); + + Guid a = Guid.Parse(S11Support.SpaceAGuid); + Guid b = Guid.Parse(S11Support.SpaceBGuid); + Assert.That(a.ToString("N").Substring(0, 8), Is.EqualTo(b.ToString("N").Substring(0, 8)), "the signed-off fixture's shared prefix"); + Assert.That(run.Synthetic.Zones.Select(x => x.SpaceGuid), Is.EquivalentTo(new[] { a, b })); + Assert.That(run.Compare.Verdict, Is.EqualTo("PASS"), run.Summary()); + Assert.That(run.Compare.Failed + run.Compare.Blocked, Is.Zero); + + // Native identity: two distinct spaces, zones and Ideal Loads; each is the production key of its own SAM space. + Assert.That(run.Reference.Spaces.All(x => x.IdentityProblem == null), Is.True); + foreach (Func part in new Func[] { x => x.NativeSpace, x => x.Zone, x => x.IdealLoads }) + { + Assert.That(run.Reference.Spaces.Select(part).Distinct(StringComparer.OrdinalIgnoreCase).Count(), Is.EqualTo(2)); + } + + foreach (ProductionKey key in run.Subject.ProductionKeys) + { + SpaceReference native = run.Reference.Space(key.SpaceGuid); + Assert.That(key.IdealLoads, Is.EqualTo(native.IdealLoads).IgnoreCase, key.SpaceName); + Assert.That(key.ThermalZone, Is.EqualTo(native.Zone).IgnoreCase, key.SpaceName); + } + + List identity = run.Compare.Rows.Where(x => x.Layer == "identity" && (x.Quantity == "native" || x.Quantity == "idealLoadsKey" || x.Quantity == "zoneKey")).ToList(); + Assert.That(identity, Has.Count.EqualTo(6)); + Assert.That(identity.All(x => x.Status == "PASS"), Is.True, run.Summary()); + + // Zone-specific fractional unmet hours: exact in the independent reference, and each layer carries its own space's. + for (int z = 0; z < run.Synthetic.Zones.Count; z++) + { + Guid space = run.Synthetic.Zones[z].SpaceGuid; + double heating = z == 0 ? 14.5 : 7.25; + double cooling = z == 0 ? 14.75 : 5.25; + Assert.That(run.Reference.Space(space).Quantities[S11Names.UnmetHeating].Sum, Is.EqualTo(heating)); + Assert.That(run.Reference.Space(space).Quantities[S11Names.UnmetCooling].Sum, Is.EqualTo(cooling)); + foreach (string layer in new[] { "samP", "samI", "b1a" }) + { + Assert.That(run.Row(layer + "|" + run.Space(z) + "|Heating|unmetHours").Subject, Is.EqualTo(heating), layer); + Assert.That(run.Row(layer + "|" + run.Space(z) + "|Cooling|unmetHours").Subject, Is.EqualTo(cooling), layer); + } + } + + // SAM-I: every annual result is attached to the space it references; no zone or space was left unmatched. + List annual = run.Subject.SamI.Where(x => x.IsAnnual).ToList(); + Assert.That(annual, Has.Count.EqualTo(4)); + Assert.That(annual.All(x => x.AttachedTo.HasValue && Guid.Parse(x.Reference) == x.AttachedTo.Value), Is.True); + Assert.That(annual.Select(x => x.AttachedTo.Value).Distinct(), Is.EquivalentTo(new[] { a, b })); + Assert.That(run.Subject.SamIDiagnostics.Where(x => x.Contains("matched no SAM space") || x.Contains("No simulation result found")), Is.Empty); + } + + /// + /// F-S11-2 (SAM#219): fractional EnergyPlus not-met time, 14.5 h heating and 14.75 h cooling (binary-exact), survives + /// the production SQL import, a SAM JSON save and reload (the producer's sam.json, and the imported model) and B1a, + /// and S11 PASSes. Before SAM#219 the integer metadata rounded half to even (14 and 15) and these rows FAILed. + /// + [Test] + public void FractionalUnmetHours_SurviveSqlImport_SamSaveReload_AndB1a() + { + string model = S11Support.Model(TestSupport.Box); + S11Case run = S11Support.Evaluate(SyntheticRun.For(model, "box", fractionalUnmet: true), model, "box-fractional-unmet"); + Guid space = run.Synthetic.Zones.Single().SpaceGuid; + + Assert.That(run.Compare.Verdict, Is.EqualTo("PASS"), run.Summary()); + Assert.That(run.Reference.Space(space).Quantities[S11Names.UnmetHeating].Sum, Is.EqualTo(14.5)); + Assert.That(run.Reference.Space(space).Quantities[S11Names.UnmetCooling].Sum, Is.EqualTo(14.75)); + foreach (string layer in new[] { "samP", "samI", "b1a" }) + { + foreach ((string loadType, double expected) in new[] { ("Heating", 14.5), ("Cooling", 14.75) }) + { + ResultRow row = run.Row(layer + "|" + run.Space(0) + "|" + loadType + "|unmetHours"); + Assert.That(row.Status, Is.EqualTo("PASS"), row.Id); + Assert.That(row.Reference, Is.EqualTo(expected), row.Id); + Assert.That(row.Subject, Is.EqualTo(expected), row.Id); + } + } + + // SQL import (the producer's objects), then saved as sam.json and read back. + Assert.That(Unmet(S11Support.Annual(run.Inputs, space, LoadKind.Heating)), Is.EqualTo(14.5)); + Assert.That(Unmet(S11Support.Annual(run.Inputs, space, LoadKind.Cooling)), Is.EqualTo(14.75)); + OpenStudioBenchmarkInputs reloaded = OpenStudioBenchmarkInputs.Read(Path.Combine(run.Run, RunLayout.SamResultsFile)); + Assert.That(Unmet(S11Support.Annual(reloaded, space, LoadKind.Heating)), Is.EqualTo(14.5)); + Assert.That(Unmet(S11Support.Annual(reloaded, space, LoadKind.Cooling)), Is.EqualTo(14.75)); + + // B1a as written. + BenchmarkDocument b1a = BenchmarkSerializer.Deserialize(File.ReadAllText(Path.Combine(run.Run, RunLayout.B1aFile))); + BenchmarkSpaceResult b1aSpace = b1a.Spaces.Single(x => x.Guid == space.ToString("N")); + Assert.That(b1aSpace.Heating.UnmetHours.Value, Is.EqualTo(14.5)); + Assert.That(b1aSpace.Cooling.UnmetHours.Value, Is.EqualTo(14.75)); + + // SAM-I: the SAM_UI/Grasshopper import into the model, saved as SAM JSON and reopened. + AdjacencyCluster cluster = new AdjacencyCluster(run.Model.AdjacencyCluster); + Analytical.OpenStudio.Modify.AddResults(cluster, Path.Combine(run.Run, "work", "run", "eplusout.sql"), out List _); + string saved = Path.Combine(run.Run, "imported-model.json"); + Assert.That(Core.Convert.ToFile(new AnalyticalModel(run.Model, cluster), saved, Core.SAMFileType.Json), Is.True); + AnalyticalModel reopened = Core.Convert.ToSAM(saved).Single(); + Space reopenedSpace = reopened.AdjacencyCluster.GetSpaces().Single(x => x.Guid == space); + List attached = reopened.AdjacencyCluster.GetResults(reopenedSpace); + Assert.That(Unmet(attached.Where(x => IsLoadType(x, LoadKind.Heating))), Is.EqualTo(14.5)); + Assert.That(Unmet(attached.Where(x => IsLoadType(x, LoadKind.Cooling))), Is.EqualTo(14.75)); + } + + private static bool IsLoadType(SpaceSimulationResult result, LoadKind kind) + { + return result.TryGetValue(Analytical.SpaceSimulationResultParameter.LoadType, out string loadType) && loadType == kind.ToString(); + } + + /// The unmet hours of the single result, read as a double (the stored value, not a rounded one). + private static double Unmet(IEnumerable results) + { + SpaceSimulationResult result = results.Single(); + Assert.That(result.TryGetValue(Analytical.SpaceSimulationResultParameter.UnmetHours, out double value), Is.True); + return value; + } + } +} diff --git a/benchmark/SAM.Analytical.OpenStudio.InputReadBack.Tests/S11SourceScanTests.cs b/benchmark/SAM.Analytical.OpenStudio.InputReadBack.Tests/S11SourceScanTests.cs new file mode 100644 index 0000000..bbb2fde --- /dev/null +++ b/benchmark/SAM.Analytical.OpenStudio.InputReadBack.Tests/S11SourceScanTests.cs @@ -0,0 +1,122 @@ +// SPDX-License-Identifier: LGPL-3.0-or-later +// Copyright (c) 2020–2026 Michal Dengusiak & Jakub Ziolkowski and contributors + +using NUnit.Framework; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.RegularExpressions; + +namespace SAM.Analytical.OpenStudio.ResultsReadBack.Tests +{ + /// + /// S11 reference independence (ported from SAM_Tas S10SourceScanTests): no file of the reference library (native + /// readers, reference, compare, report, classes) and not its project file may reach the code under test: the + /// OpenStudio runner, the importer and producer conversions, the production naming and unit helpers, AddResults, the + /// SAM.Analytical.OpenStudio / SAM.Core.OpenStudio namespaces or the OpenStudio SDK. Only the CLI's subject step calls + /// production code. + /// + [TestFixture] + public class S11SourceScanTests + { + public static readonly string[] Forbidden = + { + "OpenStudioSimulationRunner", "Convert.ToSAM", "ToSAM_", "OpenStudioName", "ConvertUnit", "JoulesToKilowattHours", + "JoulesPerIntervalToWatts", "AddResults", "Core.OpenStudio", "OpenStudioSimulationResultSet", "OpenStudioBenchmarkInputs", + "ToBenchmark", "BenchmarkInputs", "using OpenStudio", "OpenStudio.Model", "SqlFile", + }; + + /// The production namespace, but not this library's own or the S09 reader's. + private static readonly Regex ProductionNamespace = new Regex(@"\bSAM\.Analytical\.OpenStudio(?!\.(ResultsReadBack|InputReadBack)\b)(\.|;|\s)", RegexOptions.CultureInvariant); + + private static string LibraryDirectory() + { + DirectoryInfo directory = new DirectoryInfo(TestContext.CurrentContext.TestDirectory); + while (directory != null) + { + string candidate = Path.Combine(directory.FullName, "benchmark", "SAM.Analytical.OpenStudio.ResultsReadBack"); + if (Directory.Exists(candidate)) return candidate; + candidate = Path.Combine(directory.FullName, "SAM.Analytical.OpenStudio.ResultsReadBack"); + if (File.Exists(Path.Combine(candidate, "SAM.Analytical.OpenStudio.ResultsReadBack.csproj"))) return candidate; + directory = directory.Parent; + } + + throw new DirectoryNotFoundException("SAM.Analytical.OpenStudio.ResultsReadBack not found above " + TestContext.CurrentContext.TestDirectory); + } + + /// Every forbidden token found in the code lines of (comment lines are not code). + public static List Scan(string source) + { + List result = new List(); + string[] lines = source.Split('\n'); + for (int i = 0; i < lines.Length; i++) + { + string line = lines[i].Trim(); + if (line.StartsWith("//", StringComparison.Ordinal)) continue; + foreach (string token in Forbidden.Where(x => line.Contains(x))) + { + result.Add("line " + (i + 1) + ": " + token); + } + + if (ProductionNamespace.IsMatch(line)) + { + result.Add("line " + (i + 1) + ": SAM.Analytical.OpenStudio namespace"); + } + } + + return result; + } + + private static IEnumerable ReferenceFiles() + { + return Directory.GetFiles(LibraryDirectory(), "*.cs", SearchOption.AllDirectories).Where(x => !x.Contains(Path.DirectorySeparatorChar + "obj" + Path.DirectorySeparatorChar) && !x.Contains(Path.DirectorySeparatorChar + "bin" + Path.DirectorySeparatorChar)); + } + + [Test] + public void ReferenceFiles_CallNothingOfTheCodeUnderTest() + { + List files = ReferenceFiles().ToList(); + Assert.That(files.Select(Path.GetFileName), Is.SupersetOf(new[] { "SqlResultRead.cs", "EsoResultRead.cs", "RunCalendar.cs", "ResultReference.cs", "ResultCompare.cs", "ResultReport.cs" })); + foreach (string file in files) + { + Assert.That(Scan(File.ReadAllText(file)), Is.Empty, file); + } + } + + [Test] + public void ReferenceProject_ReferencesNoProductionProjectAndNoSdk() + { + System.Xml.Linq.XDocument project = System.Xml.Linq.XDocument.Load(Path.Combine(LibraryDirectory(), "SAM.Analytical.OpenStudio.ResultsReadBack.csproj")); + List includes = project.Descendants().Where(x => x.Name.LocalName == "ProjectReference" || x.Name.LocalName == "PackageReference" || x.Name.LocalName == "Reference").Select(x => (string)x.Attribute("Include")).ToList(); + Assert.That(includes, Is.EquivalentTo(new[] + { + "System.Data.SQLite.Core", + @"..\SAM.Analytical.OpenStudio.InputReadBack\SAM.Analytical.OpenStudio.InputReadBack.csproj", + "SAM.Analytical.Benchmark", "SAM.Core", "SAM.Geometry", "SAM.Analytical", "SAM.Architectural", "SAM.Weather", + }), "the reference library references only SQLite, the S09 reader and SAM core/schema assemblies"); + } + + [Test] + public void ReferenceAssembly_LoadsNoProductionAssembly() + { + string[] references = typeof(ResultReference).Assembly.GetReferencedAssemblies().Select(x => x.Name).ToArray(); + Assert.That(references, Has.None.EqualTo("SAM.Analytical.OpenStudio")); + Assert.That(references, Has.None.EqualTo("SAM.Core.OpenStudio")); + Assert.That(references, Has.None.EqualTo("OpenStudio")); + Assert.That(references, Has.None.EqualTo("benchmark-openstudio")); + } + + [Test] + public void Scanner_FindsAPlantedForbiddenCall() + { + string path = ReferenceFiles().Single(x => Path.GetFileName(x) == "ResultReference.cs"); + string planted = File.ReadAllText(path).Replace("public static ReferenceDocument Build(", "public static object Cheat(string p) => SAM.Analytical.OpenStudio.Modify.AddResults(null, p, out _);\n public static ReferenceDocument Build("); + List found = Scan(planted); + Assert.That(found, Has.Some.Contains("AddResults")); + Assert.That(found, Has.Some.Contains("namespace")); + Assert.That(Scan("string key = Core.OpenStudio.Query.OpenStudioName(\"IdealLoads\", name, guid);"), Has.Count.EqualTo(2)); + Assert.That(Scan("// a comment may name OpenStudioSimulationRunner"), Is.Empty); + } + } +} diff --git a/benchmark/SAM.Analytical.OpenStudio.InputReadBack.Tests/S11Support.cs b/benchmark/SAM.Analytical.OpenStudio.InputReadBack.Tests/S11Support.cs new file mode 100644 index 0000000..80b0870 --- /dev/null +++ b/benchmark/SAM.Analytical.OpenStudio.InputReadBack.Tests/S11Support.cs @@ -0,0 +1,161 @@ +// SPDX-License-Identifier: LGPL-3.0-or-later +// Copyright (c) 2020–2026 Michal Dengusiak & Jakub Ziolkowski and contributors + +using NUnit.Framework; +using SAM.Analytical.Benchmark; +using SAM.Analytical.OpenStudio.Benchmark; +using SAM.Analytical.OpenStudio.ResultsReadBack; +using SAM.Analytical.OpenStudio.ResultsReadBack.Cli; +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; + +namespace SAM.Analytical.OpenStudio.InputReadBack.Tests +{ + /// One prepared synthetic S11 run: its directory, model and the objects the steps produced. + internal sealed class S11Case + { + public string Run; + public string ModelPath; + public AnalyticalModel Model; + public SyntheticRun Synthetic; + public OpenStudioBenchmarkInputs Inputs; + public ReferenceDocument Reference; + public SubjectDocument Subject; + public CompareDocument Compare; + + public ResultRow Row(string id) => Compare.Rows.SingleOrDefault(x => x.Id == id) ?? throw new AssertionException("no row " + id + "; rows: " + string.Join(", ", Compare.Rows.Select(x => x.Id).Take(80))); + + public string Space(int index) => "space:" + Synthetic.Zones[index].SpaceGuid.ToString("N"); + + public string Summary() => Compare.Verdict + ": " + string.Join(" | ", Compare.Rows.Where(x => x.Status != "PASS" && x.Status != "INFO").Take(12).Select(x => x.Id + " " + x.Status + " " + x.Reason)); + } + + internal static class S11Support + { + public static string Model(string fixtureId) => TestSupport.Fixture(fixtureId); + + /// Space A's (FixA) GUID in s08-fix-two. + public const string SpaceAGuid = "5080c000-0000-0000-0000-00000000c0a1"; + + /// Space B's (FixB) GUID in s08-fix-two; it shares its first 8 hex digits (5080c000) with space A. + public const string SpaceBGuid = "5080c000-0000-0000-0000-00000000c0b1"; + + /// + /// The signed-off s08-fix-two fixture, unchanged: FixA and FixB share the GUID prefix 5080c000. The importer + /// associates their zone results by name identity proven unique by the SQL Zones inventory (F-S11-1, #35), so no + /// re-GUIDed variant is needed. + /// + public static string Two() => Model(TestSupport.Two); + + /// A text variant of a signed-off fixture, written once to a stable path (so its export is cached). + public static string Variant(string fixtureId, string name, params (string From, string To)[] replacements) + { + string text = File.ReadAllText(TestSupport.Fixture(fixtureId)); + foreach ((string from, string to) in replacements) + { + Assert.That(text, Does.Contain(from), name + ": the fixture must contain " + from); + text = text.Replace(from, to); + } + + string path = Path.Combine(TestContext.CurrentContext.WorkDirectory, "s11-models", name + ".json"); + Directory.CreateDirectory(Path.GetDirectoryName(path)); + if (!File.Exists(path) || File.ReadAllText(path) != text) + { + File.WriteAllText(path, text); + } + + return path; + } + + /// + /// Writes the synthetic files, produces b1a.json and sam.json through the production extraction and the benchmark + /// producer, then runs read, subject and compare IN PROCESS (the separate-process path has its own tests). + /// may edit the run (b1a.json, sam.json) before the reference and subject read it; + /// may edit the subject, as a mutated subject would be. + /// + public static S11Case Evaluate(SyntheticRun synthetic, string modelPath, string label, DesignDaySource designDaySource = DesignDaySource.None, Action mutateInputs = null, Action afterProduce = null, Action beforeCompare = null) + { + S11Case result = Prepare(synthetic, modelPath, label, designDaySource, mutateInputs); + afterProduce?.Invoke(result); + return Judge(result, beforeCompare); + } + + public static S11Case Prepare(SyntheticRun synthetic, string modelPath, string label, DesignDaySource designDaySource = DesignDaySource.None, Action mutateInputs = null) + { + S11Case result = new S11Case { Run = NewRun(label), ModelPath = modelPath, Model = Core.Convert.ToSAM(modelPath).Single(), Synthetic = synthetic }; + synthetic.Write(result.Run); + result.Inputs = SyntheticProducer.Produce(result.Model, result.Run, designDaySource, mutateInputs); + return result; + } + + public static S11Case Judge(S11Case result, Action beforeCompare = null) + { + RunLayout layout = RunLayout.Resolve(result.Run, out List problems); + Assert.That(problems, Is.Empty); + result.Reference = ResultReference.Build(result.Model, result.ModelPath, layout.Idf, layout.Sql, layout.Eso, "s11-test"); + result.Subject = SubjectStep.Capture(result.Model, result.ModelPath, layout); + beforeCompare?.Invoke(result); + BenchmarkDocument b1a = BenchmarkSerializer.Deserialize(File.ReadAllText(layout.B1a)); + result.Compare = ResultCompare.Run(result.Model, result.Reference, result.Subject, b1a, SamResults.Read(layout.SamResults)); + return result; + } + + public static string NewRun(string label) + { + string path = Path.Combine(TestContext.CurrentContext.WorkDirectory, "s11-runs", label + "-" + Guid.NewGuid().ToString("N").Substring(0, 6)); + Directory.CreateDirectory(path); + return path; + } + + /// Runs os-results-readback in its own process; returns the exit code with its stdout and stderr in output. + public static int Cli(out string output, params string[] arguments) + { + string exe = Path.Combine(TestContext.CurrentContext.TestDirectory, "os-results-readback.exe"); + Assert.That(File.Exists(exe), Is.True, exe); + ProcessStartInfo info = new ProcessStartInfo(exe) + { + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true, + WorkingDirectory = TestContext.CurrentContext.TestDirectory + }; + + foreach (string argument in arguments) + { + info.ArgumentList.Add(argument); + } + + using (Process process = Process.Start(info)) + { + System.Threading.Tasks.Task error = process.StandardError.ReadToEndAsync(); + output = process.StandardOutput.ReadToEnd(); + process.WaitForExit(); + output = "[stdout]" + Environment.NewLine + output + "[stderr]" + Environment.NewLine + error.Result; + return process.ExitCode; + } + } + + /// The reference SENSIBLE value of a space quantity, for substitution negatives. + public static QuantityReference Sensible(ReferenceDocument reference, Guid space, LoadKind kind) + { + return reference.Space(space).Quantities[S11Names.Sensible(kind)]; + } + + public static IEnumerable Annual(OpenStudioBenchmarkInputs inputs, Guid space, LoadKind kind) + { + return inputs.SpaceResults.Where(x => x.Reference == space.ToString("N") && x.TryGetValue(Analytical.SpaceSimulationResultParameter.LoadType, out string loadType) && loadType == kind.ToString()); + } + + /// Rewrites b1a.json through the schema types (a test-side edit of the produced document). + public static void EditB1a(S11Case run, Action edit) + { + string path = Path.Combine(run.Run, RunLayout.B1aFile); + BenchmarkDocument document = BenchmarkSerializer.Deserialize(File.ReadAllText(path)); + edit(document); + BenchmarkSerializer.Write(path, document); + } + } +} diff --git a/benchmark/SAM.Analytical.OpenStudio.InputReadBack.Tests/S11Synthetic.cs b/benchmark/SAM.Analytical.OpenStudio.InputReadBack.Tests/S11Synthetic.cs new file mode 100644 index 0000000..f1f7053 --- /dev/null +++ b/benchmark/SAM.Analytical.OpenStudio.InputReadBack.Tests/S11Synthetic.cs @@ -0,0 +1,531 @@ +// SPDX-License-Identifier: LGPL-3.0-or-later +// Copyright (c) 2020–2026 Michal Dengusiak & Jakub Ziolkowski and contributors + +using NUnit.Framework; +using SAM.Analytical.Benchmark; +using SAM.Analytical.OpenStudio.Benchmark; +using SAM.Analytical.OpenStudio.ResultsReadBack; +using SAM.Analytical.Tas.DirectT3D.Validation.ReadBack; +using System; +using System.Collections.Generic; +using System.Data.SQLite; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Text; + +namespace SAM.Analytical.OpenStudio.InputReadBack.Tests +{ + /// One synthetic hourly value as a file will stamp it. + internal sealed class SyntheticPoint + { + public int Month; + public int Day; + public int Hour; + public double Value; + public string Environment; + public bool Warmup; + public int Dst; + + public SyntheticPoint Copy() => (SyntheticPoint)MemberwiseClone(); + } + + /// One identified zone of the exported fixture IDF: its SAM space, native Zone and Ideal Loads names. + internal sealed class SyntheticZone + { + public Guid SpaceGuid; + public string Zone; + public string IdealLoads; + } + + /// + /// A synthetic EnergyPlus run for S11 offline tests: the S09 export of a fixture (OpenStudio SDK translation only, no + /// EnergyPlus) gives the in.idf; deterministic hourly series keyed by that IDF's native names are written as a + /// real-schema eplusout.sql and an eplusout.eso. Each file holds its own copy of every series, so a test can break one + /// file, both, or one series. Values: heating in the cold months and cooling in summer, both with a latent part of at + /// least 10 % of Total, and a Total peak at a different hour from the Sensible peak (F3 discrimination). + /// + internal sealed class SyntheticRun + { + private static readonly int[] DaysInMonth = { 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 }; + private static readonly CultureInfo ci = CultureInfo.InvariantCulture; + private static readonly Dictionary exports = new Dictionary(StringComparer.Ordinal); + + public string Idf; + public string RunPeriodName; + public List Zones = new List(); + public Dictionary> Sql = new Dictionary>(StringComparer.Ordinal); + public Dictionary> Eso = new Dictionary>(StringComparer.Ordinal); + public List> SqlEnvironments = new List>(); + public List EsoEnvironments = new List(); + public string SqlVersion = "EnergyPlus, Version 25.1.0-1c11a3d85f, YMD=2026.10.24 08:00"; + public string EsoVersion = "Program Version,EnergyPlus, Version 25.1.0-1c11a3d85f, YMD=2026.10.24 08:00"; + public bool Meters = true; + public List ZoneSizes = new List(); + + /// Extra SQL edits after the file is written (e.g. a second dictionary entry). + public Action SqlEdit; + + /// Extra ESO text edits after the file is written (e.g. a repeated value line). + public Func, List> EsoEdit; + + public static string Key(string variable, string key) => S11Names.SeriesKey(variable, key); + + /// The exported in.idf of a model (cached per model file hash); the export runs os-input-readback in its own process. + public static string ExportIdf(string modelPath, string label) + { + string hash = Hashing.Sha256(modelPath); + lock (exports) + { + if (!exports.TryGetValue(hash, out string idf)) + { + string run = TestSupport.NewDirectory("s11-export-" + label); + int exit = TestSupport.Cli(out string output, "export", "--model", modelPath, "--epw", TestSupport.Budapest(), "--out", run, "--label", label); + Assert.That(exit, Is.EqualTo(0), output); + idf = File.ReadAllText(Path.Combine(run, "in.idf")); + exports[hash] = idf; + } + + return idf; + } + } + + /// A run for the model at , every series complete and consistent in both files. + public static SyntheticRun For(string modelPath, string label, bool fractionalUnmet = false) + { + AnalyticalModel model = Core.Convert.ToSAM(modelPath).Single(); + SyntheticRun result = new SyntheticRun { Idf = ExportIdf(modelPath, label) }; + List idf = IdfText.Parse(result.Idf).Where(x => NativeReader.IdfTypes.Contains(x.Type, StringComparer.OrdinalIgnoreCase)).ToList(); + result.RunPeriodName = idf.Single(x => x.Type == "RunPeriod").Name.ToUpperInvariant(); + NativeModel native = new NativeModel(new NativeSnapshot { Idf = idf }); + foreach (NativeZoneIdentityEntry entry in NativeZoneIdentity.Resolve(native, ReadBackExpected.Build(model, null, label))) + { + Assert.That(entry.Problem, Is.Null, "the exported fixture must have a clean identity"); + result.Zones.Add(new SyntheticZone { SpaceGuid = entry.SpaceGuid, Zone = entry.Zone.ToUpperInvariant(), IdealLoads = entry.IdealLoads.ToUpperInvariant() }); + } + + result.SqlEnvironments.Add(new KeyValuePair(result.RunPeriodName, 3)); + result.EsoEnvironments.Add(result.RunPeriodName); + for (int z = 0; z < result.Zones.Count; z++) + { + SyntheticZone zone = result.Zones[z]; + foreach (string variable in S11Names.IdealLoadsVariables.Concat(S11Names.ZoneVariables)) + { + string key = S11Names.ZoneVariables.Contains(variable) ? zone.Zone : zone.IdealLoads; + List points = new List(); + int hourOfYear = 0; + for (int month = 1; month <= 12; month++) + { + for (int day = 1; day <= DaysInMonth[month - 1]; day++) + { + for (int hour = 1; hour <= 24; hour++) + { + points.Add(new SyntheticPoint { Month = month, Day = day, Hour = hour, Value = Value(variable, z, month, day, hour, hourOfYear, fractionalUnmet), Environment = result.RunPeriodName }); + hourOfYear++; + } + } + } + + result.Sql[Key(variable, key)] = points; + result.Eso[Key(variable, key)] = points.Select(x => x.Copy()).ToList(); + } + } + + return result; + } + + /// + /// Deterministic, non-round values [J per hour or hr]. Cooling: May-September, EnergyPlus hours 10-19 (1530 hours). + /// Sensible cooling peaks on 15 July h15; Total peaks on 10 August h14 where the latent is large. Heating likewise in + /// November-March with a latent (humidification) part. Zone index shifts the magnitudes. Not-met time is whole hours + /// (29 heating, 20 cooling, equal in every zone) unless : EnergyPlus reports + /// timestep fractions, here binary-exact quarter hours that differ per zone so the annual sums are exact and a swap + /// between zones is visible. Zone 0: heating 29 x 0.5 = 14.5 h, cooling 19 x 0.75 + 0.5 = 14.75 h; every other + /// zone: heating 29 x 0.25 = 7.25 h, cooling 19 x 0.25 + 0.5 = 5.25 h (0.5 on 28 September). + /// + public static double Value(string variable, int zone, int month, int day, int hour, int hourOfYear, bool fractionalUnmet = false) + { + double scale = 1 + 0.37 * zone; + bool cooling = month >= 5 && month <= 9 && hour >= 10 && hour <= 19; + bool heating = (month >= 11 || month <= 3) && (hour <= 9 || hour >= 18); + double wave = 0.5 + 0.5 * Math.Sin(hourOfYear * 0.0137 + zone); + double sensibleCooling = cooling ? scale * (1_000_000.123 + 400_000.0 * wave) : 0; + double sensibleHeating = heating ? scale * (1_500_000.321 + 500_000.0 * wave) : 0; + if (month == 7 && day == 15 && hour == 15) sensibleCooling = scale * 2_000_000.5; + if (month == 1 && day == 20 && hour == 6) sensibleHeating = scale * 2_600_000.25; + double latentCooling = sensibleCooling * 0.15; + double latentHeating = sensibleHeating * 0.12; + if (month == 8 && day == 10 && hour == 14) { sensibleCooling = scale * 1_950_000.75; latentCooling = scale * 600_000.0; } + if (month == 2 && day == 3 && hour == 7) { sensibleHeating = scale * 2_550_000.5; latentHeating = scale * 700_000.0; } + + switch (variable) + { + case S11Names.SensibleCooling: return sensibleCooling; + case S11Names.TotalCooling: return sensibleCooling + latentCooling; + case S11Names.SensibleHeating: return sensibleHeating; + case S11Names.TotalHeating: return sensibleHeating + latentHeating; + case S11Names.UnmetHeating: return heating && hour == 7 && day % 5 == 0 ? (fractionalUnmet ? (zone == 0 ? 0.5 : 0.25) : 1.0) : 0; + case S11Names.UnmetCooling: return cooling && hour == 16 && day % 7 == 0 ? (fractionalUnmet ? (month == 9 && day == 28 ? 0.5 : zone == 0 ? 0.75 : 0.25) : 1.0) : 0; + default: throw new ArgumentException(variable); + } + } + + /// Applies to every series of the chosen file(s). + public void EditSeries(bool sql, bool eso, Action> edit, Func which = null) + { + foreach (Dictionary> file in new[] { sql ? Sql : null, eso ? Eso : null }.Where(x => x != null)) + { + foreach (KeyValuePair> series in file.Where(x => which == null || which(x.Key)).ToList()) + { + edit(series.Value); + } + } + } + + // ---- writing --------------------------------------------------------------------------------------------------- + + /// Writes run/{in.idf, eplusout.sql, eplusout.eso} below . + public void Write(string runDirectory) + { + string folder = Path.Combine(runDirectory, "work", "run"); + Directory.CreateDirectory(folder); + File.WriteAllText(Path.Combine(folder, "in.idf"), Idf); + WriteSql(Path.Combine(folder, "eplusout.sql")); + WriteEso(Path.Combine(folder, "eplusout.eso")); + } + + private void WriteSql(string path) + { + using (SQLiteConnection connection = new SQLiteConnection("Data Source=" + path)) + { + connection.Open(); + using (SQLiteTransaction transaction = connection.BeginTransaction()) + using (SQLiteCommand command = connection.CreateCommand()) + { + void Exec(string sql) + { + command.CommandText = sql; + command.ExecuteNonQuery(); + } + + Exec("CREATE TABLE Simulations (SimulationIndex INTEGER PRIMARY KEY, EnergyPlusVersion TEXT, TimeStamp TEXT, NumTimestepsPerHour INTEGER, Completed BOOL, CompletedSuccessfully BOOL)"); + Exec("CREATE TABLE EnvironmentPeriods (EnvironmentPeriodIndex INTEGER PRIMARY KEY, SimulationIndex INTEGER, EnvironmentName TEXT, EnvironmentType INTEGER)"); + Exec("CREATE TABLE Time (TimeIndex INTEGER PRIMARY KEY, Year INTEGER, Month INTEGER, Day INTEGER, Hour INTEGER, Minute INTEGER, Dst INTEGER, Interval INTEGER, IntervalType INTEGER, SimulationDays INTEGER, DayType TEXT, EnvironmentPeriodIndex INTEGER, WarmupFlag INTEGER)"); + Exec("CREATE TABLE ReportDataDictionary (ReportDataDictionaryIndex INTEGER PRIMARY KEY, IsMeter INTEGER, Type TEXT, IndexGroup TEXT, TimestepType TEXT, KeyValue TEXT, Name TEXT, ReportingFrequency TEXT, ScheduleName TEXT, Units TEXT)"); + Exec("CREATE TABLE ReportData (ReportDataIndex INTEGER PRIMARY KEY, TimeIndex INTEGER, ReportDataDictionaryIndex INTEGER, Value REAL)"); + Exec("CREATE TABLE Zones (ZoneIndex INTEGER PRIMARY KEY, ZoneName TEXT, Volume REAL, FloorArea REAL)"); + Exec("CREATE TABLE Surfaces (SurfaceIndex INTEGER PRIMARY KEY, SurfaceName TEXT, ConstructionIndex INTEGER, ClassName TEXT, Area REAL, GrossArea REAL, Perimeter REAL, Azimuth REAL, Height REAL, Reveal REAL, Shape INTEGER, Sides INTEGER, Tilt REAL, Width REAL, HeatTransferSurf INTEGER, BaseSurfaceIndex INTEGER, ZoneIndex INTEGER, ExtBoundCond INTEGER, ExtSolar INTEGER, ExtWind INTEGER)"); + Exec("CREATE TABLE ZoneSizes (ZoneSizesIndex INTEGER PRIMARY KEY, ZoneName TEXT, LoadType TEXT, CalcDesLoad REAL, UserDesLoad REAL, CalcDesFlow REAL, UserDesFlow REAL, DesDayName TEXT, PeakHrMin TEXT, PeakTemp REAL, PeakHumRat REAL, CalcOutsideAirFlow REAL, DOASHeatAddRate REAL)"); + Exec("CREATE TABLE NominalLighting (NominalLightingIndex INTEGER PRIMARY KEY, ObjectName TEXT, ZoneIndex INTEGER, ScheduleIndex INTEGER, DesignLevel REAL)"); + Exec("CREATE TABLE NominalInfiltration (NominalInfiltrationIndex INTEGER PRIMARY KEY, ObjectName TEXT, ZoneIndex INTEGER, ScheduleIndex INTEGER, DesignLevel REAL)"); + Exec("CREATE TABLE NominalElectricEquipment (NominalElectricEquipmentIndex INTEGER PRIMARY KEY, ObjectName TEXT, ZoneIndex INTEGER, ScheduleIndex INTEGER, DesignLevel REAL)"); + + Parameter(command, "INSERT INTO Simulations VALUES (1, @v, 'YMD=2026.10.24 08:00', 6, 1, 1)", "@v", SqlVersion); + Dictionary environments = new Dictionary(StringComparer.OrdinalIgnoreCase); + for (int i = 0; i < SqlEnvironments.Count; i++) + { + environments[SqlEnvironments[i].Key] = i + 1; + Parameter(command, "INSERT INTO EnvironmentPeriods VALUES (" + (i + 1) + ", 1, @n, " + SqlEnvironments[i].Value + ")", "@n", SqlEnvironments[i].Key); + } + + for (int i = 0; i < Zones.Count; i++) + { + Parameter(command, "INSERT INTO Zones VALUES (" + (i + 1) + ", @n, 60, 20)", "@n", Zones[i].Zone); + } + + foreach (ZoneSizeRow row in ZoneSizes) + { + Parameter(command, "INSERT INTO ZoneSizes (ZoneName, LoadType, CalcDesLoad, UserDesLoad, DesDayName, PeakHrMin) VALUES (@n, '" + row.LoadType + "', " + row.CalcDesLoad.ToString("R", ci) + ", " + (row.CalcDesLoad * 1.25).ToString("R", ci) + ", 'DESIGN DAY', ' 1/21 06:00')", "@n", row.ZoneName); + } + + // Time rows: one per distinct stamp and occurrence (a repeated stamp gets its own Time row, as a merged + // environment would), in calendar order per environment. + Dictionary times = new Dictionary(StringComparer.Ordinal); + long timeIndex = 0; + long Time(SyntheticPoint point, int occurrence) + { + string stamp = string.Join("|", point.Environment, point.Month, point.Day, point.Hour, point.Warmup, point.Dst, occurrence); + if (!times.TryGetValue(stamp, out long index)) + { + index = ++timeIndex; + times[stamp] = index; + int environment = environments.TryGetValue(point.Environment, out int e) ? e : 99; + Exec(string.Format(ci, "INSERT INTO Time VALUES ({0}, 2006, {1}, {2}, {3}, 0, {4}, 60, 1, 1, 'Monday', {5}, {6})", index, point.Month, point.Day, point.Hour, point.Dst, environment, point.Warmup ? 1 : 0)); + } + + return index; + } + + long dictionary = 0; + long data = 0; + Dictionary meterTotals = new Dictionary(StringComparer.Ordinal); + foreach (KeyValuePair> series in Sql) + { + string[] parts = series.Key.Split('|'); + bool zoneVariable = S11Names.ZoneVariables.Contains(parts[0]); + dictionary++; + Parameters(command, "INSERT INTO ReportDataDictionary VALUES (" + dictionary + ", 0, 'Sum', @g, @t, @k, @n, 'Hourly', '', @u)", ("@g", zoneVariable ? "Zone" : "System"), ("@t", zoneVariable ? "Zone" : "HVAC System"), ("@k", parts[1]), ("@n", parts[0]), ("@u", S11Names.UnitOf(parts[0]))); + Dictionary occurrences = new Dictionary(StringComparer.Ordinal); + foreach (SyntheticPoint point in series.Value) + { + string stamp = string.Join("|", point.Environment, point.Month, point.Day, point.Hour, point.Warmup, point.Dst); + occurrences.TryGetValue(stamp, out int occurrence); + occurrences[stamp] = occurrence + 1; + long time = Time(point, occurrence); + data++; + Exec(string.Format(ci, "INSERT INTO ReportData VALUES ({0}, {1}, {2}, {3})", data, time, dictionary, point.Value.ToString("R", ci))); + } + } + + if (Meters) + { + foreach (LoadKind kind in new[] { LoadKind.Heating, LoadKind.Cooling }) + { + dictionary++; + Parameters(command, "INSERT INTO ReportDataDictionary VALUES (" + dictionary + ", 1, 'Sum', 'Facility', 'Zone', '', @n, 'Hourly', '', 'J')", ("@n", S11Names.Meter(kind))); + foreach (SyntheticPoint point in MeterSeries(kind)) + { + data++; + Exec(string.Format(ci, "INSERT INTO ReportData VALUES ({0}, {1}, {2}, {3})", data, Time(point, 0), dictionary, point.Value.ToString("R", ci))); + } + } + } + + transaction.Commit(); + } + + SqlEdit?.Invoke(connection); + } + + SQLiteConnection.ClearAllPools(); + } + + /// + /// A facility meter: per stamp, the sum of the zones' Total energy in the SQL series of the RunPeriod (as EnergyPlus + /// meters the Ideal Loads), so a test that edits the series edits the meter with them. + /// + private List MeterSeries(LoadKind kind) + { + Dictionary sums = new Dictionary(StringComparer.Ordinal); + List result = new List(); + foreach (SyntheticZone zone in Zones) + { + if (!Sql.TryGetValue(Key(S11Names.Total(kind), zone.IdealLoads), out List series)) + { + continue; + } + + HashSet seen = new HashSet(StringComparer.Ordinal); + foreach (SyntheticPoint point in series.Where(x => x.Environment == RunPeriodName && !x.Warmup)) + { + string stamp = point.Month + "|" + point.Day + "|" + point.Hour; + if (!seen.Add(stamp)) + { + continue; + } + + if (!sums.TryGetValue(stamp, out SyntheticPoint sum)) + { + sum = new SyntheticPoint { Month = point.Month, Day = point.Day, Hour = point.Hour, Environment = RunPeriodName }; + sums[stamp] = sum; + result.Add(sum); + } + + sum.Value += point.Value; + } + } + + return result; + } + + private void WriteEso(string path) + { + List lines = new List { EsoVersion }; + lines.Add("1,5,Environment Title[],Latitude[deg],Longitude[deg],Time Zone[],Elevation[m]"); + lines.Add("2,8,Day of Simulation[],Month[],Day of Month[],DST Indicator[1=yes 0=no],Hour[],StartMinute[],EndMinute[],DayType"); + lines.Add("3,5,Cumulative Day of Simulation[],Month[],Day of Month[],DST Indicator[1=yes 0=no],DayType ! When Daily Report Variables Requested"); + lines.Add("4,2,Cumulative Days of Simulation[],Month[] ! When Monthly Report Variables Requested"); + lines.Add("5,1,Cumulative Days of Simulation[] ! When Run Period Report Variables Requested"); + lines.Add("6,1,Calendar Year of Simulation[] ! When Annual Report Variables Requested"); + Dictionary ids = new Dictionary(StringComparer.Ordinal); + int id = 7; + foreach (string series in Eso.Keys) + { + string[] parts = series.Split('|'); + ids[series] = id; + lines.Add(id + ",1," + parts[1] + "," + parts[0] + " [" + S11Names.UnitOf(parts[0]) + "] !Hourly"); + id++; + } + + Dictionary meterIds = new Dictionary(StringComparer.Ordinal); + if (Meters) + { + foreach (LoadKind kind in new[] { LoadKind.Heating, LoadKind.Cooling }) + { + meterIds[S11Names.Meter(kind)] = id; + lines.Add(id + ",1," + S11Names.Meter(kind) + " [J] !Hourly"); + id++; + } + } + + lines.Add("End of Data Dictionary"); + foreach (string environment in EsoEnvironments) + { + lines.Add("1," + environment + ", 47.52, 19.03, 1.00, 129.00"); + + // Stamps in first-appearance order over all series of this environment, each occurrence its own block. + List order = new List(); + Dictionary>> blocks = new Dictionary>>(StringComparer.Ordinal); + foreach (KeyValuePair> series in Eso) + { + Dictionary occurrences = new Dictionary(StringComparer.Ordinal); + foreach (SyntheticPoint point in series.Value.Where(x => string.Equals(x.Environment, environment, StringComparison.OrdinalIgnoreCase))) + { + string stamp = string.Join("|", point.Month, point.Day, point.Hour, point.Dst); + occurrences.TryGetValue(stamp, out int occurrence); + occurrences[stamp] = occurrence + 1; + string block = stamp + "|" + occurrence; + if (!blocks.TryGetValue(block, out List> list)) + { + list = new List>(); + blocks[block] = list; + order.Add(block); + } + + list.Add(new KeyValuePair(ids[series.Key], point)); + } + } + + Dictionary meters = new Dictionary(StringComparer.Ordinal); + if (Meters && environment == RunPeriodName) + { + foreach (LoadKind kind in new[] { LoadKind.Heating, LoadKind.Cooling }) + { + foreach (SyntheticPoint point in MeterSeries(kind)) + { + meters[S11Names.Meter(kind) + "|" + point.Month + "|" + point.Day + "|" + point.Hour] = point.Value; + } + } + } + + int dayOfSimulation = 0; + string lastDay = null; + foreach (string block in order) + { + SyntheticPoint first = blocks[block][0].Value; + string dayKey = first.Month + "/" + first.Day; + if (dayKey != lastDay) + { + dayOfSimulation++; + lastDay = dayKey; + } + + lines.Add(string.Format(ci, "2,{0},{1,2},{2,2},{3,2},{4,2}, 0.00,60.00,Monday", dayOfSimulation, first.Month, first.Day, first.Dst, first.Hour)); + foreach (KeyValuePair value in blocks[block]) + { + lines.Add(value.Key.ToString(ci) + "," + value.Value.Value.ToString("R", ci)); + } + + if (block.EndsWith("|0", StringComparison.Ordinal)) + { + foreach (KeyValuePair meter in meterIds) + { + if (meters.TryGetValue(meter.Key + "|" + first.Month + "|" + first.Day + "|" + first.Hour, out double value)) + { + lines.Add(meter.Value.ToString(ci) + "," + value.ToString("R", ci)); + } + } + } + } + } + + lines.Add("End of Data"); + lines.Add(" Number of Records Written= " + lines.Count); + if (EsoEdit != null) + { + lines = EsoEdit(lines); + } + + File.WriteAllText(path, string.Join("\n", lines) + "\n", new UTF8Encoding(false)); + } + + private static void Parameter(SQLiteCommand command, string sql, string name, object value) + { + Parameters(command, sql, (name, value)); + } + + private static void Parameters(SQLiteCommand command, string sql, params (string Name, object Value)[] values) + { + command.CommandText = sql; + command.Parameters.Clear(); + foreach ((string name, object value) in values) + { + command.Parameters.AddWithValue(name, value); + } + + command.ExecuteNonQuery(); + command.Parameters.Clear(); + } + } + + /// + /// The producer side of a synthetic run, offline: the production C5 extraction of the SQL (OpenStudioSimulationRunner, + /// reached by reflection because it is internal; the route's own Program would call it after the simulation), then the + /// benchmark producer's BenchmarkInputs/ToBenchmark, writing b1a.json and sam.json exactly as benchmark-openstudio does. + /// + internal static class SyntheticProducer + { + public static OpenStudioSimulationResultSet ResultSet(string sqlPath) + { + Type runner = typeof(OpenStudioSimulationResultSet).Assembly.GetType("SAM.Analytical.OpenStudio.OpenStudioSimulationRunner", true); + MethodInfo extractLoads = runner.GetMethod("ExtractLoads", BindingFlags.NonPublic | BindingFlags.Static); + MethodInfo extractResultSet = runner.GetMethod("ExtractResultSet", BindingFlags.NonPublic | BindingFlags.Static); + object[] arguments = { sqlPath, null }; + object summary = extractLoads.Invoke(null, arguments); + Assert.That(summary, Is.Not.Null, "production ExtractLoads failed: " + arguments[1]); + return (OpenStudioSimulationResultSet)extractResultSet.Invoke(null, new object[] { sqlPath, summary, false, 12.5, 0, 0, 0 }); + } + + public static OpenStudioBenchmarkContext Context(AnalyticalModel model, OpenStudioSimulationResultSet resultSet, DesignDaySource designDaySource = DesignDaySource.None) + { + OpenStudioBenchmarkContext context = new OpenStudioBenchmarkContext + { + SourceModelName = model.Name ?? "model", + SourceModelGuid = model.Guid.ToString("N"), + SourceFileHash = BenchmarkHash.ComputeSha256(new byte[] { 1, 2, 3 }), + CanonicalModelHash = BenchmarkCanonicalJson.ComputeSha256("{\"s11\":\"synthetic\"}"), + CanonicalizationVersion = BenchmarkCanonicalization.CurrentVersion, + SamCommit = "0123456789abcdef", + RunnerCommit = "fedcba9876543210", + EngineName = "EnergyPlus", + EngineVersion = "25.1.0", + SdkVersion = "3.10.0", + WeatherIdentity = "budapest", + WeatherHash = BenchmarkHash.ComputeSha256(new byte[] { 4, 5, 6 }), + DesignDaySource = designDaySource, + RunTimestampUtc = new DateTimeOffset(2026, 10, 24, 8, 0, 0, TimeSpan.Zero), + DurationSeconds = 12.5, + State = RunState.Success, + ResultSet = resultSet, + }; + + context.SpaceDesignLoadResults = resultSet.ToSAM_SpaceDesignLoadResults(model.AdjacencyCluster.GetSpaces(), out List _); + return context; + } + + /// Writes b1a.json and sam.json for the run in ; returns the producer objects. + public static OpenStudioBenchmarkInputs Produce(AnalyticalModel model, string runDirectory, DesignDaySource designDaySource = DesignDaySource.None, Action mutate = null) + { + OpenStudioSimulationResultSet resultSet = ResultSet(Path.Combine(runDirectory, "work", "run", "eplusout.sql")); + OpenStudioBenchmarkContext context = Context(model, resultSet, designDaySource); + OpenStudioBenchmarkInputs inputs = model.BenchmarkInputs(context); + mutate?.Invoke(inputs); + BenchmarkDocument document = model.ToBenchmark(context, inputs); + Assert.That(BenchmarkValidator.Validate(document).IsValid, Is.True, "the produced B1a must validate"); + BenchmarkSerializer.Write(Path.Combine(runDirectory, RunLayout.B1aFile), document); + inputs.Write(Path.Combine(runDirectory, RunLayout.SamResultsFile)); + return inputs; + } + } +} diff --git a/benchmark/SAM.Analytical.OpenStudio.InputReadBack.Tests/SAM.Analytical.OpenStudio.InputReadBack.Tests.csproj b/benchmark/SAM.Analytical.OpenStudio.InputReadBack.Tests/SAM.Analytical.OpenStudio.InputReadBack.Tests.csproj index 9080740..96167b2 100644 --- a/benchmark/SAM.Analytical.OpenStudio.InputReadBack.Tests/SAM.Analytical.OpenStudio.InputReadBack.Tests.csproj +++ b/benchmark/SAM.Analytical.OpenStudio.InputReadBack.Tests/SAM.Analytical.OpenStudio.InputReadBack.Tests.csproj @@ -13,7 +13,7 @@ @@ -25,15 +25,27 @@ + + + + + + + + + + + ..\..\..\SAM_Validation\build\SAM.Analytical.Benchmark.dll + ..\..\..\SAM\build\SAM.Core.dll diff --git a/benchmark/SAM.Analytical.OpenStudio.InputReadBack/Compare/NativeZoneIdentity.cs b/benchmark/SAM.Analytical.OpenStudio.InputReadBack/Compare/NativeZoneIdentity.cs new file mode 100644 index 0000000..0e0645d --- /dev/null +++ b/benchmark/SAM.Analytical.OpenStudio.InputReadBack/Compare/NativeZoneIdentity.cs @@ -0,0 +1,215 @@ +// SPDX-License-Identifier: LGPL-3.0-or-later +// Copyright (c) 2020–2026 Michal Dengusiak & Jakub Ziolkowski and contributors + +using SAM.Analytical.InputEquivalence; +using SAM.Analytical.Tas.DirectT3D.Validation.ReadBack; +using SAM.Geometry.Spatial; +using System; +using System.Collections.Generic; +using System.Linq; +using EQ = SAM.Analytical.InputEquivalence.Query; + +namespace SAM.Analytical.OpenStudio.InputReadBack +{ + /// One native surface of a space with its vertices in building coordinates, or the reason they are unreadable. + public sealed class NativeSurface + { + public IdfObject Object; + public List Vertices; + public string Problem; + } + + /// + /// The native space whose surfaces are, one to one, a SAM space's panel polygons. is null when + /// none matches or when more than one does (); and then + /// still hold the first bijective candidate, exactly as the S09 comparison has always kept them. + /// + public sealed class NativeSpaceMatch + { + public string NativeSpace; + public GeometryMatchResult Match; + public List Surfaces; + public bool Ambiguous; + } + + /// The native identity of one SAM space: its native Space, the Zone that holds it and the Zone's Ideal Loads. + public sealed class NativeZoneIdentityEntry + { + public Guid SpaceGuid { get; set; } + public string SpaceName { get; set; } + public string NativeSpace { get; set; } + public int SurfacePairs { get; set; } + + /// The IDF Zone name: the report key of zone variables (setpoint not-met time). + public string Zone { get; set; } + + /// The ZoneHVAC:IdealLoadsAirSystem name: the report key of the Ideal Loads variables. + public string IdealLoads { get; set; } + + public int IdealLoadsCount { get; set; } + + /// True when the writer's GUID suffix agrees with the geometry match (it may only contradict it, never make it). + public bool NameCarrierAgrees { get; set; } + + /// Null when the identity is established; otherwise why it is not. + public string Problem { get; set; } + } + + /// + /// The S09 native identity, shared by the S09 input comparison and the S11 results reference. A SAM space is matched to + /// the native IDF Space whose surfaces are its panel polygons one to one (geometry alone); the IDF Zone of that Space is + /// followed through ZoneHVAC:EquipmentConnections and ZoneHVAC:EquipmentList to its ZoneHVAC:IdealLoadsAirSystem. Names + /// are never used to make a match, so two SAM spaces with the same name are told apart by their geometry. + /// + public static class NativeZoneIdentity + { + /// The IDF BuildingSurface:Detailed objects of , with their vertices in building coordinates. + public static List SurfacesOf(NativeModel native, string spaceName) + { + List result = new List(); + foreach (IdfObject surface in native.Idf("BuildingSurface:Detailed")) + { + if (!string.Equals(surface.Value(4, "Space Name", out string _), spaceName, StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + IdfObject zone = native.IdfByName("Zone", surface.Value(3)); + List vertices = native.Vertices(surface, 11, zone, out string problem); + result.Add(new NativeSurface { Object = surface, Vertices = vertices, Problem = problem }); + } + + return result; + } + + /// + /// The native space whose surfaces are exactly the space's panel polygons. A candidate with an unreadable surface is + /// skipped; a second bijective candidate makes the match ambiguous and ends the search. + /// + public static NativeSpaceMatch MatchSpace(NativeModel native, ExpectedSpace space) + { + NativeSpaceMatch result = new NativeSpaceMatch(); + List expectedPolygons = space.Panels.Select(x => x.Outer).ToList(); + foreach (IdfObject candidate in native.Idf("Space")) + { + List candidateSurfaces = SurfacesOf(native, candidate.Name); + if (candidateSurfaces.Any(x => x.Vertices == null)) + { + continue; + } + + GeometryMatchResult candidateMatch = EQ.MatchPolygons(expectedPolygons, candidateSurfaces.Select(x => OsGeometry.ToPolygon3D(x.Vertices)).ToList()); + if (candidateMatch.IsBijection) + { + if (result.NativeSpace != null) + { + result.NativeSpace = null; + result.Ambiguous = true; + break; + } + + result.NativeSpace = candidate.Name; + result.Match = candidateMatch; + result.Surfaces = candidateSurfaces; + } + } + + return result; + } + + /// Every ZoneHVAC:IdealLoadsAirSystem the zone's equipment list names (empty when the chain is broken). + public static List IdealLoadsOf(NativeModel native, IdfObject zone) + { + List result = new List(); + if (zone == null) + { + return result; + } + + IdfObject connections = native.Idf("ZoneHVAC:EquipmentConnections").FirstOrDefault(x => string.Equals(x.Value(0, "Zone Name", out string _), zone.Name, StringComparison.OrdinalIgnoreCase)); + IdfObject list = native.IdfByName("ZoneHVAC:EquipmentList", connections?.Value(1, "Zone Conditioning Equipment List Name", out string _)); + if (list == null) + { + return result; + } + + for (int i = 1; i + 1 < list.Fields.Count; i++) + { + if (string.Equals(list.Value(i), "ZoneHVAC:IdealLoadsAirSystem", StringComparison.OrdinalIgnoreCase)) + { + IdfObject o = native.IdfByName("ZoneHVAC:IdealLoadsAirSystem", list.Value(i + 1)); + if (o != null) + { + result.Add(o); + } + } + } + + return result; + } + + /// + /// The native identity of every expected space. An entry has a when its + /// space matches no native space or more than one, when its native space or Zone is shared with another SAM space + /// (not bijective), or when its Zone does not have exactly one Ideal Loads system. + /// + public static List Resolve(NativeModel native, ExpectedModel expected) + { + List result = new List(); + foreach (ExpectedSpace space in expected.Spaces) + { + NativeZoneIdentityEntry entry = new NativeZoneIdentityEntry { SpaceGuid = space.Guid, SpaceName = space.Name }; + NativeSpaceMatch match = MatchSpace(native, space); + if (match.Ambiguous) + { + entry.Problem = "more than one native space matches its geometry"; + } + else if (match.NativeSpace == null) + { + entry.Problem = "no native space has exactly its panel polygons"; + } + else + { + entry.NativeSpace = match.NativeSpace; + entry.SurfacePairs = match.Match.Pairs.Count; + entry.NameCarrierAgrees = match.NativeSpace.EndsWith("_" + space.Guid.ToString("N").Substring(0, 8), StringComparison.OrdinalIgnoreCase); + IdfObject zone = native.IdfByName("Zone", native.IdfByName("Space", match.NativeSpace)?.Value(1, "Zone Name", out string _)); + if (zone == null) + { + entry.Problem = "native space '" + match.NativeSpace + "' names no IDF Zone"; + } + else + { + entry.Zone = zone.Name; + List idealLoads = IdealLoadsOf(native, zone); + entry.IdealLoadsCount = idealLoads.Count; + if (idealLoads.Count != 1) + { + entry.Problem = "zone '" + zone.Name + "' has " + idealLoads.Count + " Ideal Loads systems (exactly one is required)"; + } + else + { + entry.IdealLoads = idealLoads[0].Name; + } + } + } + + result.Add(entry); + } + + // Bijective: no native space, zone or Ideal Loads may serve two SAM spaces. + foreach (Func facet in new Func[] { x => x.NativeSpace, x => x.Zone, x => x.IdealLoads }) + { + foreach (IGrouping group in result.Where(x => facet(x) != null).GroupBy(facet, StringComparer.OrdinalIgnoreCase).Where(x => x.Count() > 1)) + { + foreach (NativeZoneIdentityEntry entry in group) + { + entry.Problem = (entry.Problem == null ? string.Empty : entry.Problem + "; ") + "native '" + group.Key + "' serves " + group.Count() + " SAM spaces (not bijective)"; + } + } + } + + return result; + } + } +} diff --git a/benchmark/SAM.Analytical.OpenStudio.InputReadBack/Compare/OsComparison.cs b/benchmark/SAM.Analytical.OpenStudio.InputReadBack/Compare/OsComparison.cs index 05edc73..11f8803 100644 --- a/benchmark/SAM.Analytical.OpenStudio.InputReadBack/Compare/OsComparison.cs +++ b/benchmark/SAM.Analytical.OpenStudio.InputReadBack/Compare/OsComparison.cs @@ -330,64 +330,21 @@ private AlignmentStatus Alignment(params NativeSchedule[] schedules) // ---- spaces, surfaces, apertures -------------------------------------------------------------------------------- - private sealed class NativeSurface - { - public IdfObject Object; - public List Vertices; - public string Problem; - } - - private List SurfacesOf(string spaceName) - { - List result = new List(); - foreach (IdfObject surface in native.Idf("BuildingSurface:Detailed")) - { - if (!string.Equals(surface.Value(4, "Space Name", out string _), spaceName, StringComparison.OrdinalIgnoreCase)) - { - continue; - } - - IdfObject zone = native.IdfByName("Zone", surface.Value(3)); - List vertices = native.Vertices(surface, 11, zone, out string problem); - result.Add(new NativeSurface { Object = surface, Vertices = vertices, Problem = problem }); - } - - return result; - } - private void CompareSpace(ExpectedSpace space, List matchedSpaces, List matchedSurfaces, List matchedSubSurfaces) { string key = Keys.Space(space.Guid); string carrier = "_" + space.Guid.ToString("N").Substring(0, 8); - List expectedPolygons = space.Panels.Select(x => x.Outer).ToList(); // Identity: the native space whose surfaces are, one to one, the space's panel polygons (geometry alone). The - // name suffix the writer mints from the SAM GUID may only contradict that match, never make it. - string nativeSpace = null; - GeometryMatchResult match = null; - List surfaces = null; - foreach (IdfObject candidate in native.Idf("Space")) + // name suffix the writer mints from the SAM GUID may only contradict that match, never make it. The match is the + // shared NativeZoneIdentity (also the S11 results reference), unchanged in behaviour. + NativeSpaceMatch spaceMatch = NativeZoneIdentity.MatchSpace(native, space); + string nativeSpace = spaceMatch.NativeSpace; + GeometryMatchResult match = spaceMatch.Match; + List surfaces = spaceMatch.Surfaces; + if (spaceMatch.Ambiguous) { - List candidateSurfaces = SurfacesOf(candidate.Name); - if (candidateSurfaces.Any(x => x.Vertices == null)) - { - continue; - } - - GeometryMatchResult candidateMatch = EQ.MatchPolygons(expectedPolygons, candidateSurfaces.Select(x => OsGeometry.ToPolygon3D(x.Vertices)).ToList()); - if (candidateMatch.IsBijection) - { - if (nativeSpace != null) - { - nativeSpace = null; - Supporting.Add(key + ": more than one native space matches its geometry"); - break; - } - - nativeSpace = candidate.Name; - match = candidateMatch; - surfaces = candidateSurfaces; - } + Supporting.Add(key + ": more than one native space matches its geometry"); } IdentityStatus identity; @@ -2100,31 +2057,7 @@ private void CompareVentilation(ExpectedSpace space, string nativeSpace, IdfObje private IdfObject IdealLoadsOf(IdfObject zone) { - if (zone == null) - { - return null; - } - - IdfObject connections = native.Idf("ZoneHVAC:EquipmentConnections").FirstOrDefault(x => string.Equals(x.Value(0, "Zone Name", out string _), zone.Name, StringComparison.OrdinalIgnoreCase)); - IdfObject list = native.IdfByName("ZoneHVAC:EquipmentList", connections?.Value(1, "Zone Conditioning Equipment List Name", out string _)); - if (list == null) - { - return null; - } - - List idealLoads = new List(); - for (int i = 1; i + 1 < list.Fields.Count; i++) - { - if (string.Equals(list.Value(i), "ZoneHVAC:IdealLoadsAirSystem", StringComparison.OrdinalIgnoreCase)) - { - IdfObject o = native.IdfByName("ZoneHVAC:IdealLoadsAirSystem", list.Value(i + 1)); - if (o != null) - { - idealLoads.Add(o); - } - } - } - + List idealLoads = NativeZoneIdentity.IdealLoadsOf(native, zone); if (idealLoads.Count > 1) { Consistency.Add("zone '" + zone.Name + "' has " + idealLoads.Count + " Ideal Loads systems"); diff --git a/benchmark/SAM.Analytical.OpenStudio.ResultsReadBack.Cli/CompareStep.cs b/benchmark/SAM.Analytical.OpenStudio.ResultsReadBack.Cli/CompareStep.cs new file mode 100644 index 0000000..f1ff7b8 --- /dev/null +++ b/benchmark/SAM.Analytical.OpenStudio.ResultsReadBack.Cli/CompareStep.cs @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: LGPL-3.0-or-later +// Copyright (c) 2020–2026 Michal Dengusiak & Jakub Ziolkowski and contributors + +using SAM.Analytical.Benchmark; +using SAM.Analytical.OpenStudio.InputReadBack; +using System.Collections.Generic; +using System.IO; + +namespace SAM.Analytical.OpenStudio.ResultsReadBack.Cli +{ + /// The comparison: library code only. Its inputs must be the files the read and subject steps saw (by hash). + internal static class CompareStep + { + public static int Run(string runDirectory, string modelPath, TextWriter output, TextWriter error) + { + RunLayout layout = Program.Layout(runDirectory, error, out bool ok); + if (!ok) + { + return ReadBackExit.Infrastructure; + } + + foreach (string file in new[] { layout.Reference, layout.Subject }) + { + if (!File.Exists(file)) + { + error.WriteLine("infrastructure: missing " + file + " (run read and subject first)"); + return ReadBackExit.Infrastructure; + } + } + + AnalyticalModel model = Program.LoadModel(modelPath); + ReferenceDocument reference = S11Json.Load(layout.Reference); + SubjectDocument subject = S11Json.Load(layout.Subject); + + List problems = Consistency(modelPath, layout, reference, subject); + if (problems.Count > 0) + { + foreach (string problem in problems) + { + error.WriteLine("infrastructure: " + problem); + } + + return ReadBackExit.Infrastructure; + } + + BenchmarkDocument b1a = BenchmarkSerializer.Deserialize(File.ReadAllText(layout.B1a)); + SamResults samP = SamResults.Read(layout.SamResults); + CompareDocument compare = ResultCompare.Run(model, reference, subject, b1a, samP); + compare.Inputs["model"] = Sha256.Of(modelPath); + compare.Inputs["in.idf"] = reference.IdfSha256Before; + compare.Inputs["eplusout.sql"] = reference.SqlSha256Before; + compare.Inputs["eplusout.eso"] = reference.EsoSha256Before; + compare.Inputs["b1a.json"] = Sha256.Of(layout.B1a); + compare.Inputs["sam.json"] = Sha256.Of(layout.SamResults); + compare.Inputs["reference.json"] = Sha256.Of(layout.Reference); + compare.Inputs["subject.json"] = Sha256.Of(layout.Subject); + compare.Inputs["readerPid"] = reference.ProcessId.ToString(System.Globalization.CultureInfo.InvariantCulture); + compare.Inputs["subjectPid"] = subject.ProcessId.ToString(System.Globalization.CultureInfo.InvariantCulture); + compare.Inputs["comparePid"] = compare.ProcessId.ToString(System.Globalization.CultureInfo.InvariantCulture); + compare.Inputs["mutation"] = subject.Mutation; + ResultReport.Save(compare, reference, layout.Compare); + output.WriteLine("compare: " + compare.Verdict + " (" + compare.Passed + " PASS, " + compare.Failed + " FAIL, " + compare.Blocked + " BLOCKED of " + compare.Required + "); " + Path.Combine(layout.Compare, "compare.json")); + return compare.ExitCode; + } + + /// The reference, the subject and the files on disk must describe the same run; otherwise nothing is judged. + internal static List Consistency(string modelPath, RunLayout layout, ReferenceDocument reference, SubjectDocument subject) + { + List problems = new List(); + if (reference.Schema != ReferenceDocument.CurrentSchema) problems.Add("reference.json schema " + reference.Schema); + if (subject.Schema != SubjectDocument.CurrentSchema) problems.Add("subject.json schema " + subject.Schema); + string sql = Sha256.Of(layout.Sql); + if (reference.SqlSha256Before != sql || subject.SqlSha256Before != sql || subject.SqlSha256After != sql) problems.Add("eplusout.sql is not the file the read and subject steps saw"); + if (reference.EsoSha256Before != Sha256.Of(layout.Eso)) problems.Add("eplusout.eso is not the file the read step saw"); + if (reference.IdfSha256Before != Sha256.Of(layout.Idf)) problems.Add("in.idf is not the file the read step saw"); + if (subject.B1aSha256 != Sha256.Of(layout.B1a)) problems.Add("b1a.json is not the file the subject step saw"); + if (subject.SamResultsSha256 != Sha256.Of(layout.SamResults)) problems.Add("sam.json is not the file the subject step saw"); + string model = Sha256.Of(modelPath); + if (reference.ModelSha256 != model || subject.ModelSha256 != model) problems.Add("the model is not the one the read and subject steps used"); + if (reference.ProcessId == subject.ProcessId && reference.ProcessStartedUtc == subject.ProcessStartedUtc) problems.Add("read and subject ran in the same process"); + return problems; + } + } +} diff --git a/benchmark/SAM.Analytical.OpenStudio.ResultsReadBack.Cli/NegativeStep.cs b/benchmark/SAM.Analytical.OpenStudio.ResultsReadBack.Cli/NegativeStep.cs new file mode 100644 index 0000000..eb2e020 --- /dev/null +++ b/benchmark/SAM.Analytical.OpenStudio.ResultsReadBack.Cli/NegativeStep.cs @@ -0,0 +1,323 @@ +// SPDX-License-Identifier: LGPL-3.0-or-later +// Copyright (c) 2020–2026 Michal Dengusiak & Jakub Ziolkowski and contributors + +using SAM.Analytical.Benchmark; +using SAM.Analytical.OpenStudio.Benchmark; +using SAM.Analytical.OpenStudio.InputReadBack; +using System; +using System.Collections.Generic; +using System.Data.SQLite; +using System.Globalization; +using System.IO; +using System.Linq; + +namespace SAM.Analytical.OpenStudio.ResultsReadBack.Cli +{ + /// The record of a negative copy (negative.json): what was mutated and the exit compare must give. + public sealed class NegativeRecord + { + public string Case { get; set; } + public string Source { get; set; } + public int ExpectedExit { get; set; } + public string Mutation { get; set; } + public Dictionary SourceSha256 { get; set; } = new Dictionary(StringComparer.Ordinal); + public DateTime CreatedUtc { get; set; } + } + + /// + /// Disposable negatives on COPIES of a produced run (no extra simulation; the original is never touched): + /// + /// ctl-unmutated-copy: a plain copy; compare must PASS when the original passes. + /// neg-delete-hour-both: one hour of every S11 series removed from BOTH the SQL and the ESO copy; must BLOCK. + /// neg-duplicate-hour-both: one hour of every S11 series repeated in BOTH copies; must BLOCK. + /// neg-sensible-cooling-substitution: the producer objects (sam.json) and the import (SAM-I) carry the reference + /// SENSIBLE cooling in place of Total; the B1a copy is re-mapped from those objects, so the closure holds and the + /// substitution must FAIL on the cooling rows. For the hour cases the read/subject/compare steps run fresh on the copy; + /// for the substitution the subject is captured here, then its SAM-I cooling values are replaced. + /// + /// + internal static class NegativeStep + { + public static readonly Dictionary Cases = new Dictionary(StringComparer.Ordinal) + { + ["ctl-unmutated-copy"] = ReadBackExit.Pass, + ["neg-delete-hour-both"] = ReadBackExit.BlockedTechnical, + ["neg-duplicate-hour-both"] = ReadBackExit.BlockedTechnical, + ["neg-sensible-cooling-substitution"] = ReadBackExit.Fail, + }; + + /// The mutated hour: 0-based hour 4500 of a non-leap year is 7 July, interval 12:00-13:00 (EnergyPlus hour 13). + public static readonly HourKey MutatedHour = new HourKey(7, 7, 13); + + public static int Run(string runDirectory, string modelPath, string caseId, string outputDirectory, TextWriter output) + { + if (!Cases.TryGetValue(caseId, out int expectedExit)) + { + throw new ArgumentException("unknown case '" + caseId + "'; cases: " + string.Join(", ", Cases.Keys)); + } + + if (Directory.Exists(outputDirectory) && Directory.EnumerateFileSystemEntries(outputDirectory).Any()) + { + throw new IOException("The negative directory must be new or empty: " + outputDirectory); + } + + RunLayout source = RunLayout.Resolve(runDirectory, out List problems); + if (problems.Count > 0) + { + throw new IOException(string.Join("; ", problems)); + } + + NegativeRecord record = new NegativeRecord { Case = caseId, Source = source.Directory, ExpectedExit = expectedExit, CreatedUtc = DateTime.UtcNow }; + foreach (string file in new[] { source.B1a, source.SamResults, source.Sql, source.Eso, source.Idf }) + { + record.SourceSha256[Path.GetFileName(file)] = Sha256.Of(file); + } + + string runFolder = Path.Combine(outputDirectory, "work", "run"); + Directory.CreateDirectory(runFolder); + File.Copy(source.B1a, Path.Combine(outputDirectory, RunLayout.B1aFile)); + File.Copy(source.SamResults, Path.Combine(outputDirectory, RunLayout.SamResultsFile)); + File.Copy(source.Sql, Path.Combine(runFolder, "eplusout.sql")); + File.Copy(source.Eso, Path.Combine(runFolder, "eplusout.eso")); + File.Copy(source.Idf, Path.Combine(runFolder, "in.idf")); + RunLayout copy = RunLayout.Resolve(outputDirectory, out _); + + switch (caseId) + { + case "ctl-unmutated-copy": + record.Mutation = "none (control)"; + break; + case "neg-delete-hour-both": + record.Mutation = "deleted " + MutatedHour + " of every S11 series in SQL (" + SqlDeleteHour(copy.Sql, MutatedHour) + " rows) and ESO (" + EsoEditHour(copy.Eso, MutatedHour, false) + " lines)"; + break; + case "neg-duplicate-hour-both": + record.Mutation = "repeated " + MutatedHour + " of every S11 series in SQL (" + SqlDuplicateHour(copy.Sql, MutatedHour) + " rows) and ESO (" + EsoEditHour(copy.Eso, MutatedHour, true) + " lines)"; + break; + case "neg-sensible-cooling-substitution": + record.Mutation = SensibleSubstitution(source, copy, modelPath); + break; + } + + S11Json.Save(record, Path.Combine(outputDirectory, RunLayout.NegativeFile)); + output.WriteLine("negative " + caseId + ": " + record.Mutation + "; compare must exit " + expectedExit); + return ReadBackExit.Pass; + } + + // ---- SQL --------------------------------------------------------------------------------------------------------- + + private static List HourlyIndexes(SQLiteConnection connection) + { + List result = new List(); + using (SQLiteCommand command = connection.CreateCommand()) + { + string names = string.Join(",", S11Names.IdealLoadsVariables.Concat(S11Names.ZoneVariables).Select(x => "'" + x + "'")); + command.CommandText = "SELECT ReportDataDictionaryIndex FROM ReportDataDictionary WHERE ReportingFrequency = 'Hourly' AND IsMeter = 0 AND Name IN (" + names + ")"; + using (SQLiteDataReader reader = command.ExecuteReader()) + { + while (reader.Read()) + { + result.Add(reader.GetInt64(0)); + } + } + } + + return result; + } + + private static long HourlyTimeIndex(SQLiteConnection connection, HourKey hour) + { + using (SQLiteCommand command = connection.CreateCommand()) + { + command.CommandText = "SELECT t.TimeIndex FROM Time t JOIN EnvironmentPeriods e ON e.EnvironmentPeriodIndex = t.EnvironmentPeriodIndex WHERE e.EnvironmentType = 3 AND t.Interval = 60 AND t.Month = @m AND t.Day = @d AND t.Hour = @h AND t.WarmupFlag = 0"; + command.Parameters.AddWithValue("@m", hour.Month); + command.Parameters.AddWithValue("@d", hour.Day); + command.Parameters.AddWithValue("@h", hour.Hour); + object value = command.ExecuteScalar(); + if (value == null) + { + throw new InvalidDataException("no hourly Time row for " + hour); + } + + return System.Convert.ToInt64(value, CultureInfo.InvariantCulture); + } + } + + private static SQLiteConnection Open(string path) + { + SQLiteConnection connection = new SQLiteConnection(new SQLiteConnectionStringBuilder { DataSource = path, FailIfMissing = true }.ConnectionString); + connection.Open(); + return connection; + } + + internal static int SqlDeleteHour(string sqlPath, HourKey hour) + { + try + { + using (SQLiteConnection connection = Open(sqlPath)) + using (SQLiteCommand command = connection.CreateCommand()) + { + long timeIndex = HourlyTimeIndex(connection, hour); + command.CommandText = "DELETE FROM ReportData WHERE TimeIndex = @t AND ReportDataDictionaryIndex IN (" + string.Join(",", HourlyIndexes(connection)) + ")"; + command.Parameters.AddWithValue("@t", timeIndex); + return command.ExecuteNonQuery(); + } + } + finally + { + SQLiteConnection.ClearAllPools(); + } + } + + /// A second hourly Time row with the same stamp, and a copy of every S11 value at it. + internal static int SqlDuplicateHour(string sqlPath, HourKey hour) + { + try + { + using (SQLiteConnection connection = Open(sqlPath)) + using (SQLiteCommand command = connection.CreateCommand()) + { + long timeIndex = HourlyTimeIndex(connection, hour); + command.CommandText = "SELECT MAX(TimeIndex) + 1 FROM Time"; + long duplicate = System.Convert.ToInt64(command.ExecuteScalar(), CultureInfo.InvariantCulture); + command.CommandText = "INSERT INTO Time (TimeIndex, Year, Month, Day, Hour, Minute, Dst, Interval, IntervalType, SimulationDays, DayType, EnvironmentPeriodIndex, WarmupFlag) " + + "SELECT @n, Year, Month, Day, Hour, Minute, Dst, Interval, IntervalType, SimulationDays, DayType, EnvironmentPeriodIndex, WarmupFlag FROM Time WHERE TimeIndex = @t"; + command.Parameters.AddWithValue("@n", duplicate); + command.Parameters.AddWithValue("@t", timeIndex); + command.ExecuteNonQuery(); + command.CommandText = "INSERT INTO ReportData (TimeIndex, ReportDataDictionaryIndex, Value) SELECT @n, ReportDataDictionaryIndex, Value FROM ReportData WHERE TimeIndex = @t AND ReportDataDictionaryIndex IN (" + string.Join(",", HourlyIndexes(connection)) + ")"; + return command.ExecuteNonQuery(); + } + } + finally + { + SQLiteConnection.ClearAllPools(); + } + } + + // ---- ESO --------------------------------------------------------------------------------------------------------- + + /// Removes (or repeats) the S11 value lines after the RunPeriod's hourly stamp of . + internal static int EsoEditHour(string esoPath, HourKey hour, bool duplicate) + { + List lines = File.ReadAllLines(esoPath).ToList(); + HashSet wanted = new HashSet(S11Names.IdealLoadsVariables.Concat(S11Names.ZoneVariables), StringComparer.OrdinalIgnoreCase); + HashSet ids = new HashSet(); + int end = lines.FindIndex(x => x.StartsWith("End of Data Dictionary", StringComparison.Ordinal)); + for (int i = 1; i < end; i++) + { + EsoVariable variable = EsoResultRead.DictionaryLine(lines[i]); + if (variable != null && !variable.IsMeter && wanted.Contains(variable.Name) && string.Equals(variable.Frequency, "Hourly", StringComparison.OrdinalIgnoreCase)) + { + ids.Add(variable.Id); + } + } + + for (int i = end + 1; i < lines.Count; i++) + { + string[] f = lines[i].Split(','); + if (f[0] != "2" || f.Length < 8) + { + continue; + } + + if (int.Parse(f[2].Trim(), CultureInfo.InvariantCulture) != hour.Month || int.Parse(f[3].Trim(), CultureInfo.InvariantCulture) != hour.Day || int.Parse(f[5].Trim(), CultureInfo.InvariantCulture) != hour.Hour + || double.Parse(f[6].Trim(), CultureInfo.InvariantCulture) != 0 || double.Parse(f[7].Trim(), CultureInfo.InvariantCulture) != 60) + { + continue; + } + + List block = new List(); + for (int j = i + 1; j < lines.Count; j++) + { + string[] g = lines[j].Split(','); + if (!int.TryParse(g[0], NumberStyles.Integer, CultureInfo.InvariantCulture, out int id) || id <= 6) + { + break; + } + + if (ids.Contains(id)) + { + block.Add(j); + } + } + + if (duplicate) + { + List copy = new List { lines[i] }; + copy.AddRange(block.Select(x => lines[x])); + int after = block.Count > 0 ? block.Max() + 1 : i + 1; + while (after < lines.Count && int.TryParse(lines[after].Split(',')[0], NumberStyles.Integer, CultureInfo.InvariantCulture, out int id) && id > 6) + { + after++; + } + + lines.InsertRange(after, copy); + File.WriteAllLines(esoPath, lines); + return copy.Count; + } + + for (int k = block.Count - 1; k >= 0; k--) + { + lines.RemoveAt(block[k]); + } + + File.WriteAllLines(esoPath, lines); + return block.Count; + } + + throw new InvalidDataException("no hourly ESO stamp for " + hour); + } + + // ---- F3 substitution --------------------------------------------------------------------------------------------- + + /// + /// Replaces every cooling quantity of the producer objects and of the import by the reference SENSIBLE cooling, and + /// re-maps the B1a copy from the mutated objects with the producer (so the copy is self-consistent). + /// + private static string SensibleSubstitution(RunLayout source, RunLayout copy, string modelPath) + { + if (!File.Exists(source.Reference) || !File.Exists(source.Subject)) + { + throw new IOException("the source run needs reference.json and subject.json (run read and subject first)"); + } + + ReferenceDocument reference = S11Json.Load(source.Reference); + AnalyticalModel model = Program.LoadModel(modelPath); + ModelLoadReference cooling = reference.ModelOf(LoadKind.Cooling); + if (!cooling.Sensible.Available || reference.Spaces.Any(x => !x.Quantities[S11Names.SensibleCooling].Available)) + { + throw new InvalidDataException("the reference has no complete Sensible cooling to substitute"); + } + + OpenStudioBenchmarkInputs inputs = OpenStudioBenchmarkInputs.Read(source.SamResults); + inputs.ModelResult.SetValue(AnalyticalModelSimulationResultParameter.ConsumptionCooling, cooling.Sensible.ConsumptionKwh); + inputs.ModelResult.SetValue(AnalyticalModelSimulationResultParameter.PeakCoolingLoad, cooling.Sensible.CoincidentPeakKw); + inputs.ModelResult.SetValue(AnalyticalModelSimulationResultParameter.PeakCoolingHour, cooling.Sensible.PeakHour); + foreach (SpaceSimulationResult result in inputs.SpaceResults) + { + if (result.TryGetValue(Analytical.SpaceSimulationResultParameter.LoadType, out string loadType) && loadType == LoadType.Cooling.ToString()) + { + QuantityReference sensible = reference.Spaces.Single(x => x.SpaceGuid.ToString("N") == result.Reference).Quantities[S11Names.SensibleCooling]; + result.SetValue(Analytical.SpaceSimulationResultParameter.Load, sensible.PeakWatts); + result.SetValue(Analytical.SpaceSimulationResultParameter.LoadIndex, sensible.PeakHour); + } + } + + inputs.Write(copy.SamResults); + BenchmarkDocument produced = BenchmarkSerializer.Deserialize(File.ReadAllText(source.B1a)); + BenchmarkSerializer.Write(copy.B1a, model.ToBenchmark(SubjectStep.Context(produced.Provenance), inputs)); + + SubjectDocument subject = SubjectStep.Capture(model, modelPath, copy); + foreach (SubjectResult result in subject.SamI.Where(x => x.IsAnnual && x.LoadType == LoadType.Cooling.ToString())) + { + QuantityReference sensible = reference.Spaces.Single(x => x.SpaceGuid.ToString("N") == result.Reference).Quantities[S11Names.SensibleCooling]; + result.Load = sensible.PeakWatts; + result.LoadIndex = sensible.PeakHour; + } + + subject.Mutation = "SAM-I cooling Load/LoadIndex replaced by the reference Sensible cooling peak"; + S11Json.Save(subject, copy.Subject); + return "producer objects and SAM-I carry the reference Sensible cooling (consumption " + cooling.Sensible.ConsumptionKwh.ToString("R", CultureInfo.InvariantCulture) + " kWh for Total " + cooling.Total.ConsumptionKwh.ToString("R", CultureInfo.InvariantCulture) + " kWh); B1a re-mapped from the mutated objects"; + } + } +} diff --git a/benchmark/SAM.Analytical.OpenStudio.ResultsReadBack.Cli/Program.cs b/benchmark/SAM.Analytical.OpenStudio.ResultsReadBack.Cli/Program.cs new file mode 100644 index 0000000..3275124 --- /dev/null +++ b/benchmark/SAM.Analytical.OpenStudio.ResultsReadBack.Cli/Program.cs @@ -0,0 +1,132 @@ +// SPDX-License-Identifier: LGPL-3.0-or-later +// Copyright (c) 2020–2026 Michal Dengusiak & Jakub Ziolkowski and contributors + +using SAM.Analytical.OpenStudio.InputReadBack; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; + +namespace SAM.Analytical.OpenStudio.ResultsReadBack.Cli +{ + /// + /// os-results-readback: the S11 OpenStudio results round-trip harness. Each step runs in its own process. + /// + /// read --run <run dir> --model <fixture.json> [--fixture-id id]: the independent SQL/ESO reference (reference.json) + /// subject --run <run dir> --model <fixture.json>: the code under test (subject.json) + /// compare --run <run dir> --model <fixture.json>: the verdict (compare/compare.json, compare.md) + /// negative --run <run dir> --model <fixture.json> --case <id> --out <new dir>: a mutated copy + /// + /// Exit codes follow S08/S09: 0 PASS, 10 FAIL, 11 BLOCKED technical, 64 usage, 70 infrastructure. read, subject and + /// negative return 0 when they completed (their findings are judged by compare). + /// + public static class Program + { + public static int Main(string[] args) + { + if (args == null || args.Length == 0 || (args.Length - 1) % 2 != 0) + { + return Usage(); + } + + Dictionary options = new Dictionary(StringComparer.OrdinalIgnoreCase); + for (int i = 1; i + 1 < args.Length; i += 2) + { + if (!args[i].StartsWith("--", StringComparison.Ordinal) || options.ContainsKey(args[i].Substring(2))) + { + return Usage(); + } + + options[args[i].Substring(2)] = args[i + 1]; + } + + try + { + switch (args[0].ToLowerInvariant()) + { + case "read": + return ReadStep.Run(Required(options, "run"), Required(options, "model"), Optional(options, "fixture-id"), Console.Out, Console.Error); + case "subject": + return SubjectStep.Run(Required(options, "run"), Required(options, "model"), Console.Out, Console.Error); + case "compare": + return CompareStep.Run(Required(options, "run"), Required(options, "model"), Console.Out, Console.Error); + case "negative": + return NegativeStep.Run(Required(options, "run"), Required(options, "model"), Required(options, "case"), Required(options, "out"), Console.Out); + default: + return Usage(); + } + } + catch (UsageException exception) + { + Console.Error.WriteLine(exception.Message); + return Usage(); + } + catch (Exception exception) + { + Console.Error.WriteLine("infrastructure: " + exception); + return ReadBackExit.Infrastructure; + } + } + + /// The frozen fixture model (the SAM inputs: expected spaces, areas, volumes). + internal static AnalyticalModel LoadModel(string modelPath) + { + if (!File.Exists(modelPath)) + { + throw new FileNotFoundException("model not found", modelPath); + } + + AnalyticalModel model = Core.Convert.ToSAM(modelPath)?.FirstOrDefault(); + if (model == null) + { + throw new InvalidDataException(modelPath + " is not a SAM AnalyticalModel"); + } + + return model; + } + + /// The run layout, or the infrastructure problems that stop a step. + internal static RunLayout Layout(string runDirectory, TextWriter error, out bool ok) + { + RunLayout layout = RunLayout.Resolve(runDirectory, out List problems); + foreach (string problem in problems) + { + error.WriteLine("infrastructure: " + problem); + } + + ok = problems.Count == 0; + return layout; + } + + private static string Required(Dictionary options, string name) + { + if (!options.TryGetValue(name, out string value) || string.IsNullOrWhiteSpace(value)) + { + throw new UsageException("missing --" + name); + } + + return value; + } + + private static string Optional(Dictionary options, string name) + { + return options.TryGetValue(name, out string value) ? value : null; + } + + private static int Usage() + { + Console.Error.WriteLine("usage: os-results-readback read --run --model [--fixture-id id]"); + Console.Error.WriteLine(" os-results-readback subject --run --model "); + Console.Error.WriteLine(" os-results-readback compare --run --model "); + Console.Error.WriteLine(" os-results-readback negative --run --model --case <" + string.Join("|", NegativeStep.Cases.Keys) + "> --out "); + return ReadBackExit.Usage; + } + + private sealed class UsageException : Exception + { + public UsageException(string message) : base(message) + { + } + } + } +} diff --git a/benchmark/SAM.Analytical.OpenStudio.ResultsReadBack.Cli/ReadStep.cs b/benchmark/SAM.Analytical.OpenStudio.ResultsReadBack.Cli/ReadStep.cs new file mode 100644 index 0000000..ae204d4 --- /dev/null +++ b/benchmark/SAM.Analytical.OpenStudio.ResultsReadBack.Cli/ReadStep.cs @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: LGPL-3.0-or-later +// Copyright (c) 2020–2026 Michal Dengusiak & Jakub Ziolkowski and contributors + +using SAM.Analytical.OpenStudio.InputReadBack; +using System.IO; + +namespace SAM.Analytical.OpenStudio.ResultsReadBack.Cli +{ + /// The reference read: library code only (no production importer, writer or naming). + internal static class ReadStep + { + public static int Run(string runDirectory, string modelPath, string fixtureId, TextWriter output, TextWriter error) + { + RunLayout layout = Program.Layout(runDirectory, error, out bool ok); + if (!ok) + { + return ReadBackExit.Infrastructure; + } + + AnalyticalModel model = Program.LoadModel(modelPath); + ReferenceDocument reference = ResultReference.Build(model, modelPath, layout.Idf, layout.Sql, layout.Eso, fixtureId ?? Path.GetFileNameWithoutExtension(modelPath)); + S11Json.Save(reference, layout.Reference); + output.WriteLine("read: " + layout.Reference + " (" + reference.Spaces.Count + " spaces, " + reference.GlobalBlocks.Count + " global block(s), " + reference.Completeness.Count + " series judged)"); + return ReadBackExit.Pass; + } + } +} diff --git a/benchmark/SAM.Analytical.OpenStudio.ResultsReadBack.Cli/SAM.Analytical.OpenStudio.ResultsReadBack.Cli.csproj b/benchmark/SAM.Analytical.OpenStudio.ResultsReadBack.Cli/SAM.Analytical.OpenStudio.ResultsReadBack.Cli.csproj new file mode 100644 index 0000000..470e23a --- /dev/null +++ b/benchmark/SAM.Analytical.OpenStudio.ResultsReadBack.Cli/SAM.Analytical.OpenStudio.ResultsReadBack.Cli.csproj @@ -0,0 +1,61 @@ + + + + Exe + net8.0 + SAM.Analytical.OpenStudio.ResultsReadBack.Cli + os-results-readback + false + x64 + x64 + x64 + latest + disable + disable + + + + + + + + + + + + + + + + + + + + + + + ..\..\..\SAM_Validation\build\SAM.Analytical.Benchmark.dll + + + ..\..\..\SAM\build\SAM.Core.dll + + + ..\..\..\SAM\build\SAM.Geometry.dll + + + ..\..\..\SAM\build\SAM.Analytical.dll + + + ..\..\..\SAM\build\SAM.Architectural.dll + + + ..\..\..\SAM\build\SAM.Weather.dll + + + + diff --git a/benchmark/SAM.Analytical.OpenStudio.ResultsReadBack.Cli/SubjectStep.cs b/benchmark/SAM.Analytical.OpenStudio.ResultsReadBack.Cli/SubjectStep.cs new file mode 100644 index 0000000..051e665 --- /dev/null +++ b/benchmark/SAM.Analytical.OpenStudio.ResultsReadBack.Cli/SubjectStep.cs @@ -0,0 +1,153 @@ +// SPDX-License-Identifier: LGPL-3.0-or-later +// Copyright (c) 2020–2026 Michal Dengusiak & Jakub Ziolkowski and contributors + +using SAM.Analytical.Benchmark; +using SAM.Analytical.OpenStudio.Benchmark; +using SAM.Analytical.OpenStudio.InputReadBack; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.Json.Nodes; + +namespace SAM.Analytical.OpenStudio.ResultsReadBack.Cli +{ + /// + /// The subject capture: the ONLY step that runs the code under test. It (1) re-maps the captured producer objects + /// (sam.json) with the producer's own ToBenchmark and requires byte-identical B1a model and spaces sections (closure), + /// (2) records the production report keys of every SAM space, and (3) imports the same SQL into a fresh copy of the + /// model with the SAM_UI/Grasshopper path Modify.AddResults. + /// + internal static class SubjectStep + { + public static int Run(string runDirectory, string modelPath, TextWriter output, TextWriter error) + { + RunLayout layout = Program.Layout(runDirectory, error, out bool ok); + if (!ok) + { + return ReadBackExit.Infrastructure; + } + + AnalyticalModel model = Program.LoadModel(modelPath); + SubjectDocument subject = Capture(model, modelPath, layout); + S11Json.Save(subject, layout.Subject); + output.WriteLine("subject: " + layout.Subject + " (closure " + (subject.ClosureIdentical ? "identical" : "DIFFERS") + ", " + subject.SamI.Count + " SAM-I space results)"); + return ReadBackExit.Pass; + } + + public static SubjectDocument Capture(AnalyticalModel model, string modelPath, RunLayout layout) + { + SubjectDocument result = new SubjectDocument + { + ProcessId = Environment.ProcessId, + ProcessStartedUtc = ProcessIdentity.CurrentStartedUtc(), + StartedUtc = DateTime.UtcNow, + SubjectAssemblySha256 = Sha256.Of(typeof(SubjectStep).Assembly.Location), + ProducerAssemblySha256 = Sha256.Of(typeof(OpenStudioBenchmarkInputs).Assembly.Location), + ImporterAssemblySha256 = Sha256.Of(typeof(Analytical.OpenStudio.Modify).Assembly.Location), + ModelSha256 = Sha256.Of(modelPath), + B1aSha256 = Sha256.Of(layout.B1a), + SamResultsSha256 = Sha256.Of(layout.SamResults), + SqlSha256Before = Sha256.Of(layout.Sql), + }; + + Closure(model, layout, result); + + foreach (Space space in model.AdjacencyCluster?.GetSpaces() ?? new List()) + { + result.ProductionKeys.Add(new ProductionKey + { + SpaceGuid = space.Guid, + SpaceName = space.Name, + IdealLoads = Core.OpenStudio.Query.OpenStudioName("IdealLoads", space.Name, space.Guid), + ThermalZone = Core.OpenStudio.Query.OpenStudioName("ThermalZone", space.Name, space.Guid), + }); + } + + Import(model, layout.Sql, result); + result.SqlSha256After = Sha256.Of(layout.Sql); + result.FinishedUtc = DateTime.UtcNow; + return result; + } + + /// + /// The captured objects, mapped again by the producer with a context rebuilt from the produced document's own + /// provenance, must reproduce its model and spaces sections byte for byte. + /// + private static void Closure(AnalyticalModel model, RunLayout layout, SubjectDocument result) + { + try + { + string producedJson = File.ReadAllText(layout.B1a); + BenchmarkDocument produced = BenchmarkSerializer.Deserialize(producedJson); + OpenStudioBenchmarkInputs inputs = OpenStudioBenchmarkInputs.Read(layout.SamResults); + BenchmarkDocument regenerated = model.ToBenchmark(Context(produced.Provenance), inputs); + JsonObject producedNode = JsonNode.Parse(producedJson) as JsonObject; + JsonObject regeneratedNode = JsonNode.Parse(BenchmarkSerializer.Serialize(regenerated)) as JsonObject; + string producedModel = producedNode?["model"]?.ToJsonString(); + string regeneratedModel = regeneratedNode?["model"]?.ToJsonString(); + string producedSpaces = producedNode?["spaces"]?.ToJsonString(); + string regeneratedSpaces = regeneratedNode?["spaces"]?.ToJsonString(); + result.ClosureModelSha256Produced = Hash(producedModel); + result.ClosureModelSha256Regenerated = Hash(regeneratedModel); + result.ClosureSpacesSha256Produced = Hash(producedSpaces); + result.ClosureSpacesSha256Regenerated = Hash(regeneratedSpaces); + bool model_ = producedModel != null && producedModel == regeneratedModel; + bool spaces = producedSpaces != null && producedSpaces == regeneratedSpaces; + result.ClosureIdentical = model_ && spaces; + result.ClosureDetail = result.ClosureIdentical + ? "the captured objects re-map to byte-identical B1a model and spaces sections" + : "the captured objects re-map to a different B1a: model section " + (model_ ? "identical" : "DIFFERS") + ", spaces section " + (spaces ? "identical" : "DIFFERS"); + } + catch (Exception exception) when (exception is IOException || exception is System.Text.Json.JsonException || exception is InvalidDataException || exception is InvalidCastException) + { + result.ClosureIdentical = false; + result.ClosureDetail = "closure could not be evaluated: " + exception.Message; + } + } + + /// A producer context carrying the produced document's provenance (the mapping reads only provenance from it). + internal static OpenStudioBenchmarkContext Context(BenchmarkProvenance provenance) + { + OpenStudioBenchmarkContext context = new OpenStudioBenchmarkContext + { + SourceModelName = provenance.SourceModelName, + SourceModelGuid = provenance.SourceModelGuid, + SourceFileHash = provenance.SourceFileHash, + CanonicalModelHash = provenance.CanonicalModelHash, + CanonicalizationVersion = provenance.CanonicalizationVersion, + SamCommit = provenance.SamCommit, + RunnerCommit = provenance.RunnerCommit, + EngineName = provenance.Engine?.Name, + EngineVersion = provenance.Engine?.Version, + SdkVersion = provenance.Engine?.SdkVersion, + WeatherIdentity = provenance.Weather?.Identity, + WeatherHash = provenance.Weather?.Hash, + DesignDaySource = provenance.DesignDaySource, + RunTimestampUtc = provenance.RunTimestampUtc, + DurationSeconds = provenance.DurationSeconds ?? 0, + State = provenance.State, + }; + + context.Warnings.AddRange(provenance.Warnings ?? new List()); + context.Notes.AddRange(provenance.Notes ?? new List()); + return context; + } + + /// SAM-I: the production import of the SQL into a fresh copy of the model, with its diagnostics and relations. + private static void Import(AnalyticalModel model, string sqlPath, SubjectDocument result) + { + AdjacencyCluster cluster = new AdjacencyCluster(model.AdjacencyCluster); + List results = Analytical.OpenStudio.Modify.AddResults(cluster, sqlPath, out List diagnostics) ?? new List(); + result.SamIDiagnostics.AddRange(diagnostics ?? new List()); + List spaces = cluster.GetSpaces() ?? new List(); + foreach (SpaceSimulationResult spaceResult in results.OfType()) + { + Space attached = spaces.FirstOrDefault(x => cluster.GetResults(x)?.Any(r => r.Guid == spaceResult.Guid) == true); + result.SamI.Add(SamResults.Describe(spaceResult, attached?.Guid)); + } + } + + private static string Hash(string text) => text == null ? null : Sha256.OfBytes(System.Text.Encoding.UTF8.GetBytes(text)); + } +} diff --git a/benchmark/SAM.Analytical.OpenStudio.ResultsReadBack/Build-S11-Receipt.ps1 b/benchmark/SAM.Analytical.OpenStudio.ResultsReadBack/Build-S11-Receipt.ps1 new file mode 100644 index 0000000..44b87c5 --- /dev/null +++ b/benchmark/SAM.Analytical.OpenStudio.ResultsReadBack/Build-S11-Receipt.ps1 @@ -0,0 +1,155 @@ +# SPDX-License-Identifier: LGPL-3.0-or-later +# Copyright (c) 2020-2026 Michal Dengusiak & Jakub Ziolkowski and contributors +# +# S11: the controlled, engine-free Release build whose receipt Run-S11-Results.ps1 requires (ported from SAM_Tas +# Build-S10-Receipt.ps1). Nothing here starts OpenStudio or EnergyPlus. +# +# 1. SAM, SAM_Validation, SAM_Tas and SAM_OpenStudio must each be a clean checkout (no change, no untracked file). Their +# HEADs are the build commits. +# 2. MSBuild rebuilds, one project at a time, the SAM core libraries into SAM\build, the B1a schema into +# SAM_Validation\build, then the two S11 executables: benchmark-openstudio (the producer) and os-results-readback +# (the harness, which builds the S09 reader and the SAM_Tas evaluation core it references). Each build gets +# /p:SAMSourceRevision=, so every assembly carries InformationalVersion +. +# The Grasshopper projects are NOT built: their post-build step deploys into %APPDATA%\SAM. +# 3. Only after every build exits 0: the checkouts are still clean at the same HEADs; each declared artifact was written +# by this build and carries its checkout's stamp; every SAM\build library beside the executables is byte-identical +# to its SAM\build copy. +# 4. The receipt (default: s11-build-receipt.json beside os-results-readback.exe) records the commits, the commands and +# the sha256 of every .dll/.exe in both executable folders. It is removed first, so a refused build leaves none. +# +# Usage (Windows PowerShell 5.1; the defaults are this workspace's paths): +# powershell -ExecutionPolicy Bypass -File Build-S11-Receipt.ps1 +param( + [string]$MSBuild = 'C:\Program Files\Microsoft Visual Studio\18\Community\MSBuild\Current\Bin\MSBuild.exe', + [string]$RunnerRepository = '', + [string]$BuildReceipt = '' +) + +$ErrorActionPreference = 'Stop' +if (-not $RunnerRepository) { $RunnerRepository = Join-Path $PSScriptRoot '..\..' } +$RunnerRepository = [IO.Path]::GetFullPath($RunnerRepository) +$workspace = Split-Path $RunnerRepository +$SamRepository = Join-Path $workspace 'SAM' +$ValidationRepository = Join-Path $workspace 'SAM_Validation' +$TasRepository = Join-Path $workspace 'SAM_Tas' +$benchmarkProject = Join-Path $RunnerRepository 'benchmark\SAM.Analytical.OpenStudio.Benchmark\SAM.Analytical.OpenStudio.Benchmark.csproj' +$harnessProject = Join-Path $RunnerRepository 'benchmark\SAM.Analytical.OpenStudio.ResultsReadBack.Cli\SAM.Analytical.OpenStudio.ResultsReadBack.Cli.csproj' +$benchmarkExe = Join-Path $RunnerRepository 'benchmark\SAM.Analytical.OpenStudio.Benchmark\bin\x64\Release\net8.0\benchmark-openstudio.exe' +$harnessExe = Join-Path $RunnerRepository 'benchmark\SAM.Analytical.OpenStudio.ResultsReadBack.Cli\bin\x64\Release\net8.0\os-results-readback.exe' +if (-not $BuildReceipt) { $BuildReceipt = Join-Path (Split-Path $harnessExe) 's11-build-receipt.json' } +$BuildReceipt = [IO.Path]::GetFullPath($BuildReceipt) + +# The SAM core libraries the S11 executables load from SAM\build, in dependency order. +$samProjects = 'SAM.Core', 'SAM.Math', 'SAM.Units', 'SAM.Geometry', 'SAM.Weather', 'SAM.Architectural', 'SAM.Analytical' + +function Refuse([string]$Reason) { + Write-Host "REFUSED (no receipt written): $Reason" + exit 3 +} + +# SHA-256 through .NET (Get-FileHash is a module function Windows PowerShell 5.1 may not autoload in a child host). +function Sha([string]$Path) { + if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) { return $null } + $stream = [IO.File]::OpenRead($Path) + try { + $sha = [Security.Cryptography.SHA256]::Create() + try { return ([BitConverter]::ToString($sha.ComputeHash($stream)) -replace '-', '').ToLowerInvariant() } finally { $sha.Dispose() } + } + finally { $stream.Dispose() } +} + +$git = 'C:\Program Files\Git\cmd\git.exe' +if (-not (Test-Path $git)) { $git = 'git' } + +function Git([string]$Directory) { + $ErrorActionPreference = 'Continue' + & $git -C $Directory @args 2>$null +} + +function Clean-Head([string]$Directory) { + if (-not (Test-Path (Join-Path $Directory '.git'))) { Refuse "$Directory is not a git checkout" } + $head = (Git $Directory rev-parse HEAD | Out-String).Trim() + if ($head -notmatch '^[0-9a-f]{40}$') { Refuse "HEAD of $Directory is not a full commit SHA ('$head')" } + $status = @(Git $Directory status --porcelain --untracked-files=all) + if ($LASTEXITCODE -ne 0) { Refuse "git status failed in $Directory" } + if ($status.Count -gt 0) { Refuse "$Directory has $($status.Count) uncommitted or untracked change(s), e.g. $($status[0])" } + return $head +} + +function Test-Stamp([string]$Path, [string]$Commit) { + $text = [Text.Encoding]::GetEncoding(28591).GetString([IO.File]::ReadAllBytes($Path)) + return $text.Contains('+' + $Commit) +} + +if (Test-Path -LiteralPath $BuildReceipt) { Remove-Item -LiteralPath $BuildReceipt -Force } +$heads = [ordered]@{} +foreach ($repository in @($SamRepository, $ValidationRepository, $TasRepository, $RunnerRepository)) { $heads[$repository] = Clean-Head $repository } +if (-not (Test-Path -LiteralPath $MSBuild)) { Refuse "MSBuild not found: $MSBuild" } + +$samBuild = Join-Path $SamRepository 'build' +$started = (Get-Date).ToUniversalTime() +$commands = New-Object System.Collections.Generic.List[object] + +function Build([string]$Project, [string]$Commit, [string[]]$Extra) { + $arguments = @($Project, '-restore', '-t:Rebuild', '-p:Configuration=Release', "-p:SAMSourceRevision=$Commit", '-nr:false', '-v:m') + $Extra + Write-Host "==> $Project" + & $MSBuild @arguments | Out-Host + $code = $LASTEXITCODE + $commands.Add([ordered]@{ file = $MSBuild; arguments = ($arguments -join ' '); exitCode = $code }) + if ($code -ne 0) { Refuse "MSBuild exited $code for $Project" } +} + +foreach ($name in $samProjects) { Build (Join-Path $SamRepository "SAM\$name\$name.csproj") $heads[$SamRepository] @() } +Build (Join-Path $ValidationRepository 'SAM.Analytical.Benchmark\SAM.Analytical.Benchmark.csproj') $heads[$ValidationRepository] @() +Build $benchmarkProject $heads[$RunnerRepository] @('-p:Platform=x64') +Build $harnessProject $heads[$RunnerRepository] @('-p:Platform=x64') + +foreach ($repository in @($heads.Keys)) { + if ((Clean-Head $repository) -ne $heads[$repository]) { Refuse "the HEAD of $repository moved during the build" } +} + +$benchmarkDir = Split-Path $benchmarkExe +$harnessDir = Split-Path $harnessExe +$stamped = New-Object System.Collections.Generic.List[object] +$declared = @( + [ordered]@{ path = $benchmarkExe; commit = $heads[$RunnerRepository]; compiled = $false }, + [ordered]@{ path = [IO.Path]::ChangeExtension($benchmarkExe, '.dll'); commit = $heads[$RunnerRepository]; compiled = $true }, + [ordered]@{ path = $harnessExe; commit = $heads[$RunnerRepository]; compiled = $false }, + [ordered]@{ path = [IO.Path]::ChangeExtension($harnessExe, '.dll'); commit = $heads[$RunnerRepository]; compiled = $true }, + [ordered]@{ path = (Join-Path $harnessDir 'SAM.Analytical.OpenStudio.ResultsReadBack.dll'); commit = $heads[$RunnerRepository]; compiled = $true }, + [ordered]@{ path = (Join-Path $harnessDir 'SAM.Analytical.OpenStudio.InputReadBack.dll'); commit = $heads[$RunnerRepository]; compiled = $true }, + [ordered]@{ path = (Join-Path $harnessDir 'SAM.Analytical.OpenStudio.dll'); commit = $heads[$RunnerRepository]; compiled = $true }, + # The SAM_Tas evaluation core is compiled by the harness build (project reference), so it carries the runner's stamp; + # its source commit is the clean SAM_Tas HEAD recorded under commits. + [ordered]@{ path = (Join-Path $harnessDir 'SAM.Analytical.InputEquivalence.dll'); commit = $heads[$RunnerRepository]; compiled = $true }, + [ordered]@{ path = (Join-Path $ValidationRepository 'build\SAM.Analytical.Benchmark.dll'); commit = $heads[$ValidationRepository]; compiled = $true } +) + @($samProjects | ForEach-Object { [ordered]@{ path = (Join-Path $samBuild "$_.dll"); commit = $heads[$SamRepository]; compiled = $true } }) +foreach ($artifact in $declared) { + $item = Get-Item -LiteralPath $artifact.path -ErrorAction SilentlyContinue + if (-not $item) { Refuse "the build did not produce $($artifact.path)" } + if ($item.LastWriteTimeUtc -lt $started) { Refuse "$($artifact.path) was not written by this build" } + if ($artifact.compiled) { + if (-not (Test-Stamp $artifact.path $artifact.commit)) { Refuse "$($artifact.path) does not carry the stamp +$($artifact.commit)" } + $stamped.Add([ordered]@{ path = $item.FullName; commit = $artifact.commit }) + } +} + +foreach ($library in @(Get-ChildItem -LiteralPath $benchmarkDir, $harnessDir -Filter 'SAM.*.dll' | Where-Object { Test-Path -LiteralPath (Join-Path $samBuild $_.Name) })) { + if ($samProjects -notcontains $library.BaseName) { Refuse "$($library.FullName) is a SAM\build library this build does not rebuild" } + if ((Sha $library.FullName) -ne (Sha (Join-Path $samBuild $library.Name))) { Refuse "$($library.FullName) differs from $(Join-Path $samBuild $library.Name)" } +} + +$files = @(Get-ChildItem -LiteralPath $benchmarkDir, $harnessDir -File | Where-Object { $_.Extension -in '.dll', '.exe' } | Sort-Object FullName | ForEach-Object { [ordered]@{ path = $_.FullName; sha256 = (Sha $_.FullName) } }) + +[ordered]@{ + schema = 's11-build-receipt/1' + note = 'Written by Build-S11-Receipt.ps1 after a successful controlled build of clean checkouts; Run-S11-Results.ps1 verifies it and never writes it.' + startedUtc = $started.ToString('o'); completedUtc = (Get-Date).ToUniversalTime().ToString('o'); machine = $env:COMPUTERNAME + msbuild = $MSBuild; commands = $commands + commits = @($heads.Keys | ForEach-Object { [ordered]@{ repository = $_; commit = $heads[$_]; branch = (Git $_ rev-parse --abbrev-ref HEAD | Out-String).Trim() } }) + benchmarkExe = $benchmarkExe; harnessExe = $harnessExe + stamped = $stamped; files = $files +} | ConvertTo-Json -Depth 5 | Set-Content -Encoding utf8 $BuildReceipt + +Write-Host "receipt: $BuildReceipt ($($files.Count) files)" +exit 0 diff --git a/benchmark/SAM.Analytical.OpenStudio.ResultsReadBack/Classes/RunLayout.cs b/benchmark/SAM.Analytical.OpenStudio.ResultsReadBack/Classes/RunLayout.cs new file mode 100644 index 0000000..0d790ce --- /dev/null +++ b/benchmark/SAM.Analytical.OpenStudio.ResultsReadBack/Classes/RunLayout.cs @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: LGPL-3.0-or-later +// Copyright (c) 2020–2026 Michal Dengusiak & Jakub Ziolkowski and contributors + +using System.Collections.Generic; +using System.IO; +using System.Linq; + +namespace SAM.Analytical.OpenStudio.ResultsReadBack +{ + /// + /// The files of one S11 run directory: b1a.json and sam.json written by benchmark-openstudio, the EnergyPlus run folder + /// somewhere below it (exactly one eplusout.sql, with eplusout.eso and in.idf beside it), and the S11 outputs. + /// + public sealed class RunLayout + { + public const string B1aFile = "b1a.json"; + public const string SamResultsFile = "sam.json"; + public const string ReferenceFile = "reference.json"; + public const string SubjectFile = "subject.json"; + public const string NegativeFile = "negative.json"; + public const string CompareDirectory = "compare"; + + public string Directory { get; private set; } + public string B1a { get; private set; } + public string SamResults { get; private set; } + public string Sql { get; private set; } + public string Eso { get; private set; } + public string Idf { get; private set; } + public string Reference => Path.Combine(Directory, ReferenceFile); + public string Subject => Path.Combine(Directory, SubjectFile); + public string Compare => Path.Combine(Directory, CompareDirectory); + + /// The layout of ; lists every missing or ambiguous file. + public static RunLayout Resolve(string directory, out List problems) + { + problems = new List(); + RunLayout result = new RunLayout { Directory = Path.GetFullPath(directory) }; + if (!System.IO.Directory.Exists(result.Directory)) + { + problems.Add("run directory does not exist: " + result.Directory); + return result; + } + + result.B1a = Path.Combine(result.Directory, B1aFile); + result.SamResults = Path.Combine(result.Directory, SamResultsFile); + foreach (string file in new[] { result.B1a, result.SamResults }) + { + if (!File.Exists(file)) + { + problems.Add("missing " + file); + } + } + + List sql = System.IO.Directory.GetFiles(result.Directory, "eplusout.sql", SearchOption.AllDirectories).OrderBy(x => x, System.StringComparer.Ordinal).ToList(); + if (sql.Count != 1) + { + problems.Add(sql.Count + " eplusout.sql files below " + result.Directory + " (exactly one is required)"); + return result; + } + + result.Sql = sql[0]; + string runFolder = Path.GetDirectoryName(result.Sql); + result.Eso = Path.Combine(runFolder, "eplusout.eso"); + result.Idf = Path.Combine(runFolder, "in.idf"); + foreach (string file in new[] { result.Eso, result.Idf }) + { + if (!File.Exists(file)) + { + problems.Add("missing " + file + " beside eplusout.sql"); + } + } + + int esoCount = System.IO.Directory.GetFiles(result.Directory, "eplusout.eso", SearchOption.AllDirectories).Length; + if (esoCount != 1) + { + problems.Add(esoCount + " eplusout.eso files below " + result.Directory + " (exactly one is required)"); + } + + return result; + } + } +} diff --git a/benchmark/SAM.Analytical.OpenStudio.ResultsReadBack/Classes/SubjectDocument.cs b/benchmark/SAM.Analytical.OpenStudio.ResultsReadBack/Classes/SubjectDocument.cs new file mode 100644 index 0000000..b5b0900 --- /dev/null +++ b/benchmark/SAM.Analytical.OpenStudio.ResultsReadBack/Classes/SubjectDocument.cs @@ -0,0 +1,159 @@ +// SPDX-License-Identifier: LGPL-3.0-or-later +// Copyright (c) 2020–2026 Michal Dengusiak & Jakub Ziolkowski and contributors + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.Json.Nodes; + +namespace SAM.Analytical.OpenStudio.ResultsReadBack +{ + /// One SAM SpaceSimulationResult as the subject saw it (parameter values, identity, relation). + public sealed class SubjectResult + { + public Guid ResultGuid { get; set; } + public string Name { get; set; } + public string Reference { get; set; } + public string Source { get; set; } + public string LoadType { get; set; } + public double? Load { get; set; } + public int? LoadIndex { get; set; } + public double? UnmetHours { get; set; } + public double? DesignLoad { get; set; } + + /// The SAM space the result is related to in the cluster; null when unattached. + public Guid? AttachedTo { get; set; } + + public bool IsDesignLoad => DesignLoad.HasValue; + public bool IsAnnual => !DesignLoad.HasValue && Load.HasValue; + } + + /// The report keys the production naming gives one SAM space (cross-checked against the native identity). + public sealed class ProductionKey + { + public Guid SpaceGuid { get; set; } + public string SpaceName { get; set; } + public string IdealLoads { get; set; } + public string ThermalZone { get; set; } + } + + /// + /// subject.json: what the code under test produced, captured by the CLI's subject step (the only step that runs + /// production code): the closure of the captured producer objects onto the produced B1a, the production report keys, + /// and the SAM_UI/Grasshopper import (Modify.AddResults) of the same SQL on a fresh copy of the model. + /// + public sealed class SubjectDocument + { + public const string CurrentSchema = "s11-subject/1"; + + public string Schema { get; set; } = CurrentSchema; + public int ProcessId { get; set; } + public DateTime ProcessStartedUtc { get; set; } + public DateTime StartedUtc { get; set; } + public DateTime FinishedUtc { get; set; } + public string SubjectAssemblySha256 { get; set; } + public string ProducerAssemblySha256 { get; set; } + public string ImporterAssemblySha256 { get; set; } + public string ModelSha256 { get; set; } + public string B1aSha256 { get; set; } + public string SamResultsSha256 { get; set; } + public string SqlSha256Before { get; set; } + public string SqlSha256After { get; set; } + + /// True when the captured objects re-mapped offline give byte-identical B1a model and spaces sections. + public bool ClosureIdentical { get; set; } + public string ClosureDetail { get; set; } + public string ClosureModelSha256Produced { get; set; } + public string ClosureModelSha256Regenerated { get; set; } + public string ClosureSpacesSha256Produced { get; set; } + public string ClosureSpacesSha256Regenerated { get; set; } + + public List ProductionKeys { get; set; } = new List(); + public List SamI { get; set; } = new List(); + public List SamIDiagnostics { get; set; } = new List(); + + /// Set on a negative copy whose subject was mutated rather than captured. + public string Mutation { get; set; } + } + + /// + /// The captured producer objects (SAM-P) read straight from the --sam-results file with the SAM JSON + /// deserializer, independently of the subject step: the compare judges the very objects the closure tied to the B1a. + /// + public sealed class SamResults + { + public const string Schema = "s11-sam-results/1"; + + public AnalyticalModelSimulationResult ModelResult { get; set; } + public List SpaceResults { get; set; } = new List(); + public List DesignLoadResults { get; set; } = new List(); + public bool HeatingConsumptionKeyed { get; set; } + public bool CoolingConsumptionKeyed { get; set; } + + public static SamResults Read(string path) + { + JsonObject jsonObject = JsonNode.Parse(File.ReadAllText(path)) as JsonObject; + if (jsonObject == null || (string)jsonObject["schema"] != Schema) + { + throw new InvalidDataException(path + " is not a " + Schema + " document"); + } + + return new SamResults + { + HeatingConsumptionKeyed = (bool)jsonObject["heatingConsumptionKeyed"], + CoolingConsumptionKeyed = (bool)jsonObject["coolingConsumptionKeyed"], + ModelResult = jsonObject["modelResult"] is JsonObject model ? Core.Create.IJSAMObject(model) : null, + SpaceResults = Results(jsonObject["spaceResults"] as JsonArray), + DesignLoadResults = Results(jsonObject["designLoadResults"] as JsonArray), + }; + } + + private static List Results(JsonArray jsonArray) + { + return jsonArray == null + ? new List() + : jsonArray.OfType().Select(x => Core.Create.IJSAMObject(x)).ToList(); + } + + /// A SAM result's parameter values in the subject's neutral form. + public static SubjectResult Describe(SpaceSimulationResult result, Guid? attachedTo) + { + return new SubjectResult + { + ResultGuid = result.Guid, + Name = result.Name, + Reference = result.Reference, + Source = result.Source, + LoadType = result.TryGetValue(global::SAM.Analytical.SpaceSimulationResultParameter.LoadType, out string loadType) ? loadType : null, + Load = Double(result, global::SAM.Analytical.SpaceSimulationResultParameter.Load), + LoadIndex = Integer(result, global::SAM.Analytical.SpaceSimulationResultParameter.LoadIndex), + UnmetHours = Double(result, global::SAM.Analytical.SpaceSimulationResultParameter.UnmetHours), + DesignLoad = Double(result, global::SAM.Analytical.SpaceSimulationResultParameter.DesignLoad), + AttachedTo = attachedTo, + }; + } + + public static double? Double(Core.SAMObject sAMObject, Enum parameter) + { + return sAMObject != null && sAMObject.TryGetValue(parameter, out double value) ? value : (double?)null; + } + + /// An integer parameter, accepting a whole-number double (JSON round trips may store it so). + public static int? Integer(Core.SAMObject sAMObject, Enum parameter) + { + if (sAMObject == null) + { + return null; + } + + if (sAMObject.TryGetValue(parameter, out int value)) + { + return value; + } + + double? asDouble = Double(sAMObject, parameter); + return asDouble.HasValue && asDouble.Value == Math.Truncate(asDouble.Value) ? (int)asDouble.Value : (int?)null; + } + } +} diff --git a/benchmark/SAM.Analytical.OpenStudio.ResultsReadBack/Compare/ResultCompare.cs b/benchmark/SAM.Analytical.OpenStudio.ResultsReadBack/Compare/ResultCompare.cs new file mode 100644 index 0000000..9c09068 --- /dev/null +++ b/benchmark/SAM.Analytical.OpenStudio.ResultsReadBack/Compare/ResultCompare.cs @@ -0,0 +1,630 @@ +// SPDX-License-Identifier: LGPL-3.0-or-later +// Copyright (c) 2020–2026 Michal Dengusiak & Jakub Ziolkowski and contributors + +using SAM.Analytical.Benchmark; +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; + +namespace SAM.Analytical.OpenStudio.ResultsReadBack +{ + /// One compared row: layer|scope|loadType|quantity, PASS, FAIL, BLOCKED or INFO, with its evidence. + public sealed class ResultRow + { + public string Id { get; set; } + public string Layer { get; set; } + public string Scope { get; set; } + public string LoadType { get; set; } + public string Quantity { get; set; } + + /// R0 exact, R1 single value, R2 annual sum, Text, Identity. + public string Class { get; set; } + public string Status { get; set; } + public string Reason { get; set; } + public double? Reference { get; set; } + public double? Subject { get; set; } + public double? Delta { get; set; } + public double? Tolerance { get; set; } + + /// F3 discrimination: |reference(Total) - reference(Sensible)| / this row's tolerance (value rows of a Total quantity). + public double? Margin { get; set; } + + /// F3 discrimination for an hour row: the Sensible series peaks at another hour. + public bool? SensibleHourDiffers { get; set; } + } + + /// The per-fixture F3 discrimination summary the OD-7 real-data requirement is judged on (informational here). + public sealed class F3Discrimination + { + public double? CoolingConsumptionMargin { get; set; } + public double? BestSpaceCoolingPeakMargin { get; set; } + + /// Both margins at least 1000 x tolerance (OD-7 recommended requirement for one fixture). + public bool MeetsOd7 { get; set; } + public string Detail { get; set; } + } + + /// compare.json: the S11 verdict for one run. + public sealed class CompareDocument + { + public const string CurrentSchema = "s11-compare/1"; + + /// Carried in every S11 record (plan section 10). + public const string S10Limitation = "Phase B remains subject to S10's declared validation limitation (SAM|FixB|Heating|designDayPeak.hourOfDay BLOCKED); an S11 PASS is not a Phase B PASS."; + + public string Schema { get; set; } = CurrentSchema; + public int ProcessId { get; set; } + public DateTime ProcessStartedUtc { get; set; } + public string CompareAssemblySha256 { get; set; } + public string FixtureId { get; set; } + public string Verdict { get; set; } + public int ExitCode { get; set; } + public int Required { get; set; } + public int Passed { get; set; } + public int Failed { get; set; } + public int Blocked { get; set; } + public int Informational { get; set; } + public Dictionary Inputs { get; set; } = new Dictionary(StringComparer.Ordinal); + public List GlobalBlocks { get; set; } = new List(); + public F3Discrimination F3 { get; set; } = new F3Discrimination(); + public List Rows { get; set; } = new List(); + public List Notes { get; set; } = new List { S10Limitation }; + } + + /// + /// The S11 comparison (the CLI's compare step, a separate process). The reference is reference.json; the subject + /// layers are SAM-P (the captured producer objects, read from sam.json here), SAM-I (the import, from subject.json) and + /// B1a (the produced document). Every row is judged against the reference with the S11 classes; a row whose reference + /// depends on a blocked quantity is BLOCKED, never passed. + /// + public static class ResultCompare + { + private static readonly CultureInfo ci = CultureInfo.InvariantCulture; + private static readonly LoadKind[] Kinds = { LoadKind.Heating, LoadKind.Cooling }; + + public static double ToleranceR1(double reference) => 1e-12 * Math.Max(1, Math.Abs(reference)); + + public static double ToleranceR2(double reference) => 1e-9 * Math.Abs(reference) + 1e-9; + + public static CompareDocument Run(AnalyticalModel model, ReferenceDocument reference, SubjectDocument subject, BenchmarkDocument b1a, SamResults samP) + { + CompareDocument result = new CompareDocument + { + ProcessId = Environment.ProcessId, + ProcessStartedUtc = InputReadBack.ProcessIdentity.CurrentStartedUtc(), + CompareAssemblySha256 = Sha256.Of(typeof(ResultCompare).Assembly.Location), + FixtureId = reference.FixtureId, + }; + + result.GlobalBlocks.AddRange(reference.GlobalBlocks); + Context context = new Context { Model = model, Reference = reference, Subject = subject, B1a = b1a, SamP = samP, Document = result }; + context.Spaces = model?.AdjacencyCluster?.GetSpaces() ?? new List(); + context.Sizing = Sizing(reference, b1a); + + ReferenceChecks(context); + Identity(context); + Closure(context); + SamPModel(context); + foreach (Space space in context.Spaces) + { + foreach (LoadKind kind in Kinds) + { + SpaceRows(context, "samP", space, kind, SamPAnnual(context, space, kind), SamPDesign(context, space, kind)); + SpaceRows(context, "samI", space, kind, SamIAnnual(context, space, kind), SamIDesign(context, space, kind)); + B1aSpace(context, space, kind); + } + } + + SamIUnattached(context); + B1aModel(context); + F3(context); + Verdict(result); + return result; + } + + private sealed class Context + { + public AnalyticalModel Model; + public ReferenceDocument Reference; + public SubjectDocument Subject; + public BenchmarkDocument B1a; + public SamResults SamP; + public CompareDocument Document; + public List Spaces; + + /// Null when sizing is consistent with the run's design-day basis (S11-C8); otherwise why not. + public string Sizing; + } + + // ---- dependencies ----------------------------------------------------------------------------------------------- + + /// The reasons a row that depends on these quantities must block (empty when none). + private static List Blocks(Context context, IEnumerable spaces, IEnumerable variables, LoadKind? basis) + { + List result = new List(context.Reference.GlobalBlocks); + foreach (Guid guid in spaces) + { + SpaceReference space = context.Reference.Space(guid); + if (space == null) + { + result.Add("space " + guid.ToString("N") + " has no reference"); + continue; + } + + if (space.IdentityProblem != null) + { + result.Add("identity " + guid.ToString("N") + ": " + space.IdentityProblem); + } + + foreach (string variable in variables) + { + if (space.Quantities.TryGetValue(variable, out QuantityReference quantity) && !quantity.Available) + { + result.AddRange(quantity.BlockReasons.Select(x => guid.ToString("N") + " " + x)); + } + } + + if (basis.HasValue && space.BasisProblems.TryGetValue(basis.Value.ToString(), out string problem)) + { + result.Add(guid.ToString("N") + " " + problem); + } + } + + return result.Distinct().ToList(); + } + + private static string[] Energy(LoadKind kind) => new[] { S11Names.Total(kind), S11Names.Sensible(kind) }; + + private static IEnumerable All(Context context) => context.Spaces.Select(x => x.Guid); + + // ---- rows ------------------------------------------------------------------------------------------------------- + + private static ResultRow Row(Context context, string layer, string scope, string loadType, string quantity, string @class) + { + ResultRow row = new ResultRow { Layer = layer, Scope = scope, LoadType = loadType ?? "-", Quantity = quantity, Class = @class }; + row.Id = layer + "|" + scope + "|" + row.LoadType + "|" + quantity; + context.Document.Rows.Add(row); + return row; + } + + private static void Status(ResultRow row, bool pass, string reason) + { + row.Status = pass ? "PASS" : "FAIL"; + row.Reason = reason; + } + + /// BLOCKED when any reason is given; true when the row was blocked. + private static bool Block(ResultRow row, IEnumerable reasons) + { + List list = reasons.ToList(); + if (list.Count == 0) + { + return false; + } + + row.Status = "BLOCKED"; + row.Reason = string.Join("; ", list.Take(4)) + (list.Count > 4 ? "; (+" + (list.Count - 4) + " more)" : string.Empty); + return true; + } + + /// A value row: the subject must be present and within the class tolerance of the reference. + private static void Value(ResultRow row, double reference, double? subject, string @class) + { + row.Reference = reference; + row.Subject = subject; + if (!subject.HasValue) + { + Status(row, false, "the subject has no value"); + return; + } + + double tolerance = @class == "R2" ? ToleranceR2(reference) : @class == "R1" ? ToleranceR1(reference) : 0; + double delta = subject.Value - reference; + row.Delta = delta; + row.Tolerance = tolerance; + bool pass = !double.IsNaN(subject.Value) && Math.Abs(delta) <= tolerance; + Status(row, pass, pass ? null : "|delta| " + Math.Abs(delta).ToString("R", ci) + " > " + @class + " tolerance " + tolerance.ToString("R", ci)); + } + + /// F3 margin of a value row: how many tolerances the Sensible value lies from the Total reference. + private static void Margin(ResultRow row, double? sensible) + { + if (row.Reference.HasValue && row.Tolerance.HasValue && row.Tolerance.Value > 0 && sensible.HasValue) + { + row.Margin = Math.Abs(row.Reference.Value - sensible.Value) / row.Tolerance.Value; + } + } + + // ---- reference and identity ------------------------------------------------------------------------------------- + + private static void ReferenceChecks(Context context) + { + ResultRow row = Row(context, "reference", "model", null, "checks", "Identity"); + List blocked = context.Reference.Checks.Where(x => x.Status == "BLOCKED").ToList(); + if (!Block(row, context.Reference.GlobalBlocks.Concat(blocked.Select(x => x.Id + " " + x.Scope + ": " + x.Detail)))) + { + Status(row, true, context.Reference.Checks.Count(x => x.Status == "PASS") + " checks PASS, " + context.Reference.Checks.Count(x => x.Status == "INFO") + " informational"); + } + } + + private static void Identity(Context context) + { + foreach (Space space in context.Spaces) + { + string scope = "space:" + space.Guid.ToString("N"); + SpaceReference reference = context.Reference.Space(space.Guid); + ResultRow native = Row(context, "identity", scope, null, "native", "Identity"); + if (!Block(native, reference == null ? new[] { "no reference" } : reference.IdentityProblem == null ? new string[0] : new[] { reference.IdentityProblem })) + { + Status(native, true, "'" + space.Name + "' = native space '" + reference.NativeSpace + "', zone '" + reference.Zone + "', Ideal Loads '" + reference.IdealLoads + "' (geometry, " + reference.SurfacePairs + " surfaces)"); + } + + ProductionKey key = context.Subject.ProductionKeys.FirstOrDefault(x => x.SpaceGuid == space.Guid); + foreach (string facet in new[] { "idealLoadsKey", "zoneKey" }) + { + ResultRow row = Row(context, "identity", scope, null, facet, "Identity"); + if (Block(row, reference?.IdentityProblem == null && reference != null ? new string[0] : new[] { "no native identity" })) + { + continue; + } + + string nativeKey = facet == "idealLoadsKey" ? reference.IdealLoads : reference.Zone; + string productionKey = key == null ? null : facet == "idealLoadsKey" ? key.IdealLoads : key.ThermalZone; + bool pass = productionKey != null && string.Equals(nativeKey, productionKey, StringComparison.OrdinalIgnoreCase); + Status(row, pass, (pass ? "production key = native key '" : "production key '" + productionKey + "' is not the native key '") + nativeKey + (pass ? "'" : "'")); + } + } + + ResultRow set = Row(context, "identity", "model", null, "b1aSpaceSet", "Identity"); + List expected = context.Spaces.Select(x => x.Guid.ToString("N")).OrderBy(x => x, StringComparer.Ordinal).ToList(); + List actual = (context.B1a?.Spaces ?? new List()).Select(x => x.Guid).OrderBy(x => x, StringComparer.Ordinal).ToList(); + Status(set, expected.SequenceEqual(actual), expected.SequenceEqual(actual) ? actual.Count + " B1a spaces = the SAM spaces" : "B1a spaces [" + string.Join(",", actual) + "] are not the SAM spaces [" + string.Join(",", expected) + "]"); + + ResultRow provenance = Row(context, "identity", "model", null, "b1aProvenance", "Identity"); + BenchmarkProvenance p = context.B1a?.Provenance; + List problems = new List(); + if (p == null) problems.Add("no provenance"); + else + { + if (p.Route != BenchmarkRoute.NativeOpenStudio) problems.Add("route " + p.Route); + if (p.State != RunState.Success) problems.Add("state " + p.State); + if (p.ResultSources == null || p.ResultSources.Count != 1 || p.ResultSources[0] != S11Names.ResultSource) problems.Add("resultSources [" + string.Join(",", p.ResultSources ?? new List()) + "]"); + } + + Status(provenance, problems.Count == 0, problems.Count == 0 ? "Native-OpenStudio, Success, source " + S11Names.ResultSource : string.Join("; ", problems)); + } + + private static void Closure(Context context) + { + ResultRow row = Row(context, "closure", "model", null, "samPToB1a", "Identity"); + Status(row, context.Subject.ClosureIdentical, context.Subject.ClosureDetail); + } + + /// Every SAM-P row also blocks when the captured objects are not shown to be the ones that produced the B1a. + private static IEnumerable ClosureBlock(Context context, string layer) + { + return layer == "samP" && !context.Subject.ClosureIdentical ? new[] { "closure: the captured objects do not re-map to the produced B1a" } : new string[0]; + } + + // ---- SAM-P model ------------------------------------------------------------------------------------------------ + + private static void SamPModel(Context context) + { + AnalyticalModelSimulationResult model = context.SamP?.ModelResult; + foreach (LoadKind kind in Kinds) + { + ModelLoadReference reference = context.Reference.ModelOf(kind); + List blocks = Blocks(context, All(context), Energy(kind), kind).Concat(ClosureBlock(context, "samP")).ToList(); + bool heating = kind == LoadKind.Heating; + + ResultRow consumption = Row(context, "samP", "model", kind.ToString(), "consumption", "R2"); + if (!Block(consumption, blocks.Concat(reference.MeterProblem == null ? new string[0] : new[] { reference.MeterProblem }))) + { + Value(consumption, reference.Total.ConsumptionKwh, SamResults.Double(model, heating ? AnalyticalModelSimulationResultParameter.ConsumptionHeating : AnalyticalModelSimulationResultParameter.ConsumptionCooling), "R2"); + Margin(consumption, reference.Sensible.Available ? reference.Sensible.ConsumptionKwh : (double?)null); + } + + ResultRow peak = Row(context, "samP", "model", kind.ToString(), "peakLoad", "R1"); + if (!Block(peak, blocks)) + { + Value(peak, reference.Total.CoincidentPeakKw, SamResults.Double(model, heating ? AnalyticalModelSimulationResultParameter.PeakHeatingLoad : AnalyticalModelSimulationResultParameter.PeakCoolingLoad), "R1"); + Margin(peak, reference.Sensible.Available ? reference.Sensible.CoincidentPeakKw : (double?)null); + } + + // SAM's own rule: earliest tie; an all-zero year ties everywhere, so its earliest hour 0 is expected exactly. + ResultRow hour = Row(context, "samP", "model", kind.ToString(), "peakHour", "R0"); + if (!Block(hour, blocks)) + { + Value(hour, reference.Total.PeakHour, SamResults.Integer(model, heating ? AnalyticalModelSimulationResultParameter.PeakHeatingHour : AnalyticalModelSimulationResultParameter.PeakCoolingHour), "R0"); + hour.SensibleHourDiffers = reference.Sensible.Available ? reference.Sensible.PeakHour != reference.Total.PeakHour : (bool?)null; + } + } + + ResultRow area = Row(context, "samP", "model", null, "floorArea", "R1"); + if (!Block(area, ClosureBlock(context, "samP"))) + { + Value(area, context.Reference.FloorArea, SamResults.Double(model, AnalyticalModelSimulationResultParameter.FloorArea), "R1"); + } + + ResultRow volume = Row(context, "samP", "model", null, "volume", "R1"); + if (!Block(volume, ClosureBlock(context, "samP"))) + { + Value(volume, context.Reference.Volume, SamResults.Double(model, AnalyticalModelSimulationResultParameter.Volume), "R1"); + } + } + + // ---- space rows (SAM-P and SAM-I) ------------------------------------------------------------------------------- + + private static List SamPAnnual(Context context, Space space, LoadKind kind) + { + return (context.SamP?.SpaceResults ?? new List()).Select(x => SamResults.Describe(x, null)).Where(x => Matches(x, space, kind) && x.IsAnnual).ToList(); + } + + private static List SamPDesign(Context context, Space space, LoadKind kind) + { + return (context.SamP?.DesignLoadResults ?? new List()).Select(x => SamResults.Describe(x, null)).Where(x => Matches(x, space, kind) && x.IsDesignLoad).ToList(); + } + + private static List SamIAnnual(Context context, Space space, LoadKind kind) + { + return context.Subject.SamI.Where(x => Matches(x, space, kind) && x.IsAnnual).ToList(); + } + + private static List SamIDesign(Context context, Space space, LoadKind kind) + { + return context.Subject.SamI.Where(x => Matches(x, space, kind) && x.IsDesignLoad).ToList(); + } + + private static bool Matches(SubjectResult result, Space space, LoadKind kind) + { + return string.Equals(result.Reference, space.Guid.ToString("N"), StringComparison.OrdinalIgnoreCase) && string.Equals(result.LoadType, kind.ToString(), StringComparison.Ordinal); + } + + private static void SpaceRows(Context context, string layer, Space space, LoadKind kind, List annual, List design) + { + string scope = "space:" + space.Guid.ToString("N"); + SpaceReference reference = context.Reference.Space(space.Guid); + List closure = ClosureBlock(context, layer).ToList(); + List energy = Blocks(context, new[] { space.Guid }, Energy(kind), kind).Concat(closure).ToList(); + + // The annual result object exists because the Total series exists: whether it should exist, and the not-met + // hours it carries, can only be judged when the reference has that series. + List exists = Blocks(context, new[] { space.Guid }, new[] { S11Names.Total(kind) }, null).Concat(closure).ToList(); + List unmet = Blocks(context, new[] { space.Guid }, new[] { S11Names.Unmet(kind), S11Names.Total(kind) }, null).Concat(closure).ToList(); + + // Exactly one annual result per space and load type, from the OpenStudio source; SAM-I also attached to its space. + ResultRow one = Row(context, layer, scope, kind.ToString(), "result", "Identity"); + SubjectResult result = annual.Count == 1 ? annual[0] : null; + if (!Block(one, exists)) + { + List problems = new List(); + if (annual.Count != 1) problems.Add(annual.Count + " annual results (exactly one is required)"); + if (result != null && result.Source != S11Names.ResultSource) problems.Add("source '" + result.Source + "'"); + if (layer == "samI" && result != null && result.AttachedTo != space.Guid) problems.Add("not attached to its space"); + Status(one, problems.Count == 0, problems.Count == 0 ? "one result, source " + S11Names.ResultSource + (layer == "samI" ? ", attached" : string.Empty) : string.Join("; ", problems)); + } + + QuantityReference total = reference?.Quantities.TryGetValue(S11Names.Total(kind), out QuantityReference t) == true ? t : null; + QuantityReference sensible = reference?.Quantities.TryGetValue(S11Names.Sensible(kind), out QuantityReference s) == true ? s : null; + QuantityReference unmetReference = reference?.Quantities.TryGetValue(S11Names.Unmet(kind), out QuantityReference u) == true ? u : null; + + ResultRow peak = Row(context, layer, scope, kind.ToString(), "peakLoad", "R1"); + if (!Block(peak, energy)) + { + Value(peak, total.PeakWatts, result?.Load, "R1"); + Margin(peak, sensible?.Available == true ? sensible.PeakWatts : (double?)null); + } + + ResultRow hour = Row(context, layer, scope, kind.ToString(), "peakHour", "R0"); + if (!Block(hour, energy)) + { + Value(hour, total.PeakHour, result?.LoadIndex, "R0"); + hour.SensibleHourDiffers = sensible?.Available == true ? sensible.PeakHour != total.PeakHour : (bool?)null; + } + + ResultRow unmetRow = Row(context, layer, scope, kind.ToString(), "unmetHours", "R2"); + if (!Block(unmetRow, unmet)) + { + Value(unmetRow, unmetReference.Sum, result?.UnmetHours, "R2"); + } + + DesignRow(context, layer, scope, kind, reference, design.Select(x => x.DesignLoad).ToList(), closure); + } + + /// + /// The design load: absent when the run did no sizing (no ZoneSizes); otherwise ZoneSizes.CalcDesLoad of the space's + /// zone [W]. For SAM-I (the legacy design-day import path) only availability is gated; its value is informational. + /// + private static void DesignRow(Context context, string layer, string scope, LoadKind kind, SpaceReference reference, List designLoads, List closure) + { + ResultRow row = Row(context, layer, scope, kind.ToString(), "designLoad", "R0"); + List blocks = new List(context.Reference.GlobalBlocks.Concat(closure)); + if (context.Sizing != null) blocks.Add(context.Sizing); + if (reference?.IdentityProblem != null) blocks.Add("identity: " + reference.IdentityProblem); + if (Block(row, blocks)) + { + return; + } + + List sizes = context.Reference.ZoneSizes.Where(x => string.Equals(x.ZoneName, reference.Zone, StringComparison.OrdinalIgnoreCase) && string.Equals(x.LoadType, kind.ToString(), StringComparison.OrdinalIgnoreCase)).ToList(); + if (sizes.Count == 0) + { + Status(row, designLoads.Count == 0, designLoads.Count == 0 ? "no sizing: design load absent" : designLoads.Count + " design load(s) present without a ZoneSizes row"); + return; + } + + if (layer == "samI") + { + row.Status = "INFO"; + row.Reason = designLoads.Count + " SAM-I design load(s) beside " + sizes.Count + " ZoneSizes row(s): the legacy design-day import path is out of S11 scope"; + return; + } + + row.Class = "R1"; + if (sizes.Count > 1 || designLoads.Count != 1) + { + Status(row, false, sizes.Count + " ZoneSizes rows and " + designLoads.Count + " design load results (exactly one of each is required)"); + return; + } + + Value(row, sizes[0].CalcDesLoad, designLoads[0], "R1"); + } + + private static void SamIUnattached(Context context) + { + ResultRow row = Row(context, "samI", "model", null, "unattached", "Identity"); + HashSet spaces = new HashSet(context.Spaces.Select(x => x.Guid)); + List unattached = context.Subject.SamI.Where(x => !x.AttachedTo.HasValue || !spaces.Contains(x.AttachedTo.Value)).ToList(); + Status(row, unattached.Count == 0, unattached.Count == 0 ? context.Subject.SamI.Count + " space results, all attached" : unattached.Count + " unattached result(s), e.g. '" + unattached[0].Name + "'"); + } + + // ---- B1a -------------------------------------------------------------------------------------------------------- + + private static void B1aModel(Context context) + { + BenchmarkModelResult model = context.B1a?.Model; + foreach (LoadKind kind in Kinds) + { + ModelLoadReference reference = context.Reference.ModelOf(kind); + List blocks = Blocks(context, All(context), Energy(kind), kind); + bool heating = kind == LoadKind.Heating; + + ResultRow consumption = Row(context, "b1a", "model", kind.ToString(), "consumption", "R2"); + if (!Block(consumption, blocks.Concat(reference.MeterProblem == null ? new string[0] : new[] { reference.MeterProblem }))) + { + Value(consumption, reference.Total.ConsumptionKwh, Available(heating ? model?.ConsumptionHeating : model?.ConsumptionCooling), "R2"); + Margin(consumption, reference.Sensible.Available ? reference.Sensible.ConsumptionKwh : (double?)null); + } + + PeakRows(context, "model", kind, heating ? model?.PeakHeatingLoad : model?.PeakCoolingLoad, heating ? model?.PeakHeatingHour : model?.PeakCoolingHour, blocks, reference.Total.CoincidentPeakKw, reference.Total.PeakHour, reference.Total.AllZero, reference.Sensible.Available ? reference.Sensible.CoincidentPeakKw : (double?)null, reference.Sensible.Available ? reference.Sensible.PeakHour : (int?)null); + } + + Value(Row(context, "b1a", "model", null, "floorArea", "R1"), context.Reference.FloorArea, Available(model?.FloorArea), "R1"); + Value(Row(context, "b1a", "model", null, "volume", "R1"), context.Reference.Volume, Available(model?.Volume), "R1"); + + ResultRow note = Row(context, "b1a", "model", null, "totalBasisNote", "Text"); + bool hasNote = context.B1a?.Provenance?.Notes?.Contains(S11Names.TotalBasisNote) == true; + Status(note, hasNote, hasNote ? "notes carry the Total-basis label" : "notes do not carry the Total-basis label '" + S11Names.TotalBasisNote + "'"); + + ResultRow sizing = Row(context, "b1a", "model", null, "designDaySource", "Identity"); + if (!Block(sizing, context.Sizing == null ? new string[0] : new[] { context.Sizing })) + { + Status(sizing, true, "designDaySource " + context.B1a?.Provenance?.DesignDaySource + " with " + context.Reference.ZoneSizes.Count + " ZoneSizes rows"); + } + } + + private static void B1aSpace(Context context, Space space, LoadKind kind) + { + string scope = "space:" + space.Guid.ToString("N"); + BenchmarkSpaceResult b1aSpace = context.B1a?.Spaces?.FirstOrDefault(x => string.Equals(x.Guid, space.Guid.ToString("N"), StringComparison.OrdinalIgnoreCase)); + BenchmarkConditionResult condition = kind == LoadKind.Heating ? b1aSpace?.Heating : b1aSpace?.Cooling; + SpaceReference reference = context.Reference.Space(space.Guid); + QuantityReference total = reference?.Quantities.TryGetValue(S11Names.Total(kind), out QuantityReference t) == true ? t : null; + QuantityReference sensible = reference?.Quantities.TryGetValue(S11Names.Sensible(kind), out QuantityReference s) == true ? s : null; + QuantityReference unmet = reference?.Quantities.TryGetValue(S11Names.Unmet(kind), out QuantityReference u) == true ? u : null; + + List blocks = Blocks(context, new[] { space.Guid }, Energy(kind), kind); + PeakRows(context, scope, kind, condition?.PeakLoad, condition?.PeakHour, blocks, total?.PeakWatts ?? double.NaN, total?.PeakHour ?? -1, total?.AllZero == true, sensible?.Available == true ? sensible.PeakWatts : (double?)null, sensible?.Available == true ? sensible.PeakHour : (int?)null); + + ResultRow unmetRow = Row(context, "b1a", scope, kind.ToString(), "unmetHours", "R2"); + if (!Block(unmetRow, Blocks(context, new[] { space.Guid }, new[] { S11Names.Unmet(kind), S11Names.Total(kind) }, null))) + { + Value(unmetRow, unmet.Sum, Available(condition?.UnmetHours), "R2"); + } + + List design = condition?.DesignLoad?.Available == true ? new List { condition.DesignLoad.Value } : new List(); + DesignRow(context, "b1a", scope, kind, reference, design, new List()); + } + + /// + /// The B1a peak and its hour under the S11 zero rule: an all-zero series is a load of 0 with NO hour; otherwise the + /// load and the earliest peak hour. + /// + private static void PeakRows(Context context, string scope, LoadKind kind, MetricValue load, MetricValue hour, List blocks, double reference, int referenceHour, bool allZero, double? sensible, int? sensibleHour) + { + ResultRow peak = Row(context, "b1a", scope, kind.ToString(), "peakLoad", "R1"); + if (!Block(peak, blocks)) + { + Value(peak, reference, Available(load), "R1"); + Margin(peak, sensible); + } + + ResultRow hourRow = Row(context, "b1a", scope, kind.ToString(), "peakHour", "R0"); + if (Block(hourRow, blocks)) + { + return; + } + + if (allZero) + { + bool unavailable = hour != null && !hour.Available; + hourRow.Subject = Available(hour); + Status(hourRow, unavailable, unavailable ? "all-zero year: hour unavailable (zero rule)" : "all-zero year but the B1a publishes hour " + Available(hour)?.ToString("R", ci) + " (the zero rule requires it unavailable)"); + return; + } + + Value(hourRow, referenceHour, Available(hour), "R0"); + hourRow.SensibleHourDiffers = sensibleHour.HasValue ? sensibleHour.Value != referenceHour : (bool?)null; + } + + private static double? Available(MetricValue metric) + { + return metric != null && metric.Available && metric.Value.HasValue ? metric.Value.Value : (double?)null; + } + + /// S11-C8: ZoneSizes present exactly when the run had a design-day basis. + private static string Sizing(ReferenceDocument reference, BenchmarkDocument b1a) + { + DesignDaySource? source = b1a?.Provenance?.DesignDaySource; + if (source == null) + { + return "S11-C8: the B1a states no designDaySource"; + } + + if (source == DesignDaySource.None && reference.ZoneSizes.Count > 0) + { + return "S11-C8: " + reference.ZoneSizes.Count + " ZoneSizes rows although designDaySource is None"; + } + + if (source != DesignDaySource.None && reference.ZoneSizes.Count == 0) + { + return "S11-C8: designDaySource " + source + " but no ZoneSizes rows"; + } + + return null; + } + + // ---- F3 and verdict --------------------------------------------------------------------------------------------- + + private static void F3(Context context) + { + F3Discrimination f3 = context.Document.F3; + f3.CoolingConsumptionMargin = context.Document.Rows.FirstOrDefault(x => x.Id == "b1a|model|Cooling|consumption")?.Margin; + f3.BestSpaceCoolingPeakMargin = context.Document.Rows.Where(x => x.Layer == "b1a" && x.Scope.StartsWith("space:", StringComparison.Ordinal) && x.LoadType == "Cooling" && x.Quantity == "peakLoad" && x.Margin.HasValue).Select(x => x.Margin).DefaultIfEmpty(null).Max(); + f3.MeetsOd7 = f3.CoolingConsumptionMargin >= 1000 && f3.BestSpaceCoolingPeakMargin >= 1000; + f3.Detail = f3.MeetsOd7 + ? "Total and Sensible cooling are distinguishable on this run (consumption and at least one space peak >= 1000 x tolerance)" + : "Total and Sensible cooling are NOT distinguishable to 1000 x tolerance on this run (consumption margin " + Text(f3.CoolingConsumptionMargin) + ", best space peak margin " + Text(f3.BestSpaceCoolingPeakMargin) + ")"; + } + + private static string Text(double? value) => value.HasValue ? value.Value.ToString("G6", ci) : "n/a"; + + private static void Verdict(CompareDocument document) + { + List gated = document.Rows.Where(x => x.Status != "INFO").ToList(); + document.Required = gated.Count; + document.Passed = gated.Count(x => x.Status == "PASS"); + document.Failed = gated.Count(x => x.Status == "FAIL"); + document.Blocked = gated.Count(x => x.Status == "BLOCKED"); + document.Informational = document.Rows.Count - gated.Count; + if (gated.Any(x => x.Status == null)) + { + throw new InvalidOperationException("row without a status: " + gated.First(x => x.Status == null).Id); + } + + document.Verdict = document.Failed > 0 ? "FAIL" : document.Blocked > 0 ? "BLOCKED" : "PASS"; + document.ExitCode = document.Failed > 0 ? InputReadBack.ReadBackExit.Fail : document.Blocked > 0 ? InputReadBack.ReadBackExit.BlockedTechnical : InputReadBack.ReadBackExit.Pass; + } + } +} diff --git a/benchmark/SAM.Analytical.OpenStudio.ResultsReadBack/Native/EsoResultRead.cs b/benchmark/SAM.Analytical.OpenStudio.ResultsReadBack/Native/EsoResultRead.cs new file mode 100644 index 0000000..57bd386 --- /dev/null +++ b/benchmark/SAM.Analytical.OpenStudio.ResultsReadBack/Native/EsoResultRead.cs @@ -0,0 +1,292 @@ +// SPDX-License-Identifier: LGPL-3.0-or-later +// Copyright (c) 2020–2026 Michal Dengusiak & Jakub Ziolkowski and contributors + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; + +namespace SAM.Analytical.OpenStudio.ResultsReadBack +{ + /// One ESO data-dictionary entry. + public sealed class EsoVariable + { + public int Id { get; set; } + public string Key { get; set; } + public string Name { get; set; } + public string Units { get; set; } + public string Frequency { get; set; } + public bool IsMeter { get; set; } + } + + /// + /// The ESO channel: eplusout.eso read as text by its own parser (it shares no reader code with the SQL channel). The + /// data dictionary maps report ids to key, variable, units and frequency. In the data section, record 1 opens an + /// environment, record 2 stamps the following values (an hourly stamp has StartMinute 0 and EndMinute 60), and each + /// Hourly variable must appear exactly once after each hourly stamp of the RunPeriod environment. + /// + public sealed class EsoResultRead + { + private static readonly CultureInfo ci = CultureInfo.InvariantCulture; + + public string Path { get; set; } + public string Sha256Before { get; set; } + public string Sha256After { get; set; } + + /// The first line of the file (EnergyPlus version and build). + public string ProgramVersion { get; set; } + + public List Environments { get; set; } = new List(); + + /// How many environment blocks carry the RunPeriod's name (exactly one is required). + public int RunPeriodBlocks { get; set; } + + public Dictionary Series { get; set; } = new Dictionary(StringComparer.Ordinal); + public List Meters { get; set; } = new List(); + public List Problems { get; set; } = new List(); + + public static EsoResultRead Read(string esoPath, RunCalendar calendar) + { + EsoResultRead result = new EsoResultRead { Path = System.IO.Path.GetFullPath(esoPath), Sha256Before = Sha256.Of(esoPath) }; + try + { + Parse(File.ReadAllLines(esoPath), calendar, result); + } + catch (IOException exception) + { + result.Problems.Add("eplusout.eso could not be read: " + exception.Message); + } + + result.Sha256After = Sha256.Of(esoPath); + return result; + } + + public static void Parse(string[] lines, RunCalendar calendar, EsoResultRead result) + { + HashSet wanted = new HashSet(S11Names.IdealLoadsVariables.Concat(S11Names.ZoneVariables), StringComparer.OrdinalIgnoreCase); + HashSet meters = new HashSet(S11Names.Meters, StringComparer.OrdinalIgnoreCase); + Dictionary seriesById = new Dictionary(); + Dictionary meterById = new Dictionary(); + + int i = 0; + if (lines.Length == 0) + { + result.Problems.Add("eplusout.eso is empty"); + return; + } + + result.ProgramVersion = lines[0].Trim(); + bool dictionaryEnded = false; + for (i = 1; i < lines.Length; i++) + { + string line = lines[i]; + if (line.StartsWith("End of Data Dictionary", StringComparison.Ordinal)) + { + dictionaryEnded = true; + i++; + break; + } + + EsoVariable variable = DictionaryLine(line); + if (variable == null || variable.Id <= 6) + { + continue; + } + + if (!variable.IsMeter && wanted.Contains(variable.Name)) + { + // Every dictionary entry of a required variable is judged: Hourly entries are the series, any other + // reporting frequency for the same key is unsupported and blocks it (its values are not read). + string seriesKey = S11Names.SeriesKey(Canonical(variable.Name), variable.Key); + bool hourly = string.Equals(variable.Frequency, "Hourly", StringComparison.OrdinalIgnoreCase); + if (!result.Series.TryGetValue(seriesKey, out ChannelSeries series)) + { + series = new ChannelSeries { Channel = "eso", Variable = Canonical(variable.Name), Key = variable.Key, Units = hourly ? variable.Units : null }; + result.Series[seriesKey] = series; + } + + if (!hourly) + { + series.Problem("S11-C1", "unsupported reporting frequency '" + variable.Frequency + "' (report id " + variable.Id + "; only Hourly is permitted)"); + continue; + } + + series.DictionaryEntries++; + if (series.Units == null) + { + series.Units = variable.Units; + } + else if (!string.Equals(series.Units, variable.Units, StringComparison.Ordinal)) + { + series.Units = series.Units + "," + variable.Units; + } + + seriesById[variable.Id] = series; + } + else if (variable.IsMeter && meters.Contains(variable.Name) && !result.Meters.Any(x => string.Equals(x.Name, variable.Name, StringComparison.OrdinalIgnoreCase))) + { + // The finest frequency comes first in the dictionary only by convention; any one entry sums the run. + MeterSum sum = new MeterSum { Name = variable.Name, Frequency = variable.Frequency }; + result.Meters.Add(sum); + meterById[variable.Id] = sum; + } + } + + if (!dictionaryEnded) + { + result.Problems.Add("eplusout.eso has no 'End of Data Dictionary' line"); + return; + } + + bool inRunPeriod = false; + bool hourlyStamp = false; + HourKey stamp = default(HourKey); + int dst = 0; + HashSet seenInBlock = new HashSet(); + bool dataEnded = false; + for (; i < lines.Length; i++) + { + string line = lines[i]; + if (line.StartsWith("End of Data", StringComparison.Ordinal)) + { + dataEnded = true; + break; + } + + string[] fields = line.Split(','); + if (!int.TryParse(fields[0].Trim(), NumberStyles.Integer, ci, out int id)) + { + result.Problems.Add("eplusout.eso data line " + (i + 1) + " does not start with a report id"); + continue; + } + + if (id == 1) + { + string name = fields.Length > 1 ? fields[1].Trim() : string.Empty; + result.Environments.Add(name); + inRunPeriod = calendar.RunPeriodName != null && string.Equals(name, calendar.RunPeriodName, StringComparison.OrdinalIgnoreCase); + if (inRunPeriod) + { + result.RunPeriodBlocks++; + } + + hourlyStamp = false; + seenInBlock.Clear(); + continue; + } + + if (id == 2) + { + // 2,DayOfSimulation,Month,DayOfMonth,DST,Hour,StartMinute,EndMinute,DayType + seenInBlock.Clear(); + hourlyStamp = false; + if (fields.Length < 8 + || !int.TryParse(fields[2].Trim(), NumberStyles.Integer, ci, out int month) + || !int.TryParse(fields[3].Trim(), NumberStyles.Integer, ci, out int day) + || !int.TryParse(fields[4].Trim(), NumberStyles.Integer, ci, out dst) + || !int.TryParse(fields[5].Trim(), NumberStyles.Integer, ci, out int hour) + || !double.TryParse(fields[6].Trim(), NumberStyles.Float, ci, out double startMinute) + || !double.TryParse(fields[7].Trim(), NumberStyles.Float, ci, out double endMinute)) + { + result.Problems.Add("eplusout.eso line " + (i + 1) + " is not a readable time stamp"); + continue; + } + + hourlyStamp = startMinute == 0 && endMinute == 60; + stamp = new HourKey(month, day, hour); + continue; + } + + if (id <= 6) + { + // Daily, monthly, run-period and annual stamps: no hourly value may follow them. + seenInBlock.Clear(); + hourlyStamp = false; + continue; + } + + if (!inRunPeriod) + { + continue; + } + + if (seriesById.TryGetValue(id, out ChannelSeries series)) + { + if (!hourlyStamp) + { + series.Problem("S11-C1", "a value of id " + id + " follows a non-hourly stamp (line " + (i + 1) + ")"); + continue; + } + + if (!seenInBlock.Add(id)) + { + series.Problem("S11-C1", "id " + id + " appears more than once after the hourly stamp " + stamp); + } + + if (dst != 0) + { + series.Problem("S11-C2", "the hourly stamp " + stamp + " has DST " + dst + " (0 is required)"); + } + + double value = fields.Length > 1 && double.TryParse(fields[1].Trim(), NumberStyles.Float, ci, out double parsed) ? parsed : double.NaN; + series.Points.Add(new HourlyPoint { Month = stamp.Month, Day = stamp.Day, Hour = stamp.Hour, Value = value }); + } + else if (meterById.TryGetValue(id, out MeterSum meter) && fields.Length > 1 && double.TryParse(fields[1].Trim(), NumberStyles.Float, ci, out double meterValue)) + { + meter.Rows++; + meter.Sum += meterValue; + } + } + + if (!dataEnded) + { + result.Problems.Add("eplusout.eso has no 'End of Data' line (the run may not have finished)"); + } + } + + /// The S11 spelling of a variable name read case-insensitively. + private static string Canonical(string name) + { + return S11Names.IdealLoadsVariables.Concat(S11Names.ZoneVariables).First(x => string.Equals(x, name, StringComparison.OrdinalIgnoreCase)); + } + + /// + /// "id,count,KEY,Variable Name [Units] !Frequency" for a variable, "id,count,Meter:Name [Units] !Frequency" for a + /// meter. Null for a line this reader does not recognise. + /// + public static EsoVariable DictionaryLine(string line) + { + int bang = line.IndexOf('!'); + string frequency = bang >= 0 ? line.Substring(bang + 1).Trim() : null; + string body = bang >= 0 ? line.Substring(0, bang) : line; + if (frequency != null) + { + int space = frequency.IndexOf(' '); + frequency = space >= 0 ? frequency.Substring(0, space) : frequency; + } + + string[] parts = body.Split(new[] { ',' }, 3); + if (parts.Length < 3 || !int.TryParse(parts[0].Trim(), NumberStyles.Integer, ci, out int id)) + { + return null; + } + + string rest = parts[2].Trim(); + string key = null; + int comma = rest.IndexOf(','); + int bracket = rest.IndexOf('['); + if (comma >= 0 && (bracket < 0 || comma < bracket)) + { + key = rest.Substring(0, comma).Trim(); + rest = rest.Substring(comma + 1).Trim(); + bracket = rest.IndexOf('['); + } + + string name = bracket >= 0 ? rest.Substring(0, bracket).Trim() : rest; + int close = bracket >= 0 ? rest.IndexOf(']', bracket) : -1; + string units = bracket >= 0 && close > bracket ? rest.Substring(bracket + 1, close - bracket - 1).Trim() : null; + return new EsoVariable { Id = id, Key = key ?? string.Empty, Name = name, Units = units, Frequency = frequency, IsMeter = key == null }; + } + } +} diff --git a/benchmark/SAM.Analytical.OpenStudio.ResultsReadBack/Native/HourlySeries.cs b/benchmark/SAM.Analytical.OpenStudio.ResultsReadBack/Native/HourlySeries.cs new file mode 100644 index 0000000..a767c1f --- /dev/null +++ b/benchmark/SAM.Analytical.OpenStudio.ResultsReadBack/Native/HourlySeries.cs @@ -0,0 +1,195 @@ +// SPDX-License-Identifier: LGPL-3.0-or-later +// Copyright (c) 2020–2026 Michal Dengusiak & Jakub Ziolkowski and contributors + +using System; +using System.Collections.Generic; +using System.Linq; + +namespace SAM.Analytical.OpenStudio.ResultsReadBack +{ + /// The EnergyPlus output names S11 reads, and the label the producer must carry (an independent copy of its text). + public static class S11Names + { + public const string TotalHeating = "Zone Ideal Loads Supply Air Total Heating Energy"; + public const string TotalCooling = "Zone Ideal Loads Supply Air Total Cooling Energy"; + public const string SensibleHeating = "Zone Ideal Loads Supply Air Sensible Heating Energy"; + public const string SensibleCooling = "Zone Ideal Loads Supply Air Sensible Cooling Energy"; + public const string UnmetHeating = "Zone Heating Setpoint Not Met Time"; + public const string UnmetCooling = "Zone Cooling Setpoint Not Met Time"; + public const string HeatingMeter = "DistrictHeatingWater:Facility"; + public const string CoolingMeter = "DistrictCooling:Facility"; + + /// The F3 basis label every successful OpenStudio B1a document must carry (S11 plan section 2). + public const string TotalBasisNote = "Cooling metrics are EnergyPlus Ideal Loads Supply Air Total cooling (sensible + latent); heating is Total heating."; + + /// The SAM result source the OpenStudio importer stamps on its results. + public const string ResultSource = "SAM.Analytical.OpenStudio"; + + /// Ideal Loads variables: keyed by the ZoneHVAC:IdealLoadsAirSystem name, unit J. + public static readonly string[] IdealLoadsVariables = { TotalHeating, TotalCooling, SensibleHeating, SensibleCooling }; + + /// Zone variables: keyed by the IDF Zone name, unit hr. + public static readonly string[] ZoneVariables = { UnmetHeating, UnmetCooling }; + + public static readonly string[] Meters = { HeatingMeter, CoolingMeter }; + + public static string UnitOf(string variable) => ZoneVariables.Contains(variable) ? "hr" : "J"; + + public static string Total(LoadKind kind) => kind == LoadKind.Heating ? TotalHeating : TotalCooling; + + public static string Sensible(LoadKind kind) => kind == LoadKind.Heating ? SensibleHeating : SensibleCooling; + + public static string Unmet(LoadKind kind) => kind == LoadKind.Heating ? UnmetHeating : UnmetCooling; + + public static string Meter(LoadKind kind) => kind == LoadKind.Heating ? HeatingMeter : CoolingMeter; + + /// The series key of a variable and a report key (EnergyPlus keys are case-insensitive). + public static string SeriesKey(string variable, string key) => variable + "|" + (key ?? string.Empty).ToUpperInvariant(); + } + + public enum LoadKind + { + Heating, + Cooling + } + + /// One value of an hourly series as a file stamps it: month, day, EnergyPlus hour 1-24 (interval end). + public sealed class HourlyPoint + { + public int Month { get; set; } + public int Day { get; set; } + public int Hour { get; set; } + public double Value { get; set; } + + public HourKey Key => new HourKey(Month, Day, Hour); + } + + /// A structural problem one channel found in one series, with the S11 check it belongs to. + public sealed class SeriesProblem + { + public string Check { get; set; } + public string Message { get; set; } + } + + /// + /// One hourly series (variable x key) as ONE channel (SQL or ESO) holds it in the RunPeriod environment, in file order. + /// Channel-specific readers fill it; judges it against the calendar alone. + /// + public sealed class ChannelSeries + { + public string Channel { get; set; } + public string Variable { get; set; } + public string Key { get; set; } + public string Units { get; set; } + + /// The Hourly data-dictionary entries for this variable and key (exactly one is required). + public int DictionaryEntries { get; set; } + + public List Points { get; set; } = new List(); + public List Problems { get; set; } = new List(); + + public void Problem(string check, string message, int limit = 5) + { + if (Problems.Count(x => x.Check == check) < limit) + { + Problems.Add(new SeriesProblem { Check = check, Message = message }); + } + } + } + + /// Completeness of one series in one file against the expected calendar (S11-C1). + public sealed class CompletenessResult + { + public string Channel { get; set; } + public string Variable { get; set; } + public string Key { get; set; } + public int DictionaryEntries { get; set; } + public int Expected { get; set; } + public int Found { get; set; } + public int Missing { get; set; } + public int Duplicated { get; set; } + public int Extra { get; set; } + public List Examples { get; set; } = new List(); + public List Problems { get; set; } = new List(); + public bool Complete { get; set; } + + /// The value of each expected hour (by 0-based hour of year) when ; otherwise null. + [System.Text.Json.Serialization.JsonIgnore] + public double[] Values { get; set; } + } + + public static class Completeness + { + /// + /// The multiset of the series' timestamps must equal the calendar exactly: each expected hour once, none missing, + /// none repeated, none outside the calendar. A count of 8760 alone is never accepted (a repeated hour and a + /// missing one keep the count). Every structural problem the channel reported also makes the series incomplete. + /// + public static CompletenessResult Check(RunCalendar calendar, ChannelSeries series) + { + CompletenessResult result = new CompletenessResult + { + Channel = series.Channel, + Variable = series.Variable, + Key = series.Key, + DictionaryEntries = series.DictionaryEntries, + Expected = calendar.Count, + Found = series.Points.Count, + }; + + result.Problems.AddRange(series.Problems.Select(x => x.Check + ": " + x.Message)); + if (series.DictionaryEntries != 1) + { + result.Problems.Add("S11-C1: " + series.DictionaryEntries + " Hourly data-dictionary entries (exactly one is required)"); + } + + if (calendar.Problem != null) + { + result.Problems.Add("calendar: " + calendar.Problem); + return result; + } + + int[] counts = new int[calendar.Count]; + double[] values = new double[calendar.Count]; + foreach (HourlyPoint point in series.Points) + { + int hour = calendar.IndexOf(point.Key); + if (hour < 0) + { + result.Extra++; + Example(result, "extra " + point.Key); + continue; + } + + counts[hour]++; + values[hour] = point.Value; + } + + for (int i = 0; i < counts.Length; i++) + { + if (counts[i] == 0) + { + result.Missing++; + Example(result, "missing " + calendar.Hours[i]); + } + else if (counts[i] > 1) + { + result.Duplicated++; + Example(result, "duplicated x" + counts[i] + " " + calendar.Hours[i]); + } + } + + result.Complete = result.Problems.Count == 0 && result.Missing == 0 && result.Duplicated == 0 && result.Extra == 0; + result.Values = result.Complete ? values : null; + return result; + } + + private static void Example(CompletenessResult result, string text) + { + if (result.Examples.Count < 6) + { + result.Examples.Add(text); + } + } + } +} diff --git a/benchmark/SAM.Analytical.OpenStudio.ResultsReadBack/Native/RunCalendar.cs b/benchmark/SAM.Analytical.OpenStudio.ResultsReadBack/Native/RunCalendar.cs new file mode 100644 index 0000000..842c690 --- /dev/null +++ b/benchmark/SAM.Analytical.OpenStudio.ResultsReadBack/Native/RunCalendar.cs @@ -0,0 +1,149 @@ +// SPDX-License-Identifier: LGPL-3.0-or-later +// Copyright (c) 2020–2026 Michal Dengusiak & Jakub Ziolkowski and contributors + +using SAM.Analytical.OpenStudio.InputReadBack; +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; + +namespace SAM.Analytical.OpenStudio.ResultsReadBack +{ + /// One expected hourly interval: month, day and EnergyPlus hour 1-24 (the interval END), and its 0-based hour of year. + public readonly struct HourKey : IEquatable + { + public HourKey(int month, int day, int hour) + { + Month = month; + Day = day; + Hour = hour; + } + + public int Month { get; } + public int Day { get; } + public int Hour { get; } + + public bool Equals(HourKey other) => Month == other.Month && Day == other.Day && Hour == other.Hour; + + public override bool Equals(object obj) => obj is HourKey other && Equals(other); + + public override int GetHashCode() => (Month * 32 + Day) * 32 + Hour; + + public override string ToString() => Month.ToString(CultureInfo.InvariantCulture) + "/" + Day.ToString(CultureInfo.InvariantCulture) + " h" + Hour.ToString(CultureInfo.InvariantCulture); + } + + /// + /// The expected hourly calendar of the run, built from the single RunPeriod of the run's in.idf (the input EnergyPlus + /// ran), never from the result data. S11 requires a full non-leap year 1/1-12/31; anything else is a + /// and every row blocks. Hour h of the year (0-based) is the interval h:00-h+1:00, which + /// EnergyPlus stamps with its END hour h mod 24 + 1 on that day. + /// + public sealed class RunCalendar + { + private static readonly int[] DaysInMonth = { 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 }; + + private readonly Dictionary index = new Dictionary(); + + public string RunPeriodName { get; private set; } + + /// The run period's evidence as read (begin/end month/day/year). + public string Evidence { get; private set; } + + /// Null when the calendar is valid; otherwise why no calendar can be defined. + public string Problem { get; private set; } + + /// The expected hourly keys in order (8760 when valid, empty otherwise). + public List Hours { get; } = new List(); + + public int Count => Hours.Count; + + /// The 0-based hour of year of , or -1 when the key is not in the calendar. + public int IndexOf(HourKey key) => index.TryGetValue(key, out int value) ? value : -1; + + public static RunCalendar FromIdf(IEnumerable idf) + { + RunCalendar result = new RunCalendar(); + List runPeriods = (idf ?? Enumerable.Empty()).Where(x => string.Equals(x.Type, "RunPeriod", StringComparison.OrdinalIgnoreCase)).ToList(); + if (runPeriods.Count != 1) + { + result.Problem = "in.idf has " + runPeriods.Count + " RunPeriod objects (exactly one is required)"; + return result; + } + + IdfObject runPeriod = runPeriods[0]; + result.RunPeriodName = runPeriod.Name; + int? beginMonth = Integer(runPeriod, 1, "Begin Month", out string p1); + int? beginDay = Integer(runPeriod, 2, "Begin Day of Month", out string p2); + int? beginYear = Integer(runPeriod, 3, "Begin Year", out string p3); + int? endMonth = Integer(runPeriod, 4, "End Month", out string p4); + int? endDay = Integer(runPeriod, 5, "End Day of Month", out string p5); + int? endYear = Integer(runPeriod, 6, "End Year", out string p6); + string problem = new[] { p1, p2, p3, p4, p5, p6 }.FirstOrDefault(x => x != null); + result.Evidence = string.Format(CultureInfo.InvariantCulture, "RunPeriod '{0}': {1}/{2}/{3} to {4}/{5}/{6}", runPeriod.Name, beginMonth, beginDay, beginYear?.ToString(CultureInfo.InvariantCulture) ?? "(no year)", endMonth, endDay, endYear?.ToString(CultureInfo.InvariantCulture) ?? "(no year)"); + if (problem != null) + { + result.Problem = problem; + return result; + } + + if (string.IsNullOrWhiteSpace(runPeriod.Name)) + { + result.Problem = "the RunPeriod has no name, so its environment cannot be identified"; + } + else if (beginMonth != 1 || beginDay != 1 || endMonth != 12 || endDay != 31) + { + result.Problem = result.Evidence + " is not the full year 1/1-12/31"; + } + else if (beginYear.HasValue && DateTime.IsLeapYear(beginYear.Value)) + { + result.Problem = result.Evidence + " is a leap year (S11 requires 8760 hours)"; + } + else if (beginYear.HasValue && endYear.HasValue && beginYear.Value != endYear.Value) + { + result.Problem = result.Evidence + " spans more than one year"; + } + else if (!beginYear.HasValue && endYear.HasValue) + { + result.Problem = result.Evidence + " states an end year without a begin year"; + } + + if (result.Problem != null) + { + return result; + } + + for (int month = 1; month <= 12; month++) + { + for (int day = 1; day <= DaysInMonth[month - 1]; day++) + { + for (int hour = 1; hour <= 24; hour++) + { + HourKey key = new HourKey(month, day, hour); + result.index[key] = result.Hours.Count; + result.Hours.Add(key); + } + } + } + + return result; + } + + /// A blank field is null (EnergyPlus default); text that is not an integer is a problem. + private static int? Integer(IdfObject o, int field, string label, out string problem) + { + string text = o.Value(field, label, out problem); + if (problem != null || string.IsNullOrWhiteSpace(text)) + { + return null; + } + + if (int.TryParse(text, NumberStyles.Integer, CultureInfo.InvariantCulture, out int value)) + { + return value; + } + + problem = "RunPeriod '" + o.Name + "' " + label + " is not an integer: '" + text + "'"; + return null; + } + } +} diff --git a/benchmark/SAM.Analytical.OpenStudio.ResultsReadBack/Native/SqlResultRead.cs b/benchmark/SAM.Analytical.OpenStudio.ResultsReadBack/Native/SqlResultRead.cs new file mode 100644 index 0000000..d8c2ac5 --- /dev/null +++ b/benchmark/SAM.Analytical.OpenStudio.ResultsReadBack/Native/SqlResultRead.cs @@ -0,0 +1,306 @@ +// SPDX-License-Identifier: LGPL-3.0-or-later +// Copyright (c) 2020–2026 Michal Dengusiak & Jakub Ziolkowski and contributors + +using System; +using System.Collections.Generic; +using System.Data.SQLite; +using System.Globalization; +using System.IO; +using System.Linq; + +namespace SAM.Analytical.OpenStudio.ResultsReadBack +{ + /// One EnvironmentPeriods row. + public sealed class SqlEnvironment + { + public long Index { get; set; } + public string Name { get; set; } + public long Type { get; set; } + } + + /// A facility meter's sum over the RunPeriod environment as one channel reports it. + public sealed class MeterSum + { + public string Name { get; set; } + public string Frequency { get; set; } + public int Rows { get; set; } + public double Sum { get; set; } + } + + /// One ZoneSizes row (design-day sizing outcome). + public sealed class ZoneSizeRow + { + public string ZoneName { get; set; } + public string LoadType { get; set; } + public double CalcDesLoad { get; set; } + } + + /// + /// The SQL channel: eplusout.sql opened read-only with raw System.Data.SQLite (no OpenStudio SqlFile, no production + /// reader). Every dictionary entry of the S11 variables is judged (any frequency other than Hourly blocks its key), and + /// every Hourly series, for every key, is read row by row with its Time stamp; nothing is + /// summed or filtered by SQL, so a missing, repeated or foreign row stays visible to . + /// + public sealed class SqlResultRead + { + public string Path { get; set; } + public string Sha256Before { get; set; } + public string Sha256After { get; set; } + public List EnergyPlusVersions { get; set; } = new List(); + public List Environments { get; set; } = new List(); + + /// + /// The RunPeriod environment: the ONLY EnvironmentType 3 (weather-run) environment, which must be named as the in.idf + /// RunPeriod; otherwise null with . The production reader reads every type-3 + /// environment, so a second one (whatever its name) would be summed there and must not be ignored here. + /// + public long? RunPeriodEnvironment { get; set; } + public string EnvironmentProblem { get; set; } + + /// Rows of the S11 series outside the RunPeriod environment (design days, sizing): ignored, counted. + public int OtherEnvironmentRows { get; set; } + + public Dictionary Series { get; set; } = new Dictionary(StringComparer.Ordinal); + public List Meters { get; set; } = new List(); + public List ZoneSizes { get; set; } = new List(); + public string Problem { get; set; } + + public static SqlResultRead Read(string sqlPath, RunCalendar calendar) + { + SqlResultRead result = new SqlResultRead { Path = System.IO.Path.GetFullPath(sqlPath), Sha256Before = Sha256.Of(sqlPath) }; + try + { + using (SQLiteConnection connection = new SQLiteConnection(new SQLiteConnectionStringBuilder { DataSource = sqlPath, ReadOnly = true, FailIfMissing = true }.ConnectionString)) + { + connection.Open(); + HashSet tables = new HashSet(Query(connection, "SELECT name FROM sqlite_master WHERE type = 'table'", null, r => r.GetString(0)), StringComparer.OrdinalIgnoreCase); + foreach (string table in new[] { "Simulations", "EnvironmentPeriods", "Time", "ReportDataDictionary", "ReportData" }) + { + if (!tables.Contains(table)) + { + result.Problem = "eplusout.sql has no " + table + " table"; + return result; + } + } + + result.EnergyPlusVersions = Query(connection, "SELECT EnergyPlusVersion FROM Simulations ORDER BY SimulationIndex", null, r => r.IsDBNull(0) ? null : r.GetString(0)); + result.Environments = Query(connection, "SELECT EnvironmentPeriodIndex, EnvironmentName, EnvironmentType FROM EnvironmentPeriods ORDER BY EnvironmentPeriodIndex", null, r => new SqlEnvironment { Index = r.GetInt64(0), Name = r.IsDBNull(1) ? null : r.GetString(1), Type = r.IsDBNull(2) ? -1 : r.GetInt64(2) }); + + List weatherRuns = result.Environments.Where(x => x.Type == 3).ToList(); + if (weatherRuns.Count == 1 && calendar.RunPeriodName != null && string.Equals(weatherRuns[0].Name, calendar.RunPeriodName, StringComparison.OrdinalIgnoreCase)) + { + result.RunPeriodEnvironment = weatherRuns[0].Index; + } + else + { + result.EnvironmentProblem = weatherRuns.Count + " SQL environment(s) of type 3 (exactly one, named '" + calendar.RunPeriodName + "', is required); environments: " + string.Join(", ", result.Environments.Select(x => x.Name + " (type " + x.Type + ")")); + } + + foreach (string variable in S11Names.IdealLoadsVariables.Concat(S11Names.ZoneVariables)) + { + ReadVariable(connection, result, variable); + } + + foreach (string meter in S11Names.Meters) + { + ReadMeter(connection, result, meter); + } + + if (tables.Contains("ZoneSizes")) + { + result.ZoneSizes = Query(connection, "SELECT ZoneName, LoadType, CalcDesLoad FROM ZoneSizes", null, r => new ZoneSizeRow { ZoneName = r.IsDBNull(0) ? null : r.GetString(0), LoadType = r.IsDBNull(1) ? null : r.GetString(1), CalcDesLoad = r.IsDBNull(2) ? double.NaN : r.GetDouble(2) }); + } + } + } + catch (Exception exception) when (exception is SQLiteException || exception is InvalidCastException || exception is IOException) + { + result.Problem = "eplusout.sql could not be read: " + exception.Message; + } + finally + { + SQLiteConnection.ClearAllPools(); + result.Sha256After = Sha256.Of(sqlPath); + } + + return result; + } + + private sealed class DictionaryEntry + { + public long Index; + public string Key; + public string Units; + public string Frequency; + } + + private static void ReadVariable(SQLiteConnection connection, SqlResultRead result, string variable) + { + List entries = Query(connection, "SELECT ReportDataDictionaryIndex, KeyValue, Units, ReportingFrequency FROM ReportDataDictionary WHERE Name = @name COLLATE NOCASE AND IsMeter = 0 ORDER BY ReportDataDictionaryIndex", new Dictionary { ["@name"] = variable }, r => new DictionaryEntry { Index = r.GetInt64(0), Key = r.IsDBNull(1) ? string.Empty : r.GetString(1), Units = r.IsDBNull(2) ? null : r.GetString(2), Frequency = r.IsDBNull(3) ? null : r.GetString(3) }); + // Every dictionary entry of a required variable is judged: the Hourly ones are the series, any other reporting + // frequency for the same key is unsupported and blocks it (the production reader does not filter frequency). + foreach (IGrouping group in entries.GroupBy(x => x.Key.ToUpperInvariant())) + { + List hourly = group.Where(x => string.Equals(x.Frequency, "Hourly", StringComparison.OrdinalIgnoreCase)).ToList(); + ChannelSeries series = new ChannelSeries + { + Channel = "sql", + Variable = variable, + Key = group.First().Key, + Units = string.Join(",", hourly.Select(x => x.Units).Distinct()), + DictionaryEntries = hourly.Count, + }; + + foreach (DictionaryEntry entry in group.Except(hourly)) + { + series.Problem("S11-C1", "unsupported reporting frequency '" + entry.Frequency + "' (ReportDataDictionaryIndex " + entry.Index + "; only Hourly is permitted)"); + } + + foreach (DictionaryEntry entry in hourly) + { + ReadRows(connection, result, series, entry.Index); + } + + result.Series[S11Names.SeriesKey(variable, series.Key)] = series; + } + } + + private static void ReadRows(SQLiteConnection connection, SqlResultRead result, ChannelSeries series, long dictionaryIndex) + { + HashSet timeIndexes = new HashSet(); + const string sql = "SELECT rd.TimeIndex, t.TimeIndex, t.Month, t.Day, t.Hour, t.Minute, t.Interval, t.Dst, t.WarmupFlag, t.EnvironmentPeriodIndex, rd.Value " + + "FROM ReportData rd LEFT JOIN Time t ON t.TimeIndex = rd.TimeIndex WHERE rd.ReportDataDictionaryIndex = @index ORDER BY rd.TimeIndex, rd.ReportDataIndex"; + using (SQLiteCommand command = connection.CreateCommand()) + { + command.CommandText = sql; + command.Parameters.AddWithValue("@index", dictionaryIndex); + using (SQLiteDataReader reader = command.ExecuteReader()) + { + while (reader.Read()) + { + long timeIndex = reader.GetInt64(0); + if (reader.IsDBNull(1)) + { + series.Problem("S11-C1", "ReportData row at TimeIndex " + timeIndex + " has no Time row"); + continue; + } + + long environment = reader.IsDBNull(9) ? -1 : reader.GetInt64(9); + if (result.RunPeriodEnvironment == null || environment != result.RunPeriodEnvironment.Value) + { + result.OtherEnvironmentRows++; + continue; + } + + if (!timeIndexes.Add(timeIndex)) + { + series.Problem("S11-C1", "TimeIndex " + timeIndex + " repeats in ReportData"); + } + + int month = Int(reader, 2); + int day = Int(reader, 3); + int hour = Int(reader, 4); + int minute = Int(reader, 5); + int interval = Int(reader, 6); + int dst = Int(reader, 7); + int warmup = Int(reader, 8); + string stamp = month + "/" + day + " h" + hour + ":" + minute.ToString("00", CultureInfo.InvariantCulture); + if (warmup != 0) + { + series.Problem("S11-C1", "warmup row in the RunPeriod environment at " + stamp); + continue; + } + + if (interval != 60 || minute != 0) + { + series.Problem("S11-C2", "row at " + stamp + " has Interval " + interval + " and Minute " + minute + " (an hourly row is Interval 60, Minute 0)"); + } + + if (dst != 0) + { + series.Problem("S11-C2", "row at " + stamp + " has Dst " + dst + " (0 is required)"); + } + + series.Points.Add(new HourlyPoint { Month = month, Day = day, Hour = hour, Value = reader.IsDBNull(10) ? double.NaN : reader.GetDouble(10) }); + } + } + } + } + + private static void ReadMeter(SQLiteConnection connection, SqlResultRead result, string meter) + { + List entries = Query(connection, "SELECT ReportDataDictionaryIndex, KeyValue, Units, ReportingFrequency FROM ReportDataDictionary WHERE Name = @name AND IsMeter = 1 ORDER BY ReportDataDictionaryIndex", new Dictionary { ["@name"] = meter }, r => new DictionaryEntry { Index = r.GetInt64(0), Key = r.IsDBNull(1) ? string.Empty : r.GetString(1), Units = r.IsDBNull(2) ? null : r.GetString(2), Frequency = r.IsDBNull(3) ? null : r.GetString(3) }); + + // The finest reported frequency sums every interval of the run once. + DictionaryEntry entry = entries.FirstOrDefault(x => string.Equals(x.Frequency, "Zone Timestep", StringComparison.OrdinalIgnoreCase)) + ?? entries.FirstOrDefault(x => string.Equals(x.Frequency, "Hourly", StringComparison.OrdinalIgnoreCase)); + if (entry == null || result.RunPeriodEnvironment == null) + { + return; + } + + MeterSum sum = new MeterSum { Name = meter, Frequency = entry.Frequency }; + const string sql = "SELECT rd.Value FROM ReportData rd JOIN Time t ON t.TimeIndex = rd.TimeIndex WHERE rd.ReportDataDictionaryIndex = @index AND t.EnvironmentPeriodIndex = @environment AND t.WarmupFlag = 0 ORDER BY rd.TimeIndex, rd.ReportDataIndex"; + foreach (double value in Query(connection, sql, new Dictionary { ["@index"] = entry.Index, ["@environment"] = result.RunPeriodEnvironment.Value }, r => r.GetDouble(0))) + { + sum.Rows++; + sum.Sum += value; + } + + result.Meters.Add(sum); + } + + private static int Int(SQLiteDataReader reader, int column) + { + return reader.IsDBNull(column) ? int.MinValue : System.Convert.ToInt32(reader.GetValue(column), CultureInfo.InvariantCulture); + } + + private static List Query(SQLiteConnection connection, string sql, Dictionary parameters, Func read) + { + List result = new List(); + using (SQLiteCommand command = connection.CreateCommand()) + { + command.CommandText = sql; + foreach (KeyValuePair parameter in parameters ?? new Dictionary()) + { + command.Parameters.AddWithValue(parameter.Key, parameter.Value); + } + + using (SQLiteDataReader reader = command.ExecuteReader()) + { + while (reader.Read()) + { + result.Add(read(reader)); + } + } + } + + return result; + } + } + + /// SHA-256 of a file in lowercase hexadecimal; null when the file does not exist. + public static class Sha256 + { + public static string Of(string path) + { + if (string.IsNullOrWhiteSpace(path) || !File.Exists(path)) + { + return null; + } + + using (FileStream stream = File.OpenRead(path)) + using (System.Security.Cryptography.SHA256 sha = System.Security.Cryptography.SHA256.Create()) + { + return string.Concat(sha.ComputeHash(stream).Select(x => x.ToString("x2", CultureInfo.InvariantCulture))); + } + } + + public static string OfBytes(byte[] bytes) + { + using (System.Security.Cryptography.SHA256 sha = System.Security.Cryptography.SHA256.Create()) + { + return string.Concat(sha.ComputeHash(bytes).Select(x => x.ToString("x2", CultureInfo.InvariantCulture))); + } + } + } +} diff --git a/benchmark/SAM.Analytical.OpenStudio.ResultsReadBack/Pack-S11-Evidence.ps1 b/benchmark/SAM.Analytical.OpenStudio.ResultsReadBack/Pack-S11-Evidence.ps1 new file mode 100644 index 0000000..32f3591 --- /dev/null +++ b/benchmark/SAM.Analytical.OpenStudio.ResultsReadBack/Pack-S11-Evidence.ps1 @@ -0,0 +1,56 @@ +# SPDX-License-Identifier: LGPL-3.0-or-later +# Copyright (c) 2020-2026 Michal Dengusiak & Jakub Ziolkowski and contributors + +<# +.SYNOPSIS + Copies the S11 acceptance evidence from -Root into the repository (documentation/S11-OpenStudio-ResultsRoundTrip-evidence). + +.DESCRIPTION + Copies the text evidence only: run manifests, b1a.json, sam.json, reference.json, subject.json, compare.json/.md, the + negatives' negative.json and compare output, console logs, the phase summaries and the build receipt, then writes + SHA256SUMS.txt over the copied files. eplusout.sql, eplusout.eso and the EnergyPlus work folders stay machine-local; the + manifests record their paths and hashes. +#> +param( + [string]$Root = 'C:\TasOut\s11\acc1', + [Parameter(Mandatory = $true)][string]$Label, + [string]$BuildReceipt = '' +) + +$ErrorActionPreference = 'Stop' +$repository = [IO.Path]::GetFullPath((Join-Path $PSScriptRoot '..\..')) +if (-not $BuildReceipt) { $BuildReceipt = Join-Path $repository 'benchmark\SAM.Analytical.OpenStudio.ResultsReadBack.Cli\bin\x64\Release\net8.0\s11-build-receipt.json' } +$target = Join-Path $repository "documentation\S11-OpenStudio-ResultsRoundTrip-evidence\$Label" +if (Test-Path -LiteralPath $target) { throw "The evidence folder exists already: $target" } + +function Sha([string]$Path) { + $stream = [IO.File]::OpenRead($Path) + try { + $sha = [Security.Cryptography.SHA256]::Create() + try { return ([BitConverter]::ToString($sha.ComputeHash($stream)) -replace '-', '').ToLowerInvariant() } finally { $sha.Dispose() } + } + finally { $stream.Dispose() } +} + +$include = @('run-manifest.json', 'b1a.json', 'sam.json', 'reference.json', 'subject.json', 'negative.json', 'compare.json', 'compare.md', 'summary-*.json', '*.console.txt', '*.readback.txt') +$copied = New-Object System.Collections.Generic.List[string] +foreach ($file in Get-ChildItem -LiteralPath $Root -Recurse -File) { + $relative = $file.FullName.Substring($Root.TrimEnd('\').Length + 1) + if ($relative -match '(^|\\)work\\') { continue } + $match = $false + foreach ($pattern in $include) { if ($file.Name -like $pattern) { $match = $true } } + if (-not $match) { continue } + $destination = Join-Path $target $relative + New-Item -ItemType Directory -Force -Path (Split-Path $destination) | Out-Null + Copy-Item -LiteralPath $file.FullName -Destination $destination + $copied.Add($relative) +} + +if (Test-Path -LiteralPath $BuildReceipt) { + Copy-Item -LiteralPath $BuildReceipt -Destination (Join-Path $target 's11-build-receipt.json') + $copied.Add('s11-build-receipt.json') +} + +$sums = @($copied | Sort-Object | ForEach-Object { (Sha (Join-Path $target $_)) + ' ' + ($_ -replace '\\', '/') }) +[IO.File]::WriteAllText((Join-Path $target 'SHA256SUMS.txt'), (($sums -join "`n") + "`n"), (New-Object Text.UTF8Encoding($false))) +Write-Host "packed $($copied.Count) files into $target" diff --git a/benchmark/SAM.Analytical.OpenStudio.ResultsReadBack/Reference/ResultReference.cs b/benchmark/SAM.Analytical.OpenStudio.ResultsReadBack/Reference/ResultReference.cs new file mode 100644 index 0000000..5fab0c5 --- /dev/null +++ b/benchmark/SAM.Analytical.OpenStudio.ResultsReadBack/Reference/ResultReference.cs @@ -0,0 +1,598 @@ +// SPDX-License-Identifier: LGPL-3.0-or-later +// Copyright (c) 2020–2026 Michal Dengusiak & Jakub Ziolkowski and contributors + +using SAM.Analytical.OpenStudio.InputReadBack; +using SAM.Analytical.Tas.DirectT3D.Validation.ReadBack; +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace SAM.Analytical.OpenStudio.ResultsReadBack +{ + public static class S11Json + { + public static readonly JsonSerializerOptions Options = new JsonSerializerOptions + { + WriteIndented = true, + NumberHandling = JsonNumberHandling.AllowNamedFloatingPointLiterals, + Converters = { new JsonStringEnumConverter() }, + }; + + public static void Save(T value, string path) + { + File.WriteAllText(path, JsonSerializer.Serialize(value, Options).Replace("\r\n", "\n") + "\n", new System.Text.UTF8Encoding(false)); + } + + public static T Load(string path) + { + return JsonSerializer.Deserialize(File.ReadAllText(path), Options); + } + } + + /// One S11 check (S11-C1..C9) as the reference read judged it. + public sealed class ReferenceCheck + { + public string Id { get; set; } + public string Scope { get; set; } + + /// PASS, BLOCKED or INFO (informational, never gating). + public string Status { get; set; } + public string Detail { get; set; } + } + + /// The reference value of one quantity (variable x zone) from the raw SQL series after S11-C1..C4. + public sealed class QuantityReference + { + public string Variable { get; set; } + public string Key { get; set; } + public bool Available { get; set; } + public List BlockReasons { get; set; } = new List(); + + /// Annual sum in the file unit (J or hr). + public double Sum { get; set; } + + /// Largest hourly value (J per hour, or hr). + public double Peak { get; set; } + + /// Peak power: the largest hourly energy / 3600 s [W] (energy variables only). + public double PeakWatts { get; set; } + + /// Earliest 0-based hour of year holding the largest value (SAM's own tie rule). + public int PeakHour { get; set; } = -1; + + /// Hours holding exactly the largest value. + public int TieCount { get; set; } + + /// Every hour is exactly 0. + public bool AllZero { get; set; } + } + + /// The reference of one SAM space: its native identity and its quantities. + public sealed class SpaceReference + { + public Guid SpaceGuid { get; set; } + public string SpaceName { get; set; } + public string NativeSpace { get; set; } + public string Zone { get; set; } + public string IdealLoads { get; set; } + public int SurfacePairs { get; set; } + public bool NameCarrierAgrees { get; set; } + public string IdentityProblem { get; set; } + + /// Area and volume as the SAM space states them (inputs, not engine values). + public double Area { get; set; } = double.NaN; + public double Volume { get; set; } = double.NaN; + + public Dictionary Quantities { get; set; } = new Dictionary(StringComparer.Ordinal); + + /// Per load kind, why the Total/Sensible basis cannot be trusted (S11-C6: negative latent); absent when it can. + public Dictionary BasisProblems { get; set; } = new Dictionary(StringComparer.Ordinal); + + /// Total minus Sensible over the year [J], per load kind (informational). + public Dictionary LatentJoules { get; set; } = new Dictionary(StringComparer.Ordinal); + } + + /// The model-level reference of one load kind for one basis (Total or Sensible). + public sealed class ModelBasisReference + { + public bool Available { get; set; } + public double ConsumptionKwh { get; set; } + + /// Coincident peak: the largest hourly sum over zones / 3600 s / 1000 [kW]. + public double CoincidentPeakKw { get; set; } + public int PeakHour { get; set; } = -1; + public int TieCount { get; set; } + public bool AllZero { get; set; } + } + + public sealed class ModelLoadReference + { + public LoadKind Kind { get; set; } + public ModelBasisReference Total { get; set; } = new ModelBasisReference(); + public ModelBasisReference Sensible { get; set; } = new ModelBasisReference(); + + /// Why the model consumption cannot be trusted (S11-C7 meter disagreement); null when it can. + public string MeterProblem { get; set; } + } + + /// reference.json: everything the fresh-process reference read established, with its inputs' hashes. + public sealed class ReferenceDocument + { + public const string CurrentSchema = "s11-reference/1"; + + public string Schema { get; set; } = CurrentSchema; + public int ProcessId { get; set; } + public DateTime ProcessStartedUtc { get; set; } + public DateTime StartedUtc { get; set; } + public DateTime FinishedUtc { get; set; } + public string ReaderAssemblySha256 { get; set; } + public string FixtureId { get; set; } + + public string ModelPath { get; set; } + public string ModelSha256 { get; set; } + public string IdfPath { get; set; } + public string IdfSha256Before { get; set; } + public string IdfSha256After { get; set; } + public string SqlPath { get; set; } + public string SqlSha256Before { get; set; } + public string SqlSha256After { get; set; } + public string EsoPath { get; set; } + public string EsoSha256Before { get; set; } + public string EsoSha256After { get; set; } + + public string RunPeriod { get; set; } + public int CalendarHours { get; set; } + public List SqlEnergyPlusVersions { get; set; } = new List(); + public string EsoProgramVersion { get; set; } + public List SqlEnvironments { get; set; } = new List(); + public List EsoEnvironments { get; set; } = new List(); + public int SqlOtherEnvironmentRows { get; set; } + + /// Reasons that block every row (calendar, environment, version, unreadable file, unknown keys). + public List GlobalBlocks { get; set; } = new List(); + + public List Checks { get; set; } = new List(); + public List Completeness { get; set; } = new List(); + public List Spaces { get; set; } = new List(); + public List Model { get; set; } = new List(); + public double FloorArea { get; set; } + public double Volume { get; set; } + public List SqlMeters { get; set; } = new List(); + public List EsoMeters { get; set; } = new List(); + public List ZoneSizes { get; set; } = new List(); + + public SpaceReference Space(Guid guid) => Spaces.FirstOrDefault(x => x.SpaceGuid == guid); + + public ModelLoadReference ModelOf(LoadKind kind) => Model.FirstOrDefault(x => x.Kind == kind); + } + + /// + /// The S11 reference read (the CLI's read step, a separate process). Inputs are the frozen SAM model (expected + /// spaces only), the run's in.idf, eplusout.sql and eplusout.eso. Nothing here calls the SAM_OpenStudio importer, + /// writer or naming: the space identity is the S09 native identity and the values are the raw SQL series. + /// + public static class ResultReference + { + /// The S11 class E tolerance (ESO text against SQL double, and Total >= Sensible). + public static double ToleranceE(double reference) => 1e-12 * Math.Max(1, Math.Abs(reference)); + + /// The S11 class R2 tolerance (sums over the year), in the row's unit. + public static double ToleranceR2(double reference) => 1e-9 * Math.Abs(reference) + 1e-9; + + public static ReferenceDocument Build(AnalyticalModel model, string modelPath, string idfPath, string sqlPath, string esoPath, string fixtureId) + { + ReferenceDocument result = new ReferenceDocument + { + ProcessId = Environment.ProcessId, + ProcessStartedUtc = ProcessIdentity.CurrentStartedUtc(), + StartedUtc = DateTime.UtcNow, + ReaderAssemblySha256 = Sha256.Of(typeof(ResultReference).Assembly.Location), + FixtureId = fixtureId, + ModelPath = modelPath == null ? null : Path.GetFullPath(modelPath), + ModelSha256 = Sha256.Of(modelPath), + IdfPath = Path.GetFullPath(idfPath), + IdfSha256Before = Sha256.Of(idfPath), + }; + + // Native IDF (the input EnergyPlus ran) as text; the calendar and the identity come from it alone. + List idf = IdfText.Parse(File.ReadAllText(idfPath)).Where(x => NativeReader.IdfTypes.Contains(x.Type, StringComparer.OrdinalIgnoreCase)).ToList(); + result.IdfSha256After = Sha256.Of(idfPath); + RunCalendar calendar = RunCalendar.FromIdf(idf); + result.RunPeriod = calendar.Evidence; + result.CalendarHours = calendar.Count; + Check(result, "S11-C1", "calendar", calendar.Problem == null, calendar.Problem ?? calendar.Evidence + ": " + calendar.Count + " expected hourly intervals"); + if (calendar.Problem != null) + { + result.GlobalBlocks.Add("calendar: " + calendar.Problem); + } + + SqlResultRead sql = SqlResultRead.Read(sqlPath, calendar); + EsoResultRead eso = EsoResultRead.Read(esoPath, calendar); + result.SqlPath = sql.Path; + result.SqlSha256Before = sql.Sha256Before; + result.SqlSha256After = sql.Sha256After; + result.EsoPath = eso.Path; + result.EsoSha256Before = eso.Sha256Before; + result.EsoSha256After = eso.Sha256After; + result.SqlEnergyPlusVersions = sql.EnergyPlusVersions; + result.EsoProgramVersion = eso.ProgramVersion; + result.SqlEnvironments = sql.Environments.Select(x => x.Name + " (type " + x.Type.ToString(CultureInfo.InvariantCulture) + ")").ToList(); + result.EsoEnvironments = eso.Environments; + result.SqlOtherEnvironmentRows = sql.OtherEnvironmentRows; + result.SqlMeters = sql.Meters; + result.EsoMeters = eso.Meters; + result.ZoneSizes = sql.ZoneSizes; + + Files(result, sql, eso); + Version(result, sql, eso); + + ExpectedModel expected = ReadBackExpected.Build(model, null, fixtureId); + NativeModel native = new NativeModel(new NativeSnapshot { Idf = idf }); + List identity = NativeZoneIdentity.Resolve(native, expected); + Identity(result, model, identity); + KeySets(result, identity, sql, eso); + Dictionary hourly = Quantities(result, calendar, sql, eso); + ModelValues(result, hourly); + Meters(result); + + Check(result, "S11-C8", "sizing", true, "ZoneSizes rows: " + result.ZoneSizes.Count + " (judged against designDaySource in compare)", "INFO"); + result.FinishedUtc = DateTime.UtcNow; + return result; + } + + private static void Check(ReferenceDocument document, string id, string scope, bool pass, string detail, string status = null) + { + document.Checks.Add(new ReferenceCheck { Id = id, Scope = scope, Status = status ?? (pass ? "PASS" : "BLOCKED"), Detail = detail }); + } + + private static void Files(ReferenceDocument document, SqlResultRead sql, EsoResultRead eso) + { + List problems = new List(); + if (sql.Problem != null) problems.Add(sql.Problem); + if (sql.EnvironmentProblem != null) problems.Add("sql: " + sql.EnvironmentProblem); + problems.AddRange(eso.Problems.Select(x => "eso: " + x)); + if (eso.RunPeriodBlocks != 1) problems.Add("eso: " + eso.RunPeriodBlocks + " environment blocks are named as the RunPeriod (exactly one is required); environments: " + string.Join(", ", eso.Environments)); + if (sql.Sha256Before != sql.Sha256After) problems.Add("eplusout.sql changed while it was read"); + if (eso.Sha256Before != eso.Sha256After) problems.Add("eplusout.eso changed while it was read"); + if (document.IdfSha256Before != document.IdfSha256After) problems.Add("in.idf changed while it was read"); + document.GlobalBlocks.AddRange(problems); + Check(document, "S11-C1", "files and RunPeriod environment", problems.Count == 0, problems.Count == 0 ? "SQL environment and one ESO RunPeriod block found; files unchanged by the read" : string.Join("; ", problems)); + } + + private static void Version(ReferenceDocument document, SqlResultRead sql, EsoResultRead eso) + { + bool sqlOk = sql.EnergyPlusVersions.Count == 1 && sql.EnergyPlusVersions[0] != null && sql.EnergyPlusVersions[0].Contains("Version 25.1."); + bool esoOk = eso.ProgramVersion != null && eso.ProgramVersion.Contains("Version 25.1."); + string detail = "SQL Simulations: " + string.Join(" | ", sql.EnergyPlusVersions) + "; ESO: " + eso.ProgramVersion; + Check(document, "S11-C9", "EnergyPlus version", sqlOk && esoOk, detail); + if (!(sqlOk && esoOk)) + { + document.GlobalBlocks.Add("S11-C9: EnergyPlus 25.1 is required in both files (" + detail + ")"); + } + } + + private static void Identity(ReferenceDocument document, AnalyticalModel model, List identity) + { + List spaces = model?.AdjacencyCluster?.GetSpaces() ?? new List(); + foreach (NativeZoneIdentityEntry entry in identity) + { + Space space = spaces.FirstOrDefault(x => x.Guid == entry.SpaceGuid); + SpaceReference reference = new SpaceReference + { + SpaceGuid = entry.SpaceGuid, + SpaceName = entry.SpaceName, + NativeSpace = entry.NativeSpace, + Zone = entry.Zone, + IdealLoads = entry.IdealLoads, + SurfacePairs = entry.SurfacePairs, + NameCarrierAgrees = entry.NameCarrierAgrees, + IdentityProblem = entry.Problem, + }; + + if (space != null && space.TryGetValue(SpaceParameter.Area, out double area)) reference.Area = area; + if (space != null && space.TryGetValue(SpaceParameter.Volume, out double volume)) reference.Volume = volume; + document.Spaces.Add(reference); + Check(document, "S11-C5", "identity " + entry.SpaceGuid.ToString("N"), entry.Problem == null, entry.Problem ?? ("'" + entry.SpaceName + "' = native space '" + entry.NativeSpace + "' (" + entry.SurfacePairs + " surfaces one to one), zone '" + entry.Zone + "', Ideal Loads '" + entry.IdealLoads + "'; name carrier " + (entry.NameCarrierAgrees ? "agrees" : "CONTRADICTS"))); + } + + // Floor area and volume of the model are the sums of the SAM space inputs (as SAM states them). + document.FloorArea = 0; + document.Volume = 0; + foreach (Space space in spaces) + { + if (space.TryGetValue(SpaceParameter.Area, out double area) && !double.IsNaN(area)) document.FloorArea += area; + if (space.TryGetValue(SpaceParameter.Volume, out double volume) && !double.IsNaN(volume)) document.Volume += volume; + } + + if (identity.Count == 0) + { + document.GlobalBlocks.Add("S11-C5: the model has no space to identify"); + } + } + + /// S11-C5: the report keys in each file are exactly the identity's Ideal Loads and Zone names. + private static void KeySets(ReferenceDocument document, List identity, SqlResultRead sql, EsoResultRead eso) + { + HashSet idealLoads = new HashSet(identity.Where(x => x.IdealLoads != null).Select(x => x.IdealLoads.ToUpperInvariant())); + HashSet zones = new HashSet(identity.Where(x => x.Zone != null).Select(x => x.Zone.ToUpperInvariant())); + List problems = new List(); + foreach (KeyValuePair> channel in new[] { new KeyValuePair>("sql", sql.Series), new KeyValuePair>("eso", eso.Series) }) + { + foreach (ChannelSeries series in channel.Value.Values) + { + HashSet expected = S11Names.ZoneVariables.Contains(series.Variable) ? zones : idealLoads; + if (!expected.Contains((series.Key ?? string.Empty).ToUpperInvariant())) + { + problems.Add(channel.Key + ": '" + series.Variable + "' key '" + series.Key + "' is not a native identity of any SAM space"); + } + } + } + + Check(document, "S11-C5", "report keys", problems.Count == 0, problems.Count == 0 ? "every SQL and ESO key is an identified Zone or Ideal Loads" : string.Join("; ", problems.Take(10))); + if (problems.Count > 0) + { + document.GlobalBlocks.Add("S11-C5: " + problems.Count + " unknown report key(s), e.g. " + problems[0]); + } + } + + /// + /// Judges every (space x variable) in both files and fills its reference values. Returns the hourly SQL values of + /// each available quantity for the model sums (reference.json carries the derived values, not the 8760-value arrays). + /// + private static Dictionary Quantities(ReferenceDocument document, RunCalendar calendar, SqlResultRead sql, EsoResultRead eso) + { + Dictionary hourly = new Dictionary(); + foreach (SpaceReference space in document.Spaces) + { + Dictionary values = new Dictionary(StringComparer.Ordinal); + foreach (string variable in S11Names.IdealLoadsVariables.Concat(S11Names.ZoneVariables)) + { + string key = S11Names.ZoneVariables.Contains(variable) ? space.Zone : space.IdealLoads; + QuantityReference quantity = new QuantityReference { Variable = variable, Key = key }; + space.Quantities[variable] = quantity; + if (space.IdentityProblem != null || key == null) + { + quantity.BlockReasons.Add("identity: " + (space.IdentityProblem ?? "no native key")); + continue; + } + + sql.Series.TryGetValue(S11Names.SeriesKey(variable, key), out ChannelSeries sqlSeries); + eso.Series.TryGetValue(S11Names.SeriesKey(variable, key), out ChannelSeries esoSeries); + double[] sqlValues = Judge(document, calendar, "sql", variable, key, sqlSeries, quantity); + double[] esoValues = Judge(document, calendar, "eso", variable, key, esoSeries, quantity); + if (sqlValues == null || esoValues == null) + { + continue; + } + + // S11-C4: the two channels agree hour by hour (class E), judged only once both are complete. + int disagreements = 0; + string first = null; + for (int h = 0; h < sqlValues.Length; h++) + { + if (!(Math.Abs(sqlValues[h] - esoValues[h]) <= ToleranceE(sqlValues[h]))) + { + disagreements++; + first = first ?? (calendar.Hours[h] + " sql " + sqlValues[h].ToString("R", CultureInfo.InvariantCulture) + " eso " + esoValues[h].ToString("R", CultureInfo.InvariantCulture)); + } + } + + if (disagreements > 0) + { + quantity.BlockReasons.Add("S11-C4: SQL and ESO disagree in " + disagreements + " hour(s), first " + first); + Check(document, "S11-C4", variable + " " + key, false, "SQL and ESO disagree in " + disagreements + " hour(s), first " + first); + continue; + } + + values[variable] = sqlValues; + hourly[quantity] = sqlValues; + Fill(quantity, sqlValues); + quantity.Available = true; + } + + foreach (LoadKind kind in new[] { LoadKind.Heating, LoadKind.Cooling }) + { + if (values.TryGetValue(S11Names.Total(kind), out double[] total) && values.TryGetValue(S11Names.Sensible(kind), out double[] sensible)) + { + int negative = 0; + string first = null; + double latent = 0; + for (int h = 0; h < total.Length; h++) + { + latent += total[h] - sensible[h]; + if (total[h] < sensible[h] - ToleranceE(total[h])) + { + negative++; + first = first ?? (calendar.Hours[h] + " Total " + total[h].ToString("R", CultureInfo.InvariantCulture) + " < Sensible " + sensible[h].ToString("R", CultureInfo.InvariantCulture)); + } + } + + space.LatentJoules[kind.ToString()] = latent; + if (negative > 0) + { + space.BasisProblems[kind.ToString()] = "S11-C6: Total < Sensible (negative latent) in " + negative + " hour(s), first " + first; + } + + Check(document, "S11-C6", kind + " " + space.SpaceGuid.ToString("N"), negative == 0, negative == 0 ? "Total >= Sensible every hour; annual latent " + latent.ToString("R", CultureInfo.InvariantCulture) + " J" : space.BasisProblems[kind.ToString()]); + } + } + } + + return hourly; + } + + /// One channel's series judged against the calendar (S11-C1/C1b) and S11-C3; its values, or null with the reason recorded. + private static double[] Judge(ReferenceDocument document, RunCalendar calendar, string channel, string variable, string key, ChannelSeries series, QuantityReference quantity) + { + if (series == null) + { + string missing = channel + ": no Hourly '" + variable + "' for key '" + key + "'"; + quantity.BlockReasons.Add("S11-C1b: " + missing); + document.Completeness.Add(new CompletenessResult { Channel = channel, Variable = variable, Key = key, Expected = calendar.Count, Problems = { "absent from the file" } }); + return null; + } + + CompletenessResult completeness = Completeness.Check(calendar, series); + document.Completeness.Add(completeness); + if (!completeness.Complete) + { + quantity.BlockReasons.Add("S11-C1: " + channel + " '" + variable + "' '" + key + "' incomplete: missing " + completeness.Missing + ", duplicated " + completeness.Duplicated + ", extra " + completeness.Extra + (completeness.Problems.Count > 0 ? "; " + string.Join("; ", completeness.Problems.Take(3)) : string.Empty) + (completeness.Examples.Count > 0 ? "; e.g. " + string.Join(", ", completeness.Examples.Take(3)) : string.Empty)); + return null; + } + + // S11-C3: units, finite, non-negative. + string unit = S11Names.UnitOf(variable); + if (!string.Equals(series.Units, unit, StringComparison.Ordinal)) + { + quantity.BlockReasons.Add("S11-C3: " + channel + " '" + variable + "' unit is '" + series.Units + "', not '" + unit + "'"); + return null; + } + + int bad = completeness.Values.Count(x => double.IsNaN(x) || double.IsInfinity(x) || x < 0); + if (bad > 0) + { + quantity.BlockReasons.Add("S11-C3: " + channel + " '" + variable + "' '" + key + "' has " + bad + " non-finite or negative value(s)"); + return null; + } + + return completeness.Values; + } + + private static void Fill(QuantityReference quantity, double[] values) + { + double sum = 0; + double peak = double.MinValue; + int peakHour = -1; + foreach (double value in values) + { + sum += value; + } + + for (int h = 0; h < values.Length; h++) + { + if (values[h] > peak) + { + peak = values[h]; + peakHour = h; + } + } + + quantity.Sum = sum; + quantity.Peak = peak; + quantity.PeakHour = peakHour; + quantity.TieCount = values.Count(x => x == peak); + quantity.AllZero = values.All(x => x == 0); + quantity.PeakWatts = S11Names.ZoneVariables.Contains(quantity.Variable) ? double.NaN : peak / 3600.0; + } + + /// Model consumption and coincident peak per load kind and basis, from the identified zones' SQL series. + private static void ModelValues(ReferenceDocument document, Dictionary hourly) + { + foreach (LoadKind kind in new[] { LoadKind.Heating, LoadKind.Cooling }) + { + ModelLoadReference model = new ModelLoadReference { Kind = kind }; + model.Total = Basis(document, hourly, S11Names.Total(kind)); + model.Sensible = Basis(document, hourly, S11Names.Sensible(kind)); + document.Model.Add(model); + } + } + + /// Available only when every identified space's series of is available. + private static ModelBasisReference Basis(ReferenceDocument document, Dictionary hourly, string variable) + { + ModelBasisReference result = new ModelBasisReference(); + List series = new List(); + foreach (SpaceReference space in document.Spaces) + { + if (!space.Quantities.TryGetValue(variable, out QuantityReference quantity) || !quantity.Available || !hourly.TryGetValue(quantity, out double[] values)) + { + return result; + } + + series.Add(values); + } + + if (series.Count == 0) + { + return result; + } + + double joules = 0; + foreach (double[] values in series) + { + double zone = 0; + foreach (double value in values) + { + zone += value; + } + + joules += zone; + } + + double[] totals = new double[series[0].Length]; + foreach (double[] values in series) + { + for (int h = 0; h < totals.Length; h++) + { + totals[h] += values[h]; + } + } + + double peak = double.MinValue; + int peakHour = -1; + for (int h = 0; h < totals.Length; h++) + { + if (totals[h] > peak) + { + peak = totals[h]; + peakHour = h; + } + } + + result.Available = true; + result.ConsumptionKwh = joules / 3.6e6; + result.CoincidentPeakKw = peak / 3600.0 / 1000.0; + result.PeakHour = peakHour; + result.TieCount = totals.Count(x => x == peak); + result.AllZero = totals.All(x => x == 0); + return result; + } + + /// S11-C7: each facility meter equals the sum of the zones' Total energy (class R2), in each file where it exists. + private static void Meters(ReferenceDocument document) + { + foreach (LoadKind kind in new[] { LoadKind.Heating, LoadKind.Cooling }) + { + ModelLoadReference model = document.ModelOf(kind); + if (!model.Total.Available) + { + continue; + } + + double zonesJoules = model.Total.ConsumptionKwh * 3.6e6; + foreach (KeyValuePair> channel in new[] { new KeyValuePair>("sql", document.SqlMeters), new KeyValuePair>("eso", document.EsoMeters) }) + { + MeterSum meter = channel.Value.FirstOrDefault(x => string.Equals(x.Name, S11Names.Meter(kind), StringComparison.OrdinalIgnoreCase)); + if (meter == null) + { + Check(document, "S11-C7", kind + " " + channel.Key, true, channel.Key + " has no " + S11Names.Meter(kind) + " meter (informational)", "INFO"); + continue; + } + + double delta = Math.Abs(meter.Sum - zonesJoules); + bool pass = delta <= ToleranceR2(zonesJoules); + string detail = channel.Key + " " + meter.Name + " (" + meter.Frequency + ", " + meter.Rows + " rows) " + meter.Sum.ToString("R", CultureInfo.InvariantCulture) + " J vs zones " + zonesJoules.ToString("R", CultureInfo.InvariantCulture) + " J, |delta| " + delta.ToString("R", CultureInfo.InvariantCulture); + Check(document, "S11-C7", kind + " " + channel.Key, pass, detail); + if (!pass) + { + model.MeterProblem = (model.MeterProblem == null ? string.Empty : model.MeterProblem + "; ") + "S11-C7: " + detail; + } + } + } + } + } +} diff --git a/benchmark/SAM.Analytical.OpenStudio.ResultsReadBack/Report/ResultReport.cs b/benchmark/SAM.Analytical.OpenStudio.ResultsReadBack/Report/ResultReport.cs new file mode 100644 index 0000000..101a896 --- /dev/null +++ b/benchmark/SAM.Analytical.OpenStudio.ResultsReadBack/Report/ResultReport.cs @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: LGPL-3.0-or-later +// Copyright (c) 2020–2026 Michal Dengusiak & Jakub Ziolkowski and contributors + +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text; + +namespace SAM.Analytical.OpenStudio.ResultsReadBack +{ + /// The human-readable S11 report (compare.md) beside compare.json; byte-stable for the same inputs. + public static class ResultReport + { + private static readonly CultureInfo ci = CultureInfo.InvariantCulture; + + public static string Markdown(CompareDocument compare, ReferenceDocument reference) + { + StringBuilder text = new StringBuilder(); + text.Append("# S11 OpenStudio results round-trip: ").Append(compare.FixtureId).Append("\n\n"); + text.Append("Verdict: **").Append(compare.Verdict).Append("** (exit ").Append(compare.ExitCode.ToString(ci)).Append("): ") + .Append(compare.Passed.ToString(ci)).Append(" PASS, ").Append(compare.Failed.ToString(ci)).Append(" FAIL, ") + .Append(compare.Blocked.ToString(ci)).Append(" BLOCKED of ").Append(compare.Required.ToString(ci)).Append(" required rows; ") + .Append(compare.Informational.ToString(ci)).Append(" informational.\n\n"); + foreach (string note in compare.Notes) + { + text.Append("> ").Append(note).Append("\n\n"); + } + + text.Append("Run period: ").Append(reference.RunPeriod).Append(" (").Append(reference.CalendarHours.ToString(ci)).Append(" hours). "); + text.Append("SQL ").Append(reference.SqlSha256Before).Append(", ESO ").Append(reference.EsoSha256Before).Append(", in.idf ").Append(reference.IdfSha256Before).Append(".\n\n"); + text.Append("F3: ").Append(compare.F3.Detail).Append(".\n\n"); + + if (compare.GlobalBlocks.Count > 0) + { + text.Append("## Global blocks\n\n"); + foreach (string block in compare.GlobalBlocks) + { + text.Append("- ").Append(block).Append('\n'); + } + + text.Append('\n'); + } + + text.Append("## Checks\n\n| Check | Scope | Status | Detail |\n|---|---|---|---|\n"); + foreach (ReferenceCheck check in reference.Checks) + { + text.Append("| ").Append(check.Id).Append(" | ").Append(Cell(check.Scope)).Append(" | ").Append(check.Status).Append(" | ").Append(Cell(check.Detail)).Append(" |\n"); + } + + text.Append("\n## Completeness (per file, against the RunPeriod calendar)\n\n| File | Variable | Key | Entries | Found | Missing | Duplicated | Extra | Complete |\n|---|---|---|---|---|---|---|---|---|\n"); + foreach (CompletenessResult row in reference.Completeness) + { + text.Append("| ").Append(row.Channel).Append(" | ").Append(row.Variable).Append(" | ").Append(Cell(row.Key)).Append(" | ") + .Append(row.DictionaryEntries.ToString(ci)).Append(" | ").Append(row.Found.ToString(ci)).Append(" | ").Append(row.Missing.ToString(ci)).Append(" | ") + .Append(row.Duplicated.ToString(ci)).Append(" | ").Append(row.Extra.ToString(ci)).Append(" | ").Append(row.Complete ? "yes" : "NO").Append(" |\n"); + } + + text.Append("\n## Rows\n\n| Row | Class | Status | Reference | Subject | Delta | Margin | Reason |\n|---|---|---|---|---|---|---|---|\n"); + foreach (ResultRow row in compare.Rows) + { + text.Append("| ").Append(row.Id).Append(" | ").Append(row.Class).Append(" | ").Append(row.Status).Append(" | ") + .Append(Number(row.Reference)).Append(" | ").Append(Number(row.Subject)).Append(" | ").Append(Number(row.Delta)).Append(" | ") + .Append(row.Margin.HasValue ? row.Margin.Value.ToString("G6", ci) : row.SensibleHourDiffers.HasValue ? (row.SensibleHourDiffers.Value ? "hour differs" : "same hour") : string.Empty) + .Append(" | ").Append(Cell(row.Reason)).Append(" |\n"); + } + + return text.ToString(); + } + + public static void Save(CompareDocument compare, ReferenceDocument reference, string directory) + { + Directory.CreateDirectory(directory); + S11Json.Save(compare, Path.Combine(directory, "compare.json")); + File.WriteAllText(Path.Combine(directory, "compare.md"), Markdown(compare, reference), new UTF8Encoding(false)); + } + + private static string Number(double? value) => value.HasValue ? value.Value.ToString("R", ci) : string.Empty; + + private static string Cell(string value) => (value ?? string.Empty).Replace("|", "\\|").Replace("\n", " "); + } +} diff --git a/benchmark/SAM.Analytical.OpenStudio.ResultsReadBack/Run-S11-Results.ps1 b/benchmark/SAM.Analytical.OpenStudio.ResultsReadBack/Run-S11-Results.ps1 new file mode 100644 index 0000000..35f62de --- /dev/null +++ b/benchmark/SAM.Analytical.OpenStudio.ResultsReadBack/Run-S11-Results.ps1 @@ -0,0 +1,379 @@ +# SPDX-License-Identifier: LGPL-3.0-or-later +# Copyright (c) 2020-2026 Michal Dengusiak & Jakub Ziolkowski and contributors + +<# +.SYNOPSIS + S11 OpenStudio results round-trip acceptance (revision 2 of the S11 plan; NOT authorised by revision 1). + +.DESCRIPTION + -Phase run (2 EnergyPlus simulations; refuses without -ConfirmEngineRuns): preflight (fixture hashes against + InputReadBack\Fixtures\SHA256SUMS.txt, Budapest EPW hash, -Root new or empty, the build receipt of + Build-S11-Receipt.ps1 against the files on disk and the clean HEADs, the portable OpenStudio CLI 3.10.0), then for + s08-fix-box and s08-fix-two: benchmark-openstudio with --sam-results, the CLI forced by prepending its bin to the + child PATH only. Writes run-manifest.json per run. + -Phase readback: read, subject and compare for both runs, each a separate process. + -Phase negative: the os-results-readback negative cases on copies of both runs, each read/subject/compared fresh. + + Before and after each run is processed, its model, in.idf, SQL, ESO, b1a.json and sam.json must still have the + run-manifest hashes, and the active build receipt (and the harness and producer it lists) must be the one the run + recorded: evidence of different builds is never combined. + + Exit codes: 70 STOPPED (infrastructure or integrity: a step that did not complete, an output it did not write fresh, + a report whose verdict disagrees with its exit or that judged other files, a changed original); the phase stops at + once and no earlier report is read in its place. Otherwise readback gives 0 PASS (every required row of both + fixtures PASS and OD-7 met by at least one), 10 FAIL (a FAIL in either), 11 BLOCKED; numerical FAIL/BLOCKED outcomes + are collected for both fixtures first. negative gives 0 when every case gave its required exit on a copy mutated as + designed and each control PASSed, else 10; without a PASSing control no mutation outcome counts as evidence. + summary-.json under -Root records each attempt, including a stopped one. SQL/ESO stay machine-local. + + -BenchmarkExe and -HarnessExe exist for the offline driver tests (stand-in commands); whatever is used must be a file + of the build receipt. +#> +param( + [ValidateSet('run', 'readback', 'negative')] + [string]$Phase = 'readback', + [string]$Root = 'C:\TasOut\s11\acc1', + [string]$WeatherEpw = '', + [string]$WeatherEpwSha256 = '472ffb4e2f10867600949a392ce46891fc1bd05f2fc8d8917a99bca7d26209c3', + [string]$OpenStudioCli = 'C:\Users\michal\Documents\GitHub\SAM-BIM\.icon-migration\runtime\OpenStudio-3.10.0+86d7e215a1-Windows\bin\openstudio.exe', + [string]$OpenStudioCliVersion = '3.10.0+86d7e215a1', + [string]$BuildReceipt = '', + [string]$BenchmarkExe = '', + [string]$HarnessExe = '', + [switch]$ConfirmEngineRuns +) + +$ErrorActionPreference = 'Stop' +$repository = [IO.Path]::GetFullPath((Join-Path $PSScriptRoot '..\..')) +$workspace = Split-Path $repository +$fixtures = Join-Path $repository 'benchmark\SAM.Analytical.OpenStudio.InputReadBack\Fixtures' +if (-not $BenchmarkExe) { $BenchmarkExe = Join-Path $repository 'benchmark\SAM.Analytical.OpenStudio.Benchmark\bin\x64\Release\net8.0\benchmark-openstudio.exe' } +if (-not $HarnessExe) { $HarnessExe = Join-Path $repository 'benchmark\SAM.Analytical.OpenStudio.ResultsReadBack.Cli\bin\x64\Release\net8.0\os-results-readback.exe' } +$BenchmarkExe = [IO.Path]::GetFullPath($BenchmarkExe) +$HarnessExe = [IO.Path]::GetFullPath($HarnessExe) +$benchmarkDll = [IO.Path]::ChangeExtension($BenchmarkExe, '.dll') +$harnessDll = [IO.Path]::ChangeExtension($HarnessExe, '.dll') +if (-not $BuildReceipt) { $BuildReceipt = Join-Path (Split-Path $HarnessExe) 's11-build-receipt.json' } +$BuildReceipt = [IO.Path]::GetFullPath($BuildReceipt) +$runs = @(@('s08-fix-box', 'box'), @('s08-fix-two', 'two')) + +# The exit each negative case must give and the copied files it must change. Owned by this driver: a case's own +# negative.json is checked against it, never trusted for it. The control comes first. +$negatives = [ordered]@{ + 'ctl-unmutated-copy' = 0 + 'neg-delete-hour-both' = 11 + 'neg-duplicate-hour-both' = 11 + 'neg-sensible-cooling-substitution' = 10 +} +$mutates = @{ + 'ctl-unmutated-copy' = @() + 'neg-delete-hour-both' = @('eplusout.sql', 'eplusout.eso') + 'neg-duplicate-hour-both' = @('eplusout.sql', 'eplusout.eso') + 'neg-sensible-cooling-substitution' = @('b1a.json', 'sam.json') +} +$verdicts = @{ 0 = 'PASS'; 10 = 'FAIL'; 11 = 'BLOCKED' } + +$summary = [ordered]@{ phase = $Phase; started = (Get-Date).ToUniversalTime().ToString('o'); root = $Root; buildReceipt = $BuildReceipt; harness = $HarnessExe; runs = @() } +$summaryWritable = $false + +function Write-Summary([int]$Code, [string]$Result) { + $summary.exit = $Code + $summary.result = $Result + $summary.finished = (Get-Date).ToUniversalTime().ToString('o') + if ($script:summaryWritable) { $summary | ConvertTo-Json -Depth 8 | Set-Content -Encoding utf8 (Join-Path $Root "summary-$Phase.json") } +} + +function Stop-Phase([string]$Reason) { + Write-Host "STOPPED: $Reason" + $summary.stopped = $Reason + Write-Summary 70 'STOPPED' + exit 70 +} + +function Sha([string]$Path) { + if (-not $Path -or -not (Test-Path -LiteralPath $Path -PathType Leaf)) { return $null } + $stream = [IO.File]::OpenRead($Path) + try { + $sha = [Security.Cryptography.SHA256]::Create() + try { return ([BitConverter]::ToString($sha.ComputeHash($stream)) -replace '-', '').ToLowerInvariant() } finally { $sha.Dispose() } + } + finally { $stream.Dispose() } +} + +function Git([string]$Directory) { + $ErrorActionPreference = 'Continue' + & git -C $Directory @args 2>$null +} + +# A step's exit code; a step that cannot start or reports none stops the phase. +function Invoke-Step([string]$Exe, [string[]]$Arguments, [string]$Log) { + $ErrorActionPreference = 'Continue' + $global:LASTEXITCODE = $null + try { & $Exe @Arguments *>> $Log } + catch { Stop-Phase "$Exe $($Arguments[0]) could not run: $($_.Exception.Message)" } + if ($null -eq $global:LASTEXITCODE) { Stop-Phase "$Exe $($Arguments[0]) reported no exit code (see $Log)" } + return [int]$global:LASTEXITCODE +} + +# A file as a step finds it (write time and hash); null when absent. Taken before and after a step, so freshness never +# depends on comparing clocks. +function Get-Stamp([string]$Path) { + $item = Get-Item -LiteralPath $Path -ErrorAction SilentlyContinue + if (-not $item) { return $null } + return "$($item.LastWriteTimeUtc.Ticks)|$(Sha $Path)" +} + +# The output a step that exited 0 must have written itself: present, and not the file an earlier attempt left. +function Assert-Fresh([string]$Path, $Before, [string]$What) { + $after = Get-Stamp $Path + if (-not $after) { Stop-Phase "$What wrote no $Path" } + if ($after -eq $Before) { Stop-Phase "$What did not write $Path (it is the file from before the step)" } +} + +# The receipt's files must be the files on disk, its commits the clean HEADs now, and the executables this driver +# starts must be among its files. +function Test-Receipt { + if (-not (Test-Path -LiteralPath $BuildReceipt -PathType Leaf)) { Stop-Phase "no build receipt $BuildReceipt (run Build-S11-Receipt.ps1)" } + $receipt = Get-Content -LiteralPath $BuildReceipt -Raw | ConvertFrom-Json + if ($receipt.schema -ne 's11-build-receipt/1') { Stop-Phase "receipt schema $($receipt.schema)" } + $listed = @{} + foreach ($file in @($receipt.files)) { + if (-not $file.path -or -not $file.sha256 -or (Sha $file.path) -ne $file.sha256) { Stop-Phase "$($file.path) is not the file the receipt recorded" } + $listed[[IO.Path]::GetFullPath($file.path)] = $true + } + foreach ($file in @($BenchmarkExe, $benchmarkDll, $HarnessExe, $harnessDll)) { + if (-not $listed.ContainsKey($file)) { Stop-Phase "$file is not a file of the build receipt $BuildReceipt" } + } + foreach ($commit in @($receipt.commits)) { + $head = (Git $commit.repository rev-parse HEAD | Out-String).Trim() + $status = @(Git $commit.repository status --porcelain --untracked-files=all) + if ($head -ne $commit.commit -or $status.Count -gt 0) { Stop-Phase "$($commit.repository) is not clean at the receipt commit $($commit.commit)" } + } +} + +# Every file a run's evidence rests on, with the hash its run-manifest recorded. +function Get-RunFiles($Manifest, [string]$Directory, [string]$FixtureId) { + return [ordered]@{ + 'model' = @((Join-Path $fixtures "$FixtureId.json"), $Manifest.fixtureSha256) + 'in.idf' = @([string]$Manifest.idf, $Manifest.idfSha256) + 'eplusout.sql' = @([string]$Manifest.sql, $Manifest.sqlSha256) + 'eplusout.eso' = @([string]$Manifest.eso, $Manifest.esoSha256) + 'b1a.json' = @((Join-Path $Directory 'b1a.json'), $Manifest.b1aSha256) + 'sam.json' = @((Join-Path $Directory 'sam.json'), $Manifest.samResultsSha256) + 'build receipt' = @($BuildReceipt, $Manifest.receiptSha256) + 'harness' = @($harnessDll, $Manifest.harnessDll) + 'producer' = @($benchmarkDll, $Manifest.benchmarkDll) + } +} + +# Stops unless the run's files are still those of its manifest and the active build is the one it recorded. +function Test-Run([string]$Label, [string]$FixtureId, [string]$Directory, [string]$When) { + $path = Join-Path $Directory 'run-manifest.json' + if (-not (Test-Path -LiteralPath $path -PathType Leaf)) { Stop-Phase "$Label has no run-manifest.json ($When)" } + $manifest = Get-Content -LiteralPath $path -Raw | ConvertFrom-Json + if ($manifest.schema -ne 's11-run-manifest/1' -or $manifest.label -ne $Label -or $manifest.fixture -ne $FixtureId) { Stop-Phase "$path is not the s11-run-manifest/1 of $Label ($FixtureId)" } + $below = [IO.Path]::GetFullPath($Directory).TrimEnd('\') + '\' + $problems = @() + foreach ($name in @('idf', 'sql', 'eso')) { + $file = [string]$manifest.$name + if (-not $file -or -not [IO.Path]::GetFullPath($file).StartsWith($below, [StringComparison]::OrdinalIgnoreCase)) { $problems += "the manifest $name '$file' is not below $Directory" } + } + $files = Get-RunFiles $manifest $Directory $FixtureId + foreach ($name in $files.Keys) { + $recorded = $files[$name][1] + if (-not $recorded -or (Sha $files[$name][0]) -ne $recorded) { $problems += "$name is not the file the run recorded" } + } + if ($problems.Count -gt 0) { + $summary["original-unchanged:$Label"] = $false + Stop-Phase "$Label ${When}: $($problems -join '; ')" + } + return $manifest +} + +# A compare that completed (0/10/11) must have written a fresh report whose verdict is its exit and whose inputs are +# exactly the files given; any other exit stops the phase before a report is read. +function Read-Compare([string]$Directory, [int]$Exit, $Before, $Inputs, [string]$What) { + if (-not $verdicts.ContainsKey($Exit)) { Stop-Phase "$What exit $Exit (infrastructure or usage; no report is read)" } + $path = Join-Path $Directory 'compare\compare.json' + Assert-Fresh $path $Before $What + $report = Get-Content -LiteralPath $path -Raw | ConvertFrom-Json + if ($report.Schema -ne 's11-compare/1') { Stop-Phase "$path schema $($report.Schema)" } + if ($report.Verdict -ne $verdicts[$Exit] -or [int]$report.ExitCode -ne $Exit) { Stop-Phase "$What exit $Exit but its report says $($report.Verdict) (exit $($report.ExitCode))" } + foreach ($name in $Inputs.Keys) { + if (-not $Inputs[$name] -or $report.Inputs.$name -ne $Inputs[$name]) { Stop-Phase "$What judged a $name other than $($Inputs[$name])" } + } + return $report +} + +# read, then subject (unless the case captured it), each in its own process and each writing its output fresh. +function Invoke-ReadSubject([string]$Directory, [string]$Model, [string]$FixtureId, [string]$Log, [string]$What, [bool]$Subject) { + $steps = @(, @('read', 'reference.json', @('--fixture-id', $FixtureId))) + if ($Subject) { $steps += , @('subject', 'subject.json', @()) } + foreach ($step in $steps) { + $before = Get-Stamp (Join-Path $Directory $step[1]) + $exit = Invoke-Step $HarnessExe (@($step[0], '--run', $Directory, '--model', $Model) + $step[2]) $Log + if ($exit -ne 0) { Stop-Phase "$($step[0]) $What exit $exit (see $Log)" } + Assert-Fresh (Join-Path $Directory $step[1]) $before "$($step[0]) $What" + } +} + +if ($Phase -eq 'run') { + if (-not $ConfirmEngineRuns) { Stop-Phase 'the run phase starts 2 EnergyPlus simulations: pass -ConfirmEngineRuns only under an explicit owner authorisation' } + if (Test-Path -LiteralPath $Root) { + if (@(Get-ChildItem -LiteralPath $Root -Force).Count -gt 0) { Stop-Phase "Root must be new or empty: $Root" } + } + else { New-Item -ItemType Directory -Path $Root | Out-Null } + $summaryWritable = $true + + Test-Receipt + foreach ($line in Get-Content (Join-Path $fixtures 'SHA256SUMS.txt')) { + if ([string]::IsNullOrWhiteSpace($line)) { continue } + $hash, $name = $line -split '\s+', 2 + if ((Sha (Join-Path $fixtures $name.Trim())) -ne $hash.ToLowerInvariant()) { Stop-Phase "fixture $name is not the signed-off $hash" } + } + if (-not $WeatherEpw -or (Sha $WeatherEpw) -ne $WeatherEpwSha256) { Stop-Phase "the EPW $WeatherEpw is not $WeatherEpwSha256" } + if (-not (Test-Path -LiteralPath $OpenStudioCli)) { Stop-Phase "OpenStudio CLI not found: $OpenStudioCli" } + $cliVersion = (& $OpenStudioCli --version | Out-String).Trim() + if ($cliVersion -ne $OpenStudioCliVersion) { Stop-Phase "OpenStudio CLI version '$cliVersion' is not $OpenStudioCliVersion" } + + $runnerHead = (Git $repository rev-parse HEAD | Out-String).Trim() + $samHead = (Git (Join-Path $workspace 'SAM') rev-parse HEAD | Out-String).Trim() + foreach ($run in $runs) { + $fixtureId, $label = $run + $dir = Join-Path $Root $label + New-Item -ItemType Directory -Path $dir | Out-Null + $log = Join-Path $Root "$label.console.txt" + $model = Join-Path $fixtures "$fixtureId.json" + $path = $env:PATH + try { + # The child resolves openstudio.exe from PATH first, and that CLI matches the SDK, so it is selected. + $env:PATH = (Split-Path $OpenStudioCli) + ';' + $path + $exit = Invoke-Step $BenchmarkExe @('--model', $model, '--weather', $WeatherEpw, '--out', (Join-Path $dir 'b1a.json'), '--work', (Join-Path $dir 'work'), '--sam-results', (Join-Path $dir 'sam.json'), '--runner-commit', $runnerHead, '--sam-commit', $samHead) $log + } + finally { $env:PATH = $path } + if ($exit -ne 0) { Stop-Phase "benchmark-openstudio $label exit $exit (no retry; see $log)" } + $sql = @(Get-ChildItem -LiteralPath (Join-Path $dir 'work') -Recurse -Filter 'eplusout.sql') + $eso = @(Get-ChildItem -LiteralPath (Join-Path $dir 'work') -Recurse -Filter 'eplusout.eso') + if ($sql.Count -ne 1 -or $eso.Count -ne 1) { Stop-Phase "$label has $($sql.Count) eplusout.sql and $($eso.Count) eplusout.eso (exactly one each)" } + $idf = Join-Path $sql[0].DirectoryName 'in.idf' + $manifest = [ordered]@{ + schema = 's11-run-manifest/1'; label = $label; fixture = $fixtureId; fixtureSha256 = (Sha $model); epwSha256 = (Sha $WeatherEpw) + openStudioCli = $OpenStudioCli; openStudioCliVersion = $cliVersion; runnerCommit = $runnerHead; samCommit = $samHead + buildReceipt = $BuildReceipt; receiptSha256 = (Sha $BuildReceipt); benchmarkExe = $BenchmarkExe; benchmarkDll = (Sha $benchmarkDll); harnessExe = $HarnessExe; harnessDll = (Sha $harnessDll) + idf = $idf; idfSha256 = (Sha $idf); sql = $sql[0].FullName; sqlSha256 = (Sha $sql[0].FullName); eso = $eso[0].FullName; esoSha256 = (Sha $eso[0].FullName) + b1aSha256 = (Sha (Join-Path $dir 'b1a.json')); samResultsSha256 = (Sha (Join-Path $dir 'sam.json')) + } + foreach ($name in @('idfSha256', 'sqlSha256', 'esoSha256', 'b1aSha256', 'samResultsSha256')) { + if (-not $manifest[$name]) { Stop-Phase "$label produced no file for $name" } + } + $manifest | ConvertTo-Json -Depth 4 | Set-Content -Encoding utf8 (Join-Path $dir 'run-manifest.json') + $summary.runs += $manifest + Write-Host "$label simulated: SQL $($manifest.sqlSha256)" + } + Write-Summary 0 'SIMULATED' + exit 0 +} + +if (-not (Test-Path -LiteralPath $Root -PathType Container)) { Stop-Phase "Root does not exist: $Root" } +$summaryWritable = $true +Test-Receipt + +if ($Phase -eq 'readback') { + $exits = @() + foreach ($run in $runs) { + $fixtureId, $label = $run + $dir = Join-Path $Root $label + $log = Join-Path $Root "$label.readback.txt" + $model = Join-Path $fixtures "$fixtureId.json" + $manifest = Test-Run $label $fixtureId $dir 'before readback' + Invoke-ReadSubject $dir $model $fixtureId $log $label $true + $before = Get-Stamp (Join-Path $dir 'compare\compare.json') + $compare = Invoke-Step $HarnessExe @('compare', '--run', $dir, '--model', $model) $log + Test-Run $label $fixtureId $dir 'after readback' | Out-Null + $inputs = [ordered]@{} + foreach ($entry in (Get-RunFiles $manifest $dir $fixtureId).GetEnumerator()) { + if (@('model', 'in.idf', 'eplusout.sql', 'eplusout.eso', 'b1a.json', 'sam.json') -contains $entry.Key) { $inputs[$entry.Key] = $entry.Value[1] } + } + $inputs['reference.json'] = Sha (Join-Path $dir 'reference.json') + $inputs['subject.json'] = Sha (Join-Path $dir 'subject.json') + $report = Read-Compare $dir $compare $before $inputs "compare $label" + $summary["original-unchanged:$label"] = $true + $exits += $compare + $summary.runs += [ordered]@{ label = $label; fixture = $fixtureId; exit = $compare; verdict = $report.Verdict; required = $report.Required; passed = $report.Passed; failed = $report.Failed; blocked = $report.Blocked; f3 = $report.F3; compareSha256 = (Sha (Join-Path $dir 'compare\compare.json')) } + Write-Host ("{0,-6} compare exit {1} ({2}: {3}/{4} PASS, {5} FAIL, {6} BLOCKED)" -f $label, $compare, $report.Verdict, $report.Passed, $report.Required, $report.Failed, $report.Blocked) + } + $od7 = @($summary.runs | Where-Object { $_.f3.MeetsOd7 -eq $true }).Count -gt 0 + $summary.od7 = [ordered]@{ anyFixtureMeets = $od7; note = 'OD-7: on at least one fixture, cooling consumption and one space cooling peak discriminate Total from Sensible by >= 1000 x tolerance (required for PASS)' } + if ($exits -contains 10) { $code = 10 } + elseif ($exits -contains 11 -or -not $od7) { $code = 11 } + else { $code = 0 } + Write-Host ("readback: {0}{1}" -f $verdicts[$code], $(if ($code -eq 11 -and -not ($exits -contains 11)) { ' (OD-7 not met by either fixture)' } else { '' })) + Write-Summary $code $verdicts[$code] + exit $code +} + +# negative +$failures = @() +foreach ($run in $runs) { + $fixtureId, $label = $run + $source = Join-Path $Root $label + $model = Join-Path $fixtures "$fixtureId.json" + $manifest = Test-Run $label $fixtureId $source 'before negatives' + $originals = Get-RunFiles $manifest $source $fixtureId + $controlPassed = $false + foreach ($caseId in $negatives.Keys) { + $required = $negatives[$caseId] + $dir = Join-Path $Root "negatives\$label-$caseId" + $log = Join-Path $Root "negatives\$label-$caseId.console.txt" + $what = "$label-$caseId" + New-Item -ItemType Directory -Force -Path (Split-Path $dir) | Out-Null + $beforeRecord = Get-Stamp (Join-Path $dir 'negative.json') + $beforeSubject = Get-Stamp (Join-Path $dir 'subject.json') + $exit = Invoke-Step $HarnessExe @('negative', '--run', $source, '--model', $model, '--case', $caseId, '--out', $dir) $log + if ($exit -ne 0) { Stop-Phase "negative $what exit $exit (see $log)" } + Assert-Fresh (Join-Path $dir 'negative.json') $beforeRecord "negative $what" + $record = Get-Content -LiteralPath (Join-Path $dir 'negative.json') -Raw | ConvertFrom-Json + if ($record.Case -ne $caseId -or [int]$record.ExpectedExit -ne $required) { Stop-Phase "negative.json of $what describes case '$($record.Case)' with exit $($record.ExpectedExit), not '$caseId' with exit $required" } + + # The copy must differ from the original in exactly the files the case mutates. + $copy = [ordered]@{ + 'in.idf' = Sha (Join-Path $dir 'work\run\in.idf'); 'eplusout.sql' = Sha (Join-Path $dir 'work\run\eplusout.sql'); 'eplusout.eso' = Sha (Join-Path $dir 'work\run\eplusout.eso') + 'b1a.json' = Sha (Join-Path $dir 'b1a.json'); 'sam.json' = Sha (Join-Path $dir 'sam.json') + } + $design = @() + foreach ($name in $copy.Keys) { + if (-not $copy[$name]) { Stop-Phase "negative $what copied no $name" } + $changed = $copy[$name] -ne $originals[$name][1] + if ($changed -ne ($mutates[$caseId] -contains $name)) { $design += "$name $(if ($changed) { 'changed' } else { 'unchanged' })" } + } + + $captured = $caseId -eq 'neg-sensible-cooling-substitution' + if ($captured) { Assert-Fresh (Join-Path $dir 'subject.json') $beforeSubject "negative $what (captured subject)" } + Invoke-ReadSubject $dir $model $fixtureId $log $what (-not $captured) + $before = Get-Stamp (Join-Path $dir 'compare\compare.json') + $exit = Invoke-Step $HarnessExe @('compare', '--run', $dir, '--model', $model) $log + + # The original must be untouched by every case. + Test-Run $label $fixtureId $source "after $caseId" | Out-Null + $inputs = [ordered]@{ 'model' = $originals['model'][1] } + foreach ($name in $copy.Keys) { $inputs[$name] = $copy[$name] } + $inputs['reference.json'] = Sha (Join-Path $dir 'reference.json') + $inputs['subject.json'] = Sha (Join-Path $dir 'subject.json') + $report = Read-Compare $dir $exit $before $inputs "compare $what" + + $asRequired = ($exit -eq $required) -and ($design.Count -eq 0) + if ($caseId -eq 'ctl-unmutated-copy') { $controlPassed = $asRequired } + $evidence = $controlPassed + if (-not $asRequired) { $failures += "$what exit $exit (required $required)$(if ($design.Count -gt 0) { '; copy not mutated as designed: ' + ($design -join ', ') })" } + $summary.runs += [ordered]@{ label = $what; exit = $exit; required = $required; asRequired = $asRequired; evidence = $evidence; copyAsDesigned = ($design.Count -eq 0); copyProblems = $design; verdict = $report.Verdict; mutation = $record.Mutation; f3 = $report.F3 } + Write-Host ("{0,-44} exit {1} (required {2}) {3}" -f $what, $exit, $required, $(if ($asRequired) { 'as required' } else { 'NOT AS REQUIRED' })) + } + $summary["original-unchanged:$label"] = $true + if (-not $controlPassed) { $failures += "$label control did not PASS: no mutation outcome of $label is evidence" } +} + +$summary.failures = $failures +$code = $(if ($failures.Count -gt 0) { 10 } else { 0 }) +Write-Host ("negative: {0}" -f $(if ($code -eq 0) { 'every case as required' } else { "$($failures.Count) failure(s): " + ($failures -join '; ') })) +Write-Summary $code $(if ($code -eq 0) { 'AS REQUIRED' } else { 'NOT AS REQUIRED' }) +exit $code diff --git a/benchmark/SAM.Analytical.OpenStudio.ResultsReadBack/SAM.Analytical.OpenStudio.ResultsReadBack.csproj b/benchmark/SAM.Analytical.OpenStudio.ResultsReadBack/SAM.Analytical.OpenStudio.ResultsReadBack.csproj new file mode 100644 index 0000000..b439dab --- /dev/null +++ b/benchmark/SAM.Analytical.OpenStudio.ResultsReadBack/SAM.Analytical.OpenStudio.ResultsReadBack.csproj @@ -0,0 +1,57 @@ + + + + Library + net8.0 + SAM.Analytical.OpenStudio.ResultsReadBack + false + x64 + x64 + x64 + latest + disable + disable + + + + + + + + + + + + + + + + ..\..\..\SAM_Validation\build\SAM.Analytical.Benchmark.dll + true + + + ..\..\..\SAM\build\SAM.Core.dll + + + ..\..\..\SAM\build\SAM.Geometry.dll + + + ..\..\..\SAM\build\SAM.Analytical.dll + + + ..\..\..\SAM\build\SAM.Architectural.dll + + + ..\..\..\SAM\build\SAM.Weather.dll + + + + diff --git a/documentation/S11-OpenStudio-ResultsRoundTrip-PR.md b/documentation/S11-OpenStudio-ResultsRoundTrip-PR.md new file mode 100644 index 0000000..87afb47 --- /dev/null +++ b/documentation/S11-OpenStudio-ResultsRoundTrip-PR.md @@ -0,0 +1,478 @@ +# S11: OpenStudio results round-trip validation + +Roadmap row S11 (`SAM/documentation/SIMULATION_ROADMAP_Q4_2026.md`): SAM-imported results and B1a against an +independent SQL/ESO read; label total versus sensible cooling (F3). + +> Phase B remains subject to S10's declared validation limitation (`SAM|FixB|Heating|designDayPeak.hourOfDay` BLOCKED); +> an S11 PASS is not a Phase B PASS. + +## Status + +- **Revision 1 (code + offline tests) only**, owner-authorised 2026-10-24 together with approval of the revised S11 + plan (local, non-Git: `SAM-BIM\S11_IMPLEMENTATION_PLAN.md`, plan revision 2). Draft PR into `sow/2026-Q4` from + `feature/s11-openstudio-results-roundtrip`, base `ec204cbb`. +- Not authorised and not done: EnergyPlus/OpenStudio runs (plan section 9, Revision 2), merge, Ready, any other stage. + No simulation, CLI engine run or Tas/COM operation was performed. No S11 acceptance claim exists yet. +- Independent review of Revision 1 (2026-10-24, reviewed head `06084afb`, tested `246c8fed`): **REVISE** with two + findings, R1 (acceptance driver) and R2 (reporting frequencies / weather-run environments). Both are corrected in + "Revision 1 correction" below, code and offline tests only, on this same Draft PR. +- The R1/R2 correction was independently accepted (2026-10-24, tested `059aed6f`, head `9ce146cd`). +- **Offline integration of the production fixes (2026-10-24, owner-authorised):** F-S11-1 (SAM_OpenStudio#35, merged + `4b020585`) and F-S11-2 (SAM#219, merged `ac68ad0f`) are integrated; the tests that pinned both defects are replaced by + positive regressions on the signed-off fixtures. Tested content `76d61612ebe477f57cb44517f82e5353b2abe3b0` (on Q4 merge + `43e3541c3cdcc8b0a43f4c73f062db694aafbdf2`); see "Integration of F-S11-1 and F-S11-2". + No engine run, Ready, merge, SAM_UI run, S12 or Phase B acceptance. +- Tested content SHA: see "Tests and validation" (recorded after each code commit, in a record-only commit). +- The integrated head `01344e65` was independently reviewed (2026-10-26): **ACCEPT for the offline integration only** + (machine-local `SAM-BIM\.review-s11-integration-20261026\REVIEW.md`). +- **Revision 2 (owner-authorised 2026-10-26): BLOCKED before engine execution.** The controlled receipt build succeeded, + but `Run-S11-Results.ps1 -Phase run` stopped in its preflight (exit 70) because the driver's `Git` helper calls itself + instead of `git.exe`, so no HEAD can be read for a real receipt. **0 of the 2 authorised engine attempts were used**; + no simulation, readback, compare or negative ran. See "Revision 2 execution (2026-10-26): BLOCKED". +- Next: owner decision on the bounded driver correction recommended there; Revision 2 is then repeated on a new receipt. + +## Owner directions applied (2026-10-24) + +1. Revision 1 only; plan saved as `S11_IMPLEMENTATION_PLAN.md` (verbatim, with the approval header). +2. Both new projects (`SAM.Analytical.OpenStudio.ResultsReadBack`, `SAM.Analytical.OpenStudio.ResultsReadBack.Cli`) + are in `SAM_OpenStudio.sln` and in the CI build checks: a new `build.yml` step fails if either is missing from the + solution or produced no output after the solution rebuild; the S11 tests run in the existing read-back test step + (renamed "S09 native-input and S11 results read-back tests"). This is the only workflow change. +3. Duplicate room names are handled through geometry and identity, never rejected because names match: the reference + identity is the S09 geometry bijection (names only ever contradict it). Tests: two spaces with the same name PASS + (`S11IdentityTests.DuplicateSpaceNames_DistinctGeometry_ResolvedByGeometryAndPass`); with the same name AND the + fixture's shared GUID prefix the identity still resolves both by geometry and only the production naming collision is + reported (`..._SharedGuidPrefix_IdentityStillByGeometry_ProductionKeyCollisionReported`). + +## Revision 1 correction (review findings R1, R2; 2026-10-24) + +Scope as directed: code and offline tests only; no EnergyPlus run, acceptance run, merge, Ready or next stage; the +importer, SAM core and the fixtures are unchanged. + +| Finding | Disposition | +|---|---| +| **R1** `Run-S11-Results.ps1` recorded comparator exits without enforcing them, could read an earlier report after an infrastructure error, and recorded unexpected negative outcomes and changed originals without failing (review probe: two comparator exits 70 + old PASS reports -> printed PASS, exit 0). | **Fixed.** Exit 70 STOPPED at once on any step that does not complete (exit other than 0/10/11, no exit code, cannot start), on an output a step did not write itself (each output's write time + hash is taken before and after the step, so no clock comparison), on a compare report whose verdict/ExitCode disagree with the exit or whose `Inputs` hashes are not the files of that directory, and on a changed original; a stop writes `summary-.json` with `result = STOPPED`, never an earlier verdict. Before and after each run (readback: after compare; negative: after every case) the model, `in.idf`, SQL, ESO, `b1a.json` and `sam.json` must equal the run-manifest hashes, the manifest paths must lie in the run directory, and the active receipt, harness and producer must equal the hashes the run recorded; the receipt must list the executables the driver starts. readback collects numerical FAIL/BLOCKED for both fixtures and exits 10 (any FAIL) / 11 (any BLOCKED, or OD-7 met by neither fixture) / 0. negative takes each case's required exit from the driver (negative.json must agree or the phase stops), checks that each copy differs from the original in exactly the files the case mutates, exits 10 on any case not as required and on a control that did not PASS; without a PASSing control no mutation outcome of that fixture is marked evidence. `-HarnessExe`/`-BenchmarkExe` added for the stand-in tests (must still be receipt files). | +| **R2** `SqlResultRead`/`EsoResultRead` dropped non-Hourly dictionary entries of the required variables (review copy `extra-frequency-two/`, an extra ESO `!TimeStep` Total Cooling entry + record `999999,0`: still PASS 85/85); the SQL reader accepted a second type-3 environment named otherwise (production reads every type-3 environment). | **Fixed.** Both readers judge every dictionary entry of a required variable (SQL name match now case-insensitive, as ESO): Hourly entries form the series; any other frequency for that key is an S11-C1 problem that blocks it, whatever its values. Meter frequency handling is unchanged (separate, optional). SQL: exactly one EnvironmentType 3 environment, named as the RunPeriod, else a global block. Rerun of the review copy with the rebuilt CLI: **BLOCKED** (68 PASS, 0 FAIL, 17 BLOCKED; exit 11), reason `unsupported reporting frequency 'TimeStep' (report id 999999 ...)`. | + +Tests added: `S11DriverTests` (42 cases; Windows PowerShell 5.1 runs the real driver against stand-in run folders, +manifests and receipt, with `S11DriverStandIn.ps1` as the harness and an inert producer that is asserted never to +start): fully passing control (0); the review reproduction, both comparators 70 with old PASS reports (70, no `two` +step, no PASS printed); read 70, subject 64, subject/compare exit 0 without a fresh output, verdict/exit disagreement +both ways, report judging other files (each 70); numeric FAIL collected for both (10); BLOCKED (11); FAIL + BLOCKED (10); +OD-7 met by neither (11); 10 changed inputs/builds x 2 phases (idf, SQL, ESO, b1a, sam, recorded model hash, receipt of +another build, changed harness assembly, harness not in receipt, manifest path outside the run; each 70 before any step +of that run); an original changed by a readback step and by a negative case (70); negatives all as required (0); +mutation not detected (10, collected); failing control (10, no evidence); copy not mutated (10); negative.json +disagreeing with the driver (70); negative compare 70 (70); run phase without `-ConfirmEngineRuns` (70, nothing +created). `S11CompletenessTests` gains three BLOCKING defects: SQL extra `HVAC System Timestep` entry with a zero value, +ESO extra `!TimeStep` entry with a zero record (the review case), and a second SQL type-3 environment with another name +and rows in both files (paired production-reader check: it still returns values). + +## Integration of F-S11-1 and F-S11-2 (offline, 2026-10-24) + +Owner-authorised offline integration on this Draft PR. Write scope SAM_OpenStudio (this branch); SAM, SAM_Tas, +SAM_Validation and SAM_SQLite reference-only. No EnergyPlus/OpenStudio CLI simulation, Tas/COM operation, SAM_UI run, +Ready, merge, deployment, S12 or Phase B acceptance. Revit compatibility (SAM#219) stays deferred by the owner. + +**Branch update.** `sow/2026-Q4` (`194ee1e6`, containing #35 merge `4b020585` and its closeout) merged into this branch +as `43e3541c` (merge commit, no conflicts, no history rewrite). The merge brings the importer fix +(`OpenStudioSimulationRunner.cs`), its tests (`tests/.../C5/ZoneResultIdentityTests.cs`), its record and the Q4 +progress closeout; this PR's own diff is unchanged by it. + +**Tests replaced** (tested content `76d6161`, test project only; no production, library, CLI, script, fixture or +workflow change): + +| Former test (pinned a defect) | Replacement (positive regression) | +|---|---| +| `Finding_SharedGuidPrefix_ImporterLosesUnmetHoursOfOneZone_S11Fails` (expected FAIL on the unmet rows of FixB) | `SignedOffTwoZone_SharedGuidPrefix_EachSpaceKeepsItsOwnResults`: the **unchanged** signed-off `s08-fix-two` (FixA `5080c000-…c0a1`, FixB `5080c000-…c0b1`) with zone-specific fractional not-met time; verdict PASS with 0 FAIL/BLOCKED; native Space, Zone and Ideal Loads distinct (2 each); each space's production Ideal Loads/ThermalZone key = its own native key (6 identity rows PASS); reference, SAM-P, SAM-I and B1a unmet hours exact per space (zone 0 14.5/14.75 h, zone 1 7.25/5.25 h, so a swap FAILs); the 4 SAM-I annual results attached to the space they reference (both spaces); no "matched no SAM space" / "No simulation result found" diagnostic. | +| `Finding_FractionalNotMetTime_StoredAsWholeHours_S11Fails` (expected b1a heating 14) | `FractionalUnmetHours_SurviveSqlImport_SamSaveReload_AndB1a` (`s08-fix-box`): independent SQL/ESO reference 14.5 h heating / 14.75 h cooling; SAM-P, SAM-I and B1a rows PASS with reference = subject = 14.5 / 14.75 exactly; the producer's SAM objects after SQL import, the same objects after `sam.json` save and `OpenStudioBenchmarkInputs.Read`, the written `b1a.json` (`unmetHours.value`), and the SAM_UI/GH import (`Modify.AddResults`) saved as an `AnalyticalModel` SAM JSON and reopened, read through `TryGetValue`: all 14.5 / 14.75. | + +Other test changes: the re-GUIDed two-zone variant (`TwoDistinctPrefix`, space B -> `6181d111-…`) is retired; the +consistent two-zone CLI run, completeness, F3 and negative tests now use the signed-off fixture +(`S11Support.Two()`), so those offline proofs now also run on the shared GUID prefix. Synthetic fractional not-met time +is binary-exact quarter hours that differ per zone (was 0.5 h heating / 1/6 h cooling in every zone); the default +whole-hour data, every other series and all negatives are unchanged. The same-name tests in `S11IdentityTests` are kept: +same name + distinct prefix PASSes; same name + same prefix still reports the production key collision (the separately +tracked generated-name identity residual, left unmapped by #35 rather than guessed) and is now documented as such. + +**Fail-before controls** (the tested test binaries with one dependency swapped for its pre-fix build; same filter): +- Pre-#35 importer (`SAM.Analytical.OpenStudio.dll` `b251fec0…`, built 2026-10-24 10:42 from this branch before the + merge; has no `UniqueZoneIdentities`/`ReadZoneNames`): two-zone regression **FAILs** (FixA reads FixB's not-met time: + delta 7.25 h heating, 9.5 h cooling, in SAM-P, SAM-I and B1a); the box regression passes (one zone). +- Pre-#219 SAM (`SAM.Analytical.dll` `1492f3ed…`, local `SAM\build` built 2026-10-23 from `b7c2c36a`): both + regressions **FAIL** (half-to-even rounding: 14.5 -> 14, 14.75 -> 15; deltas 0.5 / 0.25 h). +- Baseline before the test change (merged code `43e3541c`, old tests): 240/242; exactly the two `Finding_` tests fail + with "Expected FAIL, but was PASS", i.e. the integrated fixes remove both defects on the signed-off fixtures. + +### Dependency provenance (local, isolated) + +Isolated sibling workspace `SAM-BIM\.s11i\ws` (machine-local; short path for MAX_PATH): SAM exported with `git archive` +at merged Q4 `457defbb033fa4a58128f2395b87587f0210c608` (contains #219 merge `ac68ad0f…`) and built there +(`dotnet build SAM/SAM.Analytical/SAM.Analytical.csproj -c Release -p:APPDATA=`; 0 errors); SAM_Tas source +export `922044fa9ca35062979689f4e359ea85bee577dd`; SAM_Validation `build/` from the local checkout at Q4 +`2a3acfafb3919b4a0b81d67ef16a4b9d1e222f5d`; SAM_SQLite `build/` from the local checkout at `ec2eb86d00cac6e4cadf52b429f9d293eeabbd5e` +(both checkouts clean, equal to remote Q4); SAM_OpenStudio exported at `43e3541c` then the test edits (tree compared +equal to `76d6161`, excluding generated output). `APPDATA` redirected for every build; no Grasshopper project built, no +`%APPDATA%\SAM` deployment; the repository checkouts and their `build/` folders were not rebuilt or modified. + +| Assembly (loaded by the test binaries) | SHA-256 | +|---|---| +| `SAM.Analytical.dll` (SAM `457defbb`; all 8 copies in the test/CLI outputs) | `21bec17f66e1c7cff13444ef29025adc96cab8a324237cd58fc2750f76e86536` | +| `SAM.Core.dll` | `43ad048540a48310598b4910ace9287b0dcbb5eb116533adc3124a5fd154c96b` | +| `SAM.Geometry.dll` | `f51b88d0691a4c160e059308096586aec618e8f4193ef550cc12715edf8274d5` | +| `SAM.Architectural.dll` | `fdacd08c21180048ff98b2596c0b8dbb48d36c225303332d8558b0df3d18c338` | +| `SAM.Weather.dll` | `02310d05ecd746c36f5fd660a50702645ebd4fc3a470f4935ecc9c703eca4fec` | +| `SAM.Math.dll` | `f33840c1fbf79f1bae1725eb6808a6344ad6fcbf463ae85139b7e0c53ef53ac4` | +| `SAM.Analytical.OpenStudio.dll` (importer incl. #35, built in the workspace) | `5d1e50df731b07ebc3a85eb4ded0a0ef6d24e1c536a5515fc28ccec561a49cfb` | +| `SAM.Analytical.Benchmark.dll` (SAM_Validation `2a3acfaf`) | `0dfdc8ec8f9b65f33d50a914ae4dd48c7d3be151abeb435e81a8ae4cee9d087b` | +| `SAM.Core.SQLite.dll` (SAM_SQLite `ec2eb86`) | `ba9400e9b9969ed8f16d3c1c4dac987bd4d70d9e067501546ba1cbe8d5a260c4` | + +The SAM_Validation and SAM_SQLite hashes equal those recorded for SAM#219's validation. The SAM build is not +byte-identical to SAM#219's tested build (`eacd1ed8…`, other path); its behaviour is shown by the tests above. + +### Validation (local, Release x64, tested content `76d6161`) + +| Check | Command (from the isolated `SAM_OpenStudio`, `$A` = scratch APPDATA) | Result | +|---|---|---| +| Affected builds | `MSBuild.exe -restore -t:Rebuild -p:Configuration=Release -p:Platform=x64 -p:UseSharedCompilation=false -p:APPDATA=$A -nr:false -v:m` for `InputReadBack.Tests` (S11 library, CLI, producer, S09 reader/CLI), `Benchmark.Tests` and `tests/SAM.Analytical.OpenStudio.Tests` | exit 0, 0 errors, 0 CS warnings (existing MSB3277 System.Memory only). Note: `-t:Restore;Rebuild` in one call failed on a fresh tree (CS0400 `global::OpenStudio`: restore output not visible to the same evaluation); `-restore` fixes it. | +| Read-back tests (S09 + S11) | `dotnet test benchmark/SAM.Analytical.OpenStudio.InputReadBack.Tests -c Release -p:Platform=x64 --no-build` | **242 passed, 0 failed, 0 skipped**, 6 min 22 s (120 S09 + 122 S11; the two replacements counted in place of the two `Finding_` tests) | +| Producer tests | `dotnet test benchmark/SAM.Analytical.OpenStudio.Benchmark.Tests -c Release -p:Platform=x64 --no-build --filter "TestCategory!=Simulation"` | 58 passed, 0 failed, 0 skipped | +| Importer offline tests (incl. #35's 17 `ZoneResultIdentityTests`) | `dotnet test tests/SAM.Analytical.OpenStudio.Tests -c Release -p:Platform=x64 --no-build --filter "TestCategory!=Simulation"` | 488 passed, 0 failed, 2 skipped (the two real-SQL reproduction fixtures are not committed), 6 min 58 s | +| S09 saved-run replays (offline) | `os-input-readback compare --run C:/TasOut/s09/acc7-u1/ --fixture-id s08-fix- --rule-set approved --out `, on merged `43e3541c` and on `76d6161` | exit 0 all four; `report.txt` byte-identical to committed `acceptance-2026-10-23/runs//report-approved.txt` (`868a5808…`, `9e3a3f0e…`) | +| Fail-before controls | the two regressions on copies of the tested binaries with the pre-fix DLL | as above: pre-#35 importer 1 FAIL / 1 PASS; pre-#219 SAM 2 FAIL | +| Fixtures | SHA-256 of `benchmark/SAM.Analytical.OpenStudio.InputReadBack/Fixtures/s08-fix-{box,two}.json` | `45524172…`, `20da9b58…` (unchanged, signed-off) | + +Machine-local evidence (copy before switching machines): `SAM-BIM\.s11i\ev\` (`sources.txt`, `sam-build-hashes.sha256`, +`control-dll-hashes.sha256`, `logs\` with build logs, TRX and console logs, `s09-replay-base\`, `s09-replay-tested\`) and +the control binaries `SAM-BIM\.s11i\ctl\`. Not run: any `Simulation`-category test, `Run-S11-Results.ps1` run phase, +`Build-S11-Receipt.ps1`, SAM_UI, Tas. After the runs the isolated workspace holds no `eplusout.end`, `eplusout.err` or +`run.log` and no EnergyPlus/OpenStudio process was running; the `eplusout.sql/.eso` files under `s11-runs` are the +synthetic generator's. + +**Separately tracked identity concerns (visible, not changed here; none blocks this integration):** +- `Modify.AddResults` `LookupSpace`/panel lookups resolve design-day (ZoneSizes) and surface results by the 8-character + Guid suffix first. On `s08-fix-two` they would pick FixA for FixB's design-day or surface rows. S11 is not affected on + these fixtures: the annual family references the space Guid directly (proved by the SAM-I attachment assertions), + the fixtures have no design days (no ZoneSizes, OD-4 scope) and S11 compares no surface results. The same lookup + feeds the annual "SQL zone '...' matched no SAM space" diagnostic, which therefore cannot fire for a FixB key whose + prefix matches FixA (it never names a valid zone wrongly; it can only stay silent). +- Generated-name identity: two spaces with the same name AND the same Guid prefix get identical generated names; + #35 leaves such keys unmapped and `ToSAM_SpaceSimulationResults` looks values up by the exact generated name. Not + present on the signed-off fixtures; pinned by `S11IdentityTests.DuplicateSpaceNames_SharedGuidPrefix_...`. + +## Revision 2 execution (2026-10-26): BLOCKED before engine execution + +**Authorisation.** The owner authorised exactly two OpenStudio simulations (box and two-zone, one attempt each, no retry), +then the readback, compare and negative phases, evidence packing, record/description update, commit and push. Ready, +merge, S12, Phase B acceptance, `PROJECT_PROGRESS.md`, the roadmap and other repositories are excluded. + +**Result: BLOCKED. Engine attempts used: 0 of 2.** No `benchmark-openstudio` process, EnergyPlus or OpenStudio +simulation started. The acceptance root holds only the driver's `summary-run.json` (`STOPPED`, exit 70). There is no SQL, +ESO, `in.idf`, B1a, `sam.json`, readback, compare or negative output, so no acceptance criterion (completeness, comparison +rows, SAM-P/SAM-I/B1a agreement, closure, negatives, OD-7) has been evaluated on real data. + +**Workspace (machine-local).** +- Root: `SAM-BIM\.s11r2-20261026\`. Fresh Git clones of `github.com/SAM-BIM/*` (fetch and push verified) sit under + `ws\` as siblings, mapped temporarily as `R:` for Windows path length. Each clone was checked out at its pin: + - SAM_OpenStudio `01344e659528ec49a746d9c165ee3914d7b1cca0` (branch `feature/s11-openstudio-results-roundtrip`); + - SAM `457defbb033fa4a58128f2395b87587f0210c608`; + - SAM_Tas `922044fa9ca35062979689f4e359ea85bee577dd`; + - SAM_Validation `2a3acfafb3919b4a0b81d67ef16a4b9d1e222f5d`; + - SAM_SQLite `ec2eb86d00cac6e4cadf52b429f9d293eeabbd5e`. +- Redirected for every step: `APPDATA`, `LOCALAPPDATA`, `TEMP`/`TMP`, `DOTNET_CLI_HOME` and the NuGet caches, all to + `.s11r2-20261026\env\` (`scripts\env.ps1`). No Grasshopper project was built and nothing was written to `%APPDATA%\SAM`. +- Existing checkouts, their `build/` folders, `.s11i`, earlier reviews and historical `C:\TasOut` evidence were not touched. + +**Inputs (verified).** +- Fixtures `s08-fix-box.json` `45524172…acd08` and `s08-fix-two.json` `20da9b58…9baa0`, equal to `SHA256SUMS.txt`. +- Budapest EPW decompressed from SAM `457defbb` `SAM/SAM.Tests/Resources/Weather/HUN_BU_Budapest.Met.Center.128400_TMYx.epw.gz` + (`24b14f83…`) to `.s11r2-20261026\weather\budapest.epw`, `472ffb4e2f10867600949a392ce46891fc1bd05f2fc8d8917a99bca7d26209c3`. +- Portable CLI `SAM-BIM\.icon-migration\runtime\OpenStudio-3.10.0+86d7e215a1-Windows\bin\openstudio.exe`; `--version` + prints `3.10.0+86d7e215a1`. + +**Build (local, succeeded).** +- `SAM.Core.SQLite` was built first: `MSBuild.exe R:\SAM_SQLite\SAM_SQLite\SAM.Core.SQLite\SAM.Core.SQLite.csproj -restore + -t:Rebuild -p:Configuration=Release -p:SAMSourceRevision=ec2eb86d… -nr:false -v:m` (exit 0; the stamp is present). +- Then `Build-S11-Receipt.ps1` ran with its defaults from the R: clone (exit 0, 0 errors). The receipt + (`s11-build-receipt.json`, 51 files, `c057cf47ca82c2eb5d61fd5ea2f714a059184d19bc48cf4a748747ceffa2f602`) records the + four clean HEADs. +- C# warnings are only in the SAM core projects (Core 618, Geometry 3, Analytical 2; existing). SAM_Validation and the + SAM_OpenStudio projects have none; there are 50 existing MSB3277 warnings. +- All five clones were still clean at their pins after the build. + +| Fresh DLL (identical in the producer and harness folders) | SHA-256 | +|---|---| +| `SAM.Analytical.dll` (SAM `457defbb`) | `feac8dab69ff3483384cac4fa19a376e528c681f7e0037023442360bcaf23977` | +| `SAM.Analytical.OpenStudio.dll` | `deddf7140715cf59942c912d363e7ef4a93627b5adf2e8a217b74552c4364489` | +| `SAM.Analytical.Benchmark.dll` (SAM_Validation `2a3acfaf`) | `17766896e9f2eb7eb80b0fcb555b68983df7ed9217c85ebd1c72ec1992970935` | +| `SAM.Core.SQLite.dll` (SAM_SQLite `ec2eb86d`) | `3d1d9033a4edce3644c65a15644b2e96513a25f722684a03c806659825e514ee` | +| `benchmark-openstudio.dll` | `5866ce57cd878d55d065e7d708b9ae51a87355c9d2af6e0434b3934164271de2` | +| `os-results-readback.dll` | `def1a9f7b612ad77182d40dcd4f28ad07ea5650f13d2c38b3dd1338e349cc73a` | + +These differ from historical builds, which is expected for a new build path; the receipt lists every `.dll`/`.exe`. + +**Run phase (stopped in preflight).** +- Command: `Run-S11-Results.ps1 -Phase run -Root C:\TasOut\s11\r2-20261026 -WeatherEpw + -WeatherEpwSha256 472ffb4e… -OpenStudioCli -OpenStudioCliVersion 3.10.0+86d7e215a1 + -BuildReceipt -BenchmarkExe -HarnessExe + -ConfirmEngineRuns`. The root was new; no engine process was running. +- Started 17:08:46Z and stopped 17:15:19Z: `STOPPED: R:\SAM is not clean at the receipt commit 457defbb…`, exit 70. +- The stop came from `Test-Receipt` (`Run-S11-Results.ps1:146-151`), which runs before the fixture, EPW and CLI checks + and before any simulation. +- At that moment `R:\SAM` was clean at `457defbb` (`git status` showed 0 entries; no file under any clone was newer + than the receipt). + +**Diagnosis (defect in the driver; the checkout was not dirty).** +- `Run-S11-Results.ps1:103-106` defines `function Git([string]$Directory) { ... & git -C $Directory @args 2>$null }`. +- In PowerShell a function shadows an application of the same name, case-insensitively. So `& git` inside `Git` calls + `Git` again: inside the helper, `(Get-Command git).CommandType` is `Function`. +- The recursion ends in a call-depth overflow. `2>$null` and `ErrorActionPreference = 'Continue'` swallow it, so the + helper returns nothing after about 200 s per call. +- The replayed helper (`proof-git-recursion.txt`) returned `head=[]` after 192.4 s, with the recorded error + `System.Management.Automation.RuntimeException: The script failed due to call depth overflow.` The `Test-Receipt` + replay (`diag-receipt-1.txt`) returned an empty HEAD after 194.1 s. +- Consequences: `Test-Receipt` sees an empty HEAD for the first receipt commit and stops all three phases (run, readback + and negative) on every real receipt. The run phase's `$runnerHead`/`$samHead` (`:239-240`) would also be empty. +- `git.exe` called directly returns `457defbb…` at once. +- Why the offline tests missed it: `S11DriverTests` writes every stand-in receipt with `"commits": []` + (`S11DriverTests.cs:102`), so neither the commit check nor the head lookup was ever executed. +- `Build-S11-Receipt.ps1:61-66` is not affected while `C:\Program Files\Git\cmd\git.exe` exists, because it calls `& $git` + with that full path. Its fallback `$git = 'git'` would recurse the same way. + +**Bounded correction recommended (not implemented here; it needs an owner decision and the usual review).** +- **Driver:** in `Run-S11-Results.ps1`, resolve git as an application, e.g. + `$gitExe = (Get-Command git -CommandType Application -ErrorAction Stop | Select-Object -First 1).Source`, and call + `& $gitExe`. Alternatively rename the helper (e.g. `Invoke-Git`). Fail closed when git exits non-zero or prints no + 40-hex HEAD, instead of swallowing the error. +- **Receipt build:** apply the same application resolution to the fallback in `Build-S11-Receipt.ps1`. +- **Tests:** add `S11DriverTests` cases with a receipt that lists a real temporary Git repository: + - clean at the recorded commit → passes the check; + - dirty or at another commit → 70; + - the run phase records non-empty `runnerCommit`/`samCommit` using the inert producer stand-in. +- **Out of scope for the correction:** fixtures, tolerances, gates, metric definitions and production code. +- **Afterwards:** a new receipt build at the corrected runner commit, then Revision 2 under a fresh owner instruction. + The engine allowance (2 attempts) is unused. + +**Evidence.** +- Committed: `documentation/S11-OpenStudio-ResultsRoundTrip-evidence/revision-2-2026-10-26/`, holding the stopped + `summary-run.json`, the receipt, `SHA256SUMS.txt` (written by `Pack-S11-Evidence.ps1 -Root C:\TasOut\s11\r2-20261026 + -Label revision-2-2026-10-26`) and a README. +- Machine-local (copy before switching computers): + - `C:\TasOut\s11\r2-20261026\summary-run.json` (`192ec65f…`); + - `SAM-BIM\.s11r2-20261026\` (`scripts\`, `logs\`: build logs, `phase-run.log`, `phase-commands.txt`, + `diag-receipt-1.txt`, `proof-git-recursion.txt`, `engine-check.txt`; `hashes\`; `weather\`; `ws\` clones and builds). + +## Work completed + +| Plan item (section 6) | Implementation | +|---|---| +| 1. `ToBenchmark` split, zero rule, no-zone consumption, docs | `Modify.BenchmarkInputs(model, context)` returns `OpenStudioBenchmarkInputs` (model result, space results, design-load results, consumption-keyed flags); `ToBenchmark(model, context, inputs)` maps exactly those instances and reads only provenance from the context; the two-argument form calls both. `Peak()`: load exactly 0 = available 0, hour unavailable (space and model). Consumption with no zone key = unavailable. | +| 2. F3 note, `--sam-results` | `Modify.TotalBasisNote` added once to every Success document. `benchmark-openstudio --sam-results ` writes the same instances (envelope `s11-sam-results/1`, SAM JSON, UTF-8 no BOM, LF) after the B1a and prints their SHA-256; equal to `--out` is a usage error before any run. | +| 3. RB09 identity helper | `InputReadBack/Compare/NativeZoneIdentity.cs` (`SurfacesOf`, `MatchSpace`, `IdealLoadsOf`, `Resolve`); `OsComparison` now calls it. Behaviour-neutral: offline replay of the S09 `compare` on `C:\TasOut\s09\acc7-u1\{box,two}` gives `report.txt` byte-identical to the committed `acceptance-2026-10-23/runs/*/report-approved.txt` (`868a5808…` box, `9e3a3f0e…` two) before and after. | +| 4. `ResultsReadBack` library + CLI, scripts | Library (no production project, no SDK): `Native/RunCalendar.cs`, `HourlySeries.cs` (completeness), `SqlResultRead.cs`, `EsoResultRead.cs`, `Reference/ResultReference.cs`, `Compare/ResultCompare.cs`, `Report/ResultReport.cs`, `Classes/RunLayout.cs`, `SubjectDocument.cs`. CLI `os-results-readback`: `read`, `subject`, `compare`, `negative`. Scripts `Build-S11-Receipt.ps1`, `Run-S11-Results.ps1` (run phase refuses without `-ConfirmEngineRuns`), `Pack-S11-Evidence.ps1`; parse-checked under Windows PowerShell 5.1, not executed beyond the refusal guard. | +| 5. Tests in CI-run projects | `InputReadBack.Tests`: `S11Synthetic.cs` (real-schema SQL + ESO generator keyed by the S09 export IDF of a fixture), `S11Support.cs`, `S11ResultsTests`, `S11CompletenessTests`, `S11F3Tests`, `S11NegativeTests`, `S11IdentityTests` (+ unit tests), `S11SourceScanTests`. `Benchmark.Tests`: `S11ProducerTests`; golden updated by the note only. | +| 6. Record | This file. | + +### Reference design as built + +- **Calendar (C1):** from the single `RunPeriod` of the run's `in.idf`: must be 1/1-12/31 of a non-leap year (blank year + allowed); 8760 keys (month, day, EnergyPlus hour 1-24). Anything else blocks every row. +- **Per file, per (zone x quantity) (C1/C1b):** every dictionary entry is judged: exactly one Hourly entry and no + entry of another reporting frequency (both channels); SQL: rows of the RunPeriod environment (the only type-3 + environment, named as the RunPeriod) with `Interval 60`, `Minute 0`, `Dst 0`, `WarmupFlag 0`, no repeated + `TimeIndex`; ESO: exactly one RunPeriod environment block, hourly stamps `StartMinute 0 / EndMinute 60`, DST 0, each id + once per hourly block. The multiset of stamps must equal the calendar exactly (count alone never accepted). A series + absent from either file blocks. +- **C3** units J/hr, finite, non-negative. **C4** SQL = ESO per hour, class E. **C5** geometry identity bijective, exactly + one Ideal Loads per zone, every SQL/ESO key an identified Zone/Ideal Loads (unknown key blocks). **C6** Total >= Sensible + each hour (else that load type's value rows block). **C7** each facility meter = sum of zone Total (R2) in each file + where present (else informational); a disagreement blocks that consumption. **C8** ZoneSizes present iff + `designDaySource != None`. **C9** EnergyPlus 25.1 in SQL `Simulations` and the ESO header. +- **Rows:** `reference`, `identity` (native, production Ideal Loads/Zone key = native key, B1a space set, provenance), + `closure` (sam.json re-mapped by the producer = byte-identical B1a `model` and `spaces`; otherwise every SAM-P row + blocks), SAM-P (model consumption R2, coincident peak R1, hour R0 earliest tie with SAM's hour 0 for an all-zero year, + floor area/volume R1; per space x load type result, peak W R1, hour R0, unmet R2, design load), SAM-I (same space rows + plus attachment; design-load value informational), B1a (as SAM-P with the zero rule, unmet, design load, Total-basis + note). Verdict PASS only when every gated row PASSes; any FAIL gives FAIL (exit 10), otherwise BLOCKED (exit 11). +- **F3 margins:** every value row of a Total quantity reports `|ref(Total) - ref(Sensible)| / tolerance`; hour rows + report whether the Sensible peak hour differs. `F3.MeetsOd7` (consumption and one space cooling peak >= 1000) is + informational per run, for the OD-7 cross-fixture judgement in Revision 2. +- **Independence:** the library references only System.Data.SQLite, the S09 reader (`InputReadBack`) and SAM core/schema + assemblies (asserted from the project file and the loaded assembly references); a source scan forbids the runner, + importer/producer conversions, production naming/unit helpers, `AddResults` and the production namespaces in every + library file, with a planted negative. Only `subject` (and the substitution negative) run production code. + +## Findings F-S11-1 and F-S11-2 (RESOLVED by separate production PRs; integrated below) + +Both were fixed outside this PR, each in its own reviewed and merged production PR, and are integrated here (section +"Integration of F-S11-1 and F-S11-2"). The text below is the original Revision 1 finding and proposal, kept as history. +F-S11-1 was fixed differently from the proposal: identity is proven unique by the SQL `Zones` inventory (see +`documentation/F-S11-1-Zone-Result-Identity-PR.md`). F-S11-2 was fixed as proposed (`UnmetHours` is a +`DoubleParameterValue(0, 8760)`; `OccupiedUnmetHours` unchanged). + +- **F-S11-1 (importer, likely to FAIL on the real two-zone fixture):** both `s08-fix-two` spaces have GUIDs beginning + `5080c000`. The importer remaps zone-level report keys (setpoint not-met time) onto Ideal Loads keys by the **last 8 + characters** (`OpenStudioSimulationRunner.NormalizeKey`), so one zone's unmet hours overwrite the other's and one space + gets none. The geometry identity resolves both zones; S11 FAILs the unmet rows of the affected space in all three layers + (`S11ResultsTests.Finding_SharedGuidPrefix_ImporterLosesUnmetHoursOfOneZone_S11Fails`). The positive two-zone tests use + a variant with space B re-GUIDed (`6181d111-…`), recorded as such. +- **F-S11-2 (SAM core semantics, likely to FAIL on real runs):** `SpaceSimulationResultParameter.UnmetHours` is declared + `IntegerParameterValue(0, 8760)` (SAM `SAM.Analytical/Enums/Parameter/SpaceSimulationResultParameter.cs:48`), so a + fractional EnergyPlus not-met total (timestep fractions; the fixtures run 6 timesteps/hour) is stored as whole hours + (14.5 h -> 14, 3.33 h -> 3). S11 compares the exact sum (R2) and FAILs + (`Finding_FractionalNotMetTime_StoredAsWholeHours_S11Fails`). The default synthetic data use whole-hour not-met values + so the other rows are exercised; no tolerance or semantic was changed. +- The independent review confirmed both as production defects, not engine differences. They stay visible FAILs on the + signed-off fixtures: the fixtures, tolerances and reference values are unchanged, no row is informational, and the + re-GUIDed synthetic variant remains unit-test coverage only, never acceptance evidence. + +### Recommended bounded correction (historical proposal; superseded by SAM_OpenStudio#35 and SAM#219) + +Recommendation: fix both before Revision 2 (real acceptance), each in its own PR, then rerun this PR's offline tests. + +- **F-S11-1, SAM_OpenStudio importer only.** `OpenStudioName` (SAM.Core.OpenStudio `Query/OpenStudioName.cs`) is + `SAM___`; `NormalizeKey` (`OpenStudioSimulationRunner.cs:1033`) compares only the + last 8 characters and returns the FIRST energy key that shares them. Smallest correction: compare the identity part + after `SAM__` (`_`, case-insensitive), and leave a key unmapped when zero or more + than one energy key match (no first-wins overwrite). This fixes `s08-fix-two` (FixA/FixB share `5080c000` but not + names). It also corrects the other `NormalizeKeys` consumers that share the defect (internal gains, window solar, + infiltration, temperature/RH series, `OpenStudioSimulationRunner.cs:968-981`). Compatibility: models whose 8-character + prefixes are unique map exactly as now; colliding models change from silently wrong to correct. Residual (stated, not + fixed): same sanitized name AND same GUID prefix stays ambiguous; it becomes visibly unmapped. Only full-GUID OpenStudio + names would remove it, and that changes IDF names and the S09 evidence, so it needs a separate decision. Tests: the + `Finding_SharedGuidPrefix_...` test should then PASS on the signed-off fixture, plus a runner unit test for ambiguity. +- **F-S11-2, SAM core (shared).** `SpaceSimulationResultParameter.UnmetHours` is `IntegerParameterValue(0, 8760)` + (`SAM.Analytical/Enums/Parameter/SpaceSimulationResultParameter.cs:48`). `IntegerParameterValue` converts the + importer's double to int, so 14.5 h is stored as 14 h; the B1a `unmetHours` is already a double `MetricValue`. + Smallest correction: declare `UnmetHours` as `DoubleParameterValue(0, 8760)`, and consider `OccupiedUnmetHours` the + same way for consistency. Compatibility checked by search (not by running): Tas writes integer counts + (`SAM_Tas Query/UnmetHours.cs:136,155`), which stay exact as doubles; the Tas and OpenStudio B1a mappers already read + `UnmetHours` as a double (`SAM_Tas .../ToBenchmark.cs:209`, this repo `ToBenchmark.cs:238`). No non-test + `GetValue` on the space parameter was found in SAM, SAM_Tas, SAM_UI or SAM_Validation (DesignExplorer reads the + separate `AdjacencyClusterSimulationResultParameter.UnmetHours`, which is unchanged). Must be verified in that PR: saved + JSON results holding integer `UnmetHours` still load and read (an int read as a double), Grasshopper/SAM_UI + displays, the remaining SAM-BIM repositories, and S10's Tas results round-trip unchanged. + Alternative with a smaller blast radius: a new double parameter written by the OpenStudio importer alongside the + integer one. It is not recommended because SAM-I (the SAM_UI/Grasshopper layer) would keep showing whole hours. +- Revision 2 should not start until the owner decides both (fix as above, or accept and record a limitation and how the + row is judged). + +## Decisions and assumptions + +- OD-2..OD-7 taken as proposed in the approved plan (Total basis + note, zero rule, missing = unavailable, earliest tie, + engine-defined unmet hours, R0/R1/R2 + class E, ESO mandatory, completeness per file against the RunPeriod, design-load + scope OD-4, OD-7 real-data requirement evaluated in Revision 2). OD-5 (SAM_Validation METRICS.md) is a later docs PR. +- The plan put the "production reader still returns values for the shared-error files" check in `Benchmark.Tests`; it is + in `InputReadBack.Tests` (`S11CompletenessTests`), which references the importer too and owns the synthetic generator. +- The synthetic producer path calls the internal `OpenStudioSimulationRunner.ExtractLoads/ExtractResultSet` by reflection + in tests only (as `benchmark-openstudio` does after a simulation); no `InternalsVisibleTo` or production change. +- SAM-I design loads come from the legacy design-day path (`Create.SpaceSimulationResults`); only their availability is + gated, their value is informational (the `CalcDesLoad` mapping is gated through SAM-P/B1a, OD-4). +- `subject.json` for the Sensible-substitution negative is captured by the `negative` process and then mutated + (recorded in `Mutation`); for the hour negatives read/subject/compare run fresh on the copy. + +## Files changed + +- `benchmark/SAM.Analytical.OpenStudio.Benchmark/`: `Classes/OpenStudioBenchmarkInputs.cs` (new), `Modify/ToBenchmark.cs`, + `Program.cs`. +- `benchmark/SAM.Analytical.OpenStudio.Benchmark.Tests/`: `S11ProducerTests.cs` (new), `DiagnosticPropagationTests.cs` + (the note), `Fixtures/golden-openstudio-benchmark.json` (one line: the note). +- `benchmark/SAM.Analytical.OpenStudio.InputReadBack/Compare/`: `NativeZoneIdentity.cs` (new), `OsComparison.cs`. +- `benchmark/SAM.Analytical.OpenStudio.ResultsReadBack/` (new project, 9 sources, 3 scripts) and + `benchmark/SAM.Analytical.OpenStudio.ResultsReadBack.Cli/` (new project, 6 sources). +- `benchmark/SAM.Analytical.OpenStudio.InputReadBack.Tests/`: csproj (references), 9 new `S11*.cs`. +- `SAM_OpenStudio.sln` (two projects), `.github/workflows/build.yml` (build check step, step label), this record. +- Revision 1 correction: `ResultsReadBack/Native/SqlResultRead.cs`, `Native/EsoResultRead.cs`, `Run-S11-Results.ps1`; + tests `S11CompletenessTests.cs` (3 defects), new `S11DriverTests.cs` and `S11DriverStandIn.ps1` (copied to the test + output by the test csproj). +- Integration (2026-10-24): merge of `sow/2026-Q4` (`43e3541c`; brings #35's runner change, tests and record, already + on the base); tests `S11ResultsTests.cs` (two replacements), `S11Support.cs` (`Two()`, `SpaceAGuid`; re-GUIDed variant + removed), `S11Synthetic.cs` (fractional not-met profile), `S11CompletenessTests.cs`, `S11F3Tests.cs`, + `S11NegativeTests.cs` (signed-off fixture), `S11IdentityTests.cs` (comments); this record. +- Revision 2 attempt (2026-10-26, documentation only): `documentation/S11-OpenStudio-ResultsRoundTrip-evidence/revision-2-2026-10-26/` + (stopped `summary-run.json`, receipt, `SHA256SUMS.txt`, README); this record. No code, test or script change. +- Not changed by this PR: importer, writer, SQL readers, schema, SAM_UI, SAM_Tas, SAM core, S10, fixtures, tolerances, + metric definitions, comparison gates, scripts, workflow. + +## Tests and validation (local, Release x64) + +Tested content: `246c8fedd1dcf45618df9241af682feadc1fa9bc` (code, tests, scripts, solution, workflow; this record section +was added afterwards in a record-only commit). Dependencies as built locally: SAM `b7c2c36a` (`SAM\build`), SAM_Validation +`2a3acfaf` (`SAM_Validation\build` schema), SAM_Tas `922044fa` (InputEquivalence + linked ReadBackExpected), SAM_SQLite +`ec2eb86`, all on `sow/2026-Q4`. Machine: Windows 11, VS 18 MSBuild, .NET SDK 10.0.401 targeting net8.0. + +| Check | Command (from `SAM_OpenStudio`) | Result | +|---|---|---| +| Full solution, as CI | `MSBuild.exe SAM_OpenStudio.sln -t:Restore;Rebuild -p:Configuration=Release -p:Platform=x64 -p:UseSharedCompilation=false "-p:APPDATA=" -m -nr:false -v:m` | exit 0, 0 errors; no compiler warning from a new or changed project (existing MSB3277 System.Memory warnings only). `APPDATA` redirected so the Grasshopper post-build did not deploy into `%APPDATA%\SAM` (verified unchanged). | +| New CI build-check logic, run locally | the `Check S11 results read-back projects are built` step body | both projects listed in the solution; both outputs present | +| Producer tests | `dotnet test benchmark/SAM.Analytical.OpenStudio.Benchmark.Tests -c Release -p:Platform=x64 --no-build --filter "TestCategory!=Simulation"` | 58 passed, 0 failed (51 existing + 7 `S11ProducerTests`) | +| Read-back tests (S09 + S11) | `dotnet test benchmark/SAM.Analytical.OpenStudio.InputReadBack.Tests -c Release -p:Platform=x64 --no-build` | 197 passed, 0 failed, 0 skipped, 5 min 9 s (120 existing S09 + 77 S11) | +| S09 behaviour-neutral replay (offline, saved runs) | `os-input-readback compare --run C:\TasOut\s09\acc7-u1\ --fixture-id s08-fix- --rule-set approved --out ` before and after the extraction | exit 0 both; `report.txt` byte-identical to committed `acceptance-2026-10-23/runs//report-approved.txt` (`868a5808…b959fe`, `9e3a3f0e…a6c466d`) | +| Scripts | Windows PowerShell 5.1 `Parser.ParseFile` on the three `.ps1`; `Run-S11-Results.ps1 -Phase run` without `-ConfirmEngineRuns` | 0 parse errors, ASCII only; the run phase stops with exit 70 before creating anything | + +The S11 tests include: the consistent two-zone run PASSing with `read`, `subject`, `compare` in three processes and the +four CLI negatives on copies (control 0, deleted hour 11, duplicated hour 11, Sensible substitution 10, original +unchanged); 22 completeness defects each BLOCKING (shared, one-sided, count-preserving shift, absent series/zone, repeated +ESO id, second dictionary entry, extra hour, Feb 29, warmup rows, merged design-day rows, non-annual/leap RunPeriod, +second ESO RunPeriod block, DST stamp) with the paired production-reader check for the shared ones; the F3 precondition +(every targeted row >= 10^6 tolerances, other peak hour) and 8 substitutions (consumption, space peak, model peak, all; +heating and cooling) each FAILing with no BLOCKED row; zero rule, ties, value slips, C3-C9, closure, identity and source +scan. Hosted CI at the earlier record head `06084afb`: Build `40067946762` and SPDX `40067946736` SUCCESS (the current +head and its checks are in the PR description). + +### Revision 1 correction (local, Release x64) + +Tested content: `059aed6f7efe2a43291e8b6433528c5819b8707f` (code, tests and stand-in; this record section was added +in a following record-only commit). Same dependency builds and machine as above. Only the affected projects were +rebuilt; no Grasshopper project was built; `%APPDATA%\SAM` has no file written after the build (its newest files date +from 07:00 that day, before this work). + +| Check | Command (from `SAM_OpenStudio`) | Result | +|---|---|---| +| Affected projects | `MSBuild.exe benchmark/SAM.Analytical.OpenStudio.InputReadBack.Tests/SAM.Analytical.OpenStudio.InputReadBack.Tests.csproj -t:Rebuild -p:Configuration=Release -p:Platform=x64 -p:UseSharedCompilation=false "-p:APPDATA=" -nr:false -v:m` (rebuilds the S11 library, CLI, producer, S09 reader/CLI and their references) | exit 0, 0 errors, 0 CS warnings (existing MSB3277 only) | +| Read-back tests (S09 + S11) | `dotnet test benchmark/SAM.Analytical.OpenStudio.InputReadBack.Tests -c Release -p:Platform=x64 --no-build` | **242 passed, 0 failed, 0 skipped**, 7 min 13 s (120 S09 + 122 S11: 77 earlier + 42 `S11DriverTests` + 3 completeness defects) | +| Producer tests | `dotnet test benchmark/SAM.Analytical.OpenStudio.Benchmark.Tests -c Release -p:Platform=x64 --no-build --filter "TestCategory!=Simulation"` | 58 passed, 0 failed (existing test output; producer source unchanged by the correction) | +| S09 saved-run replays (offline) | `os-input-readback compare --run C:/TasOut/s09/acc7-u1/ --fixture-id s08-fix- --rule-set approved --out ` | exit 0 both; `report.txt` byte-identical to committed `acceptance-2026-10-23/runs//report-approved.txt` (`868a5808…`, `9e3a3f0e…`) | +| Review reproduction R2 | `os-results-readback read/subject/compare` on a scratch copy of the review's `extra-frequency-two/` (machine-local, parent workspace `.review-s11-r1-20261024`), model `s11-models/two-distinct-prefix.json` | BLOCKED 68 PASS / 0 FAIL / 17 BLOCKED, exit 11 (was PASS 85/85) | +| Scripts | Windows PowerShell 5.1.26100 `Parser.ParseFile` on `Run-S11-Results.ps1` and `S11DriverStandIn.ps1` | 0 parse errors, ASCII only | + +The review's original `probe-driver.ps1` is superseded by `S11DriverTests.Readback_InfrastructureErrorWithOldPassReport_StopsAt70AndReadsNoReport` +(same scenario through the new `-HarnessExe` parameter; the old probe rewrote variable lines that no longer exist). +No EnergyPlus or OpenStudio run, no acceptance phase and no Grasshopper deployment took place. + +## Remaining limitations and risks + +- No S11 run exists. The channel readers were exercised on synthetic files and, as a development probe only (not + evidence; older content, one zone, no SAM model match), read-only on the S05 output + `C:\TasOut\s05\runs\c2-openstudio-explicit\000000_…_275dbafb\run` (EnergyPlus `25.1.0-1c11a3d85f`, SQL `32b5dcc8…`, + hashes unchanged by the read): calendar 8760 from `SAM_RunPeriod` 2006; one SQL environment and one ESO block; all six + series complete in both files with units J/hr; SQL = ESO with relative difference 0 in every hour (the ESO prints full + precision); timestep meters equal the zone Total within ~1e-15 relative. Real cooling latent there is ~9 % of Total + (Total 16.40 GJ, Sensible 14.91 GJ); heating Total = Sensible, so heating F3 margins are 0 on such data (OD-7 concerns + cooling). The full read/subject/compare chain has not seen a real run; Revision 2 may still BLOCK. +- F-S11-1 and F-S11-2 are fixed and integrated (offline proof only). Real EnergyPlus output has not been read by the + integrated chain; whether real not-met totals are fractional on these fixtures is unknown until Revision 2. +- Separately tracked, not blocking on these fixtures (see the integration section): AddResults suffix-only + design-day/surface lookups and the generated-name identity residual. SAM#219's Revit compatibility is deferred by the + owner (SAM_Revit is not touched). S10's declared heating design-day hour limitation is unchanged. +- `Build-S11-Receipt.ps1` has now run successfully on clean clones (2026-10-26). **Revision 2 is BLOCKED by a driver + defect:** `Run-S11-Results.ps1`'s `Git` helper recurses into itself (`:103-106`), so `Test-Receipt` stops every phase + with exit 70 on a real receipt. The offline driver tests never listed a receipt commit (`S11DriverTests.cs:102`), so + they could not catch it. The bounded correction is recommended in "Revision 2 execution"; it is not implemented here. + The run, readback and negative phases have still not processed a real run. +- The full-solution local build redirects `APPDATA` because the Grasshopper post-build deploys into `%APPDATA%\SAM`. + +## Recommended next step + +1. **Owner decision** on the bounded driver correction ("Revision 2 execution", "Bounded correction recommended"). + - Scope: `Run-S11-Results.ps1` git resolution, the same hardening of the `Build-S11-Receipt.ps1` fallback, and + `S11DriverTests` with a real Git commit in the receipt. + - Code and offline tests only, on this Draft PR; no engine run. + - Recommended executor: Claude Code / Claude Opus 5.5 / High / new session / Build mode. It is a small, test-first + script fix. +2. **Independent review** of that correction. +3. **Revision 2 repeated** under a fresh owner instruction: a new receipt build at the corrected head, then exactly 2 + `benchmark-openstudio` runs (the allowance is still unused), readback/compare/negative and packing. +4. **Independent review of the real-run evidence.** + +None of this is authorised by this record. diff --git a/documentation/S11-OpenStudio-ResultsRoundTrip-evidence/revision-2-2026-10-26/README.md b/documentation/S11-OpenStudio-ResultsRoundTrip-evidence/revision-2-2026-10-26/README.md new file mode 100644 index 0000000..c307d47 --- /dev/null +++ b/documentation/S11-OpenStudio-ResultsRoundTrip-evidence/revision-2-2026-10-26/README.md @@ -0,0 +1,58 @@ +# S11 Revision 2 attempt, 2026-10-26: BLOCKED before engine execution + +> Phase B remains subject to S10's declared validation limitation (`SAM|FixB|Heating|designDayPeak.hourOfDay` BLOCKED); +> an S11 PASS is not a Phase B PASS. + +**Outcome:** BLOCKED. **Engine attempts used: 0 of 2.** No simulation, readback, compare or negative ran. This folder +is bounded evidence of the stopped run-phase preflight. It is not acceptance evidence. + +## Files + +| File | What it is | +|---|---| +| `summary-run.json` | `Run-S11-Results.ps1 -Phase run` summary: `STOPPED`, exit 70, no runs, reason `R:\SAM is not clean at the receipt commit 457defbb…` | +| `s11-build-receipt.json` | Receipt of the successful controlled build (`Build-S11-Receipt.ps1`): four clean HEADs, 51 `.dll`/`.exe` hashes | +| `SHA256SUMS.txt` | Written by `Pack-S11-Evidence.ps1 -Root C:\TasOut\s11\r2-20261026 -Label revision-2-2026-10-26` | + +## Build provenance + +| Item | Value | +|---|---| +| SAM_OpenStudio | `01344e659528ec49a746d9c165ee3914d7b1cca0` | +| SAM | `457defbb033fa4a58128f2395b87587f0210c608` | +| SAM_Tas | `922044fa9ca35062979689f4e359ea85bee577dd` | +| SAM_Validation | `2a3acfafb3919b4a0b81d67ef16a4b9d1e222f5d` | +| SAM_SQLite | `ec2eb86d00cac6e4cadf52b429f9d293eeabbd5e` (built first, stamped; not part of the receipt's commit list) | +| Receipt SHA-256 | `c057cf47ca82c2eb5d61fd5ea2f714a059184d19bc48cf4a748747ceffa2f602` | +| `SAM.Core.SQLite.dll` | `3d1d9033a4edce3644c65a15644b2e96513a25f722684a03c806659825e514ee` | +| EPW | `472ffb4e2f10867600949a392ce46891fc1bd05f2fc8d8917a99bca7d26209c3` | +| Fixtures | box `45524172…acd08`, two `20da9b58…9baa0` | +| CLI | portable `3.10.0+86d7e215a1` (`--version` checked; never started for a simulation) | + +**Line endings.** `SHA256SUMS.txt` hashes the files as packed on Windows, with CRLF. Git stores these JSON files with LF +(no `.gitattributes` rule covers `documentation/`) and restores CRLF on a Windows `core.autocrlf=true` checkout, where the +sums verify. LF blob hashes: `summary-run.json` `aeb17bd65aa2f64edd115b44d875968a13ab62837eb88d0a827fa0c36cdd4da9`; +`s11-build-receipt.json` `23552ab6978c69cf46aa0531d421c7f20d7d34da5e2d53c90850d95c24d8d3c8`. + +The `R:` paths in the receipt are a temporary drive mapping of +`SAM-BIM\.s11r2-20261026\ws`, used for Windows path length. The mapping was removed after the session. + +## Cause + +- `Run-S11-Results.ps1:103-106`: the `Git` helper calls `& git`, which PowerShell resolves to the helper itself. +- The recursion overflows silently: errors are swallowed by `2>$null` and `ErrorActionPreference = 'Continue'`. After + about 200 s the helper returns an empty HEAD, so `Test-Receipt` stops every phase with exit 70. +- The checkout was clean. The offline driver tests use receipts without commits (`S11DriverTests.cs:102`), so they could + not catch this. + +A bounded correction is recommended, not implemented, in `../../S11-OpenStudio-ResultsRoundTrip-PR.md` +("Revision 2 execution (2026-10-26): BLOCKED"). + +## Machine-local (not committed; copy before switching computers) + +- `C:\TasOut\s11\r2-20261026\summary-run.json` (`192ec65f…`), the acceptance root. +- `SAM-BIM\.s11r2-20261026\`: + - `scripts\` (`env.ps1`, `build.ps1`, `phase.ps1`, diagnostics); + - `logs\` (build logs, `phase-run.log`, `phase-commands.txt`, `diag-receipt-1.txt`, `proof-git-recursion.txt`, + `engine-check.txt`); + - `hashes\`, `weather\budapest.epw`, and the `ws\` clones with their builds. diff --git a/documentation/S11-OpenStudio-ResultsRoundTrip-evidence/revision-2-2026-10-26/SHA256SUMS.txt b/documentation/S11-OpenStudio-ResultsRoundTrip-evidence/revision-2-2026-10-26/SHA256SUMS.txt new file mode 100644 index 0000000..8cf2c45 --- /dev/null +++ b/documentation/S11-OpenStudio-ResultsRoundTrip-evidence/revision-2-2026-10-26/SHA256SUMS.txt @@ -0,0 +1,2 @@ +c057cf47ca82c2eb5d61fd5ea2f714a059184d19bc48cf4a748747ceffa2f602 s11-build-receipt.json +192ec65f4da11ff7965b6e600825e252af7795fbe92ac65bea079ac342b5ee42 summary-run.json diff --git a/documentation/S11-OpenStudio-ResultsRoundTrip-evidence/revision-2-2026-10-26/s11-build-receipt.json b/documentation/S11-OpenStudio-ResultsRoundTrip-evidence/revision-2-2026-10-26/s11-build-receipt.json new file mode 100644 index 0000000..fc812dd --- /dev/null +++ b/documentation/S11-OpenStudio-ResultsRoundTrip-evidence/revision-2-2026-10-26/s11-build-receipt.json @@ -0,0 +1,348 @@ +{ + "schema": "s11-build-receipt/1", + "note": "Written by Build-S11-Receipt.ps1 after a successful controlled build of clean checkouts; Run-S11-Results.ps1 verifies it and never writes it.", + "startedUtc": "2026-10-26T17:07:30.5223699Z", + "completedUtc": "2026-10-26T17:08:06.0988070Z", + "machine": "LEGION5MD", + "msbuild": "C:\\Program Files\\Microsoft Visual Studio\\18\\Community\\MSBuild\\Current\\Bin\\MSBuild.exe", + "commands": [ + { + "file": "C:\\Program Files\\Microsoft Visual Studio\\18\\Community\\MSBuild\\Current\\Bin\\MSBuild.exe", + "arguments": "R:\\SAM\\SAM\\SAM.Core\\SAM.Core.csproj -restore -t:Rebuild -p:Configuration=Release -p:SAMSourceRevision=457defbb033fa4a58128f2395b87587f0210c608 -nr:false -v:m", + "exitCode": 0 + }, + { + "file": "C:\\Program Files\\Microsoft Visual Studio\\18\\Community\\MSBuild\\Current\\Bin\\MSBuild.exe", + "arguments": "R:\\SAM\\SAM\\SAM.Math\\SAM.Math.csproj -restore -t:Rebuild -p:Configuration=Release -p:SAMSourceRevision=457defbb033fa4a58128f2395b87587f0210c608 -nr:false -v:m", + "exitCode": 0 + }, + { + "file": "C:\\Program Files\\Microsoft Visual Studio\\18\\Community\\MSBuild\\Current\\Bin\\MSBuild.exe", + "arguments": "R:\\SAM\\SAM\\SAM.Units\\SAM.Units.csproj -restore -t:Rebuild -p:Configuration=Release -p:SAMSourceRevision=457defbb033fa4a58128f2395b87587f0210c608 -nr:false -v:m", + "exitCode": 0 + }, + { + "file": "C:\\Program Files\\Microsoft Visual Studio\\18\\Community\\MSBuild\\Current\\Bin\\MSBuild.exe", + "arguments": "R:\\SAM\\SAM\\SAM.Geometry\\SAM.Geometry.csproj -restore -t:Rebuild -p:Configuration=Release -p:SAMSourceRevision=457defbb033fa4a58128f2395b87587f0210c608 -nr:false -v:m", + "exitCode": 0 + }, + { + "file": "C:\\Program Files\\Microsoft Visual Studio\\18\\Community\\MSBuild\\Current\\Bin\\MSBuild.exe", + "arguments": "R:\\SAM\\SAM\\SAM.Weather\\SAM.Weather.csproj -restore -t:Rebuild -p:Configuration=Release -p:SAMSourceRevision=457defbb033fa4a58128f2395b87587f0210c608 -nr:false -v:m", + "exitCode": 0 + }, + { + "file": "C:\\Program Files\\Microsoft Visual Studio\\18\\Community\\MSBuild\\Current\\Bin\\MSBuild.exe", + "arguments": "R:\\SAM\\SAM\\SAM.Architectural\\SAM.Architectural.csproj -restore -t:Rebuild -p:Configuration=Release -p:SAMSourceRevision=457defbb033fa4a58128f2395b87587f0210c608 -nr:false -v:m", + "exitCode": 0 + }, + { + "file": "C:\\Program Files\\Microsoft Visual Studio\\18\\Community\\MSBuild\\Current\\Bin\\MSBuild.exe", + "arguments": "R:\\SAM\\SAM\\SAM.Analytical\\SAM.Analytical.csproj -restore -t:Rebuild -p:Configuration=Release -p:SAMSourceRevision=457defbb033fa4a58128f2395b87587f0210c608 -nr:false -v:m", + "exitCode": 0 + }, + { + "file": "C:\\Program Files\\Microsoft Visual Studio\\18\\Community\\MSBuild\\Current\\Bin\\MSBuild.exe", + "arguments": "R:\\SAM_Validation\\SAM.Analytical.Benchmark\\SAM.Analytical.Benchmark.csproj -restore -t:Rebuild -p:Configuration=Release -p:SAMSourceRevision=2a3acfafb3919b4a0b81d67ef16a4b9d1e222f5d -nr:false -v:m", + "exitCode": 0 + }, + { + "file": "C:\\Program Files\\Microsoft Visual Studio\\18\\Community\\MSBuild\\Current\\Bin\\MSBuild.exe", + "arguments": "R:\\SAM_OpenStudio\\benchmark\\SAM.Analytical.OpenStudio.Benchmark\\SAM.Analytical.OpenStudio.Benchmark.csproj -restore -t:Rebuild -p:Configuration=Release -p:SAMSourceRevision=01344e659528ec49a746d9c165ee3914d7b1cca0 -nr:false -v:m -p:Platform=x64", + "exitCode": 0 + }, + { + "file": "C:\\Program Files\\Microsoft Visual Studio\\18\\Community\\MSBuild\\Current\\Bin\\MSBuild.exe", + "arguments": "R:\\SAM_OpenStudio\\benchmark\\SAM.Analytical.OpenStudio.ResultsReadBack.Cli\\SAM.Analytical.OpenStudio.ResultsReadBack.Cli.csproj -restore -t:Rebuild -p:Configuration=Release -p:SAMSourceRevision=01344e659528ec49a746d9c165ee3914d7b1cca0 -nr:false -v:m -p:Platform=x64", + "exitCode": 0 + } + ], + "commits": [ + { + "repository": "R:\\SAM", + "commit": "457defbb033fa4a58128f2395b87587f0210c608", + "branch": "HEAD" + }, + { + "repository": "R:\\SAM_Validation", + "commit": "2a3acfafb3919b4a0b81d67ef16a4b9d1e222f5d", + "branch": "HEAD" + }, + { + "repository": "R:\\SAM_Tas", + "commit": "922044fa9ca35062979689f4e359ea85bee577dd", + "branch": "HEAD" + }, + { + "repository": "R:\\SAM_OpenStudio", + "commit": "01344e659528ec49a746d9c165ee3914d7b1cca0", + "branch": "feature/s11-openstudio-results-roundtrip" + } + ], + "benchmarkExe": "R:\\SAM_OpenStudio\\benchmark\\SAM.Analytical.OpenStudio.Benchmark\\bin\\x64\\Release\\net8.0\\benchmark-openstudio.exe", + "harnessExe": "R:\\SAM_OpenStudio\\benchmark\\SAM.Analytical.OpenStudio.ResultsReadBack.Cli\\bin\\x64\\Release\\net8.0\\os-results-readback.exe", + "stamped": [ + { + "path": "R:\\SAM_OpenStudio\\benchmark\\SAM.Analytical.OpenStudio.Benchmark\\bin\\x64\\Release\\net8.0\\benchmark-openstudio.dll", + "commit": "01344e659528ec49a746d9c165ee3914d7b1cca0" + }, + { + "path": "R:\\SAM_OpenStudio\\benchmark\\SAM.Analytical.OpenStudio.ResultsReadBack.Cli\\bin\\x64\\Release\\net8.0\\os-results-readback.dll", + "commit": "01344e659528ec49a746d9c165ee3914d7b1cca0" + }, + { + "path": "R:\\SAM_OpenStudio\\benchmark\\SAM.Analytical.OpenStudio.ResultsReadBack.Cli\\bin\\x64\\Release\\net8.0\\SAM.Analytical.OpenStudio.ResultsReadBack.dll", + "commit": "01344e659528ec49a746d9c165ee3914d7b1cca0" + }, + { + "path": "R:\\SAM_OpenStudio\\benchmark\\SAM.Analytical.OpenStudio.ResultsReadBack.Cli\\bin\\x64\\Release\\net8.0\\SAM.Analytical.OpenStudio.InputReadBack.dll", + "commit": "01344e659528ec49a746d9c165ee3914d7b1cca0" + }, + { + "path": "R:\\SAM_OpenStudio\\benchmark\\SAM.Analytical.OpenStudio.ResultsReadBack.Cli\\bin\\x64\\Release\\net8.0\\SAM.Analytical.OpenStudio.dll", + "commit": "01344e659528ec49a746d9c165ee3914d7b1cca0" + }, + { + "path": "R:\\SAM_OpenStudio\\benchmark\\SAM.Analytical.OpenStudio.ResultsReadBack.Cli\\bin\\x64\\Release\\net8.0\\SAM.Analytical.InputEquivalence.dll", + "commit": "01344e659528ec49a746d9c165ee3914d7b1cca0" + }, + { + "path": "R:\\SAM_Validation\\build\\SAM.Analytical.Benchmark.dll", + "commit": "2a3acfafb3919b4a0b81d67ef16a4b9d1e222f5d" + }, + { + "path": "R:\\SAM\\build\\SAM.Core.dll", + "commit": "457defbb033fa4a58128f2395b87587f0210c608" + }, + { + "path": "R:\\SAM\\build\\SAM.Math.dll", + "commit": "457defbb033fa4a58128f2395b87587f0210c608" + }, + { + "path": "R:\\SAM\\build\\SAM.Units.dll", + "commit": "457defbb033fa4a58128f2395b87587f0210c608" + }, + { + "path": "R:\\SAM\\build\\SAM.Geometry.dll", + "commit": "457defbb033fa4a58128f2395b87587f0210c608" + }, + { + "path": "R:\\SAM\\build\\SAM.Weather.dll", + "commit": "457defbb033fa4a58128f2395b87587f0210c608" + }, + { + "path": "R:\\SAM\\build\\SAM.Architectural.dll", + "commit": "457defbb033fa4a58128f2395b87587f0210c608" + }, + { + "path": "R:\\SAM\\build\\SAM.Analytical.dll", + "commit": "457defbb033fa4a58128f2395b87587f0210c608" + } + ], + "files": [ + { + "path": "R:\\SAM_OpenStudio\\benchmark\\SAM.Analytical.OpenStudio.Benchmark\\bin\\x64\\Release\\net8.0\\benchmark-openstudio.dll", + "sha256": "5866ce57cd878d55d065e7d708b9ae51a87355c9d2af6e0434b3934164271de2" + }, + { + "path": "R:\\SAM_OpenStudio\\benchmark\\SAM.Analytical.OpenStudio.Benchmark\\bin\\x64\\Release\\net8.0\\benchmark-openstudio.exe", + "sha256": "a9553d11c5663a11ca80f4ff52b9e29df0265ca67272859f008840b26bcd0f77" + }, + { + "path": "R:\\SAM_OpenStudio\\benchmark\\SAM.Analytical.OpenStudio.Benchmark\\bin\\x64\\Release\\net8.0\\MathNet.Numerics.dll", + "sha256": "8f4b1b9c35726b9bf3d51f0e806e12996a849d547b5321690abad66fb0e04e6d" + }, + { + "path": "R:\\SAM_OpenStudio\\benchmark\\SAM.Analytical.OpenStudio.Benchmark\\bin\\x64\\Release\\net8.0\\NetTopologySuite.dll", + "sha256": "55288513bdd9a834098fc4a995f5f96d99b9456f1a26c134fe0051084983e134" + }, + { + "path": "R:\\SAM_OpenStudio\\benchmark\\SAM.Analytical.OpenStudio.Benchmark\\bin\\x64\\Release\\net8.0\\OpenStudio.dll", + "sha256": "678bdace11401e751fd661d9638944613b8802e6e8dc01809d1dbfa7fbede499" + }, + { + "path": "R:\\SAM_OpenStudio\\benchmark\\SAM.Analytical.OpenStudio.Benchmark\\bin\\x64\\Release\\net8.0\\openstudio_csharp.dll", + "sha256": "6d79e1e6e5b4fd0e25ed14293a4a1f5823eb670ce6307ac629bcd176524f73bc" + }, + { + "path": "R:\\SAM_OpenStudio\\benchmark\\SAM.Analytical.OpenStudio.Benchmark\\bin\\x64\\Release\\net8.0\\openstudio_model_csharp.dll", + "sha256": "8ffdb629129b4a3995805100bf4817076a363a656d613df14bc15f0c79a98f5b" + }, + { + "path": "R:\\SAM_OpenStudio\\benchmark\\SAM.Analytical.OpenStudio.Benchmark\\bin\\x64\\Release\\net8.0\\openstudio_translators_csharp.dll", + "sha256": "973b2b49afb6cb32be8efd3f11f99834e5fc72d8236b35927dca7bcc5045c834" + }, + { + "path": "R:\\SAM_OpenStudio\\benchmark\\SAM.Analytical.OpenStudio.Benchmark\\bin\\x64\\Release\\net8.0\\openstudiolib.dll", + "sha256": "549eb1a7492b86adcf87ba7015579370acb1738d995f53e0ae551b85af3e1451" + }, + { + "path": "R:\\SAM_OpenStudio\\benchmark\\SAM.Analytical.OpenStudio.Benchmark\\bin\\x64\\Release\\net8.0\\QuickGraph.NETStandard.dll", + "sha256": "50eeead03278bdf43bf0a9ad0d2348bc977f882ecd0bfe50527602446002af88" + }, + { + "path": "R:\\SAM_OpenStudio\\benchmark\\SAM.Analytical.OpenStudio.Benchmark\\bin\\x64\\Release\\net8.0\\SAM.Analytical.Benchmark.dll", + "sha256": "17766896e9f2eb7eb80b0fcb555b68983df7ed9217c85ebd1c72ec1992970935" + }, + { + "path": "R:\\SAM_OpenStudio\\benchmark\\SAM.Analytical.OpenStudio.Benchmark\\bin\\x64\\Release\\net8.0\\SAM.Analytical.dll", + "sha256": "feac8dab69ff3483384cac4fa19a376e528c681f7e0037023442360bcaf23977" + }, + { + "path": "R:\\SAM_OpenStudio\\benchmark\\SAM.Analytical.OpenStudio.Benchmark\\bin\\x64\\Release\\net8.0\\SAM.Analytical.OpenStudio.dll", + "sha256": "deddf7140715cf59942c912d363e7ef4a93627b5adf2e8a217b74552c4364489" + }, + { + "path": "R:\\SAM_OpenStudio\\benchmark\\SAM.Analytical.OpenStudio.Benchmark\\bin\\x64\\Release\\net8.0\\SAM.Architectural.dll", + "sha256": "99d39499e0a4959fa316e1f7f121a932ace72855ba4741fbcde44360d70b6ed1" + }, + { + "path": "R:\\SAM_OpenStudio\\benchmark\\SAM.Analytical.OpenStudio.Benchmark\\bin\\x64\\Release\\net8.0\\SAM.Core.dll", + "sha256": "8ab264ad00c0ca35cd988285d1ef97513267f4655385f4dbcc3425835923abda" + }, + { + "path": "R:\\SAM_OpenStudio\\benchmark\\SAM.Analytical.OpenStudio.Benchmark\\bin\\x64\\Release\\net8.0\\SAM.Core.OpenStudio.dll", + "sha256": "4565d71d00079ff658325c338abf9a2d30c3620d70382fa8b30bb199758747c1" + }, + { + "path": "R:\\SAM_OpenStudio\\benchmark\\SAM.Analytical.OpenStudio.Benchmark\\bin\\x64\\Release\\net8.0\\SAM.Core.SQLite.dll", + "sha256": "3d1d9033a4edce3644c65a15644b2e96513a25f722684a03c806659825e514ee" + }, + { + "path": "R:\\SAM_OpenStudio\\benchmark\\SAM.Analytical.OpenStudio.Benchmark\\bin\\x64\\Release\\net8.0\\SAM.Geometry.dll", + "sha256": "a0e79678b948db33c4989b99a25f9b9f2c468b68de141a7976d8d92d159f3292" + }, + { + "path": "R:\\SAM_OpenStudio\\benchmark\\SAM.Analytical.OpenStudio.Benchmark\\bin\\x64\\Release\\net8.0\\SAM.Geometry.OpenStudio.dll", + "sha256": "664cfac216a983dce6c476fd2f19bd09283be734a5d2370b5fbc5ae9d0931f09" + }, + { + "path": "R:\\SAM_OpenStudio\\benchmark\\SAM.Analytical.OpenStudio.Benchmark\\bin\\x64\\Release\\net8.0\\SAM.Math.dll", + "sha256": "ff2a40355d977a9ed6ade2a3fcc659683c0a674c0f92ea3f21ba5b6f48d6e936" + }, + { + "path": "R:\\SAM_OpenStudio\\benchmark\\SAM.Analytical.OpenStudio.Benchmark\\bin\\x64\\Release\\net8.0\\SAM.Weather.dll", + "sha256": "8fe7de206f78f8492d92ee14c393566c10323045135d2cbb9a662d2f84a52e9c" + }, + { + "path": "R:\\SAM_OpenStudio\\benchmark\\SAM.Analytical.OpenStudio.Benchmark\\bin\\x64\\Release\\net8.0\\System.Data.SQLite.dll", + "sha256": "fa3d7678272b10dfa0be3d959f0aea38a58b75caf1bba06d6781218ced489620" + }, + { + "path": "R:\\SAM_OpenStudio\\benchmark\\SAM.Analytical.OpenStudio.Benchmark\\bin\\x64\\Release\\net8.0\\System.Drawing.Common.dll", + "sha256": "b6ac309471f2ba20e84a67bbd20d49bf2025ddb4e12219344ef4be97c549fb3c" + }, + { + "path": "R:\\SAM_OpenStudio\\benchmark\\SAM.Analytical.OpenStudio.ResultsReadBack.Cli\\bin\\x64\\Release\\net8.0\\benchmark-openstudio.dll", + "sha256": "5866ce57cd878d55d065e7d708b9ae51a87355c9d2af6e0434b3934164271de2" + }, + { + "path": "R:\\SAM_OpenStudio\\benchmark\\SAM.Analytical.OpenStudio.ResultsReadBack.Cli\\bin\\x64\\Release\\net8.0\\benchmark-openstudio.exe", + "sha256": "a9553d11c5663a11ca80f4ff52b9e29df0265ca67272859f008840b26bcd0f77" + }, + { + "path": "R:\\SAM_OpenStudio\\benchmark\\SAM.Analytical.OpenStudio.ResultsReadBack.Cli\\bin\\x64\\Release\\net8.0\\MathNet.Numerics.dll", + "sha256": "8f4b1b9c35726b9bf3d51f0e806e12996a849d547b5321690abad66fb0e04e6d" + }, + { + "path": "R:\\SAM_OpenStudio\\benchmark\\SAM.Analytical.OpenStudio.ResultsReadBack.Cli\\bin\\x64\\Release\\net8.0\\NetTopologySuite.dll", + "sha256": "55288513bdd9a834098fc4a995f5f96d99b9456f1a26c134fe0051084983e134" + }, + { + "path": "R:\\SAM_OpenStudio\\benchmark\\SAM.Analytical.OpenStudio.ResultsReadBack.Cli\\bin\\x64\\Release\\net8.0\\OpenStudio.dll", + "sha256": "678bdace11401e751fd661d9638944613b8802e6e8dc01809d1dbfa7fbede499" + }, + { + "path": "R:\\SAM_OpenStudio\\benchmark\\SAM.Analytical.OpenStudio.ResultsReadBack.Cli\\bin\\x64\\Release\\net8.0\\openstudio_csharp.dll", + "sha256": "6d79e1e6e5b4fd0e25ed14293a4a1f5823eb670ce6307ac629bcd176524f73bc" + }, + { + "path": "R:\\SAM_OpenStudio\\benchmark\\SAM.Analytical.OpenStudio.ResultsReadBack.Cli\\bin\\x64\\Release\\net8.0\\openstudio_model_csharp.dll", + "sha256": "8ffdb629129b4a3995805100bf4817076a363a656d613df14bc15f0c79a98f5b" + }, + { + "path": "R:\\SAM_OpenStudio\\benchmark\\SAM.Analytical.OpenStudio.ResultsReadBack.Cli\\bin\\x64\\Release\\net8.0\\openstudio_translators_csharp.dll", + "sha256": "973b2b49afb6cb32be8efd3f11f99834e5fc72d8236b35927dca7bcc5045c834" + }, + { + "path": "R:\\SAM_OpenStudio\\benchmark\\SAM.Analytical.OpenStudio.ResultsReadBack.Cli\\bin\\x64\\Release\\net8.0\\openstudiolib.dll", + "sha256": "549eb1a7492b86adcf87ba7015579370acb1738d995f53e0ae551b85af3e1451" + }, + { + "path": "R:\\SAM_OpenStudio\\benchmark\\SAM.Analytical.OpenStudio.ResultsReadBack.Cli\\bin\\x64\\Release\\net8.0\\os-results-readback.dll", + "sha256": "def1a9f7b612ad77182d40dcd4f28ad07ea5650f13d2c38b3dd1338e349cc73a" + }, + { + "path": "R:\\SAM_OpenStudio\\benchmark\\SAM.Analytical.OpenStudio.ResultsReadBack.Cli\\bin\\x64\\Release\\net8.0\\os-results-readback.exe", + "sha256": "e7e604a02e363742f11fefd957817ccdd167080dfe88c6734f1329365ed88ce6" + }, + { + "path": "R:\\SAM_OpenStudio\\benchmark\\SAM.Analytical.OpenStudio.ResultsReadBack.Cli\\bin\\x64\\Release\\net8.0\\QuickGraph.NETStandard.dll", + "sha256": "50eeead03278bdf43bf0a9ad0d2348bc977f882ecd0bfe50527602446002af88" + }, + { + "path": "R:\\SAM_OpenStudio\\benchmark\\SAM.Analytical.OpenStudio.ResultsReadBack.Cli\\bin\\x64\\Release\\net8.0\\SAM.Analytical.Benchmark.dll", + "sha256": "17766896e9f2eb7eb80b0fcb555b68983df7ed9217c85ebd1c72ec1992970935" + }, + { + "path": "R:\\SAM_OpenStudio\\benchmark\\SAM.Analytical.OpenStudio.ResultsReadBack.Cli\\bin\\x64\\Release\\net8.0\\SAM.Analytical.dll", + "sha256": "feac8dab69ff3483384cac4fa19a376e528c681f7e0037023442360bcaf23977" + }, + { + "path": "R:\\SAM_OpenStudio\\benchmark\\SAM.Analytical.OpenStudio.ResultsReadBack.Cli\\bin\\x64\\Release\\net8.0\\SAM.Analytical.InputEquivalence.dll", + "sha256": "83554e610732df6678e8e36d8c2addb02ade2f70f24f70b2742af1cb1d8e31f3" + }, + { + "path": "R:\\SAM_OpenStudio\\benchmark\\SAM.Analytical.OpenStudio.ResultsReadBack.Cli\\bin\\x64\\Release\\net8.0\\SAM.Analytical.OpenStudio.dll", + "sha256": "deddf7140715cf59942c912d363e7ef4a93627b5adf2e8a217b74552c4364489" + }, + { + "path": "R:\\SAM_OpenStudio\\benchmark\\SAM.Analytical.OpenStudio.ResultsReadBack.Cli\\bin\\x64\\Release\\net8.0\\SAM.Analytical.OpenStudio.InputReadBack.dll", + "sha256": "9e4b58ab1838c0ec627d11ae6ca324b6962b2b2a77f0541c710054bcf9218af6" + }, + { + "path": "R:\\SAM_OpenStudio\\benchmark\\SAM.Analytical.OpenStudio.ResultsReadBack.Cli\\bin\\x64\\Release\\net8.0\\SAM.Analytical.OpenStudio.ResultsReadBack.dll", + "sha256": "f2012d5819c1fe59a7fd2978b778808688ab71aa28f53eeabc66ab9f242fbd8b" + }, + { + "path": "R:\\SAM_OpenStudio\\benchmark\\SAM.Analytical.OpenStudio.ResultsReadBack.Cli\\bin\\x64\\Release\\net8.0\\SAM.Architectural.dll", + "sha256": "99d39499e0a4959fa316e1f7f121a932ace72855ba4741fbcde44360d70b6ed1" + }, + { + "path": "R:\\SAM_OpenStudio\\benchmark\\SAM.Analytical.OpenStudio.ResultsReadBack.Cli\\bin\\x64\\Release\\net8.0\\SAM.Core.dll", + "sha256": "8ab264ad00c0ca35cd988285d1ef97513267f4655385f4dbcc3425835923abda" + }, + { + "path": "R:\\SAM_OpenStudio\\benchmark\\SAM.Analytical.OpenStudio.ResultsReadBack.Cli\\bin\\x64\\Release\\net8.0\\SAM.Core.OpenStudio.dll", + "sha256": "4565d71d00079ff658325c338abf9a2d30c3620d70382fa8b30bb199758747c1" + }, + { + "path": "R:\\SAM_OpenStudio\\benchmark\\SAM.Analytical.OpenStudio.ResultsReadBack.Cli\\bin\\x64\\Release\\net8.0\\SAM.Core.SQLite.dll", + "sha256": "3d1d9033a4edce3644c65a15644b2e96513a25f722684a03c806659825e514ee" + }, + { + "path": "R:\\SAM_OpenStudio\\benchmark\\SAM.Analytical.OpenStudio.ResultsReadBack.Cli\\bin\\x64\\Release\\net8.0\\SAM.Geometry.dll", + "sha256": "a0e79678b948db33c4989b99a25f9b9f2c468b68de141a7976d8d92d159f3292" + }, + { + "path": "R:\\SAM_OpenStudio\\benchmark\\SAM.Analytical.OpenStudio.ResultsReadBack.Cli\\bin\\x64\\Release\\net8.0\\SAM.Geometry.OpenStudio.dll", + "sha256": "664cfac216a983dce6c476fd2f19bd09283be734a5d2370b5fbc5ae9d0931f09" + }, + { + "path": "R:\\SAM_OpenStudio\\benchmark\\SAM.Analytical.OpenStudio.ResultsReadBack.Cli\\bin\\x64\\Release\\net8.0\\SAM.Math.dll", + "sha256": "ff2a40355d977a9ed6ade2a3fcc659683c0a674c0f92ea3f21ba5b6f48d6e936" + }, + { + "path": "R:\\SAM_OpenStudio\\benchmark\\SAM.Analytical.OpenStudio.ResultsReadBack.Cli\\bin\\x64\\Release\\net8.0\\SAM.Weather.dll", + "sha256": "8fe7de206f78f8492d92ee14c393566c10323045135d2cbb9a662d2f84a52e9c" + }, + { + "path": "R:\\SAM_OpenStudio\\benchmark\\SAM.Analytical.OpenStudio.ResultsReadBack.Cli\\bin\\x64\\Release\\net8.0\\System.Data.SQLite.dll", + "sha256": "fa3d7678272b10dfa0be3d959f0aea38a58b75caf1bba06d6781218ced489620" + }, + { + "path": "R:\\SAM_OpenStudio\\benchmark\\SAM.Analytical.OpenStudio.ResultsReadBack.Cli\\bin\\x64\\Release\\net8.0\\System.Drawing.Common.dll", + "sha256": "b6ac309471f2ba20e84a67bbd20d49bf2025ddb4e12219344ef4be97c549fb3c" + } + ] +} diff --git a/documentation/S11-OpenStudio-ResultsRoundTrip-evidence/revision-2-2026-10-26/summary-run.json b/documentation/S11-OpenStudio-ResultsRoundTrip-evidence/revision-2-2026-10-26/summary-run.json new file mode 100644 index 0000000..cc57f7a --- /dev/null +++ b/documentation/S11-OpenStudio-ResultsRoundTrip-evidence/revision-2-2026-10-26/summary-run.json @@ -0,0 +1,14 @@ +{ + "phase": "run", + "started": "2026-10-26T17:08:46.7523039Z", + "root": "C:\\TasOut\\s11\\r2-20261026", + "buildReceipt": "R:\\SAM_OpenStudio\\benchmark\\SAM.Analytical.OpenStudio.ResultsReadBack.Cli\\bin\\x64\\Release\\net8.0\\s11-build-receipt.json", + "harness": "R:\\SAM_OpenStudio\\benchmark\\SAM.Analytical.OpenStudio.ResultsReadBack.Cli\\bin\\x64\\Release\\net8.0\\os-results-readback.exe", + "runs": [ + + ], + "stopped": "R:\\SAM is not clean at the receipt commit 457defbb033fa4a58128f2395b87587f0210c608", + "exit": 70, + "result": "STOPPED", + "finished": "2026-10-26T17:15:19.0050007Z" +}