From 3db8d7bfa99b59c2be2e87429e1ed36aba3c22a1 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 17 Oct 2026 12:02:12 +0000 Subject: [PATCH 01/13] feat: Tas vs OpenStudio comparison workflow and view (SAM_UI#231) Adds Simulate > Compare Tas / OpenStudio, a lightweight WPF view over SAM_Validation's comparator library (SAM#211). SAM_UI computes no comparison: values, differences, bands, notes, gates, space alignment and reports are SAM_Validation's (Query.Compare, Format, WriteReports). - Run Tas + OpenStudio: freezes the open model once (snapshot.json), prepares one EPW, then runs the validated B1a producers one after the other in their own processes - benchmark-tas (SAM_Tas) and benchmark-openstudio (SAM_OpenStudio) - with logs, exit codes and a cancel that kills the process tree; then compares and writes comparison.md/.csv/comparison-summary.json into the run folder, with an engine-comparison-run.json record. The open model and the Tas Energy Simulation command are not changed. - Load existing B1a documents or a comparison run folder. - Model and per-space grids (units, Tas/OpenStudio values, absolute and % difference, band) with each row labelled Compared, Not equivalent (informational or unit mismatch), Missing on Tas/OpenStudio or Not reported - never shown as zero. - Provenance (gate, mismatches, hashes, weather, design days; input manifests named as not yet available), diagnostics (gates, unmatched and duplicate spaces, reconciliation, B1a warnings/notes, producer exit codes and logs), open/export reports, open run folder. Tests (fabricated SAM_Validation documents, fake producers): every cell equals SAM_Validation's comparison.csv; status labels follow its flags; orchestration feeds both producers one snapshot and one weather file, compares and reports; failed producer, cancel and no-weather paths; producer location and arguments. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Fgbs3ko6Xix9Ut15onTY7R --- .../EngineComparisonTests.cs | 362 ++++++++++++++ .../Resources/EngineComparison/README.md | 9 + .../scenario-benchmark-OpenStudio.json | 251 ++++++++++ .../scenario-benchmark-TAS.json | 251 ++++++++++ ...weather-mismatch-benchmark-OpenStudio.json | 251 ++++++++++ .../SAM.Analytical.UI.WPF.Tests.csproj | 7 + .../Classes/Comparison/EngineComparison.cs | 111 +++++ .../Classes/Comparison/EngineComparisonRun.cs | 152 ++++++ .../Modify/RunEngineComparison.cs | 261 ++++++++++ .../Query/BenchmarkProducer.cs | 109 +++++ .../Query/EngineComparison.cs | 272 +++++++++++ .../SAM.Analytical.UI.WPF.csproj | 8 + .../Windows/AnalyticalWindow.xaml | 3 + .../Windows/AnalyticalWindow.xaml.cs | 29 ++ .../Windows/EngineComparisonWindow.xaml | 153 ++++++ .../Windows/EngineComparisonWindow.xaml.cs | 450 ++++++++++++++++++ 16 files changed, 2679 insertions(+) create mode 100644 WPF/SAM.Analytical.UI.WPF.Tests/EngineComparisonTests.cs create mode 100644 WPF/SAM.Analytical.UI.WPF.Tests/Resources/EngineComparison/README.md create mode 100644 WPF/SAM.Analytical.UI.WPF.Tests/Resources/EngineComparison/scenario-benchmark-OpenStudio.json create mode 100644 WPF/SAM.Analytical.UI.WPF.Tests/Resources/EngineComparison/scenario-benchmark-TAS.json create mode 100644 WPF/SAM.Analytical.UI.WPF.Tests/Resources/EngineComparison/weather-mismatch-benchmark-OpenStudio.json create mode 100644 WPF/SAM.Analytical.UI.WPF/Classes/Comparison/EngineComparison.cs create mode 100644 WPF/SAM.Analytical.UI.WPF/Classes/Comparison/EngineComparisonRun.cs create mode 100644 WPF/SAM.Analytical.UI.WPF/Modify/RunEngineComparison.cs create mode 100644 WPF/SAM.Analytical.UI.WPF/Query/BenchmarkProducer.cs create mode 100644 WPF/SAM.Analytical.UI.WPF/Query/EngineComparison.cs create mode 100644 WPF/SAM.Analytical.UI.WPF/Windows/EngineComparisonWindow.xaml create mode 100644 WPF/SAM.Analytical.UI.WPF/Windows/EngineComparisonWindow.xaml.cs diff --git a/WPF/SAM.Analytical.UI.WPF.Tests/EngineComparisonTests.cs b/WPF/SAM.Analytical.UI.WPF.Tests/EngineComparisonTests.cs new file mode 100644 index 00000000..134dd3fe --- /dev/null +++ b/WPF/SAM.Analytical.UI.WPF.Tests/EngineComparisonTests.cs @@ -0,0 +1,362 @@ +// SPDX-License-Identifier: LGPL-3.0-or-later +// Copyright (c) 2020-2026 Michal Dengusiak & Jakub Ziolkowski and contributors + +using SAM.Analytical.Benchmark.Compare; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Text.Json.Nodes; +using System.Threading; +using Xunit; + +namespace SAM.Analytical.UI.WPF.Tests +{ + /// + /// Simulate > Compare Tas / OpenStudio (SAM_UI#231). The documents are SAM_Validation's FABRICATED regression documents + /// (Resources/EngineComparison/README.md), not engine results. These tests prove the view computes nothing - every cell equals + /// SAM_Validation's own comparison.csv - and that the orchestration feeds both producers one snapshot and one weather file, then + /// compares and reports through SAM_Validation. The producers are faked here; the real engines are the manual Windows test. + /// + public class EngineComparisonTests : IDisposable + { + private readonly string directory; + + public EngineComparisonTests() + { + directory = Path.Combine(Path.GetTempPath(), "SAM-UI-231-tests", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(directory); + } + + public void Dispose() + { + try + { + Directory.Delete(directory, true); + } + catch (IOException) + { + } + catch (UnauthorizedAccessException) + { + } + } + + private static string Resource(string name, [CallerFilePath] string sourceFilePath = null) + { + return Path.Combine(Path.GetDirectoryName(sourceFilePath), "Resources", "EngineComparison", name); + } + + private static string Tas => Resource("scenario-benchmark-TAS.json"); + + private static string OpenStudio => Resource("scenario-benchmark-OpenStudio.json"); + + private static string OpenStudio_WeatherMismatch => Resource("weather-mismatch-benchmark-OpenStudio.json"); + + [Theory] + [InlineData("scenario-benchmark-OpenStudio.json")] + [InlineData("weather-mismatch-benchmark-OpenStudio.json")] + public void EveryCell_EqualsSamValidationsComparisonCsv(string openStudio) + { + EngineComparison engineComparison = Query.LoadEngineComparison(Tas, Resource(openStudio)); + + List csv = Benchmark.Compare.Modify.WriteCsv(engineComparison.ComparisonResult).Split('\n', StringSplitOptions.RemoveEmptyEntries).Skip(1).Select(ParseCsvLine).ToList(); + List rows = engineComparison.ModelRows.Concat(engineComparison.SpaceRows).ToList(); + + Assert.Equal(csv.Count, rows.Count); + for (int i = 0; i < rows.Count; i++) + { + string[] expected = csv[i]; + EngineComparisonRow row = rows[i]; + Assert.Equal(expected[0], row.Scope); + Assert.Equal(expected[1], row.Metric); + Assert.Equal(expected[2], row.Unit); + Assert.Equal(expected[4], row.Tas); + Assert.Equal(expected[5], row.OpenStudio); + Assert.Equal(expected[6], row.AbsoluteDifference); + Assert.Equal(expected[8], row.PercentDifference); + Assert.Equal(expected[10], row.Band); + Assert.Equal(expected[11], row.Note); + } + } + + [Fact] + public void Status_FollowsSamValidationsFlags_AndMissingValuesAreNeverZero() + { + EngineComparison engineComparison = Query.LoadEngineComparison(Tas, OpenStudio); + List rows = engineComparison.ModelRows.Concat(engineComparison.SpaceRows).ToList(); + + foreach (EngineComparisonRow row in rows) + { + MetricComparison metric = row.MetricComparison; + switch (row.Status) + { + case EngineComparisonRowStatus.Compared: + Assert.Equal(ComparisonAvailability.Both, metric.Availability); + Assert.False(Benchmark.Compare.Query.IsInformationalForNumericalGate(metric)); + Assert.NotEqual("N/A", row.Band); + break; + case EngineComparisonRowStatus.NotEquivalent: + Assert.Equal(ComparisonAvailability.Both, metric.Availability); + Assert.True(Benchmark.Compare.Query.IsInformationalForNumericalGate(metric) || metric.NotApplicableReason == NotApplicableReason.UnitMismatch); + break; + case EngineComparisonRowStatus.MissingOnTas: + Assert.Equal(ComparisonAvailability.OpenStudioOnly, metric.Availability); + Assert.Equal(string.Empty, row.Tas); + Assert.Equal(string.Empty, row.AbsoluteDifference); + break; + case EngineComparisonRowStatus.MissingOnOpenStudio: + Assert.Equal(ComparisonAvailability.TasOnly, metric.Availability); + Assert.Equal(string.Empty, row.OpenStudio); + Assert.Equal(string.Empty, row.AbsoluteDifference); + break; + case EngineComparisonRowStatus.Unavailable: + Assert.Equal(ComparisonAvailability.Neither, metric.Availability); + Assert.Equal(string.Empty, row.Tas); + Assert.Equal(string.Empty, row.OpenStudio); + break; + } + } + + //The scenario spans the cases the view must tell apart. + Assert.Contains(rows, x => x.Status == EngineComparisonRowStatus.Compared); + Assert.Contains(rows, x => x.Status == EngineComparisonRowStatus.NotEquivalent); + Assert.Contains(rows, x => x.Status == EngineComparisonRowStatus.MissingOnTas || x.Status == EngineComparisonRowStatus.MissingOnOpenStudio); + Assert.Contains(rows, x => x.Status == EngineComparisonRowStatus.Unavailable); + Assert.Contains(rows, x => x.Band == "Fail"); + } + + [Fact] + public void Provenance_NamesTheWeatherMismatch_AndTheMissingInputManifests() + { + EngineComparison engineComparison = Query.LoadEngineComparison(Tas, OpenStudio_WeatherMismatch); + + string provenance = Query.EngineComparisonProvenance(engineComparison); + + Assert.False(engineComparison.ComparisonResult.ProvenanceCompatibility.IsCompatible); + Assert.Contains("MISMATCH weatherHash", provenance); + Assert.Contains("Input manifests", provenance); + Assert.Contains("not available", provenance); + } + + [Fact] + public void Diagnostics_ListUnmatchedSpaces_GateAndReconciliation() + { + EngineComparison engineComparison = Query.LoadEngineComparison(Tas, OpenStudio); + SpaceMatchDiagnostics spaceMatchDiagnostics = engineComparison.ComparisonResult.SpaceDiagnostics; + + string diagnostics = Query.EngineComparisonDiagnostics(engineComparison); + + Assert.Contains("Overall gate: " + Format.Gate(engineComparison.ComparisonResult.Gate), diagnostics); + Assert.NotEmpty(spaceMatchDiagnostics.OnlyInTas.Concat(spaceMatchDiagnostics.OnlyInOpenStudio)); + foreach (string space in spaceMatchDiagnostics.OnlyInTas.Concat(spaceMatchDiagnostics.OnlyInOpenStudio)) + { + Assert.Contains(space, diagnostics); + } + } + + [Fact] + public void Run_FeedsBothProducersOneSnapshotAndOneWeatherFile_ThenComparesAndReports_WithoutChangingTheModel() + { + AnalyticalModel analyticalModel = Model(); + string json_Before = Core.Convert.ToString(analyticalModel); + string epwPath = Epw(); + + List calls = new List(); + EngineComparisonRun run = Modify.RunEngineComparison(analyticalModel, directory, epwPath, ProducerPath("benchmark-tas.exe"), ProducerPath("benchmark-openstudio.exe"), @"C:\models\source.json", null, CancellationToken.None, (producerRun, workDirectory, cancellationToken) => + { + calls.Add(producerRun); + File.Copy(producerRun.Engine == ComparisonEngine.Tas ? Tas : OpenStudio, Argument(producerRun, "--out")); + return 0; + }); + + Assert.Equal(EngineComparisonRunStatus.Compared, run.Status); + Assert.Equal(new[] { ComparisonEngine.Tas, ComparisonEngine.OpenStudio }, calls.Select(x => x.Engine)); + + //One snapshot and one weather file, both in the run folder, for both engines. + Assert.All(calls, x => Assert.Equal(run.SnapshotPath, Argument(x, "--model"))); + Assert.All(calls, x => Assert.Equal(run.WeatherPath, Argument(x, "--weather"))); + Assert.Equal(Query.Sha256(epwPath), run.WeatherSha256); + Assert.Equal("File", run.WeatherSource); + Assert.NotNull(Argument(calls[0], "--tbd")); + Assert.NotNull(Argument(calls[1], "--work")); + + //The reports are SAM_Validation's, for these documents. + Assert.Equal(3, run.ReportPaths.Count); + EngineComparison engineComparison = Query.LoadEngineComparison(Tas, OpenStudio); + Assert.Equal(Benchmark.Compare.Modify.WriteCsv(engineComparison.ComparisonResult), File.ReadAllText(Path.Combine(run.Directory, Benchmark.Compare.Modify.CsvFileName))); + Assert.Equal(Format.Gate(engineComparison.ComparisonResult.Gate), run.Gate); + + Assert.True(File.Exists(run.ManifestPath)); + Assert.Equal("Compared", (string)JsonNode.Parse(File.ReadAllText(run.ManifestPath))["status"]); + Assert.Equal(json_Before, Core.Convert.ToString(analyticalModel)); + } + + [Fact] + public void Run_WithAFailedProducer_IsFailed_AndComparesNothing() + { + EngineComparisonRun run = Modify.RunEngineComparison(Model(), directory, Epw(), ProducerPath("benchmark-tas.exe"), ProducerPath("benchmark-openstudio.exe"), null, null, CancellationToken.None, (producerRun, workDirectory, cancellationToken) => + { + if (producerRun.Engine == ComparisonEngine.OpenStudio) + { + File.Copy(OpenStudio, Argument(producerRun, "--out")); + return 0; + } + + return 5; + }); + + Assert.Equal(EngineComparisonRunStatus.Failed, run.Status); + Assert.Empty(run.ReportPaths); + Assert.Null(run.Gate); + Assert.Contains("exited with code 5", run.ProducerRuns[0].Error); + Assert.Contains(run.Diagnostics, x => x.Contains("Tas wrote no B1a document")); + } + + [Fact] + public void Run_Cancelled_StopsAndSaysSo() + { + using CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); + EngineComparisonRun run = Modify.RunEngineComparison(Model(), directory, Epw(), ProducerPath("benchmark-tas.exe"), ProducerPath("benchmark-openstudio.exe"), null, null, cancellationTokenSource.Token, (producerRun, workDirectory, cancellationToken) => + { + cancellationTokenSource.Cancel(); + cancellationToken.ThrowIfCancellationRequested(); + return 0; + }); + + Assert.Equal(EngineComparisonRunStatus.Cancelled, run.Status); + Assert.Single(run.ProducerRuns); + Assert.Empty(run.ReportPaths); + } + + [Fact] + public void Run_WithoutWeather_RunsNothing() + { + int calls = 0; + EngineComparisonRun run = Modify.RunEngineComparison(Model(), directory, null, ProducerPath("benchmark-tas.exe"), ProducerPath("benchmark-openstudio.exe"), null, null, CancellationToken.None, (producerRun, workDirectory, cancellationToken) => ++calls); + + //No EPW was given and the model has no WeatherData; on a machine without the OpenStudio stack the embedded-weather export + //itself fails to load. Either way nothing runs. + Assert.Equal(EngineComparisonRunStatus.Failed, run.Status); + Assert.Equal(0, calls); + Assert.True(run.Exception != null || run.Diagnostics.Any(x => x.Contains("no weather"))); + } + + [Fact] + public void Producer_IsFoundByVariableOrSavedLocation_AndArgumentsMatchTheProducers() + { + string settingsPath = Path.Combine(directory, "producers.json"); + string producer = ProducerPath("benchmark-openstudio.exe"); + + string variable = Query.BenchmarkProducerVariable(ComparisonEngine.OpenStudio); + string previous = Environment.GetEnvironmentVariable(variable); + try + { + Environment.SetEnvironmentVariable(variable, null); + Assert.Null(Query.BenchmarkProducerPath(ComparisonEngine.OpenStudio, settingsPath)); + + Assert.True(Modify.SaveBenchmarkProducerPath(ComparisonEngine.OpenStudio, producer, settingsPath)); + Assert.Equal(producer, Query.BenchmarkProducerPath(ComparisonEngine.OpenStudio, settingsPath)); + + //The variable may name the folder. + Environment.SetEnvironmentVariable(variable, Path.GetDirectoryName(producer)); + Assert.Equal(producer, Query.BenchmarkProducerPath(ComparisonEngine.OpenStudio, Path.Combine(directory, "none.json"))); + } + finally + { + Environment.SetEnvironmentVariable(variable, previous); + } + + //benchmark-tas --model --weather --tbd --out; benchmark-openstudio --model --weather --out --work. + List tas = Query.BenchmarkProducerArguments(ComparisonEngine.Tas, "m.json", "w.epw", "b.json", "work", "Block A"); + Assert.Equal(new[] { "--model", "m.json", "--weather", "w.epw", "--tbd", Path.Combine("work", "Block_A.tbd"), "--out", "b.json" }, tas); + List openStudio = Query.BenchmarkProducerArguments(ComparisonEngine.OpenStudio, "m.json", "w.epw", "b.json", "work", "Block A"); + Assert.Equal(new[] { "--model", "m.json", "--weather", "w.epw", "--out", "b.json", "--work", "work" }, openStudio); + } + + [Fact] + public void Folder_WithoutB1aDocuments_FindsNone() + { + (string tas, string openStudio) = Query.EngineComparisonDocuments(directory); + Assert.Null(tas); + Assert.Null(openStudio); + + File.Copy(Tas, Path.Combine(directory, Query.TasBenchmarkFileName)); + (tas, openStudio) = Query.EngineComparisonDocuments(directory); + Assert.NotNull(tas); + Assert.Null(openStudio); + } + + private static AnalyticalModel Model() + { + AnalyticalModel analyticalModel = new AnalyticalModel(Guid.NewGuid(), "Test model"); + Assert.True(analyticalModel.AddSpace(new Space("Room 1", new Geometry.Spatial.Point3D(1, 1, 1)))); + return analyticalModel; + } + + private string Epw() + { + string path = Path.Combine(directory, "test.epw"); + File.WriteAllText(path, "LOCATION,Test,,,TMY,000000,0,0,0,0\n"); + return path; + } + + private string ProducerPath(string fileName) + { + string path = Path.Combine(directory, "producers", fileName); + Directory.CreateDirectory(Path.GetDirectoryName(path)); + File.WriteAllText(path, string.Empty); + return path; + } + + private static string Argument(EngineComparisonProducerRun producerRun, string name) + { + int index = producerRun.Arguments.IndexOf(name); + return index < 0 ? null : producerRun.Arguments[index + 1]; + } + + private static string[] ParseCsvLine(string line) + { + List fields = new List(); + System.Text.StringBuilder field = new System.Text.StringBuilder(); + bool quoted = false; + for (int i = 0; i < line.Length; i++) + { + char c = line[i]; + if (quoted) + { + if (c == '"' && i + 1 < line.Length && line[i + 1] == '"') + { + field.Append('"'); + i++; + } + else if (c == '"') + { + quoted = false; + } + else + { + field.Append(c); + } + } + else if (c == '"') + { + quoted = true; + } + else if (c == ',') + { + fields.Add(field.ToString()); + field.Clear(); + } + else + { + field.Append(c); + } + } + + fields.Add(field.ToString()); + return fields.ToArray(); + } + } +} diff --git a/WPF/SAM.Analytical.UI.WPF.Tests/Resources/EngineComparison/README.md b/WPF/SAM.Analytical.UI.WPF.Tests/Resources/EngineComparison/README.md new file mode 100644 index 00000000..6eb64898 --- /dev/null +++ b/WPF/SAM.Analytical.UI.WPF.Tests/Resources/EngineComparison/README.md @@ -0,0 +1,9 @@ +# Engine comparison test documents (fabricated) + +B1a documents copied verbatim from SAM_Validation `SAM.Analytical.Benchmark.Compare.Tests/Fixtures/Regression/` +(branch `claude/tas-openstudio-comparison`, SAM#211). They are **fabricated test documents, not Tas or +EnergyPlus results**: a feature-rich pair spanning Match/Warn/Fail bands, a year-wrapping peak hour, an +unavailable model metric and spaces present on only one side. `weather-mismatch-*` is the same OpenStudio +document with a different weather hash, so SAM_Validation's provenance gate reports a mismatch. + +`EngineComparisonTests` checks that the comparison view shows exactly what SAM_Validation computes for them. diff --git a/WPF/SAM.Analytical.UI.WPF.Tests/Resources/EngineComparison/scenario-benchmark-OpenStudio.json b/WPF/SAM.Analytical.UI.WPF.Tests/Resources/EngineComparison/scenario-benchmark-OpenStudio.json new file mode 100644 index 00000000..07f0fb4d --- /dev/null +++ b/WPF/SAM.Analytical.UI.WPF.Tests/Resources/EngineComparison/scenario-benchmark-OpenStudio.json @@ -0,0 +1,251 @@ +{ + "schemaVersion": "1.0.0", + "provenance": { + "sourceModelName": "SingleBox", + "sourceModelGuid": "0123456789abcdef0123456789abcdef", + "sourceFileHash": "sha256:1111111111111111111111111111111111111111111111111111111111111111", + "canonicalModelHash": "sha256:2222222222222222222222222222222222222222222222222222222222222222", + "canonicalizationVersion": "1.0.0", + "samCommit": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "runnerCommit": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "engine": { + "kind": "OpenStudio", + "name": "EnergyPlus", + "version": "24.1.0", + "sdkVersion": "3.10.0" + }, + "route": "Native-OpenStudio", + "weather": { + "identity": "London-Gatwick.epw", + "hash": "sha256:3333333333333333333333333333333333333333333333333333333333333333" + }, + "designDaySource": "DDY", + "runTimestampUtc": "2026-07-21T12:00:00Z", + "durationSeconds": 12.5, + "state": "Success", + "resultSources": [ + "AnalyticalModelSimulationResult", + "SpaceSimulationResult" + ], + "warnings": [], + "notes": [] + }, + "model": { + "consumptionHeating": { + "value": 1000, + "unit": "kWh", + "available": true + }, + "consumptionCooling": { + "value": null, + "unit": "kWh", + "available": false + }, + "peakHeatingLoad": { + "value": 30, + "unit": "kW", + "available": true + }, + "peakHeatingHour": { + "value": 205, + "unit": "hourOfYear", + "available": true + }, + "peakCoolingLoad": { + "value": 8.8, + "unit": "kW", + "available": true + }, + "peakCoolingHour": { + "value": 0, + "unit": "hourOfYear", + "available": true + }, + "floorArea": { + "value": 240, + "unit": "m2", + "available": true + }, + "volume": { + "value": 720, + "unit": "m3", + "available": true + } + }, + "spaces": [ + { + "guid": "0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a", + "name": "Office A", + "area": { + "value": 200, + "unit": "m2", + "available": true + }, + "volume": { + "value": 600, + "unit": "m3", + "available": true + }, + "heating": { + "designLoad": { + "value": null, + "unit": "W", + "available": false + }, + "peakLoad": { + "value": 2000, + "unit": "W", + "available": true + }, + "peakHour": { + "value": 205, + "unit": "hourOfYear", + "available": true + }, + "unmetHours": { + "value": 0, + "unit": "h", + "available": true + } + }, + "cooling": { + "designLoad": { + "value": null, + "unit": "W", + "available": false + }, + "peakLoad": { + "value": null, + "unit": "W", + "available": false + }, + "peakHour": { + "value": null, + "unit": "hourOfYear", + "available": false + }, + "unmetHours": { + "value": null, + "unit": "h", + "available": false + } + } + }, + { + "guid": "0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b", + "name": "Office B (OS)", + "area": { + "value": 44, + "unit": "m2", + "available": true + }, + "volume": { + "value": 132, + "unit": "m3", + "available": true + }, + "heating": { + "designLoad": { + "value": null, + "unit": "W", + "available": false + }, + "peakLoad": { + "value": 2600, + "unit": "W", + "available": true + }, + "peakHour": { + "value": 240, + "unit": "hourOfYear", + "available": true + }, + "unmetHours": { + "value": 0, + "unit": "h", + "available": true + } + }, + "cooling": { + "designLoad": { + "value": null, + "unit": "W", + "available": false + }, + "peakLoad": { + "value": null, + "unit": "W", + "available": false + }, + "peakHour": { + "value": null, + "unit": "hourOfYear", + "available": false + }, + "unmetHours": { + "value": null, + "unit": "h", + "available": false + } + } + }, + { + "guid": "0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d", + "name": "Atrium", + "area": { + "value": 18, + "unit": "m2", + "available": true + }, + "volume": { + "value": 54, + "unit": "m3", + "available": true + }, + "heating": { + "designLoad": { + "value": null, + "unit": "W", + "available": false + }, + "peakLoad": { + "value": 2000, + "unit": "W", + "available": true + }, + "peakHour": { + "value": 205, + "unit": "hourOfYear", + "available": true + }, + "unmetHours": { + "value": 0, + "unit": "h", + "available": true + } + }, + "cooling": { + "designLoad": { + "value": null, + "unit": "W", + "available": false + }, + "peakLoad": { + "value": null, + "unit": "W", + "available": false + }, + "peakHour": { + "value": null, + "unit": "hourOfYear", + "available": false + }, + "unmetHours": { + "value": null, + "unit": "h", + "available": false + } + } + } + ] +} \ No newline at end of file diff --git a/WPF/SAM.Analytical.UI.WPF.Tests/Resources/EngineComparison/scenario-benchmark-TAS.json b/WPF/SAM.Analytical.UI.WPF.Tests/Resources/EngineComparison/scenario-benchmark-TAS.json new file mode 100644 index 00000000..346d6140 --- /dev/null +++ b/WPF/SAM.Analytical.UI.WPF.Tests/Resources/EngineComparison/scenario-benchmark-TAS.json @@ -0,0 +1,251 @@ +{ + "schemaVersion": "1.0.0", + "provenance": { + "sourceModelName": "SingleBox", + "sourceModelGuid": "0123456789abcdef0123456789abcdef", + "sourceFileHash": "sha256:1111111111111111111111111111111111111111111111111111111111111111", + "canonicalModelHash": "sha256:2222222222222222222222222222222222222222222222222222222222222222", + "canonicalizationVersion": "1.0.0", + "samCommit": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "runnerCommit": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "engine": { + "kind": "TAS", + "name": "TAS", + "version": "9.5.3", + "sdkVersion": null + }, + "route": "Native-TAS", + "weather": { + "identity": "London-Gatwick.epw", + "hash": "sha256:3333333333333333333333333333333333333333333333333333333333333333" + }, + "designDaySource": "DDY", + "runTimestampUtc": "2026-07-21T12:00:00Z", + "durationSeconds": 12.5, + "state": "Success", + "resultSources": [ + "AnalyticalModelSimulationResult", + "SpaceSimulationResult" + ], + "warnings": [], + "notes": [] + }, + "model": { + "consumptionHeating": { + "value": 1000, + "unit": "kWh", + "available": true + }, + "consumptionCooling": { + "value": 500, + "unit": "kWh", + "available": true + }, + "peakHeatingLoad": { + "value": 10, + "unit": "kW", + "available": true + }, + "peakHeatingHour": { + "value": 200, + "unit": "hourOfYear", + "available": true + }, + "peakCoolingLoad": { + "value": 8, + "unit": "kW", + "available": true + }, + "peakCoolingHour": { + "value": 8759, + "unit": "hourOfYear", + "available": true + }, + "floorArea": { + "value": 240, + "unit": "m2", + "available": true + }, + "volume": { + "value": 720, + "unit": "m3", + "available": true + } + }, + "spaces": [ + { + "guid": "0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a", + "name": "Office A", + "area": { + "value": 200, + "unit": "m2", + "available": true + }, + "volume": { + "value": 600, + "unit": "m3", + "available": true + }, + "heating": { + "designLoad": { + "value": null, + "unit": "W", + "available": false + }, + "peakLoad": { + "value": 2000, + "unit": "W", + "available": true + }, + "peakHour": { + "value": 205, + "unit": "hourOfYear", + "available": true + }, + "unmetHours": { + "value": 0, + "unit": "h", + "available": true + } + }, + "cooling": { + "designLoad": { + "value": null, + "unit": "W", + "available": false + }, + "peakLoad": { + "value": null, + "unit": "W", + "available": false + }, + "peakHour": { + "value": null, + "unit": "hourOfYear", + "available": false + }, + "unmetHours": { + "value": null, + "unit": "h", + "available": false + } + } + }, + { + "guid": "0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b", + "name": "Office B", + "area": { + "value": 40, + "unit": "m2", + "available": true + }, + "volume": { + "value": 120, + "unit": "m3", + "available": true + }, + "heating": { + "designLoad": { + "value": null, + "unit": "W", + "available": false + }, + "peakLoad": { + "value": 2100, + "unit": "W", + "available": true + }, + "peakHour": { + "value": 205, + "unit": "hourOfYear", + "available": true + }, + "unmetHours": { + "value": 0, + "unit": "h", + "available": true + } + }, + "cooling": { + "designLoad": { + "value": null, + "unit": "W", + "available": false + }, + "peakLoad": { + "value": null, + "unit": "W", + "available": false + }, + "peakHour": { + "value": null, + "unit": "hourOfYear", + "available": false + }, + "unmetHours": { + "value": null, + "unit": "h", + "available": false + } + } + }, + { + "guid": "0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c", + "name": "Plant, level 1", + "area": { + "value": 12, + "unit": "m2", + "available": true + }, + "volume": { + "value": 36, + "unit": "m3", + "available": true + }, + "heating": { + "designLoad": { + "value": null, + "unit": "W", + "available": false + }, + "peakLoad": { + "value": 2000, + "unit": "W", + "available": true + }, + "peakHour": { + "value": 205, + "unit": "hourOfYear", + "available": true + }, + "unmetHours": { + "value": 0, + "unit": "h", + "available": true + } + }, + "cooling": { + "designLoad": { + "value": null, + "unit": "W", + "available": false + }, + "peakLoad": { + "value": null, + "unit": "W", + "available": false + }, + "peakHour": { + "value": null, + "unit": "hourOfYear", + "available": false + }, + "unmetHours": { + "value": null, + "unit": "h", + "available": false + } + } + } + ] +} \ No newline at end of file diff --git a/WPF/SAM.Analytical.UI.WPF.Tests/Resources/EngineComparison/weather-mismatch-benchmark-OpenStudio.json b/WPF/SAM.Analytical.UI.WPF.Tests/Resources/EngineComparison/weather-mismatch-benchmark-OpenStudio.json new file mode 100644 index 00000000..189ca333 --- /dev/null +++ b/WPF/SAM.Analytical.UI.WPF.Tests/Resources/EngineComparison/weather-mismatch-benchmark-OpenStudio.json @@ -0,0 +1,251 @@ +{ + "schemaVersion": "1.0.0", + "provenance": { + "sourceModelName": "SingleBox", + "sourceModelGuid": "0123456789abcdef0123456789abcdef", + "sourceFileHash": "sha256:1111111111111111111111111111111111111111111111111111111111111111", + "canonicalModelHash": "sha256:2222222222222222222222222222222222222222222222222222222222222222", + "canonicalizationVersion": "1.0.0", + "samCommit": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "runnerCommit": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "engine": { + "kind": "OpenStudio", + "name": "EnergyPlus", + "version": "24.1.0", + "sdkVersion": "3.10.0" + }, + "route": "Native-OpenStudio", + "weather": { + "identity": "Boston.epw", + "hash": "sha256:9999999999999999999999999999999999999999999999999999999999999999" + }, + "designDaySource": "DDY", + "runTimestampUtc": "2026-07-21T12:00:00Z", + "durationSeconds": 12.5, + "state": "Success", + "resultSources": [ + "AnalyticalModelSimulationResult", + "SpaceSimulationResult" + ], + "warnings": [], + "notes": [] + }, + "model": { + "consumptionHeating": { + "value": 1000, + "unit": "kWh", + "available": true + }, + "consumptionCooling": { + "value": null, + "unit": "kWh", + "available": false + }, + "peakHeatingLoad": { + "value": 30, + "unit": "kW", + "available": true + }, + "peakHeatingHour": { + "value": 205, + "unit": "hourOfYear", + "available": true + }, + "peakCoolingLoad": { + "value": 8.8, + "unit": "kW", + "available": true + }, + "peakCoolingHour": { + "value": 0, + "unit": "hourOfYear", + "available": true + }, + "floorArea": { + "value": 240, + "unit": "m2", + "available": true + }, + "volume": { + "value": 720, + "unit": "m3", + "available": true + } + }, + "spaces": [ + { + "guid": "0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a", + "name": "Office A", + "area": { + "value": 200, + "unit": "m2", + "available": true + }, + "volume": { + "value": 600, + "unit": "m3", + "available": true + }, + "heating": { + "designLoad": { + "value": null, + "unit": "W", + "available": false + }, + "peakLoad": { + "value": 2000, + "unit": "W", + "available": true + }, + "peakHour": { + "value": 205, + "unit": "hourOfYear", + "available": true + }, + "unmetHours": { + "value": 0, + "unit": "h", + "available": true + } + }, + "cooling": { + "designLoad": { + "value": null, + "unit": "W", + "available": false + }, + "peakLoad": { + "value": null, + "unit": "W", + "available": false + }, + "peakHour": { + "value": null, + "unit": "hourOfYear", + "available": false + }, + "unmetHours": { + "value": null, + "unit": "h", + "available": false + } + } + }, + { + "guid": "0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b", + "name": "Office B (OS)", + "area": { + "value": 44, + "unit": "m2", + "available": true + }, + "volume": { + "value": 132, + "unit": "m3", + "available": true + }, + "heating": { + "designLoad": { + "value": null, + "unit": "W", + "available": false + }, + "peakLoad": { + "value": 2600, + "unit": "W", + "available": true + }, + "peakHour": { + "value": 240, + "unit": "hourOfYear", + "available": true + }, + "unmetHours": { + "value": 0, + "unit": "h", + "available": true + } + }, + "cooling": { + "designLoad": { + "value": null, + "unit": "W", + "available": false + }, + "peakLoad": { + "value": null, + "unit": "W", + "available": false + }, + "peakHour": { + "value": null, + "unit": "hourOfYear", + "available": false + }, + "unmetHours": { + "value": null, + "unit": "h", + "available": false + } + } + }, + { + "guid": "0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d", + "name": "Atrium", + "area": { + "value": 18, + "unit": "m2", + "available": true + }, + "volume": { + "value": 54, + "unit": "m3", + "available": true + }, + "heating": { + "designLoad": { + "value": null, + "unit": "W", + "available": false + }, + "peakLoad": { + "value": 2000, + "unit": "W", + "available": true + }, + "peakHour": { + "value": 205, + "unit": "hourOfYear", + "available": true + }, + "unmetHours": { + "value": 0, + "unit": "h", + "available": true + } + }, + "cooling": { + "designLoad": { + "value": null, + "unit": "W", + "available": false + }, + "peakLoad": { + "value": null, + "unit": "W", + "available": false + }, + "peakHour": { + "value": null, + "unit": "hourOfYear", + "available": false + }, + "unmetHours": { + "value": null, + "unit": "h", + "available": false + } + } + } + ] +} \ No newline at end of file diff --git a/WPF/SAM.Analytical.UI.WPF.Tests/SAM.Analytical.UI.WPF.Tests.csproj b/WPF/SAM.Analytical.UI.WPF.Tests/SAM.Analytical.UI.WPF.Tests.csproj index f01e3cfd..36208a2f 100644 --- a/WPF/SAM.Analytical.UI.WPF.Tests/SAM.Analytical.UI.WPF.Tests.csproj +++ b/WPF/SAM.Analytical.UI.WPF.Tests/SAM.Analytical.UI.WPF.Tests.csproj @@ -103,6 +103,13 @@ ..\..\..\SAM_OpenStudio\build\SAM.Core.OpenStudio.dll + + + ..\..\..\SAM_Validation\build\SAM.Analytical.Benchmark.dll + + + ..\..\..\SAM_Validation\build\SAM.Analytical.Benchmark.Compare.dll + diff --git a/WPF/SAM.Analytical.UI.WPF/Classes/Comparison/EngineComparison.cs b/WPF/SAM.Analytical.UI.WPF/Classes/Comparison/EngineComparison.cs new file mode 100644 index 00000000..c76a0c3c --- /dev/null +++ b/WPF/SAM.Analytical.UI.WPF/Classes/Comparison/EngineComparison.cs @@ -0,0 +1,111 @@ +// SPDX-License-Identifier: LGPL-3.0-or-later +// Copyright (c) 2020-2026 Michal Dengusiak & Jakub Ziolkowski and contributors + +using SAM.Analytical.Benchmark; +using SAM.Analytical.Benchmark.Compare; +using System.Collections.Generic; + +namespace SAM.Analytical.UI.WPF +{ + /// What a comparison row is, as the Tas/OpenStudio comparison view shows it (SAM_UI#231). + public enum EngineComparisonRowStatus + { + /// Both engines report the metric in the same unit; SAM_Validation banded it Match, Warn or Fail. + Compared, + + /// + /// Both engines report a value, but the two are not the same quantity: SAM_Validation treats the metric as informational + /// (a peak hour, or a whole-model peak under different peak definitions), or the units differ. Shown, never banded as a result. + /// + NotEquivalent, + + /// Only OpenStudio reports the metric. + MissingOnTas, + + /// Only Tas reports the metric. + MissingOnOpenStudio, + + /// Neither engine reports the metric. + Unavailable, + } + + /// + /// One metric of a Tas/OpenStudio comparison, ready to display. Every value is SAM_Validation's: the numbers, differences, band and + /// note are its MetricComparison, rendered with its own Format, so the view shows exactly what comparison.csv says. The + /// only thing decided here is , a label over SAM_Validation's availability, unit and informational flags. + /// + public sealed class EngineComparisonRow + { + public EngineComparisonRow(MetricComparison metricComparison, string? spaceName) + { + MetricComparison = metricComparison; + SpaceName = spaceName; + + Status = Query.ComparisonRowStatus(metricComparison); + } + + public MetricComparison MetricComparison { get; } + + /// "model" for a model metric, else the space's SAM GUID (or its name when it has none). + public string Scope => MetricComparison.Scope; + + /// The space's name as the engines report it ("Tas name / OpenStudio name" when they differ); null for a model metric. + public string? SpaceName { get; } + + public string Metric => MetricComparison.Key; + + public string Unit => Format.Unit(MetricComparison.Unit); + + public string Tas => Format.Number(MetricComparison.TasValue); + + public string OpenStudio => Format.Number(MetricComparison.OpenStudioValue); + + public string AbsoluteDifference => Format.Number(MetricComparison.AbsoluteDifference); + + /// The relative difference in percent, as comparison.csv's relativePercent column. + public string PercentDifference => Format.Percent(MetricComparison.RelativeDifference); + + /// SAM_Validation's band: Match, Warn, Fail or N/A. A tolerance band, not an engineering verdict. + public string Band => Format.Band(MetricComparison.Band); + + public EngineComparisonRowStatus Status { get; } + + public string StatusText => Query.ComparisonRowStatusText(Status); + + /// SAM_Validation's note for the metric (why it is N/A or informational), as in comparison.csv. + public string Note => Format.MetricNote(MetricComparison); + } + + /// + /// A loaded Tas/OpenStudio comparison: the two B1a documents and SAM_Validation's for them, with the rows + /// the view shows. Built by . + /// + public sealed class EngineComparison + { + public EngineComparison(string tasPath, BenchmarkDocument tas, string openStudioPath, BenchmarkDocument openStudio, ComparisonResult comparisonResult) + { + TasPath = tasPath; + Tas = tas; + OpenStudioPath = openStudioPath; + OpenStudio = openStudio; + ComparisonResult = comparisonResult; + + ModelRows = Query.ComparisonModelRows(comparisonResult); + SpaceRows = Query.ComparisonSpaceRows(comparisonResult); + } + + public string TasPath { get; } + + public BenchmarkDocument Tas { get; } + + public string OpenStudioPath { get; } + + public BenchmarkDocument OpenStudio { get; } + + public ComparisonResult ComparisonResult { get; } + + public IReadOnlyList ModelRows { get; } + + public IReadOnlyList SpaceRows { get; } + } +} diff --git a/WPF/SAM.Analytical.UI.WPF/Classes/Comparison/EngineComparisonRun.cs b/WPF/SAM.Analytical.UI.WPF/Classes/Comparison/EngineComparisonRun.cs new file mode 100644 index 00000000..16ec4957 --- /dev/null +++ b/WPF/SAM.Analytical.UI.WPF/Classes/Comparison/EngineComparisonRun.cs @@ -0,0 +1,152 @@ +// SPDX-License-Identifier: LGPL-3.0-or-later +// Copyright (c) 2020-2026 Michal Dengusiak & Jakub Ziolkowski and contributors + +using System; +using System.Collections.Generic; +using System.Text.Json; +using System.Text.Json.Nodes; + +namespace SAM.Analytical.UI.WPF +{ + /// The simulation engines of the Tas/OpenStudio comparison. + public enum ComparisonEngine + { + Tas, + OpenStudio, + } + + /// How a Tas/OpenStudio comparison run ended. + public enum EngineComparisonRunStatus + { + /// Both producers wrote their B1a document and SAM_Validation compared them; the reports are written. + Compared, + + /// A producer failed, or its document could not be compared; the diagnostics and logs say which. + Failed, + + /// The user cancelled; the running producer's process tree was killed. + Cancelled, + } + + /// One producer run (benchmark-tas or benchmark-openstudio) of a comparison. + public sealed class EngineComparisonProducerRun + { + public ComparisonEngine Engine { get; set; } + + public string? ExecutablePath { get; set; } + + public List Arguments { get; } = new List(); + + public int? ExitCode { get; set; } + + public double DurationSeconds { get; set; } + + /// The producer's standard output and error. + public string? LogPath { get; set; } + + /// The B1a document it wrote; null when it wrote none. + public string? BenchmarkPath { get; set; } + + public string? Error { get; set; } + } + + /// + /// The record of one Simulate > Compare Tas/OpenStudio run (SAM_UI#231): one frozen snapshot and one weather file, both engines' + /// producers run on them in turn, and SAM_Validation's comparison of the two B1a documents. Written to the run folder as + /// . It holds no result value: the results are the B1a documents and SAM_Validation's reports. + /// + public sealed class EngineComparisonRun + { + public const string Schema = "sam-ui.engine-comparison-run/1"; + + public const string ManifestFileName = "engine-comparison-run.json"; + + public string? Id { get; set; } + + public EngineComparisonRunStatus Status { get; set; } = EngineComparisonRunStatus.Failed; + + public DateTime StartedUtc { get; set; } + + public DateTime FinishedUtc { get; set; } + + public string? Directory { get; set; } + + public string? SourcePath { get; set; } + + public string? ModelName { get; set; } + + public string? SnapshotPath { get; set; } + + public string? SnapshotSha256 { get; set; } + + /// "File" (chosen by the user), "Embedded" (the model's WeatherData exported to EPW) or "None". + public string? WeatherSource { get; set; } + + public string? WeatherPath { get; set; } + + public string? WeatherSha256 { get; set; } + + public List ProducerRuns { get; } = new List(); + + /// SAM_Validation's overall gate ("Pass", "Warn", "Fail") when the documents were compared. + public string? Gate { get; set; } + + /// comparison.md, comparison.csv and comparison-summary.json, as SAM_Validation wrote them. + public List ReportPaths { get; } = new List(); + + public List Diagnostics { get; } = new List(); + + public Exception? Exception { get; set; } + + public string? ManifestPath { get; set; } + + /// Strings are added with JsonValue.Create: the generic JsonArray.Add<string> cannot be written with these options. + public string ToJson() + { + JsonArray producerRuns = new JsonArray(); + foreach (EngineComparisonProducerRun producerRun in ProducerRuns) + { + JsonArray arguments = new JsonArray(); + producerRun.Arguments.ForEach(x => arguments.Add(JsonValue.Create(x))); + + producerRuns.Add(new JsonObject + { + ["engine"] = producerRun.Engine.ToString(), + ["executable"] = producerRun.ExecutablePath, + ["arguments"] = arguments, + ["exitCode"] = producerRun.ExitCode, + ["durationSeconds"] = System.Math.Round(producerRun.DurationSeconds, 3), + ["log"] = producerRun.LogPath, + ["benchmark"] = producerRun.BenchmarkPath, + ["error"] = producerRun.Error, + }); + } + + JsonArray reportPaths = new JsonArray(); + ReportPaths.ForEach(x => reportPaths.Add(JsonValue.Create(x))); + + JsonArray diagnostics = new JsonArray(); + Diagnostics.ForEach(x => diagnostics.Add(JsonValue.Create(x))); + + JsonObject jsonObject = new JsonObject + { + ["schema"] = Schema, + ["id"] = Id, + ["status"] = Status.ToString(), + ["startedUtc"] = StartedUtc.ToString("o"), + ["finishedUtc"] = FinishedUtc.ToString("o"), + ["sourcePath"] = SourcePath, + ["modelName"] = ModelName, + ["snapshot"] = new JsonObject { ["path"] = SnapshotPath, ["sha256"] = SnapshotSha256 }, + ["weather"] = new JsonObject { ["source"] = WeatherSource, ["path"] = WeatherPath, ["sha256"] = WeatherSha256 }, + ["producers"] = producerRuns, + ["gate"] = Gate, + ["reports"] = reportPaths, + ["diagnostics"] = diagnostics, + ["error"] = Exception == null ? null : Exception.GetType().Name + ": " + Exception.Message, + }; + + return jsonObject.ToJsonString(new JsonSerializerOptions { WriteIndented = true }); + } + } +} diff --git a/WPF/SAM.Analytical.UI.WPF/Modify/RunEngineComparison.cs b/WPF/SAM.Analytical.UI.WPF/Modify/RunEngineComparison.cs new file mode 100644 index 00000000..0da30c97 --- /dev/null +++ b/WPF/SAM.Analytical.UI.WPF/Modify/RunEngineComparison.cs @@ -0,0 +1,261 @@ +// SPDX-License-Identifier: LGPL-3.0-or-later +// Copyright (c) 2020-2026 Michal Dengusiak & Jakub Ziolkowski and contributors + +using SAM.Analytical.OpenStudio; +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Runtime.CompilerServices; +using System.Text.Json.Nodes; +using System.Threading; + +namespace SAM.Analytical.UI.WPF +{ + public static partial class Modify + { + /// + /// Simulate > Compare Tas/OpenStudio (SAM_UI#231): runs both engines, one after the other, on ONE frozen snapshot of + /// and ONE weather file, then compares their B1a documents with SAM_Validation. + /// + /// The engines run through their validated B1a producers, each in its own process: benchmark-tas (SAM_Tas, the native Tas route + /// with that producer's fixed workflow settings - not the Energy Simulation dialog's) and benchmark-openstudio (SAM_OpenStudio, the + /// same ToOpenStudio pipeline as Simulate > OpenStudio Simulation). SAM_UI neither maps results to B1a nor compares anything + /// itself: the documents are the producers', and the comparison and reports are SAM_Validation's (Query.Compare, WriteReports). + /// + /// + /// The model given is never changed. Never throws for a failed run: the outcome, each producer's exit code and log, and any error + /// are on the returned record, which is also written to the run folder (). + /// + /// + /// The model to simulate; read only. + /// The folder the run folder is created in. + /// An EPW chosen by the user; null to export the model's embedded WeatherData to EPW once, for both engines. + /// benchmark-tas.exe. + /// benchmark-openstudio.exe. + /// The open model's file, recorded for provenance only. + /// One line per step. + /// Cancels the run; the running producer's process tree is killed. + public static EngineComparisonRun RunEngineComparison(this AnalyticalModel analyticalModel, string directory, string? epwPath, string tasProducerPath, string openStudioProducerPath, string? sourcePath = null, IProgress? progress = null, CancellationToken cancellationToken = default) + { + return RunEngineComparison(analyticalModel, directory, epwPath, tasProducerPath, openStudioProducerPath, sourcePath, progress, cancellationToken, RunBenchmarkProducer); + } + + /// The same run with the producer runner given, so the orchestration can be tested without either engine. + internal static EngineComparisonRun RunEngineComparison(AnalyticalModel analyticalModel, string directory, string? epwPath, string tasProducerPath, string openStudioProducerPath, string? sourcePath, IProgress? progress, CancellationToken cancellationToken, Func runProducer) + { + EngineComparisonRun result = new EngineComparisonRun() + { + Id = Guid.NewGuid().ToString("N").Substring(0, 8), + StartedUtc = DateTime.UtcNow, + SourcePath = sourcePath, + ModelName = analyticalModel?.Name, + }; + + if (analyticalModel == null || string.IsNullOrWhiteSpace(directory)) + { + result.Diagnostics.Add("Error: no model or no output folder was given; nothing was run"); + result.FinishedUtc = DateTime.UtcNow; + return result; + } + + result.Directory = Query.EngineComparisonRunDirectory(directory, analyticalModel.Name, result.StartedUtc, result.Id!); + + try + { + System.IO.Directory.CreateDirectory(result.Directory); + + progress?.Report("Freezing the model"); + result.SnapshotPath = Path.Combine(result.Directory, OpenStudioSimulationRun.SnapshotFileName); + AnalyticalModel? analyticalModel_Frozen = analyticalModel.FreezeAnalyticalModel(result.SnapshotPath); + if (analyticalModel_Frozen == null) + { + result.Diagnostics.Add("Error: the model could not be written to and read back from the snapshot file; nothing was run"); + return result; + } + + result.SnapshotSha256 = Query.Sha256(result.SnapshotPath); + + //One weather file for both engines, so both documents carry the same weather hash. + progress?.Report("Preparing the weather file"); + string? epwPath_User = !string.IsNullOrWhiteSpace(epwPath) && File.Exists(epwPath) ? epwPath : null; + string? epwPath_Source = epwPath_User ?? EmbeddedWeatherPath(analyticalModel_Frozen); + result.WeatherSource = epwPath_User != null ? "File" : epwPath_Source != null ? "Embedded" : "None"; + if (epwPath_Source == null) + { + result.Diagnostics.Add("Error: no weather - choose an EPW file, or give the model annual WeatherData; nothing was run"); + return result; + } + + result.WeatherPath = Path.Combine(result.Directory, OpenStudioSimulationRun.WeatherFileName); + File.Copy(epwPath_Source, result.WeatherPath, true); + result.WeatherSha256 = Query.Sha256(result.WeatherPath); + + foreach ((ComparisonEngine engine, string producerPath) in new[] { (ComparisonEngine.Tas, tasProducerPath), (ComparisonEngine.OpenStudio, openStudioProducerPath) }) + { + cancellationToken.ThrowIfCancellationRequested(); + + progress?.Report(engine == ComparisonEngine.Tas ? "Running Tas (benchmark-tas)" : "Running OpenStudio/EnergyPlus (benchmark-openstudio)"); + + string workDirectory = Path.Combine(result.Directory, engine == ComparisonEngine.Tas ? "tas" : "openstudio"); + System.IO.Directory.CreateDirectory(workDirectory); + + string benchmarkPath = Path.Combine(result.Directory, engine == ComparisonEngine.Tas ? Query.TasBenchmarkFileName : Query.OpenStudioBenchmarkFileName); + EngineComparisonProducerRun producerRun = new EngineComparisonProducerRun() + { + Engine = engine, + ExecutablePath = producerPath, + LogPath = Path.Combine(result.Directory, engine == ComparisonEngine.Tas ? "benchmark-tas.log" : "benchmark-openstudio.log"), + }; + producerRun.Arguments.AddRange(Query.BenchmarkProducerArguments(engine, result.SnapshotPath, result.WeatherPath, benchmarkPath, workDirectory, analyticalModel_Frozen.Name)); + result.ProducerRuns.Add(producerRun); + + if (string.IsNullOrWhiteSpace(producerPath) || !File.Exists(producerPath)) + { + producerRun.Error = "The producer executable was not found: " + (producerPath ?? "(none)"); + continue; + } + + Stopwatch stopwatch = Stopwatch.StartNew(); + try + { + producerRun.ExitCode = runProducer(producerRun, workDirectory, cancellationToken); + } + catch (Exception exception) when (!(exception is OperationCanceledException)) + { + producerRun.Error = exception.GetType().Name + ": " + exception.Message; + } + finally + { + producerRun.DurationSeconds = stopwatch.Elapsed.TotalSeconds; + } + + producerRun.BenchmarkPath = File.Exists(benchmarkPath) ? benchmarkPath : null; + if (producerRun.ExitCode != 0 && producerRun.Error == null) + { + producerRun.Error = string.Format("The producer exited with code {0} (2 usage, 3 input/IO, 4 validation, 5 producer failure); see {1}", producerRun.ExitCode, Path.GetFileName(producerRun.LogPath)); + } + } + + cancellationToken.ThrowIfCancellationRequested(); + + EngineComparisonProducerRun tas = result.ProducerRuns[0]; + EngineComparisonProducerRun openStudio = result.ProducerRuns[1]; + if (tas.BenchmarkPath == null || openStudio.BenchmarkPath == null) + { + result.Diagnostics.Add("Error: no comparison - " + (tas.BenchmarkPath == null ? "Tas" : "OpenStudio") + (tas.BenchmarkPath == null && openStudio.BenchmarkPath == null ? " and OpenStudio" : string.Empty) + " wrote no B1a document"); + return result; + } + + progress?.Report("Comparing (SAM_Validation)"); + EngineComparison engineComparison = Query.LoadEngineComparison(tas.BenchmarkPath, openStudio.BenchmarkPath); + result.ReportPaths.AddRange(Benchmark.Compare.Modify.WriteReports(engineComparison.ComparisonResult, result.Directory)); + result.Gate = Benchmark.Compare.Format.Gate(engineComparison.ComparisonResult.Gate); + + //Compared is about the workflow, not the engines' agreement: a failed producer still fails the run even when an older + //document happened to be compared. + result.Status = tas.ExitCode == 0 && openStudio.ExitCode == 0 ? EngineComparisonRunStatus.Compared : EngineComparisonRunStatus.Failed; + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + result.Status = EngineComparisonRunStatus.Cancelled; + } + catch (Exception exception) + { + result.Status = EngineComparisonRunStatus.Failed; + result.Exception = exception; + } + finally + { + result.FinishedUtc = DateTime.UtcNow; + WriteEngineComparisonManifest(result); + } + + return result; + } + + /// Remembers the producer the user located, for . + public static bool SaveBenchmarkProducerPath(ComparisonEngine comparisonEngine, string path, string? settingsPath = null) + { + settingsPath ??= Query.BenchmarkProducerSettingsPath(); + try + { + JsonObject jsonObject = File.Exists(settingsPath) ? JsonNode.Parse(File.ReadAllText(settingsPath)) as JsonObject ?? new JsonObject() : new JsonObject(); + jsonObject[comparisonEngine.ToString()] = path; + + System.IO.Directory.CreateDirectory(Path.GetDirectoryName(settingsPath)!); + File.WriteAllText(settingsPath, jsonObject.ToJsonString()); + return true; + } + catch (Exception exception) when (exception is IOException || exception is UnauthorizedAccessException || exception is System.Text.Json.JsonException) + { + return false; + } + } + + // Its own method, never inlined: SAM_OpenStudio's embedded-weather export is reached here, so a missing SAM_OpenStudio assembly + // fails inside the caller's try and is recorded on the run. + [MethodImpl(MethodImplOptions.NoInlining)] + private static string? EmbeddedWeatherPath(AnalyticalModel analyticalModel) + { + return analyticalModel.EmbeddedAnnualWeatherPath(); + } + + /// Runs a producer as a child process, its output to its log; cancellation kills the whole process tree. + private static int RunBenchmarkProducer(EngineComparisonProducerRun producerRun, string workDirectory, CancellationToken cancellationToken) + { + ProcessStartInfo processStartInfo = new ProcessStartInfo(producerRun.ExecutablePath!) + { + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true, + CreateNoWindow = true, + WorkingDirectory = workDirectory, + }; + producerRun.Arguments.ForEach(x => processStartInfo.ArgumentList.Add(x)); + + using StreamWriter streamWriter = new StreamWriter(producerRun.LogPath!); + object @lock = new object(); + + using Process process = Process.Start(processStartInfo) ?? throw new InvalidOperationException("The producer process did not start"); + process.OutputDataReceived += (s, e) => { if (e.Data != null) { lock (@lock) { streamWriter.WriteLine(e.Data); } } }; + process.ErrorDataReceived += (s, e) => { if (e.Data != null) { lock (@lock) { streamWriter.WriteLine("[stderr] " + e.Data); } } }; + process.BeginOutputReadLine(); + process.BeginErrorReadLine(); + + while (!process.WaitForExit(250)) + { + if (cancellationToken.IsCancellationRequested) + { + process.Kill(true); + process.WaitForExit(); + cancellationToken.ThrowIfCancellationRequested(); + } + } + + //Drains the asynchronous output before the log is closed. + process.WaitForExit(); + return process.ExitCode; + } + + private static void WriteEngineComparisonManifest(EngineComparisonRun engineComparisonRun) + { + if (string.IsNullOrWhiteSpace(engineComparisonRun.Directory) || !System.IO.Directory.Exists(engineComparisonRun.Directory)) + { + return; + } + + string path = Path.Combine(engineComparisonRun.Directory!, EngineComparisonRun.ManifestFileName); + try + { + File.WriteAllText(path, engineComparisonRun.ToJson()); + engineComparisonRun.ManifestPath = path; + } + catch (Exception exception) + { + //Written from a finally on a worker thread: whatever goes wrong here must not take the run, or the application, down. + engineComparisonRun.Diagnostics.Add("Warning: the run manifest could not be written: " + exception.GetType().Name + ": " + exception.Message); + } + } + } +} diff --git a/WPF/SAM.Analytical.UI.WPF/Query/BenchmarkProducer.cs b/WPF/SAM.Analytical.UI.WPF/Query/BenchmarkProducer.cs new file mode 100644 index 00000000..cd83e0af --- /dev/null +++ b/WPF/SAM.Analytical.UI.WPF/Query/BenchmarkProducer.cs @@ -0,0 +1,109 @@ +// SPDX-License-Identifier: LGPL-3.0-or-later +// Copyright (c) 2020-2026 Michal Dengusiak & Jakub Ziolkowski and contributors + +using System; +using System.Collections.Generic; +using System.IO; +using System.Text.Json.Nodes; + +namespace SAM.Analytical.UI.WPF +{ + public static partial class Query + { + /// + /// The producer executable of an engine: benchmark-tas (SAM_Tas\build) or benchmark-openstudio (SAM_OpenStudio's benchmark + /// project). These are the validated B1a producers; SAM_UI runs them rather than mapping results to B1a a second way. + /// + public static string BenchmarkProducerFileName(ComparisonEngine comparisonEngine) + { + return comparisonEngine == ComparisonEngine.Tas ? "benchmark-tas.exe" : "benchmark-openstudio.exe"; + } + + /// The environment variable that may name an engine's producer (the executable, or the folder holding it). + public static string BenchmarkProducerVariable(ComparisonEngine comparisonEngine) + { + return comparisonEngine == ComparisonEngine.Tas ? "SAM_BENCHMARK_TAS" : "SAM_BENCHMARK_OPENSTUDIO"; + } + + /// Where the producer locations chosen by the user are remembered. + public static string BenchmarkProducerSettingsPath() + { + return Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), "SAM", "engine-comparison-producers.json"); + } + + /// + /// The producer executable to run for , or null when none is found: the environment variable + /// first, then the application folder, then the location the user chose before (). + /// + public static string? BenchmarkProducerPath(ComparisonEngine comparisonEngine, string? settingsPath = null) + { + string fileName = BenchmarkProducerFileName(comparisonEngine); + + List candidates = new List + { + Environment.GetEnvironmentVariable(BenchmarkProducerVariable(comparisonEngine)), + Path.Combine(AppContext.BaseDirectory, fileName), + SavedBenchmarkProducerPath(comparisonEngine, settingsPath ?? BenchmarkProducerSettingsPath()), + }; + + foreach (string? candidate in candidates) + { + if (string.IsNullOrWhiteSpace(candidate)) + { + continue; + } + + string path = Directory.Exists(candidate) ? Path.Combine(candidate!, fileName) : candidate!; + if (File.Exists(path)) + { + return path; + } + } + + return null; + } + + /// + /// The command line of a producer run: the frozen snapshot and one weather file for both engines, so both B1a documents carry the + /// same model and weather provenance. Tas writes its TBD (and gbXML) under ; OpenStudio runs there. + /// + public static List BenchmarkProducerArguments(ComparisonEngine comparisonEngine, string snapshotPath, string weatherPath, string benchmarkPath, string workDirectory, string? modelName) + { + List result = new List { "--model", snapshotPath, "--weather", weatherPath }; + + if (comparisonEngine == ComparisonEngine.Tas) + { + result.Add("--tbd"); + result.Add(Path.Combine(workDirectory, FileNameSafe(modelName) + ".tbd")); + } + + result.Add("--out"); + result.Add(benchmarkPath); + + if (comparisonEngine == ComparisonEngine.OpenStudio) + { + result.Add("--work"); + result.Add(workDirectory); + } + + return result; + } + + private static string? SavedBenchmarkProducerPath(ComparisonEngine comparisonEngine, string settingsPath) + { + try + { + if (!File.Exists(settingsPath)) + { + return null; + } + + return JsonNode.Parse(File.ReadAllText(settingsPath))?[comparisonEngine.ToString()]?.GetValue(); + } + catch (Exception exception) when (exception is IOException || exception is UnauthorizedAccessException || exception is System.Text.Json.JsonException || exception is InvalidOperationException || exception is FormatException) + { + return null; + } + } + } +} diff --git a/WPF/SAM.Analytical.UI.WPF/Query/EngineComparison.cs b/WPF/SAM.Analytical.UI.WPF/Query/EngineComparison.cs new file mode 100644 index 00000000..3e9266cc --- /dev/null +++ b/WPF/SAM.Analytical.UI.WPF/Query/EngineComparison.cs @@ -0,0 +1,272 @@ +// SPDX-License-Identifier: LGPL-3.0-or-later +// Copyright (c) 2020-2026 Michal Dengusiak & Jakub Ziolkowski and contributors + +using SAM.Analytical.Benchmark; +using SAM.Analytical.Benchmark.Compare; +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; + +namespace SAM.Analytical.UI.WPF +{ + public static partial class Query + { + /// The B1a document file names the producers write and this workflow looks for in a run folder. + public const string TasBenchmarkFileName = "benchmark-TAS.json"; + + public const string OpenStudioBenchmarkFileName = "benchmark-OpenStudio.json"; + + /// + /// The folder of one Tas/OpenStudio comparison run: <directory>\<model>_TasOpenStudio_<yyyyMMdd-HHmmss>_<id>, so a run + /// never writes into another run's folder (as ). + /// + public static string EngineComparisonRunDirectory(string directory, string? modelName, DateTime startedUtc, string id) + { + return Path.Combine(directory, string.Format("{0}_TasOpenStudio_{1:yyyyMMdd-HHmmss}_{2}", FileNameSafe(modelName), startedUtc, id)); + } + + /// + /// Reads two B1a documents and compares them with SAM_Validation (Query.Compare, default tolerance profile) - the same + /// comparison benchmark-compare runs. Nothing is computed here. Throws what SAM_Validation throws for an unreadable, + /// invalid or incompatible document, so the caller can show it. + /// + public static EngineComparison LoadEngineComparison(string tasPath, string openStudioPath) + { + BenchmarkDocument tas = BenchmarkSerializer.Read(tasPath); + BenchmarkDocument openStudio = BenchmarkSerializer.Read(openStudioPath); + + ComparisonResult comparisonResult = Benchmark.Compare.Query.Compare(tas, openStudio, ToleranceProfile.Default); + + return new EngineComparison(tasPath, tas, openStudioPath, openStudio, comparisonResult); + } + + /// + /// The two B1a documents in (a Tas/OpenStudio comparison run folder, or any folder holding both + /// producer outputs); null for a document that is not there. + /// + public static (string? Tas, string? OpenStudio) EngineComparisonDocuments(string? directory) + { + if (string.IsNullOrWhiteSpace(directory) || !Directory.Exists(directory)) + { + return (null, null); + } + + string tas = Path.Combine(directory!, TasBenchmarkFileName); + string openStudio = Path.Combine(directory!, OpenStudioBenchmarkFileName); + + return (File.Exists(tas) ? tas : null, File.Exists(openStudio) ? openStudio : null); + } + + /// + /// How a row is shown. A label over SAM_Validation's own flags - availability, unit mismatch and its informational rule - never a + /// second comparison: no value, difference or band is looked at. + /// + public static EngineComparisonRowStatus ComparisonRowStatus(MetricComparison metricComparison) + { + switch (metricComparison.Availability) + { + case ComparisonAvailability.Neither: + return EngineComparisonRowStatus.Unavailable; + case ComparisonAvailability.TasOnly: + return EngineComparisonRowStatus.MissingOnOpenStudio; + case ComparisonAvailability.OpenStudioOnly: + return EngineComparisonRowStatus.MissingOnTas; + } + + if (metricComparison.NotApplicableReason == NotApplicableReason.UnitMismatch || Benchmark.Compare.Query.IsInformationalForNumericalGate(metricComparison)) + { + return EngineComparisonRowStatus.NotEquivalent; + } + + return EngineComparisonRowStatus.Compared; + } + + public static string ComparisonRowStatusText(EngineComparisonRowStatus status) + { + switch (status) + { + case EngineComparisonRowStatus.Compared: + return "Compared"; + case EngineComparisonRowStatus.NotEquivalent: + return "Not equivalent"; + case EngineComparisonRowStatus.MissingOnTas: + return "Missing on Tas"; + case EngineComparisonRowStatus.MissingOnOpenStudio: + return "Missing on OpenStudio"; + default: + return "Not reported by either engine"; + } + } + + /// The model-level rows, in SAM_Validation's order. + public static IReadOnlyList ComparisonModelRows(ComparisonResult comparisonResult) + { + return comparisonResult.ModelMetrics.Select(x => new EngineComparisonRow(x, null)).ToList(); + } + + /// Every space's rows, in SAM_Validation's order (spaces aligned by GUID, then name). + public static IReadOnlyList ComparisonSpaceRows(ComparisonResult comparisonResult) + { + List result = new List(); + foreach (SpaceComparison spaceComparison in comparisonResult.Spaces) + { + string? name = spaceComparison.TasName ?? spaceComparison.OpenStudioName; + if (spaceComparison.TasName != null && spaceComparison.OpenStudioName != null && !string.Equals(spaceComparison.TasName, spaceComparison.OpenStudioName, StringComparison.Ordinal)) + { + name = spaceComparison.TasName + " / " + spaceComparison.OpenStudioName; + } + + result.AddRange(spaceComparison.Metrics.Select(x => new EngineComparisonRow(x, name))); + } + + return result; + } + + /// + /// The input provenance of a comparison: SAM_Validation's provenance gate and mismatches, then what each B1a document says about + /// its inputs. Per-space input manifests are named as not available: they do not exist yet (SAM_OpenStudio#30, SAM_UI#230). + /// + public static string EngineComparisonProvenance(EngineComparison engineComparison) + { + ComparisonResult comparisonResult = engineComparison.ComparisonResult; + + List lines = new List + { + "Provenance gate: " + Format.Gate(comparisonResult.ProvenanceStatus) + (comparisonResult.ProvenanceCompatibility.IsCompatible ? " (same model, weather and design days)" : string.Empty), + }; + + foreach (ProvenanceMismatch provenanceMismatch in comparisonResult.ProvenanceCompatibility.Mismatches) + { + lines.Add(string.Format(" MISMATCH {0}: Tas = {1}; OpenStudio = {2}", provenanceMismatch.Field, provenanceMismatch.Tas ?? "(none)", provenanceMismatch.OpenStudio ?? "(none)")); + } + + lines.Add(string.Empty); + lines.AddRange(ProvenanceLines("Tas", engineComparison.TasPath, engineComparison.Tas.Provenance)); + lines.Add(string.Empty); + lines.AddRange(ProvenanceLines("OpenStudio", engineComparison.OpenStudioPath, engineComparison.OpenStudio.Provenance)); + lines.Add(string.Empty); + lines.Add("Input manifests (per-space inputs each engine received): not available - not implemented yet (SAM_OpenStudio#30, SAM_UI#230)."); + lines.Add("Input equivalence is therefore shown only as the provenance above (same canonical model, weather file and design-day source)."); + + return string.Join(Environment.NewLine, lines); + } + + /// + /// The diagnostics of a comparison: the gate statuses, each run's state, warnings and notes from its B1a document, space alignment + /// problems, model-total reconciliation, and - for a run made by this workflow - each producer's exit code and log. + /// + public static string EngineComparisonDiagnostics(EngineComparison engineComparison, EngineComparisonRun? engineComparisonRun = null) + { + ComparisonResult comparisonResult = engineComparison.ComparisonResult; + + List lines = new List + { + string.Format(CultureInfo.InvariantCulture, "Overall gate: {0} (numerical {1}, coverage {2}, provenance {3}, reconciliation {4})", + Format.Gate(comparisonResult.Gate), Format.Gate(comparisonResult.NumericalStatus), Format.Gate(comparisonResult.CoverageStatus), Format.Gate(comparisonResult.ProvenanceStatus), Format.Gate(comparisonResult.ReconciliationStatus)), + string.Format(CultureInfo.InvariantCulture, "Metrics: {0} match, {1} warn, {2} fail, {3} N/A; {4} of {5} required metrics comparable. Tolerance profile '{6}'. A band is a tolerance band, not an engineering verdict (SAM_UI#232/#233).", + comparisonResult.MatchCount, comparisonResult.WarnCount, comparisonResult.FailCount, comparisonResult.NotApplicableCount, comparisonResult.ComparableMetricCount, comparisonResult.RequiredMetricCount, comparisonResult.ToleranceProfileName), + }; + + if (!string.IsNullOrWhiteSpace(comparisonResult.SchemaDriftNote)) + { + lines.Add("Schema: " + comparisonResult.SchemaDriftNote); + } + + SpaceMatchDiagnostics spaceMatchDiagnostics = comparisonResult.SpaceDiagnostics; + lines.Add(string.Empty); + lines.Add(string.Format(CultureInfo.InvariantCulture, "Spaces: Tas {0}, OpenStudio {1}; matched by GUID {2}, by name {3}.", spaceMatchDiagnostics.TasCount, spaceMatchDiagnostics.OpenStudioCount, spaceMatchDiagnostics.MatchedByGuid, spaceMatchDiagnostics.MatchedByName)); + AddList(lines, "Unmatched - only in Tas", spaceMatchDiagnostics.OnlyInTas); + AddList(lines, "Unmatched - only in OpenStudio", spaceMatchDiagnostics.OnlyInOpenStudio); + AddList(lines, "Duplicate GUIDs in Tas", spaceMatchDiagnostics.DuplicateTasGuids); + AddList(lines, "Duplicate GUIDs in OpenStudio", spaceMatchDiagnostics.DuplicateOpenStudioGuids); + AddList(lines, "Duplicate names in Tas", spaceMatchDiagnostics.DuplicateTasNames); + AddList(lines, "Duplicate names in OpenStudio", spaceMatchDiagnostics.DuplicateOpenStudioNames); + AddList(lines, "Ambiguous name matches", spaceMatchDiagnostics.AmbiguousNameMatches); + + if (comparisonResult.Reconciliations.Count != 0) + { + lines.Add(string.Empty); + lines.Add("Model total vs sum of spaces:"); + foreach (ReconciliationResult reconciliationResult in comparisonResult.Reconciliations) + { + lines.Add(string.Format(CultureInfo.InvariantCulture, " {0} {1}: model {2}, sum of {3} spaces {4} {5} -> {6}{7}", + reconciliationResult.Engine, reconciliationResult.Key, reconciliationResult.ModelTotalAvailable ? Format.Number(reconciliationResult.ModelTotal) : "(not reported)", reconciliationResult.ContributingSpaces, + Format.Number(reconciliationResult.SumOfSpaces), Format.Unit(reconciliationResult.Unit), Format.Band(reconciliationResult.Band), reconciliationResult.Complete ? string.Empty : " (incomplete)")); + } + } + + lines.Add(string.Empty); + lines.AddRange(RunDiagnosticLines("Tas", engineComparison.Tas.Provenance)); + lines.Add(string.Empty); + lines.AddRange(RunDiagnosticLines("OpenStudio", engineComparison.OpenStudio.Provenance)); + + if (engineComparisonRun != null) + { + foreach (EngineComparisonProducerRun producerRun in engineComparisonRun.ProducerRuns) + { + lines.Add(string.Empty); + lines.Add(string.Format(CultureInfo.InvariantCulture, "{0} producer: {1}, exit code {2}, {3:0.#} s, log {4}", producerRun.Engine, producerRun.ExecutablePath ?? "(not found)", producerRun.ExitCode?.ToString(CultureInfo.InvariantCulture) ?? "-", producerRun.DurationSeconds, producerRun.LogPath ?? "-")); + if (!string.IsNullOrWhiteSpace(producerRun.Error)) + { + lines.Add(" " + producerRun.Error); + } + } + } + + return string.Join(Environment.NewLine, lines); + } + + private static IEnumerable ProvenanceLines(string engine, string path, BenchmarkProvenance? provenance) + { + yield return engine + ": " + path; + if (provenance == null) + { + yield return " (the document has no provenance)"; + yield break; + } + + yield return string.Format(CultureInfo.InvariantCulture, " Route {0}; engine {1} {2} (SDK {3}); state {4}", provenance.Route, provenance.Engine?.Name ?? "?", provenance.Engine?.Version ?? "?", provenance.Engine?.SdkVersion ?? "-", provenance.State); + yield return " Model: " + (provenance.SourceModelName ?? "?") + " (" + (provenance.SourceModelGuid ?? "?") + ")"; + yield return " Canonical model hash: " + (provenance.CanonicalModelHash ?? "(none)") + " (canonicalization " + (provenance.CanonicalizationVersion ?? "?") + ")"; + yield return " Source file hash: " + (provenance.SourceFileHash ?? "(none)"); + yield return " Weather: " + (provenance.Weather?.Identity ?? "(none)") + ", " + (provenance.Weather?.Hash ?? "(no hash)"); + yield return " Design-day source: " + provenance.DesignDaySource; + yield return string.Format(CultureInfo.InvariantCulture, " Run: {0:u}, {1} s; SAM {2}; runner {3}", provenance.RunTimestampUtc, provenance.DurationSeconds?.ToString("0.#", CultureInfo.InvariantCulture) ?? "?", provenance.SamCommit ?? "?", provenance.RunnerCommit ?? "?"); + } + + private static IEnumerable RunDiagnosticLines(string engine, BenchmarkProvenance? provenance) + { + yield return engine + " run: " + (provenance == null ? "(no provenance)" : provenance.State.ToString()); + if (provenance?.Warnings != null) + { + foreach (string warning in provenance.Warnings) + { + yield return " Warning: " + warning; + } + } + + if (provenance?.Notes != null) + { + foreach (string note in provenance.Notes) + { + yield return " Note: " + note; + } + } + + if (provenance?.ResultSources != null && provenance.ResultSources.Count != 0) + { + yield return " Result sources: " + string.Join(", ", provenance.ResultSources); + } + } + + private static void AddList(List lines, string title, IReadOnlyList values) + { + if (values != null && values.Count != 0) + { + lines.Add(" " + title + ": " + string.Join(", ", values)); + } + } + } +} diff --git a/WPF/SAM.Analytical.UI.WPF/SAM.Analytical.UI.WPF.csproj b/WPF/SAM.Analytical.UI.WPF/SAM.Analytical.UI.WPF.csproj index b91acc6d..08a3176a 100644 --- a/WPF/SAM.Analytical.UI.WPF/SAM.Analytical.UI.WPF.csproj +++ b/WPF/SAM.Analytical.UI.WPF/SAM.Analytical.UI.WPF.csproj @@ -184,6 +184,14 @@ ..\..\..\SAM_OpenStudio\build\SAM.Geometry.OpenStudio.dll + + + ..\..\..\SAM_Validation\build\SAM.Analytical.Benchmark.dll + + + ..\..\..\SAM_Validation\build\SAM.Analytical.Benchmark.Compare.dll + + + diff --git a/WPF/SAM.Analytical.UI.WPF/Windows/AnalyticalWindow.xaml.cs b/WPF/SAM.Analytical.UI.WPF/Windows/AnalyticalWindow.xaml.cs index 61ca8094..e4e23d32 100644 --- a/WPF/SAM.Analytical.UI.WPF/Windows/AnalyticalWindow.xaml.cs +++ b/WPF/SAM.Analytical.UI.WPF/Windows/AnalyticalWindow.xaml.cs @@ -712,6 +712,11 @@ private void InitializeWindow() RibbonButton_OpenStudioSimulation.LargeImageSource = SAM.Core.UI.WPF.Convert.ToBitmapSource(Properties.Resources.SAM_UI_ModelRun); RibbonButton_OpenStudioSimulation.Click += RibbonButton_OpenStudioSimulation_Click; RibbonButton_OpenStudioSimulation.ToolTipTitle = "OpenStudio Simulation"; + RibbonButton_CompareTasOpenStudio.LargeImageSource = SAM.Core.UI.WPF.Convert.ToBitmapSource(Properties.Resources.SAM_UI_ModelValidate); + RibbonButton_CompareTasOpenStudio.Click += RibbonButton_CompareTasOpenStudio_Click; + RibbonButton_CompareTasOpenStudio.ToolTipTitle = "Compare Tas / OpenStudio"; + RibbonButton_CompareTasOpenStudio.ToolTipDescription = "Run Tas and OpenStudio/EnergyPlus on one frozen copy of the open model and compare them, or compare existing B1a documents. Model and per-space results, differences and bands come from SAM_Validation. The open model and the Tas workflow are not changed."; + RibbonButton_OpenStudioSimulation.ToolTipDescription = "Run an annual OpenStudio/EnergyPlus simulation of a frozen copy of the open model (SAM_OpenStudio). The results are saved as a separate model in a new run folder; the open model and the Tas workflow are not changed. Needs OpenStudio 3.10 (openstudio.exe) installed."; RibbonButton_Optimisation.LargeImageSource = SAM.Core.UI.WPF.Convert.ToBitmapSource(Properties.Resources.SAM_UI_AlgorithmRun); @@ -2307,6 +2312,30 @@ private async void RibbonButton_OpenStudioSimulation_Click(object sender, Routed } } + private void RibbonButton_CompareTasOpenStudio_Click(object sender, RoutedEventArgs e) + { + //A missing SAM_Validation or SAM_OpenStudio assembly fails when ShowEngineComparison is compiled; say what is missing. + try + { + ShowEngineComparison(); + } + catch (Exception exception) when (Query.IsOpenStudioLoadFailure(exception)) + { + MessageBox.Show(this, "Compare Tas / OpenStudio could not load its components (SAM.Analytical.Benchmark.Compare.dll and SAM.Analytical.Benchmark.dll from SAM_Validation\\build, and the SAM_OpenStudio assemblies).\n\n" + Query.OpenStudioLoadFailure(exception), "Compare Tas / OpenStudio", MessageBoxButton.OK, MessageBoxImage.Error); + } + } + + [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] + private void ShowEngineComparison() + { + EngineComparisonWindow engineComparisonWindow = new EngineComparisonWindow(uIAnalyticalModel) + { + Owner = this, + }; + + engineComparisonWindow.Show(); + } + [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] private System.Threading.Tasks.Task SimulateOpenStudio() { diff --git a/WPF/SAM.Analytical.UI.WPF/Windows/EngineComparisonWindow.xaml b/WPF/SAM.Analytical.UI.WPF/Windows/EngineComparisonWindow.xaml new file mode 100644 index 00000000..2f5a6be1 --- /dev/null +++ b/WPF/SAM.Analytical.UI.WPF/Windows/EngineComparisonWindow.xaml @@ -0,0 +1,153 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +