diff --git a/WPF/SAM.Analytical.UI.WPF.Tests/EngineComparisonScreenshotHarness.cs b/WPF/SAM.Analytical.UI.WPF.Tests/EngineComparisonScreenshotHarness.cs new file mode 100644 index 00000000..40ddee58 --- /dev/null +++ b/WPF/SAM.Analytical.UI.WPF.Tests/EngineComparisonScreenshotHarness.cs @@ -0,0 +1,122 @@ +// SPDX-License-Identifier: LGPL-3.0-or-later +// Copyright (c) 2020-2026 Michal Dengusiak & Jakub Ziolkowski and contributors + +using System; +using System.IO; +using System.Linq; +using System.Windows; +using System.Windows.Controls; +using System.Windows.Media; +using Xunit; + +namespace SAM.Analytical.UI.WPF.Tests +{ + /// + /// Opt-in. Renders the Tas/OpenStudio comparison window at the two reviewed widths (1180 px and 820 px), on both the Model and + /// the Spaces tab, including a narrow view scrolled to the right. Nothing is run and nothing is written beside the screenshots. + /// + /// SAM_ENGINE_COMPARISON_RUN is a folder holding benchmark-TAS.json and benchmark-OpenStudio.json (a saved + /// comparison run folder works); SAM_ENGINE_COMPARISON_SCREENSHOTS is where the images go. Without them this passes having done + /// nothing. + /// + /// + [Collection(WpfCollection.Name)] + public class EngineComparisonScreenshotHarness + { + [WpfFact] + public void Render_both_grids_at_1180_and_820() + { + string directory_Run = Environment.GetEnvironmentVariable("SAM_ENGINE_COMPARISON_RUN"); + string directory_Screenshots = Environment.GetEnvironmentVariable("SAM_ENGINE_COMPARISON_SCREENSHOTS"); + + if (string.IsNullOrWhiteSpace(directory_Run) || string.IsNullOrWhiteSpace(directory_Screenshots) || !Directory.Exists(directory_Run)) + { + return; + } + + string tas = Path.Combine(directory_Run, Query.TasBenchmarkFileName); + string openStudio = Path.Combine(directory_Run, Query.OpenStudioBenchmarkFileName); + if (!File.Exists(tas) || !File.Exists(openStudio)) + { + return; + } + + Directory.CreateDirectory(directory_Screenshots); + + EngineComparisonWindow window = new EngineComparisonWindow(null) + { + WindowStartupLocation = System.Windows.WindowStartupLocation.Manual, + Left = 0, + Top = 0, + ShowActivated = false, + }; + + try + { + Assert.True(window.Compare(tas, openStudio, null)); + window.Show(); + Flush(window); + + TabControl tabControl = (TabControl)window.FindName("TabControl_Main"); + DataGrid model = (DataGrid)window.FindName("DataGrid_Model"); + DataGrid spaces = (DataGrid)window.FindName("DataGrid_Spaces"); + + tabControl.SelectedIndex = 0; + Flush(window); + PartOWorkflowEvidenceHarness.Render(window, Path.Combine(directory_Screenshots, "compare-model-1180.png"), 1180, 780); + + //At 820 px the columns keep their readable widths and the grid scrolls horizontally. + tabControl.SelectedIndex = 0; + Flush(window); + PartOWorkflowEvidenceHarness.Render(window, Path.Combine(directory_Screenshots, "compare-model-820.png"), 820, 780); + ScrollRight(model); + PartOWorkflowEvidenceHarness.Render(window, Path.Combine(directory_Screenshots, "compare-model-820-scrolled.png"), 820, 780); + + tabControl.SelectedIndex = 1; + Flush(window); + PartOWorkflowEvidenceHarness.Render(window, Path.Combine(directory_Screenshots, "compare-spaces-1180.png"), 1180, 780); + + tabControl.SelectedIndex = 1; + Flush(window); + PartOWorkflowEvidenceHarness.Render(window, Path.Combine(directory_Screenshots, "compare-spaces-820.png"), 820, 780); + ScrollRight(spaces); + PartOWorkflowEvidenceHarness.Render(window, Path.Combine(directory_Screenshots, "compare-spaces-820-scrolled.png"), 820, 780); + } + finally + { + window.Close(); + } + } + + private static void ScrollRight(DataGrid dataGrid) + { + ScrollViewer scrollViewer = VisualChildren(dataGrid).First(); + scrollViewer.ScrollToHorizontalOffset(scrollViewer.ScrollableWidth); + scrollViewer.UpdateLayout(); + } + + private static void Flush(System.Windows.Window window) + { + window.Dispatcher.Invoke(() => { }, System.Windows.Threading.DispatcherPriority.ApplicationIdle); + window.UpdateLayout(); + window.Dispatcher.Invoke(() => { }, System.Windows.Threading.DispatcherPriority.ApplicationIdle); + } + + private static System.Collections.Generic.IEnumerable VisualChildren(DependencyObject parent) where T : DependencyObject + { + for (int index = 0; index < VisualTreeHelper.GetChildrenCount(parent); index++) + { + DependencyObject child = VisualTreeHelper.GetChild(parent, index); + if (child is T match) + { + yield return match; + } + + foreach (T descendant in VisualChildren(child)) + { + yield return descendant; + } + } + } + } +} diff --git a/WPF/SAM.Analytical.UI.WPF.Tests/EngineComparisonTests.cs b/WPF/SAM.Analytical.UI.WPF.Tests/EngineComparisonTests.cs new file mode 100644 index 00000000..140549ce --- /dev/null +++ b/WPF/SAM.Analytical.UI.WPF.Tests/EngineComparisonTests.cs @@ -0,0 +1,625 @@ +// SPDX-License-Identifier: LGPL-3.0-or-later +// Copyright (c) 2020-2026 Michal Dengusiak & Jakub Ziolkowski and contributors + +using SAM.Analytical.Benchmark; +using SAM.Analytical.Benchmark.Compare; +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Text.Json.Nodes; +using System.Threading; +using Xunit; + +namespace SAM.Analytical.UI.WPF.Tests +{ + /// + /// Simulate > Compare Tas / OpenStudio (SAM_UI#231). The documents are SAM_Validation's FABRICATED regression documents + /// (Resources/EngineComparison/README.md), not engine results. These tests prove the view computes nothing - every cell equals + /// SAM_Validation's own comparison.csv - and that the orchestration feeds both producers one snapshot and one weather file, then + /// compares and reports through SAM_Validation. The producers are faked here; the real engines are the manual Windows test. + /// + public class EngineComparisonTests : IDisposable + { + private readonly string directory; + + public EngineComparisonTests() + { + directory = Path.Combine(Path.GetTempPath(), "SAM-UI-231-tests", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(directory); + } + + public void Dispose() + { + try + { + //The run's snapshot and weather file are read-only, which Directory.Delete refuses on Windows. + foreach (string path in Directory.EnumerateFiles(directory, "*", SearchOption.AllDirectories)) + { + File.SetAttributes(path, FileAttributes.Normal); + } + + Directory.Delete(directory, true); + } + catch (IOException) + { + } + catch (UnauthorizedAccessException) + { + } + } + + private static string Resource(string name, [CallerFilePath] string sourceFilePath = null) + { + return Path.Combine(Path.GetDirectoryName(sourceFilePath), "Resources", "EngineComparison", name); + } + + private static string Tas => Resource("scenario-benchmark-TAS.json"); + + private static string OpenStudio => Resource("scenario-benchmark-OpenStudio.json"); + + private static string OpenStudio_WeatherMismatch => Resource("weather-mismatch-benchmark-OpenStudio.json"); + + [Theory] + [InlineData("scenario-benchmark-OpenStudio.json")] + [InlineData("weather-mismatch-benchmark-OpenStudio.json")] + public void EveryCell_EqualsSamValidationsComparisonCsv(string openStudio) + { + EngineComparison engineComparison = Query.LoadEngineComparison(Tas, Resource(openStudio)); + + List csv = Benchmark.Compare.Modify.WriteCsv(engineComparison.ComparisonResult).Split('\n', StringSplitOptions.RemoveEmptyEntries).Skip(1).Select(ParseCsvLine).ToList(); + List rows = engineComparison.ModelRows.Concat(engineComparison.SpaceRows).ToList(); + + Assert.Equal(csv.Count, rows.Count); + for (int i = 0; i < rows.Count; i++) + { + string[] expected = csv[i]; + EngineComparisonRow row = rows[i]; + Assert.Equal(expected[0], row.Scope); + Assert.Equal(expected[1], row.Metric); + Assert.Equal(expected[2], row.Unit); + Assert.Equal(expected[10], row.Band); + + //The CSV stays the raw, full-precision contract; the grid is the two-decimal presentation. + Assert.Equal(expected[4], row.TasFull); + Assert.Equal(expected[5], row.OpenStudioFull); + Assert.Equal(expected[6], row.AbsoluteDifferenceFull); + Assert.Equal(expected[8], row.PercentDifferenceFull); + Assert.Equal(PresentationFormat.Note(row.MetricComparison), row.Note); + + Assert.Equal(PresentationFormat.Number(row.MetricComparison.TasValue, row.MetricComparison.Unit, System.Globalization.CultureInfo.CurrentCulture, string.Empty), row.Tas); + Assert.Equal(PresentationFormat.Number(row.MetricComparison.OpenStudioValue, row.MetricComparison.Unit, System.Globalization.CultureInfo.CurrentCulture, string.Empty), row.OpenStudio); + Assert.Equal( + PresentationFormat.Number(row.MetricComparison.AbsoluteDifference, row.MetricComparison.Circular ? MetricUnit.HourOfYear : row.MetricComparison.Unit, System.Globalization.CultureInfo.CurrentCulture, string.Empty), + row.AbsoluteDifference); + Assert.Equal( + row.MetricComparison.Circular ? string.Empty : PresentationFormat.Percent(row.MetricComparison.RelativeDifference, System.Globalization.CultureInfo.CurrentCulture, string.Empty), + row.PercentDifference); + } + } + + [Fact] + public void Status_FollowsSamValidationsFlags_AndMissingValuesAreNeverZero() + { + EngineComparison engineComparison = Query.LoadEngineComparison(Tas, OpenStudio); + List rows = engineComparison.ModelRows.Concat(engineComparison.SpaceRows).ToList(); + + foreach (EngineComparisonRow row in rows) + { + MetricComparison metric = row.MetricComparison; + switch (row.Status) + { + case EngineComparisonRowStatus.Compared: + Assert.Equal(ComparisonAvailability.Both, metric.Availability); + Assert.False(Benchmark.Compare.Query.IsInformationalForNumericalGate(metric)); + Assert.NotEqual("N/A", row.Band); + break; + case EngineComparisonRowStatus.NotEquivalent: + Assert.Equal(ComparisonAvailability.Both, metric.Availability); + Assert.True(Benchmark.Compare.Query.IsInformationalForNumericalGate(metric) || metric.NotApplicableReason == NotApplicableReason.UnitMismatch); + break; + case EngineComparisonRowStatus.MissingOnTas: + Assert.Equal(ComparisonAvailability.OpenStudioOnly, metric.Availability); + Assert.Equal(string.Empty, row.Tas); + Assert.Equal(string.Empty, row.AbsoluteDifference); + break; + case EngineComparisonRowStatus.MissingOnOpenStudio: + Assert.Equal(ComparisonAvailability.TasOnly, metric.Availability); + Assert.Equal(string.Empty, row.OpenStudio); + Assert.Equal(string.Empty, row.AbsoluteDifference); + break; + case EngineComparisonRowStatus.Unavailable: + Assert.Equal(ComparisonAvailability.Neither, metric.Availability); + Assert.Equal(string.Empty, row.Tas); + Assert.Equal(string.Empty, row.OpenStudio); + break; + } + } + + //The scenario spans the cases the view must tell apart. + Assert.Contains(rows, x => x.Status == EngineComparisonRowStatus.Compared); + Assert.Contains(rows, x => x.Status == EngineComparisonRowStatus.NotEquivalent); + Assert.Contains(rows, x => x.Status == EngineComparisonRowStatus.MissingOnTas || x.Status == EngineComparisonRowStatus.MissingOnOpenStudio); + Assert.Contains(rows, x => x.Status == EngineComparisonRowStatus.Unavailable); + Assert.Contains(rows, x => x.Band == "Fail"); + } + + [Fact] + public void Provenance_NamesTheWeatherMismatch_AndTheMissingInputManifests() + { + EngineComparison engineComparison = Query.LoadEngineComparison(Tas, OpenStudio_WeatherMismatch); + + string provenance = Query.EngineComparisonProvenance(engineComparison); + + Assert.False(engineComparison.ComparisonResult.ProvenanceCompatibility.IsCompatible); + Assert.Contains("MISMATCH weatherHash", provenance); + Assert.Contains("Input manifests", provenance); + Assert.Contains("not available", provenance); + } + + [Fact] + public void Diagnostics_ListUnmatchedSpaces_GateAndReconciliation() + { + EngineComparison engineComparison = Query.LoadEngineComparison(Tas, OpenStudio); + SpaceMatchDiagnostics spaceMatchDiagnostics = engineComparison.ComparisonResult.SpaceDiagnostics; + + string diagnostics = Query.EngineComparisonDiagnostics(engineComparison); + + Assert.Contains("Overall gate: " + Format.Gate(engineComparison.ComparisonResult.Gate), diagnostics); + Assert.NotEmpty(spaceMatchDiagnostics.OnlyInTas.Concat(spaceMatchDiagnostics.OnlyInOpenStudio)); + foreach (string space in spaceMatchDiagnostics.OnlyInTas.Concat(spaceMatchDiagnostics.OnlyInOpenStudio)) + { + Assert.Contains(space, diagnostics); + } + } + + [Fact] + public void Run_FeedsBothProducersOneSnapshotAndOneWeatherFile_ThenComparesAndReports_WithoutChangingTheModel() + { + AnalyticalModel analyticalModel = Model(); + string json_Before = Core.Convert.ToString(analyticalModel); + string epwPath = Epw(); + + List calls = new List(); + EngineComparisonRun run = Modify.RunEngineComparison(analyticalModel, directory, epwPath, ProducerPath("benchmark-tas.exe"), ProducerPath("benchmark-openstudio.exe"), @"C:\models\source.json", null, CancellationToken.None, (producerRun, workDirectory, cancellationToken) => + { + calls.Add(producerRun); + File.Copy(producerRun.Engine == ComparisonEngine.Tas ? Tas : OpenStudio, Argument(producerRun, "--out")); + return 0; + }); + + Assert.Equal(EngineComparisonRunStatus.Compared, run.Status); + Assert.Equal(new[] { ComparisonEngine.Tas, ComparisonEngine.OpenStudio }, calls.Select(x => x.Engine)); + + //One snapshot and one weather file, both in the run folder, for both engines. + Assert.All(calls, x => Assert.Equal(run.SnapshotPath, Argument(x, "--model"))); + Assert.All(calls, x => Assert.Equal(run.WeatherPath, Argument(x, "--weather"))); + Assert.Equal(Query.Sha256(epwPath), run.WeatherSha256); + Assert.Equal("File", run.WeatherSource); + Assert.NotNull(Argument(calls[0], "--tbd")); + Assert.NotNull(Argument(calls[1], "--work")); + + //The reports are SAM_Validation's, for these documents. + Assert.Equal(3, run.ReportPaths.Count); + EngineComparison engineComparison = Query.LoadEngineComparison(Tas, OpenStudio); + Assert.Equal(Benchmark.Compare.Modify.WriteCsv(engineComparison.ComparisonResult), File.ReadAllText(Path.Combine(run.Directory, Benchmark.Compare.Modify.CsvFileName))); + Assert.Equal(Format.Gate(engineComparison.ComparisonResult.Gate), run.Gate); + + Assert.True(File.Exists(run.ManifestPath)); + Assert.Equal("Compared", (string)JsonNode.Parse(File.ReadAllText(run.ManifestPath))["status"]); + Assert.Equal(json_Before, Core.Convert.ToString(analyticalModel)); + } + + [Fact] + public void Run_WithTasFailed_IsPartiallyCompleted_KeepsOpenStudio_AndComparesNothing() + { + EngineComparisonRun run = Modify.RunEngineComparison(Model(), directory, Epw(), ProducerPath("benchmark-tas.exe"), ProducerPath("benchmark-openstudio.exe"), null, null, CancellationToken.None, (producerRun, workDirectory, cancellationToken) => + { + if (producerRun.Engine == ComparisonEngine.OpenStudio) + { + File.Copy(OpenStudio, Argument(producerRun, "--out")); + return 0; + } + + return 5; + }); + + Assert.Equal(EngineComparisonRunStatus.PartiallyCompleted, run.Status); + Assert.Empty(run.ReportPaths); + Assert.Null(run.Gate); + Assert.Contains("exited with code 5", run.ProducerRuns[0].Error); + Assert.False(run.ProducerRuns[0].Succeeded); + Assert.True(run.ProducerRuns[1].Succeeded); + Assert.True(File.Exists(run.ProducerRuns[1].BenchmarkPath)); + Assert.Contains(run.Diagnostics, x => x.Contains("not compared - Tas did not complete")); + + JsonNode manifest = JsonNode.Parse(File.ReadAllText(run.ManifestPath)); + Assert.Equal("PartiallyCompleted", (string)manifest["status"]); + Assert.False((bool)manifest["producers"][0]["succeeded"]); + Assert.True((bool)manifest["producers"][1]["succeeded"]); + } + + [Fact] + public void Run_WithAProducerNotFound_IsPartiallyCompleted_AndTheOtherEngineStillRuns() + { + List calls = new List(); + EngineComparisonRun run = Modify.RunEngineComparison(Model(), directory, Epw(), ProducerPath("benchmark-tas.exe"), Path.Combine(directory, "missing", "benchmark-openstudio.exe"), null, null, CancellationToken.None, (producerRun, workDirectory, cancellationToken) => + { + calls.Add(producerRun.Engine); + File.Copy(Tas, Argument(producerRun, "--out")); + return 0; + }); + + Assert.Equal(new[] { ComparisonEngine.Tas }, calls); + Assert.Equal(EngineComparisonRunStatus.PartiallyCompleted, run.Status); + Assert.Contains("not found", run.ProducerRuns[1].Error); + Assert.Contains(run.Diagnostics, x => x.Contains("not compared - OpenStudio did not complete")); + Assert.Empty(run.ReportPaths); + } + + [Fact] + public void Run_DocumentOfAFailedProducer_IsKeptButNeverUsed() + { + //Both write a document, but Tas exits with 5: its document is not a result, so nothing is compared. + EngineComparisonRun run = Modify.RunEngineComparison(Model(), directory, Epw(), ProducerPath("benchmark-tas.exe"), ProducerPath("benchmark-openstudio.exe"), null, null, CancellationToken.None, (producerRun, workDirectory, cancellationToken) => + { + File.Copy(producerRun.Engine == ComparisonEngine.Tas ? Tas : OpenStudio, Argument(producerRun, "--out")); + return producerRun.Engine == ComparisonEngine.Tas ? 5 : 0; + }); + + Assert.Equal(EngineComparisonRunStatus.PartiallyCompleted, run.Status); + Assert.NotNull(run.ProducerRuns[0].BenchmarkPath); + Assert.False(run.ProducerRuns[0].Succeeded); + Assert.Empty(run.ReportPaths); + Assert.Null(run.Gate); + } + + [Fact] + public void Run_WithBothFailed_IsFailed() + { + EngineComparisonRun run = Modify.RunEngineComparison(Model(), directory, Epw(), ProducerPath("benchmark-tas.exe"), ProducerPath("benchmark-openstudio.exe"), null, null, CancellationToken.None, (producerRun, workDirectory, cancellationToken) => 5); + + Assert.Equal(EngineComparisonRunStatus.Failed, run.Status); + Assert.Equal(2, run.ProducerRuns.Count); + Assert.Contains(run.Diagnostics, x => x.Contains("neither Tas nor OpenStudio completed")); + Assert.Empty(run.ReportPaths); + } + + [Fact] + public void Run_SnapshotAndWeatherAreReadOnly_AndAChangedSnapshotStopsTheRun() + { + List calls = new List(); + EngineComparisonRun run = Modify.RunEngineComparison(Model(), directory, Epw(), ProducerPath("benchmark-tas.exe"), ProducerPath("benchmark-openstudio.exe"), null, null, CancellationToken.None, (producerRun, workDirectory, cancellationToken) => + { + calls.Add(producerRun.Engine); + + string snapshot = Argument(producerRun, "--model"); + Assert.True(File.GetAttributes(snapshot).HasFlag(FileAttributes.ReadOnly)); + Assert.True(File.GetAttributes(Argument(producerRun, "--weather")).HasFlag(FileAttributes.ReadOnly)); + + //Something changes the frozen snapshot while Tas runs. + File.SetAttributes(snapshot, FileAttributes.Normal); + File.AppendAllText(snapshot, " "); + + File.Copy(Tas, Argument(producerRun, "--out")); + return 0; + }); + + //OpenStudio is not run on the changed snapshot, and the Tas document is not shown or compared. + Assert.Equal(new[] { ComparisonEngine.Tas }, calls); + Assert.Equal(EngineComparisonRunStatus.Failed, run.Status); + Assert.Contains("snapshot changed after it was frozen", run.ProducerRuns[1].Error); + Assert.Contains(run.Diagnostics, x => x.Contains("no comparison - the snapshot changed")); + Assert.Empty(run.ReportPaths); + } + + [Fact] + public void EngineResult_ShowsOneEngineOnly_UnderSamValidationsKeys_NotComparedAndNothingFilledIn() + { + EngineResult engineResult = Query.LoadEngineResult(ComparisonEngine.OpenStudio, OpenStudio, ComparisonEngine.Tas); + Benchmark.BenchmarkDocument document = Benchmark.BenchmarkSerializer.Read(OpenStudio); + EngineComparison engineComparison = Query.LoadEngineComparison(Tas, OpenStudio); + + //The same metric keys, in the same order, as SAM_Validation's comparison rows. + Assert.Equal(engineComparison.ModelRows.Select(x => x.Metric), engineResult.ModelRows.Select(x => x.Metric)); + Assert.Equal(document.Spaces.Count * Benchmark.Compare.Query.SpaceMetricValues(null).Count, engineResult.SpaceRows.Count); + + IReadOnlyList> values = Benchmark.Compare.Query.ModelMetricValues(document.Model); + for (int i = 0; i < values.Count; i++) + { + EngineComparisonRow row = engineResult.ModelRows[i]; + Assert.Equal(EngineComparisonRowStatus.NotCompared, row.Status); + Assert.Null(row.MetricComparison); + Assert.Equal(string.Empty, row.Tas); + Assert.Equal(string.Empty, row.AbsoluteDifference); + Assert.Equal(string.Empty, row.PercentDifference); + Assert.Equal(string.Empty, row.Band); + Assert.Contains("Tas did not complete", row.Note); + + Benchmark.MetricValue value = values[i].Value; + if (value != null && value.Available) + { + Assert.Equal(PresentationFormat.Number(value.Value, value.Unit, CultureInfo.CurrentCulture, string.Empty), row.OpenStudio); + } + else + { + //Never a zero or another engine's value. + Assert.Equal(string.Empty, row.OpenStudio); + Assert.Contains("Not reported by OpenStudio", row.Note); + } + } + + Assert.All(engineResult.SpaceRows, x => Assert.Equal(EngineComparisonRowStatus.NotCompared, x.Status)); + Assert.Contains("not evaluated", Query.EngineResultProvenance(engineResult)); + Assert.Contains("Not compared: only OpenStudio completed", Query.EngineResultDiagnostics(engineResult)); + } + + [Fact] + public void Run_Cancelled_StopsAndSaysSo() + { + using CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); + EngineComparisonRun run = Modify.RunEngineComparison(Model(), directory, Epw(), ProducerPath("benchmark-tas.exe"), ProducerPath("benchmark-openstudio.exe"), null, null, cancellationTokenSource.Token, (producerRun, workDirectory, cancellationToken) => + { + cancellationTokenSource.Cancel(); + cancellationToken.ThrowIfCancellationRequested(); + return 0; + }); + + Assert.Equal(EngineComparisonRunStatus.Cancelled, run.Status); + Assert.Single(run.ProducerRuns); + Assert.Empty(run.ReportPaths); + } + + [Fact] + public void Run_WithoutWeather_RunsNothing() + { + int calls = 0; + EngineComparisonRun run = Modify.RunEngineComparison(Model(), directory, null, ProducerPath("benchmark-tas.exe"), ProducerPath("benchmark-openstudio.exe"), null, null, CancellationToken.None, (producerRun, workDirectory, cancellationToken) => ++calls); + + //No EPW was given and the model has no WeatherData; on a machine without the OpenStudio stack the embedded-weather export + //itself fails to load. Either way nothing runs. + Assert.Equal(EngineComparisonRunStatus.Failed, run.Status); + Assert.Equal(0, calls); + Assert.True(run.Exception != null || run.Diagnostics.Any(x => x.Contains("no weather"))); + } + + [Fact] + public void Producer_IsFoundByVariableOrSavedLocation_AndArgumentsMatchTheProducers() + { + string settingsPath = Path.Combine(directory, "producers.json"); + string producer = ProducerPath("benchmark-openstudio.exe"); + + string variable = Query.BenchmarkProducerVariable(ComparisonEngine.OpenStudio); + string previous = Environment.GetEnvironmentVariable(variable); + try + { + Environment.SetEnvironmentVariable(variable, null); + Assert.Null(Query.BenchmarkProducerPath(ComparisonEngine.OpenStudio, settingsPath)); + + Assert.True(Modify.SaveBenchmarkProducerPath(ComparisonEngine.OpenStudio, producer, settingsPath)); + Assert.Equal(producer, Query.BenchmarkProducerPath(ComparisonEngine.OpenStudio, settingsPath)); + + //The variable may name the folder. + Environment.SetEnvironmentVariable(variable, Path.GetDirectoryName(producer)); + Assert.Equal(producer, Query.BenchmarkProducerPath(ComparisonEngine.OpenStudio, Path.Combine(directory, "none.json"))); + } + finally + { + Environment.SetEnvironmentVariable(variable, previous); + } + + //benchmark-tas --model --weather --tbd --out; benchmark-openstudio --model --weather --out --work. + List tas = Query.BenchmarkProducerArguments(ComparisonEngine.Tas, "m.json", "w.epw", "b.json", "work", "Block A"); + Assert.Equal(new[] { "--model", "m.json", "--weather", "w.epw", "--tbd", Path.Combine("work", "Block_A.tbd"), "--out", "b.json" }, tas); + List openStudio = Query.BenchmarkProducerArguments(ComparisonEngine.OpenStudio, "m.json", "w.epw", "b.json", "work", "Block A"); + Assert.Equal(new[] { "--model", "m.json", "--weather", "w.epw", "--out", "b.json", "--work", "work" }, openStudio); + } + + [Fact] + public void Folder_WithoutB1aDocuments_FindsNone() + { + (string tas, string openStudio) = Query.EngineComparisonDocuments(directory); + Assert.Null(tas); + Assert.Null(openStudio); + + File.Copy(Tas, Path.Combine(directory, Query.TasBenchmarkFileName)); + (tas, openStudio) = Query.EngineComparisonDocuments(directory); + Assert.NotNull(tas); + Assert.Null(openStudio); + } + + [Fact] + public void Presentation_TwoDecimals_HoursStayInteger_AndMissingStaysEmpty() + { + MetricComparison heating = Metric( + "model", "consumptionHeating", MetricUnit.KilowattHour, + 742.7750866578222, true, 2810.0182949034006, true, + circular: false, absoluteDifference: 2067.2432082455784, relativeDifference: 0.7356689499121726, + band: ComparisonBand.Fail, reason: NotApplicableReason.None); + + EngineComparisonRow heatingRow = new EngineComparisonRow(heating, null); + Assert.Equal("742.78", heatingRow.Tas); + Assert.Equal("742.7750866578222", heatingRow.TasFull); + Assert.Equal("2810.02", heatingRow.OpenStudio); + Assert.Equal("2067.24", heatingRow.AbsoluteDifference); + Assert.Equal("73.57", heatingRow.PercentDifference); + Assert.Equal("Compared", heatingRow.StatusText); + Assert.False(heatingRow.IsInformational); + + MetricComparison hour = Metric( + "model", "peakHeatingHour", MetricUnit.HourOfYear, + 871, true, 919, true, + circular: true, absoluteDifference: 48, relativeDifference: null, + band: ComparisonBand.Fail, reason: NotApplicableReason.None); + + EngineComparisonRow hourRow = new EngineComparisonRow(hour, null); + Assert.Equal("871", hourRow.Tas); + Assert.Equal("919", hourRow.OpenStudio); + Assert.Equal("48", hourRow.AbsoluteDifference); + Assert.Equal(string.Empty, hourRow.PercentDifference); + Assert.Equal(48.ToString(CultureInfo.InvariantCulture), hourRow.AbsoluteDifferenceFull); + Assert.True(hourRow.IsInformational); + Assert.Equal("Informational", hourRow.StatusText); + Assert.Contains(PresentationFormat.InformationalNote, hourRow.Note); + + MetricComparison missing = Metric( + "model", "heating.designLoad", MetricUnit.Watt, + 1927.0106201171875, true, null, false, + circular: false, absoluteDifference: null, relativeDifference: null, + band: ComparisonBand.NotApplicable, reason: NotApplicableReason.Unavailable); + + EngineComparisonRow missingRow = new EngineComparisonRow(missing, "Office"); + Assert.Equal("1927.01", missingRow.Tas); + Assert.Equal(string.Empty, missingRow.OpenStudio); + Assert.Equal(string.Empty, missingRow.AbsoluteDifference); + Assert.Equal(string.Empty, missingRow.PercentDifference); + Assert.Equal("Missing on OpenStudio", missingRow.StatusText); + Assert.Equal(string.Empty, missingRow.OpenStudioFull); + } + + [Fact] + public void ScopeDisplay_IsCompactWhileFullGuidStaysAvailable() + { + string guid = "cb68bed195af496bb255a13ffa5c58e9"; + EngineComparisonRow row = new EngineComparisonRow( + Metric("model", "area", MetricUnit.SquareMetre, 50, true, 50, true, false, 0, 0, ComparisonBand.Match, NotApplicableReason.None), + "Office"); + + //The comparison scope is the model for this metric; the space GUID is carried by the space row factory. + Assert.Equal("Model", row.ScopeDisplay); + Assert.Equal("model", row.ScopeFull); + + MetricComparison spaceMetric = Metric( + guid, "area", MetricUnit.SquareMetre, 50, true, 50, true, + circular: false, absoluteDifference: 0, relativeDifference: 0, + band: ComparisonBand.Match, reason: NotApplicableReason.None); + + EngineComparisonRow spaceRow = new EngineComparisonRow(spaceMetric, "Office"); + Assert.Equal(guid, spaceRow.ScopeFull); + Assert.Equal(guid.Substring(0, 8) + "…", spaceRow.ScopeDisplay); + } + + [Fact] + public void UnitMismatch_HasItsOwnStatusAndNote() + { + MetricComparison mismatch = new MetricComparison( + "model", "consumptionHeating", MetricUnit.Kilowatt, + 1, true, 2, true, false, null, null, null, + ComparisonBand.NotApplicable, NotApplicableReason.UnitMismatch, + MetricUnit.Kilowatt, MetricUnit.Watt); + + EngineComparisonRow row = new EngineComparisonRow(mismatch, null); + Assert.True(row.IsUnitMismatch); + Assert.False(row.IsInformational); + Assert.Equal("Unit mismatch", row.StatusText); + Assert.Contains("Unit mismatch (TAS kW vs OpenStudio W)", row.Note); + Assert.DoesNotContain(PresentationFormat.InformationalNote, row.Note); + } + + [Fact] + public void InformationalFailCountsAreSeparateFromContributingFailCounts() + { + EngineComparison comparison = Query.LoadEngineComparison(Tas, OpenStudio); + Assert.True(comparison.InformationalFailCount > 0); + Assert.True(comparison.ContributingFailCount > 0); + Assert.Equal(comparison.ComparisonResult.FailCount, comparison.ContributingFailCount + comparison.InformationalFailCount); + Assert.All(comparison.AllRows.Where(x => x.IsInformational), x => Assert.Equal("Informational", x.StatusText)); + } + + private static MetricComparison Metric( + string scope, + string key, + MetricUnit unit, + double? tas, + bool tasAvailable, + double? openStudio, + bool openStudioAvailable, + bool circular, + double? absoluteDifference, + double? relativeDifference, + ComparisonBand band, + NotApplicableReason reason) + { + return new MetricComparison( + scope, key, unit, + tas, tasAvailable, + openStudio, openStudioAvailable, + circular, absoluteDifference, relativeDifference, null, + band, reason, + unit, unit); + } + + private static AnalyticalModel Model() + { + AnalyticalModel analyticalModel = new AnalyticalModel(Guid.NewGuid(), "Test model"); + Assert.True(analyticalModel.AddSpace(new Space("Room 1", new Geometry.Spatial.Point3D(1, 1, 1)))); + return analyticalModel; + } + + private string Epw() + { + string path = Path.Combine(directory, "test.epw"); + File.WriteAllText(path, "LOCATION,Test,,,TMY,000000,0,0,0,0\n"); + return path; + } + + private string ProducerPath(string fileName) + { + string path = Path.Combine(directory, "producers", fileName); + Directory.CreateDirectory(Path.GetDirectoryName(path)); + File.WriteAllText(path, string.Empty); + return path; + } + + private static string Argument(EngineComparisonProducerRun producerRun, string name) + { + int index = producerRun.Arguments.IndexOf(name); + return index < 0 ? null : producerRun.Arguments[index + 1]; + } + + private static string[] ParseCsvLine(string line) + { + List fields = new List(); + System.Text.StringBuilder field = new System.Text.StringBuilder(); + bool quoted = false; + for (int i = 0; i < line.Length; i++) + { + char c = line[i]; + if (quoted) + { + if (c == '"' && i + 1 < line.Length && line[i + 1] == '"') + { + field.Append('"'); + i++; + } + else if (c == '"') + { + quoted = false; + } + else + { + field.Append(c); + } + } + else if (c == '"') + { + quoted = true; + } + else if (c == ',') + { + fields.Add(field.ToString()); + field.Clear(); + } + else + { + field.Append(c); + } + } + + fields.Add(field.ToString()); + return fields.ToArray(); + } + } +} diff --git a/WPF/SAM.Analytical.UI.WPF.Tests/EngineComparisonWindowTests.cs b/WPF/SAM.Analytical.UI.WPF.Tests/EngineComparisonWindowTests.cs new file mode 100644 index 00000000..d58e7265 --- /dev/null +++ b/WPF/SAM.Analytical.UI.WPF.Tests/EngineComparisonWindowTests.cs @@ -0,0 +1,181 @@ +// SPDX-License-Identifier: LGPL-3.0-or-later +// Copyright (c) 2020-2026 Michal Dengusiak & Jakub Ziolkowski and contributors + +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Windows; +using System.Windows.Controls; +using System.Windows.Media; +using System.Windows.Threading; +using Xunit; + +namespace SAM.Analytical.UI.WPF.Tests +{ + /// + /// The Tas/OpenStudio comparison window as a person meets it: the banner's counts and the two grids at the wide and the narrow + /// reviewed widths. The documents are SAM_Validation's FABRICATED regression documents, not engine results; the grid shows what + /// SAM_Validation computed, so the only product decisions here are labels and layout. + /// + [Collection(WpfCollection.Name)] + public class EngineComparisonWindowTests + { + private static readonly double[] ModelWidths = { 160, 85, 80, 90, 100, 135, 55, 140, 275 }; + + private static readonly double[] SpaceWidths = { 100, 80, 125, 85, 60, 75, 90, 140, 45, 140, 160 }; + + private static string Resource(string name, [CallerFilePath] string sourceFilePath = null) + { + return Path.Combine(Path.GetDirectoryName(sourceFilePath), "Resources", "EngineComparison", name); + } + + private static string Tas => Resource("scenario-benchmark-TAS.json"); + + private static string OpenStudio => Resource("scenario-benchmark-OpenStudio.json"); + + [WpfFact] + public void Banner_CountsInformationalRowsSeparatelyFromUnitMismatch() + { + //Informational rows (peak hours, whole-model peaks) are excluded from the numerical status. They are not "not equivalent" + //in the incompatible sense, so the banner must not describe them that way. + EngineComparisonWindow window = Window(1180); + try + { + Assert.True(window.Compare(Tas, OpenStudio, null)); + Flush(window); + + EngineComparison comparison = Query.LoadEngineComparison(Tas, OpenStudio); + string banner = Control(window, "TextBlock_Banner").Text; + + Assert.Contains(comparison.ComparisonResult.FailCount + " fail", banner); + Assert.Contains(comparison.ContributingFailCount + " contributing to numerical status", banner); + Assert.Contains(comparison.InformationalFailCount + " informational and excluded", banner); + Assert.Contains(comparison.UnitMismatchCount + " unit mismatch", banner); + Assert.DoesNotContain("not equivalent", banner); + + //The counts themselves are SAM_Validation's: contributing + informational is every Fail band. + Assert.Equal(comparison.ComparisonResult.FailCount, comparison.ContributingFailCount + comparison.InformationalFailCount); + Assert.Equal(comparison.ComparisonResult.ContributingFailCount, comparison.ContributingFailCount); + Assert.Equal(comparison.ComparisonResult.InformationalFailCount, comparison.InformationalFailCount); + } + finally + { + window.Close(); + } + } + + [WpfFact] + public void Grids_KeepReadableColumns_AndScrollHorizontallyWhenNarrow() + { + EngineComparisonWindow wide = Window(1180); + try + { + Assert.True(wide.Compare(Tas, OpenStudio, null)); + Flush(wide); + + AssertGrid(wide, "DataGrid_Model", ModelWidths, expectHorizontalScroll: false); + ((TabControl)wide.FindName("TabControl_Main")).SelectedIndex = 1; + Flush(wide); + AssertGrid(wide, "DataGrid_Spaces", SpaceWidths, expectHorizontalScroll: false); + } + finally + { + wide.Close(); + } + + EngineComparisonWindow narrow = Window(820); + try + { + Assert.True(narrow.Compare(Tas, OpenStudio, null)); + Flush(narrow); + + //At 820 px every column keeps its readable width; the grid scrolls instead of compressing the columns. + AssertGrid(narrow, "DataGrid_Model", ModelWidths, expectHorizontalScroll: true); + ((TabControl)narrow.FindName("TabControl_Main")).SelectedIndex = 1; + Flush(narrow); + AssertGrid(narrow, "DataGrid_Spaces", SpaceWidths, expectHorizontalScroll: true); + } + finally + { + narrow.Close(); + } + } + + private static void AssertGrid(EngineComparisonWindow window, string name, double[] widths, bool expectHorizontalScroll) + { + DataGrid dataGrid = Control(window, name); + + Assert.Equal(widths.Length, dataGrid.Columns.Count); + for (int index = 0; index < widths.Length; index++) + { + //Fixed widths are preserved: a compressed column was the defect this guards against. + Assert.True( + dataGrid.Columns[index].ActualWidth >= widths[index] - 1, + string.Format("{0} column {1} ({2}) compressed to {3:0.#} px", name, index, dataGrid.Columns[index].Header, dataGrid.Columns[index].ActualWidth)); + } + + ScrollViewer scrollViewer = VisualChildren(dataGrid).First(); + if (expectHorizontalScroll) + { + Assert.True(scrollViewer.ScrollableWidth > 0, name + " should scroll horizontally at a narrow width"); + Assert.Equal(Visibility.Visible, scrollViewer.ComputedHorizontalScrollBarVisibility); + } + else + { + Assert.True(scrollViewer.ScrollableWidth < 1, name + " should fit at the reviewed wide width"); + } + + //The Note column stays readable at both widths (its full text is also on the cell tooltip). + DataGridColumn note = dataGrid.Columns.Last(); + Assert.Equal("Note", note.Header); + Assert.True(note.ActualWidth >= widths.Last() - 1); + } + + private static EngineComparisonWindow Window(double width) + { + EngineComparisonWindow window = new EngineComparisonWindow(null) + { + WindowStartupLocation = WindowStartupLocation.Manual, + Left = 0, + Top = 0, + Width = width, + ShowActivated = false, + }; + + window.Show(); + Flush(window); + + return window; + } + + private static T Control(EngineComparisonWindow window, string name) where T : FrameworkElement + { + return (T)window.FindName(name); + } + + private static void Flush(System.Windows.Window window) + { + window.Dispatcher.Invoke(() => { }, DispatcherPriority.ApplicationIdle); + window.UpdateLayout(); + window.Dispatcher.Invoke(() => { }, DispatcherPriority.ApplicationIdle); + } + + private static IEnumerable VisualChildren(DependencyObject parent) where T : DependencyObject + { + for (int index = 0; index < VisualTreeHelper.GetChildrenCount(parent); index++) + { + DependencyObject child = VisualTreeHelper.GetChild(parent, index); + if (child is T match) + { + yield return match; + } + + foreach (T descendant in VisualChildren(child)) + { + yield return descendant; + } + } + } + } +} diff --git a/WPF/SAM.Analytical.UI.WPF.Tests/Resources/EngineComparison/README.md b/WPF/SAM.Analytical.UI.WPF.Tests/Resources/EngineComparison/README.md new file mode 100644 index 00000000..6eb64898 --- /dev/null +++ b/WPF/SAM.Analytical.UI.WPF.Tests/Resources/EngineComparison/README.md @@ -0,0 +1,9 @@ +# Engine comparison test documents (fabricated) + +B1a documents copied verbatim from SAM_Validation `SAM.Analytical.Benchmark.Compare.Tests/Fixtures/Regression/` +(branch `claude/tas-openstudio-comparison`, SAM#211). They are **fabricated test documents, not Tas or +EnergyPlus results**: a feature-rich pair spanning Match/Warn/Fail bands, a year-wrapping peak hour, an +unavailable model metric and spaces present on only one side. `weather-mismatch-*` is the same OpenStudio +document with a different weather hash, so SAM_Validation's provenance gate reports a mismatch. + +`EngineComparisonTests` checks that the comparison view shows exactly what SAM_Validation computes for them. diff --git a/WPF/SAM.Analytical.UI.WPF.Tests/Resources/EngineComparison/scenario-benchmark-OpenStudio.json b/WPF/SAM.Analytical.UI.WPF.Tests/Resources/EngineComparison/scenario-benchmark-OpenStudio.json new file mode 100644 index 00000000..07f0fb4d --- /dev/null +++ b/WPF/SAM.Analytical.UI.WPF.Tests/Resources/EngineComparison/scenario-benchmark-OpenStudio.json @@ -0,0 +1,251 @@ +{ + "schemaVersion": "1.0.0", + "provenance": { + "sourceModelName": "SingleBox", + "sourceModelGuid": "0123456789abcdef0123456789abcdef", + "sourceFileHash": "sha256:1111111111111111111111111111111111111111111111111111111111111111", + "canonicalModelHash": "sha256:2222222222222222222222222222222222222222222222222222222222222222", + "canonicalizationVersion": "1.0.0", + "samCommit": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "runnerCommit": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "engine": { + "kind": "OpenStudio", + "name": "EnergyPlus", + "version": "24.1.0", + "sdkVersion": "3.10.0" + }, + "route": "Native-OpenStudio", + "weather": { + "identity": "London-Gatwick.epw", + "hash": "sha256:3333333333333333333333333333333333333333333333333333333333333333" + }, + "designDaySource": "DDY", + "runTimestampUtc": "2026-07-21T12:00:00Z", + "durationSeconds": 12.5, + "state": "Success", + "resultSources": [ + "AnalyticalModelSimulationResult", + "SpaceSimulationResult" + ], + "warnings": [], + "notes": [] + }, + "model": { + "consumptionHeating": { + "value": 1000, + "unit": "kWh", + "available": true + }, + "consumptionCooling": { + "value": null, + "unit": "kWh", + "available": false + }, + "peakHeatingLoad": { + "value": 30, + "unit": "kW", + "available": true + }, + "peakHeatingHour": { + "value": 205, + "unit": "hourOfYear", + "available": true + }, + "peakCoolingLoad": { + "value": 8.8, + "unit": "kW", + "available": true + }, + "peakCoolingHour": { + "value": 0, + "unit": "hourOfYear", + "available": true + }, + "floorArea": { + "value": 240, + "unit": "m2", + "available": true + }, + "volume": { + "value": 720, + "unit": "m3", + "available": true + } + }, + "spaces": [ + { + "guid": "0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a", + "name": "Office A", + "area": { + "value": 200, + "unit": "m2", + "available": true + }, + "volume": { + "value": 600, + "unit": "m3", + "available": true + }, + "heating": { + "designLoad": { + "value": null, + "unit": "W", + "available": false + }, + "peakLoad": { + "value": 2000, + "unit": "W", + "available": true + }, + "peakHour": { + "value": 205, + "unit": "hourOfYear", + "available": true + }, + "unmetHours": { + "value": 0, + "unit": "h", + "available": true + } + }, + "cooling": { + "designLoad": { + "value": null, + "unit": "W", + "available": false + }, + "peakLoad": { + "value": null, + "unit": "W", + "available": false + }, + "peakHour": { + "value": null, + "unit": "hourOfYear", + "available": false + }, + "unmetHours": { + "value": null, + "unit": "h", + "available": false + } + } + }, + { + "guid": "0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b", + "name": "Office B (OS)", + "area": { + "value": 44, + "unit": "m2", + "available": true + }, + "volume": { + "value": 132, + "unit": "m3", + "available": true + }, + "heating": { + "designLoad": { + "value": null, + "unit": "W", + "available": false + }, + "peakLoad": { + "value": 2600, + "unit": "W", + "available": true + }, + "peakHour": { + "value": 240, + "unit": "hourOfYear", + "available": true + }, + "unmetHours": { + "value": 0, + "unit": "h", + "available": true + } + }, + "cooling": { + "designLoad": { + "value": null, + "unit": "W", + "available": false + }, + "peakLoad": { + "value": null, + "unit": "W", + "available": false + }, + "peakHour": { + "value": null, + "unit": "hourOfYear", + "available": false + }, + "unmetHours": { + "value": null, + "unit": "h", + "available": false + } + } + }, + { + "guid": "0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d", + "name": "Atrium", + "area": { + "value": 18, + "unit": "m2", + "available": true + }, + "volume": { + "value": 54, + "unit": "m3", + "available": true + }, + "heating": { + "designLoad": { + "value": null, + "unit": "W", + "available": false + }, + "peakLoad": { + "value": 2000, + "unit": "W", + "available": true + }, + "peakHour": { + "value": 205, + "unit": "hourOfYear", + "available": true + }, + "unmetHours": { + "value": 0, + "unit": "h", + "available": true + } + }, + "cooling": { + "designLoad": { + "value": null, + "unit": "W", + "available": false + }, + "peakLoad": { + "value": null, + "unit": "W", + "available": false + }, + "peakHour": { + "value": null, + "unit": "hourOfYear", + "available": false + }, + "unmetHours": { + "value": null, + "unit": "h", + "available": false + } + } + } + ] +} \ No newline at end of file diff --git a/WPF/SAM.Analytical.UI.WPF.Tests/Resources/EngineComparison/scenario-benchmark-TAS.json b/WPF/SAM.Analytical.UI.WPF.Tests/Resources/EngineComparison/scenario-benchmark-TAS.json new file mode 100644 index 00000000..346d6140 --- /dev/null +++ b/WPF/SAM.Analytical.UI.WPF.Tests/Resources/EngineComparison/scenario-benchmark-TAS.json @@ -0,0 +1,251 @@ +{ + "schemaVersion": "1.0.0", + "provenance": { + "sourceModelName": "SingleBox", + "sourceModelGuid": "0123456789abcdef0123456789abcdef", + "sourceFileHash": "sha256:1111111111111111111111111111111111111111111111111111111111111111", + "canonicalModelHash": "sha256:2222222222222222222222222222222222222222222222222222222222222222", + "canonicalizationVersion": "1.0.0", + "samCommit": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "runnerCommit": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "engine": { + "kind": "TAS", + "name": "TAS", + "version": "9.5.3", + "sdkVersion": null + }, + "route": "Native-TAS", + "weather": { + "identity": "London-Gatwick.epw", + "hash": "sha256:3333333333333333333333333333333333333333333333333333333333333333" + }, + "designDaySource": "DDY", + "runTimestampUtc": "2026-07-21T12:00:00Z", + "durationSeconds": 12.5, + "state": "Success", + "resultSources": [ + "AnalyticalModelSimulationResult", + "SpaceSimulationResult" + ], + "warnings": [], + "notes": [] + }, + "model": { + "consumptionHeating": { + "value": 1000, + "unit": "kWh", + "available": true + }, + "consumptionCooling": { + "value": 500, + "unit": "kWh", + "available": true + }, + "peakHeatingLoad": { + "value": 10, + "unit": "kW", + "available": true + }, + "peakHeatingHour": { + "value": 200, + "unit": "hourOfYear", + "available": true + }, + "peakCoolingLoad": { + "value": 8, + "unit": "kW", + "available": true + }, + "peakCoolingHour": { + "value": 8759, + "unit": "hourOfYear", + "available": true + }, + "floorArea": { + "value": 240, + "unit": "m2", + "available": true + }, + "volume": { + "value": 720, + "unit": "m3", + "available": true + } + }, + "spaces": [ + { + "guid": "0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a", + "name": "Office A", + "area": { + "value": 200, + "unit": "m2", + "available": true + }, + "volume": { + "value": 600, + "unit": "m3", + "available": true + }, + "heating": { + "designLoad": { + "value": null, + "unit": "W", + "available": false + }, + "peakLoad": { + "value": 2000, + "unit": "W", + "available": true + }, + "peakHour": { + "value": 205, + "unit": "hourOfYear", + "available": true + }, + "unmetHours": { + "value": 0, + "unit": "h", + "available": true + } + }, + "cooling": { + "designLoad": { + "value": null, + "unit": "W", + "available": false + }, + "peakLoad": { + "value": null, + "unit": "W", + "available": false + }, + "peakHour": { + "value": null, + "unit": "hourOfYear", + "available": false + }, + "unmetHours": { + "value": null, + "unit": "h", + "available": false + } + } + }, + { + "guid": "0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b", + "name": "Office B", + "area": { + "value": 40, + "unit": "m2", + "available": true + }, + "volume": { + "value": 120, + "unit": "m3", + "available": true + }, + "heating": { + "designLoad": { + "value": null, + "unit": "W", + "available": false + }, + "peakLoad": { + "value": 2100, + "unit": "W", + "available": true + }, + "peakHour": { + "value": 205, + "unit": "hourOfYear", + "available": true + }, + "unmetHours": { + "value": 0, + "unit": "h", + "available": true + } + }, + "cooling": { + "designLoad": { + "value": null, + "unit": "W", + "available": false + }, + "peakLoad": { + "value": null, + "unit": "W", + "available": false + }, + "peakHour": { + "value": null, + "unit": "hourOfYear", + "available": false + }, + "unmetHours": { + "value": null, + "unit": "h", + "available": false + } + } + }, + { + "guid": "0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c", + "name": "Plant, level 1", + "area": { + "value": 12, + "unit": "m2", + "available": true + }, + "volume": { + "value": 36, + "unit": "m3", + "available": true + }, + "heating": { + "designLoad": { + "value": null, + "unit": "W", + "available": false + }, + "peakLoad": { + "value": 2000, + "unit": "W", + "available": true + }, + "peakHour": { + "value": 205, + "unit": "hourOfYear", + "available": true + }, + "unmetHours": { + "value": 0, + "unit": "h", + "available": true + } + }, + "cooling": { + "designLoad": { + "value": null, + "unit": "W", + "available": false + }, + "peakLoad": { + "value": null, + "unit": "W", + "available": false + }, + "peakHour": { + "value": null, + "unit": "hourOfYear", + "available": false + }, + "unmetHours": { + "value": null, + "unit": "h", + "available": false + } + } + } + ] +} \ No newline at end of file diff --git a/WPF/SAM.Analytical.UI.WPF.Tests/Resources/EngineComparison/weather-mismatch-benchmark-OpenStudio.json b/WPF/SAM.Analytical.UI.WPF.Tests/Resources/EngineComparison/weather-mismatch-benchmark-OpenStudio.json new file mode 100644 index 00000000..189ca333 --- /dev/null +++ b/WPF/SAM.Analytical.UI.WPF.Tests/Resources/EngineComparison/weather-mismatch-benchmark-OpenStudio.json @@ -0,0 +1,251 @@ +{ + "schemaVersion": "1.0.0", + "provenance": { + "sourceModelName": "SingleBox", + "sourceModelGuid": "0123456789abcdef0123456789abcdef", + "sourceFileHash": "sha256:1111111111111111111111111111111111111111111111111111111111111111", + "canonicalModelHash": "sha256:2222222222222222222222222222222222222222222222222222222222222222", + "canonicalizationVersion": "1.0.0", + "samCommit": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "runnerCommit": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "engine": { + "kind": "OpenStudio", + "name": "EnergyPlus", + "version": "24.1.0", + "sdkVersion": "3.10.0" + }, + "route": "Native-OpenStudio", + "weather": { + "identity": "Boston.epw", + "hash": "sha256:9999999999999999999999999999999999999999999999999999999999999999" + }, + "designDaySource": "DDY", + "runTimestampUtc": "2026-07-21T12:00:00Z", + "durationSeconds": 12.5, + "state": "Success", + "resultSources": [ + "AnalyticalModelSimulationResult", + "SpaceSimulationResult" + ], + "warnings": [], + "notes": [] + }, + "model": { + "consumptionHeating": { + "value": 1000, + "unit": "kWh", + "available": true + }, + "consumptionCooling": { + "value": null, + "unit": "kWh", + "available": false + }, + "peakHeatingLoad": { + "value": 30, + "unit": "kW", + "available": true + }, + "peakHeatingHour": { + "value": 205, + "unit": "hourOfYear", + "available": true + }, + "peakCoolingLoad": { + "value": 8.8, + "unit": "kW", + "available": true + }, + "peakCoolingHour": { + "value": 0, + "unit": "hourOfYear", + "available": true + }, + "floorArea": { + "value": 240, + "unit": "m2", + "available": true + }, + "volume": { + "value": 720, + "unit": "m3", + "available": true + } + }, + "spaces": [ + { + "guid": "0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a", + "name": "Office A", + "area": { + "value": 200, + "unit": "m2", + "available": true + }, + "volume": { + "value": 600, + "unit": "m3", + "available": true + }, + "heating": { + "designLoad": { + "value": null, + "unit": "W", + "available": false + }, + "peakLoad": { + "value": 2000, + "unit": "W", + "available": true + }, + "peakHour": { + "value": 205, + "unit": "hourOfYear", + "available": true + }, + "unmetHours": { + "value": 0, + "unit": "h", + "available": true + } + }, + "cooling": { + "designLoad": { + "value": null, + "unit": "W", + "available": false + }, + "peakLoad": { + "value": null, + "unit": "W", + "available": false + }, + "peakHour": { + "value": null, + "unit": "hourOfYear", + "available": false + }, + "unmetHours": { + "value": null, + "unit": "h", + "available": false + } + } + }, + { + "guid": "0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b", + "name": "Office B (OS)", + "area": { + "value": 44, + "unit": "m2", + "available": true + }, + "volume": { + "value": 132, + "unit": "m3", + "available": true + }, + "heating": { + "designLoad": { + "value": null, + "unit": "W", + "available": false + }, + "peakLoad": { + "value": 2600, + "unit": "W", + "available": true + }, + "peakHour": { + "value": 240, + "unit": "hourOfYear", + "available": true + }, + "unmetHours": { + "value": 0, + "unit": "h", + "available": true + } + }, + "cooling": { + "designLoad": { + "value": null, + "unit": "W", + "available": false + }, + "peakLoad": { + "value": null, + "unit": "W", + "available": false + }, + "peakHour": { + "value": null, + "unit": "hourOfYear", + "available": false + }, + "unmetHours": { + "value": null, + "unit": "h", + "available": false + } + } + }, + { + "guid": "0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d", + "name": "Atrium", + "area": { + "value": 18, + "unit": "m2", + "available": true + }, + "volume": { + "value": 54, + "unit": "m3", + "available": true + }, + "heating": { + "designLoad": { + "value": null, + "unit": "W", + "available": false + }, + "peakLoad": { + "value": 2000, + "unit": "W", + "available": true + }, + "peakHour": { + "value": 205, + "unit": "hourOfYear", + "available": true + }, + "unmetHours": { + "value": 0, + "unit": "h", + "available": true + } + }, + "cooling": { + "designLoad": { + "value": null, + "unit": "W", + "available": false + }, + "peakLoad": { + "value": null, + "unit": "W", + "available": false + }, + "peakHour": { + "value": null, + "unit": "hourOfYear", + "available": false + }, + "unmetHours": { + "value": null, + "unit": "h", + "available": false + } + } + } + ] +} \ No newline at end of file diff --git a/WPF/SAM.Analytical.UI.WPF.Tests/SAM.Analytical.UI.WPF.Tests.csproj b/WPF/SAM.Analytical.UI.WPF.Tests/SAM.Analytical.UI.WPF.Tests.csproj index f01e3cfd..36208a2f 100644 --- a/WPF/SAM.Analytical.UI.WPF.Tests/SAM.Analytical.UI.WPF.Tests.csproj +++ b/WPF/SAM.Analytical.UI.WPF.Tests/SAM.Analytical.UI.WPF.Tests.csproj @@ -103,6 +103,13 @@ ..\..\..\SAM_OpenStudio\build\SAM.Core.OpenStudio.dll + + + ..\..\..\SAM_Validation\build\SAM.Analytical.Benchmark.dll + + + ..\..\..\SAM_Validation\build\SAM.Analytical.Benchmark.Compare.dll + diff --git a/WPF/SAM.Analytical.UI.WPF.Tests/TasOptimisationWindowTests.cs b/WPF/SAM.Analytical.UI.WPF.Tests/TasOptimisationWindowTests.cs index f4da9b8b..e195a42e 100644 --- a/WPF/SAM.Analytical.UI.WPF.Tests/TasOptimisationWindowTests.cs +++ b/WPF/SAM.Analytical.UI.WPF.Tests/TasOptimisationWindowTests.cs @@ -74,15 +74,18 @@ public void The_ribbon_button_sits_right_after_Energy_Simulation_and_needs_no_mo RibbonGroup ribbonGroup = Assert.IsType(analyticalWindow.FindName("RibbonGroup_Simulate_Simulate")); List ribbonButtons = ribbonGroup.Items.OfType().ToList(); - Assert.Equal(["RibbonButton_SolarSimulation", "RibbonButton_EnergySimulation", "RibbonButton_Optimisation"], ribbonButtons.Select(x => x.Name)); + //#231 added Run Simulation, OpenStudio Simulation and Compare Tas / OpenStudio to the same tab. + Assert.Equal( + ["RibbonButton_RunSimulation", "RibbonButton_SolarSimulation", "RibbonButton_EnergySimulation", "RibbonButton_OpenStudioSimulation", "RibbonButton_CompareTasOpenStudio", "RibbonButton_Optimisation"], + ribbonButtons.Select(x => x.Name)); - RibbonButton ribbonButton = ribbonButtons[2]; + RibbonButton ribbonButton = ribbonButtons[5]; Assert.Equal("Optimisation", ribbonButton.Label); Assert.NotNull(ribbonButton.LargeImageSource); Assert.Equal("Find the design-variable values that minimise an objective by running a Tas model repeatedly. Works on a Tas project folder, not on the open model. Not the Part O Optimise (2B) command.", ribbonButton.ToolTipDescription); //Energy Simulation needs an open model; Optimisation works on a Tas project folder. - Assert.False(ribbonButtons[1].IsEnabled); + Assert.False(ribbonButtons[2].IsEnabled); Assert.True(ribbonButton.IsEnabled); } finally diff --git a/WPF/SAM.Analytical.UI.WPF/Classes/Comparison/EngineComparison.cs b/WPF/SAM.Analytical.UI.WPF/Classes/Comparison/EngineComparison.cs new file mode 100644 index 00000000..c3c9aaf3 --- /dev/null +++ b/WPF/SAM.Analytical.UI.WPF/Classes/Comparison/EngineComparison.cs @@ -0,0 +1,282 @@ +// SPDX-License-Identifier: LGPL-3.0-or-later +// Copyright (c) 2020-2026 Michal Dengusiak & Jakub Ziolkowski and contributors + +using SAM.Analytical.Benchmark; +using SAM.Analytical.Benchmark.Compare; +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; + +namespace SAM.Analytical.UI.WPF +{ + /// What a comparison row is, as the Tas/OpenStudio comparison view shows it (SAM_UI#231). + public enum EngineComparisonRowStatus + { + /// Both engines report the metric in the same unit; SAM_Validation banded it Match, Warn or Fail. + Compared, + + /// + /// Both engines report a value, but the two are not the same quantity: SAM_Validation treats the metric as informational + /// (a peak hour, or a whole-model peak under different peak definitions), or the units differ. Shown, never banded as a result. + /// + NotEquivalent, + + /// Only OpenStudio reports the metric. + MissingOnTas, + + /// Only Tas reports the metric. + MissingOnOpenStudio, + + /// Neither engine reports the metric. + Unavailable, + + /// + /// One engine's own result, shown because the other engine did not complete (Run Simulation > Both, partially completed). + /// Nothing is compared: there is no difference or band, and the other engine's column stays empty. + /// + NotCompared, + } + + /// + /// One metric of a Tas/OpenStudio comparison, ready to display. Every value is SAM_Validation's: the numbers, differences, band and + /// note are its MetricComparison, rendered with its own Format, so the view shows exactly what comparison.csv says. The + /// only thing decided here is , a label over SAM_Validation's availability, unit and informational flags. + /// + /// A row may instead hold ONE engine's value (), when the other engine did not + /// complete: then only that engine's column, the unit and the note are filled, and there is no difference or band. + /// + /// + public sealed class EngineComparisonRow + { + public EngineComparisonRow(MetricComparison metricComparison, string? spaceName) + { + MetricComparison = metricComparison; + SpaceName = spaceName; + + Scope = metricComparison.Scope; + ScopeFull = metricComparison.Scope; + ScopeDisplay = string.Equals(metricComparison.Scope, Benchmark.Compare.Query.ModelScope, StringComparison.Ordinal) + ? "Model" + : Guid.TryParseExact(metricComparison.Scope, "N", out _) + ? PresentationFormat.CompactGuid(metricComparison.Scope) + : metricComparison.Scope; + + Metric = metricComparison.Key; + Unit = Format.Unit(metricComparison.Unit); + + //Presentation only: two decimals; the raw values and differences stay on the *Full properties and in the reports. + Tas = PresentationFormat.Number(metricComparison.TasValue, metricComparison.Unit, CultureInfo.CurrentCulture, string.Empty); + OpenStudio = PresentationFormat.Number(metricComparison.OpenStudioValue, metricComparison.Unit, CultureInfo.CurrentCulture, string.Empty); + AbsoluteDifference = PresentationFormat.Number( + metricComparison.AbsoluteDifference, + metricComparison.Circular ? MetricUnit.HourOfYear : metricComparison.Unit, + CultureInfo.CurrentCulture, + string.Empty); + PercentDifference = metricComparison.Circular + ? string.Empty + : PresentationFormat.Percent(metricComparison.RelativeDifference, CultureInfo.CurrentCulture, string.Empty); + + TasFull = Format.Number(metricComparison.TasValue); + OpenStudioFull = Format.Number(metricComparison.OpenStudioValue); + AbsoluteDifferenceFull = Format.Number(metricComparison.AbsoluteDifference); + PercentDifferenceFull = metricComparison.Circular ? string.Empty : Format.Percent(metricComparison.RelativeDifference); + + Band = Format.Band(metricComparison.Band); + Note = PresentationFormat.Note(metricComparison); + IsInformational = metricComparison.Band != ComparisonBand.NotApplicable && Benchmark.Compare.Query.IsInformationalForNumericalGate(metricComparison); + IsUnitMismatch = metricComparison.NotApplicableReason == NotApplicableReason.UnitMismatch; + + Status = Query.ComparisonRowStatus(metricComparison); + } + + /// One engine's own value of a metric; nothing is compared (see ). + public EngineComparisonRow(ComparisonEngine comparisonEngine, string scope, string key, MetricValue? metricValue, string? spaceName, string note) + { + SpaceName = spaceName; + + Scope = scope; + ScopeFull = scope; + ScopeDisplay = string.Equals(scope, Benchmark.Compare.Query.ModelScope, StringComparison.Ordinal) + ? "Model" + : Guid.TryParseExact(scope, "N", out _) + ? PresentationFormat.CompactGuid(scope) + : scope; + + Metric = key; + Unit = metricValue == null ? string.Empty : Format.Unit(metricValue.Unit); + + //An unreported metric stays empty: SAM_Validation's own rule (MetricValue.Available), never a zero. + bool available = metricValue != null && metricValue.Available; + string value = available ? PresentationFormat.Number(metricValue!.Value, metricValue.Unit, CultureInfo.CurrentCulture, string.Empty) : string.Empty; + string full = available ? Format.Number(metricValue!.Value) : string.Empty; + Tas = comparisonEngine == ComparisonEngine.Tas ? value : string.Empty; + OpenStudio = comparisonEngine == ComparisonEngine.OpenStudio ? value : string.Empty; + TasFull = comparisonEngine == ComparisonEngine.Tas ? full : string.Empty; + OpenStudioFull = comparisonEngine == ComparisonEngine.OpenStudio ? full : string.Empty; + AbsoluteDifference = string.Empty; + PercentDifference = string.Empty; + AbsoluteDifferenceFull = string.Empty; + PercentDifferenceFull = string.Empty; + Band = string.Empty; + Note = available ? note : "Not reported by " + Query.ComparisonEngineText(comparisonEngine) + ". " + note; + IsInformational = false; + IsUnitMismatch = false; + + Status = EngineComparisonRowStatus.NotCompared; + } + + /// SAM_Validation's comparison of the metric; null for a single-engine row. + public MetricComparison? MetricComparison { get; } + + /// "model" for a model metric, else the space's SAM GUID (or its name when it has none). + public string Scope { get; } + + /// A compact grid label for : "Model", a short GUID, or the space name. + public string ScopeDisplay { get; } + + /// The complete scope identifier, for inspection and copying. + public string ScopeFull { get; } + + /// The space's name as the engines report it ("Tas name / OpenStudio name" when they differ); null for a model metric. + public string? SpaceName { get; } + + public string Metric { get; } + + public string Unit { get; } + + /// Two-decimal display value; empty when the engine did not report it. + public string Tas { get; } + + public string OpenStudio { get; } + + public string AbsoluteDifference { get; } + + /// The normalized difference in percent, two decimals; empty for a circular hour or a missing value. + public string PercentDifference { get; } + + /// Full-precision Tas value for the cell tooltip; empty when unavailable. + public string TasFull { get; } + + /// Full-precision OpenStudio value for the cell tooltip; empty when unavailable. + public string OpenStudioFull { get; } + + /// Full-precision absolute difference for the cell tooltip; empty when unavailable. + public string AbsoluteDifferenceFull { get; } + + /// Full-precision normalized percentage for the cell tooltip; empty for a circular hour. + public string PercentDifferenceFull { get; } + + /// SAM_Validation's band: Match, Warn, Fail or N/A. A tolerance band, not an engineering verdict. Empty when not compared. + public string Band { get; } + + public EngineComparisonRowStatus Status { get; } + + /// True when the row carries a band that is excluded from the numerical status. + public bool IsInformational { get; } + + /// True when the two engines declared incompatible units (not an informational metric). + public bool IsUnitMismatch { get; } + + public string StatusText + { + get + { + if (Status == EngineComparisonRowStatus.NotEquivalent) + { + if (IsUnitMismatch) + { + return "Unit mismatch"; + } + + return IsInformational ? "Informational" : "Not equivalent"; + } + + return Query.ComparisonRowStatusText(Status); + } + } + + /// SAM_Validation's presentation note for the metric; for a single-engine row, why it is not compared. + public string Note { get; } + } + + /// + /// A loaded Tas/OpenStudio comparison: the two B1a documents and SAM_Validation's for them, with the rows + /// the view shows. Built by . + /// + public sealed class EngineComparison + { + public EngineComparison(string tasPath, BenchmarkDocument tas, string openStudioPath, BenchmarkDocument openStudio, ComparisonResult comparisonResult) + { + TasPath = tasPath; + Tas = tas; + OpenStudioPath = openStudioPath; + OpenStudio = openStudio; + ComparisonResult = comparisonResult; + + ModelRows = Query.ComparisonModelRows(comparisonResult); + SpaceRows = Query.ComparisonSpaceRows(comparisonResult); + } + + public string TasPath { get; } + + public BenchmarkDocument Tas { get; } + + public string OpenStudioPath { get; } + + public BenchmarkDocument OpenStudio { get; } + + public ComparisonResult ComparisonResult { get; } + + public IReadOnlyList ModelRows { get; } + + public IReadOnlyList SpaceRows { get; } + + public IReadOnlyList AllRows => ModelRows.Concat(SpaceRows).ToList(); + + /// + /// Fail bands that contribute to the numerical status, taken from SAM_Validation's authoritative split + /// () rather than re-derived from the displayed rows. + /// + public int ContributingFailCount => ComparisonResult.ContributingFailCount; + + /// + /// Fail bands shown but excluded from the numerical status, taken from SAM_Validation's authoritative split + /// (). They are still included in the gate's fail count. + /// + public int InformationalFailCount => ComparisonResult.InformationalFailCount; + + /// + /// Rows whose two engines declared incompatible units. Unlike an informational row (a peak hour or a whole-model peak + /// under different definitions) this is a genuine not-equivalent condition, so the view counts the two separately. + /// + public int UnitMismatchCount => AllRows.Count(x => x.IsUnitMismatch); + } + + /// + /// One engine's B1a document shown on its own, because the other engine of a Run Simulation > Both run did not complete. Built by + /// . Holds no comparison: every row is . + /// + public sealed class EngineResult + { + public EngineResult(ComparisonEngine comparisonEngine, string path, BenchmarkDocument document, string note) + { + Engine = comparisonEngine; + Path = path; + Document = document; + + ModelRows = Query.EngineResultModelRows(comparisonEngine, document, note); + SpaceRows = Query.EngineResultSpaceRows(comparisonEngine, document, note); + } + + public ComparisonEngine Engine { get; } + + public string Path { get; } + + public BenchmarkDocument Document { get; } + + public IReadOnlyList ModelRows { get; } + + public IReadOnlyList SpaceRows { get; } + } +} diff --git a/WPF/SAM.Analytical.UI.WPF/Classes/Comparison/EngineComparisonRun.cs b/WPF/SAM.Analytical.UI.WPF/Classes/Comparison/EngineComparisonRun.cs new file mode 100644 index 00000000..4fff5107 --- /dev/null +++ b/WPF/SAM.Analytical.UI.WPF/Classes/Comparison/EngineComparisonRun.cs @@ -0,0 +1,169 @@ +// SPDX-License-Identifier: LGPL-3.0-or-later +// Copyright (c) 2020-2026 Michal Dengusiak & Jakub Ziolkowski and contributors + +using System; +using System.Collections.Generic; +using System.Text.Json; +using System.Text.Json.Nodes; + +namespace SAM.Analytical.UI.WPF +{ + /// The simulation engines of the Tas/OpenStudio comparison. + public enum ComparisonEngine + { + Tas, + OpenStudio, + } + + /// How a Tas/OpenStudio comparison run ended. + public enum EngineComparisonRunStatus + { + /// + /// Both producers completed and SAM_Validation compared their B1a documents; the reports are written. Says nothing about whether + /// the engines agree: that is SAM_Validation's gate. + /// + Compared, + + /// + /// One producer completed and the other did not (failed, not found or not run): the completed engine's B1a document is kept and + /// shown on its own, and nothing is compared. The missing engine's results are never substituted. + /// + PartiallyCompleted, + + /// No producer completed, the inputs could not be prepared or changed during the run, or the documents could not be + /// compared; the diagnostics and logs say which. + Failed, + + /// The user cancelled; the running producer's process tree was killed. + Cancelled, + } + + /// One producer run (benchmark-tas or benchmark-openstudio) of a comparison. + public sealed class EngineComparisonProducerRun + { + public ComparisonEngine Engine { get; set; } + + public string? ExecutablePath { get; set; } + + public List Arguments { get; } = new List(); + + public int? ExitCode { get; set; } + + public double DurationSeconds { get; set; } + + /// The producer's standard output and error. + public string? LogPath { get; set; } + + /// The B1a document it wrote; null when it wrote none. + public string? BenchmarkPath { get; set; } + + public string? Error { get; set; } + + /// + /// True when the producer ran, exited with 0 and wrote its B1a document. Only such a document is shown or compared: a document + /// left by a failed producer is kept in the run folder for diagnosis but never treated as a result. + /// + public bool Succeeded => ExitCode == 0 && BenchmarkPath != null && Error == null; + } + + /// + /// The record of one Run Simulation > Both run (SAM_UI#231): one frozen, read-only snapshot and weather file, both engines' + /// producers run on them in turn, and SAM_Validation's comparison of the two B1a documents when both completed. Written to the run folder as + /// . It holds no result value: the results are the B1a documents and SAM_Validation's reports. + /// + public sealed class EngineComparisonRun + { + public const string Schema = "sam-ui.engine-comparison-run/1"; + + public const string ManifestFileName = "engine-comparison-run.json"; + + public string? Id { get; set; } + + public EngineComparisonRunStatus Status { get; set; } = EngineComparisonRunStatus.Failed; + + public DateTime StartedUtc { get; set; } + + public DateTime FinishedUtc { get; set; } + + public string? Directory { get; set; } + + public string? SourcePath { get; set; } + + public string? ModelName { get; set; } + + public string? SnapshotPath { get; set; } + + public string? SnapshotSha256 { get; set; } + + /// "File" (chosen by the user), "Embedded" (the model's WeatherData exported to EPW) or "None". + public string? WeatherSource { get; set; } + + public string? WeatherPath { get; set; } + + public string? WeatherSha256 { get; set; } + + public List ProducerRuns { get; } = new List(); + + /// SAM_Validation's overall gate ("Pass", "Warn", "Fail") when the documents were compared. + public string? Gate { get; set; } + + /// comparison.md, comparison.csv and comparison-summary.json, as SAM_Validation wrote them. + public List ReportPaths { get; } = new List(); + + public List Diagnostics { get; } = new List(); + + public Exception? Exception { get; set; } + + public string? ManifestPath { get; set; } + + /// Strings are added with JsonValue.Create: the generic JsonArray.Add<string> cannot be written with these options. + public string ToJson() + { + JsonArray producerRuns = new JsonArray(); + foreach (EngineComparisonProducerRun producerRun in ProducerRuns) + { + JsonArray arguments = new JsonArray(); + producerRun.Arguments.ForEach(x => arguments.Add(JsonValue.Create(x))); + + producerRuns.Add(new JsonObject + { + ["engine"] = producerRun.Engine.ToString(), + ["executable"] = producerRun.ExecutablePath, + ["arguments"] = arguments, + ["exitCode"] = producerRun.ExitCode, + ["durationSeconds"] = System.Math.Round(producerRun.DurationSeconds, 3), + ["log"] = producerRun.LogPath, + ["benchmark"] = producerRun.BenchmarkPath, + ["error"] = producerRun.Error, + ["succeeded"] = producerRun.Succeeded, + }); + } + + JsonArray reportPaths = new JsonArray(); + ReportPaths.ForEach(x => reportPaths.Add(JsonValue.Create(x))); + + JsonArray diagnostics = new JsonArray(); + Diagnostics.ForEach(x => diagnostics.Add(JsonValue.Create(x))); + + JsonObject jsonObject = new JsonObject + { + ["schema"] = Schema, + ["id"] = Id, + ["status"] = Status.ToString(), + ["startedUtc"] = StartedUtc.ToString("o"), + ["finishedUtc"] = FinishedUtc.ToString("o"), + ["sourcePath"] = SourcePath, + ["modelName"] = ModelName, + ["snapshot"] = new JsonObject { ["path"] = SnapshotPath, ["sha256"] = SnapshotSha256 }, + ["weather"] = new JsonObject { ["source"] = WeatherSource, ["path"] = WeatherPath, ["sha256"] = WeatherSha256 }, + ["producers"] = producerRuns, + ["gate"] = Gate, + ["reports"] = reportPaths, + ["diagnostics"] = diagnostics, + ["error"] = Exception == null ? null : Exception.GetType().Name + ": " + Exception.Message, + }; + + return jsonObject.ToJsonString(new JsonSerializerOptions { WriteIndented = true }); + } + } +} diff --git a/WPF/SAM.Analytical.UI.WPF/Modify/RunEngineComparison.cs b/WPF/SAM.Analytical.UI.WPF/Modify/RunEngineComparison.cs new file mode 100644 index 00000000..2787482d --- /dev/null +++ b/WPF/SAM.Analytical.UI.WPF/Modify/RunEngineComparison.cs @@ -0,0 +1,319 @@ +// SPDX-License-Identifier: LGPL-3.0-or-later +// Copyright (c) 2020-2026 Michal Dengusiak & Jakub Ziolkowski and contributors + +using SAM.Analytical.OpenStudio; +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Runtime.CompilerServices; +using System.Text.Json.Nodes; +using System.Threading; + +namespace SAM.Analytical.UI.WPF +{ + public static partial class Modify + { + /// + /// Run Simulation > Both (SAM_UI#231): runs both engines, one after the other, on ONE frozen snapshot of + /// and ONE weather file, then compares their B1a documents with SAM_Validation when both completed. + /// + /// The engines run through their validated B1a producers, each in its own process: benchmark-tas (SAM_Tas, the native Tas route + /// with that producer's fixed workflow settings - not the Energy Simulation dialog's) and benchmark-openstudio (SAM_OpenStudio, the + /// same ToOpenStudio pipeline as Simulate > OpenStudio Simulation). SAM_UI neither maps results to B1a nor compares anything + /// itself: the documents are the producers', and the comparison and reports are SAM_Validation's (Query.Compare, WriteReports). + /// + /// + /// The snapshot and weather file are made read-only once written, and their SHA-256 is checked again before each producer and + /// before comparing: a changed input stops the run rather than compare results made from different inputs. + /// + /// + /// A producer that fails does not stop the other one. Both completed: . One completed: + /// - its document is kept and nothing is compared or substituted for the + /// other engine. None: . + /// + /// + /// The model given is never changed. Never throws for a failed run: the outcome, each producer's exit code and log, and any error + /// are on the returned record, which is also written to the run folder (). + /// + /// + /// The model to simulate; read only. + /// The folder the run folder is created in. + /// An EPW chosen by the user; null to export the model's embedded WeatherData to EPW once, for both engines. + /// benchmark-tas.exe. + /// benchmark-openstudio.exe. + /// The open model's file, recorded for provenance only. + /// One line per step. + /// Cancels the run; the running producer's process tree is killed. + public static EngineComparisonRun RunEngineComparison(this AnalyticalModel analyticalModel, string directory, string? epwPath, string tasProducerPath, string openStudioProducerPath, string? sourcePath = null, IProgress? progress = null, CancellationToken cancellationToken = default) + { + return RunEngineComparison(analyticalModel, directory, epwPath, tasProducerPath, openStudioProducerPath, sourcePath, progress, cancellationToken, RunBenchmarkProducer); + } + + /// The same run with the producer runner given, so the orchestration can be tested without either engine. + internal static EngineComparisonRun RunEngineComparison(AnalyticalModel analyticalModel, string directory, string? epwPath, string tasProducerPath, string openStudioProducerPath, string? sourcePath, IProgress? progress, CancellationToken cancellationToken, Func runProducer) + { + EngineComparisonRun result = new EngineComparisonRun() + { + Id = Guid.NewGuid().ToString("N").Substring(0, 8), + StartedUtc = DateTime.UtcNow, + SourcePath = sourcePath, + ModelName = analyticalModel?.Name, + }; + + if (analyticalModel == null || string.IsNullOrWhiteSpace(directory)) + { + result.Diagnostics.Add("Error: no model or no output folder was given; nothing was run"); + result.FinishedUtc = DateTime.UtcNow; + return result; + } + + result.Directory = Query.EngineComparisonRunDirectory(directory, analyticalModel.Name, result.StartedUtc, result.Id!); + + try + { + System.IO.Directory.CreateDirectory(result.Directory); + + progress?.Report("Freezing the model"); + result.SnapshotPath = Path.Combine(result.Directory, OpenStudioSimulationRun.SnapshotFileName); + AnalyticalModel? analyticalModel_Frozen = analyticalModel.FreezeAnalyticalModel(result.SnapshotPath); + if (analyticalModel_Frozen == null) + { + result.Diagnostics.Add("Error: the model could not be written to and read back from the snapshot file; nothing was run"); + return result; + } + + result.SnapshotSha256 = Query.Sha256(result.SnapshotPath); + ProtectInput(result.SnapshotPath); + + //One weather file for both engines, so both documents carry the same weather hash. + progress?.Report("Preparing the weather file"); + string? epwPath_User = !string.IsNullOrWhiteSpace(epwPath) && File.Exists(epwPath) ? epwPath : null; + string? epwPath_Source = epwPath_User ?? EmbeddedWeatherPath(analyticalModel_Frozen); + result.WeatherSource = epwPath_User != null ? "File" : epwPath_Source != null ? "Embedded" : "None"; + if (epwPath_Source == null) + { + result.Diagnostics.Add("Error: no weather - choose an EPW file, or give the model annual WeatherData; nothing was run"); + return result; + } + + result.WeatherPath = Path.Combine(result.Directory, OpenStudioSimulationRun.WeatherFileName); + File.Copy(epwPath_Source, result.WeatherPath, true); + result.WeatherSha256 = Query.Sha256(result.WeatherPath); + ProtectInput(result.WeatherPath); + + foreach ((ComparisonEngine engine, string producerPath) in new[] { (ComparisonEngine.Tas, tasProducerPath), (ComparisonEngine.OpenStudio, openStudioProducerPath) }) + { + cancellationToken.ThrowIfCancellationRequested(); + + progress?.Report(engine == ComparisonEngine.Tas ? "Running Tas (benchmark-tas)" : "Running OpenStudio/EnergyPlus (benchmark-openstudio)"); + + string workDirectory = Path.Combine(result.Directory, engine == ComparisonEngine.Tas ? "tas" : "openstudio"); + System.IO.Directory.CreateDirectory(workDirectory); + + string benchmarkPath = Path.Combine(result.Directory, engine == ComparisonEngine.Tas ? Query.TasBenchmarkFileName : Query.OpenStudioBenchmarkFileName); + EngineComparisonProducerRun producerRun = new EngineComparisonProducerRun() + { + Engine = engine, + ExecutablePath = producerPath, + LogPath = Path.Combine(result.Directory, engine == ComparisonEngine.Tas ? "benchmark-tas.log" : "benchmark-openstudio.log"), + }; + producerRun.Arguments.AddRange(Query.BenchmarkProducerArguments(engine, result.SnapshotPath, result.WeatherPath, benchmarkPath, workDirectory, analyticalModel_Frozen.Name)); + result.ProducerRuns.Add(producerRun); + + if (string.IsNullOrWhiteSpace(producerPath) || !File.Exists(producerPath)) + { + producerRun.Error = "The producer executable was not found: " + (producerPath ?? "(none)"); + continue; + } + + string? inputChange = InputChange(result); + if (inputChange != null) + { + producerRun.Error = "Not run: " + inputChange; + continue; + } + + Stopwatch stopwatch = Stopwatch.StartNew(); + try + { + producerRun.ExitCode = runProducer(producerRun, workDirectory, cancellationToken); + } + catch (Exception exception) when (!(exception is OperationCanceledException)) + { + producerRun.Error = exception.GetType().Name + ": " + exception.Message; + } + finally + { + producerRun.DurationSeconds = stopwatch.Elapsed.TotalSeconds; + } + + producerRun.BenchmarkPath = File.Exists(benchmarkPath) ? benchmarkPath : null; + if (producerRun.ExitCode != 0 && producerRun.Error == null) + { + producerRun.Error = string.Format("The producer exited with code {0} (2 usage, 3 input/IO, 4 validation, 5 producer failure); see {1}", producerRun.ExitCode, Path.GetFileName(producerRun.LogPath)); + } + } + + cancellationToken.ThrowIfCancellationRequested(); + + EngineComparisonProducerRun tas = result.ProducerRuns[0]; + EngineComparisonProducerRun openStudio = result.ProducerRuns[1]; + + string? inputChange_Final = InputChange(result); + if (inputChange_Final != null) + { + //Which producer saw which input can no longer be told: keep the documents for diagnosis, show and compare none. + result.Diagnostics.Add("Error: no comparison - " + inputChange_Final + "; the documents are kept in the run folder but not used"); + return result; + } + + if (!tas.Succeeded || !openStudio.Succeeded) + { + EngineComparisonProducerRun? completed = tas.Succeeded ? tas : openStudio.Succeeded ? openStudio : null; + if (completed == null) + { + result.Diagnostics.Add("Error: no comparison - neither Tas nor OpenStudio completed"); + return result; + } + + EngineComparisonProducerRun notCompleted = completed == tas ? openStudio : tas; + result.Status = EngineComparisonRunStatus.PartiallyCompleted; + result.Diagnostics.Add(string.Format("Partially completed: not compared - {0} did not complete. The {1} results ({2}) are kept and shown on their own; no {0} result is substituted.", + Query.ComparisonEngineText(notCompleted.Engine), Query.ComparisonEngineText(completed.Engine), Path.GetFileName(completed.BenchmarkPath))); + return result; + } + + progress?.Report("Comparing (SAM_Validation)"); + EngineComparison engineComparison = Query.LoadEngineComparison(tas.BenchmarkPath!, openStudio.BenchmarkPath!); + result.ReportPaths.AddRange(Benchmark.Compare.Modify.WriteReports(engineComparison.ComparisonResult, result.Directory)); + result.Gate = Benchmark.Compare.Format.Gate(engineComparison.ComparisonResult.Gate); + + //Compared is about the workflow, not the engines' agreement: that is SAM_Validation's gate. + result.Status = EngineComparisonRunStatus.Compared; + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + result.Status = EngineComparisonRunStatus.Cancelled; + } + catch (Exception exception) + { + result.Status = EngineComparisonRunStatus.Failed; + result.Exception = exception; + } + finally + { + result.FinishedUtc = DateTime.UtcNow; + WriteEngineComparisonManifest(result); + } + + return result; + } + + /// Remembers the producer the user located, for . + public static bool SaveBenchmarkProducerPath(ComparisonEngine comparisonEngine, string path, string? settingsPath = null) + { + settingsPath ??= Query.BenchmarkProducerSettingsPath(); + try + { + JsonObject jsonObject = File.Exists(settingsPath) ? JsonNode.Parse(File.ReadAllText(settingsPath)) as JsonObject ?? new JsonObject() : new JsonObject(); + jsonObject[comparisonEngine.ToString()] = path; + + System.IO.Directory.CreateDirectory(Path.GetDirectoryName(settingsPath)!); + File.WriteAllText(settingsPath, jsonObject.ToJsonString()); + return true; + } + catch (Exception exception) when (exception is IOException || exception is UnauthorizedAccessException || exception is System.Text.Json.JsonException) + { + return false; + } + } + + /// Makes a run input read-only, so neither engine nor the user changes it unnoticed; its hash is checked as well. + private static void ProtectInput(string path) + { + File.SetAttributes(path, File.GetAttributes(path) | FileAttributes.ReadOnly); + } + + /// Null when the snapshot and weather file still have the SHA-256 they were frozen with; else what changed. + private static string? InputChange(EngineComparisonRun engineComparisonRun) + { + List changes = new List(); + foreach ((string name, string? path, string? sha256) in new[] { ("snapshot", engineComparisonRun.SnapshotPath, engineComparisonRun.SnapshotSha256), ("weather file", engineComparisonRun.WeatherPath, engineComparisonRun.WeatherSha256) }) + { + string? sha256_Now = path != null && File.Exists(path) ? Query.Sha256(path) : null; + if (sha256_Now == null || !string.Equals(sha256_Now, sha256, StringComparison.OrdinalIgnoreCase)) + { + changes.Add(string.Format("the {0} changed after it was frozen (SHA-256 {1} -> {2})", name, sha256 ?? "-", sha256_Now ?? "missing")); + } + } + + return changes.Count == 0 ? null : string.Join("; ", changes); + } + + // Its own method, never inlined: SAM_OpenStudio's embedded-weather export is reached here, so a missing SAM_OpenStudio assembly + // fails inside the caller's try and is recorded on the run. + [MethodImpl(MethodImplOptions.NoInlining)] + private static string? EmbeddedWeatherPath(AnalyticalModel analyticalModel) + { + return analyticalModel.EmbeddedAnnualWeatherPath(); + } + + /// Runs a producer as a child process, its output to its log; cancellation kills the whole process tree. + private static int RunBenchmarkProducer(EngineComparisonProducerRun producerRun, string workDirectory, CancellationToken cancellationToken) + { + ProcessStartInfo processStartInfo = new ProcessStartInfo(producerRun.ExecutablePath!) + { + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true, + CreateNoWindow = true, + WorkingDirectory = workDirectory, + }; + producerRun.Arguments.ForEach(x => processStartInfo.ArgumentList.Add(x)); + + using StreamWriter streamWriter = new StreamWriter(producerRun.LogPath!); + object @lock = new object(); + + using Process process = Process.Start(processStartInfo) ?? throw new InvalidOperationException("The producer process did not start"); + process.OutputDataReceived += (s, e) => { if (e.Data != null) { lock (@lock) { streamWriter.WriteLine(e.Data); } } }; + process.ErrorDataReceived += (s, e) => { if (e.Data != null) { lock (@lock) { streamWriter.WriteLine("[stderr] " + e.Data); } } }; + process.BeginOutputReadLine(); + process.BeginErrorReadLine(); + + while (!process.WaitForExit(250)) + { + if (cancellationToken.IsCancellationRequested) + { + process.Kill(true); + process.WaitForExit(); + cancellationToken.ThrowIfCancellationRequested(); + } + } + + //Drains the asynchronous output before the log is closed. + process.WaitForExit(); + return process.ExitCode; + } + + private static void WriteEngineComparisonManifest(EngineComparisonRun engineComparisonRun) + { + if (string.IsNullOrWhiteSpace(engineComparisonRun.Directory) || !System.IO.Directory.Exists(engineComparisonRun.Directory)) + { + return; + } + + string path = Path.Combine(engineComparisonRun.Directory!, EngineComparisonRun.ManifestFileName); + try + { + File.WriteAllText(path, engineComparisonRun.ToJson()); + engineComparisonRun.ManifestPath = path; + } + catch (Exception exception) + { + //Written from a finally on a worker thread: whatever goes wrong here must not take the run, or the application, down. + engineComparisonRun.Diagnostics.Add("Warning: the run manifest could not be written: " + exception.GetType().Name + ": " + exception.Message); + } + } + } +} diff --git a/WPF/SAM.Analytical.UI.WPF/Modify/SimulateBothEngines.cs b/WPF/SAM.Analytical.UI.WPF/Modify/SimulateBothEngines.cs new file mode 100644 index 00000000..0b0a89c4 --- /dev/null +++ b/WPF/SAM.Analytical.UI.WPF/Modify/SimulateBothEngines.cs @@ -0,0 +1,172 @@ +// SPDX-License-Identifier: LGPL-3.0-or-later +// Copyright (c) 2020-2026 Michal Dengusiak & Jakub Ziolkowski and contributors + +using SAM.Core.Windows.WPF; +using SAM.Weather; +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using System.Windows; + +namespace SAM.Analytical.UI.WPF +{ + public static partial class Modify + { + private const string SimulateBothEnginesTitle = "Run Simulation - Tas + OpenStudio"; + + /// + /// Run Simulation > Both (SAM_UI#231): locates the two B1a producers, asks for the output folder and the weather, then runs + /// + /// on a worker thread, with a progress window whose Cancel kills the running producer's process tree. + /// + /// The engines run one after the other on one frozen snapshot and one weather file; the open model is never changed. Null when + /// the user cancelled a dialog before the run started. The caller shows the returned run (). + /// + /// + public static async Task SimulateBothEnginesAsync(this UIAnalyticalModel uIAnalyticalModel, System.Windows.Window? owner = null) + { + //The getter returns a deep clone, so later edits of the open model cannot reach this one. + AnalyticalModel? analyticalModel = uIAnalyticalModel?.JSAMObject; + if (analyticalModel == null) + { + return null; + } + + string? tasProducerPath = LocateBenchmarkProducer(ComparisonEngine.Tas, owner); + if (tasProducerPath == null) + { + return null; + } + + string? openStudioProducerPath = LocateBenchmarkProducer(ComparisonEngine.OpenStudio, owner); + if (openStudioProducerPath == null) + { + return null; + } + + string? path = uIAnalyticalModel!.Path; + string directory; + using (System.Windows.Forms.FolderBrowserDialog folderBrowserDialog = new() + { + Description = "Select the folder for the Tas + OpenStudio run (a new run folder is created in it)", + UseDescriptionForTitle = true, + SelectedPath = (string.IsNullOrWhiteSpace(path) ? null : Path.GetDirectoryName(path)) ?? Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments), + }) + { + if (folderBrowserDialog.ShowDialog() != System.Windows.Forms.DialogResult.OK || string.IsNullOrWhiteSpace(folderBrowserDialog.SelectedPath)) + { + return null; + } + + directory = folderBrowserDialog.SelectedPath; + } + + //One weather file for both engines: the model's WeatherData exported once, or an EPW chosen here. + string? epwPath = null; + bool embeddedWeather = analyticalModel.TryGetValue(Analytical.AnalyticalModelParameter.WeatherData, out WeatherData? weatherData) && weatherData != null; + if (embeddedWeather) + { + MessageBoxResult messageBoxResult = ShowBothEnginesMessage(owner, "Use the model's weather data for both engines?\n\nYes: the model's weather data (exported once to EPW).\nNo: choose an EPW file.", MessageBoxButton.YesNoCancel, MessageBoxImage.Question); + if (messageBoxResult == MessageBoxResult.Cancel) + { + return null; + } + + embeddedWeather = messageBoxResult == MessageBoxResult.Yes; + } + + if (!embeddedWeather) + { + Microsoft.Win32.OpenFileDialog openFileDialog = new Microsoft.Win32.OpenFileDialog() + { + Title = "Select the EPW weather file for both engines", + Filter = "epw files (*.epw)|*.epw|All files (*.*)|*.*", + }; + + bool? dialogResult = owner == null ? openFileDialog.ShowDialog() : openFileDialog.ShowDialog(owner); + if (dialogResult != true || string.IsNullOrWhiteSpace(openFileDialog.FileName)) + { + return null; + } + + epwPath = openFileDialog.FileName; + } + + using CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); + ProgressWindowHost progressWindowHost = new ProgressWindowHost(SimulateBothEnginesTitle, 6, true, "Tas runs first, then OpenStudio/EnergyPlus. Cancel stops the running engine (its process tree is killed). The open model is not changed."); + try + { + progressWindowHost.CancelRequested += (s, e) => cancellationTokenSource.Cancel(); + + EngineComparisonProgressReport progress = new EngineComparisonProgressReport(progressWindowHost); + CancellationToken cancellationToken = cancellationTokenSource.Token; + + return await Task.Run(() => analyticalModel.RunEngineComparison(directory, epwPath, tasProducerPath, openStudioProducerPath, path, progress, cancellationToken)); + } + finally + { + progressWindowHost.Dispose(); + } + } + + /// The producer to run, asking the user to locate it (once; remembered) when it is not found. Null when the user declined. + public static string? LocateBenchmarkProducer(ComparisonEngine comparisonEngine, System.Windows.Window? owner = null) + { + string? path = Query.BenchmarkProducerPath(comparisonEngine); + if (path != null) + { + return path; + } + + string fileName = Query.BenchmarkProducerFileName(comparisonEngine); + string hint = comparisonEngine == ComparisonEngine.Tas + ? "It is built by SAM_Tas into SAM_Tas\\build (needs Visual Studio / .NET Framework MSBuild for the Tas COM references)." + : "It is built by SAM_OpenStudio (benchmark\\SAM.Analytical.OpenStudio.Benchmark\\bin\\x64\\Release\\...)."; + + string text = string.Format("{0} was not found (environment variable {1}, the application folder, or a location chosen before).\n\n{2}\n\nLocate it now?", fileName, Query.BenchmarkProducerVariable(comparisonEngine), hint); + if (ShowBothEnginesMessage(owner, text, MessageBoxButton.OKCancel, MessageBoxImage.Question) != MessageBoxResult.OK) + { + return null; + } + + Microsoft.Win32.OpenFileDialog openFileDialog = new Microsoft.Win32.OpenFileDialog() + { + Title = "Locate " + fileName, + Filter = fileName + "|" + fileName + "|Executables (*.exe)|*.exe", + }; + + bool? dialogResult = owner == null ? openFileDialog.ShowDialog() : openFileDialog.ShowDialog(owner); + if (dialogResult != true) + { + return null; + } + + SaveBenchmarkProducerPath(comparisonEngine, openFileDialog.FileName); + return openFileDialog.FileName; + } + + private static MessageBoxResult ShowBothEnginesMessage(System.Windows.Window? owner, string text, MessageBoxButton messageBoxButton, MessageBoxImage messageBoxImage) + { + return owner == null + ? MessageBox.Show(text, SimulateBothEnginesTitle, messageBoxButton, messageBoxImage) + : MessageBox.Show(owner, text, SimulateBothEnginesTitle, messageBoxButton, messageBoxImage); + } + + /// Step lines into the progress window; it marshals onto its own thread, so any thread may report. + private sealed class EngineComparisonProgressReport : IProgress + { + private readonly ProgressWindowHost progressWindowHost; + + public EngineComparisonProgressReport(ProgressWindowHost progressWindowHost) + { + this.progressWindowHost = progressWindowHost; + } + + public void Report(string value) + { + progressWindowHost.Update(value); + } + } + } +} diff --git a/WPF/SAM.Analytical.UI.WPF/Query/BenchmarkProducer.cs b/WPF/SAM.Analytical.UI.WPF/Query/BenchmarkProducer.cs new file mode 100644 index 00000000..cd83e0af --- /dev/null +++ b/WPF/SAM.Analytical.UI.WPF/Query/BenchmarkProducer.cs @@ -0,0 +1,109 @@ +// SPDX-License-Identifier: LGPL-3.0-or-later +// Copyright (c) 2020-2026 Michal Dengusiak & Jakub Ziolkowski and contributors + +using System; +using System.Collections.Generic; +using System.IO; +using System.Text.Json.Nodes; + +namespace SAM.Analytical.UI.WPF +{ + public static partial class Query + { + /// + /// The producer executable of an engine: benchmark-tas (SAM_Tas\build) or benchmark-openstudio (SAM_OpenStudio's benchmark + /// project). These are the validated B1a producers; SAM_UI runs them rather than mapping results to B1a a second way. + /// + public static string BenchmarkProducerFileName(ComparisonEngine comparisonEngine) + { + return comparisonEngine == ComparisonEngine.Tas ? "benchmark-tas.exe" : "benchmark-openstudio.exe"; + } + + /// The environment variable that may name an engine's producer (the executable, or the folder holding it). + public static string BenchmarkProducerVariable(ComparisonEngine comparisonEngine) + { + return comparisonEngine == ComparisonEngine.Tas ? "SAM_BENCHMARK_TAS" : "SAM_BENCHMARK_OPENSTUDIO"; + } + + /// Where the producer locations chosen by the user are remembered. + public static string BenchmarkProducerSettingsPath() + { + return Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), "SAM", "engine-comparison-producers.json"); + } + + /// + /// The producer executable to run for , or null when none is found: the environment variable + /// first, then the application folder, then the location the user chose before (). + /// + public static string? BenchmarkProducerPath(ComparisonEngine comparisonEngine, string? settingsPath = null) + { + string fileName = BenchmarkProducerFileName(comparisonEngine); + + List candidates = new List + { + Environment.GetEnvironmentVariable(BenchmarkProducerVariable(comparisonEngine)), + Path.Combine(AppContext.BaseDirectory, fileName), + SavedBenchmarkProducerPath(comparisonEngine, settingsPath ?? BenchmarkProducerSettingsPath()), + }; + + foreach (string? candidate in candidates) + { + if (string.IsNullOrWhiteSpace(candidate)) + { + continue; + } + + string path = Directory.Exists(candidate) ? Path.Combine(candidate!, fileName) : candidate!; + if (File.Exists(path)) + { + return path; + } + } + + return null; + } + + /// + /// The command line of a producer run: the frozen snapshot and one weather file for both engines, so both B1a documents carry the + /// same model and weather provenance. Tas writes its TBD (and gbXML) under ; OpenStudio runs there. + /// + public static List BenchmarkProducerArguments(ComparisonEngine comparisonEngine, string snapshotPath, string weatherPath, string benchmarkPath, string workDirectory, string? modelName) + { + List result = new List { "--model", snapshotPath, "--weather", weatherPath }; + + if (comparisonEngine == ComparisonEngine.Tas) + { + result.Add("--tbd"); + result.Add(Path.Combine(workDirectory, FileNameSafe(modelName) + ".tbd")); + } + + result.Add("--out"); + result.Add(benchmarkPath); + + if (comparisonEngine == ComparisonEngine.OpenStudio) + { + result.Add("--work"); + result.Add(workDirectory); + } + + return result; + } + + private static string? SavedBenchmarkProducerPath(ComparisonEngine comparisonEngine, string settingsPath) + { + try + { + if (!File.Exists(settingsPath)) + { + return null; + } + + return JsonNode.Parse(File.ReadAllText(settingsPath))?[comparisonEngine.ToString()]?.GetValue(); + } + catch (Exception exception) when (exception is IOException || exception is UnauthorizedAccessException || exception is System.Text.Json.JsonException || exception is InvalidOperationException || exception is FormatException) + { + return null; + } + } + } +} diff --git a/WPF/SAM.Analytical.UI.WPF/Query/EngineComparison.cs b/WPF/SAM.Analytical.UI.WPF/Query/EngineComparison.cs new file mode 100644 index 00000000..ddc031c6 --- /dev/null +++ b/WPF/SAM.Analytical.UI.WPF/Query/EngineComparison.cs @@ -0,0 +1,388 @@ +// SPDX-License-Identifier: LGPL-3.0-or-later +// Copyright (c) 2020-2026 Michal Dengusiak & Jakub Ziolkowski and contributors + +using SAM.Analytical.Benchmark; +using SAM.Analytical.Benchmark.Compare; +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; + +namespace SAM.Analytical.UI.WPF +{ + public static partial class Query + { + /// The B1a document file names the producers write and this workflow looks for in a run folder. + public const string TasBenchmarkFileName = "benchmark-TAS.json"; + + public const string OpenStudioBenchmarkFileName = "benchmark-OpenStudio.json"; + + /// + /// The folder of one Tas/OpenStudio comparison run: <directory>\<model>_TasOpenStudio_<yyyyMMdd-HHmmss>_<id>, so a run + /// never writes into another run's folder (as ). + /// + public static string EngineComparisonRunDirectory(string directory, string? modelName, DateTime startedUtc, string id) + { + return Path.Combine(directory, string.Format("{0}_TasOpenStudio_{1:yyyyMMdd-HHmmss}_{2}", FileNameSafe(modelName), startedUtc, id)); + } + + /// + /// Reads two B1a documents and compares them with SAM_Validation (Query.Compare, default tolerance profile) - the same + /// comparison benchmark-compare runs. Nothing is computed here. Throws what SAM_Validation throws for an unreadable, + /// invalid or incompatible document, so the caller can show it. + /// + public static EngineComparison LoadEngineComparison(string tasPath, string openStudioPath) + { + BenchmarkDocument tas = BenchmarkSerializer.Read(tasPath); + BenchmarkDocument openStudio = BenchmarkSerializer.Read(openStudioPath); + + ComparisonResult comparisonResult = Benchmark.Compare.Query.Compare(tas, openStudio, ToleranceProfile.Default); + + return new EngineComparison(tasPath, tas, openStudioPath, openStudio, comparisonResult); + } + + /// + /// The two B1a documents in (a Tas/OpenStudio comparison run folder, or any folder holding both + /// producer outputs); null for a document that is not there. + /// + public static (string? Tas, string? OpenStudio) EngineComparisonDocuments(string? directory) + { + if (string.IsNullOrWhiteSpace(directory) || !Directory.Exists(directory)) + { + return (null, null); + } + + string tas = Path.Combine(directory!, TasBenchmarkFileName); + string openStudio = Path.Combine(directory!, OpenStudioBenchmarkFileName); + + return (File.Exists(tas) ? tas : null, File.Exists(openStudio) ? openStudio : null); + } + + /// + /// How a row is shown. A label over SAM_Validation's own flags - availability, unit mismatch and its informational rule - never a + /// second comparison: no value, difference or band is looked at. + /// + public static EngineComparisonRowStatus ComparisonRowStatus(MetricComparison metricComparison) + { + switch (metricComparison.Availability) + { + case ComparisonAvailability.Neither: + return EngineComparisonRowStatus.Unavailable; + case ComparisonAvailability.TasOnly: + return EngineComparisonRowStatus.MissingOnOpenStudio; + case ComparisonAvailability.OpenStudioOnly: + return EngineComparisonRowStatus.MissingOnTas; + } + + if (metricComparison.NotApplicableReason == NotApplicableReason.UnitMismatch || Benchmark.Compare.Query.IsInformationalForNumericalGate(metricComparison)) + { + return EngineComparisonRowStatus.NotEquivalent; + } + + return EngineComparisonRowStatus.Compared; + } + + public static string ComparisonRowStatusText(EngineComparisonRowStatus status) + { + switch (status) + { + case EngineComparisonRowStatus.Compared: + return "Compared"; + case EngineComparisonRowStatus.NotEquivalent: + return "Not equivalent"; + case EngineComparisonRowStatus.MissingOnTas: + return "Missing on Tas"; + case EngineComparisonRowStatus.MissingOnOpenStudio: + return "Missing on OpenStudio"; + case EngineComparisonRowStatus.NotCompared: + return "Not compared"; + default: + return "Not reported by either engine"; + } + } + + public static string ComparisonEngineText(ComparisonEngine comparisonEngine) + { + return comparisonEngine == ComparisonEngine.Tas ? "Tas" : "OpenStudio"; + } + + /// + /// Reads one engine's B1a document (SAM_Validation's reader, which validates it) to show it on its own: Run Simulation > Both + /// when the other engine did not complete. Nothing is compared. Throws what SAM_Validation throws for an unreadable or invalid + /// document. + /// + public static EngineResult LoadEngineResult(ComparisonEngine comparisonEngine, string path, ComparisonEngine? notCompleted = null) + { + BenchmarkDocument benchmarkDocument = BenchmarkSerializer.Read(path); + + string note = notCompleted == null + ? "Not compared." + : "Not compared - " + ComparisonEngineText(notCompleted.Value) + " did not complete."; + + return new EngineResult(comparisonEngine, path, benchmarkDocument, note); + } + + /// One engine's model metrics, under SAM_Validation's keys and in its order (ModelMetricValues). + public static IReadOnlyList EngineResultModelRows(ComparisonEngine comparisonEngine, BenchmarkDocument benchmarkDocument, string note) + { + return Benchmark.Compare.Query.ModelMetricValues(benchmarkDocument.Model) + .Select(x => new EngineComparisonRow(comparisonEngine, Benchmark.Compare.Query.ModelScope, x.Key, x.Value, null, note)).ToList(); + } + + /// One engine's per-space metrics, under SAM_Validation's keys and in its order (SpaceMetricValues), in document order. + public static IReadOnlyList EngineResultSpaceRows(ComparisonEngine comparisonEngine, BenchmarkDocument benchmarkDocument, string note) + { + List result = new List(); + foreach (BenchmarkSpaceResult benchmarkSpaceResult in benchmarkDocument.Spaces ?? new List()) + { + string scope = benchmarkSpaceResult.Guid ?? benchmarkSpaceResult.Name ?? "(unnamed)"; + result.AddRange(Benchmark.Compare.Query.SpaceMetricValues(benchmarkSpaceResult) + .Select(x => new EngineComparisonRow(comparisonEngine, scope, x.Key, x.Value, benchmarkSpaceResult.Name, note))); + } + + return result; + } + + /// The input provenance of one engine's result shown on its own. + public static string EngineResultProvenance(EngineResult engineResult) + { + List lines = new List + { + "Provenance gate: not evaluated - only " + ComparisonEngineText(engineResult.Engine) + " completed, so there is nothing to check it against.", + string.Empty, + }; + + lines.AddRange(ProvenanceLines(ComparisonEngineText(engineResult.Engine), engineResult.Path, engineResult.Document.Provenance)); + lines.Add(string.Empty); + lines.Add("Input manifests (per-space inputs each engine received): not available - not implemented yet (SAM_OpenStudio#30, SAM_UI#230)."); + + return string.Join(Environment.NewLine, lines); + } + + /// The diagnostics of one engine's result shown on its own, and the run's producers and diagnostics. + public static string EngineResultDiagnostics(EngineResult engineResult, EngineComparisonRun? engineComparisonRun = null) + { + List lines = new List + { + "Not compared: only " + ComparisonEngineText(engineResult.Engine) + " completed. No gate, difference or band is given, and no result is substituted for the other engine.", + string.Empty, + }; + + lines.AddRange(RunDiagnosticLines(ComparisonEngineText(engineResult.Engine), engineResult.Document.Provenance)); + lines.AddRange(EngineComparisonRunLines(engineComparisonRun)); + + return string.Join(Environment.NewLine, lines); + } + + /// + /// A Run Simulation > Both run's status, producers (exit code, duration, log, error) and diagnostics, as text; empty for no run. + /// + public static IEnumerable EngineComparisonRunLines(EngineComparisonRun? engineComparisonRun) + { + if (engineComparisonRun == null) + { + yield break; + } + + yield return string.Empty; + yield return "Run: " + EngineComparisonRunStatusText(engineComparisonRun.Status) + (engineComparisonRun.Directory == null ? string.Empty : " - " + engineComparisonRun.Directory); + foreach (EngineComparisonProducerRun producerRun in engineComparisonRun.ProducerRuns) + { + yield return string.Format(CultureInfo.InvariantCulture, "{0} producer: {1}, {2}, exit code {3}, {4:0.#} s, log {5}", producerRun.Engine, producerRun.Succeeded ? "completed" : "did not complete", producerRun.ExecutablePath ?? "(not found)", producerRun.ExitCode?.ToString(CultureInfo.InvariantCulture) ?? "-", producerRun.DurationSeconds, producerRun.LogPath ?? "-"); + if (!string.IsNullOrWhiteSpace(producerRun.Error)) + { + yield return " " + producerRun.Error; + } + } + + foreach (string diagnostic in engineComparisonRun.Diagnostics) + { + yield return diagnostic; + } + + if (engineComparisonRun.Exception != null) + { + yield return "Error: " + engineComparisonRun.Exception.GetType().Name + ": " + engineComparisonRun.Exception.Message; + } + + if (engineComparisonRun.ManifestPath != null) + { + yield return "Run record: " + engineComparisonRun.ManifestPath; + } + } + + public static string EngineComparisonRunStatusText(EngineComparisonRunStatus engineComparisonRunStatus) + { + switch (engineComparisonRunStatus) + { + case EngineComparisonRunStatus.Compared: + return "Compared"; + case EngineComparisonRunStatus.PartiallyCompleted: + return "Partially completed"; + case EngineComparisonRunStatus.Cancelled: + return "Cancelled"; + default: + return "Failed"; + } + } + + /// The model-level rows, in SAM_Validation's order. + public static IReadOnlyList ComparisonModelRows(ComparisonResult comparisonResult) + { + return comparisonResult.ModelMetrics.Select(x => new EngineComparisonRow(x, null)).ToList(); + } + + /// Every space's rows, in SAM_Validation's order (spaces aligned by GUID, then name). + public static IReadOnlyList ComparisonSpaceRows(ComparisonResult comparisonResult) + { + List result = new List(); + foreach (SpaceComparison spaceComparison in comparisonResult.Spaces) + { + string? name = spaceComparison.TasName ?? spaceComparison.OpenStudioName; + if (spaceComparison.TasName != null && spaceComparison.OpenStudioName != null && !string.Equals(spaceComparison.TasName, spaceComparison.OpenStudioName, StringComparison.Ordinal)) + { + name = spaceComparison.TasName + " / " + spaceComparison.OpenStudioName; + } + + result.AddRange(spaceComparison.Metrics.Select(x => new EngineComparisonRow(x, name))); + } + + return result; + } + + /// + /// The input provenance of a comparison: SAM_Validation's provenance gate and mismatches, then what each B1a document says about + /// its inputs. Per-space input manifests are named as not available: they do not exist yet (SAM_OpenStudio#30, SAM_UI#230). + /// + public static string EngineComparisonProvenance(EngineComparison engineComparison) + { + ComparisonResult comparisonResult = engineComparison.ComparisonResult; + + List lines = new List + { + "Provenance gate: " + Format.Gate(comparisonResult.ProvenanceStatus) + (comparisonResult.ProvenanceCompatibility.IsCompatible ? " (same model, weather and design days)" : string.Empty), + }; + + foreach (ProvenanceMismatch provenanceMismatch in comparisonResult.ProvenanceCompatibility.Mismatches) + { + lines.Add(string.Format(" MISMATCH {0}: Tas = {1}; OpenStudio = {2}", provenanceMismatch.Field, provenanceMismatch.Tas ?? "(none)", provenanceMismatch.OpenStudio ?? "(none)")); + } + + lines.Add(string.Empty); + lines.AddRange(ProvenanceLines("Tas", engineComparison.TasPath, engineComparison.Tas.Provenance)); + lines.Add(string.Empty); + lines.AddRange(ProvenanceLines("OpenStudio", engineComparison.OpenStudioPath, engineComparison.OpenStudio.Provenance)); + lines.Add(string.Empty); + lines.Add("Input manifests (per-space inputs each engine received): not available - not implemented yet (SAM_OpenStudio#30, SAM_UI#230)."); + lines.Add("Input equivalence is therefore shown only as the provenance above (same canonical model, weather file and design-day source)."); + + return string.Join(Environment.NewLine, lines); + } + + /// + /// The diagnostics of a comparison: the gate statuses, each run's state, warnings and notes from its B1a document, space alignment + /// problems, model-total reconciliation, and - for a run made by this workflow - each producer's exit code and log. + /// + public static string EngineComparisonDiagnostics(EngineComparison engineComparison, EngineComparisonRun? engineComparisonRun = null) + { + ComparisonResult comparisonResult = engineComparison.ComparisonResult; + + List lines = new List + { + string.Format(CultureInfo.InvariantCulture, "Overall gate: {0} (numerical {1}, coverage {2}, provenance {3}, reconciliation {4})", + Format.Gate(comparisonResult.Gate), Format.Gate(comparisonResult.NumericalStatus), Format.Gate(comparisonResult.CoverageStatus), Format.Gate(comparisonResult.ProvenanceStatus), Format.Gate(comparisonResult.ReconciliationStatus)), + string.Format(CultureInfo.InvariantCulture, "Metrics: {0} match, {1} warn, {2} fail, {3} N/A; {4} of {5} required metrics comparable. Tolerance profile '{6}'. A band is a tolerance band, not an engineering verdict (SAM_UI#232/#233).", + comparisonResult.MatchCount, comparisonResult.WarnCount, comparisonResult.FailCount, comparisonResult.NotApplicableCount, comparisonResult.ComparableMetricCount, comparisonResult.RequiredMetricCount, comparisonResult.ToleranceProfileName), + }; + + if (!string.IsNullOrWhiteSpace(comparisonResult.SchemaDriftNote)) + { + lines.Add("Schema: " + comparisonResult.SchemaDriftNote); + } + + SpaceMatchDiagnostics spaceMatchDiagnostics = comparisonResult.SpaceDiagnostics; + lines.Add(string.Empty); + lines.Add(string.Format(CultureInfo.InvariantCulture, "Spaces: Tas {0}, OpenStudio {1}; matched by GUID {2}, by name {3}.", spaceMatchDiagnostics.TasCount, spaceMatchDiagnostics.OpenStudioCount, spaceMatchDiagnostics.MatchedByGuid, spaceMatchDiagnostics.MatchedByName)); + AddList(lines, "Unmatched - only in Tas", spaceMatchDiagnostics.OnlyInTas); + AddList(lines, "Unmatched - only in OpenStudio", spaceMatchDiagnostics.OnlyInOpenStudio); + AddList(lines, "Duplicate GUIDs in Tas", spaceMatchDiagnostics.DuplicateTasGuids); + AddList(lines, "Duplicate GUIDs in OpenStudio", spaceMatchDiagnostics.DuplicateOpenStudioGuids); + AddList(lines, "Duplicate names in Tas", spaceMatchDiagnostics.DuplicateTasNames); + AddList(lines, "Duplicate names in OpenStudio", spaceMatchDiagnostics.DuplicateOpenStudioNames); + AddList(lines, "Ambiguous name matches", spaceMatchDiagnostics.AmbiguousNameMatches); + + if (comparisonResult.Reconciliations.Count != 0) + { + lines.Add(string.Empty); + lines.Add("Model total vs sum of spaces:"); + foreach (ReconciliationResult reconciliationResult in comparisonResult.Reconciliations) + { + lines.Add(string.Format(CultureInfo.InvariantCulture, " {0} {1}: model {2}, sum of {3} spaces {4} {5} -> {6}{7}", + reconciliationResult.Engine, reconciliationResult.Key, reconciliationResult.ModelTotalAvailable ? Format.Number(reconciliationResult.ModelTotal) : "(not reported)", reconciliationResult.ContributingSpaces, + Format.Number(reconciliationResult.SumOfSpaces), Format.Unit(reconciliationResult.Unit), Format.Band(reconciliationResult.Band), reconciliationResult.Complete ? string.Empty : " (incomplete)")); + } + } + + lines.Add(string.Empty); + lines.AddRange(RunDiagnosticLines("Tas", engineComparison.Tas.Provenance)); + lines.Add(string.Empty); + lines.AddRange(RunDiagnosticLines("OpenStudio", engineComparison.OpenStudio.Provenance)); + + lines.AddRange(EngineComparisonRunLines(engineComparisonRun)); + + return string.Join(Environment.NewLine, lines); + } + + private static IEnumerable ProvenanceLines(string engine, string path, BenchmarkProvenance? provenance) + { + yield return engine + ": " + path; + if (provenance == null) + { + yield return " (the document has no provenance)"; + yield break; + } + + yield return string.Format(CultureInfo.InvariantCulture, " Route {0}; engine {1} {2} (SDK {3}); state {4}", provenance.Route, provenance.Engine?.Name ?? "?", provenance.Engine?.Version ?? "?", provenance.Engine?.SdkVersion ?? "-", provenance.State); + yield return " Model: " + (provenance.SourceModelName ?? "?") + " (" + (provenance.SourceModelGuid ?? "?") + ")"; + yield return " Canonical model hash: " + (provenance.CanonicalModelHash ?? "(none)") + " (canonicalization " + (provenance.CanonicalizationVersion ?? "?") + ")"; + yield return " Source file hash: " + (provenance.SourceFileHash ?? "(none)"); + yield return " Weather: " + (provenance.Weather?.Identity ?? "(none)") + ", " + (provenance.Weather?.Hash ?? "(no hash)"); + yield return " Design-day source: " + provenance.DesignDaySource; + yield return string.Format(CultureInfo.InvariantCulture, " Run: {0:u}, {1} s; SAM {2}; runner {3}", provenance.RunTimestampUtc, provenance.DurationSeconds?.ToString("0.#", CultureInfo.InvariantCulture) ?? "?", provenance.SamCommit ?? "?", provenance.RunnerCommit ?? "?"); + } + + private static IEnumerable RunDiagnosticLines(string engine, BenchmarkProvenance? provenance) + { + yield return engine + " run: " + (provenance == null ? "(no provenance)" : provenance.State.ToString()); + if (provenance?.Warnings != null) + { + foreach (string warning in provenance.Warnings) + { + yield return " Warning: " + warning; + } + } + + if (provenance?.Notes != null) + { + foreach (string note in provenance.Notes) + { + yield return " Note: " + note; + } + } + + if (provenance?.ResultSources != null && provenance.ResultSources.Count != 0) + { + yield return " Result sources: " + string.Join(", ", provenance.ResultSources); + } + } + + private static void AddList(List lines, string title, IReadOnlyList values) + { + if (values != null && values.Count != 0) + { + lines.Add(" " + title + ": " + string.Join(", ", values)); + } + } + } +} diff --git a/WPF/SAM.Analytical.UI.WPF/SAM.Analytical.UI.WPF.csproj b/WPF/SAM.Analytical.UI.WPF/SAM.Analytical.UI.WPF.csproj index b91acc6d..08a3176a 100644 --- a/WPF/SAM.Analytical.UI.WPF/SAM.Analytical.UI.WPF.csproj +++ b/WPF/SAM.Analytical.UI.WPF/SAM.Analytical.UI.WPF.csproj @@ -184,6 +184,14 @@ ..\..\..\SAM_OpenStudio\build\SAM.Geometry.OpenStudio.dll + + + ..\..\..\SAM_Validation\build\SAM.Analytical.Benchmark.dll + + + ..\..\..\SAM_Validation\build\SAM.Analytical.Benchmark.Compare.dll + + + + diff --git a/WPF/SAM.Analytical.UI.WPF/Windows/AnalyticalWindow.xaml.cs b/WPF/SAM.Analytical.UI.WPF/Windows/AnalyticalWindow.xaml.cs index 61ca8094..732925c3 100644 --- a/WPF/SAM.Analytical.UI.WPF/Windows/AnalyticalWindow.xaml.cs +++ b/WPF/SAM.Analytical.UI.WPF/Windows/AnalyticalWindow.xaml.cs @@ -63,6 +63,12 @@ public partial class AnalyticalWindow : System.Windows.Window /// what keeps a second click from starting a second run; leaves the button off while it is set. /// private bool openStudioSimulationRunning; + + /// True while a Run Simulation > Both run is going; it does not block the window either. + private bool engineComparisonRunning; + + /// The engine last chosen in Run Simulation, offered again next time. + private SimulationEngineChoice runSimulationChoice = SimulationEngineChoice.Tas; public AnalyticalWindow() { @@ -708,11 +714,23 @@ private void InitializeWindow() RibbonButton_EnergySimulation.LargeImageSource = SAM.Core.UI.WPF.Convert.ToBitmapSource(Properties.Resources.SAM_UI_ModelRun); RibbonButton_EnergySimulation.Click += RibbonButton_EnergySimulation_Click; + RibbonButton_EnergySimulation.ToolTipTitle = "Energy Simulation"; + RibbonButton_EnergySimulation.ToolTipDescription = "The Tas energy simulation of the open model, as before. Also available as Run Simulation > Tas only."; + + RibbonButton_RunSimulation.LargeImageSource = SAM.Core.UI.WPF.Convert.ToBitmapSource(Properties.Resources.SAM_UI_ModelRun); + RibbonButton_RunSimulation.Click += RibbonButton_RunSimulation_Click; + RibbonButton_RunSimulation.ToolTipTitle = "Run Simulation"; + RibbonButton_RunSimulation.ToolTipDescription = "Choose Tas only (the Energy Simulation, unchanged), OpenStudio/EnergyPlus only, or Both: one frozen snapshot simulated by Tas then OpenStudio and compared by SAM_Validation. If one engine fails, the other's results are kept and shown as partially completed."; RibbonButton_OpenStudioSimulation.LargeImageSource = SAM.Core.UI.WPF.Convert.ToBitmapSource(Properties.Resources.SAM_UI_ModelRun); RibbonButton_OpenStudioSimulation.Click += RibbonButton_OpenStudioSimulation_Click; RibbonButton_OpenStudioSimulation.ToolTipTitle = "OpenStudio Simulation"; - RibbonButton_OpenStudioSimulation.ToolTipDescription = "Run an annual OpenStudio/EnergyPlus simulation of a frozen copy of the open model (SAM_OpenStudio). The results are saved as a separate model in a new run folder; the open model and the Tas workflow are not changed. Needs OpenStudio 3.10 (openstudio.exe) installed."; + RibbonButton_CompareTasOpenStudio.LargeImageSource = SAM.Core.UI.WPF.Convert.ToBitmapSource(Properties.Resources.SAM_UI_ModelValidate); + RibbonButton_CompareTasOpenStudio.Click += RibbonButton_CompareTasOpenStudio_Click; + RibbonButton_CompareTasOpenStudio.ToolTipTitle = "Compare Tas / OpenStudio"; + RibbonButton_CompareTasOpenStudio.ToolTipDescription = "Compare existing Tas and OpenStudio B1a documents or a comparison run folder, or run both engines (as Run Simulation > Both). Model and per-space results, differences and bands come from SAM_Validation. The open model and the Tas workflow are not changed."; + + RibbonButton_OpenStudioSimulation.ToolTipDescription = "Run an annual OpenStudio/EnergyPlus simulation of a frozen copy of the open model (SAM_OpenStudio). The results are saved as a separate model in a new run folder; the open model and the Tas workflow are not changed. Needs OpenStudio 3.10 (openstudio.exe) installed. Also available as Run Simulation > OpenStudio/EnergyPlus only."; RibbonButton_Optimisation.LargeImageSource = SAM.Core.UI.WPF.Convert.ToBitmapSource(Properties.Resources.SAM_UI_AlgorithmRun); RibbonButton_Optimisation.Click += RibbonButton_Optimisation_Click; @@ -2272,6 +2290,12 @@ private void RibbonButton_EditWeatherData_Click(object sender, RoutedEventArgs e } private void RibbonButton_EnergySimulation_Click(object sender, RoutedEventArgs e) + { + SimulateTas(); + } + + /// Energy Simulation, and Run Simulation > Tas only: the one Tas path, so both entry points behave the same. + private void SimulateTas() { //uIAnalyticalModel?.EnergySimulation(windowHandle); //The ordinary simulation of the open model. It never completes a Part O run: the open model is the @@ -2282,7 +2306,45 @@ private void RibbonButton_EnergySimulation_Click(object sender, RoutedEventArgs RefreshPartOButtons(); } + private async void RibbonButton_RunSimulation_Click(object sender, RoutedEventArgs e) + { + if (uIAnalyticalModel == null) + { + return; + } + + RunSimulationWindow runSimulationWindow = new RunSimulationWindow(runSimulationChoice, !openStudioSimulationRunning, !engineComparisonRunning) + { + Owner = this, + }; + + if (runSimulationWindow.ShowDialog() != true) + { + return; + } + + runSimulationChoice = runSimulationWindow.EngineChoice; + switch (runSimulationChoice) + { + case SimulationEngineChoice.Tas: + SimulateTas(); + break; + case SimulationEngineChoice.OpenStudio: + await RunOpenStudioSimulationAsync(); + break; + case SimulationEngineChoice.Both: + await RunBothEnginesAsync(); + break; + } + } + private async void RibbonButton_OpenStudioSimulation_Click(object sender, RoutedEventArgs e) + { + await RunOpenStudioSimulationAsync(); + } + + /// OpenStudio Simulation, and Run Simulation > OpenStudio/EnergyPlus only. + private async System.Threading.Tasks.Task RunOpenStudioSimulationAsync() { if (openStudioSimulationRunning || uIAnalyticalModel == null) { @@ -2307,6 +2369,71 @@ private async void RibbonButton_OpenStudioSimulation_Click(object sender, Routed } } + /// + /// Run Simulation > Both: one frozen snapshot, Tas then OpenStudio, SAM_Validation's comparison; the comparison window opens + /// with the run when it finishes, whatever its status (compared, partially completed, failed or cancelled). + /// + private async System.Threading.Tasks.Task RunBothEnginesAsync() + { + if (engineComparisonRunning || uIAnalyticalModel == null) + { + return; + } + + engineComparisonRunning = true; + try + { + EngineComparisonRun? engineComparisonRun = await SimulateBothEngines(); + if (engineComparisonRun != null) + { + ShowEngineComparison(engineComparisonRun); + } + } + //A missing SAM_Validation or SAM_OpenStudio assembly fails when SimulateBothEngines is compiled; say what is missing. + catch (Exception exception) when (Query.IsOpenStudioLoadFailure(exception)) + { + MessageBox.Show(this, "Run Simulation (Both) could not load its components (SAM.Analytical.Benchmark.Compare.dll and SAM.Analytical.Benchmark.dll from SAM_Validation\\build, and the SAM_OpenStudio assemblies).\n\n" + Query.OpenStudioLoadFailure(exception), "Run Simulation", MessageBoxButton.OK, MessageBoxImage.Error); + } + finally + { + engineComparisonRunning = false; + } + } + + [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] + private System.Threading.Tasks.Task SimulateBothEngines() + { + return uIAnalyticalModel!.SimulateBothEnginesAsync(this); + } + + private void RibbonButton_CompareTasOpenStudio_Click(object sender, RoutedEventArgs e) + { + //A missing SAM_Validation or SAM_OpenStudio assembly fails when ShowEngineComparison is compiled; say what is missing. + try + { + ShowEngineComparison(); + } + catch (Exception exception) when (Query.IsOpenStudioLoadFailure(exception)) + { + MessageBox.Show(this, "Compare Tas / OpenStudio could not load its components (SAM.Analytical.Benchmark.Compare.dll and SAM.Analytical.Benchmark.dll from SAM_Validation\\build, and the SAM_OpenStudio assemblies).\n\n" + Query.OpenStudioLoadFailure(exception), "Compare Tas / OpenStudio", MessageBoxButton.OK, MessageBoxImage.Error); + } + } + + [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] + private void ShowEngineComparison(EngineComparisonRun? engineComparisonRun = null) + { + EngineComparisonWindow engineComparisonWindow = new EngineComparisonWindow(uIAnalyticalModel) + { + Owner = this, + }; + + engineComparisonWindow.Show(); + if (engineComparisonRun != null) + { + engineComparisonWindow.ShowRun(engineComparisonRun); + } + } + [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] private System.Threading.Tasks.Task SimulateOpenStudio() { @@ -3244,6 +3371,7 @@ private void SetEnabled() RibbonButton_OpenT3D.IsEnabled = false; RibbonButton_EditLibrary.IsEnabled = false; RibbonButton_EnergySimulation.IsEnabled = false; + RibbonButton_RunSimulation.IsEnabled = false; RibbonButton_OpenStudioSimulation.IsEnabled = false; RibbonButton_SolarSimulation.IsEnabled = false; RibbonButton_ImportWeatherData.IsEnabled = false; @@ -3264,6 +3392,7 @@ private void SetEnabled() //RibbonButton_CreateTBD.IsEnabled = false; RibbonButton_EditLibrary.IsEnabled = false; RibbonButton_EnergySimulation.IsEnabled = false; + RibbonButton_RunSimulation.IsEnabled = false; RibbonButton_OpenStudioSimulation.IsEnabled = false; RibbonButton_SolarSimulation.IsEnabled = false; RibbonButton_ImportWeatherData.IsEnabled = false; @@ -3323,6 +3452,7 @@ private void SetEnabled() RibbonButton_MapInternalConditions.IsEnabled = true; RibbonButton_MapInternalConditionsByTM59.IsEnabled = true; RibbonButton_EnergySimulation.IsEnabled = true; + RibbonButton_RunSimulation.IsEnabled = true; RibbonButton_OpenStudioSimulation.IsEnabled = !openStudioSimulationRunning; RibbonButton_SolarSimulation.IsEnabled = true; RibbonButton_ImportWeatherData.IsEnabled = true; diff --git a/WPF/SAM.Analytical.UI.WPF/Windows/EngineComparisonWindow.xaml b/WPF/SAM.Analytical.UI.WPF/Windows/EngineComparisonWindow.xaml new file mode 100644 index 00000000..42d087e9 --- /dev/null +++ b/WPF/SAM.Analytical.UI.WPF/Windows/EngineComparisonWindow.xaml @@ -0,0 +1,240 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +