From 6b1a08111473d4084010e6317c25ae1c9b366b23 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 1 Sep 2026 13:48:39 +0000 Subject: [PATCH 1/2] feat: write the calibration report as JSON and YAML srcmorph:calibrate built a CalibrationReport and printed it as INFO lines, and that was the whole output. So the numbers a calibration run produces -- prefill and decode throughput, chars per token, per model -- could not be diffed across runs, committed as a baseline, or fed back into aiDefinitions by anything other than a human re-reading the console, which is most of the point of measuring them. execute() now also writes srcmorph-calibration.json and srcmorph-calibration.yaml into the configured outputDirectory, creating it when missing. Both carry the same keys in the same order and add loadSeconds, midPrefillTokensPerSecond and cachedPromptTokens, which the pasteable block does not. The three figures the XML does carry use the identical format strings, so the JSON and the paste block cannot disagree about what was measured -- pinned by a test that asserts both spellings of the same number. The renderers are hand-rolled in CalibrationReport rather than delegated to Jackson: this module is framework-free and carries no JSON dependency (only srcmorph-cli does), and renderXml() already set that precedent. Model keys come from user configuration, so they are escaped for both formats; an unescaped quote would produce a file neither parser accepts. The split follows the PIT boundary rather than convenience: rendering is pure and lives in CalibrationReport, which is on the gate at threshold 100, so the documents are asserted whole rather than field by field -- a per-field check passes on output that is not valid JSON at all, which is exactly how a hand-rolled writer fails. The file writing lives in CalibrateEngine, which is deliberately off the gate as orchestration. Also fixed while here: outputDirectory defaults to the source-tree path src/site/ai, so every existing CalibrateEngineTest would have started leaving two files behind in the checkout. They now use a @TempDir. --- CHANGELOG.md | 18 +++ TODO.md | 9 -- srcmorph-maven-plugin/README.md | 16 ++ .../out-dir/srcmorph-calibration.json | 13 ++ .../out-dir/srcmorph-calibration.yaml | 8 + .../srcmorph/engine/CalibrateEngine.java | 50 +++++- .../srcmorph/engine/CalibrationReport.java | 149 ++++++++++++++++++ .../srcmorph/engine/CalibrateEngineTest.java | 52 ++++++ .../engine/CalibrationReportTest.java | 127 +++++++++++++++ 9 files changed, 432 insertions(+), 10 deletions(-) create mode 100644 srcmorph-maven-plugin/out-dir/srcmorph-calibration.json create mode 100644 srcmorph-maven-plugin/out-dir/srcmorph-calibration.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index c5f43b97..94589f6e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,24 @@ The release procedure (prompt template and step-by-step instructions) lives in [ ## [Unreleased] +### Added +- **`srcmorph:calibrate` now writes a machine-readable report.** The goal built a `CalibrationReport` + and printed it as `INFO` lines, and that was the only output — so the numbers a calibration run + produces (prefill / decode throughput, chars per token per model) could not be diffed across runs, + committed as a baseline, or fed back into `aiDefinitions` by anything other than a human re-reading + the console, which is most of the point of measuring them. + + `execute()` now also writes `srcmorph-calibration.json` and `srcmorph-calibration.yaml` into the + configured `outputDirectory`, creating it if needed. Both carry the same keys in the same order and + add `loadSeconds`, `midPrefillTokensPerSecond` and `cachedPromptTokens`, which the pasteable + `` block does not. The three figures the XML *does* carry are formatted identically, so + the JSON and the paste block can never disagree about what was measured. + + The renderers are hand-rolled in `CalibrationReport` rather than delegated to Jackson: the core + module is framework-free and carries no JSON dependency (only `srcmorph-cli` does), and + `renderXml()` already set that precedent. Model keys come from user configuration, so they are + escaped for both formats — an unescaped quote would produce a file neither parser accepts. + ### Added - **`flashAttn` works.** The knob was documented and settable since it was introduced, but could not be forwarded: `--flash-attn` takes a mandatory `on|off|auto` and `net.ladenthin:llama` offered only diff --git a/TODO.md b/TODO.md index 164a0477..78d4ae1e 100644 --- a/TODO.md +++ b/TODO.md @@ -24,15 +24,6 @@ everything below is genuinely still open. not dropping the idea. Deliberately out of scope for 1.2.0: it is a build-time question, not a correctness one. -- **`srcmorph:calibrate` reports only through the log.** `CalibrateEngine` builds a - `CalibrationReport` and `CalibrateMojo` prints it as `INFO` lines. There is no machine-readable - output, so the numbers a calibration run produces (prefill / decode throughput, chars per token per - model) cannot be diffed across runs, fed back into `aiDefinitions`, or committed as a baseline -- - which is most of the point of measuring them. Emitting the same report as JSON and YAML next to - the log (the CLI already carries both Jackson mappers, and `SrcMorphConfiguration` round-trips - through them) would close it. **Was announced during the 1.2.0 audit cycle and never landed**; - it is a feature, not a fix, so it is not a 1.2.0 blocker. - - **The sixteen GPU classifier fat jars are verified structurally, never launched.** Since 1.2.0 `.github/verify-classifier-fatjars.sh` asserts each is the artifact its name claims (one jar per classifier, a native for the promised OS/arch, a native set that differs from the default jar's, so diff --git a/srcmorph-maven-plugin/README.md b/srcmorph-maven-plugin/README.md index 09dc9540..867a6a00 100644 --- a/srcmorph-maven-plugin/README.md +++ b/srcmorph-maven-plugin/README.md @@ -480,6 +480,22 @@ src/site/ai/ ├── AnotherClass.java.ai.md └── package.ai.md ``` + +The `calibrate` goal writes two more files into the same `outputDirectory`: + +``` +src/site/ai/ +├── srcmorph-calibration.json +└── srcmorph-calibration.yaml +``` + +Both carry one entry per measured model — `modelKey`, `loadSeconds`, +`prefillTokensPerSecond`, `decodeTokensPerSecond`, `charsPerToken`, +`midPrefillTokensPerSecond`, `cachedPromptTokens` — so a calibration run can be diffed +across machines or committed as a baseline, instead of existing only as console output. +The three figures that also appear in the paste-ready `` block are formatted +identically, so the two can never disagree about what was measured. + ## Design Principles - Deterministic metadata (hash-based change detection) - Separation of concerns (header = metadata, body = summary) diff --git a/srcmorph-maven-plugin/out-dir/srcmorph-calibration.json b/srcmorph-maven-plugin/out-dir/srcmorph-calibration.json new file mode 100644 index 00000000..52b44881 --- /dev/null +++ b/srcmorph-maven-plugin/out-dir/srcmorph-calibration.json @@ -0,0 +1,13 @@ +{ + "models": [ + { + "modelKey": "mock-model", + "loadSeconds": 0.002, + "prefillTokensPerSecond": 1000.0, + "decodeTokensPerSecond": 100.0, + "charsPerToken": 4.00, + "midPrefillTokensPerSecond": 1000.0, + "cachedPromptTokens": 32 + } + ] +} diff --git a/srcmorph-maven-plugin/out-dir/srcmorph-calibration.yaml b/srcmorph-maven-plugin/out-dir/srcmorph-calibration.yaml new file mode 100644 index 00000000..6ea25325 --- /dev/null +++ b/srcmorph-maven-plugin/out-dir/srcmorph-calibration.yaml @@ -0,0 +1,8 @@ +models: + - modelKey: "mock-model" + loadSeconds: 0.002 + prefillTokensPerSecond: 1000.0 + decodeTokensPerSecond: 100.0 + charsPerToken: 4.00 + midPrefillTokensPerSecond: 1000.0 + cachedPromptTokens: 32 diff --git a/srcmorph/src/main/java/net/ladenthin/srcmorph/engine/CalibrateEngine.java b/srcmorph/src/main/java/net/ladenthin/srcmorph/engine/CalibrateEngine.java index 77e9beb1..15a5dae1 100644 --- a/srcmorph/src/main/java/net/ladenthin/srcmorph/engine/CalibrateEngine.java +++ b/srcmorph/src/main/java/net/ladenthin/srcmorph/engine/CalibrateEngine.java @@ -4,6 +4,11 @@ package net.ladenthin.srcmorph.engine; import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.Writer; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; @@ -103,7 +108,50 @@ public CalibrationReport execute() throws SrcMorphException { providerFactory)); } - return new CalibrationReport(measurements); + final CalibrationReport report = new CalibrationReport(measurements); + writeMachineReadableReport(report); + return report; + } + + /** + * Writes the report next to the log, as JSON and YAML, into the configured output directory. + * + *

Without this the numbers a calibration run produces exist only as log lines: they cannot be + * diffed across runs, committed as a baseline, or fed back into {@code aiDefinitions} by anything + * other than a human re-reading the console — which is most of the point of measuring + * them.

+ * + * @param report the report to write + * @throws SrcMorphException if the directory cannot be created or a file cannot be written + */ + private void writeMachineReadableReport(final CalibrationReport report) throws SrcMorphException { + final Path directory = config.getOutputDirectory().toPath(); + final Path json = directory.resolve(CalibrationReport.JSON_FILE_NAME); + final Path yaml = directory.resolve(CalibrationReport.YAML_FILE_NAME); + try { + Files.createDirectories(directory); + writeUtf8(json, report.renderJson()); + writeUtf8(yaml, report.renderYaml()); + } catch (final IOException e) { + throw new SrcMorphException("Failed to write the calibration report to " + directory + ": " + e, e); + } + LOGGER.info(""); + LOGGER.info("Machine-readable calibration report written to:"); + LOGGER.info(" {}", json); + LOGGER.info(" {}", yaml); + } + + /** + * Writes UTF-8 text to a file, replacing any previous content. + * + * @param target the file + * @param text the content + * @throws IOException if the write fails + */ + private static void writeUtf8(final Path target, final String text) throws IOException { + try (Writer writer = new OutputStreamWriter(Files.newOutputStream(target), StandardCharsets.UTF_8)) { + writer.write(text); + } } /** diff --git a/srcmorph/src/main/java/net/ladenthin/srcmorph/engine/CalibrationReport.java b/srcmorph/src/main/java/net/ladenthin/srcmorph/engine/CalibrationReport.java index b1ad648a..46847217 100644 --- a/srcmorph/src/main/java/net/ladenthin/srcmorph/engine/CalibrationReport.java +++ b/srcmorph/src/main/java/net/ladenthin/srcmorph/engine/CalibrationReport.java @@ -57,6 +57,155 @@ public String renderXml() { return out.toString(); } + /** + * File name of the machine-readable JSON report, written by {@link CalibrateEngine#execute()} + * into the configured output directory. + */ + public static final String JSON_FILE_NAME = "srcmorph-calibration.json"; + + /** File name of the machine-readable YAML report, written alongside {@link #JSON_FILE_NAME}. */ + public static final String YAML_FILE_NAME = "srcmorph-calibration.yaml"; + + /** Format of the two throughput figures; identical to the one {@link #renderXml()} pastes. */ + private static final String FORMAT_TOKENS_PER_SECOND = "%.1f"; + + /** Format of the chars-per-token figure; identical to the one {@link #renderXml()} pastes. */ + private static final String FORMAT_CHARS_PER_TOKEN = "%.2f"; + + /** Format of the load duration, which has no counterpart in the XML block. */ + private static final String FORMAT_LOAD_SECONDS = "%.3f"; + + /** + * Renders the report as JSON. + * + *

Hand-rolled rather than delegated to Jackson on purpose: this module is framework-free and + * carries no JSON dependency (only {@code srcmorph-cli} does), and {@link #renderXml()} sets the + * same precedent. The document is small and fully determined by six numbers and a key.

+ * + *

The three figures that also appear in the {@code } block are formatted + * identically, so the JSON and the paste-ready XML can never disagree about what was measured.

+ * + * @return the JSON document; a {@code models} array that is empty when no model was measured + */ + public String renderJson() { + if (measurements.isEmpty()) { + return "{\n \"models\": []\n}\n"; + } + final StringBuilder out = new StringBuilder("{\n \"models\": [\n"); + for (int i = 0; i < measurements.size(); i++) { + final ModelMeasurement entry = measurements.get(i); + final AiCalibrationMeasurement m = entry.measurement(); + out.append(" {\n"); + out.append(" \"modelKey\": ").append(quote(entry.modelKey())).append(",\n"); + appendJsonNumber(out, "loadSeconds", FORMAT_LOAD_SECONDS, m.loadSeconds()); + appendJsonNumber(out, "prefillTokensPerSecond", FORMAT_TOKENS_PER_SECOND, m.prefillTokensPerSecond()); + appendJsonNumber(out, "decodeTokensPerSecond", FORMAT_TOKENS_PER_SECOND, m.decodeTokensPerSecond()); + appendJsonNumber(out, "charsPerToken", FORMAT_CHARS_PER_TOKEN, m.charsPerToken()); + appendJsonNumber(out, "midPrefillTokensPerSecond", FORMAT_TOKENS_PER_SECOND, m.midPrefillTokensPerSecond()); + out.append(" \"cachedPromptTokens\": ").append(m.cachedPromptTokens()).append('\n'); + out.append(" }"); + out.append(i + 1 < measurements.size() ? ",\n" : "\n"); + } + out.append(" ]\n}\n"); + return out.toString(); + } + + /** + * Renders the report as YAML, carrying exactly the same keys, order and number formatting as + * {@link #renderJson()}. + * + *

Every scalar is emitted double-quoted or numeric, which keeps the output inside the subset of + * YAML that needs no emitter to get right — no block scalars, no anchors, no indentation + * subtleties beyond a fixed two-level indent.

+ * + * @return the YAML document; {@code models: []} when no model was measured + */ + public String renderYaml() { + if (measurements.isEmpty()) { + return "models: []\n"; + } + final StringBuilder out = new StringBuilder("models:\n"); + for (final ModelMeasurement entry : measurements) { + final AiCalibrationMeasurement m = entry.measurement(); + out.append(" - modelKey: ").append(quote(entry.modelKey())).append('\n'); + appendYamlNumber(out, "loadSeconds", FORMAT_LOAD_SECONDS, m.loadSeconds()); + appendYamlNumber(out, "prefillTokensPerSecond", FORMAT_TOKENS_PER_SECOND, m.prefillTokensPerSecond()); + appendYamlNumber(out, "decodeTokensPerSecond", FORMAT_TOKENS_PER_SECOND, m.decodeTokensPerSecond()); + appendYamlNumber(out, "charsPerToken", FORMAT_CHARS_PER_TOKEN, m.charsPerToken()); + appendYamlNumber(out, "midPrefillTokensPerSecond", FORMAT_TOKENS_PER_SECOND, m.midPrefillTokensPerSecond()); + out.append(" cachedPromptTokens: ").append(m.cachedPromptTokens()).append('\n'); + } + return out.toString(); + } + + /** + * Appends one {@code "key": number,} line to the JSON buffer. + * + * @param out the buffer + * @param key the JSON key + * @param format the {@link String#format} pattern for the value + * @param value the value + */ + private static void appendJsonNumber( + final StringBuilder out, final String key, final String format, final double value) { + out.append(" \"") + .append(key) + .append("\": ") + .append(String.format(Locale.ROOT, format, value)) + .append(",\n"); + } + + /** + * Appends one {@code key: number} line to the YAML buffer. + * + * @param out the buffer + * @param key the YAML key + * @param format the {@link String#format} pattern for the value + * @param value the value + */ + private static void appendYamlNumber( + final StringBuilder out, final String key, final String format, final double value) { + out.append(" ") + .append(key) + .append(": ") + .append(String.format(Locale.ROOT, format, value)) + .append('\n'); + } + + /** + * Renders a string as a double-quoted scalar that is valid in both JSON and YAML. + * + *

A model key comes from user configuration, so it can carry a quote, a backslash or a control + * character; without escaping, one such key would produce a file neither parser accepts.

+ * + * @param value the string + * @return the quoted, escaped scalar + */ + private static String quote(final String value) { + final StringBuilder out = new StringBuilder(value.length() + 2); + out.append('"'); + for (int i = 0; i < value.length(); i++) { + final char c = value.charAt(i); + if (c == '"') { + out.append("\\\""); + } else if (c == '\\') { + out.append("\\\\"); + } else if (c == '\n') { + out.append("\\n"); + } else if (c == '\r') { + out.append("\\r"); + } else if (c == '\t') { + out.append("\\t"); + } else if (c < ' ') { + out.append(String.format(Locale.ROOT, "\\u%04x", (int) c)); + } else { + out.append(c); + } + } + out.append('"'); + return out.toString(); + } + /** * Appends a copy-pasteable {@code } block (with a comment naming the model) to the buffer. * diff --git a/srcmorph/src/test/java/net/ladenthin/srcmorph/engine/CalibrateEngineTest.java b/srcmorph/src/test/java/net/ladenthin/srcmorph/engine/CalibrateEngineTest.java index a12fdc94..87fa506b 100644 --- a/srcmorph/src/test/java/net/ladenthin/srcmorph/engine/CalibrateEngineTest.java +++ b/srcmorph/src/test/java/net/ladenthin/srcmorph/engine/CalibrateEngineTest.java @@ -8,12 +8,16 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.jupiter.api.Assertions.assertThrows; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.Collections; import net.ladenthin.srcmorph.CommonTestFixtures; import net.ladenthin.srcmorph.config.AiFieldGenerationConfig; import net.ladenthin.srcmorph.config.AiModelDefinition; import net.ladenthin.srcmorph.config.SrcMorphConfiguration; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; public class CalibrateEngineTest { @@ -31,8 +35,17 @@ private static AiModelDefinition mockModelDefinition() { return definition; } + /** + * Per-test output directory. {@code execute()} writes the machine-readable report into + * {@code outputDirectory}, whose default is the source-tree path {@code src/site/ai} -- without + * this every calibration test would leave two files behind in the checkout. + */ + @TempDir + Path outputDirectory; + private SrcMorphConfiguration baseConfig() { final SrcMorphConfiguration config = new SrcMorphConfiguration(); + config.setOutputDirectory(outputDirectory.toFile()); config.setGenerationProvider("mock"); config.setPromptDefinitions(CommonTestFixtures.createFilePromptDefinitions()); config.setAiDefinitions(Collections.singletonList(mockModelDefinition())); @@ -94,4 +107,43 @@ public void execute_dedupesMultipleRulesRoutedToTheSameModel() throws Exception assertThat(report.measurements().size(), is(1)); } + + /** + * The point of the feature: a calibration run's numbers must survive as something diffable and + * committable, not only as log lines a human has to re-read. Asserted end to end through the real + * engine -- rendering is covered separately in {@link CalibrationReportTest}, but only this proves + * the files are actually written, to the documented names, with the documented content. + */ + @Test + public void execute_writesTheReportAsJsonAndYamlIntoTheOutputDirectory() throws Exception { + final SrcMorphConfiguration config = baseConfig(); + config.setFieldGenerations(CommonTestFixtures.createFileFieldGenerations()); + + final CalibrationReport report = new CalibrateEngine(config).execute(); + + final Path json = outputDirectory.resolve(CalibrationReport.JSON_FILE_NAME); + final Path yaml = outputDirectory.resolve(CalibrationReport.YAML_FILE_NAME); + assertThat(Files.exists(json), is(true)); + assertThat(Files.exists(yaml), is(true)); + + // Byte-for-byte what the report renders -- so the file cannot drift from the API. + assertThat(new String(Files.readAllBytes(json), StandardCharsets.UTF_8), is(report.renderJson())); + assertThat(new String(Files.readAllBytes(yaml), StandardCharsets.UTF_8), is(report.renderYaml())); + assertThat( + new String(Files.readAllBytes(json), StandardCharsets.UTF_8), + containsString("\"prefillTokensPerSecond\": 1000.0")); + } + + /** A missing output directory is created rather than being a reason to fail the run. */ + @Test + public void execute_createsTheOutputDirectoryWhenItDoesNotExist() throws Exception { + final SrcMorphConfiguration config = baseConfig(); + final Path nested = outputDirectory.resolve("does/not/exist/yet"); + config.setOutputDirectory(nested.toFile()); + config.setFieldGenerations(CommonTestFixtures.createFileFieldGenerations()); + + new CalibrateEngine(config).execute(); + + assertThat(Files.exists(nested.resolve(CalibrationReport.JSON_FILE_NAME)), is(true)); + } } diff --git a/srcmorph/src/test/java/net/ladenthin/srcmorph/engine/CalibrationReportTest.java b/srcmorph/src/test/java/net/ladenthin/srcmorph/engine/CalibrationReportTest.java index 3578cab6..89e3b2ac 100644 --- a/srcmorph/src/test/java/net/ladenthin/srcmorph/engine/CalibrationReportTest.java +++ b/srcmorph/src/test/java/net/ladenthin/srcmorph/engine/CalibrationReportTest.java @@ -3,12 +3,15 @@ // SPDX-License-Identifier: Apache-2.0 package net.ladenthin.srcmorph.engine; +import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.not; import static org.hamcrest.MatcherAssert.assertThat; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; import net.ladenthin.srcmorph.indexer.AiCalibrationMeasurement; import org.junit.jupiter.api.Test; @@ -90,4 +93,128 @@ public void toString_containsModelKey() { "unique-model-key", new AiCalibrationMeasurement(0.0d, 0.0d, 0.0d, 0.0d, 0.0d, 0)))); assertThat(report.toString(), org.hamcrest.CoreMatchers.containsString("unique-model-key")); } + + // + + private static CalibrationReport reportOf(final CalibrationReport.ModelMeasurement... entries) { + return new CalibrationReport(Arrays.asList(entries)); + } + + private static CalibrationReport.ModelMeasurement entry(final String key) { + return new CalibrationReport.ModelMeasurement( + key, new AiCalibrationMeasurement(1.2345d, 1234.56d, 45.67d, 3.214d, 1200.4d, 128)); + } + + /** + * Pinned as a whole document, not field by field. A per-field assertion would pass on output that + * is not valid JSON at all -- a missing brace, a stray comma -- which is precisely the failure + * mode a hand-rolled writer has. + */ + @Test + public void renderJson_singleModel_isExactlyThisDocument() { + assertThat( + reportOf(entry("qwen35-4b")).renderJson(), + is(equalTo("{\n" + " \"models\": [\n" + + " {\n" + + " \"modelKey\": \"qwen35-4b\",\n" + + " \"loadSeconds\": 1.235,\n" + + " \"prefillTokensPerSecond\": 1234.6,\n" + + " \"decodeTokensPerSecond\": 45.7,\n" + + " \"charsPerToken\": 3.21,\n" + + " \"midPrefillTokensPerSecond\": 1200.4,\n" + + " \"cachedPromptTokens\": 128\n" + + " }\n" + + " ]\n" + + "}\n"))); + } + + /** The separator between entries is the one thing a single-model test cannot see. */ + @Test + public void renderJson_twoModels_separatesEntriesWithACommaAndNoTrailingComma() { + final String json = reportOf(entry("a"), entry("b")).renderJson(); + + assertThat(json, containsString(" },\n {\n")); + assertThat(json, not(containsString(" },\n ]"))); + } + + /** An empty run must still produce a parseable document, not a dangling array. */ + @Test + public void renderJson_noModels_isAnEmptyArray() { + assertThat(new CalibrationReport(Collections.emptyList()).renderJson(), is(equalTo("{\n \"models\": []\n}\n"))); + } + + @Test + public void renderYaml_singleModel_isExactlyThisDocument() { + assertThat( + reportOf(entry("qwen35-4b")).renderYaml(), + is(equalTo("models:\n" + " - modelKey: \"qwen35-4b\"\n" + + " loadSeconds: 1.235\n" + + " prefillTokensPerSecond: 1234.6\n" + + " decodeTokensPerSecond: 45.7\n" + + " charsPerToken: 3.21\n" + + " midPrefillTokensPerSecond: 1200.4\n" + + " cachedPromptTokens: 128\n"))); + } + + @Test + public void renderYaml_twoModels_emitsOneSequenceItemPerModel() { + final String yaml = reportOf(entry("a"), entry("b")).renderYaml(); + + assertThat(yaml, containsString(" - modelKey: \"a\"\n")); + assertThat(yaml, containsString(" - modelKey: \"b\"\n")); + } + + @Test + public void renderYaml_noModels_isAnEmptySequence() { + assertThat(new CalibrationReport(Collections.emptyList()).renderYaml(), is(equalTo("models: []\n"))); + } + + /** + * The numbers the two documents share must be byte-identical to the ones {@link + * CalibrationReport#renderXml()} pastes, or a user comparing the JSON against the XML block would + * see two different measurements of the same run. + */ + @Test + public void renderJson_reusesTheExactNumberFormattingOfTheXmlBlock() { + final CalibrationReport report = reportOf(entry("m")); + final String xml = report.renderXml(); + final String json = report.renderJson(); + + assertThat(xml, containsString("1234.6")); + assertThat(json, containsString("\"prefillTokensPerSecond\": 1234.6,")); + assertThat(xml, containsString("3.21")); + assertThat(json, containsString("\"charsPerToken\": 3.21,")); + } + + /** + * A model key comes from user configuration. Unescaped, a single quote in it produces a file + * neither parser accepts -- so every escape branch is exercised, including the numeric fallback + * for control characters and the boundary just above it. + */ + @Test + public void renderJson_modelKeyWithSpecialCharacters_isEscapedForBothFormats() { + final String key = "a\"b\\c\nd\re\tf\u0001g h"; + + final String json = reportOf(entry(key)).renderJson(); + final String yaml = reportOf(entry(key)).renderYaml(); + final String expected = "\"a\\\"b\\\\c\\nd\\re\\tf\\u0001g h\""; + + assertThat(json, containsString("\"modelKey\": " + expected + ",")); + assertThat(yaml, containsString(" - modelKey: " + expected + "\n")); + } + + /** The space is the boundary of the {@code c < ' '} control-character branch: it must pass through. */ + @Test + public void renderJson_spaceIsNotEscaped() { + assertThat(reportOf(entry("two words")).renderJson(), containsString("\"modelKey\": \"two words\",")); + } + + /** The two file names are part of the contract with CalibrateEngine and its documentation. */ + @Test + public void fileNames_areTheDocumentedOnes() { + assertThat(CalibrationReport.JSON_FILE_NAME, is("srcmorph-calibration.json")); + assertThat(CalibrationReport.YAML_FILE_NAME, is("srcmorph-calibration.yaml")); + } + + // } From 256cab7fb88d6fa77485436b52e2a5298a70f4cc Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 1 Sep 2026 13:49:56 +0000 Subject: [PATCH 2/2] fix(test): stop the calibrate goal writing into the checkout The previous commit shipped two generated files -- srcmorph-maven-plugin/out-dir/srcmorph-calibration.{json,yaml} -- into the repository. They are output, not source. Root cause, and it is the same one the engine tests already hit: execute() now writes the report into outputDirectory, and MojoConfigurationMappingTest's fillSharedParameters points that at the relative path "out-dir". So calibrateMojo_execute_logsTheInstructionsAndTheRenderedReport left two files in the working tree on every run, and 'git add -A' swept them in. The test now uses a @TempDir like the engine tests, and the two committed files are removed. What actually went wrong on my side: 'git add -A' on a tree I had not looked at first. The untracked directory was listed in the status output of that very commit. --- .../out-dir/srcmorph-calibration.json | 13 ------------- .../out-dir/srcmorph-calibration.yaml | 8 -------- .../srcmorph/mojo/MojoConfigurationMappingTest.java | 10 ++++++++++ 3 files changed, 10 insertions(+), 21 deletions(-) delete mode 100644 srcmorph-maven-plugin/out-dir/srcmorph-calibration.json delete mode 100644 srcmorph-maven-plugin/out-dir/srcmorph-calibration.yaml diff --git a/srcmorph-maven-plugin/out-dir/srcmorph-calibration.json b/srcmorph-maven-plugin/out-dir/srcmorph-calibration.json deleted file mode 100644 index 52b44881..00000000 --- a/srcmorph-maven-plugin/out-dir/srcmorph-calibration.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "models": [ - { - "modelKey": "mock-model", - "loadSeconds": 0.002, - "prefillTokensPerSecond": 1000.0, - "decodeTokensPerSecond": 100.0, - "charsPerToken": 4.00, - "midPrefillTokensPerSecond": 1000.0, - "cachedPromptTokens": 32 - } - ] -} diff --git a/srcmorph-maven-plugin/out-dir/srcmorph-calibration.yaml b/srcmorph-maven-plugin/out-dir/srcmorph-calibration.yaml deleted file mode 100644 index 6ea25325..00000000 --- a/srcmorph-maven-plugin/out-dir/srcmorph-calibration.yaml +++ /dev/null @@ -1,8 +0,0 @@ -models: - - modelKey: "mock-model" - loadSeconds: 0.002 - prefillTokensPerSecond: 1000.0 - decodeTokensPerSecond: 100.0 - charsPerToken: 4.00 - midPrefillTokensPerSecond: 1000.0 - cachedPromptTokens: 32 diff --git a/srcmorph-maven-plugin/src/test/java/net/ladenthin/maven/srcmorph/mojo/MojoConfigurationMappingTest.java b/srcmorph-maven-plugin/src/test/java/net/ladenthin/maven/srcmorph/mojo/MojoConfigurationMappingTest.java index b37c3001..33143987 100644 --- a/srcmorph-maven-plugin/src/test/java/net/ladenthin/maven/srcmorph/mojo/MojoConfigurationMappingTest.java +++ b/srcmorph-maven-plugin/src/test/java/net/ladenthin/maven/srcmorph/mojo/MojoConfigurationMappingTest.java @@ -9,6 +9,7 @@ import static org.hamcrest.MatcherAssert.assertThat; import java.io.File; +import java.nio.file.Path; import java.lang.reflect.Field; import java.util.Arrays; import java.util.Collections; @@ -19,6 +20,7 @@ import net.ladenthin.srcmorph.config.SrcMorphConfiguration; import net.ladenthin.srcmorph.prompt.AiPromptDefinition; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; /** * Covers each concrete goal's own {@code @Parameter} mapping. @@ -35,6 +37,10 @@ */ public class MojoConfigurationMappingTest { + /** Per-test output directory, so a goal that writes files cannot pollute the checkout. */ + @TempDir + Path outputDirectory; + // /** The goal parameters are private, so the same reflective assignment the sibling skip test uses. */ @@ -199,6 +205,10 @@ public void calibrateMojo_execute_logsTheInstructionsAndTheRenderedReport() thro final CalibrateMojo mojo = new CalibrateMojo(); fillSharedParameters(mojo); wireMockModelAndRule(mojo); + // execute() writes the machine-readable report into outputDirectory; fillSharedParameters + // points that at the relative path "out-dir", so without this the goal leaves two files + // behind in the checkout on every run. + mojo.outputDirectory = outputDirectory.toFile(); final CapturingLog log = new CapturingLog(); mojo.setLog(log);