From 37aca67a7e899f64474771d1d306c6e3bd9f44f7 Mon Sep 17 00:00:00 2001 From: Enksodsoon <229880765+Enksodsoon@users.noreply.github.com> Date: Sat, 22 Aug 2026 02:05:54 +0700 Subject: [PATCH 01/50] feat: add autonomous Evidence Mentor runner --- contracts/evidence-mentor/README.md | 9 + .../pathlab-ai-evidence-v1.schema.json | 30 ++ .../pathlab-ai-pack-v1.schema.json | 22 ++ .../pathlab-knowledge-pack-v1.schema.json | 16 ++ docs/evidence-mentor-runner.md | 23 ++ scripts/evidence-mentor-task.ps1 | 71 +++++ scripts/submit-evidence-job.ps1 | 51 ++++ .../evidence/BrightfieldTileAnalyzer.java | 103 +++++++ .../forge/evidence/EvidenceArtifactStore.java | 49 ++++ .../forge/evidence/EvidenceBundleWriter.java | 113 ++++++++ .../pathlab/forge/evidence/EvidenceJob.java | 34 +++ .../forge/evidence/EvidenceJobProcessor.java | 257 ++++++++++++++++++ .../forge/evidence/EvidenceJobQueue.java | 255 +++++++++++++++++ .../forge/evidence/EvidenceJobState.java | 18 ++ .../forge/evidence/EvidenceMentorRunner.java | 241 ++++++++++++++++ .../forge/evidence/EvidencePackManifest.java | 186 +++++++++++++ .../forge/evidence/EvidenceSigner.java | 78 ++++++ .../viewer/PrivateResultsBundleBuilder.java | 16 ++ .../forge/viewer/ViewerPairingService.java | 7 +- .../evidence-packs/cell-od-watershed-v1.json | 34 +++ .../evidence-packs/he-dinov2-small-v1.json | 33 +++ .../he-gigapath-benchmark-v1.json | 33 +++ .../evidence-packs/he-hibou-b-v1.json | 33 +++ .../evidence-packs/ihc-descriptive-v1.json | 41 +++ .../evidence/BrightfieldTileAnalyzerTest.java | 41 +++ .../evidence/EvidenceArtifactStoreTest.java | 35 +++ .../evidence/EvidenceBundleWriterTest.java | 58 ++++ .../evidence/EvidenceJobProcessorTest.java | 170 ++++++++++++ .../forge/evidence/EvidenceJobQueueTest.java | 55 ++++ .../evidence/EvidenceMentorRunnerTest.java | 52 ++++ .../evidence/EvidencePackManifestTest.java | 70 +++++ .../PrivateResultsBundleBuilderTest.java | 14 + 32 files changed, 2247 insertions(+), 1 deletion(-) create mode 100644 contracts/evidence-mentor/README.md create mode 100644 contracts/evidence-mentor/pathlab-ai-evidence-v1.schema.json create mode 100644 contracts/evidence-mentor/pathlab-ai-pack-v1.schema.json create mode 100644 contracts/evidence-mentor/pathlab-knowledge-pack-v1.schema.json create mode 100644 docs/evidence-mentor-runner.md create mode 100644 scripts/evidence-mentor-task.ps1 create mode 100644 scripts/submit-evidence-job.ps1 create mode 100644 src/main/java/org/pathlab/forge/evidence/BrightfieldTileAnalyzer.java create mode 100644 src/main/java/org/pathlab/forge/evidence/EvidenceArtifactStore.java create mode 100644 src/main/java/org/pathlab/forge/evidence/EvidenceBundleWriter.java create mode 100644 src/main/java/org/pathlab/forge/evidence/EvidenceJob.java create mode 100644 src/main/java/org/pathlab/forge/evidence/EvidenceJobProcessor.java create mode 100644 src/main/java/org/pathlab/forge/evidence/EvidenceJobQueue.java create mode 100644 src/main/java/org/pathlab/forge/evidence/EvidenceJobState.java create mode 100644 src/main/java/org/pathlab/forge/evidence/EvidenceMentorRunner.java create mode 100644 src/main/java/org/pathlab/forge/evidence/EvidencePackManifest.java create mode 100644 src/main/java/org/pathlab/forge/evidence/EvidenceSigner.java create mode 100644 src/main/resources/evidence-packs/cell-od-watershed-v1.json create mode 100644 src/main/resources/evidence-packs/he-dinov2-small-v1.json create mode 100644 src/main/resources/evidence-packs/he-gigapath-benchmark-v1.json create mode 100644 src/main/resources/evidence-packs/he-hibou-b-v1.json create mode 100644 src/main/resources/evidence-packs/ihc-descriptive-v1.json create mode 100644 src/test/java/org/pathlab/forge/evidence/BrightfieldTileAnalyzerTest.java create mode 100644 src/test/java/org/pathlab/forge/evidence/EvidenceArtifactStoreTest.java create mode 100644 src/test/java/org/pathlab/forge/evidence/EvidenceBundleWriterTest.java create mode 100644 src/test/java/org/pathlab/forge/evidence/EvidenceJobProcessorTest.java create mode 100644 src/test/java/org/pathlab/forge/evidence/EvidenceJobQueueTest.java create mode 100644 src/test/java/org/pathlab/forge/evidence/EvidenceMentorRunnerTest.java create mode 100644 src/test/java/org/pathlab/forge/evidence/EvidencePackManifestTest.java diff --git a/contracts/evidence-mentor/README.md b/contracts/evidence-mentor/README.md new file mode 100644 index 0000000..f59f72e --- /dev/null +++ b/contracts/evidence-mentor/README.md @@ -0,0 +1,9 @@ +# Evidence Mentor contracts + +These schemas are the cross-repository truth boundary for the private research MVP. + +- `pathlab.ai-pack/1` describes a locally installed expert pack and its rights/resource gates. +- `pathlab.ai-evidence/1` is the immutable, signed, slide-bound result accepted by Viewer. +- `pathlab.knowledge-pack/1` contains reviewed atomic claims; the tutor may select claim IDs but may not invent factual prose. + +Raw WSI pixels and reusable embeddings are never valid evidence-bundle members. All contracts are research-only and fail closed. diff --git a/contracts/evidence-mentor/pathlab-ai-evidence-v1.schema.json b/contracts/evidence-mentor/pathlab-ai-evidence-v1.schema.json new file mode 100644 index 0000000..debc9c5 --- /dev/null +++ b/contracts/evidence-mentor/pathlab-ai-evidence-v1.schema.json @@ -0,0 +1,30 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://pathlab.local/schemas/pathlab-ai-evidence-v1.schema.json", + "title": "PathLab AI Evidence v1", + "type": "object", + "additionalProperties": false, + "required": ["schema", "bundleId", "source", "pack", "status", "researchOnly", "notDiagnostic", "reviewRequired", "coordinates", "evidence", "cellAggregates", "ihcDescriptors", "citations", "qc", "provenance", "manifestSha256", "signature"], + "properties": { + "schema": {"const": "pathlab.ai-evidence/1"}, + "bundleId": {"type": "string", "pattern": "^[A-Za-z0-9._-]{1,120}$"}, + "source": {"type": "object", "additionalProperties": false, "required": ["slideSha256", "revision"], "properties": {"slideSha256": {"type": "string", "pattern": "^[a-f0-9]{64}$"}, "revision": {"type": "string", "minLength": 1, "maxLength": 160}}}, + "pack": {"type": "object", "additionalProperties": false, "required": ["id", "version", "manifestSha256", "preprocessing", "artifacts", "allowedUse", "validationStatus"], "properties": {"id": {"type": "string", "maxLength": 120}, "version": {"type": "string", "maxLength": 64}, "manifestSha256": {"type": "string", "pattern": "^[a-f0-9]{64}$"}, "preprocessing": {"type": "string", "maxLength": 120}, "artifacts": {"type": "array", "maxItems": 16, "items": {"type": "string", "pattern": "^[a-f0-9]{64}$"}}, "allowedUse": {"const": "private-research"}, "validationStatus": {"enum": ["experimental", "qualified"]}}}, + "status": {"enum": ["completed", "partial", "abstained", "unsupported", "failed"]}, + "researchOnly": {"const": true}, + "notDiagnostic": {"const": true}, + "reviewRequired": {"const": true}, + "coordinates": {"type": "object", "additionalProperties": false, "required": ["space", "originX", "originY", "scaleX", "scaleY"], "properties": {"space": {"const": "source-pixel"}, "originX": {"type": "number", "minimum": 0}, "originY": {"type": "number", "minimum": 0}, "scaleX": {"type": "number", "exclusiveMinimum": 0}, "scaleY": {"type": "number", "exclusiveMinimum": 0}}}, + "evidence": {"type": "array", "maxItems": 1000, "items": {"$ref": "#/$defs/region"}}, + "cellAggregates": {"type": "array", "maxItems": 1000, "items": {"type": "object", "additionalProperties": false, "required": ["regionId", "algorithm", "count", "densityPerMm2", "meanNucleusAreaPx2"], "properties": {"regionId": {"type": "string", "maxLength": 120}, "algorithm": {"enum": ["hovernet-fast", "od-watershed"]}, "count": {"type": "integer", "minimum": 0}, "densityPerMm2": {"type": ["number", "null"], "minimum": 0}, "meanNucleusAreaPx2": {"type": ["number", "null"], "minimum": 0}}}}, + "ihcDescriptors": {"type": "array", "maxItems": 1000, "items": {"type": "object", "additionalProperties": false, "required": ["regionId", "marker", "compartment", "dabAreaFraction", "meanDabOd", "researchEstimate"], "properties": {"regionId": {"type": "string", "maxLength": 120}, "marker": {"enum": ["generic", "er", "pr", "ki-67", "her2", "pd-l1"]}, "compartment": {"enum": ["nuclear", "membrane", "tumor-immune-region", "generic-region"]}, "dabAreaFraction": {"type": "number", "minimum": 0, "maximum": 1}, "meanDabOd": {"type": "number", "minimum": 0}, "researchEstimate": {"const": true}}}}, + "citations": {"type": "array", "maxItems": 100, "items": {"type": "object", "additionalProperties": false, "required": ["claimId", "sourceUrl"], "properties": {"claimId": {"type": "string", "maxLength": 160}, "sourceUrl": {"type": "string", "format": "uri", "maxLength": 1000}}}}, + "qc": {"type": "object", "additionalProperties": false, "required": ["focus", "tissueFraction", "uncertainty", "abstentionReasons"], "properties": {"focus": {"type": "number", "minimum": 0, "maximum": 1}, "tissueFraction": {"type": "number", "minimum": 0, "maximum": 1}, "uncertainty": {"type": "number", "minimum": 0, "maximum": 1}, "abstentionReasons": {"type": "array", "maxItems": 20, "items": {"type": "string", "maxLength": 500}}}}, + "provenance": {"type": "object", "additionalProperties": false, "required": ["createdAt", "codeRevision", "offlineAnalysis"], "properties": {"createdAt": {"type": "string", "format": "date-time"}, "codeRevision": {"type": "string", "maxLength": 160}, "offlineAnalysis": {"const": true}}}, + "manifestSha256": {"type": "string", "pattern": "^[a-f0-9]{64}$"}, + "signature": {"type": "object", "additionalProperties": false, "required": ["algorithm", "keyId", "publicKeyDer", "value"], "properties": {"algorithm": {"const": "Ed25519"}, "keyId": {"type": "string", "maxLength": 160}, "publicKeyDer": {"type": "string", "minLength": 1}, "value": {"type": "string", "minLength": 1}}} + }, + "$defs": { + "region": {"type": "object", "additionalProperties": false, "required": ["id", "stage", "kind", "x", "y", "width", "height", "score"], "properties": {"id": {"type": "string", "maxLength": 120}, "stage": {"enum": ["coarse", "refined"]}, "kind": {"enum": ["support", "similar", "contrast"]}, "x": {"type": "number", "minimum": 0}, "y": {"type": "number", "minimum": 0}, "width": {"type": "number", "exclusiveMinimum": 0}, "height": {"type": "number", "exclusiveMinimum": 0}, "score": {"type": "number", "minimum": 0, "maximum": 1}, "thumbnail": {"type": "string", "pattern": "^thumbnails/[A-Za-z0-9._-]+\\.(png|jpg|webp)$"}}} + } +} diff --git a/contracts/evidence-mentor/pathlab-ai-pack-v1.schema.json b/contracts/evidence-mentor/pathlab-ai-pack-v1.schema.json new file mode 100644 index 0000000..1179ae0 --- /dev/null +++ b/contracts/evidence-mentor/pathlab-ai-pack-v1.schema.json @@ -0,0 +1,22 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://pathlab.local/schemas/pathlab-ai-pack-v1.schema.json", + "title": "PathLab AI Pack v1", + "type": "object", + "additionalProperties": false, + "required": ["schema", "packId", "version", "capability", "acceptedStains", "preprocessing", "artifacts", "rights", "resourceEnvelope", "validation", "outputSchema"], + "properties": { + "schema": {"const": "pathlab.ai-pack/1"}, + "packId": {"type": "string", "pattern": "^[a-z0-9][a-z0-9._-]{0,119}$"}, + "version": {"type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$"}, + "capability": {"enum": ["he-evidence", "cell-morphology", "ihc-descriptive"]}, + "acceptedStains": {"type": "array", "minItems": 1, "maxItems": 16, "uniqueItems": true, "items": {"enum": ["he", "ihc_dab"]}}, + "preprocessing": {"type": "object", "additionalProperties": false, "required": ["id", "tilePixels"], "properties": {"id": {"type": "string", "maxLength": 120}, "tilePixels": {"type": "integer", "minimum": 64, "maximum": 2048}}}, + "artifacts": {"type": "array", "maxItems": 16, "items": {"type": "object", "additionalProperties": false, "required": ["name", "sha256", "source"], "properties": {"name": {"type": "string", "maxLength": 160}, "sha256": {"type": "string", "pattern": "^[a-f0-9]{64}$"}, "source": {"type": "string", "format": "uri", "maxLength": 1000}}}, + "rights": {"type": "object", "additionalProperties": false, "required": ["license", "allowedUse", "redistributable", "derivativesAllowed", "reviewedAt"], "properties": {"license": {"type": "string", "maxLength": 200}, "allowedUse": {"enum": ["private-research", "benchmark-only"]}, "redistributable": {"type": "boolean"}, "derivativesAllowed": {"type": "boolean"}, "reviewedAt": {"type": "string", "format": "date-time"}}}, + "resourceEnvelope": {"type": "object", "additionalProperties": false, "required": ["maxRamMiB", "maxVramMiB", "maxSeconds", "network"], "properties": {"maxRamMiB": {"type": "integer", "minimum": 64, "maximum": 16384}, "maxVramMiB": {"type": "integer", "minimum": 0, "maximum": 4608}, "maxSeconds": {"type": "integer", "minimum": 1, "maximum": 1200}, "network": {"const": false}}}, + "validation": {"type": "object", "additionalProperties": false, "required": ["status", "modelCard", "heldOutEvaluation"], "properties": {"status": {"enum": ["experimental", "qualified", "not-evaluable", "blocked"]}, "modelCard": {"type": "string", "maxLength": 1000}, "heldOutEvaluation": {"type": "string", "maxLength": 1000}}}, + "outputSchema": {"const": "pathlab.ai-evidence/1"}, + "markers": {"type": "array", "uniqueItems": true, "items": {"enum": ["generic", "er", "pr", "ki-67", "her2", "pd-l1"]}} + } +} diff --git a/contracts/evidence-mentor/pathlab-knowledge-pack-v1.schema.json b/contracts/evidence-mentor/pathlab-knowledge-pack-v1.schema.json new file mode 100644 index 0000000..f675662 --- /dev/null +++ b/contracts/evidence-mentor/pathlab-knowledge-pack-v1.schema.json @@ -0,0 +1,16 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://pathlab.local/schemas/pathlab-knowledge-pack-v1.schema.json", + "title": "PathLab Knowledge Pack v1", + "type": "object", + "additionalProperties": false, + "required": ["schema", "packId", "version", "language", "claims", "checksum"], + "properties": { + "schema": {"const": "pathlab.knowledge-pack/1"}, + "packId": {"type": "string", "pattern": "^[a-z0-9][a-z0-9._-]{0,119}$"}, + "version": {"type": "string", "maxLength": 64}, + "language": {"const": "en"}, + "claims": {"type": "array", "minItems": 1, "maxItems": 10000, "items": {"type": "object", "additionalProperties": false, "required": ["id", "text", "retrievalText", "source", "license", "allowedUse", "reviewedAt"], "properties": {"id": {"type": "string", "pattern": "^[A-Za-z0-9._-]{1,160}$"}, "text": {"type": "string", "minLength": 1, "maxLength": 1000}, "retrievalText": {"type": "string", "minLength": 1, "maxLength": 4000}, "source": {"type": "object", "additionalProperties": false, "required": ["title", "url", "revision"], "properties": {"title": {"type": "string", "maxLength": 500}, "url": {"type": "string", "format": "uri", "maxLength": 1000}, "revision": {"type": "string", "maxLength": 160}}}, "license": {"type": "string", "maxLength": 200}, "allowedUse": {"const": "private-research-education"}, "reviewedAt": {"type": "string", "format": "date-time"}, "tags": {"type": "array", "maxItems": 32, "uniqueItems": true, "items": {"type": "string", "maxLength": 80}}}}}, + "checksum": {"type": "string", "pattern": "^[a-f0-9]{64}$"} + } +} diff --git a/docs/evidence-mentor-runner.md b/docs/evidence-mentor-runner.md new file mode 100644 index 0000000..6afe188 --- /dev/null +++ b/docs/evidence-mentor-runner.md @@ -0,0 +1,23 @@ +# Evidence Mentor runner + +`EvidenceMentorRunner` is a standalone per-user process. It binds only to +`127.0.0.1`, requires a 256-bit bearer token from `ipc-token`, and owns the +SQLite WAL queue, leases, checkpoints, signing key, and result artifacts. + +Install from a versioned Forge distribution: + +```powershell +.\scripts\evidence-mentor-task.ps1 -RuntimeRoot 'C:\Path\To\pathlab-forge' +``` + +Default state is `D:\PathLabData\EvidenceMentor\state` when the data drive is +present, otherwise `%LOCALAPPDATA%\PathLab\EvidenceMentor\state`. Analysis is +offline. Acquisition is outside this process. Current runnable pack is the +deterministic cell/IHC baseline; H&E encoder requests fail closed until an +approved, checksum-matched encoder artifact is installed. + +Rollback removes the scheduled task but preserves queue, evidence, and keys: + +```powershell +.\scripts\evidence-mentor-task.ps1 -RuntimeRoot 'C:\Path\To\pathlab-forge' -Uninstall +``` diff --git a/scripts/evidence-mentor-task.ps1 b/scripts/evidence-mentor-task.ps1 new file mode 100644 index 0000000..0052a33 --- /dev/null +++ b/scripts/evidence-mentor-task.ps1 @@ -0,0 +1,71 @@ +[CmdletBinding(SupportsShouldProcess)] +param( + [Parameter(Mandatory = $true)] + [string]$RuntimeRoot, + [string]$StateRoot, + [int]$Port = 8765, + [switch]$Uninstall +) + +$ErrorActionPreference = 'Stop' +$taskName = 'PathLab Evidence Mentor Runner' +$runtime = [IO.Path]::GetFullPath($RuntimeRoot) +if (-not $StateRoot) { + $StateRoot = if (Test-Path -LiteralPath 'D:\PathLabData' -PathType Container) { + 'D:\PathLabData\EvidenceMentor\state' + } else { + Join-Path $env:LOCALAPPDATA 'PathLab\EvidenceMentor\state' + } +} +$state = [IO.Path]::GetFullPath($StateRoot) + +if ($Uninstall) { + if ($PSCmdlet.ShouldProcess($taskName, 'Unregister per-user scheduled task')) { + Unregister-ScheduledTask -TaskName $taskName -Confirm:$false -ErrorAction SilentlyContinue + } + return +} + +$lib = Join-Path $runtime 'lib\*' +if (-not (Test-Path -LiteralPath (Split-Path $lib) -PathType Container)) { + throw "Versioned Forge runtime missing lib directory: $lib" +} +$java = if ($env:JAVA_HOME -and (Test-Path -LiteralPath (Join-Path $env:JAVA_HOME 'bin\javaw.exe'))) { + Join-Path $env:JAVA_HOME 'bin\javaw.exe' +} else { + (Get-Command javaw.exe -ErrorAction Stop).Source +} + +if ($PSCmdlet.ShouldProcess($state, 'Create protected Evidence Mentor state')) { + New-Item -ItemType Directory -Path $state -Force | Out-Null + $token = Join-Path $state 'ipc-token' + if (-not (Test-Path -LiteralPath $token -PathType Leaf)) { + $bytes = [byte[]]::new(32) + [Security.Cryptography.RandomNumberGenerator]::Fill($bytes) + [IO.File]::WriteAllText($token, [Convert]::ToBase64String($bytes)) + } + & icacls.exe $token /inheritance:r /grant:r "${env:USERNAME}:(R,W)" | Out-Null +} + +$arguments = @( + '-Xmx16g' + '-cp' + ('"{0}"' -f $lib) + 'org.pathlab.forge.evidence.EvidenceMentorRunner' + '--state' + ('"{0}"' -f $state) + '--port' + $Port +) -join ' ' +$action = New-ScheduledTaskAction -Execute $java -Argument $arguments +$trigger = New-ScheduledTaskTrigger -AtLogOn -User $env:USERNAME +$settings = New-ScheduledTaskSettingsSet -StartWhenAvailable -RestartCount 3 ` + -RestartInterval (New-TimeSpan -Minutes 1) -ExecutionTimeLimit ([TimeSpan]::Zero) +$principal = New-ScheduledTaskPrincipal -UserId $env:USERNAME -LogonType Interactive ` + -RunLevel Limited + +if ($PSCmdlet.ShouldProcess($taskName, 'Register and start per-user autonomous runner')) { + Register-ScheduledTask -TaskName $taskName -Action $action -Trigger $trigger ` + -Settings $settings -Principal $principal -Force | Out-Null + Start-ScheduledTask -TaskName $taskName +} diff --git a/scripts/submit-evidence-job.ps1 b/scripts/submit-evidence-job.ps1 new file mode 100644 index 0000000..689090b --- /dev/null +++ b/scripts/submit-evidence-job.ps1 @@ -0,0 +1,51 @@ +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)][string]$SourcePath, + [Parameter(Mandatory = $true)][string]$SlideRevision, + [Parameter(Mandatory = $true)][string]$PreviewPath, + [Parameter(Mandatory = $true)][int]$SourceWidth, + [Parameter(Mandatory = $true)][int]$SourceHeight, + [Parameter(Mandatory = $true)][string]$PackManifest, + [ValidateSet('he', 'ihc_dab')][string]$Stain, + [string]$Marker = 'generic', + [string]$StateRoot, + [int]$Port = 8765 +) + +$ErrorActionPreference = 'Stop' +if (-not $StateRoot) { + $StateRoot = if (Test-Path -LiteralPath 'D:\PathLabData' -PathType Container) { + 'D:\PathLabData\EvidenceMentor\state' + } else { + Join-Path $env:LOCALAPPDATA 'PathLab\EvidenceMentor\state' + } +} +$state = [IO.Path]::GetFullPath($StateRoot) +$source = Get-Item -LiteralPath $SourcePath +$preview = Get-Item -LiteralPath $PreviewPath +$pack = Get-Item -LiteralPath $PackManifest +$tokenPath = Join-Path $state 'ipc-token' +if (-not (Test-Path -LiteralPath $tokenPath -PathType Leaf)) { + throw 'Evidence Mentor runner is not installed or has not started.' +} +$jobId = [Guid]::NewGuid().ToString() +$requestRoot = Join-Path $state 'requests' +New-Item -ItemType Directory -Path $requestRoot -Force | Out-Null +$requestPath = Join-Path $requestRoot "$jobId.json" +$request = [ordered]@{ + schema = 'pathlab.evidence-job/1' + sourcePath = $source.FullName + sourceSha256 = (Get-FileHash -LiteralPath $source.FullName -Algorithm SHA256).Hash.ToLowerInvariant() + slideRevision = $SlideRevision + previewPath = $preview.FullName + sourceWidth = $SourceWidth + sourceHeight = $SourceHeight + packManifest = $pack.FullName + stain = $Stain + marker = $Marker.ToLowerInvariant() +} +$request | ConvertTo-Json -Depth 4 | Set-Content -LiteralPath $requestPath -Encoding utf8NoBOM +$token = [IO.File]::ReadAllText($tokenPath).Trim() +$headers = @{ Authorization = "Bearer $token" } +Invoke-RestMethod -Method Post -Uri "http://127.0.0.1:$Port/v1/jobs" -Headers $headers ` + -ContentType 'application/json' -Body (@{ id = $jobId; requestPath = $requestPath } | ConvertTo-Json) diff --git a/src/main/java/org/pathlab/forge/evidence/BrightfieldTileAnalyzer.java b/src/main/java/org/pathlab/forge/evidence/BrightfieldTileAnalyzer.java new file mode 100644 index 0000000..05a7128 --- /dev/null +++ b/src/main/java/org/pathlab/forge/evidence/BrightfieldTileAnalyzer.java @@ -0,0 +1,103 @@ +package org.pathlab.forge.evidence; + +import java.awt.image.BufferedImage; +import java.util.ArrayDeque; +import java.util.Set; + +/** Bounded deterministic hematoxylin-component and DAB descriptor baseline. */ +public final class BrightfieldTileAnalyzer { + private static final Set KNOWN = Set.of("er", "pr", "ki-67", "her2", "pd-l1"); + + private BrightfieldTileAnalyzer() {} + + public static Result analyze(BufferedImage image, String requestedMarker) { + if (image == null || image.getWidth() < 1 || image.getHeight() < 1 + || (long) image.getWidth() * image.getHeight() > 4_194_304) { + throw new IllegalArgumentException("Brightfield image geometry is invalid"); + } + var marker = requestedMarker == null ? "generic" : requestedMarker.toLowerCase(); + if (!KNOWN.contains(marker)) marker = "generic"; + var width = image.getWidth(); + var height = image.getHeight(); + var nuclei = new boolean[width * height]; + var dabPixels = 0; + double dabOd = 0; + for (var y = 0; y < height; y++) { + for (var x = 0; x < width; x++) { + var rgb = image.getRGB(x, y); + var red = rgb >>> 16 & 0xff; + var green = rgb >>> 8 & 0xff; + var blue = rgb & 0xff; + nuclei[y * width + x] = blue - red >= 25 && blue - green >= 25 + && red + green + blue < 660; + if (red - green >= 15 && green - blue >= 15 && red + green + blue < 660) { + dabPixels++; + var luminance = 0.2126 * red + 0.7152 * green + 0.0722 * blue; + dabOd += -Math.log((luminance + 1) / 256.0); + } + } + } + var components = components(nuclei, width, height); + return new Result( + marker, + compartment(marker), + components.count, + components.count == 0 ? null : (double) components.pixels / components.count, + (double) dabPixels / (width * height), + dabPixels == 0 ? 0 : dabOd / dabPixels, + true); + } + + private static Components components(boolean[] mask, int width, int height) { + var seen = new boolean[mask.length]; + var queue = new ArrayDeque(); + var count = 0; + var pixels = 0; + for (var start = 0; start < mask.length; start++) { + if (!mask[start] || seen[start]) continue; + seen[start] = true; + queue.add(start); + var size = 0; + while (!queue.isEmpty()) { + var current = queue.removeFirst(); + size++; + var x = current % width; + var y = current / width; + for (var ny = Math.max(0, y - 1); ny <= Math.min(height - 1, y + 1); ny++) { + for (var nx = Math.max(0, x - 1); nx <= Math.min(width - 1, x + 1); nx++) { + var next = ny * width + nx; + if (mask[next] && !seen[next]) { + seen[next] = true; + queue.addLast(next); + } + } + } + } + if (size >= 4) { + count++; + pixels += size; + } + } + return new Components(count, pixels); + } + + private static String compartment(String marker) { + return switch (marker) { + case "er", "pr", "ki-67" -> "nuclear"; + case "her2" -> "membrane"; + case "pd-l1" -> "tumor-immune-region"; + default -> "generic-region"; + }; + } + + private record Components(int count, int pixels) {} + + public record Result( + String marker, + String compartment, + int cellCount, + Double meanNucleusAreaPx2, + double dabAreaFraction, + double meanDabOd, + boolean researchEstimate) {} +} diff --git a/src/main/java/org/pathlab/forge/evidence/EvidenceArtifactStore.java b/src/main/java/org/pathlab/forge/evidence/EvidenceArtifactStore.java new file mode 100644 index 0000000..13f8568 --- /dev/null +++ b/src/main/java/org/pathlab/forge/evidence/EvidenceArtifactStore.java @@ -0,0 +1,49 @@ +package org.pathlab.forge.evidence; + +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Comparator; +import java.util.Optional; +import java.util.Set; + +/** Finds a signed terminal artifact bound to one exact slide revision. */ +public final class EvidenceArtifactStore { + private static final ObjectMapper JSON = new ObjectMapper(); + private static final Set TERMINAL = Set.of("completed", "partial", "abstained"); + + private EvidenceArtifactStore() {} + + public static Optional find(Path stateRoot, String slideSha256, String revision) + throws IOException { + var artifacts = stateRoot.toAbsolutePath().normalize().resolve("artifacts"); + if (!Files.isDirectory(artifacts)) return Optional.empty(); + try (var paths = Files.walk(artifacts, 3)) { + return paths.filter(path -> path.getFileName().toString().equals("evidence.json")) + .filter(Files::isRegularFile) + .limit(2_000) + .filter(path -> matches(path, slideSha256, revision)) + .max(Comparator.comparingLong(EvidenceArtifactStore::modified)); + } + } + + private static boolean matches(Path path, String sha, String revision) { + try { + var root = JSON.readTree(path.toFile()); + return "pathlab.ai-evidence/1".equals(root.path("schema").asText()) + && sha.equals(root.path("source").path("slideSha256").asText()) + && revision.equals(root.path("source").path("revision").asText()) + && TERMINAL.contains(root.path("status").asText()) + && root.path("manifestSha256").asText().matches("[a-f0-9]{64}") + && "Ed25519".equals(root.path("signature").path("algorithm").asText()); + } catch (IOException ignored) { + return false; + } + } + + private static long modified(Path path) { + try { return Files.getLastModifiedTime(path).toMillis(); } + catch (IOException ignored) { return Long.MIN_VALUE; } + } +} diff --git a/src/main/java/org/pathlab/forge/evidence/EvidenceBundleWriter.java b/src/main/java/org/pathlab/forge/evidence/EvidenceBundleWriter.java new file mode 100644 index 0000000..c70e1bb --- /dev/null +++ b/src/main/java/org/pathlab/forge/evidence/EvidenceBundleWriter.java @@ -0,0 +1,113 @@ +package org.pathlab.forge.evidence; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.node.ObjectNode; +import java.io.IOException; +import java.nio.file.AtomicMoveNotSupportedException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.security.MessageDigest; +import java.util.HexFormat; + +/** Canonicalizes, signs, and atomically publishes a pathlab.ai-evidence/1 manifest. */ +public final class EvidenceBundleWriter { + public static final String SCHEMA = "pathlab.ai-evidence/1"; + private static final ObjectMapper JSON = new ObjectMapper() + .enable(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS); + private final EvidenceSigner signer; + + public EvidenceBundleWriter(Path signingRoot) { + signer = new EvidenceSigner(signingRoot); + } + + public Result write(Path output, ObjectNode unsigned) throws IOException { + require(SCHEMA.equals(unsigned.path("schema").asText()), "Evidence schema is unsupported"); + require(unsigned.path("researchOnly").asBoolean(false), "Evidence must be research-only"); + require(unsigned.path("notDiagnostic").asBoolean(false), "Evidence must be non-diagnostic"); + require(unsigned.path("reviewRequired").asBoolean(false), "Evidence must require review"); + require(unsigned.path("source").path("slideSha256").asText().matches("[a-f0-9]{64}"), + "Evidence source SHA-256 is invalid"); + require(!unsigned.has("manifestSha256") && !unsigned.has("signature"), + "Unsigned evidence must not contain signature fields"); + var canonical = canonicalBytes(unsigned); + var manifestSha = sha256(canonical); + var signed = signer.sign(SCHEMA + "\n" + manifestSha); + var complete = unsigned.deepCopy(); + complete.put("manifestSha256", manifestSha); + complete.set("signature", JSON.valueToTree(java.util.Map.of( + "algorithm", "Ed25519", + "keyId", signed.keyId(), + "publicKeyDer", signed.publicKeyDer(), + "value", signed.signature()))); + var normalized = output.toAbsolutePath().normalize(); + Files.createDirectories(normalized.getParent()); + var partial = normalized.resolveSibling(normalized.getFileName() + ".partial"); + JSON.writerWithDefaultPrettyPrinter().writeValue(partial.toFile(), complete); + try { + Files.move(partial, normalized, StandardCopyOption.REPLACE_EXISTING, + StandardCopyOption.ATOMIC_MOVE); + } catch (AtomicMoveNotSupportedException ignored) { + Files.move(partial, normalized, StandardCopyOption.REPLACE_EXISTING); + } + return new Result(normalized, manifestSha, signed.keyId()); + } + + public static byte[] canonicalBytes(JsonNode value) throws IOException { + var output = new StringBuilder(); + appendCanonical(value, output); + return output.toString().getBytes(java.nio.charset.StandardCharsets.UTF_8); + } + + private static void appendCanonical(JsonNode value, StringBuilder output) throws IOException { + if (value.isNull()) { + output.append("null"); + } else if (value.isBoolean()) { + output.append(value.booleanValue()); + } else if (value.isTextual()) { + output.append(JSON.writeValueAsString(value.textValue())); + } else if (value.isIntegralNumber()) { + output.append(value.bigIntegerValue()); + } else if (value.isFloatingPointNumber()) { + var decimal = value.decimalValue(); + if (!Double.isFinite(value.doubleValue())) throw new IOException("Evidence number is invalid"); + output.append(decimal.signum() == 0 ? "0" : decimal.stripTrailingZeros().toPlainString()); + } else if (value.isArray()) { + output.append('['); + for (var index = 0; index < value.size(); index++) { + if (index > 0) output.append(','); + appendCanonical(value.get(index), output); + } + output.append(']'); + } else if (value.isObject()) { + output.append('{'); + var names = new java.util.ArrayList(); + value.fieldNames().forEachRemaining(names::add); + names.sort(String::compareTo); + for (var index = 0; index < names.size(); index++) { + if (index > 0) output.append(','); + output.append(JSON.writeValueAsString(names.get(index))).append(':'); + appendCanonical(value.get(names.get(index)), output); + } + output.append('}'); + } else { + throw new IOException("Evidence value is invalid"); + } + } + + public static String sha256(byte[] value) throws IOException { + try { + return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(value)); + } catch (Exception error) { + throw new IOException("SHA-256 is unavailable", error); + } + } + + private static void require(boolean condition, String message) { + if (!condition) throw new IllegalArgumentException(message); + } + + public record Result(Path path, String manifestSha256, String keyId) {} +} diff --git a/src/main/java/org/pathlab/forge/evidence/EvidenceJob.java b/src/main/java/org/pathlab/forge/evidence/EvidenceJob.java new file mode 100644 index 0000000..a6de4a2 --- /dev/null +++ b/src/main/java/org/pathlab/forge/evidence/EvidenceJob.java @@ -0,0 +1,34 @@ +package org.pathlab.forge.evidence; + +import java.nio.file.Path; +import java.time.Instant; + +public record EvidenceJob( + String id, + Path requestPath, + EvidenceJobState state, + String stage, + double progress, + int retryCount, + boolean cancelRequested, + String leaseOwner, + Instant leaseExpiresAt, + String detail, + Instant createdAt, + Instant updatedAt) { + public EvidenceJob { + if (id == null || !id.matches("[A-Za-z0-9._-]{1,120}")) { + throw new IllegalArgumentException("Evidence job id is invalid"); + } + requestPath = requestPath.toAbsolutePath().normalize(); + if (!Double.isFinite(progress) || progress < 0 || progress > 1) { + throw new IllegalArgumentException("Evidence job progress is invalid"); + } + if (retryCount < 0 || retryCount > 3) { + throw new IllegalArgumentException("Evidence job retry count is invalid"); + } + detail = detail == null ? "" : detail; + stage = stage == null ? "" : stage; + leaseOwner = leaseOwner == null ? "" : leaseOwner; + } +} diff --git a/src/main/java/org/pathlab/forge/evidence/EvidenceJobProcessor.java b/src/main/java/org/pathlab/forge/evidence/EvidenceJobProcessor.java new file mode 100644 index 0000000..b757c60 --- /dev/null +++ b/src/main/java/org/pathlab/forge/evidence/EvidenceJobProcessor.java @@ -0,0 +1,257 @@ +package org.pathlab.forge.evidence; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.core.JsonProcessingException; +import java.awt.image.BufferedImage; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.time.Duration; +import java.time.Instant; +import java.util.HexFormat; +import java.util.Set; +import javax.imageio.ImageIO; + +/** Executes bounded, network-free deterministic Evidence Mentor packs. */ +public final class EvidenceJobProcessor { + private static final ObjectMapper JSON = new ObjectMapper(); + private static final Duration LEASE = Duration.ofMinutes(2); + private static final long ARTIFACT_QUOTA = 10L * 1024 * 1024 * 1024; + private static final Set REQUEST_FIELDS = Set.of( + "schema", "sourcePath", "sourceSha256", "slideRevision", "previewPath", + "sourceWidth", "sourceHeight", "packManifest", "stain", "marker"); + private final EvidenceJobQueue queue; + private final Path stateRoot; + + public EvidenceJobProcessor(EvidenceJobQueue queue, Path stateRoot) throws IOException { + this.queue = queue; + this.stateRoot = stateRoot.toAbsolutePath().normalize(); + Files.createDirectories(this.stateRoot.resolve("artifacts")); + } + + public EvidenceJob process(EvidenceJob job, String workerId, Instant now) throws IOException { + final JsonNode request; + try { + request = JSON.readTree(job.requestPath().toFile()); + } catch (JsonProcessingException invalidJson) { + throw new IllegalArgumentException("Evidence job request JSON is invalid", invalidJson); + } + require(request.isObject() && fieldNames(request).equals(REQUEST_FIELDS), + "Evidence job request is invalid"); + require("pathlab.evidence-job/1".equals(text(request, "schema")), + "Evidence job schema is unsupported"); + var source = regularPath(request, "sourcePath"); + var preview = regularPath(request, "previewPath"); + var expectedSha = text(request, "sourceSha256"); + require(expectedSha.matches("[a-f0-9]{64}") && expectedSha.equals(sha256(source)), + "Evidence source checksum does not match"); + var pack = EvidencePackManifest.load(regularPath(request, "packManifest")); + pack.requirePilotEligible(); + var stain = text(request, "stain"); + pack.requireStain(stain); + require(pack.capability() != EvidencePackManifest.Capability.HE_EVIDENCE, + "H&E encoder artifact is not installed"); + var marker = text(request, "marker").toLowerCase(); + if (!pack.markers().contains(marker)) marker = "generic"; + require(pack.capability() != EvidencePackManifest.Capability.IHC_DESCRIPTIVE + || pack.markers().contains(marker), + "Requested IHC marker is unsupported and generic fallback is unavailable"); + require(positiveInt(request, "sourceWidth") > 0 && positiveInt(request, "sourceHeight") > 0, + "Evidence source geometry is invalid"); + checkCancellation(job.id(), workerId, now); + var running = queue.checkpoint(job.id(), workerId, EvidenceJobState.RUNNING, + "running", 0.25, "Running offline bounded brightfield analysis", now, LEASE); + + final BufferedImage image; + try { + image = ImageIO.read(preview.toFile()); + } catch (javax.imageio.IIOException invalidImage) { + throw new IllegalArgumentException("Evidence preview could not be decoded", invalidImage); + } + require(image != null, "Evidence preview could not be decoded"); + var analysis = BrightfieldTileAnalyzer.analyze(image, marker); + var tissue = tissueFraction(image); + var focus = focus(image); + var abstentionReasons = new java.util.ArrayList(); + if (tissue < 0.05) abstentionReasons.add("insufficient_tissue"); + if (focus < 0.05) abstentionReasons.add("out_of_focus_or_uninformative_preview"); + var abstained = !abstentionReasons.isEmpty(); + var refiningAt = now.plusSeconds(1); + checkCancellation(job.id(), workerId, refiningAt); + var refining = queue.checkpoint(running.id(), workerId, EvidenceJobState.REFINING, + "refining", 0.65, "Refining deterministic region descriptors", refiningAt, LEASE); + var packagingAt = now.plusSeconds(2); + var packaging = queue.checkpoint(refining.id(), workerId, EvidenceJobState.PACKAGING, + "packaging", 0.9, "Signing immutable evidence bundle", packagingAt, LEASE); + + var artifactRoot = stateRoot.resolve("artifacts").resolve(job.id()).normalize(); + require(artifactRoot.startsWith(stateRoot.resolve("artifacts")), "Evidence artifact path escaped state root"); + require(directoryBytes(stateRoot.resolve("artifacts")) < ARTIFACT_QUOTA, + "Evidence artifact quota is exhausted"); + Files.createDirectories(artifactRoot); + var unsigned = evidence(request, pack, image, analysis, focus, tissue, + abstentionReasons, now); + new EvidenceBundleWriter(stateRoot.resolve("signing")) + .write(artifactRoot.resolve("evidence.json"), unsigned); + var terminal = abstained ? EvidenceJobState.ABSTAINED : EvidenceJobState.COMPLETED; + var detail = abstained + ? "Signed QC evidence abstained from descriptive analysis" + : "Signed evidence is ready for faculty review"; + return queue.checkpoint(packaging.id(), workerId, terminal, + terminal.name().toLowerCase(), 1, detail, + now.plusSeconds(3), LEASE); + } + + private static ObjectNode evidence( + JsonNode request, + EvidencePackManifest pack, + BufferedImage image, + BrightfieldTileAnalyzer.Result analysis, + double focus, + double tissue, + java.util.List abstentionReasons, + Instant now) { + var root = JSON.createObjectNode(); + root.put("schema", EvidenceBundleWriter.SCHEMA); + root.put("bundleId", "evidence-" + java.util.UUID.randomUUID()); + root.set("source", JSON.valueToTree(java.util.Map.of( + "slideSha256", text(request, "sourceSha256"), + "revision", text(request, "slideRevision")))); + root.set("pack", JSON.valueToTree(java.util.Map.of( + "id", pack.packId(), + "version", pack.version(), + "manifestSha256", pack.sha256(), + "preprocessing", pack.preprocessingId(), + "artifacts", pack.artifacts().stream().map(EvidencePackManifest.Artifact::sha256).toList(), + "allowedUse", pack.allowedUse(), + "validationStatus", pack.validationStatus().wire()))); + root.put("status", abstentionReasons.isEmpty() ? "completed" : "abstained"); + root.put("researchOnly", true); + root.put("notDiagnostic", true); + root.put("reviewRequired", true); + var sourceWidth = positiveInt(request, "sourceWidth"); + var sourceHeight = positiveInt(request, "sourceHeight"); + root.set("coordinates", JSON.valueToTree(java.util.Map.of( + "space", "source-pixel", "originX", 0, "originY", 0, + "scaleX", (double) sourceWidth / image.getWidth(), + "scaleY", (double) sourceHeight / image.getHeight()))); + root.set("evidence", JSON.valueToTree(java.util.List.of(java.util.Map.of( + "id", "region-1", "stage", "refined", "kind", "support", + "x", 0, "y", 0, "width", sourceWidth, "height", sourceHeight, + "score", Math.max(0, Math.min(1, 1 - analysis.dabAreaFraction())))))); + var aggregate = new java.util.LinkedHashMap(); + aggregate.put("regionId", "region-1"); + aggregate.put("algorithm", "od-watershed"); + aggregate.put("count", analysis.cellCount()); + aggregate.put("densityPerMm2", null); + aggregate.put("meanNucleusAreaPx2", analysis.meanNucleusAreaPx2()); + root.set("cellAggregates", JSON.valueToTree(java.util.List.of(aggregate))); + root.set("ihcDescriptors", JSON.valueToTree( + pack.capability() == EvidencePackManifest.Capability.IHC_DESCRIPTIVE + ? java.util.List.of(java.util.Map.of( + "regionId", "region-1", "marker", analysis.marker(), + "compartment", analysis.compartment(), + "dabAreaFraction", analysis.dabAreaFraction(), + "meanDabOd", analysis.meanDabOd(), + "researchEstimate", true)) + : java.util.List.of())); + root.set("citations", JSON.createArrayNode()); + root.set("qc", JSON.valueToTree(java.util.Map.of( + "focus", focus, "tissueFraction", tissue, + "uncertainty", 1 - Math.min(focus, tissue), + "abstentionReasons", abstentionReasons))); + root.set("provenance", JSON.valueToTree(java.util.Map.of( + "createdAt", now.toString(), "codeRevision", "pathlab-forge-evidence-mentor-v1", + "offlineAnalysis", true))); + return root; + } + + private void checkCancellation(String id, String workerId, Instant now) throws IOException { + var current = queue.find(id).orElseThrow(); + if (current.cancelRequested()) { + queue.checkpoint(id, workerId, EvidenceJobState.CANCELLED, "cancelled", current.progress(), + "Cancelled at a durable checkpoint", now, LEASE); + throw new CancellationException(); + } + } + + private static double tissueFraction(BufferedImage image) { + var tissue = 0; + for (var y = 0; y < image.getHeight(); y++) for (var x = 0; x < image.getWidth(); x++) { + var rgb = image.getRGB(x, y); + if (((rgb >>> 16 & 0xff) + (rgb >>> 8 & 0xff) + (rgb & 0xff)) / 3.0 < 235) tissue++; + } + return (double) tissue / (image.getWidth() * image.getHeight()); + } + + private static double focus(BufferedImage image) { + double gradients = 0; + var samples = 0; + for (var y = 1; y < image.getHeight(); y++) for (var x = 1; x < image.getWidth(); x++) { + var here = image.getRGB(x, y) & 0xff; + gradients += Math.abs(here - (image.getRGB(x - 1, y) & 0xff)); + gradients += Math.abs(here - (image.getRGB(x, y - 1) & 0xff)); + samples += 2; + } + return Math.min(1, gradients / Math.max(1, samples) / 32.0); + } + + private static Path regularPath(JsonNode request, String name) { + var path = Path.of(text(request, name)).toAbsolutePath().normalize(); + require(Files.isRegularFile(path), "Evidence input is unavailable: " + name); + return path; + } + + private static int positiveInt(JsonNode request, String name) { + var value = request.path(name); + require(value.isIntegralNumber() && value.canConvertToInt() && value.intValue() > 0, + "Evidence source geometry is invalid"); + return value.intValue(); + } + + private static Set fieldNames(JsonNode value) { + var result = new java.util.HashSet(); + value.fieldNames().forEachRemaining(result::add); + return result; + } + + private static String text(JsonNode root, String name) { + var value = root.path(name); + require(value.isTextual() && !value.textValue().isBlank(), "Evidence field is invalid: " + name); + return value.textValue(); + } + + private static String sha256(Path path) throws IOException { + try (InputStream input = Files.newInputStream(path)) { + var digest = MessageDigest.getInstance("SHA-256"); + var buffer = new byte[1024 * 1024]; + int read; + while ((read = input.read(buffer)) >= 0) digest.update(buffer, 0, read); + return HexFormat.of().formatHex(digest.digest()); + } catch (java.security.GeneralSecurityException impossible) { + throw new IOException("SHA-256 is unavailable", impossible); + } + } + + private static long directoryBytes(Path root) throws IOException { + if (!Files.exists(root)) return 0; + try (var paths = Files.walk(root)) { + return paths.filter(Files::isRegularFile).mapToLong(path -> { + try { return Files.size(path); } catch (IOException ignored) { return Long.MAX_VALUE; } + }).sum(); + } + } + + private static void require(boolean condition, String message) { + if (!condition) throw new IllegalArgumentException(message); + } + + public static final class CancellationException extends IOException { + private static final long serialVersionUID = 1L; + public CancellationException() { super("Evidence job was cancelled"); } + } +} diff --git a/src/main/java/org/pathlab/forge/evidence/EvidenceJobQueue.java b/src/main/java/org/pathlab/forge/evidence/EvidenceJobQueue.java new file mode 100644 index 0000000..8bb0eef --- /dev/null +++ b/src/main/java/org/pathlab/forge/evidence/EvidenceJobQueue.java @@ -0,0 +1,255 @@ +package org.pathlab.forge.evidence; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.time.Duration; +import java.time.Instant; +import java.util.Optional; + +/** Durable WAL queue with expiring leases and restart-safe checkpoints. */ +public final class EvidenceJobQueue implements AutoCloseable { + private static final String ACTIVE_STATES = "'VALIDATING','RUNNING','REFINING','PACKAGING'"; + private final Connection connection; + + public EvidenceJobQueue(Path database) throws IOException { + try { + var normalized = database.toAbsolutePath().normalize(); + Files.createDirectories(normalized.getParent()); + connection = DriverManager.getConnection("jdbc:sqlite:" + normalized); + try (var statement = connection.createStatement()) { + statement.execute("PRAGMA journal_mode=WAL"); + statement.execute("PRAGMA synchronous=FULL"); + statement.execute("PRAGMA busy_timeout=5000"); + statement.execute(""" + CREATE TABLE IF NOT EXISTS evidence_jobs ( + id TEXT PRIMARY KEY, + request_path TEXT NOT NULL, + state TEXT NOT NULL, + stage TEXT NOT NULL, + progress REAL NOT NULL, + retry_count INTEGER NOT NULL, + cancel_requested INTEGER NOT NULL, + lease_owner TEXT NOT NULL, + lease_expires_at TEXT NOT NULL, + detail TEXT NOT NULL, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL + )"""); + statement.execute("CREATE INDEX IF NOT EXISTS ix_evidence_jobs_state_updated ON evidence_jobs(state, updated_at)"); + } + } catch (SQLException error) { + throw new IOException("Unable to open Evidence Mentor queue", error); + } + } + + public synchronized EvidenceJob submit(String id, Path requestPath, Instant now) throws IOException { + var normalized = requestPath.toAbsolutePath().normalize(); + if (!Files.isRegularFile(normalized)) throw new IllegalArgumentException("Evidence request file is unavailable"); + var job = new EvidenceJob(id, normalized, EvidenceJobState.QUEUED, "queued", 0, 0, + false, "", Instant.EPOCH, "Queued for autonomous local analysis", now, now); + try (var statement = connection.prepareStatement(""" + INSERT INTO evidence_jobs VALUES (?,?,?,?,?,?,?,?,?,?,?,?) + """)) { + bind(statement, job); + statement.executeUpdate(); + return job; + } catch (SQLException error) { + if (error.getErrorCode() == 19) throw new IllegalArgumentException("Evidence job already exists", error); + throw new IOException("Unable to queue evidence job", error); + } + } + + public synchronized Optional find(String id) throws IOException { + try (var statement = connection.prepareStatement("SELECT * FROM evidence_jobs WHERE id=?")) { + statement.setString(1, id); + try (var rows = statement.executeQuery()) { + return rows.next() ? Optional.of(read(rows)) : Optional.empty(); + } + } catch (SQLException error) { + throw new IOException("Unable to read evidence job", error); + } + } + + public synchronized Optional claimNext( + String workerId, Instant now, Duration lease) throws IOException { + if (workerId == null || !workerId.matches("[A-Za-z0-9._-]{1,120}")) { + throw new IllegalArgumentException("Evidence worker id is invalid"); + } + if (lease.isNegative() || lease.isZero() || lease.compareTo(Duration.ofMinutes(10)) > 0) { + throw new IllegalArgumentException("Evidence lease is invalid"); + } + try { + try (var begin = connection.createStatement()) { + begin.execute("BEGIN IMMEDIATE"); + } + EvidenceJob selected = null; + var claimSql = "SELECT * FROM evidence_jobs " + + "WHERE cancel_requested=0 AND (state='QUEUED' OR (state IN (" + + ACTIVE_STATES + ") AND lease_expires_at<=?)) " + + "ORDER BY created_at, id LIMIT 1"; + try (var statement = connection.prepareStatement(claimSql)) { + statement.setString(1, now.toString()); + try (var rows = statement.executeQuery()) { + if (rows.next()) selected = read(rows); + } + } + if (selected == null) { + commit(); + return Optional.empty(); + } + var state = selected.state() == EvidenceJobState.QUEUED + ? EvidenceJobState.VALIDATING : selected.state(); + var claimed = new EvidenceJob(selected.id(), selected.requestPath(), state, + selected.state() == EvidenceJobState.QUEUED ? "validating" : selected.stage(), + selected.progress(), selected.retryCount(), false, workerId, now.plus(lease), + selected.state() == EvidenceJobState.QUEUED + ? "Validating immutable request and pack" : "Recovered after expired worker lease", + selected.createdAt(), now); + update(claimed); + commit(); + return Optional.of(claimed); + } catch (SQLException error) { + rollbackQuietly(); + throw new IOException("Unable to claim evidence job", error); + } + } + + public synchronized EvidenceJob checkpoint( + String id, String workerId, EvidenceJobState state, String stage, + double progress, String detail, Instant now, Duration lease) throws IOException { + var current = find(id).orElseThrow(() -> new IllegalArgumentException("Evidence job was not found")); + if (!workerId.equals(current.leaseOwner()) || current.leaseExpiresAt().isBefore(now)) { + throw new IllegalStateException("Evidence job lease is not owned by this worker"); + } + requireTransition(current.state(), state); + var updated = new EvidenceJob(id, current.requestPath(), state, stage, progress, + current.retryCount(), current.cancelRequested(), state.terminal() ? "" : workerId, + state.terminal() ? Instant.EPOCH : now.plus(lease), bounded(detail), current.createdAt(), now); + updateChecked(updated); + return updated; + } + + public synchronized void requestCancel(String id, Instant now) throws IOException { + var current = find(id).orElseThrow(() -> new IllegalArgumentException("Evidence job was not found")); + if (current.state().terminal()) return; + try (var statement = connection.prepareStatement( + "UPDATE evidence_jobs SET cancel_requested=1, detail=?, updated_at=? WHERE id=?")) { + statement.setString(1, "Cancellation requested by the local operator"); + statement.setString(2, now.toString()); + statement.setString(3, id); + statement.executeUpdate(); + } catch (SQLException error) { + throw new IOException("Unable to request evidence cancellation", error); + } + } + + public synchronized EvidenceJob retryOrFail(String id, String workerId, String detail, Instant now) + throws IOException { + var current = find(id).orElseThrow(() -> new IllegalArgumentException("Evidence job was not found")); + if (!workerId.equals(current.leaseOwner())) throw new IllegalStateException("Evidence job lease is not owned"); + var retries = current.retryCount() + 1; + var state = retries <= 3 ? EvidenceJobState.QUEUED : EvidenceJobState.FAILED; + var next = new EvidenceJob(id, current.requestPath(), state, + state == EvidenceJobState.QUEUED ? "retrying" : "failed", current.progress(), + Math.min(3, retries), false, "", Instant.EPOCH, bounded(detail), current.createdAt(), now); + updateChecked(next); + return next; + } + + private void updateChecked(EvidenceJob job) throws IOException { + try { update(job); } catch (SQLException error) { throw new IOException("Unable to checkpoint evidence job", error); } + } + + private void update(EvidenceJob job) throws SQLException { + try (var statement = connection.prepareStatement(""" + UPDATE evidence_jobs SET request_path=?, state=?, stage=?, progress=?, retry_count=?, + cancel_requested=?, lease_owner=?, lease_expires_at=?, detail=?, created_at=?, updated_at=? + WHERE id=? + """)) { + statement.setString(1, job.requestPath().toString()); + statement.setString(2, job.state().name()); + statement.setString(3, job.stage()); + statement.setDouble(4, job.progress()); + statement.setInt(5, job.retryCount()); + statement.setInt(6, job.cancelRequested() ? 1 : 0); + statement.setString(7, job.leaseOwner()); + statement.setString(8, job.leaseExpiresAt().toString()); + statement.setString(9, job.detail()); + statement.setString(10, job.createdAt().toString()); + statement.setString(11, job.updatedAt().toString()); + statement.setString(12, job.id()); + if (statement.executeUpdate() != 1) throw new SQLException("Evidence job was not updated"); + } + } + + private static void bind(java.sql.PreparedStatement statement, EvidenceJob job) throws SQLException { + statement.setString(1, job.id()); + statement.setString(2, job.requestPath().toString()); + statement.setString(3, job.state().name()); + statement.setString(4, job.stage()); + statement.setDouble(5, job.progress()); + statement.setInt(6, job.retryCount()); + statement.setInt(7, job.cancelRequested() ? 1 : 0); + statement.setString(8, job.leaseOwner()); + statement.setString(9, job.leaseExpiresAt().toString()); + statement.setString(10, job.detail()); + statement.setString(11, job.createdAt().toString()); + statement.setString(12, job.updatedAt().toString()); + } + + private static EvidenceJob read(ResultSet row) throws SQLException { + return new EvidenceJob(row.getString("id"), Path.of(row.getString("request_path")), + EvidenceJobState.valueOf(row.getString("state")), row.getString("stage"), + row.getDouble("progress"), row.getInt("retry_count"), + row.getInt("cancel_requested") != 0, row.getString("lease_owner"), + Instant.parse(row.getString("lease_expires_at")), row.getString("detail"), + Instant.parse(row.getString("created_at")), Instant.parse(row.getString("updated_at"))); + } + + private static void requireTransition(EvidenceJobState from, EvidenceJobState to) { + if (from.terminal()) throw new IllegalStateException("Evidence job is already terminal"); + var allowed = switch (from) { + case QUEUED -> java.util.Set.of(EvidenceJobState.VALIDATING, EvidenceJobState.CANCELLED); + case VALIDATING -> java.util.Set.of(EvidenceJobState.RUNNING, EvidenceJobState.ABSTAINED, + EvidenceJobState.UNSUPPORTED, EvidenceJobState.FAILED, EvidenceJobState.CANCELLED); + case RUNNING -> java.util.Set.of(EvidenceJobState.REFINING, EvidenceJobState.PACKAGING, + EvidenceJobState.ABSTAINED, EvidenceJobState.UNSUPPORTED, EvidenceJobState.FAILED, + EvidenceJobState.CANCELLED); + case REFINING -> java.util.Set.of(EvidenceJobState.PACKAGING, EvidenceJobState.ABSTAINED, + EvidenceJobState.FAILED, EvidenceJobState.CANCELLED); + case PACKAGING -> java.util.Set.of(EvidenceJobState.COMPLETED, EvidenceJobState.ABSTAINED, + EvidenceJobState.FAILED, EvidenceJobState.CANCELLED); + default -> java.util.Set.of(); + }; + if (!allowed.contains(to) && from != to) { + throw new IllegalStateException("Invalid evidence job transition: " + from + " -> " + to); + } + } + + private static String bounded(String value) { + if (value == null || value.isBlank()) return "No additional detail"; + return value.length() <= 500 ? value : value.substring(0, 500); + } + + private void rollbackQuietly() { + try (var statement = connection.createStatement()) { + statement.execute("ROLLBACK"); + } catch (SQLException ignored) { } + } + + private void commit() throws SQLException { + try (var statement = connection.createStatement()) { + statement.execute("COMMIT"); + } + } + + @Override + public synchronized void close() throws IOException { + try { connection.close(); } catch (SQLException error) { throw new IOException("Unable to close Evidence Mentor queue", error); } + } +} diff --git a/src/main/java/org/pathlab/forge/evidence/EvidenceJobState.java b/src/main/java/org/pathlab/forge/evidence/EvidenceJobState.java new file mode 100644 index 0000000..dfa5e78 --- /dev/null +++ b/src/main/java/org/pathlab/forge/evidence/EvidenceJobState.java @@ -0,0 +1,18 @@ +package org.pathlab.forge.evidence; + +public enum EvidenceJobState { + QUEUED(false), + VALIDATING(false), + RUNNING(false), + REFINING(false), + PACKAGING(false), + COMPLETED(true), + ABSTAINED(true), + UNSUPPORTED(true), + FAILED(true), + CANCELLED(true); + + private final boolean terminal; + EvidenceJobState(boolean terminal) { this.terminal = terminal; } + public boolean terminal() { return terminal; } +} diff --git a/src/main/java/org/pathlab/forge/evidence/EvidenceMentorRunner.java b/src/main/java/org/pathlab/forge/evidence/EvidenceMentorRunner.java new file mode 100644 index 0000000..c922184 --- /dev/null +++ b/src/main/java/org/pathlab/forge/evidence/EvidenceMentorRunner.java @@ -0,0 +1,241 @@ +package org.pathlab.forge.evidence; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpServer; +import java.io.IOException; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.security.SecureRandom; +import java.time.Duration; +import java.time.Instant; +import java.util.Base64; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +/** Standalone authenticated loopback process for unattended Evidence Mentor jobs. */ +public final class EvidenceMentorRunner implements AutoCloseable { + private static final ObjectMapper JSON = new ObjectMapper(); + private final Path stateRoot; + private final String token; + private final EvidenceJobQueue queue; + private final HttpServer server; + private final ScheduledExecutorService worker; + private final java.util.concurrent.ExecutorService http; + + private EvidenceMentorRunner(Path stateRoot, int port, String token, boolean workerEnabled) + throws IOException { + this.stateRoot = stateRoot.toAbsolutePath().normalize(); + this.token = token; + require(token != null && token.length() >= 32, "Loopback token is too short"); + Files.createDirectories(this.stateRoot); + queue = new EvidenceJobQueue(this.stateRoot.resolve("jobs.sqlite3")); + server = HttpServer.create( + new InetSocketAddress(InetAddress.getByName("127.0.0.1"), port), 16); + server.createContext("/health", this::handle); + server.createContext("/v1/jobs", this::handle); + http = Executors.newFixedThreadPool(2, runnable -> { + var thread = new Thread(runnable, "pathlab-evidence-ipc"); + thread.setDaemon(true); + return thread; + }); + server.setExecutor(http); + server.start(); + worker = Executors.newSingleThreadScheduledExecutor(runnable -> { + var thread = new Thread(runnable, "pathlab-evidence-worker"); + thread.setDaemon(true); + thread.setPriority(Thread.MIN_PRIORITY); + return thread; + }); + if (workerEnabled) worker.scheduleWithFixedDelay(this::processOne, 0, 1, TimeUnit.SECONDS); + } + + public static EvidenceMentorRunner start(Path stateRoot, int port, String token, boolean worker) + throws IOException { + return new EvidenceMentorRunner(stateRoot, port, token, worker); + } + + public URI uri(String path) { + return URI.create("http://127.0.0.1:" + server.getAddress().getPort() + path); + } + + Optional find(String id) throws IOException { + return queue.find(id); + } + + private void handle(HttpExchange exchange) throws IOException { + try { + if (!exchange.getRemoteAddress().getAddress().isLoopbackAddress()) { + respond(exchange, 403, java.util.Map.of("error", "loopback_required")); + return; + } + if (!authorized(exchange)) { + respond(exchange, 401, java.util.Map.of("error", "authentication_required")); + return; + } + var path = exchange.getRequestURI().getPath(); + if ("/health".equals(path) && "GET".equals(exchange.getRequestMethod())) { + respond(exchange, 200, java.util.Map.of( + "schema", "pathlab.evidence-runner-status/1", "status", "ready")); + return; + } + if ("/v1/jobs".equals(path) && "POST".equals(exchange.getRequestMethod())) { + submit(exchange); + return; + } + var prefix = "/v1/jobs/"; + if (path.startsWith(prefix)) { + var id = path.substring(prefix.length()); + if (!id.matches("[A-Za-z0-9._-]{1,120}")) { + respond(exchange, 404, java.util.Map.of("error", "job_not_found")); + } else if ("GET".equals(exchange.getRequestMethod())) { + var job = queue.find(id); + if (job.isEmpty()) respond(exchange, 404, java.util.Map.of("error", "job_not_found")); + else respond(exchange, 200, jobJson(job.get())); + } else if ("DELETE".equals(exchange.getRequestMethod())) { + queue.requestCancel(id, Instant.now()); + exchange.sendResponseHeaders(204, -1); + } else { + respond(exchange, 405, java.util.Map.of("error", "method_not_allowed")); + } + return; + } + respond(exchange, 404, java.util.Map.of("error", "not_found")); + } catch (IllegalArgumentException error) { + respond(exchange, 422, java.util.Map.of("error", "invalid_request", "detail", error.getMessage())); + } catch (Exception error) { + respond(exchange, 500, java.util.Map.of("error", "runner_error")); + } finally { + exchange.close(); + } + } + + private void submit(HttpExchange exchange) throws IOException { + var raw = exchange.getRequestBody().readNBytes(65_537); + require(raw.length <= 65_536, "Job request is too large"); + JsonNode value = JSON.readTree(raw); + require(value != null && value.isObject() && fieldNames(value).equals(Set.of("id", "requestPath")), + "Job submission fields are invalid"); + var id = text(value, "id"); + require(id.matches("[A-Za-z0-9._-]{1,120}"), "Job id is invalid"); + var job = queue.submit(id, Path.of(text(value, "requestPath")), Instant.now()); + respond(exchange, 202, jobJson(job)); + } + + private boolean authorized(HttpExchange exchange) { + var expected = ("Bearer " + token).getBytes(StandardCharsets.UTF_8); + var supplied = exchange.getRequestHeaders().getFirst("Authorization"); + return supplied != null && MessageDigest.isEqual( + expected, supplied.getBytes(StandardCharsets.UTF_8)); + } + + private void processOne() { + var workerId = "cpu-" + ProcessHandle.current().pid(); + try { + var claimed = queue.claimNext(workerId, Instant.now(), Duration.ofMinutes(2)); + if (claimed.isEmpty()) return; + try { + new EvidenceJobProcessor(queue, stateRoot).process(claimed.get(), workerId, Instant.now()); + } catch (EvidenceJobProcessor.CancellationException ignored) { + // Cancellation was persisted at a checkpoint. + } catch (IllegalArgumentException permanent) { + var current = queue.find(claimed.get().id()).orElseThrow(); + var state = permanent.getMessage().contains("not installed") + || permanent.getMessage().contains("unsupported") + ? EvidenceJobState.UNSUPPORTED : EvidenceJobState.FAILED; + queue.checkpoint(current.id(), workerId, state, state.name().toLowerCase(), + current.progress(), permanent.getMessage(), Instant.now(), Duration.ofMinutes(2)); + } catch (IOException transientFailure) { + var current = queue.find(claimed.get().id()).orElseThrow(); + if (!current.state().terminal()) { + queue.retryOrFail(current.id(), workerId, transientFailure.getMessage(), Instant.now()); + } + } + } catch (Exception ignored) { + // Durable lease expiry permits recovery on next iteration or restart. + } + } + + private static java.util.Map jobJson(EvidenceJob job) { + return java.util.Map.of( + "id", job.id(), "state", job.state().name().toLowerCase(), + "stage", job.stage(), "progress", job.progress(), + "retryCount", job.retryCount(), "cancelRequested", job.cancelRequested(), + "detail", job.detail(), "updatedAt", job.updatedAt().toString()); + } + + private static Set fieldNames(JsonNode value) { + var result = new java.util.HashSet(); + value.fieldNames().forEachRemaining(result::add); + return result; + } + + private static String text(JsonNode value, String field) { + var node = value.path(field); + require(node.isTextual() && !node.textValue().isBlank(), "Job field is invalid: " + field); + return node.textValue(); + } + + private static void respond(HttpExchange exchange, int status, Object body) throws IOException { + if (exchange.getResponseCode() != -1) return; + var payload = JSON.writeValueAsBytes(body); + exchange.getResponseHeaders().set("Content-Type", "application/json; charset=utf-8"); + exchange.getResponseHeaders().set("Cache-Control", "no-store"); + exchange.sendResponseHeaders(status, payload.length); + exchange.getResponseBody().write(payload); + } + + private static void require(boolean condition, String message) { + if (!condition) throw new IllegalArgumentException(message); + } + + @Override + public void close() throws IOException { + server.stop(0); + worker.shutdownNow(); + http.shutdownNow(); + queue.close(); + } + + public static void main(String[] args) throws Exception { + var state = defaultStateRoot(); + var port = 8765; + for (var index = 0; index < args.length; index++) { + if ("--state".equals(args[index]) && index + 1 < args.length) state = Path.of(args[++index]); + else if ("--port".equals(args[index]) && index + 1 < args.length) port = Integer.parseInt(args[++index]); + else throw new IllegalArgumentException("Usage: EvidenceMentorRunner [--state PATH] [--port PORT]"); + } + Files.createDirectories(state); + var tokenPath = state.resolve("ipc-token"); + if (!Files.exists(tokenPath)) { + var bytes = new byte[32]; + new SecureRandom().nextBytes(bytes); + Files.writeString(tokenPath, Base64.getUrlEncoder().withoutPadding().encodeToString(bytes)); + } + var runner = start(state, port, Files.readString(tokenPath).trim(), true); + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + try { runner.close(); } catch (IOException ignored) { } + })); + new CountDownLatch(1).await(); + } + + public static Path defaultStateRoot() { + var configured = System.getenv("PATHLAB_EVIDENCE_STATE"); + if (configured != null && !configured.isBlank()) return Path.of(configured); + var dataDrive = Path.of("D:\\PathLabData"); + if (Files.isDirectory(dataDrive)) return dataDrive.resolve("EvidenceMentor/state"); + var local = System.getenv("LOCALAPPDATA"); + return Path.of(local == null ? System.getProperty("user.home") : local) + .resolve("PathLab/EvidenceMentor/state"); + } +} diff --git a/src/main/java/org/pathlab/forge/evidence/EvidencePackManifest.java b/src/main/java/org/pathlab/forge/evidence/EvidencePackManifest.java new file mode 100644 index 0000000..27c11d2 --- /dev/null +++ b/src/main/java/org/pathlab/forge/evidence/EvidencePackManifest.java @@ -0,0 +1,186 @@ +package org.pathlab.forge.evidence; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.util.HexFormat; +import java.util.List; +import java.util.Set; +import java.util.regex.Pattern; + +/** Validated, immutable description of one locally installed Evidence Mentor pack. */ +public record EvidencePackManifest( + Path path, + String packId, + String version, + Capability capability, + Set acceptedStains, + String preprocessingId, + String allowedUse, + boolean redistributable, + boolean derivativesAllowed, + int maxRamMiB, + int maxVramMiB, + int maxSeconds, + ValidationStatus validationStatus, + String sha256, + List artifacts, + Set markers) { + public static final String SCHEMA = "pathlab.ai-pack/1"; + private static final ObjectMapper JSON = new ObjectMapper(); + private static final Pattern ID = Pattern.compile("[a-z0-9][a-z0-9._-]{0,119}"); + private static final Pattern SHA = Pattern.compile("[a-f0-9]{64}"); + private static final Set STAINS = Set.of("he", "ihc_dab"); + private static final Set MARKERS = Set.of("generic", "er", "pr", "ki-67", "her2", "pd-l1"); + + public static EvidencePackManifest load(Path manifest) throws IOException { + var normalized = manifest.toAbsolutePath().normalize(); + var bytes = Files.readAllBytes(normalized); + var root = JSON.readTree(bytes); + require(root.isObject(), "AI pack must be a JSON object"); + require(SCHEMA.equals(text(root, "schema")), "AI pack schema is unsupported"); + var packId = text(root, "packId"); + require(ID.matcher(packId).matches(), "AI pack id is invalid"); + var version = text(root, "version"); + require(version.matches("[A-Za-z0-9][A-Za-z0-9._-]{0,63}"), "AI pack version is invalid"); + var capability = Capability.fromWire(text(root, "capability")); + var acceptedStains = stringSet(root.path("acceptedStains"), STAINS, "accepted stain"); + require(!acceptedStains.isEmpty(), "AI pack requires an accepted stain"); + var preprocessing = object(root, "preprocessing"); + var preprocessingId = text(preprocessing, "id"); + require(preprocessingId.length() <= 120, "AI pack preprocessing id is invalid"); + boundedInt(preprocessing, "tilePixels", 64, 2_048); + var rights = object(root, "rights"); + var allowedUse = text(rights, "allowedUse"); + require(Set.of("private-research", "benchmark-only").contains(allowedUse), "AI pack allowed use is invalid"); + var resources = object(root, "resourceEnvelope"); + var maxRam = boundedInt(resources, "maxRamMiB", 64, 16_384); + var maxVram = boundedInt(resources, "maxVramMiB", 0, 4_608); + var maxSeconds = boundedInt(resources, "maxSeconds", 1, 1_200); + require(resources.has("network") && !resources.path("network").asBoolean(true), + "Analysis packs must be offline"); + var validation = object(root, "validation"); + var status = ValidationStatus.fromWire(text(validation, "status")); + var artifacts = parseArtifacts(root.path("artifacts")); + var markers = root.has("markers") + ? stringSet(root.path("markers"), MARKERS, "marker") + : Set.of(); + if (capability == Capability.IHC_DESCRIPTIVE) { + require(markers.contains("generic"), "IHC packs must include the generic marker"); + } else { + require(markers.isEmpty(), "Only IHC packs may declare markers"); + } + return new EvidencePackManifest( + normalized, packId, version, capability, acceptedStains, preprocessingId, allowedUse, + booleanValue(rights, "redistributable"), + booleanValue(rights, "derivativesAllowed"), + maxRam, maxVram, maxSeconds, status, sha256(bytes), artifacts, markers); + } + + public boolean pilotEligible() { + return "private-research".equals(allowedUse) + && validationStatus != ValidationStatus.BLOCKED + && validationStatus != ValidationStatus.NOT_EVALUABLE; + } + + public void requirePilotEligible() { + require(pilotEligible(), "AI pack is not eligible for the private pilot"); + } + + public void requireStain(String stain) { + require(acceptedStains.contains(stain), "AI pack does not support stain " + stain); + } + + private static List parseArtifacts(JsonNode node) { + require(node.isArray() && node.size() <= 16, "AI pack artifacts are invalid"); + var result = new java.util.ArrayList(); + for (var item : node) { + var name = text(item, "name"); + var hash = text(item, "sha256"); + var source = text(item, "source"); + require(name.length() <= 160 && SHA.matcher(hash).matches() + && source.length() <= 1_000, "AI pack artifact is invalid"); + result.add(new Artifact(name, hash, source)); + } + return List.copyOf(result); + } + + private static Set stringSet(JsonNode node, Set allowed, String label) { + require(node.isArray() && node.size() <= 16, "AI pack " + label + " list is invalid"); + var result = new java.util.LinkedHashSet(); + for (var item : node) { + require(item.isTextual() && allowed.contains(item.textValue()), + "AI pack " + label + " is invalid"); + require(result.add(item.textValue()), "AI pack " + label + " is duplicated"); + } + return Set.copyOf(result); + } + + private static JsonNode object(JsonNode root, String name) { + var value = root.path(name); + require(value.isObject(), "AI pack " + name + " is invalid"); + return value; + } + + private static String text(JsonNode root, String name) { + var value = root.path(name); + require(value.isTextual() && !value.textValue().isBlank(), "AI pack " + name + " is invalid"); + return value.textValue(); + } + + private static int boundedInt(JsonNode root, String name, int minimum, int maximum) { + var value = root.path(name); + require(value.isIntegralNumber() && value.canConvertToInt() + && value.intValue() >= minimum && value.intValue() <= maximum, + "AI pack " + name + " is outside the supported resource envelope"); + return value.intValue(); + } + + private static boolean booleanValue(JsonNode root, String name) { + var value = root.path(name); + require(value.isBoolean(), "AI pack " + name + " is invalid"); + return value.booleanValue(); + } + + private static String sha256(byte[] value) throws IOException { + try { + return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(value)); + } catch (Exception error) { + throw new IOException("SHA-256 is unavailable", error); + } + } + + private static void require(boolean condition, String message) { + if (!condition) throw new IllegalArgumentException(message); + } + + public record Artifact(String name, String sha256, String source) {} + + public enum Capability { + HE_EVIDENCE("he-evidence"), + CELL_MORPHOLOGY("cell-morphology"), + IHC_DESCRIPTIVE("ihc-descriptive"); + + private final String wire; + Capability(String wire) { this.wire = wire; } + public String wire() { return wire; } + static Capability fromWire(String value) { + return java.util.Arrays.stream(values()).filter(item -> item.wire.equals(value)).findFirst() + .orElseThrow(() -> new IllegalArgumentException("AI pack capability is invalid")); + } + } + + public enum ValidationStatus { + EXPERIMENTAL("experimental"), QUALIFIED("qualified"), NOT_EVALUABLE("not-evaluable"), BLOCKED("blocked"); + private final String wire; + ValidationStatus(String wire) { this.wire = wire; } + public String wire() { return wire; } + static ValidationStatus fromWire(String value) { + return java.util.Arrays.stream(values()).filter(item -> item.wire.equals(value)).findFirst() + .orElseThrow(() -> new IllegalArgumentException("AI pack validation status is invalid")); + } + } +} diff --git a/src/main/java/org/pathlab/forge/evidence/EvidenceSigner.java b/src/main/java/org/pathlab/forge/evidence/EvidenceSigner.java new file mode 100644 index 0000000..89eb836 --- /dev/null +++ b/src/main/java/org/pathlab/forge/evidence/EvidenceSigner.java @@ -0,0 +1,78 @@ +package org.pathlab.forge.evidence; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.GeneralSecurityException; +import java.security.KeyFactory; +import java.security.KeyPair; +import java.security.KeyPairGenerator; +import java.security.MessageDigest; +import java.security.Signature; +import java.security.spec.PKCS8EncodedKeySpec; +import java.security.spec.X509EncodedKeySpec; +import java.util.Base64; +import java.util.HexFormat; + +/** Persistent per-install Ed25519 identity for immutable evidence manifests. */ +public final class EvidenceSigner { + private final Path privateKey; + private final Path publicKey; + + public EvidenceSigner(Path root) { + var normalized = root.toAbsolutePath().normalize(); + privateKey = normalized.resolve("evidence-signing-key.pk8"); + publicKey = normalized.resolve("evidence-signing-key.pub"); + } + + public SignedValue sign(String message) throws IOException { + try { + var pair = loadOrCreate(); + var signer = Signature.getInstance("Ed25519"); + signer.initSign(pair.getPrivate()); + signer.update(message.getBytes(java.nio.charset.StandardCharsets.UTF_8)); + var publicBytes = pair.getPublic().getEncoded(); + return new SignedValue( + sha256(publicBytes), + Base64.getUrlEncoder().withoutPadding().encodeToString(publicBytes), + Base64.getUrlEncoder().withoutPadding().encodeToString(signer.sign())); + } catch (GeneralSecurityException error) { + throw new IOException("Ed25519 evidence signing is unavailable", error); + } + } + + private KeyPair loadOrCreate() throws IOException, GeneralSecurityException { + Files.createDirectories(privateKey.getParent()); + var factory = KeyFactory.getInstance("Ed25519"); + if (Files.isRegularFile(privateKey) && Files.isRegularFile(publicKey)) { + return new KeyPair( + factory.generatePublic(new X509EncodedKeySpec(Files.readAllBytes(publicKey))), + factory.generatePrivate(new PKCS8EncodedKeySpec(Files.readAllBytes(privateKey)))); + } + if (Files.exists(privateKey) || Files.exists(publicKey)) { + throw new IOException("Evidence signing keypair is incomplete"); + } + var pair = KeyPairGenerator.getInstance("Ed25519").generateKeyPair(); + writePrivate(privateKey, pair.getPrivate().getEncoded()); + writePrivate(publicKey, pair.getPublic().getEncoded()); + return pair; + } + + private static void writePrivate(Path target, byte[] value) throws IOException { + var partial = target.resolveSibling(target.getFileName() + ".partial"); + Files.write(partial, value); + try { + Files.setPosixFilePermissions(partial, + java.nio.file.attribute.PosixFilePermissions.fromString("rw-------")); + } catch (UnsupportedOperationException ignored) { + // The Windows per-user data root supplies the ACL boundary. + } + Files.move(partial, target, java.nio.file.StandardCopyOption.REPLACE_EXISTING); + } + + private static String sha256(byte[] value) throws GeneralSecurityException { + return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(value)); + } + + public record SignedValue(String keyId, String publicKeyDer, String signature) {} +} diff --git a/src/main/java/org/pathlab/forge/viewer/PrivateResultsBundleBuilder.java b/src/main/java/org/pathlab/forge/viewer/PrivateResultsBundleBuilder.java index 4fb8af0..5fbc388 100644 --- a/src/main/java/org/pathlab/forge/viewer/PrivateResultsBundleBuilder.java +++ b/src/main/java/org/pathlab/forge/viewer/PrivateResultsBundleBuilder.java @@ -29,6 +29,15 @@ public Result build( String artifactRevisionId, String slideSha256, List annotations) throws IOException { + return build(output, artifactRevisionId, slideSha256, annotations, null); + } + + public Result build( + Path output, + String artifactRevisionId, + String slideSha256, + List annotations, + Path evidenceManifest) throws IOException { var manifest = JSON.writeValueAsBytes(Map.of( "schema", "pathlab-private-results/v1", "artifactRevisionId", artifactRevisionId, @@ -68,6 +77,13 @@ public Result build( entries.put("objects.ndjson", objects.toString().getBytes(StandardCharsets.UTF_8)); entries.put("measurements.ndjson", measurements.toString().getBytes(StandardCharsets.UTF_8)); entries.put("runs.ndjson", runs.getBytes(StandardCharsets.UTF_8)); + if (evidenceManifest != null) { + var evidence = Files.readAllBytes(evidenceManifest.toAbsolutePath().normalize()); + if (evidence.length == 0 || evidence.length > 2 * 1024 * 1024) { + throw new IOException("Evidence manifest exceeds delivery limit"); + } + entries.put("evidence.json", evidence); + } var normalized = output.toAbsolutePath().normalize(); Files.createDirectories(normalized.getParent()); var partial = normalized.resolveSibling(normalized.getFileName() + ".partial"); diff --git a/src/main/java/org/pathlab/forge/viewer/ViewerPairingService.java b/src/main/java/org/pathlab/forge/viewer/ViewerPairingService.java index abedea6..31b8959 100644 --- a/src/main/java/org/pathlab/forge/viewer/ViewerPairingService.java +++ b/src/main/java/org/pathlab/forge/viewer/ViewerPairingService.java @@ -22,6 +22,8 @@ import java.util.concurrent.Executors; import java.util.regex.Pattern; import org.pathlab.forge.annotation.AnnotationRecord; +import org.pathlab.forge.evidence.EvidenceArtifactStore; +import org.pathlab.forge.evidence.EvidenceMentorRunner; import org.pathlab.forge.annotation.AnnotationTransformer; import org.pathlab.forge.conversion.ArtifactIntegrityStamp; import org.pathlab.forge.conversion.ArtifactRevision; @@ -646,8 +648,11 @@ private void syncPrivateResults( .toList(); var sidecar = Path.of(revision.omePath()).resolveSibling( revision.id() + ".plresults"); + var evidence = EvidenceArtifactStore.find( + EvidenceMentorRunner.defaultStateRoot(), revision.omeSha256(), revision.id()) + .orElse(null); var bundle = new PrivateResultsBundleBuilder().build( - sidecar, revision.id(), revision.omeSha256(), transformed); + sidecar, revision.id(), revision.omeSha256(), transformed, evidence); var create = sendJson( credential.base().resolve("/api/v2/desktop/slides/" + readyJob.remoteSlideId() + "/result-deliveries"), diff --git a/src/main/resources/evidence-packs/cell-od-watershed-v1.json b/src/main/resources/evidence-packs/cell-od-watershed-v1.json new file mode 100644 index 0000000..9a75607 --- /dev/null +++ b/src/main/resources/evidence-packs/cell-od-watershed-v1.json @@ -0,0 +1,34 @@ +{ + "schema": "pathlab.ai-pack/1", + "packId": "cell-od-watershed-v1", + "version": "1", + "capability": "cell-morphology", + "acceptedStains": [ + "he", + "ihc_dab" + ], + "preprocessing": { + "id": "od-v1", + "tilePixels": 512 + }, + "artifacts": [], + "rights": { + "license": "PathLab deterministic implementation", + "allowedUse": "private-research", + "redistributable": false, + "derivativesAllowed": false, + "reviewedAt": "2026-08-22T00:00:00Z" + }, + "resourceEnvelope": { + "maxRamMiB": 1024, + "maxVramMiB": 0, + "maxSeconds": 120, + "network": false + }, + "validation": { + "status": "experimental", + "modelCard": "docs/model-cards/cell-od-watershed-v1.md", + "heldOutEvaluation": "pending-independent-qualification" + }, + "outputSchema": "pathlab.ai-evidence/1" +} diff --git a/src/main/resources/evidence-packs/he-dinov2-small-v1.json b/src/main/resources/evidence-packs/he-dinov2-small-v1.json new file mode 100644 index 0000000..f416536 --- /dev/null +++ b/src/main/resources/evidence-packs/he-dinov2-small-v1.json @@ -0,0 +1,33 @@ +{ + "schema": "pathlab.ai-pack/1", + "packId": "he-dinov2-small-v1", + "version": "1", + "capability": "he-evidence", + "acceptedStains": [ + "he" + ], + "preprocessing": { + "id": "dinov2-v1", + "tilePixels": 512 + }, + "artifacts": [], + "rights": { + "license": "Apache-2.0 upstream; exact artifact ledger pending", + "allowedUse": "private-research", + "redistributable": false, + "derivativesAllowed": false, + "reviewedAt": "2026-08-22T00:00:00Z" + }, + "resourceEnvelope": { + "maxRamMiB": 16384, + "maxVramMiB": 4608, + "maxSeconds": 300, + "network": false + }, + "validation": { + "status": "not-evaluable", + "modelCard": "docs/model-cards/he-dinov2-small-v1.md", + "heldOutEvaluation": "not-evaluable-no-installed-artifact" + }, + "outputSchema": "pathlab.ai-evidence/1" +} diff --git a/src/main/resources/evidence-packs/he-gigapath-benchmark-v1.json b/src/main/resources/evidence-packs/he-gigapath-benchmark-v1.json new file mode 100644 index 0000000..579b980 --- /dev/null +++ b/src/main/resources/evidence-packs/he-gigapath-benchmark-v1.json @@ -0,0 +1,33 @@ +{ + "schema": "pathlab.ai-pack/1", + "packId": "he-gigapath-benchmark-v1", + "version": "1", + "capability": "he-evidence", + "acceptedStains": [ + "he" + ], + "preprocessing": { + "id": "gigapath-v1", + "tilePixels": 512 + }, + "artifacts": [], + "rights": { + "license": "Upstream model card excludes deployed use", + "allowedUse": "benchmark-only", + "redistributable": false, + "derivativesAllowed": false, + "reviewedAt": "2026-08-22T00:00:00Z" + }, + "resourceEnvelope": { + "maxRamMiB": 16384, + "maxVramMiB": 4608, + "maxSeconds": 300, + "network": false + }, + "validation": { + "status": "not-evaluable", + "modelCard": "docs/model-cards/he-gigapath-benchmark-v1.md", + "heldOutEvaluation": "not-evaluable-no-installed-artifact" + }, + "outputSchema": "pathlab.ai-evidence/1" +} diff --git a/src/main/resources/evidence-packs/he-hibou-b-v1.json b/src/main/resources/evidence-packs/he-hibou-b-v1.json new file mode 100644 index 0000000..2d1da91 --- /dev/null +++ b/src/main/resources/evidence-packs/he-hibou-b-v1.json @@ -0,0 +1,33 @@ +{ + "schema": "pathlab.ai-pack/1", + "packId": "he-hibou-b-v1", + "version": "1", + "capability": "he-evidence", + "acceptedStains": [ + "he" + ], + "preprocessing": { + "id": "hibou-v1", + "tilePixels": 512 + }, + "artifacts": [], + "rights": { + "license": "Upstream terms require exact artifact review", + "allowedUse": "private-research", + "redistributable": false, + "derivativesAllowed": false, + "reviewedAt": "2026-08-22T00:00:00Z" + }, + "resourceEnvelope": { + "maxRamMiB": 16384, + "maxVramMiB": 4608, + "maxSeconds": 300, + "network": false + }, + "validation": { + "status": "not-evaluable", + "modelCard": "docs/model-cards/he-hibou-b-v1.md", + "heldOutEvaluation": "not-evaluable-no-installed-artifact" + }, + "outputSchema": "pathlab.ai-evidence/1" +} diff --git a/src/main/resources/evidence-packs/ihc-descriptive-v1.json b/src/main/resources/evidence-packs/ihc-descriptive-v1.json new file mode 100644 index 0000000..61ecc51 --- /dev/null +++ b/src/main/resources/evidence-packs/ihc-descriptive-v1.json @@ -0,0 +1,41 @@ +{ + "schema": "pathlab.ai-pack/1", + "packId": "ihc-descriptive-v1", + "version": "1", + "capability": "ihc-descriptive", + "acceptedStains": [ + "ihc_dab" + ], + "preprocessing": { + "id": "od-v1", + "tilePixels": 512 + }, + "artifacts": [], + "rights": { + "license": "PathLab deterministic implementation", + "allowedUse": "private-research", + "redistributable": false, + "derivativesAllowed": false, + "reviewedAt": "2026-08-22T00:00:00Z" + }, + "resourceEnvelope": { + "maxRamMiB": 1024, + "maxVramMiB": 0, + "maxSeconds": 120, + "network": false + }, + "validation": { + "status": "experimental", + "modelCard": "docs/model-cards/ihc-descriptive-v1.md", + "heldOutEvaluation": "pending-independent-qualification" + }, + "outputSchema": "pathlab.ai-evidence/1", + "markers": [ + "generic", + "er", + "pr", + "ki-67", + "her2", + "pd-l1" + ] +} diff --git a/src/test/java/org/pathlab/forge/evidence/BrightfieldTileAnalyzerTest.java b/src/test/java/org/pathlab/forge/evidence/BrightfieldTileAnalyzerTest.java new file mode 100644 index 0000000..f6fb850 --- /dev/null +++ b/src/test/java/org/pathlab/forge/evidence/BrightfieldTileAnalyzerTest.java @@ -0,0 +1,41 @@ +package org.pathlab.forge.evidence; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.awt.Color; +import java.awt.image.BufferedImage; +import org.junit.jupiter.api.Test; + +final class BrightfieldTileAnalyzerTest { + @Test + void countsBoundedNuclearComponentsAndDescribesDabWithoutClinicalCalls() { + var image = new BufferedImage(32, 24, BufferedImage.TYPE_INT_RGB); + var graphics = image.createGraphics(); + graphics.setColor(Color.WHITE); + graphics.fillRect(0, 0, 32, 24); + graphics.setColor(new Color(75, 45, 125)); + graphics.fillRect(3, 4, 5, 5); + graphics.fillRect(18, 4, 6, 5); + graphics.setColor(new Color(135, 90, 45)); + graphics.fillRect(8, 14, 12, 6); + graphics.dispose(); + + var result = BrightfieldTileAnalyzer.analyze(image, "ki-67"); + + assertEquals(2, result.cellCount()); + assertTrue(result.meanNucleusAreaPx2() >= 25); + assertTrue(result.dabAreaFraction() > 0); + assertTrue(result.meanDabOd() > 0); + assertEquals("nuclear", result.compartment()); + assertTrue(result.researchEstimate()); + } + + @Test + void unknownMarkerUsesGenericDescriptiveCompartment() { + var image = new BufferedImage(8, 8, BufferedImage.TYPE_INT_RGB); + var result = BrightfieldTileAnalyzer.analyze(image, "cdx2"); + assertEquals("generic", result.marker()); + assertEquals("generic-region", result.compartment()); + } +} diff --git a/src/test/java/org/pathlab/forge/evidence/EvidenceArtifactStoreTest.java b/src/test/java/org/pathlab/forge/evidence/EvidenceArtifactStoreTest.java new file mode 100644 index 0000000..f87b263 --- /dev/null +++ b/src/test/java/org/pathlab/forge/evidence/EvidenceArtifactStoreTest.java @@ -0,0 +1,35 @@ +package org.pathlab.forge.evidence; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.file.Files; +import java.nio.file.Path; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +final class EvidenceArtifactStoreTest { + @TempDir Path temporaryDirectory; + + @Test + void selectsOnlyExactSlideRevisionAndTerminalSignedArtifact() throws Exception { + write("old", "a".repeat(64), "revision-old", "completed"); + var exact = write("exact", "a".repeat(64), "revision-1", "completed"); + write("running", "a".repeat(64), "revision-1", "running"); + + assertEquals(exact, EvidenceArtifactStore.find( + temporaryDirectory, "a".repeat(64), "revision-1").orElseThrow()); + assertTrue(EvidenceArtifactStore.find( + temporaryDirectory, "b".repeat(64), "revision-1").isEmpty()); + } + + private Path write(String job, String sha, String revision, String status) throws Exception { + var path = temporaryDirectory.resolve("artifacts").resolve(job).resolve("evidence.json"); + Files.createDirectories(path.getParent()); + Files.writeString(path, "{\"schema\":\"pathlab.ai-evidence/1\",\"source\":{" + + "\"slideSha256\":\"" + sha + "\",\"revision\":\"" + revision + "\"}," + + "\"status\":\"" + status + "\",\"manifestSha256\":\"" + "c".repeat(64) + "\"," + + "\"signature\":{\"algorithm\":\"Ed25519\"}}" ); + return path.toAbsolutePath().normalize(); + } +} diff --git a/src/test/java/org/pathlab/forge/evidence/EvidenceBundleWriterTest.java b/src/test/java/org/pathlab/forge/evidence/EvidenceBundleWriterTest.java new file mode 100644 index 0000000..1cec4fc --- /dev/null +++ b/src/test/java/org/pathlab/forge/evidence/EvidenceBundleWriterTest.java @@ -0,0 +1,58 @@ +package org.pathlab.forge.evidence; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.fasterxml.jackson.databind.ObjectMapper; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.security.KeyFactory; +import java.security.Signature; +import java.security.spec.X509EncodedKeySpec; +import java.util.Base64; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +final class EvidenceBundleWriterTest { + private static final ObjectMapper JSON = new ObjectMapper(); + @TempDir Path temporaryDirectory; + + @Test + void writesStableVerifiableSlideBoundEvidence() throws Exception { + var unsigned = JSON.createObjectNode(); + unsigned.put("schema", "pathlab.ai-evidence/1"); + unsigned.put("bundleId", "bundle-1"); + unsigned.set("source", JSON.valueToTree(java.util.Map.of( + "slideSha256", "a".repeat(64), "revision", "revision-1"))); + unsigned.put("researchOnly", true); + unsigned.put("notDiagnostic", true); + unsigned.put("reviewRequired", true); + var output = temporaryDirectory.resolve("bundle.json"); + var result = new EvidenceBundleWriter(temporaryDirectory.resolve("keys")).write(output, unsigned); + var stored = JSON.readTree(output.toFile()); + assertEquals(result.manifestSha256(), stored.path("manifestSha256").asText()); + var signature = stored.path("signature"); + var verifier = Signature.getInstance("Ed25519"); + verifier.initVerify(KeyFactory.getInstance("Ed25519").generatePublic(new X509EncodedKeySpec( + Base64.getUrlDecoder().decode(signature.path("publicKeyDer").asText())))); + verifier.update(("pathlab.ai-evidence/1\n" + result.manifestSha256()).getBytes(StandardCharsets.UTF_8)); + assertTrue(verifier.verify(Base64.getUrlDecoder().decode(signature.path("value").asText()))); + } + + @Test + void canonicalHashDoesNotDependOnObjectInsertionOrder() throws Exception { + var first = JSON.createObjectNode().put("z", 1).put("a", 2); + var second = JSON.createObjectNode().put("a", 2).put("z", 1); + assertEquals( + EvidenceBundleWriter.sha256(EvidenceBundleWriter.canonicalBytes(first)), + EvidenceBundleWriter.sha256(EvidenceBundleWriter.canonicalBytes(second))); + } + + @Test + void canonicalNumbersMatchViewerContract() throws Exception { + var value = JSON.createObjectNode().put("whole", 100.0).put("zero", -0.0).put("part", 0.125); + assertEquals( + "{\"part\":0.125,\"whole\":100,\"zero\":0}", + new String(EvidenceBundleWriter.canonicalBytes(value), StandardCharsets.UTF_8)); + } +} diff --git a/src/test/java/org/pathlab/forge/evidence/EvidenceJobProcessorTest.java b/src/test/java/org/pathlab/forge/evidence/EvidenceJobProcessorTest.java new file mode 100644 index 0000000..7bafac3 --- /dev/null +++ b/src/test/java/org/pathlab/forge/evidence/EvidenceJobProcessorTest.java @@ -0,0 +1,170 @@ +package org.pathlab.forge.evidence; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.fasterxml.jackson.databind.ObjectMapper; +import java.awt.Color; +import java.awt.image.BufferedImage; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.time.Duration; +import java.time.Instant; +import java.util.HexFormat; +import javax.imageio.ImageIO; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +final class EvidenceJobProcessorTest { + private static final ObjectMapper JSON = new ObjectMapper(); + @TempDir Path temporaryDirectory; + + @Test + void treatsMalformedRequestJsonAsPermanentValidationFailure() throws Exception { + var request = temporaryDirectory.resolve("malformed-request.json"); + Files.writeString(request, "{not-json"); + var now = Instant.parse("2026-08-22T00:00:00Z"); + + try (var queue = new EvidenceJobQueue(temporaryDirectory.resolve("state/jobs.sqlite3"))) { + queue.submit("job-invalid", request, now); + var claimed = queue.claimNext("worker-1", now, Duration.ofMinutes(2)).orElseThrow(); + + assertThrows(IllegalArgumentException.class, () -> + new EvidenceJobProcessor(queue, temporaryDirectory.resolve("state")) + .process(claimed, "worker-1", now.plusSeconds(1))); + } + } + + @Test + void treatsUndecodablePreviewAsPermanentValidationFailure() throws Exception { + var source = temporaryDirectory.resolve("source.bin"); + Files.writeString(source, "immutable source bytes"); + var preview = temporaryDirectory.resolve("preview.png"); + Files.writeString(preview, "not an image"); + var pack = temporaryDirectory.resolve("pack.json"); + Files.writeString(pack, packJson()); + var request = request(source, preview, pack, "revision-invalid-preview"); + var now = Instant.parse("2026-08-22T00:00:00Z"); + + try (var queue = new EvidenceJobQueue(temporaryDirectory.resolve("state/jobs.sqlite3"))) { + queue.submit("job-invalid-preview", request, now); + var claimed = queue.claimNext("worker-1", now, Duration.ofMinutes(2)).orElseThrow(); + + assertThrows(IllegalArgumentException.class, () -> + new EvidenceJobProcessor(queue, temporaryDirectory.resolve("state")) + .process(claimed, "worker-1", now.plusSeconds(1))); + } + } + + @Test + void signsQcEvidenceAndAbstainsForBlankPreview() throws Exception { + var source = temporaryDirectory.resolve("blank-source.bin"); + Files.writeString(source, "immutable blank source bytes"); + var preview = temporaryDirectory.resolve("blank.png"); + var image = new BufferedImage(16, 12, BufferedImage.TYPE_INT_RGB); + var graphics = image.createGraphics(); + graphics.setColor(Color.WHITE); + graphics.fillRect(0, 0, 16, 12); + graphics.dispose(); + ImageIO.write(image, "png", preview.toFile()); + var pack = temporaryDirectory.resolve("blank-pack.json"); + Files.writeString(pack, packJson()); + var request = request(source, preview, pack, "revision-blank"); + var now = Instant.parse("2026-08-22T00:00:00Z"); + + try (var queue = new EvidenceJobQueue(temporaryDirectory.resolve("blank-state/jobs.sqlite3"))) { + queue.submit("job-blank", request, now); + var claimed = queue.claimNext("worker-1", now, Duration.ofMinutes(2)).orElseThrow(); + var result = new EvidenceJobProcessor(queue, temporaryDirectory.resolve("blank-state")) + .process(claimed, "worker-1", now.plusSeconds(1)); + + assertEquals(EvidenceJobState.ABSTAINED, result.state()); + var evidence = JSON.readTree(temporaryDirectory + .resolve("blank-state/artifacts/job-blank/evidence.json").toFile()); + assertEquals("abstained", evidence.path("status").asText()); + assertTrue(evidence.path("qc").path("abstentionReasons").size() > 0); + } + } + + @Test + void completesOfflineDescriptiveIhcJobIntoSignedAtomicEvidence() throws Exception { + var source = temporaryDirectory.resolve("source.bin"); + Files.writeString(source, "immutable source bytes"); + var preview = temporaryDirectory.resolve("preview.png"); + var image = new BufferedImage(16, 12, BufferedImage.TYPE_INT_RGB); + var graphics = image.createGraphics(); + graphics.setColor(Color.WHITE); + graphics.fillRect(0, 0, 16, 12); + graphics.setColor(new Color(70, 40, 120)); + graphics.fillRect(2, 2, 4, 4); + graphics.setColor(new Color(130, 85, 40)); + graphics.fillRect(8, 4, 6, 5); + graphics.dispose(); + ImageIO.write(image, "png", preview.toFile()); + var pack = temporaryDirectory.resolve("pack.json"); + Files.writeString(pack, packJson()); + var request = temporaryDirectory.resolve("request.json"); + JSON.writeValue(request.toFile(), java.util.Map.of( + "schema", "pathlab.evidence-job/1", + "sourcePath", source.toString(), + "sourceSha256", sha256(source), + "slideRevision", "revision-1", + "previewPath", preview.toString(), + "sourceWidth", 1600, + "sourceHeight", 1200, + "packManifest", pack.toString(), + "stain", "ihc_dab", + "marker", "ki-67")); + + var now = Instant.parse("2026-08-22T00:00:00Z"); + try (var queue = new EvidenceJobQueue(temporaryDirectory.resolve("state/jobs.sqlite3"))) { + queue.submit("job-1", request, now); + var claimed = queue.claimNext("worker-1", now, Duration.ofMinutes(2)).orElseThrow(); + var completed = new EvidenceJobProcessor(queue, temporaryDirectory.resolve("state")) + .process(claimed, "worker-1", now.plusSeconds(1)); + + assertEquals(EvidenceJobState.COMPLETED, completed.state()); + var evidence = JSON.readTree( + temporaryDirectory.resolve("state/artifacts/job-1/evidence.json").toFile()); + assertEquals("pathlab.ai-evidence/1", evidence.path("schema").asText()); + assertEquals("ki-67", evidence.path("ihcDescriptors").get(0).path("marker").asText()); + assertTrue(evidence.path("researchOnly").asBoolean()); + assertTrue(evidence.path("provenance").path("offlineAnalysis").asBoolean()); + assertEquals(64, evidence.path("manifestSha256").asText().length()); + } + } + + private static String packJson() { + return "{\"schema\":\"pathlab.ai-pack/1\",\"packId\":\"ihc-descriptive-v1\",\"version\":\"1\"," + + "\"capability\":\"ihc-descriptive\",\"acceptedStains\":[\"ihc_dab\"]," + + "\"preprocessing\":{\"id\":\"od-v1\",\"tilePixels\":512},\"artifacts\":[]," + + "\"rights\":{\"license\":\"internal\",\"allowedUse\":\"private-research\"," + + "\"redistributable\":false,\"derivativesAllowed\":false,\"reviewedAt\":\"2026-08-22T00:00:00Z\"}," + + "\"resourceEnvelope\":{\"maxRamMiB\":1024,\"maxVramMiB\":0,\"maxSeconds\":300,\"network\":false}," + + "\"validation\":{\"status\":\"experimental\",\"modelCard\":\"model.md\",\"heldOutEvaluation\":\"evaluation.json\"}," + + "\"outputSchema\":\"pathlab.ai-evidence/1\",\"markers\":[\"generic\",\"ki-67\"]}"; + } + + private Path request(Path source, Path preview, Path pack, String revision) throws Exception { + var request = temporaryDirectory.resolve(revision + "-request.json"); + JSON.writeValue(request.toFile(), java.util.Map.of( + "schema", "pathlab.evidence-job/1", + "sourcePath", source.toString(), + "sourceSha256", sha256(source), + "slideRevision", revision, + "previewPath", preview.toString(), + "sourceWidth", 1600, + "sourceHeight", 1200, + "packManifest", pack.toString(), + "stain", "ihc_dab", + "marker", "generic")); + return request; + } + + private static String sha256(Path path) throws Exception { + return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256") + .digest(Files.readAllBytes(path))); + } +} diff --git a/src/test/java/org/pathlab/forge/evidence/EvidenceJobQueueTest.java b/src/test/java/org/pathlab/forge/evidence/EvidenceJobQueueTest.java new file mode 100644 index 0000000..ca87cd0 --- /dev/null +++ b/src/test/java/org/pathlab/forge/evidence/EvidenceJobQueueTest.java @@ -0,0 +1,55 @@ +package org.pathlab.forge.evidence; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.file.Files; +import java.time.Duration; +import java.time.Instant; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +final class EvidenceJobQueueTest { + @TempDir java.nio.file.Path temporaryDirectory; + + @Test + void resumesExpiredLeaseAcrossRestartAndCompletes() throws Exception { + var database = temporaryDirectory.resolve("jobs.sqlite"); + var request = temporaryDirectory.resolve("request.json"); + Files.writeString(request, "{}"); + var start = Instant.parse("2026-08-22T00:00:00Z"); + try (var queue = new EvidenceJobQueue(database)) { + queue.submit("job-1", request, start); + var claimed = queue.claimNext("worker-a", start, Duration.ofSeconds(30)).orElseThrow(); + assertEquals(EvidenceJobState.VALIDATING, claimed.state()); + queue.checkpoint("job-1", "worker-a", EvidenceJobState.RUNNING, "coarse", .2, + "Coarse evidence", start.plusSeconds(1), Duration.ofSeconds(30)); + } + try (var restarted = new EvidenceJobQueue(database)) { + assertTrue(restarted.claimNext("worker-b", start.plusSeconds(32), Duration.ofSeconds(30)).isPresent()); + restarted.checkpoint("job-1", "worker-b", EvidenceJobState.PACKAGING, "packaging", .9, + "Packaging", start.plusSeconds(33), Duration.ofSeconds(30)); + restarted.checkpoint("job-1", "worker-b", EvidenceJobState.COMPLETED, "completed", 1, + "Complete", start.plusSeconds(34), Duration.ofSeconds(30)); + assertEquals(EvidenceJobState.COMPLETED, restarted.find("job-1").orElseThrow().state()); + assertFalse(restarted.claimNext("worker-c", start.plusSeconds(60), Duration.ofSeconds(30)).isPresent()); + } + } + + @Test + void cancellationFlagSurvivesRestart() throws Exception { + var database = temporaryDirectory.resolve("jobs.sqlite"); + var request = temporaryDirectory.resolve("request.json"); + Files.writeString(request, "{}"); + var now = Instant.parse("2026-08-22T00:00:00Z"); + try (var queue = new EvidenceJobQueue(database)) { + queue.submit("job-2", request, now); + queue.requestCancel("job-2", now.plusSeconds(1)); + } + try (var queue = new EvidenceJobQueue(database)) { + assertTrue(queue.find("job-2").orElseThrow().cancelRequested()); + assertTrue(queue.claimNext("worker", now.plusSeconds(2), Duration.ofSeconds(30)).isEmpty()); + } + } +} diff --git a/src/test/java/org/pathlab/forge/evidence/EvidenceMentorRunnerTest.java b/src/test/java/org/pathlab/forge/evidence/EvidenceMentorRunnerTest.java new file mode 100644 index 0000000..501fc30 --- /dev/null +++ b/src/test/java/org/pathlab/forge/evidence/EvidenceMentorRunnerTest.java @@ -0,0 +1,52 @@ +package org.pathlab.forge.evidence; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.file.Files; +import java.nio.file.Path; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +final class EvidenceMentorRunnerTest { + @TempDir Path temporaryDirectory; + + @Test + void acceptsAuthenticatedLoopbackSubmissionIntoDurableQueue() throws Exception { + var requestFile = temporaryDirectory.resolve("request.json"); + Files.writeString(requestFile, "{}"); + try (var runner = EvidenceMentorRunner.start( + temporaryDirectory.resolve("state"), 0, "test-loopback-token-0123456789abcdef", false)) { + var client = HttpClient.newHttpClient(); + var body = "{\"id\":\"job-http-1\",\"requestPath\":" + + quote(requestFile.toString()) + "}"; + var unauthorized = client.send(HttpRequest.newBuilder(runner.uri("/v1/jobs")) + .POST(HttpRequest.BodyPublishers.ofString(body)).build(), + HttpResponse.BodyHandlers.ofString()); + assertEquals(401, unauthorized.statusCode()); + + var invalid = client.send(HttpRequest.newBuilder(runner.uri("/v1/jobs")) + .header("Authorization", "Bearer test-loopback-token-0123456789abcdef") + .POST(HttpRequest.BodyPublishers.ofString("{}")) .build(), + HttpResponse.BodyHandlers.ofString()); + assertEquals(422, invalid.statusCode()); + + var accepted = client.send(HttpRequest.newBuilder(runner.uri("/v1/jobs")) + .header("Authorization", "Bearer test-loopback-token-0123456789abcdef") + .POST(HttpRequest.BodyPublishers.ofString(body)).build(), + HttpResponse.BodyHandlers.ofString()); + assertEquals(202, accepted.statusCode()); + assertTrue(accepted.body().contains("job-http-1")); + assertTrue(runner.find("job-http-1").isPresent()); + assertEquals("127.0.0.1", runner.uri("/health").getHost()); + } + } + + private static String quote(String value) { + return "\"" + value.replace("\\", "\\\\").replace("\"", "\\\"") + "\""; + } +} diff --git a/src/test/java/org/pathlab/forge/evidence/EvidencePackManifestTest.java b/src/test/java/org/pathlab/forge/evidence/EvidencePackManifestTest.java new file mode 100644 index 0000000..1353f28 --- /dev/null +++ b/src/test/java/org/pathlab/forge/evidence/EvidencePackManifestTest.java @@ -0,0 +1,70 @@ +package org.pathlab.forge.evidence; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.file.Files; +import java.nio.file.Path; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +final class EvidencePackManifestTest { + @TempDir Path temporaryDirectory; + + @Test + void acceptsBoundedPrivateResearchPack() throws Exception { + var path = write("private-research", "experimental", "[\"generic\",\"er\"]", "ihc-descriptive", "[\"ihc_dab\"]"); + var pack = EvidencePackManifest.load(path); + assertEquals("ihc-descriptive-v1", pack.packId()); + assertTrue(pack.pilotEligible()); + assertTrue(pack.markers().contains("generic")); + assertEquals("od-v1", pack.preprocessingId()); + assertEquals(64, pack.sha256().length()); + } + + @Test + void blocksBenchmarkOnlyPackFromPilot() throws Exception { + var pack = EvidencePackManifest.load(write("benchmark-only", "experimental", "[]", "he-evidence", "[\"he\"]")); + assertFalse(pack.pilotEligible()); + assertThrows(IllegalArgumentException.class, pack::requirePilotEligible); + } + + @Test + void rejectsNetworkedOrOversizedPack() throws Exception { + var value = manifest("private-research", "qualified", "[]", "he-evidence", "[\"he\"]") + .replace("\"maxVramMiB\":4096", "\"maxVramMiB\":9000") + .replace("\"network\":false", "\"network\":true"); + var path = temporaryDirectory.resolve("invalid.json"); + Files.writeString(path, value); + assertThrows(IllegalArgumentException.class, () -> EvidencePackManifest.load(path)); + } + + @Test + void bundledPilotCatalogActivatesOnlyInstalledLawfulBaselines() throws Exception { + var root = Path.of("src/main/resources/evidence-packs"); + assertTrue(EvidencePackManifest.load(root.resolve("cell-od-watershed-v1.json")).pilotEligible()); + assertTrue(EvidencePackManifest.load(root.resolve("ihc-descriptive-v1.json")).pilotEligible()); + assertFalse(EvidencePackManifest.load(root.resolve("he-dinov2-small-v1.json")).pilotEligible()); + assertFalse(EvidencePackManifest.load(root.resolve("he-hibou-b-v1.json")).pilotEligible()); + assertFalse(EvidencePackManifest.load(root.resolve("he-gigapath-benchmark-v1.json")).pilotEligible()); + } + + private Path write(String use, String status, String markers, String capability, String stains) throws Exception { + var path = temporaryDirectory.resolve(java.util.UUID.randomUUID() + ".json"); + Files.writeString(path, manifest(use, status, markers, capability, stains)); + return path; + } + + private static String manifest(String use, String status, String markers, String capability, String stains) { + return "{\"schema\":\"pathlab.ai-pack/1\",\"packId\":\"ihc-descriptive-v1\",\"version\":\"1\"," + + "\"capability\":\"" + capability + "\",\"acceptedStains\":" + stains + "," + + "\"preprocessing\":{\"id\":\"od-v1\",\"tilePixels\":512},\"artifacts\":[]," + + "\"rights\":{\"license\":\"internal\",\"allowedUse\":\"" + use + "\"," + + "\"redistributable\":false,\"derivativesAllowed\":false,\"reviewedAt\":\"2026-08-22T00:00:00Z\"}," + + "\"resourceEnvelope\":{\"maxRamMiB\":4096,\"maxVramMiB\":4096,\"maxSeconds\":300,\"network\":false}," + + "\"validation\":{\"status\":\"" + status + "\",\"modelCard\":\"model.md\",\"heldOutEvaluation\":\"evaluation.json\"}," + + "\"outputSchema\":\"pathlab.ai-evidence/1\",\"markers\":" + markers + "}"; + } +} diff --git a/src/test/java/org/pathlab/forge/viewer/PrivateResultsBundleBuilderTest.java b/src/test/java/org/pathlab/forge/viewer/PrivateResultsBundleBuilderTest.java index 32a2452..e9de593 100644 --- a/src/test/java/org/pathlab/forge/viewer/PrivateResultsBundleBuilderTest.java +++ b/src/test/java/org/pathlab/forge/viewer/PrivateResultsBundleBuilderTest.java @@ -42,6 +42,20 @@ void createsCanonicalBoundedResultSidecarFromAvailableAnnotations() throws Excep assertFalse(entries.keySet().stream().anyMatch(name -> name.contains(":\\") || name.startsWith("/"))); } + @Test + void includesSignedEvidenceWithoutChangingDeliverySchema() throws Exception { + var evidence = temporaryDirectory.resolve("evidence.json"); + Files.writeString(evidence, "{\"schema\":\"pathlab.ai-evidence/1\"}"); + var output = temporaryDirectory.resolve("evidence-results.plresults"); + + new PrivateResultsBundleBuilder().build( + output, "revision-1", "a".repeat(64), List.of(), evidence); + + var entries = readTarGz(output); + assertEquals("pathlab.ai-evidence/1", text(entries, "evidence.json", "schema")); + assertEquals("pathlab-private-results/v1", text(entries, "manifest.json", "schema")); + } + private static String text(HashMap entries, String name, String key) throws Exception { return new com.fasterxml.jackson.databind.ObjectMapper() .readTree(entries.get(name)).get(key).asText(); From 0305eb824b0bba470c53dcad765ab5b77f61934e Mon Sep 17 00:00:00 2001 From: Enksodsoon <229880765+Enksodsoon@users.noreply.github.com> Date: Sat, 22 Aug 2026 07:36:54 +0700 Subject: [PATCH 02/50] feat: add governed model qualification packs --- .../pathlab-ai-evidence-v1.schema.json | 6 +- .../pathlab-ai-pack-v1.schema.json | 6 +- docs/evidence-mentor-runner.md | 20 ++- docs/model-cards/README.md | 18 +++ docs/model-cards/cell-hovernet-fast-v1.md | 7 + docs/model-cards/he-dinov2-small-v1.md | 5 + docs/model-cards/he-hibou-b-v1.md | 6 + frontend/src/App.tsx | 20 +++ frontend/src/api.ts | 17 +++ frontend/src/styles.css | 1 + .../forge/evidence/AtlasFeasibilityGate.java | 24 ++++ .../forge/evidence/BrightfieldStainQc.java | 47 +++++++ .../evidence/BrightfieldTileAnalyzer.java | 33 ++++- .../forge/evidence/EvidenceJobProcessor.java | 112 ++++++++++----- .../forge/evidence/EvidenceMentorRunner.java | 19 +++ .../forge/evidence/EvidencePackManifest.java | 47 ++++++- .../forge/evidence/EvidenceQuotaManager.java | 104 ++++++++++++++ .../evidence/EvidenceSampleManifest.java | 40 ++++++ .../forge/evidence/ExternalModelWorker.java | 130 ++++++++++++++++++ .../org/pathlab/forge/server/ForgeServer.java | 35 +++++ .../evidence-packs/cell-hovernet-fast-v1.json | 24 ++++ .../evidence-packs/he-dinov2-small-v1.json | 15 ++ .../evidence-packs/he-hibou-b-v1.json | 15 ++ .../evidence/AtlasFeasibilityGateTest.java | 15 ++ .../evidence/BrightfieldTileAnalyzerTest.java | 18 +++ .../evidence/EvidenceJobProcessorTest.java | 6 + .../evidence/EvidenceMentorRunnerTest.java | 7 + .../evidence/EvidencePackManifestTest.java | 4 + .../evidence/EvidenceQuotaManagerTest.java | 28 ++++ .../evidence/EvidenceSampleManifestTest.java | 20 +++ .../evidence/ExternalModelWorkerTest.java | 23 ++++ 31 files changed, 825 insertions(+), 47 deletions(-) create mode 100644 docs/model-cards/README.md create mode 100644 docs/model-cards/cell-hovernet-fast-v1.md create mode 100644 docs/model-cards/he-dinov2-small-v1.md create mode 100644 docs/model-cards/he-hibou-b-v1.md create mode 100644 src/main/java/org/pathlab/forge/evidence/AtlasFeasibilityGate.java create mode 100644 src/main/java/org/pathlab/forge/evidence/BrightfieldStainQc.java create mode 100644 src/main/java/org/pathlab/forge/evidence/EvidenceQuotaManager.java create mode 100644 src/main/java/org/pathlab/forge/evidence/EvidenceSampleManifest.java create mode 100644 src/main/java/org/pathlab/forge/evidence/ExternalModelWorker.java create mode 100644 src/main/resources/evidence-packs/cell-hovernet-fast-v1.json create mode 100644 src/test/java/org/pathlab/forge/evidence/AtlasFeasibilityGateTest.java create mode 100644 src/test/java/org/pathlab/forge/evidence/EvidenceQuotaManagerTest.java create mode 100644 src/test/java/org/pathlab/forge/evidence/EvidenceSampleManifestTest.java create mode 100644 src/test/java/org/pathlab/forge/evidence/ExternalModelWorkerTest.java diff --git a/contracts/evidence-mentor/pathlab-ai-evidence-v1.schema.json b/contracts/evidence-mentor/pathlab-ai-evidence-v1.schema.json index debc9c5..ec35fc1 100644 --- a/contracts/evidence-mentor/pathlab-ai-evidence-v1.schema.json +++ b/contracts/evidence-mentor/pathlab-ai-evidence-v1.schema.json @@ -16,10 +16,10 @@ "reviewRequired": {"const": true}, "coordinates": {"type": "object", "additionalProperties": false, "required": ["space", "originX", "originY", "scaleX", "scaleY"], "properties": {"space": {"const": "source-pixel"}, "originX": {"type": "number", "minimum": 0}, "originY": {"type": "number", "minimum": 0}, "scaleX": {"type": "number", "exclusiveMinimum": 0}, "scaleY": {"type": "number", "exclusiveMinimum": 0}}}, "evidence": {"type": "array", "maxItems": 1000, "items": {"$ref": "#/$defs/region"}}, - "cellAggregates": {"type": "array", "maxItems": 1000, "items": {"type": "object", "additionalProperties": false, "required": ["regionId", "algorithm", "count", "densityPerMm2", "meanNucleusAreaPx2"], "properties": {"regionId": {"type": "string", "maxLength": 120}, "algorithm": {"enum": ["hovernet-fast", "od-watershed"]}, "count": {"type": "integer", "minimum": 0}, "densityPerMm2": {"type": ["number", "null"], "minimum": 0}, "meanNucleusAreaPx2": {"type": ["number", "null"], "minimum": 0}}}}, - "ihcDescriptors": {"type": "array", "maxItems": 1000, "items": {"type": "object", "additionalProperties": false, "required": ["regionId", "marker", "compartment", "dabAreaFraction", "meanDabOd", "researchEstimate"], "properties": {"regionId": {"type": "string", "maxLength": 120}, "marker": {"enum": ["generic", "er", "pr", "ki-67", "her2", "pd-l1"]}, "compartment": {"enum": ["nuclear", "membrane", "tumor-immune-region", "generic-region"]}, "dabAreaFraction": {"type": "number", "minimum": 0, "maximum": 1}, "meanDabOd": {"type": "number", "minimum": 0}, "researchEstimate": {"const": true}}}}, + "cellAggregates": {"type": "array", "maxItems": 1000, "items": {"type": "object", "additionalProperties": false, "required": ["regionId", "algorithm", "count", "densityPerMm2", "meanNucleusAreaPx2"], "properties": {"regionId": {"type": "string", "maxLength": 120}, "algorithm": {"enum": ["hovernet-fast", "od-watershed"]}, "count": {"type": "integer", "minimum": 0}, "densityPerMm2": {"type": ["number", "null"], "minimum": 0}, "meanNucleusAreaPx2": {"type": ["number", "null"], "minimum": 0}, "meanNucleusPerimeterPx": {"type": ["number", "null"], "minimum": 0}, "meanNucleusEccentricity": {"type": ["number", "null"], "minimum": 0, "maximum": 1}, "meanNucleusSolidity": {"type": ["number", "null"], "minimum": 0, "maximum": 1}, "uncertainty": {"type": "number", "minimum": 0, "maximum": 1}}}}, + "ihcDescriptors": {"type": "array", "maxItems": 1000, "items": {"type": "object", "additionalProperties": false, "required": ["regionId", "marker", "compartment", "dabAreaFraction", "meanDabOd", "researchEstimate"], "properties": {"regionId": {"type": "string", "maxLength": 120}, "markerId": {"enum": ["generic", "er", "pr", "ki-67", "her2", "pd-l1"]}, "marker": {"enum": ["generic", "er", "pr", "ki-67", "her2", "pd-l1"]}, "analysisMode": {"enum": ["marker-aware", "generic-fallback"]}, "cellMaskSource": {"enum": ["hovernet-fast", "od-watershed"]}, "compartmentSource": {"enum": ["none", "faculty-authored", "faculty-approved", "model-suggested"]}, "calibrationStatus": {"enum": ["calibrated", "relative_only", "not_evaluable"]}, "compartment": {"enum": ["nuclear", "membrane", "tumor-immune-region", "generic-region"]}, "dabAreaFraction": {"type": "number", "minimum": 0, "maximum": 1}, "meanDabOd": {"type": "number", "minimum": 0}, "uncertainty": {"type": "number", "minimum": 0, "maximum": 1}, "abstentionReason": {"type": ["string", "null"], "maxLength": 500}, "researchEstimate": {"const": true}}}}, "citations": {"type": "array", "maxItems": 100, "items": {"type": "object", "additionalProperties": false, "required": ["claimId", "sourceUrl"], "properties": {"claimId": {"type": "string", "maxLength": 160}, "sourceUrl": {"type": "string", "format": "uri", "maxLength": 1000}}}}, - "qc": {"type": "object", "additionalProperties": false, "required": ["focus", "tissueFraction", "uncertainty", "abstentionReasons"], "properties": {"focus": {"type": "number", "minimum": 0, "maximum": 1}, "tissueFraction": {"type": "number", "minimum": 0, "maximum": 1}, "uncertainty": {"type": "number", "minimum": 0, "maximum": 1}, "abstentionReasons": {"type": "array", "maxItems": 20, "items": {"type": "string", "maxLength": 500}}}}, + "qc": {"type": "object", "additionalProperties": false, "required": ["focus", "tissueFraction", "uncertainty", "abstentionReasons"], "properties": {"focus": {"type": "number", "minimum": 0, "maximum": 1}, "tissueFraction": {"type": "number", "minimum": 0, "maximum": 1}, "uncertainty": {"type": "number", "minimum": 0, "maximum": 1}, "abstentionReasons": {"type": "array", "maxItems": 20, "items": {"type": "string", "maxLength": 500}}, "calibrationStatus": {"enum": ["calibrated", "relative_only", "not_evaluable"]}, "backgroundFraction": {"type": "number", "minimum": 0, "maximum": 1}, "saturationFraction": {"type": "number", "minimum": 0, "maximum": 1}, "stainSeparation": {"type": "number", "minimum": 0, "maximum": 1}, "warnings": {"type": "array", "maxItems": 20, "items": {"type": "string", "maxLength": 500}}}}, "provenance": {"type": "object", "additionalProperties": false, "required": ["createdAt", "codeRevision", "offlineAnalysis"], "properties": {"createdAt": {"type": "string", "format": "date-time"}, "codeRevision": {"type": "string", "maxLength": 160}, "offlineAnalysis": {"const": true}}}, "manifestSha256": {"type": "string", "pattern": "^[a-f0-9]{64}$"}, "signature": {"type": "object", "additionalProperties": false, "required": ["algorithm", "keyId", "publicKeyDer", "value"], "properties": {"algorithm": {"const": "Ed25519"}, "keyId": {"type": "string", "maxLength": 160}, "publicKeyDer": {"type": "string", "minLength": 1}, "value": {"type": "string", "minLength": 1}}} diff --git a/contracts/evidence-mentor/pathlab-ai-pack-v1.schema.json b/contracts/evidence-mentor/pathlab-ai-pack-v1.schema.json index 1179ae0..ddf56a2 100644 --- a/contracts/evidence-mentor/pathlab-ai-pack-v1.schema.json +++ b/contracts/evidence-mentor/pathlab-ai-pack-v1.schema.json @@ -12,10 +12,12 @@ "capability": {"enum": ["he-evidence", "cell-morphology", "ihc-descriptive"]}, "acceptedStains": {"type": "array", "minItems": 1, "maxItems": 16, "uniqueItems": true, "items": {"enum": ["he", "ihc_dab"]}}, "preprocessing": {"type": "object", "additionalProperties": false, "required": ["id", "tilePixels"], "properties": {"id": {"type": "string", "maxLength": 120}, "tilePixels": {"type": "integer", "minimum": 64, "maximum": 2048}}}, - "artifacts": {"type": "array", "maxItems": 16, "items": {"type": "object", "additionalProperties": false, "required": ["name", "sha256", "source"], "properties": {"name": {"type": "string", "maxLength": 160}, "sha256": {"type": "string", "pattern": "^[a-f0-9]{64}$"}, "source": {"type": "string", "format": "uri", "maxLength": 1000}}}, + "artifacts": {"type": "array", "maxItems": 16, "items": {"type": "object", "additionalProperties": false, "required": ["name", "sha256", "source"], "properties": {"name": {"type": "string", "maxLength": 160}, "sha256": {"type": "string", "pattern": "^[a-f0-9]{64}$"}, "source": {"type": "string", "format": "uri", "maxLength": 1000}}}}, "rights": {"type": "object", "additionalProperties": false, "required": ["license", "allowedUse", "redistributable", "derivativesAllowed", "reviewedAt"], "properties": {"license": {"type": "string", "maxLength": 200}, "allowedUse": {"enum": ["private-research", "benchmark-only"]}, "redistributable": {"type": "boolean"}, "derivativesAllowed": {"type": "boolean"}, "reviewedAt": {"type": "string", "format": "date-time"}}}, "resourceEnvelope": {"type": "object", "additionalProperties": false, "required": ["maxRamMiB", "maxVramMiB", "maxSeconds", "network"], "properties": {"maxRamMiB": {"type": "integer", "minimum": 64, "maximum": 16384}, "maxVramMiB": {"type": "integer", "minimum": 0, "maximum": 4608}, "maxSeconds": {"type": "integer", "minimum": 1, "maximum": 1200}, "network": {"const": false}}}, - "validation": {"type": "object", "additionalProperties": false, "required": ["status", "modelCard", "heldOutEvaluation"], "properties": {"status": {"enum": ["experimental", "qualified", "not-evaluable", "blocked"]}, "modelCard": {"type": "string", "maxLength": 1000}, "heldOutEvaluation": {"type": "string", "maxLength": 1000}}}, + "runtimeCompatibility": {"type": "object", "additionalProperties": false, "required": ["workerProtocol", "executionProvider", "cuda", "gpuArchitecture", "requiresExternalWorker"], "properties": {"workerProtocol": {"type": "string", "maxLength": 80}, "executionProvider": {"enum": ["cpu", "cuda"]}, "cuda": {"type": "string", "maxLength": 40}, "gpuArchitecture": {"type": "string", "maxLength": 40}, "requiresExternalWorker": {"type": "boolean"}}}, + "licenseLedger": {"type": "array", "maxItems": 32, "items": {"type": "object", "additionalProperties": false, "required": ["component", "license", "revision", "permittedUse", "redistributable", "derivativesAllowed"], "properties": {"component": {"type": "string", "maxLength": 160}, "license": {"type": "string", "maxLength": 200}, "revision": {"type": "string", "maxLength": 160}, "permittedUse": {"enum": ["private-research", "benchmark-only"]}, "redistributable": {"type": "boolean"}, "derivativesAllowed": {"type": "boolean"}}}}, + "validation": {"type": "object", "additionalProperties": false, "required": ["status", "modelCard", "heldOutEvaluation"], "properties": {"status": {"enum": ["experimental", "qualified", "not-evaluable", "unsupported", "blocked"]}, "modelCard": {"type": "string", "maxLength": 1000}, "heldOutEvaluation": {"type": "string", "maxLength": 1000}}}, "outputSchema": {"const": "pathlab.ai-evidence/1"}, "markers": {"type": "array", "uniqueItems": true, "items": {"enum": ["generic", "er", "pr", "ki-67", "her2", "pd-l1"]}} } diff --git a/docs/evidence-mentor-runner.md b/docs/evidence-mentor-runner.md index 6afe188..7d4393f 100644 --- a/docs/evidence-mentor-runner.md +++ b/docs/evidence-mentor-runner.md @@ -12,9 +12,23 @@ Install from a versioned Forge distribution: Default state is `D:\PathLabData\EvidenceMentor\state` when the data drive is present, otherwise `%LOCALAPPDATA%\PathLab\EvidenceMentor\state`. Analysis is -offline. Acquisition is outside this process. Current runnable pack is the -deterministic cell/IHC baseline; H&E encoder requests fail closed until an -approved, checksum-matched encoder artifact is installed. +offline. Acquisition is a separate allowlisted operation and must reserve quota before download or +extraction. `GET /v1/status` returns authenticated, non-outcome runtime and quota telemetry. A +dashboard failure does not stop processing. + +The deterministic cell/IHC baseline is runnable. DINOv2-small, Hibou-B and HoVer-Net remain +`not-evaluable` until an exact, rights-approved worker and model artifact are installed beneath +`state/models//`. External workers use `pathlab.model-worker-result/1`, inherit the +16 GB RAM and 4.5 GB VRAM envelope, receive offline environment controls, and are rejected on the +P2000 unless they declare CUDA 12.6 and `sm_61`. Forge signs only bounded regions and descriptors; +embeddings, raw pixels and clinical outputs are rejected. + +New material uses fixed hard buckets: 45 GB source, 25 GB derived, 10 GB models, 10 GB evidence/test, +and a 10 GB untouchable reserve. Existing BRACS remains grandfathered read-only and is not copied. + +Atlas-H&E remains a Training Lab research track. Its four-hour probe may continue locally only when +resource limits hold, no OOM occurs, validation improves, and projected completion is within seven +days. This runner performs no cloud provisioning or spending. Rollback removes the scheduled task but preserves queue, evidence, and keys: diff --git a/docs/model-cards/README.md b/docs/model-cards/README.md new file mode 100644 index 0000000..f163282 --- /dev/null +++ b/docs/model-cards/README.md @@ -0,0 +1,18 @@ +# Evidence Mentor model qualification + +All model packs are private research/education features and default off. A manifest is not proof +that its artifact is installed, licensed, qualified, or publishable. Exact artifact SHA-256 values, +source revisions, rights review, held-out evaluation, deterministic self-test, and P2000 acceptance +must be recorded before a status can move from `not-evaluable`. + +| Track | Candidate | Current status | Activation rule | +|---|---|---|---| +| H&E | DINOv2-small | `not-evaluable` | Experimental only after exact artifact and held-out retrieval run | +| H&E | Hibou-B | `not-evaluable` | User accepts gated terms; then rights, reproducibility, quality and latency gates | +| Cells | HoVer-Net fast | `not-evaluable` | Exact private weights plus PQ/count/morphometry/resource gates | +| Cells | OD watershed | `experimental` | Deterministic fallback; no clinical cell identity | +| IHC | H/DAB descriptors | `experimental` | Within-slide research descriptors; reviewed compartments for PD-L1 | +| Tutor | Qwen3-0.6B INT4 | `not-evaluable` | Complete browser asset at most 500 MiB and claim-ID-only tests | +| Distillation | Atlas-H&E ~72M | `not-evaluable` | Baselines frozen, derivative rights pass, bounded feasibility run succeeds | + +GigaPath remains benchmark-only and cannot contribute evidence to the OCI pilot. diff --git a/docs/model-cards/cell-hovernet-fast-v1.md b/docs/model-cards/cell-hovernet-fast-v1.md new file mode 100644 index 0000000..69f0567 --- /dev/null +++ b/docs/model-cards/cell-hovernet-fast-v1.md @@ -0,0 +1,7 @@ +# HoVer-Net fast cell pack + +Status: `not-evaluable`. No weights are bundled or installed by Forge. The code and weights have +separate license entries; PanNuke-derived weights are private, non-redistributed research inputs. +Qualification requires patient/source-held-out instance PQ, count error, morphometry bias, +cross-tissue failure rate, deterministic repeats, and the declared RAM/VRAM envelope. Predicted +types remain research categories and are not clinical cell identities. diff --git a/docs/model-cards/he-dinov2-small-v1.md b/docs/model-cards/he-dinov2-small-v1.md new file mode 100644 index 0000000..7c75443 --- /dev/null +++ b/docs/model-cards/he-dinov2-small-v1.md @@ -0,0 +1,5 @@ +# DINOv2-small H&E baseline + +Status: `not-evaluable`. No artifact is installed. This generic Apache-2.0 feature extractor is the +experimental executable baseline after its exact revision and hashes are pinned. Embeddings remain +local and only bounded evidence regions may leave Forge. diff --git a/docs/model-cards/he-hibou-b-v1.md b/docs/model-cards/he-hibou-b-v1.md new file mode 100644 index 0000000..bd3792e --- /dev/null +++ b/docs/model-cards/he-hibou-b-v1.md @@ -0,0 +1,6 @@ +# Hibou-B H&E candidate + +Status: `not-evaluable`. The user must personally accept upstream gated contact-sharing terms. +No credential or mutable remote code may enter analysis. Activation additionally requires an exact +revision, audited vendored model code, artifact hashes, rights review, held-out improvement and P2000 +latency qualification. diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 7f3cbb9..e82a407 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -53,6 +53,7 @@ const DEFAULT_LOCAL_FOLDER = 'Unfiled' export function App() { const [datasets, setDatasets] = useState([]) const [capabilities, setCapabilities] = useState>>() + const [evidenceStatus, setEvidenceStatus] = useState() const [selectedId, setSelectedId] = useState('') const [seriesByDataset, setSeriesByDataset] = useState>({}) const [artifactByDataset, setArtifactByDataset] = useState>({}) @@ -189,6 +190,17 @@ export function App() { .catch((nextError) => setError(message(nextError))) }, []) + useEffect(() => { + const update = () => void api.evidenceRunnerStatus() + .then(setEvidenceStatus) + .catch(() => setEvidenceStatus({ + schema: 'pathlab.evidence-runner-status/1', status: 'unavailable', detail: 'Runner is not reachable', + })) + update() + const timer = window.setInterval(update, 5_000) + return () => window.clearInterval(timer) + }, []) + useEffect(() => { if (!datasets.some((item) => ACTIVE_STATUSES.has(item.status))) return const timer = window.setInterval(() => void refresh(), 1500) @@ -798,6 +810,14 @@ export function App() { )} /> +
+ Evidence Mentor + {evidenceStatus?.status === 'ready' ? 'Autonomous runner ready' : 'Runner unavailable'} + {evidenceStatus?.status === 'ready' ? + Analysis network off · RAM {formatBytes(evidenceStatus.processMemoryUsedBytes || 0)} / {formatBytes(evidenceStatus.processMemoryLimitBytes || 0)} · VRAM cap {evidenceStatus.vramLimitMiB} MiB + : {evidenceStatus?.detail || 'Checking local runner…'}} +
{connection?.connected && viewerUpload?.viewerSlideId && ['IMAGE_READY', 'SYNCING_RESULTS', 'COMPLETE'].includes(viewerUpload.state) ? ( ('/api/capabilities') } +export type EvidenceRunnerStatus = { + schema: 'pathlab.evidence-runner-status/1' + status: 'ready' | 'unavailable' + detail?: string + networkDisabledForAnalysis?: boolean + processMemoryUsedBytes?: number + processMemoryLimitBytes?: number + vramLimitMiB?: number + diskUsableBytes?: number + updatedAt?: string + quota?: Record +} + +export async function evidenceRunnerStatus() { + return request('/api/evidence/status') +} + export async function importProjectFolder(path?: string) { const query = path?.trim() ? `?path=${encodeURIComponent(path.trim())}` : '' return request<{ datasets: Dataset[]; project?: { root: string; imported: number; failed: string } }>( diff --git a/frontend/src/styles.css b/frontend/src/styles.css index e1014ba..760ec99 100644 --- a/frontend/src/styles.css +++ b/frontend/src/styles.css @@ -6,6 +6,7 @@ button,input,select{font:inherit} button,a{-webkit-tap-highlight-color:transparent} button{cursor:pointer} .visually-hidden{position:absolute;width:1px;height:1px;overflow:hidden;clip:rect(0 0 0 0);white-space:nowrap} +.evidence-runner-dashboard{position:fixed;right:18px;bottom:70px;z-index:40;display:grid;gap:2px;max-width:420px;padding:10px 14px;border:1px solid var(--border);border-radius:10px;background:var(--surface-elevated);box-shadow:0 8px 30px rgba(35,29,24,.14);color:var(--body);font-size:12px}.evidence-runner-dashboard span{font-weight:650}.evidence-runner-dashboard small{color:var(--muted)}.evidence-runner-dashboard.ready{border-color:var(--success)} .forge-canvas-host,.forge-canvas-host>.pathlab-canvas-shell{width:100%;height:100%;min-height:0}.forge-canvas-host>.pathlab-canvas-shell{height:100dvh;grid-template:"rail navigator stage inspector" minmax(0,1fr) "rail queue queue queue" 58px/var(--library-rail-width) minmax(220px,270px) minmax(0,1fr) minmax(280px,380px)}.forge-canvas-host>.pathlab-canvas-shell.rail-expanded{--library-rail-width:176px}.forge-canvas-host.inspector-collapsed>.pathlab-canvas-shell{grid-template:"rail navigator stage" minmax(0,1fr) "rail queue queue" 58px/var(--library-rail-width) minmax(220px,270px) minmax(0,1fr)}.forge-canvas-host.navigator-collapsed>.pathlab-canvas-shell{grid-template:"rail stage inspector" minmax(0,1fr) "rail queue queue" 58px/var(--library-rail-width) minmax(0,1fr) minmax(280px,380px)}.forge-canvas-host.navigator-collapsed.inspector-collapsed>.pathlab-canvas-shell{grid-template:"rail stage" minmax(0,1fr) "rail queue" 58px/var(--library-rail-width) minmax(0,1fr)}.pathlab-viewer-inspector{min-height:0;overflow-x:hidden;overflow-y:auto;overscroll-behavior:contain;scrollbar-gutter:stable} .library-app-rail{width:var(--library-rail-width);height:100dvh;position:sticky;top:0;z-index:50;display:flex;flex-direction:column;align-items:center;gap:4px;overflow:hidden auto;padding:6px 4px;border-right:1px solid var(--border);background:var(--surface)} .library-rail-brand{width:38px;height:38px;display:grid;flex:0 0 auto;place-items:center} diff --git a/src/main/java/org/pathlab/forge/evidence/AtlasFeasibilityGate.java b/src/main/java/org/pathlab/forge/evidence/AtlasFeasibilityGate.java new file mode 100644 index 0000000..018b10b --- /dev/null +++ b/src/main/java/org/pathlab/forge/evidence/AtlasFeasibilityGate.java @@ -0,0 +1,24 @@ +package org.pathlab.forge.evidence; + +/** Deterministic decision gate for the bounded local Atlas-H&E feasibility probe. */ +public final class AtlasFeasibilityGate { + private static final double MAX_LOCAL_HOURS = 7 * 24; + private AtlasFeasibilityGate() {} + + public static Decision decide(Probe probe) { + if (probe.elapsedHours < 4 || probe.completedSteps <= probe.warmupSteps + || probe.totalSteps <= probe.completedSteps) return new Decision("PROBE_INCOMPLETE", null); + if (probe.peakVramMiB > 4608 || probe.peakRamMiB > 16_384 || probe.oomCount > 0 + || !probe.validationImproved) return new Decision("LOCAL_INSUFFICIENT", null); + var effectiveSteps = probe.completedSteps - probe.warmupSteps; + var effectiveHours = probe.elapsedHours * effectiveSteps / probe.completedSteps; + var stepsPerHour = effectiveSteps / effectiveHours; + var projected = (probe.totalSteps - probe.completedSteps) / stepsPerHour; + return new Decision(projected <= MAX_LOCAL_HOURS ? "CONTINUE_LOCAL" : "LOCAL_INSUFFICIENT", + projected); + } + + public record Probe(double elapsedHours, long completedSteps, long warmupSteps, long totalSteps, + int peakVramMiB, int peakRamMiB, int oomCount, boolean validationImproved) {} + public record Decision(String status, Double projectedRemainingHours) {} +} diff --git a/src/main/java/org/pathlab/forge/evidence/BrightfieldStainQc.java b/src/main/java/org/pathlab/forge/evidence/BrightfieldStainQc.java new file mode 100644 index 0000000..4099a00 --- /dev/null +++ b/src/main/java/org/pathlab/forge/evidence/BrightfieldStainQc.java @@ -0,0 +1,47 @@ +package org.pathlab.forge.evidence; + +import java.awt.image.BufferedImage; + +/** Deterministic preflight for bounded brightfield H/DAB measurements. */ +public final class BrightfieldStainQc { + private BrightfieldStainQc() {} + + public static Result inspect(BufferedImage image, boolean controlsValidated) { + if (image == null || image.getWidth() < 1 || image.getHeight() < 1) { + throw new IllegalArgumentException("Brightfield image geometry is invalid"); + } + long pixels = (long) image.getWidth() * image.getHeight(); + var nearWhite = 0L; + var saturated = 0L; + var blueLike = 0L; + var brownLike = 0L; + double channelSpread = 0; + for (var y = 0; y < image.getHeight(); y++) for (var x = 0; x < image.getWidth(); x++) { + var rgb = image.getRGB(x, y); + var red = rgb >>> 16 & 0xff; + var green = rgb >>> 8 & 0xff; + var blue = rgb & 0xff; + if (red > 245 && green > 245 && blue > 245) nearWhite++; + // White background is expected in brightfield and must not be treated as clipped tissue. + if (red < 3 || green < 3 || blue < 3) saturated++; + if (blue - red >= 20 && blue - green >= 20) blueLike++; + if (red - green >= 12 && green - blue >= 10) brownLike++; + channelSpread += Math.max(red, Math.max(green, blue)) - Math.min(red, Math.min(green, blue)); + } + var background = (double) nearWhite / pixels; + var saturation = (double) saturated / pixels; + var hematoxylin = (double) blueLike / pixels; + var dab = (double) brownLike / pixels; + var separation = Math.min(1, channelSpread / pixels / 64.0); + var reasons = new java.util.ArrayList(); + if (background > 0.97) reasons.add("insufficient_tissue"); + if (saturation > 0.35) reasons.add("excessive_saturation"); + if (separation < 0.03) reasons.add("weak_stain_separation"); + var status = reasons.isEmpty() ? (controlsValidated ? "calibrated" : "relative_only") : "not_evaluable"; + return new Result(status, background, saturation, hematoxylin, dab, separation, java.util.List.copyOf(reasons)); + } + + public record Result(String calibrationStatus, double backgroundFraction, double saturationFraction, + double hematoxylinLikeFraction, double dabLikeFraction, double separationScore, + java.util.List reasons) {} +} diff --git a/src/main/java/org/pathlab/forge/evidence/BrightfieldTileAnalyzer.java b/src/main/java/org/pathlab/forge/evidence/BrightfieldTileAnalyzer.java index 05a7128..eac76f6 100644 --- a/src/main/java/org/pathlab/forge/evidence/BrightfieldTileAnalyzer.java +++ b/src/main/java/org/pathlab/forge/evidence/BrightfieldTileAnalyzer.java @@ -43,6 +43,9 @@ public static Result analyze(BufferedImage image, String requestedMarker) { compartment(marker), components.count, components.count == 0 ? null : (double) components.pixels / components.count, + components.count == 0 ? null : components.perimeter / components.count, + components.count == 0 ? null : components.eccentricity / components.count, + components.count == 0 ? null : components.solidity / components.count, (double) dabPixels / (width * height), dabPixels == 0 ? 0 : dabOd / dabPixels, true); @@ -53,16 +56,32 @@ private static Components components(boolean[] mask, int width, int height) { var queue = new ArrayDeque(); var count = 0; var pixels = 0; + double perimeter = 0; + double eccentricity = 0; + double solidity = 0; for (var start = 0; start < mask.length; start++) { if (!mask[start] || seen[start]) continue; seen[start] = true; queue.add(start); var size = 0; + var minX = width; + var minY = height; + var maxX = -1; + var maxY = -1; + var componentPerimeter = 0; while (!queue.isEmpty()) { var current = queue.removeFirst(); size++; var x = current % width; var y = current / width; + minX = Math.min(minX, x); + minY = Math.min(minY, y); + maxX = Math.max(maxX, x); + maxY = Math.max(maxY, y); + if (x == 0 || !mask[current - 1]) componentPerimeter++; + if (x == width - 1 || !mask[current + 1]) componentPerimeter++; + if (y == 0 || !mask[current - width]) componentPerimeter++; + if (y == height - 1 || !mask[current + width]) componentPerimeter++; for (var ny = Math.max(0, y - 1); ny <= Math.min(height - 1, y + 1); ny++) { for (var nx = Math.max(0, x - 1); nx <= Math.min(width - 1, x + 1); nx++) { var next = ny * width + nx; @@ -76,9 +95,16 @@ private static Components components(boolean[] mask, int width, int height) { if (size >= 4) { count++; pixels += size; + perimeter += componentPerimeter; + var boxWidth = maxX - minX + 1.0; + var boxHeight = maxY - minY + 1.0; + var major = Math.max(boxWidth, boxHeight); + var minor = Math.min(boxWidth, boxHeight); + eccentricity += major == 0 ? 0 : Math.sqrt(Math.max(0, 1 - (minor * minor) / (major * major))); + solidity += size / (boxWidth * boxHeight); } } - return new Components(count, pixels); + return new Components(count, pixels, perimeter, eccentricity, solidity); } private static String compartment(String marker) { @@ -90,13 +116,16 @@ private static String compartment(String marker) { }; } - private record Components(int count, int pixels) {} + private record Components(int count, int pixels, double perimeter, double eccentricity, double solidity) {} public record Result( String marker, String compartment, int cellCount, Double meanNucleusAreaPx2, + Double meanNucleusPerimeterPx, + Double meanNucleusEccentricity, + Double meanNucleusSolidity, double dabAreaFraction, double meanDabOd, boolean researchEstimate) {} diff --git a/src/main/java/org/pathlab/forge/evidence/EvidenceJobProcessor.java b/src/main/java/org/pathlab/forge/evidence/EvidenceJobProcessor.java index b757c60..0cc612d 100644 --- a/src/main/java/org/pathlab/forge/evidence/EvidenceJobProcessor.java +++ b/src/main/java/org/pathlab/forge/evidence/EvidenceJobProcessor.java @@ -20,10 +20,11 @@ public final class EvidenceJobProcessor { private static final ObjectMapper JSON = new ObjectMapper(); private static final Duration LEASE = Duration.ofMinutes(2); - private static final long ARTIFACT_QUOTA = 10L * 1024 * 1024 * 1024; - private static final Set REQUEST_FIELDS = Set.of( + private static final Set REQUIRED_REQUEST_FIELDS = Set.of( "schema", "sourcePath", "sourceSha256", "slideRevision", "previewPath", "sourceWidth", "sourceHeight", "packManifest", "stain", "marker"); + private static final Set OPTIONAL_REQUEST_FIELDS = Set.of( + "compartmentSource", "controlsValidated"); private final EvidenceJobQueue queue; private final Path stateRoot; @@ -40,7 +41,10 @@ public EvidenceJob process(EvidenceJob job, String workerId, Instant now) throws } catch (JsonProcessingException invalidJson) { throw new IllegalArgumentException("Evidence job request JSON is invalid", invalidJson); } - require(request.isObject() && fieldNames(request).equals(REQUEST_FIELDS), + var requestFields = fieldNames(request); + require(request.isObject() && requestFields.containsAll(REQUIRED_REQUEST_FIELDS) + && java.util.stream.Stream.concat(REQUIRED_REQUEST_FIELDS.stream(), OPTIONAL_REQUEST_FIELDS.stream()) + .collect(java.util.stream.Collectors.toSet()).containsAll(requestFields), "Evidence job request is invalid"); require("pathlab.evidence-job/1".equals(text(request, "schema")), "Evidence job schema is unsupported"); @@ -53,8 +57,6 @@ public EvidenceJob process(EvidenceJob job, String workerId, Instant now) throws pack.requirePilotEligible(); var stain = text(request, "stain"); pack.requireStain(stain); - require(pack.capability() != EvidencePackManifest.Capability.HE_EVIDENCE, - "H&E encoder artifact is not installed"); var marker = text(request, "marker").toLowerCase(); if (!pack.markers().contains(marker)) marker = "generic"; require(pack.capability() != EvidencePackManifest.Capability.IHC_DESCRIPTIVE @@ -74,11 +76,29 @@ public EvidenceJob process(EvidenceJob job, String workerId, Instant now) throws } require(image != null, "Evidence preview could not be decoded"); var analysis = BrightfieldTileAnalyzer.analyze(image, marker); + JsonNode modelResult = null; + if (pack.capability() == EvidencePackManifest.Capability.HE_EVIDENCE) { + modelResult = new ExternalModelWorker(stateRoot).execute(pack, job.requestPath(), job.id()); + require("completed".equals(modelResult.path("status").asText()), + "H&E model worker returned unsupported or not_evaluable"); + } + var stainQc = BrightfieldStainQc.inspect(image, + request.path("controlsValidated").isBoolean() + && request.path("controlsValidated").booleanValue()); var tissue = tissueFraction(image); var focus = focus(image); var abstentionReasons = new java.util.ArrayList(); if (tissue < 0.05) abstentionReasons.add("insufficient_tissue"); if (focus < 0.05) abstentionReasons.add("out_of_focus_or_uninformative_preview"); + abstentionReasons.addAll(stainQc.reasons().stream() + .filter(reason -> !abstentionReasons.contains(reason)).toList()); + var compartmentSource = request.path("compartmentSource").isTextual() + ? request.path("compartmentSource").textValue() : "none"; + require(Set.of("none", "faculty-authored", "faculty-approved", "model-suggested") + .contains(compartmentSource), "Evidence compartment source is invalid"); + var reviewedCompartment = Set.of("faculty-authored", "faculty-approved").contains(compartmentSource); + var compartmentWarning = "pd-l1".equals(marker) && !reviewedCompartment + ? "COMPARTMENT_REVIEW_REQUIRED" : null; var abstained = !abstentionReasons.isEmpty(); var refiningAt = now.plusSeconds(1); checkCancellation(job.id(), workerId, refiningAt); @@ -90,11 +110,11 @@ public EvidenceJob process(EvidenceJob job, String workerId, Instant now) throws var artifactRoot = stateRoot.resolve("artifacts").resolve(job.id()).normalize(); require(artifactRoot.startsWith(stateRoot.resolve("artifacts")), "Evidence artifact path escaped state root"); - require(directoryBytes(stateRoot.resolve("artifacts")) < ARTIFACT_QUOTA, - "Evidence artifact quota is exhausted"); + new EvidenceQuotaManager(stateRoot).requireReservation(EvidenceQuotaManager.Bucket.EVIDENCE_TEST, + Math.max(1, Files.size(preview))); Files.createDirectories(artifactRoot); - var unsigned = evidence(request, pack, image, analysis, focus, tissue, - abstentionReasons, now); + var unsigned = evidence(request, pack, image, analysis, stainQc, modelResult, focus, tissue, + abstentionReasons, compartmentSource, compartmentWarning, now); new EvidenceBundleWriter(stateRoot.resolve("signing")) .write(artifactRoot.resolve("evidence.json"), unsigned); var terminal = abstained ? EvidenceJobState.ABSTAINED : EvidenceJobState.COMPLETED; @@ -111,9 +131,13 @@ private static ObjectNode evidence( EvidencePackManifest pack, BufferedImage image, BrightfieldTileAnalyzer.Result analysis, + BrightfieldStainQc.Result stainQc, + JsonNode modelResult, double focus, double tissue, java.util.List abstentionReasons, + String compartmentSource, + String compartmentWarning, Instant now) { var root = JSON.createObjectNode(); root.put("schema", EvidenceBundleWriter.SCHEMA); @@ -139,31 +163,56 @@ private static ObjectNode evidence( "space", "source-pixel", "originX", 0, "originY", 0, "scaleX", (double) sourceWidth / image.getWidth(), "scaleY", (double) sourceHeight / image.getHeight()))); - root.set("evidence", JSON.valueToTree(java.util.List.of(java.util.Map.of( - "id", "region-1", "stage", "refined", "kind", "support", - "x", 0, "y", 0, "width", sourceWidth, "height", sourceHeight, - "score", Math.max(0, Math.min(1, 1 - analysis.dabAreaFraction())))))); + root.set("evidence", modelResult == null + ? JSON.valueToTree(java.util.List.of(java.util.Map.of( + "id", "region-1", "stage", "refined", "kind", "support", + "x", 0, "y", 0, "width", sourceWidth, "height", sourceHeight, + "score", Math.max(0, Math.min(1, 1 - analysis.dabAreaFraction()))))) + : modelResult.path("regions").deepCopy()); var aggregate = new java.util.LinkedHashMap(); aggregate.put("regionId", "region-1"); aggregate.put("algorithm", "od-watershed"); aggregate.put("count", analysis.cellCount()); aggregate.put("densityPerMm2", null); aggregate.put("meanNucleusAreaPx2", analysis.meanNucleusAreaPx2()); - root.set("cellAggregates", JSON.valueToTree(java.util.List.of(aggregate))); - root.set("ihcDescriptors", JSON.valueToTree( - pack.capability() == EvidencePackManifest.Capability.IHC_DESCRIPTIVE - ? java.util.List.of(java.util.Map.of( - "regionId", "region-1", "marker", analysis.marker(), - "compartment", analysis.compartment(), - "dabAreaFraction", analysis.dabAreaFraction(), - "meanDabOd", analysis.meanDabOd(), - "researchEstimate", true)) - : java.util.List.of())); + aggregate.put("meanNucleusPerimeterPx", analysis.meanNucleusPerimeterPx()); + aggregate.put("meanNucleusEccentricity", analysis.meanNucleusEccentricity()); + aggregate.put("meanNucleusSolidity", analysis.meanNucleusSolidity()); + aggregate.put("uncertainty", 1 - Math.min(focus, tissue)); + root.set("cellAggregates", JSON.valueToTree( + pack.capability() == EvidencePackManifest.Capability.HE_EVIDENCE + ? java.util.List.of() : java.util.List.of(aggregate))); + var ihc = new java.util.ArrayList>(); + if (pack.capability() == EvidencePackManifest.Capability.IHC_DESCRIPTIVE) { + var descriptor = new java.util.LinkedHashMap(); + descriptor.put("regionId", "region-1"); + descriptor.put("markerId", analysis.marker()); + descriptor.put("marker", analysis.marker()); + descriptor.put("analysisMode", compartmentWarning == null ? "marker-aware" : "generic-fallback"); + descriptor.put("cellMaskSource", "od-watershed"); + descriptor.put("compartmentSource", compartmentSource); + descriptor.put("calibrationStatus", stainQc.calibrationStatus()); + descriptor.put("compartment", compartmentWarning == null ? analysis.compartment() : "generic-region"); + descriptor.put("dabAreaFraction", analysis.dabAreaFraction()); + descriptor.put("meanDabOd", analysis.meanDabOd()); + descriptor.put("uncertainty", 1 - Math.min(focus, tissue)); + descriptor.put("abstentionReason", compartmentWarning); + descriptor.put("researchEstimate", true); + ihc.add(descriptor); + } + root.set("ihcDescriptors", JSON.valueToTree(ihc)); root.set("citations", JSON.createArrayNode()); - root.set("qc", JSON.valueToTree(java.util.Map.of( - "focus", focus, "tissueFraction", tissue, - "uncertainty", 1 - Math.min(focus, tissue), - "abstentionReasons", abstentionReasons))); + var qc = new java.util.LinkedHashMap(); + qc.put("focus", focus); + qc.put("tissueFraction", tissue); + qc.put("uncertainty", 1 - Math.min(focus, tissue)); + qc.put("abstentionReasons", abstentionReasons); + qc.put("calibrationStatus", stainQc.calibrationStatus()); + qc.put("backgroundFraction", stainQc.backgroundFraction()); + qc.put("saturationFraction", stainQc.saturationFraction()); + qc.put("stainSeparation", stainQc.separationScore()); + qc.put("warnings", compartmentWarning == null ? java.util.List.of() : java.util.List.of(compartmentWarning)); + root.set("qc", JSON.valueToTree(qc)); root.set("provenance", JSON.valueToTree(java.util.Map.of( "createdAt", now.toString(), "codeRevision", "pathlab-forge-evidence-mentor-v1", "offlineAnalysis", true))); @@ -237,15 +286,6 @@ private static String sha256(Path path) throws IOException { } } - private static long directoryBytes(Path root) throws IOException { - if (!Files.exists(root)) return 0; - try (var paths = Files.walk(root)) { - return paths.filter(Files::isRegularFile).mapToLong(path -> { - try { return Files.size(path); } catch (IOException ignored) { return Long.MAX_VALUE; } - }).sum(); - } - } - private static void require(boolean condition, String message) { if (!condition) throw new IllegalArgumentException(message); } diff --git a/src/main/java/org/pathlab/forge/evidence/EvidenceMentorRunner.java b/src/main/java/org/pathlab/forge/evidence/EvidenceMentorRunner.java index c922184..aaaf55f 100644 --- a/src/main/java/org/pathlab/forge/evidence/EvidenceMentorRunner.java +++ b/src/main/java/org/pathlab/forge/evidence/EvidenceMentorRunner.java @@ -43,6 +43,7 @@ private EvidenceMentorRunner(Path stateRoot, int port, String token, boolean wor server = HttpServer.create( new InetSocketAddress(InetAddress.getByName("127.0.0.1"), port), 16); server.createContext("/health", this::handle); + server.createContext("/v1/status", this::handle); server.createContext("/v1/jobs", this::handle); http = Executors.newFixedThreadPool(2, runnable -> { var thread = new Thread(runnable, "pathlab-evidence-ipc"); @@ -89,6 +90,24 @@ private void handle(HttpExchange exchange) throws IOException { "schema", "pathlab.evidence-runner-status/1", "status", "ready")); return; } + if ("/v1/status".equals(path) && "GET".equals(exchange.getRequestMethod())) { + var runtime = Runtime.getRuntime(); + var fileStore = Files.getFileStore(stateRoot); + var status = new java.util.LinkedHashMap(); + status.put("schema", "pathlab.evidence-runner-status/1"); + status.put("status", "ready"); + status.put("networkDisabledForAnalysis", true); + status.put("gpuConcurrency", 1); + status.put("cpuIoConcurrency", 1); + status.put("processMemoryUsedBytes", runtime.totalMemory() - runtime.freeMemory()); + status.put("processMemoryLimitBytes", 16L * 1024 * 1024 * 1024); + status.put("vramLimitMiB", 4608); + status.put("diskUsableBytes", fileStore.getUsableSpace()); + status.put("quota", new EvidenceQuotaManager(stateRoot).snapshot().buckets()); + status.put("updatedAt", Instant.now().toString()); + respond(exchange, 200, status); + return; + } if ("/v1/jobs".equals(path) && "POST".equals(exchange.getRequestMethod())) { submit(exchange); return; diff --git a/src/main/java/org/pathlab/forge/evidence/EvidencePackManifest.java b/src/main/java/org/pathlab/forge/evidence/EvidencePackManifest.java index 27c11d2..abcb520 100644 --- a/src/main/java/org/pathlab/forge/evidence/EvidencePackManifest.java +++ b/src/main/java/org/pathlab/forge/evidence/EvidencePackManifest.java @@ -22,6 +22,8 @@ public record EvidencePackManifest( String allowedUse, boolean redistributable, boolean derivativesAllowed, + RuntimeCompatibility runtimeCompatibility, + List licenseLedger, int maxRamMiB, int maxVramMiB, int maxSeconds, @@ -73,16 +75,23 @@ public static EvidencePackManifest load(Path manifest) throws IOException { } else { require(markers.isEmpty(), "Only IHC packs may declare markers"); } + var runtime = root.has("runtimeCompatibility") + ? parseRuntime(root.path("runtimeCompatibility")) + : new RuntimeCompatibility("java17", "cpu", "none", "none", false); + var licenseLedger = root.has("licenseLedger") + ? parseLicenseLedger(root.path("licenseLedger")) : List.of(); return new EvidencePackManifest( normalized, packId, version, capability, acceptedStains, preprocessingId, allowedUse, booleanValue(rights, "redistributable"), booleanValue(rights, "derivativesAllowed"), + runtime, licenseLedger, maxRam, maxVram, maxSeconds, status, sha256(bytes), artifacts, markers); } public boolean pilotEligible() { return "private-research".equals(allowedUse) && validationStatus != ValidationStatus.BLOCKED + && validationStatus != ValidationStatus.UNSUPPORTED && validationStatus != ValidationStatus.NOT_EVALUABLE; } @@ -108,6 +117,37 @@ private static List parseArtifacts(JsonNode node) { return List.copyOf(result); } + private static RuntimeCompatibility parseRuntime(JsonNode node) { + require(node.isObject(), "AI pack runtime compatibility is invalid"); + var protocol = text(node, "workerProtocol"); + var provider = text(node, "executionProvider"); + var cuda = text(node, "cuda"); + var architecture = text(node, "gpuArchitecture"); + require(protocol.length() <= 80 && Set.of("cpu", "cuda").contains(provider) + && cuda.length() <= 40 && architecture.length() <= 40, + "AI pack runtime compatibility is invalid"); + return new RuntimeCompatibility(protocol, provider, cuda, architecture, + node.path("requiresExternalWorker").asBoolean(false)); + } + + private static List parseLicenseLedger(JsonNode node) { + require(node.isArray() && node.size() <= 32, "AI pack license ledger is invalid"); + var result = new java.util.ArrayList(); + for (var item : node) { + var component = text(item, "component"); + var license = text(item, "license"); + var revision = text(item, "revision"); + var permittedUse = text(item, "permittedUse"); + require(component.length() <= 160 && license.length() <= 200 && revision.length() <= 160 + && Set.of("private-research", "benchmark-only").contains(permittedUse), + "AI pack license ledger is invalid"); + result.add(new LicenseEntry(component, license, revision, permittedUse, + item.path("redistributable").asBoolean(false), + item.path("derivativesAllowed").asBoolean(false))); + } + return List.copyOf(result); + } + private static Set stringSet(JsonNode node, Set allowed, String label) { require(node.isArray() && node.size() <= 16, "AI pack " + label + " list is invalid"); var result = new java.util.LinkedHashSet(); @@ -158,6 +198,10 @@ private static void require(boolean condition, String message) { } public record Artifact(String name, String sha256, String source) {} + public record RuntimeCompatibility(String workerProtocol, String executionProvider, String cuda, + String gpuArchitecture, boolean requiresExternalWorker) {} + public record LicenseEntry(String component, String license, String revision, String permittedUse, + boolean redistributable, boolean derivativesAllowed) {} public enum Capability { HE_EVIDENCE("he-evidence"), @@ -174,7 +218,8 @@ static Capability fromWire(String value) { } public enum ValidationStatus { - EXPERIMENTAL("experimental"), QUALIFIED("qualified"), NOT_EVALUABLE("not-evaluable"), BLOCKED("blocked"); + EXPERIMENTAL("experimental"), QUALIFIED("qualified"), NOT_EVALUABLE("not-evaluable"), + UNSUPPORTED("unsupported"), BLOCKED("blocked"); private final String wire; ValidationStatus(String wire) { this.wire = wire; } public String wire() { return wire; } diff --git a/src/main/java/org/pathlab/forge/evidence/EvidenceQuotaManager.java b/src/main/java/org/pathlab/forge/evidence/EvidenceQuotaManager.java new file mode 100644 index 0000000..75ffb8e --- /dev/null +++ b/src/main/java/org/pathlab/forge/evidence/EvidenceQuotaManager.java @@ -0,0 +1,104 @@ +package org.pathlab.forge.evidence; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +/** Hard, fail-closed quota accounting for newly acquired Evidence Mentor material. */ +public final class EvidenceQuotaManager { + private static final long GIB = 1024L * 1024 * 1024; + private final Path root; + + public EvidenceQuotaManager(Path stateRoot) throws IOException { + root = stateRoot.toAbsolutePath().normalize(); + Files.createDirectories(root.resolve("quota")); + } + + public synchronized void requireReservation(Bucket bucket, long additionalBytes) throws IOException { + if (additionalBytes < 0) throw new IllegalArgumentException("Quota reservation is invalid"); + var used = directoryBytes(bucket.path(root)); + var reserved = reservedBytes(bucket); + if (used > bucket.limitBytes || reserved > bucket.limitBytes - used + || additionalBytes > bucket.limitBytes - used - reserved) { + throw new IllegalArgumentException("Evidence " + bucket.wire + " quota is exhausted"); + } + } + + public synchronized Reservation reserve(Bucket bucket, String id, long bytes) throws IOException { + if (bucket == Bucket.RESERVE) throw new IllegalArgumentException("Evidence reserve is untouchable"); + if (id == null || !id.matches("[A-Za-z0-9._-]{1,120}")) { + throw new IllegalArgumentException("Quota reservation id is invalid"); + } + requireReservation(bucket, bytes); + var directory = root.resolve("quota/reservations").resolve(bucket.wire); + Files.createDirectories(directory); + var target = directory.resolve(id + ".reservation"); + if (Files.exists(target)) throw new IllegalArgumentException("Quota reservation already exists"); + var partial = target.resolveSibling(target.getFileName() + ".partial"); + Files.writeString(partial, Long.toString(bytes), java.nio.file.StandardOpenOption.CREATE_NEW); + try { + Files.move(partial, target, java.nio.file.StandardCopyOption.ATOMIC_MOVE); + } catch (java.nio.file.AtomicMoveNotSupportedException ignored) { + Files.move(partial, target); + } + return new Reservation(target, bytes); + } + + public Snapshot snapshot() throws IOException { + var values = new java.util.LinkedHashMap(); + for (var bucket : Bucket.values()) { + values.put(bucket.wire, new Usage(directoryBytes(bucket.path(root)), reservedBytes(bucket), + bucket.limitBytes, bucket == Bucket.RESERVE)); + } + return new Snapshot(java.util.Map.copyOf(values)); + } + + private static long directoryBytes(Path path) throws IOException { + if (!Files.exists(path)) return 0; + try (var paths = Files.walk(path)) { + return paths.filter(Files::isRegularFile).mapToLong(item -> { + try { return Files.size(item); } catch (IOException error) { return Long.MAX_VALUE; } + }).sum(); + } + } + + private long reservedBytes(Bucket bucket) throws IOException { + var directory = root.resolve("quota/reservations").resolve(bucket.wire); + if (!Files.isDirectory(directory)) return 0; + try (var paths = Files.list(directory)) { + return paths.filter(path -> path.getFileName().toString().endsWith(".reservation")) + .mapToLong(path -> { + try { return Long.parseLong(Files.readString(path).trim()); } + catch (Exception error) { return Long.MAX_VALUE; } + }).sum(); + } + } + + public enum Bucket { + SOURCE("source", 45 * GIB, "sources"), + DERIVED("derived", 25 * GIB, "derived"), + MODELS("models", 10 * GIB, "models"), + EVIDENCE_TEST("evidence-test", 10 * GIB, "artifacts"), + RESERVE("reserve", 10 * GIB, "reserve"); + + private final String wire; + private final long limitBytes; + private final String directory; + Bucket(String wire, long limitBytes, String directory) { + this.wire = wire; this.limitBytes = limitBytes; this.directory = directory; + } + Path path(Path root) { return root.resolve(directory); } + } + + public record Usage(long usedBytes, long reservedBytes, long limitBytes, boolean untouchable) {} + public record Snapshot(java.util.Map buckets) {} + + public static final class Reservation implements AutoCloseable { + private final Path path; + private final long bytes; + private Reservation(Path path, long bytes) { this.path = path; this.bytes = bytes; } + public long bytes() { return bytes; } + public void commit() throws IOException { Files.deleteIfExists(path); } + @Override public void close() throws IOException { Files.deleteIfExists(path); } + } +} diff --git a/src/main/java/org/pathlab/forge/evidence/EvidenceSampleManifest.java b/src/main/java/org/pathlab/forge/evidence/EvidenceSampleManifest.java new file mode 100644 index 0000000..bfc5706 --- /dev/null +++ b/src/main/java/org/pathlab/forge/evidence/EvidenceSampleManifest.java @@ -0,0 +1,40 @@ +package org.pathlab.forge.evidence; + +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.nio.file.Path; +import java.util.Set; + +/** Provenance and permitted-use gate for one source slide or derived sample. */ +public record EvidenceSampleManifest(String source, String patientGroup, String slideId, + String sha256, String license, String taskLabel, String permittedUse, + long bytes, boolean grandfatheredReadOnly) { + public static final String SCHEMA = "pathlab.evidence-sample/1"; + private static final ObjectMapper JSON = new ObjectMapper(); + + public static EvidenceSampleManifest load(Path path) throws IOException { + var value = JSON.readTree(path.toFile()); + var fields = new java.util.HashSet(); value.fieldNames().forEachRemaining(fields::add); + require(fields.equals(Set.of("schema", "source", "patientGroup", "slideId", "sha256", + "license", "taskLabel", "permittedUse", "bytes", "grandfatheredReadOnly")), + "Evidence sample fields are invalid"); + require(SCHEMA.equals(value.path("schema").asText()), "Evidence sample schema is invalid"); + var result = new EvidenceSampleManifest(text(value, "source", 1000), text(value, "patientGroup", 160), + text(value, "slideId", 160), text(value, "sha256", 64), text(value, "license", 240), + text(value, "taskLabel", 160), text(value, "permittedUse", 80), + value.path("bytes").longValue(), value.path("grandfatheredReadOnly").asBoolean(false)); + require(result.sha256.matches("[a-f0-9]{64}") && result.bytes >= 0 + && Set.of("private-research", "benchmark-only").contains(result.permittedUse), + "Evidence sample provenance is invalid"); + return result; + } + + private static String text(com.fasterxml.jackson.databind.JsonNode value, String field, int max) { + var result = value.path(field).asText(); + require(!result.isBlank() && result.length() <= max, "Evidence sample provenance is invalid"); + return result; + } + private static void require(boolean condition, String message) { + if (!condition) throw new IllegalArgumentException(message); + } +} diff --git a/src/main/java/org/pathlab/forge/evidence/ExternalModelWorker.java b/src/main/java/org/pathlab/forge/evidence/ExternalModelWorker.java new file mode 100644 index 0000000..deaa5c1 --- /dev/null +++ b/src/main/java/org/pathlab/forge/evidence/ExternalModelWorker.java @@ -0,0 +1,130 @@ +package org.pathlab.forge.evidence; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.util.HexFormat; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import org.pathlab.forge.runtime.ChildProcessContainment; + +/** Executes an installed, checksum-pinned model worker through a bounded file protocol. */ +public final class ExternalModelWorker { + public static final String RESULT_SCHEMA = "pathlab.model-worker-result/1"; + private static final ObjectMapper JSON = new ObjectMapper(); + private static final Set STATUSES = Set.of("completed", "unsupported", "not_evaluable"); + private final Path stateRoot; + + public ExternalModelWorker(Path stateRoot) { + this.stateRoot = stateRoot.toAbsolutePath().normalize(); + } + + public JsonNode execute(EvidencePackManifest pack, Path request, String jobId) throws IOException { + var runtime = pack.runtimeCompatibility(); + if (!runtime.requiresExternalWorker()) { + throw new IllegalArgumentException("AI pack does not declare an external model worker"); + } + if ("cuda".equals(runtime.executionProvider()) + && (!"12.6".equals(runtime.cuda()) || !"sm_61".equals(runtime.gpuArchitecture()))) { + throw new IllegalArgumentException("Model worker is incompatible with the Pascal qualification host"); + } + var install = stateRoot.resolve("models").resolve(pack.packId()).resolve(pack.version()).normalize(); + require(install.startsWith(stateRoot.resolve("models")), "Model worker path escaped state root"); + var executable = install.resolve(System.getProperty("os.name", "").startsWith("Windows") + ? "worker.exe" : "worker"); + require(Files.isRegularFile(executable), "Model worker artifact is not installed"); + var expected = pack.artifacts().stream().filter(item -> "worker".equals(item.name())) + .map(EvidencePackManifest.Artifact::sha256).findFirst() + .orElseThrow(() -> new IllegalArgumentException("Model worker checksum is not declared")); + require(expected.equals(sha256(executable)), "Model worker checksum does not match"); + + var outputRoot = stateRoot.resolve("worker-output").resolve(jobId).normalize(); + require(outputRoot.startsWith(stateRoot.resolve("worker-output")), "Model output path escaped state root"); + Files.createDirectories(outputRoot); + var output = outputRoot.resolve("result.json"); + var partial = outputRoot.resolve("result.json.partial"); + Files.deleteIfExists(partial); + var builder = new ProcessBuilder(executable.toString(), "--request", request.toString(), + "--output", partial.toString(), "--offline").directory(install.toFile()); + var environment = builder.environment(); + environment.remove("HTTP_PROXY"); environment.remove("HTTPS_PROXY"); + environment.remove("ALL_PROXY"); environment.remove("HF_TOKEN"); + environment.put("HF_HUB_OFFLINE", "1"); + environment.put("TRANSFORMERS_OFFLINE", "1"); + environment.put("PATHLAB_ANALYSIS_NETWORK", "disabled"); + environment.put("PATHLAB_MAX_VRAM_MIB", Integer.toString(pack.maxVramMiB())); + environment.put("PATHLAB_MAX_RAM_MIB", Integer.toString(pack.maxRamMiB())); + var process = ChildProcessContainment.global().register(builder.redirectErrorStream(true) + .redirectOutput(ProcessBuilder.Redirect.DISCARD).start()); + try { + if (!process.waitFor(pack.maxSeconds(), TimeUnit.SECONDS)) { + process.destroyForcibly(); + throw new IOException("Model worker exceeded its bounded runtime"); + } + } catch (InterruptedException error) { + Thread.currentThread().interrupt(); process.destroyForcibly(); + throw new IOException("Model worker was interrupted", error); + } + require(process.exitValue() == 0 && Files.isRegularFile(partial), "Model worker failed closed"); + var result = JSON.readTree(partial.toFile()); + validateResult(result, pack); + try { + Files.move(partial, output, java.nio.file.StandardCopyOption.REPLACE_EXISTING, + java.nio.file.StandardCopyOption.ATOMIC_MOVE); + } catch (java.nio.file.AtomicMoveNotSupportedException ignored) { + Files.move(partial, output, java.nio.file.StandardCopyOption.REPLACE_EXISTING); + } + return result; + } + + static void validateResult(JsonNode result, EvidencePackManifest pack) { + require(result.isObject() && RESULT_SCHEMA.equals(result.path("schema").asText()), + "Model worker result schema is invalid"); + require(pack.sha256().equals(result.path("packManifestSha256").asText()), + "Model worker result pack identity does not match"); + require(STATUSES.contains(result.path("status").asText()), "Model worker result status is invalid"); + var regions = result.path("regions"); + require(regions.isArray() && regions.size() <= 1_000, "Model worker regions are invalid"); + for (var region : regions) { + require(region.isObject() && region.path("id").asText().matches("[A-Za-z0-9._-]{1,120}") + && Set.of("coarse", "refined").contains(region.path("stage").asText()) + && Set.of("support", "similar", "contrast").contains(region.path("kind").asText()) + && finiteNonnegative(region, "x") && finiteNonnegative(region, "y") + && finitePositive(region, "width") && finitePositive(region, "height") + && finiteUnit(region, "score"), + "Model worker region is invalid"); + } + var serialized = result.toString(); + require(!serialized.matches("(?is).*\"(embedding|embeddings|rawPixels|diagnosis|clinicalScore|tps|cps|treatment)\".*"), + "Model worker emitted prohibited reusable or clinical output"); + } + + private static String sha256(Path path) throws IOException { + try (var input = Files.newInputStream(path)) { + var digest = MessageDigest.getInstance("SHA-256"); + var buffer = new byte[1024 * 1024]; + int read; while ((read = input.read(buffer)) >= 0) digest.update(buffer, 0, read); + return HexFormat.of().formatHex(digest.digest()); + } catch (java.security.GeneralSecurityException error) { + throw new IOException("SHA-256 is unavailable", error); + } + } + + private static void require(boolean condition, String message) { + if (!condition) throw new IllegalArgumentException(message); + } + + private static boolean finiteNonnegative(JsonNode node, String field) { + return node.path(field).isNumber() && Double.isFinite(node.path(field).doubleValue()) + && node.path(field).doubleValue() >= 0; + } + private static boolean finitePositive(JsonNode node, String field) { + return finiteNonnegative(node, field) && node.path(field).doubleValue() > 0; + } + private static boolean finiteUnit(JsonNode node, String field) { + return finiteNonnegative(node, field) && node.path(field).doubleValue() <= 1; + } +} diff --git a/src/main/java/org/pathlab/forge/server/ForgeServer.java b/src/main/java/org/pathlab/forge/server/ForgeServer.java index cbc7a95..234db25 100644 --- a/src/main/java/org/pathlab/forge/server/ForgeServer.java +++ b/src/main/java/org/pathlab/forge/server/ForgeServer.java @@ -28,6 +28,7 @@ import org.pathlab.forge.conversion.SeriesInfo; import org.pathlab.forge.derivative.DerivativeEngine; import org.pathlab.forge.derivative.VipsRuntime; +import org.pathlab.forge.evidence.EvidenceMentorRunner; import org.pathlab.forge.feature.CapabilityRegistry; import org.pathlab.forge.feature.FeaturePackDescriptor; import org.pathlab.forge.feature.FeaturePackManager; @@ -302,6 +303,8 @@ private void handle(HttpExchange exchange) throws IOException { session(exchange); } else if ("/api/capabilities".equals(path) && "GET".equals(exchange.getRequestMethod())) { capabilities(exchange); + } else if ("/api/evidence/status".equals(path) && "GET".equals(exchange.getRequestMethod())) { + evidenceRunnerStatus(exchange); } else if ("/api/features".equals(path) && "GET".equals(exchange.getRequestMethod())) { features(exchange); } else if (path.matches("/api/features/[a-z0-9][a-z0-9-]{1,63}/install") @@ -757,6 +760,38 @@ private void revokeViewerConnection(HttpExchange exchange) throws IOException { } } + private void evidenceRunnerStatus(HttpExchange exchange) throws IOException { + if (!requireAuthenticated(exchange)) return; + var tokenPath = EvidenceMentorRunner.defaultStateRoot().resolve("ipc-token"); + if (!Files.isRegularFile(tokenPath)) { + respond(exchange, 503, "application/json", + "{\"schema\":\"pathlab.evidence-runner-status/1\",\"status\":\"unavailable\"," + + "\"detail\":\"Evidence Mentor runner is not installed\"}"); + return; + } + try { + var port = Integer.getInteger("pathlab.evidence.port", 8765); + var request = java.net.http.HttpRequest.newBuilder( + URI.create("http://127.0.0.1:" + port + "/v1/status")) + .timeout(java.time.Duration.ofSeconds(2)) + .header("Authorization", "Bearer " + Files.readString(tokenPath).trim()) + .GET().build(); + var response = java.net.http.HttpClient.newHttpClient().send( + request, java.net.http.HttpResponse.BodyHandlers.ofString()); + if (response.statusCode() != 200 || response.body().length() > 65_536) { + throw new IOException("Evidence Mentor status is unavailable"); + } + respond(exchange, 200, "application/json", response.body()); + } catch (InterruptedException error) { + Thread.currentThread().interrupt(); + respond(exchange, 503, "application/json", "{\"schema\":\"pathlab.evidence-runner-status/1\"," + + "\"status\":\"unavailable\",\"detail\":\"Status request was interrupted\"}"); + } catch (IOException | IllegalArgumentException error) { + respond(exchange, 503, "application/json", "{\"schema\":\"pathlab.evidence-runner-status/1\"," + + "\"status\":\"unavailable\",\"detail\":\"Runner is not reachable\"}"); + } + } + private void viewerUploadStatus(HttpExchange exchange) throws IOException { if (!requireAuthenticated(exchange)) { return; diff --git a/src/main/resources/evidence-packs/cell-hovernet-fast-v1.json b/src/main/resources/evidence-packs/cell-hovernet-fast-v1.json new file mode 100644 index 0000000..5bc4eb3 --- /dev/null +++ b/src/main/resources/evidence-packs/cell-hovernet-fast-v1.json @@ -0,0 +1,24 @@ +{ + "schema": "pathlab.ai-pack/1", + "packId": "cell-hovernet-fast-v1", + "version": "1", + "capability": "cell-morphology", + "acceptedStains": ["he", "ihc_dab"], + "preprocessing": {"id": "hovernet-fast-pannuke-v1", "tilePixels": 512}, + "artifacts": [], + "runtimeCompatibility": { + "workerProtocol": "pathlab.model-worker-result/1", + "executionProvider": "cuda", + "cuda": "12.6", + "gpuArchitecture": "sm_61", + "requiresExternalWorker": true + }, + "licenseLedger": [ + {"component": "HoVer-Net code", "license": "MIT", "revision": "PIN_REQUIRED", "permittedUse": "private-research", "redistributable": false, "derivativesAllowed": true}, + {"component": "PanNuke-derived weights", "license": "CC BY-NC-SA 4.0", "revision": "PIN_REQUIRED", "permittedUse": "private-research", "redistributable": false, "derivativesAllowed": false} + ], + "rights": {"license": "Mixed; see licenseLedger", "allowedUse": "private-research", "redistributable": false, "derivativesAllowed": false, "reviewedAt": "2026-08-22T00:00:00Z"}, + "resourceEnvelope": {"maxRamMiB": 16384, "maxVramMiB": 4608, "maxSeconds": 300, "network": false}, + "validation": {"status": "not-evaluable", "modelCard": "docs/model-cards/cell-hovernet-fast-v1.md", "heldOutEvaluation": "not-evaluable-no-installed-artifact"}, + "outputSchema": "pathlab.ai-evidence/1" +} diff --git a/src/main/resources/evidence-packs/he-dinov2-small-v1.json b/src/main/resources/evidence-packs/he-dinov2-small-v1.json index f416536..f705b6f 100644 --- a/src/main/resources/evidence-packs/he-dinov2-small-v1.json +++ b/src/main/resources/evidence-packs/he-dinov2-small-v1.json @@ -11,6 +11,21 @@ "tilePixels": 512 }, "artifacts": [], + "runtimeCompatibility": { + "workerProtocol": "pathlab.model-worker-result/1", + "executionProvider": "cuda", + "cuda": "12.6", + "gpuArchitecture": "sm_61", + "requiresExternalWorker": true + }, + "licenseLedger": [{ + "component": "facebook/dinov2-small", + "license": "Apache-2.0", + "revision": "PIN_REQUIRED_BEFORE_ACQUISITION", + "permittedUse": "private-research", + "redistributable": false, + "derivativesAllowed": false + }], "rights": { "license": "Apache-2.0 upstream; exact artifact ledger pending", "allowedUse": "private-research", diff --git a/src/main/resources/evidence-packs/he-hibou-b-v1.json b/src/main/resources/evidence-packs/he-hibou-b-v1.json index 2d1da91..29fa8a7 100644 --- a/src/main/resources/evidence-packs/he-hibou-b-v1.json +++ b/src/main/resources/evidence-packs/he-hibou-b-v1.json @@ -11,6 +11,21 @@ "tilePixels": 512 }, "artifacts": [], + "runtimeCompatibility": { + "workerProtocol": "pathlab.model-worker-result/1", + "executionProvider": "cuda", + "cuda": "12.6", + "gpuArchitecture": "sm_61", + "requiresExternalWorker": true + }, + "licenseLedger": [{ + "component": "histai/hibou-b", + "license": "Apache-2.0 with gated contact-sharing access", + "revision": "USER_ACCEPTANCE_AND_PIN_REQUIRED", + "permittedUse": "private-research", + "redistributable": false, + "derivativesAllowed": false + }], "rights": { "license": "Upstream terms require exact artifact review", "allowedUse": "private-research", diff --git a/src/test/java/org/pathlab/forge/evidence/AtlasFeasibilityGateTest.java b/src/test/java/org/pathlab/forge/evidence/AtlasFeasibilityGateTest.java new file mode 100644 index 0000000..f0e2f70 --- /dev/null +++ b/src/test/java/org/pathlab/forge/evidence/AtlasFeasibilityGateTest.java @@ -0,0 +1,15 @@ +package org.pathlab.forge.evidence; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import org.junit.jupiter.api.Test; + +final class AtlasFeasibilityGateTest { + @Test void continuesOnlyWhenProjectedInsideSevenDaysAndResourcesHold() { + assertEquals("CONTINUE_LOCAL", AtlasFeasibilityGate.decide( + new AtlasFeasibilityGate.Probe(4, 10_000, 1_000, 100_000, 4500, 12000, 0, true)).status()); + assertEquals("LOCAL_INSUFFICIENT", AtlasFeasibilityGate.decide( + new AtlasFeasibilityGate.Probe(4, 10_000, 1_000, 1_000_000, 4500, 12000, 0, true)).status()); + assertEquals("LOCAL_INSUFFICIENT", AtlasFeasibilityGate.decide( + new AtlasFeasibilityGate.Probe(4, 10_000, 1_000, 100_000, 4700, 12000, 0, true)).status()); + } +} diff --git a/src/test/java/org/pathlab/forge/evidence/BrightfieldTileAnalyzerTest.java b/src/test/java/org/pathlab/forge/evidence/BrightfieldTileAnalyzerTest.java index f6fb850..e1fb132 100644 --- a/src/test/java/org/pathlab/forge/evidence/BrightfieldTileAnalyzerTest.java +++ b/src/test/java/org/pathlab/forge/evidence/BrightfieldTileAnalyzerTest.java @@ -25,6 +25,9 @@ void countsBoundedNuclearComponentsAndDescribesDabWithoutClinicalCalls() { assertEquals(2, result.cellCount()); assertTrue(result.meanNucleusAreaPx2() >= 25); + assertTrue(result.meanNucleusPerimeterPx() > 0); + assertTrue(result.meanNucleusEccentricity() >= 0); + assertTrue(result.meanNucleusSolidity() > 0); assertTrue(result.dabAreaFraction() > 0); assertTrue(result.meanDabOd() > 0); assertEquals("nuclear", result.compartment()); @@ -38,4 +41,19 @@ void unknownMarkerUsesGenericDescriptiveCompartment() { assertEquals("generic", result.marker()); assertEquals("generic-region", result.compartment()); } + + @Test + void stainQcDefaultsToRelativeOnlyWithoutValidatedControls() { + var image = new BufferedImage(16, 16, BufferedImage.TYPE_INT_RGB); + var graphics = image.createGraphics(); + graphics.setColor(Color.WHITE); graphics.fillRect(0, 0, 16, 16); + graphics.setColor(new Color(80, 50, 125)); graphics.fillRect(1, 1, 7, 7); + graphics.setColor(new Color(140, 90, 40)); graphics.fillRect(8, 8, 7, 7); + graphics.dispose(); + + var qc = BrightfieldStainQc.inspect(image, false); + + assertEquals("relative_only", qc.calibrationStatus()); + assertTrue(qc.separationScore() > 0); + } } diff --git a/src/test/java/org/pathlab/forge/evidence/EvidenceJobProcessorTest.java b/src/test/java/org/pathlab/forge/evidence/EvidenceJobProcessorTest.java index 7bafac3..222498c 100644 --- a/src/test/java/org/pathlab/forge/evidence/EvidenceJobProcessorTest.java +++ b/src/test/java/org/pathlab/forge/evidence/EvidenceJobProcessorTest.java @@ -130,6 +130,12 @@ void completesOfflineDescriptiveIhcJobIntoSignedAtomicEvidence() throws Exceptio temporaryDirectory.resolve("state/artifacts/job-1/evidence.json").toFile()); assertEquals("pathlab.ai-evidence/1", evidence.path("schema").asText()); assertEquals("ki-67", evidence.path("ihcDescriptors").get(0).path("marker").asText()); + assertEquals("relative_only", evidence.path("ihcDescriptors").get(0) + .path("calibrationStatus").asText()); + assertEquals("od-watershed", evidence.path("ihcDescriptors").get(0) + .path("cellMaskSource").asText()); + assertTrue(evidence.path("cellAggregates").get(0) + .path("meanNucleusPerimeterPx").asDouble() > 0); assertTrue(evidence.path("researchOnly").asBoolean()); assertTrue(evidence.path("provenance").path("offlineAnalysis").asBoolean()); assertEquals(64, evidence.path("manifestSha256").asText().length()); diff --git a/src/test/java/org/pathlab/forge/evidence/EvidenceMentorRunnerTest.java b/src/test/java/org/pathlab/forge/evidence/EvidenceMentorRunnerTest.java index 501fc30..ee6eec3 100644 --- a/src/test/java/org/pathlab/forge/evidence/EvidenceMentorRunnerTest.java +++ b/src/test/java/org/pathlab/forge/evidence/EvidenceMentorRunnerTest.java @@ -43,6 +43,13 @@ void acceptsAuthenticatedLoopbackSubmissionIntoDurableQueue() throws Exception { assertTrue(accepted.body().contains("job-http-1")); assertTrue(runner.find("job-http-1").isPresent()); assertEquals("127.0.0.1", runner.uri("/health").getHost()); + + var status = client.send(HttpRequest.newBuilder(runner.uri("/v1/status")) + .header("Authorization", "Bearer test-loopback-token-0123456789abcdef") + .GET().build(), HttpResponse.BodyHandlers.ofString()); + assertEquals(200, status.statusCode()); + assertTrue(status.body().contains("networkDisabledForAnalysis")); + assertTrue(status.body().contains("evidence-test")); } } diff --git a/src/test/java/org/pathlab/forge/evidence/EvidencePackManifestTest.java b/src/test/java/org/pathlab/forge/evidence/EvidencePackManifestTest.java index 1353f28..56f7f61 100644 --- a/src/test/java/org/pathlab/forge/evidence/EvidencePackManifestTest.java +++ b/src/test/java/org/pathlab/forge/evidence/EvidencePackManifestTest.java @@ -48,6 +48,10 @@ void bundledPilotCatalogActivatesOnlyInstalledLawfulBaselines() throws Exception assertTrue(EvidencePackManifest.load(root.resolve("ihc-descriptive-v1.json")).pilotEligible()); assertFalse(EvidencePackManifest.load(root.resolve("he-dinov2-small-v1.json")).pilotEligible()); assertFalse(EvidencePackManifest.load(root.resolve("he-hibou-b-v1.json")).pilotEligible()); + var hoverNet = EvidencePackManifest.load(root.resolve("cell-hovernet-fast-v1.json")); + assertFalse(hoverNet.pilotEligible()); + assertEquals("cuda", hoverNet.runtimeCompatibility().executionProvider()); + assertEquals(2, hoverNet.licenseLedger().size()); assertFalse(EvidencePackManifest.load(root.resolve("he-gigapath-benchmark-v1.json")).pilotEligible()); } diff --git a/src/test/java/org/pathlab/forge/evidence/EvidenceQuotaManagerTest.java b/src/test/java/org/pathlab/forge/evidence/EvidenceQuotaManagerTest.java new file mode 100644 index 0000000..f355b7c --- /dev/null +++ b/src/test/java/org/pathlab/forge/evidence/EvidenceQuotaManagerTest.java @@ -0,0 +1,28 @@ +package org.pathlab.forge.evidence; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.file.Path; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +final class EvidenceQuotaManagerTest { + @TempDir Path temporaryDirectory; + + @Test + void exposesFixedBucketsAndUntouchableReserve() throws Exception { + var snapshot = new EvidenceQuotaManager(temporaryDirectory).snapshot(); + assertEquals(5, snapshot.buckets().size()); + assertEquals(45L * 1024 * 1024 * 1024, snapshot.buckets().get("source").limitBytes()); + assertTrue(snapshot.buckets().get("reserve").untouchable()); + try (var reservation = new EvidenceQuotaManager(temporaryDirectory) + .reserve(EvidenceQuotaManager.Bucket.MODELS, "dinov2-download", 1024)) { + assertEquals(1024, reservation.bytes()); + assertEquals(1024, new EvidenceQuotaManager(temporaryDirectory).snapshot() + .buckets().get("models").reservedBytes()); + } + assertEquals(0, new EvidenceQuotaManager(temporaryDirectory).snapshot() + .buckets().get("models").reservedBytes()); + } +} diff --git a/src/test/java/org/pathlab/forge/evidence/EvidenceSampleManifestTest.java b/src/test/java/org/pathlab/forge/evidence/EvidenceSampleManifestTest.java new file mode 100644 index 0000000..057c9f7 --- /dev/null +++ b/src/test/java/org/pathlab/forge/evidence/EvidenceSampleManifestTest.java @@ -0,0 +1,20 @@ +package org.pathlab.forge.evidence; + +import static org.junit.jupiter.api.Assertions.assertTrue; +import java.nio.file.Files; +import java.nio.file.Path; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +final class EvidenceSampleManifestTest { + @TempDir Path temporaryDirectory; + @Test void recordsGrandfatheredReadOnlySourceWithCompleteProvenance() throws Exception { + var path = temporaryDirectory.resolve("sample.json"); + Files.writeString(path, "{\"schema\":\"pathlab.evidence-sample/1\"," + + "\"source\":\"BRACS\",\"patientGroup\":\"patient-001\",\"slideId\":\"slide-001\"," + + "\"sha256\":\"" + "a".repeat(64) + "\",\"license\":\"research terms reviewed\"," + + "\"taskLabel\":\"breast-benign\",\"permittedUse\":\"private-research\"," + + "\"bytes\":1234,\"grandfatheredReadOnly\":true}"); + assertTrue(EvidenceSampleManifest.load(path).grandfatheredReadOnly()); + } +} diff --git a/src/test/java/org/pathlab/forge/evidence/ExternalModelWorkerTest.java b/src/test/java/org/pathlab/forge/evidence/ExternalModelWorkerTest.java new file mode 100644 index 0000000..4653a0b --- /dev/null +++ b/src/test/java/org/pathlab/forge/evidence/ExternalModelWorkerTest.java @@ -0,0 +1,23 @@ +package org.pathlab.forge.evidence; + +import static org.junit.jupiter.api.Assertions.assertThrows; + +import com.fasterxml.jackson.databind.ObjectMapper; +import java.nio.file.Path; +import org.junit.jupiter.api.Test; + +final class ExternalModelWorkerTest { + private static final ObjectMapper JSON = new ObjectMapper(); + + @Test void rejectsReusableEmbeddingsAndInvalidRegions() throws Exception { + var pack = EvidencePackManifest.load(Path.of( + "src/main/resources/evidence-packs/he-dinov2-small-v1.json")); + var validPrefix = "{\"schema\":\"pathlab.model-worker-result/1\"," + + "\"status\":\"completed\",\"packManifestSha256\":\"" + pack.sha256() + "\","; + assertThrows(IllegalArgumentException.class, () -> ExternalModelWorker.validateResult( + JSON.readTree(validPrefix + "\"regions\":[],\"embeddings\":[1,2]}"), pack)); + assertThrows(IllegalArgumentException.class, () -> ExternalModelWorker.validateResult( + JSON.readTree(validPrefix + "\"regions\":[{\"id\":\"r1\",\"stage\":\"refined\"," + + "\"kind\":\"support\",\"x\":0,\"y\":0,\"width\":0,\"height\":1,\"score\":0.5}]}"), pack)); + } +} From ec51418f3a33e8a5cbf33201e205ff8a32d47f63 Mon Sep 17 00:00:00 2001 From: Enksodsoon <229880765+Enksodsoon@users.noreply.github.com> Date: Sat, 22 Aug 2026 08:14:50 +0700 Subject: [PATCH 03/50] test: mock evidence runner status --- frontend/src/test/App.test.tsx | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/frontend/src/test/App.test.tsx b/frontend/src/test/App.test.tsx index 41c8502..9996583 100644 --- a/frontend/src/test/App.test.tsx +++ b/frontend/src/test/App.test.tsx @@ -40,6 +40,11 @@ vi.mock('../SlideViewer', () => ({ })) vi.mock('../api', () => ({ + evidenceRunnerStatus: vi.fn(async () => ({ + schema: 'pathlab.evidence-runner-status/1', + status: 'unavailable', + detail: 'Runner is not configured in frontend tests', + })), bootstrap: vi.fn(async () => [[], { conversionRuntime: 'Bio-Formats test', derivativeRuntime: 'libvips test', From 31dc44cd96be41b9d226566da0ec6aef60885a57 Mon Sep 17 00:00:00 2001 From: Enksodsoon <229880765+Enksodsoon@users.noreply.github.com> Date: Sat, 22 Aug 2026 08:22:08 +0700 Subject: [PATCH 04/50] feat: pin and acquire DINOv2 baseline --- docs/evidence-mentor-runner.md | 11 ++ docs/model-cards/he-dinov2-small-v1.md | 18 ++- scripts/acquire-dinov2-small.ps1 | 151 ++++++++++++++++++ .../evidence-packs/he-dinov2-small-v1.json | 18 ++- .../evidence/EvidencePackManifestTest.java | 8 +- 5 files changed, 199 insertions(+), 7 deletions(-) create mode 100644 scripts/acquire-dinov2-small.ps1 diff --git a/docs/evidence-mentor-runner.md b/docs/evidence-mentor-runner.md index 7d4393f..eacffa7 100644 --- a/docs/evidence-mentor-runner.md +++ b/docs/evidence-mentor-runner.md @@ -23,6 +23,17 @@ The deterministic cell/IHC baseline is runnable. DINOv2-small, Hibou-B and HoVer P2000 unless they declare CUDA 12.6 and `sm_61`. Forge signs only bounded regions and descriptors; embeddings, raw pixels and clinical outputs are rejected. +Acquire only the pinned DINOv2 safetensors baseline (never the pickle artifact) with: + +```powershell +.\scripts\acquire-dinov2-small.ps1 +``` + +The acquisition command reserves the fixed model quota, downloads from the exact upstream commit, +verifies byte counts and SHA-256 values before atomic installation, and records a local receipt. It +does not activate the pack: the status remains `not-evaluable` until the separate offline worker is +installed, checksum-bound in the pack manifest, and qualified on the P2000 host. + New material uses fixed hard buckets: 45 GB source, 25 GB derived, 10 GB models, 10 GB evidence/test, and a 10 GB untouchable reserve. Existing BRACS remains grandfathered read-only and is not copied. diff --git a/docs/model-cards/he-dinov2-small-v1.md b/docs/model-cards/he-dinov2-small-v1.md index 7c75443..1756881 100644 --- a/docs/model-cards/he-dinov2-small-v1.md +++ b/docs/model-cards/he-dinov2-small-v1.md @@ -1,5 +1,17 @@ # DINOv2-small H&E baseline -Status: `not-evaluable`. No artifact is installed. This generic Apache-2.0 feature extractor is the -experimental executable baseline after its exact revision and hashes are pinned. Embeddings remain -local and only bounded evidence regions may leave Forge. +Status: `not-evaluable-worker-not-installed`. + +The public `facebook/dinov2-small` repository is pinned at +`ed25f3a31f01632728cabb09d1542f84ab7b0056`. Acquisition accepts only the +88,249,960-byte `model.safetensors` artifact with SHA-256 +`ae1e99fcefd534ed978cdeb8326f08030c96e28b7a81ffcbc98a857c84d14be1`; the +pickle artifact is excluded. Configuration and preprocessing artifacts are +also checksum-pinned in the pack manifest. + +The local qualification host reports a Quadro P2000 (compute capability 6.1, +5,120 MiB VRAM). The model files are installed locally, but no worker has yet +passed the required CUDA 12.6 / `sm_61`, offline, memory, latency, restart, or +held-out retrieval gates. Therefore the pack cannot execute or enter Viewer +evidence and remains default-off. Embeddings remain local and only bounded +evidence regions may leave Forge. diff --git a/scripts/acquire-dinov2-small.ps1 b/scripts/acquire-dinov2-small.ps1 new file mode 100644 index 0000000..39893de --- /dev/null +++ b/scripts/acquire-dinov2-small.ps1 @@ -0,0 +1,151 @@ +[CmdletBinding()] +param( + [string]$StateRoot +) + +$ErrorActionPreference = 'Stop' +$repoId = 'facebook/dinov2-small' +$revision = 'ed25f3a31f01632728cabb09d1542f84ab7b0056' +$packId = 'he-dinov2-small-v1' +$packVersion = '1' +$modelQuotaBytes = 10GB +$files = @( + [pscustomobject]@{ + Name = 'config.json' + Bytes = 547L + Sha256 = '1809f83e3bdb1609a501a610ad4a742f4fd8ae44d72ca4aa0df52d1f2ac8628d' + }, + [pscustomobject]@{ + Name = 'preprocessor_config.json' + Bytes = 436L + Sha256 = '14e780d86fa1861f8751f868d7f45425b5feb55c38ca26f152ca5097ab30f828' + }, + [pscustomobject]@{ + Name = 'model.safetensors' + Bytes = 88249960L + Sha256 = 'ae1e99fcefd534ed978cdeb8326f08030c96e28b7a81ffcbc98a857c84d14be1' + } +) + +if (-not $StateRoot) { + $StateRoot = if (Test-Path -LiteralPath 'D:\PathLabData' -PathType Container) { + 'D:\PathLabData\EvidenceMentor\state' + } else { + Join-Path $env:LOCALAPPDATA 'PathLab\EvidenceMentor\state' + } +} + +$state = [IO.Path]::GetFullPath($StateRoot) +$modelsRoot = [IO.Path]::GetFullPath((Join-Path $state 'models')) +$installRoot = [IO.Path]::GetFullPath((Join-Path $modelsRoot "$packId\$packVersion")) +if (-not $installRoot.StartsWith($modelsRoot + [IO.Path]::DirectorySeparatorChar, + [StringComparison]::OrdinalIgnoreCase)) { + throw 'DINOv2 install path escaped the Evidence Mentor model root.' +} + +$reservationRoot = Join-Path $state 'quota\reservations\models' +New-Item -ItemType Directory -Path $installRoot -Force | Out-Null +New-Item -ItemType Directory -Path $reservationRoot -Force | Out-Null + +function Get-TreeBytes([string]$Path) { + if (-not (Test-Path -LiteralPath $Path -PathType Container)) { return 0L } + $sum = 0L + Get-ChildItem -LiteralPath $Path -Recurse -File | ForEach-Object { + $sum = [long]($sum + $_.Length) + } + return $sum +} + +$requiredBytes = 0L +foreach ($file in $files) { + $target = Join-Path $installRoot $file.Name + if (Test-Path -LiteralPath $target -PathType Leaf) { + $existing = Get-Item -LiteralPath $target + $existingHash = (Get-FileHash -LiteralPath $target -Algorithm SHA256).Hash.ToLowerInvariant() + if ($existing.Length -ne $file.Bytes -or $existingHash -ne $file.Sha256) { + throw "Existing DINOv2 artifact failed checksum: $($file.Name)" + } + } else { + $requiredBytes = [long]($requiredBytes + $file.Bytes) + } +} + +$existingReceiptPath = Join-Path $installRoot 'acquisition.json' +if ($requiredBytes -eq 0 -and (Test-Path -LiteralPath $existingReceiptPath -PathType Leaf)) { + $existingReceipt = Get-Content -Raw -LiteralPath $existingReceiptPath | ConvertFrom-Json + if ($existingReceipt.schema -ne 'pathlab.model-acquisition/1' -or + $existingReceipt.repository -ne $repoId -or $existingReceipt.revision -ne $revision) { + throw 'Existing DINOv2 acquisition receipt is invalid.' + } + Write-Output "Already acquired $repoId@$revision under $installRoot" + Write-Output 'Status remains not-evaluable until a checksum-pinned offline worker passes P2000 qualification.' + return +} + +$reservedBytes = 0L +Get-ChildItem -LiteralPath $reservationRoot -Filter '*.reservation' -File | ForEach-Object { + $value = 0L + if (-not [long]::TryParse(([IO.File]::ReadAllText($_.FullName).Trim()), [ref]$value) -or $value -lt 0) { + throw "Invalid model quota reservation: $($_.Name)" + } + $reservedBytes = [long]($reservedBytes + $value) +} +$usedBytes = Get-TreeBytes $modelsRoot +if ($usedBytes -gt $modelQuotaBytes -or $reservedBytes -gt ($modelQuotaBytes - $usedBytes) -or + $requiredBytes -gt ($modelQuotaBytes - $usedBytes - $reservedBytes)) { + throw 'Evidence model quota is exhausted.' +} + +$reservation = Join-Path $reservationRoot "acquire-$packId-$packVersion.reservation" +$reservationPartial = "$reservation.partial" +if ((Test-Path -LiteralPath $reservation) -or (Test-Path -LiteralPath $reservationPartial)) { + throw 'A DINOv2 acquisition reservation already exists; inspect it before retrying.' +} +[IO.File]::WriteAllText($reservationPartial, [string]$requiredBytes) +Move-Item -LiteralPath $reservationPartial -Destination $reservation + +try { + foreach ($file in $files) { + $target = Join-Path $installRoot $file.Name + if (Test-Path -LiteralPath $target -PathType Leaf) { continue } + $partial = "$target.partial" + if (Test-Path -LiteralPath $partial) { + throw "Partial DINOv2 artifact requires review before retry: $partial" + } + $uri = "https://huggingface.co/$repoId/resolve/$revision/$($file.Name)" + Invoke-WebRequest -Uri $uri -OutFile $partial -MaximumRedirection 10 + $download = Get-Item -LiteralPath $partial + $hash = (Get-FileHash -LiteralPath $partial -Algorithm SHA256).Hash.ToLowerInvariant() + if ($download.Length -ne $file.Bytes -or $hash -ne $file.Sha256) { + throw "Downloaded DINOv2 artifact failed checksum: $($file.Name)" + } + Move-Item -LiteralPath $partial -Destination $target + } + + $receipt = [ordered]@{ + schema = 'pathlab.model-acquisition/1' + packId = $packId + packVersion = $packVersion + repository = $repoId + revision = $revision + license = 'Apache-2.0' + permittedUse = 'private-research' + acquiredAt = [DateTimeOffset]::UtcNow.ToString('O') + artifacts = @($files | ForEach-Object { + [ordered]@{ name = $_.Name; bytes = $_.Bytes; sha256 = $_.Sha256 } + }) + analysisNetwork = 'disabled' + activationStatus = 'not-evaluable-worker-not-installed' + } + $receiptPath = Join-Path $installRoot 'acquisition.json' + $receiptPartial = "$receiptPath.partial" + $receipt | ConvertTo-Json -Depth 5 | Set-Content -LiteralPath $receiptPartial -Encoding utf8NoBOM + Move-Item -LiteralPath $receiptPartial -Destination $receiptPath -Force +} finally { + if (Test-Path -LiteralPath $reservation -PathType Leaf) { + Remove-Item -LiteralPath $reservation -Force + } +} + +Write-Output "Acquired $repoId@$revision under $installRoot" +Write-Output 'Status remains not-evaluable until a checksum-pinned offline worker passes P2000 qualification.' diff --git a/src/main/resources/evidence-packs/he-dinov2-small-v1.json b/src/main/resources/evidence-packs/he-dinov2-small-v1.json index f705b6f..7769d38 100644 --- a/src/main/resources/evidence-packs/he-dinov2-small-v1.json +++ b/src/main/resources/evidence-packs/he-dinov2-small-v1.json @@ -10,7 +10,19 @@ "id": "dinov2-v1", "tilePixels": 512 }, - "artifacts": [], + "artifacts": [{ + "name": "model", + "sha256": "ae1e99fcefd534ed978cdeb8326f08030c96e28b7a81ffcbc98a857c84d14be1", + "source": "https://huggingface.co/facebook/dinov2-small/resolve/ed25f3a31f01632728cabb09d1542f84ab7b0056/model.safetensors" + }, { + "name": "config", + "sha256": "1809f83e3bdb1609a501a610ad4a742f4fd8ae44d72ca4aa0df52d1f2ac8628d", + "source": "https://huggingface.co/facebook/dinov2-small/resolve/ed25f3a31f01632728cabb09d1542f84ab7b0056/config.json" + }, { + "name": "preprocessor", + "sha256": "14e780d86fa1861f8751f868d7f45425b5feb55c38ca26f152ca5097ab30f828", + "source": "https://huggingface.co/facebook/dinov2-small/resolve/ed25f3a31f01632728cabb09d1542f84ab7b0056/preprocessor_config.json" + }], "runtimeCompatibility": { "workerProtocol": "pathlab.model-worker-result/1", "executionProvider": "cuda", @@ -21,7 +33,7 @@ "licenseLedger": [{ "component": "facebook/dinov2-small", "license": "Apache-2.0", - "revision": "PIN_REQUIRED_BEFORE_ACQUISITION", + "revision": "ed25f3a31f01632728cabb09d1542f84ab7b0056", "permittedUse": "private-research", "redistributable": false, "derivativesAllowed": false @@ -42,7 +54,7 @@ "validation": { "status": "not-evaluable", "modelCard": "docs/model-cards/he-dinov2-small-v1.md", - "heldOutEvaluation": "not-evaluable-no-installed-artifact" + "heldOutEvaluation": "not-evaluable-worker-not-installed" }, "outputSchema": "pathlab.ai-evidence/1" } diff --git a/src/test/java/org/pathlab/forge/evidence/EvidencePackManifestTest.java b/src/test/java/org/pathlab/forge/evidence/EvidencePackManifestTest.java index 56f7f61..200fb4a 100644 --- a/src/test/java/org/pathlab/forge/evidence/EvidencePackManifestTest.java +++ b/src/test/java/org/pathlab/forge/evidence/EvidencePackManifestTest.java @@ -46,7 +46,13 @@ void bundledPilotCatalogActivatesOnlyInstalledLawfulBaselines() throws Exception var root = Path.of("src/main/resources/evidence-packs"); assertTrue(EvidencePackManifest.load(root.resolve("cell-od-watershed-v1.json")).pilotEligible()); assertTrue(EvidencePackManifest.load(root.resolve("ihc-descriptive-v1.json")).pilotEligible()); - assertFalse(EvidencePackManifest.load(root.resolve("he-dinov2-small-v1.json")).pilotEligible()); + var dino = EvidencePackManifest.load(root.resolve("he-dinov2-small-v1.json")); + assertFalse(dino.pilotEligible()); + assertEquals("ed25f3a31f01632728cabb09d1542f84ab7b0056", + dino.licenseLedger().get(0).revision()); + assertEquals(3, dino.artifacts().size()); + assertEquals("ae1e99fcefd534ed978cdeb8326f08030c96e28b7a81ffcbc98a857c84d14be1", + dino.artifacts().get(0).sha256()); assertFalse(EvidencePackManifest.load(root.resolve("he-hibou-b-v1.json")).pilotEligible()); var hoverNet = EvidencePackManifest.load(root.resolve("cell-hovernet-fast-v1.json")); assertFalse(hoverNet.pilotEligible()); From bf7270f37c4d43605514e5a6726a895d3c3e288d Mon Sep 17 00:00:00 2001 From: Enksodsoon <229880765+Enksodsoon@users.noreply.github.com> Date: Sat, 22 Aug 2026 09:10:30 +0700 Subject: [PATCH 05/50] Record offline DINOv2 worker qualification --- docs/model-cards/he-dinov2-small-v1.md | 30 ++++++++++++++----- scripts/acquire-dinov2-small.ps1 | 4 +-- .../evidence-packs/he-dinov2-small-v1.json | 14 ++++++++- .../evidence/EvidencePackManifestTest.java | 5 +++- 4 files changed, 42 insertions(+), 11 deletions(-) diff --git a/docs/model-cards/he-dinov2-small-v1.md b/docs/model-cards/he-dinov2-small-v1.md index 1756881..e5a338a 100644 --- a/docs/model-cards/he-dinov2-small-v1.md +++ b/docs/model-cards/he-dinov2-small-v1.md @@ -1,6 +1,6 @@ # DINOv2-small H&E baseline -Status: `not-evaluable-worker-not-installed`. +Status: `not-evaluable-held-out-retrieval-pending`. The public `facebook/dinov2-small` repository is pinned at `ed25f3a31f01632728cabb09d1542f84ab7b0056`. Acquisition accepts only the @@ -9,9 +9,25 @@ The public `facebook/dinov2-small` repository is pinned at pickle artifact is excluded. Configuration and preprocessing artifacts are also checksum-pinned in the pack manifest. -The local qualification host reports a Quadro P2000 (compute capability 6.1, -5,120 MiB VRAM). The model files are installed locally, but no worker has yet -passed the required CUDA 12.6 / `sm_61`, offline, memory, latency, restart, or -held-out retrieval gates. Therefore the pack cannot execute or enter Viewer -evidence and remains default-off. Embeddings remain local and only bounded -evidence regions may leave Forge. +The external feature-pack runtime uses Python 3.12, PyTorch 2.7.1+cu126, and a +compiled Windows launcher. Forge still embeds none of Python, PyTorch, CUDA, or +the weights. A runtime manifest pins the interpreter and critical CUDA/model +files, and the worker rechecks that ledger before loading the model. Network +library entry points and Hugging Face/Transformers online behavior are disabled +during analysis. + +On the local Quadro P2000 qualification host (compute capability 6.1, 5,120 MiB +VRAM), the CUDA build exposed `sm_61` and completed a real device operation. A +three-run synthetic protocol smoke produced byte-identical evidence-region +content (`9b018aa1483d3dba15cf5070e64dc4112e6ff4c6751f69cc463e2d0778889c25`). +Cold process wall time was 5.89-6.06 seconds; the bounded 16-tile inference stage +was 0.32-0.36 seconds, peak reserved VRAM was 122 MiB, and peak process RAM was +759.37-760.41 MiB. Tampered artifact metadata, a missing offline flag, and an +enabled analysis-network state all failed with no result artifact. + +These are runtime and protocol measurements on synthetic tissue-like pixels, +not pathology retrieval evidence. Patient/source-held-out breast, GI, lung, +lymph-node, benign/reactive, OOD, restart, and five-minute refinement gates have +not run. The model pack therefore remains `not-evaluable`, default-off, and +ineligible for Viewer evidence or staff/demo activation. Embeddings remain local +and only bounded evidence regions may leave Forge after model qualification. diff --git a/scripts/acquire-dinov2-small.ps1 b/scripts/acquire-dinov2-small.ps1 index 39893de..e97cf9b 100644 --- a/scripts/acquire-dinov2-small.ps1 +++ b/scripts/acquire-dinov2-small.ps1 @@ -78,7 +78,7 @@ if ($requiredBytes -eq 0 -and (Test-Path -LiteralPath $existingReceiptPath -Path throw 'Existing DINOv2 acquisition receipt is invalid.' } Write-Output "Already acquired $repoId@$revision under $installRoot" - Write-Output 'Status remains not-evaluable until a checksum-pinned offline worker passes P2000 qualification.' + Write-Output 'Acquisition never activates the pack; inspect its separate runtime and held-out qualification records.' return } @@ -148,4 +148,4 @@ try { } Write-Output "Acquired $repoId@$revision under $installRoot" -Write-Output 'Status remains not-evaluable until a checksum-pinned offline worker passes P2000 qualification.' +Write-Output 'Acquisition never activates the pack; inspect its separate runtime and held-out qualification records.' diff --git a/src/main/resources/evidence-packs/he-dinov2-small-v1.json b/src/main/resources/evidence-packs/he-dinov2-small-v1.json index 7769d38..c8d6308 100644 --- a/src/main/resources/evidence-packs/he-dinov2-small-v1.json +++ b/src/main/resources/evidence-packs/he-dinov2-small-v1.json @@ -22,6 +22,18 @@ "name": "preprocessor", "sha256": "14e780d86fa1861f8751f868d7f45425b5feb55c38ca26f152ca5097ab30f828", "source": "https://huggingface.co/facebook/dinov2-small/resolve/ed25f3a31f01632728cabb09d1542f84ab7b0056/preprocessor_config.json" + }, { + "name": "worker", + "sha256": "daa33e61d2607e23ecae29e82f274ca05c64f896f9c37061c6ca77ca81aa774b", + "source": "pathlab-local-build:dinov2-worker-v1/windows-x64" + }, { + "name": "workerSource", + "sha256": "4884827fea5f6b04ecd2ec4b5fd31800e2a6d68547f7142aea506a70dd42fd8f", + "source": "pathlab-local-build:dinov2-worker-v1/worker.py" + }, { + "name": "runtime-manifest", + "sha256": "1969c91db66d8acc70b6c8648a57d3bfc923880e03f1f7f1e30c593133c01635", + "source": "pathlab-local-build:torch-2.7.1-cu126-python-3.12/windows-x64" }], "runtimeCompatibility": { "workerProtocol": "pathlab.model-worker-result/1", @@ -54,7 +66,7 @@ "validation": { "status": "not-evaluable", "modelCard": "docs/model-cards/he-dinov2-small-v1.md", - "heldOutEvaluation": "not-evaluable-worker-not-installed" + "heldOutEvaluation": "not-evaluable-held-out-retrieval-pending" }, "outputSchema": "pathlab.ai-evidence/1" } diff --git a/src/test/java/org/pathlab/forge/evidence/EvidencePackManifestTest.java b/src/test/java/org/pathlab/forge/evidence/EvidencePackManifestTest.java index 200fb4a..adf8b4a 100644 --- a/src/test/java/org/pathlab/forge/evidence/EvidencePackManifestTest.java +++ b/src/test/java/org/pathlab/forge/evidence/EvidencePackManifestTest.java @@ -50,9 +50,12 @@ void bundledPilotCatalogActivatesOnlyInstalledLawfulBaselines() throws Exception assertFalse(dino.pilotEligible()); assertEquals("ed25f3a31f01632728cabb09d1542f84ab7b0056", dino.licenseLedger().get(0).revision()); - assertEquals(3, dino.artifacts().size()); + assertEquals(6, dino.artifacts().size()); assertEquals("ae1e99fcefd534ed978cdeb8326f08030c96e28b7a81ffcbc98a857c84d14be1", dino.artifacts().get(0).sha256()); + assertEquals("daa33e61d2607e23ecae29e82f274ca05c64f896f9c37061c6ca77ca81aa774b", + dino.artifacts().stream().filter(item -> "worker".equals(item.name())) + .findFirst().orElseThrow().sha256()); assertFalse(EvidencePackManifest.load(root.resolve("he-hibou-b-v1.json")).pilotEligible()); var hoverNet = EvidencePackManifest.load(root.resolve("cell-hovernet-fast-v1.json")); assertFalse(hoverNet.pilotEligible()); From 125c99c7146ee560d1d3cd6375279b3e2cb8b374 Mon Sep 17 00:00:00 2001 From: Enksodsoon <229880765+Enksodsoon@users.noreply.github.com> Date: Sat, 22 Aug 2026 09:34:28 +0700 Subject: [PATCH 06/50] Require immutable tiles for H&E qualification --- .gitattributes | 1 + docs/evidence-mentor-runner.md | 25 +- .../he-retrieval-qualification-protocol-v1.md | 28 ++ docs/model-cards/he-dinov2-small-v1.md | 24 +- scripts/build-bracs-dinov2-tile-cache.ps1 | 240 ++++++++++++++++++ .../forge/evidence/EvidenceJobProcessor.java | 19 +- .../forge/evidence/EvidencePackManifest.java | 5 +- .../evidence/EvidenceQualificationCohort.java | 232 +++++++++++++++++ .../evidence/EvidenceTileCacheManifest.java | 184 ++++++++++++++ .../evidence-packs/he-dinov2-small-v1.json | 8 +- .../evidence/EvidenceJobProcessorTest.java | 45 ++++ .../evidence/EvidencePackManifestTest.java | 2 +- .../EvidenceQualificationCohortTest.java | 68 +++++ .../EvidenceRealCohortIntegrationTest.java | 25 ++ .../EvidenceTileCacheManifestTest.java | 101 ++++++++ 15 files changed, 993 insertions(+), 14 deletions(-) create mode 100644 .gitattributes create mode 100644 docs/evidence/he-retrieval-qualification-protocol-v1.md create mode 100644 scripts/build-bracs-dinov2-tile-cache.ps1 create mode 100644 src/main/java/org/pathlab/forge/evidence/EvidenceQualificationCohort.java create mode 100644 src/main/java/org/pathlab/forge/evidence/EvidenceTileCacheManifest.java create mode 100644 src/test/java/org/pathlab/forge/evidence/EvidenceQualificationCohortTest.java create mode 100644 src/test/java/org/pathlab/forge/evidence/EvidenceRealCohortIntegrationTest.java create mode 100644 src/test/java/org/pathlab/forge/evidence/EvidenceTileCacheManifestTest.java diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..1ab67a5 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +docs/evidence/he-retrieval-qualification-protocol-v1.md text eol=lf diff --git a/docs/evidence-mentor-runner.md b/docs/evidence-mentor-runner.md index eacffa7..25a2d99 100644 --- a/docs/evidence-mentor-runner.md +++ b/docs/evidence-mentor-runner.md @@ -16,9 +16,11 @@ offline. Acquisition is a separate allowlisted operation and must reserve quota extraction. `GET /v1/status` returns authenticated, non-outcome runtime and quota telemetry. A dashboard failure does not stop processing. -The deterministic cell/IHC baseline is runnable. DINOv2-small, Hibou-B and HoVer-Net remain -`not-evaluable` until an exact, rights-approved worker and model artifact are installed beneath -`state/models//`. External workers use `pathlab.model-worker-result/1`, inherit the +The deterministic cell/IHC baseline is runnable. The checksum-pinned DINOv2-small worker is locally +executable on the P2000 but remains `not-evaluable` because its cross-tissue held-out cohort is +incomplete. Hibou-B and HoVer-Net remain `not-evaluable` until exact, rights-approved workers and +model artifacts are installed beneath `state/models//`. External workers use +`pathlab.model-worker-result/1`, inherit the 16 GB RAM and 4.5 GB VRAM envelope, receive offline environment controls, and are rejected on the P2000 unless they declare CUDA 12.6 and `sm_61`. Forge signs only bounded regions and descriptors; embeddings, raw pixels and clinical outputs are rejected. @@ -31,8 +33,21 @@ Acquire only the pinned DINOv2 safetensors baseline (never the pickle artifact) The acquisition command reserves the fixed model quota, downloads from the exact upstream commit, verifies byte counts and SHA-256 values before atomic installation, and records a local receipt. It -does not activate the pack: the status remains `not-evaluable` until the separate offline worker is -installed, checksum-bound in the pack manifest, and qualified on the P2000 host. +does not activate the pack. Worker installation, runtime qualification, model qualification, and +activation remain separate gates. + +H&E jobs additionally require a `pathlab.tile-cache/1` manifest and its SHA-256 in +`tileCacheManifest` and `tileCacheManifestSha256`. Forge verifies the source revision, source/sample +provenance, every 512 x 512 RGB PNG tile, coordinates, and checksums before starting the external +worker. Preview-only H&E model inference is rejected. A bounded, breast-only BRACS contract smoke can +be built idempotently with: + +```powershell +.\scripts\build-bracs-dinov2-tile-cache.ps1 +``` + +That cache is real research data but not a qualifying cohort: it lacks the frozen reference, GI, +lung, lymph-node, independent benign/reactive-source, and OOD groups. New material uses fixed hard buckets: 45 GB source, 25 GB derived, 10 GB models, 10 GB evidence/test, and a 10 GB untouchable reserve. Existing BRACS remains grandfathered read-only and is not copied. diff --git a/docs/evidence/he-retrieval-qualification-protocol-v1.md b/docs/evidence/he-retrieval-qualification-protocol-v1.md new file mode 100644 index 0000000..4cfcb5c --- /dev/null +++ b/docs/evidence/he-retrieval-qualification-protocol-v1.md @@ -0,0 +1,28 @@ +# H&E retrieval qualification protocol v1 + +Status: pre-registered before cross-tissue evaluation. + +This is an internal private-research model qualification protocol. It is not a clinical analytical +procedure, diagnostic validation, ICH Q2(R2) validation, CLSI study, or medical-device performance +claim. Public annotations are research references rather than clinical truth. + +## Frozen design + +- Evaluation groups: breast, GI, lung, lymph node, and benign/reactive. +- Splits: immutable reference, query, and OOD manifests. +- Patient, slide, and declared source-group overlap across splits: zero. +- Minimum reference samples per group: 20. +- Minimum query samples per group: 20. +- Minimum OOD samples: 20. +- Candidate: checksum-pinned DINOv2-small pack. +- Deterministic comparison: `color-histogram-v1` on exactly the same cached 512-pixel RGB tiles. +- Macro recall@5 improvement over the baseline: at least 0.05 absolute. +- Macro nDCG@10 improvement over the baseline: at least 0.03 absolute. +- OOD AUROC: at least 0.80. +- Exact repeated-run ranking agreement is required. +- Any missing rights, checksum, source grouping, group coverage, or split integrity yields + `NOT_EVALUABLE`; it cannot be repaired by lowering criteria after results are observed. + +These thresholds govern only whether DINOv2-small may be called an experimental executable H&E +retrieval baseline. They do not qualify Hibou-B, any diagnostic task, or activation for staff/demo +accounts. Activation continues to require a separately signed `qualified` pack. diff --git a/docs/model-cards/he-dinov2-small-v1.md b/docs/model-cards/he-dinov2-small-v1.md index e5a338a..9c553e2 100644 --- a/docs/model-cards/he-dinov2-small-v1.md +++ b/docs/model-cards/he-dinov2-small-v1.md @@ -1,6 +1,6 @@ # DINOv2-small H&E baseline -Status: `not-evaluable-held-out-retrieval-pending`. +Status: `not-evaluable-cross-tissue-cohort-incomplete`. The public `facebook/dinov2-small` repository is pinned at `ed25f3a31f01632728cabb09d1542f84ab7b0056`. Acquisition accepts only the @@ -31,3 +31,25 @@ lymph-node, benign/reactive, OOD, restart, and five-minute refinement gates have not run. The model pack therefore remains `not-evaluable`, default-off, and ineligible for Viewer evidence or staff/demo activation. Embeddings remain local and only bounded evidence regions may leave Forge after model qualification. + +## Real BRACS tile-cache smoke + +The frozen `bracs-roi-smoke-v1` cache contains seven patient-distinct validation +ROI samples and 28 deterministic 512 x 512 RGB PNG tiles. Its cohort manifest is +19,860,731 bytes including tiles and has SHA-256 +`4653fa49c1b488a7ab90f40a7722e802082a02dbeb967303b09078c4b2bbb2bb`. +Every source, sample manifest, tile manifest, tile pixel file, coordinate, and +revision is independently checksum-bound before inference. + +One real BRACS ROI worker smoke was repeated three times. Evidence-region content +was identical (`2e4f597caf50837bbf67de8179742a6c233a6160a723f7933adb8e5709860457`). +Cold wall time was 6.88-7.23 seconds, bounded four-tile model time was 0.19-0.23 +seconds, peak reserved VRAM was 122 MiB, and peak process RAM was 761.30-762.21 +MiB. A stale slide revision and an incorrect tile-manifest checksum both failed +without a result artifact. + +This remains a protocol smoke, not held-out retrieval qualification: BRACS ROI is +breast-only, the bounded cache is not a whole-slide cohort, and reference, GI, +lung, lymph-node, independent benign/reactive-source, and OOD coverage are +absent. The pre-registered protocol therefore returns `NOT_EVALUABLE` without +computing or relaxing model-performance thresholds. diff --git a/scripts/build-bracs-dinov2-tile-cache.ps1 b/scripts/build-bracs-dinov2-tile-cache.ps1 new file mode 100644 index 0000000..b239fad --- /dev/null +++ b/scripts/build-bracs-dinov2-tile-cache.ps1 @@ -0,0 +1,240 @@ +[CmdletBinding()] +param( + [string]$DatasetRoot = 'D:\PathLabData\BRACS', + [string]$StateRoot = 'D:\PathLabData\EvidenceMentor\state', + [string]$FrozenAt = '2026-08-22T00:00:00Z', + [ValidateRange(1, 5)][int]$SamplesPerLabel = 1, + [ValidateRange(1, 5)][int]$TilesPerSample = 4 +) + +$ErrorActionPreference = 'Stop' +$derivedQuotaBytes = 25GB +$cohortId = 'bracs-roi-smoke-v1' +$manifestPath = Join-Path $DatasetRoot 'prepared\bracs-roi-clean-v1\manifest.jsonl' +$manifestCsvPath = Join-Path $DatasetRoot 'prepared\bracs-roi-clean-v1\manifest.csv' +$provenancePath = Join-Path $DatasetRoot 'prepared\bracs-roi-clean-v1\provenance.json' +$rawRoot = Join-Path $DatasetRoot 'raw\BRACS_RoI\latest_version' +$derivedRoot = [IO.Path]::GetFullPath((Join-Path $StateRoot 'derived')) +$outputRoot = [IO.Path]::GetFullPath((Join-Path $derivedRoot "he-dinov2-small-v1\$cohortId")) +$partialRoot = "$outputRoot.partial" + +try { + [void][DateTimeOffset]::ParseExact($FrozenAt, 'yyyy-MM-ddTHH:mm:ssZ', + [Globalization.CultureInfo]::InvariantCulture, + [Globalization.DateTimeStyles]::AssumeUniversal) +} catch { + throw 'FrozenAt must be an immutable UTC timestamp such as 2026-08-22T00:00:00Z.' +} + +if (-not $outputRoot.StartsWith($derivedRoot + [IO.Path]::DirectorySeparatorChar, + [StringComparison]::OrdinalIgnoreCase)) { + throw 'BRACS tile-cache path escaped the Evidence Mentor derived-data root.' +} +if (-not (Test-Path -LiteralPath $manifestPath -PathType Leaf) -or + -not (Test-Path -LiteralPath $manifestCsvPath -PathType Leaf) -or + -not (Test-Path -LiteralPath $provenancePath -PathType Leaf) -or + -not (Test-Path -LiteralPath $rawRoot -PathType Container)) { + throw 'The reviewed local BRACS ROI dataset is unavailable.' +} +if (Test-Path -LiteralPath $outputRoot) { + $cohort = Join-Path $outputRoot 'cohort.json' + if (-not (Test-Path -LiteralPath $cohort -PathType Leaf)) { + throw 'The existing BRACS tile cache is incomplete and requires review.' + } + Write-Output "Existing immutable cohort: $cohort" + Write-Output "SHA256: $((Get-FileHash -LiteralPath $cohort -Algorithm SHA256).Hash.ToLowerInvariant())" + return +} +if (Test-Path -LiteralPath $partialRoot) { + throw 'A partial BRACS tile-cache build requires review before retrying.' +} + +function Get-TreeBytes([string]$Path) { + if (-not (Test-Path -LiteralPath $Path -PathType Container)) { return 0L } + $sum = 0L + Get-ChildItem -LiteralPath $Path -Recurse -File | ForEach-Object { + $sum = [long]($sum + $_.Length) + } + return $sum +} + +$provenance = Get-Content -Raw -LiteralPath $provenancePath | ConvertFrom-Json +if ($provenance.dataset -ne 'BRACS_RoI_latest_version' -or + $provenance.license_declared_by_current_source -ne 'CC-BY-NC-4.0' -or + -not $provenance.patient_disjoint) { + throw 'BRACS provenance does not match the reviewed patient-disjoint research dataset.' +} +$actualManifestSha = (Get-FileHash -LiteralPath $manifestCsvPath -Algorithm SHA256).Hash.ToLowerInvariant() +if ($actualManifestSha -ne $provenance.manifest_sha256) { + throw 'BRACS source manifest checksum does not match its provenance record.' +} + +$rows = @(Get-Content -LiteralPath $manifestPath | ForEach-Object { $_ | ConvertFrom-Json } | + Where-Object { $_.split -eq 'val' -and $_.mode -eq 'RGB' -and + $_.width -ge 512 -and $_.height -ge 512 } | + Sort-Object label, patient_id, slide_id, roi_id) +$selected = New-Object System.Collections.Generic.List[object] +foreach ($label in @('N', 'PB', 'UDH', 'FEA', 'ADH', 'DCIS', 'IC')) { + $seenPatients = @{} + foreach ($row in @($rows | Where-Object label -eq $label)) { + if ($seenPatients.ContainsKey([string]$row.patient_id)) { continue } + $seenPatients[[string]$row.patient_id] = $true + $selected.Add($row) + if (($selected | Where-Object label -eq $label).Count -ge $SamplesPerLabel) { break } + } + if (($selected | Where-Object label -eq $label).Count -lt $SamplesPerLabel) { + throw "BRACS validation split lacks enough distinct patients for label $label." + } +} + +$estimatedBytes = [long]$selected.Count * $TilesPerSample * 1024 * 1024 +$usedBytes = Get-TreeBytes $derivedRoot +if ($usedBytes -gt $derivedQuotaBytes -or $estimatedBytes -gt ($derivedQuotaBytes - $usedBytes)) { + throw 'Evidence Mentor derived-data quota is exhausted.' +} +$reservationRoot = Join-Path $StateRoot 'quota\reservations\derived' +New-Item -ItemType Directory -Path $reservationRoot -Force | Out-Null +$reservation = Join-Path $reservationRoot "build-$cohortId.reservation" +$reservationPartial = "$reservation.partial" +if ((Test-Path -LiteralPath $reservation) -or (Test-Path -LiteralPath $reservationPartial)) { + throw 'A BRACS tile-cache quota reservation already exists.' +} +[IO.File]::WriteAllText($reservationPartial, [string]$estimatedBytes) +Move-Item -LiteralPath $reservationPartial -Destination $reservation + +Add-Type -AssemblyName System.Drawing +try { + New-Item -ItemType Directory -Path $partialRoot -Force | Out-Null + $cohortSamples = New-Object System.Collections.Generic.List[object] + foreach ($row in $selected) { + $source = [IO.Path]::GetFullPath((Join-Path $rawRoot ([string]$row.relative_path))) + if (-not $source.StartsWith([IO.Path]::GetFullPath($rawRoot) + [IO.Path]::DirectorySeparatorChar, + [StringComparison]::OrdinalIgnoreCase) -or -not (Test-Path -LiteralPath $source -PathType Leaf)) { + throw "BRACS source escaped its reviewed root or is unavailable: $($row.roi_id)" + } + $sourceHash = (Get-FileHash -LiteralPath $source -Algorithm SHA256).Hash.ToLowerInvariant() + if ($sourceHash -ne $row.sha256 -or (Get-Item -LiteralPath $source).Length -ne $row.file_size_bytes) { + throw "BRACS source checksum failed: $($row.roi_id)" + } + $sampleRoot = Join-Path $partialRoot "samples\$($row.roi_id)" + $tileRoot = Join-Path $sampleRoot 'tiles' + New-Item -ItemType Directory -Path $tileRoot -Force | Out-Null + $sampleManifest = [ordered]@{ + schema = 'pathlab.evidence-sample/1' + source = 'BRACS_RoI_latest_version' + patientGroup = [string]$row.patient_id + slideId = [string]$row.slide_id + sha256 = $sourceHash + license = 'CC-BY-NC-4.0 declared by reviewed BRACS source' + taskLabel = "breast-$(([string]$row.label).ToLowerInvariant())" + permittedUse = 'private-research' + bytes = [long]$row.file_size_bytes + grandfatheredReadOnly = $true + } + $samplePath = Join-Path $sampleRoot 'sample.json' + $sampleManifest | ConvertTo-Json -Depth 6 | Set-Content -LiteralPath "$samplePath.partial" -Encoding utf8NoBOM + Move-Item -LiteralPath "$samplePath.partial" -Destination $samplePath + + $image = New-Object Drawing.Bitmap $source + try { + if ($image.Width -ne $row.width -or $image.Height -ne $row.height) { + throw "BRACS decoded geometry does not match its manifest: $($row.roi_id)" + } + $positions = @( + [pscustomobject]@{ X = [int](($image.Width - 512) / 2); Y = [int](($image.Height - 512) / 2) }, + [pscustomobject]@{ X = 0; Y = 0 }, + [pscustomobject]@{ X = $image.Width - 512; Y = 0 }, + [pscustomobject]@{ X = 0; Y = $image.Height - 512 }, + [pscustomobject]@{ X = $image.Width - 512; Y = $image.Height - 512 } + ) | Sort-Object X, Y -Unique | Select-Object -First $TilesPerSample + $tiles = New-Object System.Collections.Generic.List[object] + $tileIndex = 0 + foreach ($position in $positions) { + $tileIndex++ + $tileName = 'tile-{0:D3}.png' -f $tileIndex + $tilePath = Join-Path $tileRoot $tileName + $rectangle = New-Object Drawing.Rectangle $position.X, $position.Y, 512, 512 + $tile = $image.Clone($rectangle, [Drawing.Imaging.PixelFormat]::Format24bppRgb) + try { + $partial = "$tilePath.partial" + $tile.Save($partial, [Drawing.Imaging.ImageFormat]::Png) + Move-Item -LiteralPath $partial -Destination $tilePath + } finally { + $tile.Dispose() + } + $tiles.Add([ordered]@{ + id = 'tile-{0:D3}' -f $tileIndex + path = "tiles/$tileName" + sha256 = (Get-FileHash -LiteralPath $tilePath -Algorithm SHA256).Hash.ToLowerInvariant() + x = [int]$position.X + y = [int]$position.Y + width = 512 + height = 512 + }) + } + } finally { + $image.Dispose() + } + $tileManifest = [ordered]@{ + schema = 'pathlab.tile-cache/1' + source = [ordered]@{ + sha256 = $sourceHash + slideRevision = "bracs-roi:$($row.roi_id):$sourceHash" + width = [int]$row.width + height = [int]$row.height + sampleManifest = 'sample.json' + sampleManifestSha256 = (Get-FileHash -LiteralPath $samplePath -Algorithm SHA256).Hash.ToLowerInvariant() + } + tilePixels = 512 + encoding = 'png' + preprocessingInput = 'rgb-srgb-uint8' + tiles = $tiles.ToArray() + } + $tileManifestPath = Join-Path $sampleRoot 'tile-cache.json' + $tileManifest | ConvertTo-Json -Depth 10 | Set-Content -LiteralPath "$tileManifestPath.partial" -Encoding utf8NoBOM + Move-Item -LiteralPath "$tileManifestPath.partial" -Destination $tileManifestPath + $evaluationGroup = if ($row.label -in @('N', 'PB', 'UDH')) { 'benign-reactive' } else { 'breast' } + $cohortSamples.Add([ordered]@{ + id = ([string]$row.roi_id).ToLowerInvariant().Replace('_', '-') + tileCacheManifest = "samples/$($row.roi_id)/tile-cache.json" + tileCacheManifestSha256 = (Get-FileHash -LiteralPath $tileManifestPath -Algorithm SHA256).Hash.ToLowerInvariant() + evaluationGroup = $evaluationGroup + phenotypeGroup = [string]$row.label + split = 'query' + sourceGroup = 'bracs-roi-current-source' + }) + } + $cohort = [ordered]@{ + schema = 'pathlab.qualification-cohort/1' + cohortId = $cohortId + frozenAt = $FrozenAt + intendedUse = 'private-research-model-qualification' + acceptanceCriteria = [ordered]@{ + requiredEvaluationGroups = @('breast', 'gi', 'lung', 'lymph-node', 'benign-reactive') + minimumReferenceSamplesPerGroup = 20 + minimumQuerySamplesPerGroup = 20 + minimumOodSamples = 20 + maximumPatientOverlap = 0 + maximumSlideOverlap = 0 + maximumSourceOverlap = 0 + baselineId = 'color-histogram-v1' + minimumMacroRecallAt5Improvement = 0.05 + minimumMacroNdcgAt10Improvement = 0.03 + minimumOodAuRoc = 0.80 + } + samples = $cohortSamples.ToArray() + } + $cohortPath = Join-Path $partialRoot 'cohort.json' + $cohort | ConvertTo-Json -Depth 12 | Set-Content -LiteralPath "$cohortPath.partial" -Encoding utf8NoBOM + Move-Item -LiteralPath "$cohortPath.partial" -Destination $cohortPath + New-Item -ItemType Directory -Path (Split-Path $outputRoot) -Force | Out-Null + Move-Item -LiteralPath $partialRoot -Destination $outputRoot + Remove-Item -LiteralPath $reservation -Force +} catch { + throw +} + +$finalCohort = Join-Path $outputRoot 'cohort.json' +Write-Output "Built immutable BRACS breast-only smoke cohort: $finalCohort" +Write-Output "SHA256: $((Get-FileHash -LiteralPath $finalCohort -Algorithm SHA256).Hash.ToLowerInvariant())" +Write-Output 'Qualification remains NOT_EVALUABLE: reference, GI, lung, lymph-node and OOD coverage are absent.' diff --git a/src/main/java/org/pathlab/forge/evidence/EvidenceJobProcessor.java b/src/main/java/org/pathlab/forge/evidence/EvidenceJobProcessor.java index 0cc612d..c765a3e 100644 --- a/src/main/java/org/pathlab/forge/evidence/EvidenceJobProcessor.java +++ b/src/main/java/org/pathlab/forge/evidence/EvidenceJobProcessor.java @@ -24,7 +24,7 @@ public final class EvidenceJobProcessor { "schema", "sourcePath", "sourceSha256", "slideRevision", "previewPath", "sourceWidth", "sourceHeight", "packManifest", "stain", "marker"); private static final Set OPTIONAL_REQUEST_FIELDS = Set.of( - "compartmentSource", "controlsValidated"); + "compartmentSource", "controlsValidated", "tileCacheManifest", "tileCacheManifestSha256"); private final EvidenceJobQueue queue; private final Path stateRoot; @@ -62,8 +62,21 @@ public EvidenceJob process(EvidenceJob job, String workerId, Instant now) throws require(pack.capability() != EvidencePackManifest.Capability.IHC_DESCRIPTIVE || pack.markers().contains(marker), "Requested IHC marker is unsupported and generic fallback is unavailable"); - require(positiveInt(request, "sourceWidth") > 0 && positiveInt(request, "sourceHeight") > 0, - "Evidence source geometry is invalid"); + var sourceWidth = positiveInt(request, "sourceWidth"); + var sourceHeight = positiveInt(request, "sourceHeight"); + if (pack.capability() == EvidencePackManifest.Capability.HE_EVIDENCE) { + var tileManifestPath = regularPath(request, "tileCacheManifest"); + var tileManifestSha = text(request, "tileCacheManifestSha256"); + var tileCache = EvidenceTileCacheManifest.load(tileManifestPath, tileManifestSha); + tileCache.requireSource(expectedSha, text(request, "slideRevision"), + sourceWidth, sourceHeight, pack.tilePixels()); + require(tileCache.sample().bytes() == Files.size(source) + && "private-research".equals(tileCache.sample().permittedUse()), + "H&E source provenance or permitted use is invalid"); + } else { + require(!request.has("tileCacheManifest") && !request.has("tileCacheManifestSha256"), + "Only H&E evidence jobs may provide a tile cache"); + } checkCancellation(job.id(), workerId, now); var running = queue.checkpoint(job.id(), workerId, EvidenceJobState.RUNNING, "running", 0.25, "Running offline bounded brightfield analysis", now, LEASE); diff --git a/src/main/java/org/pathlab/forge/evidence/EvidencePackManifest.java b/src/main/java/org/pathlab/forge/evidence/EvidencePackManifest.java index abcb520..af1a7d9 100644 --- a/src/main/java/org/pathlab/forge/evidence/EvidencePackManifest.java +++ b/src/main/java/org/pathlab/forge/evidence/EvidencePackManifest.java @@ -19,6 +19,7 @@ public record EvidencePackManifest( Capability capability, Set acceptedStains, String preprocessingId, + int tilePixels, String allowedUse, boolean redistributable, boolean derivativesAllowed, @@ -54,7 +55,7 @@ public static EvidencePackManifest load(Path manifest) throws IOException { var preprocessing = object(root, "preprocessing"); var preprocessingId = text(preprocessing, "id"); require(preprocessingId.length() <= 120, "AI pack preprocessing id is invalid"); - boundedInt(preprocessing, "tilePixels", 64, 2_048); + var tilePixels = boundedInt(preprocessing, "tilePixels", 64, 2_048); var rights = object(root, "rights"); var allowedUse = text(rights, "allowedUse"); require(Set.of("private-research", "benchmark-only").contains(allowedUse), "AI pack allowed use is invalid"); @@ -81,7 +82,7 @@ public static EvidencePackManifest load(Path manifest) throws IOException { var licenseLedger = root.has("licenseLedger") ? parseLicenseLedger(root.path("licenseLedger")) : List.of(); return new EvidencePackManifest( - normalized, packId, version, capability, acceptedStains, preprocessingId, allowedUse, + normalized, packId, version, capability, acceptedStains, preprocessingId, tilePixels, allowedUse, booleanValue(rights, "redistributable"), booleanValue(rights, "derivativesAllowed"), runtime, licenseLedger, diff --git a/src/main/java/org/pathlab/forge/evidence/EvidenceQualificationCohort.java b/src/main/java/org/pathlab/forge/evidence/EvidenceQualificationCohort.java new file mode 100644 index 0000000..aae4c91 --- /dev/null +++ b/src/main/java/org/pathlab/forge/evidence/EvidenceQualificationCohort.java @@ -0,0 +1,232 @@ +package org.pathlab.forge.evidence; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Instant; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.regex.Pattern; + +/** Frozen patient/source-held-out cohort definition; it reports readiness but never qualifies a model. */ +public record EvidenceQualificationCohort( + Path path, + String cohortId, + Instant frozenAt, + String intendedUse, + AcceptanceCriteria acceptanceCriteria, + List samples) { + public static final String SCHEMA = "pathlab.qualification-cohort/1"; + private static final ObjectMapper JSON = new ObjectMapper(); + private static final Pattern ID = Pattern.compile("[a-z0-9][a-z0-9._-]{0,119}"); + private static final Pattern SHA = Pattern.compile("[a-f0-9]{64}"); + private static final Set GROUPS = Set.of( + "breast", "gi", "lung", "lymph-node", "benign-reactive"); + private static final Set SPLITS = Set.of("reference", "query", "ood"); + private static final Set ROOT_FIELDS = Set.of( + "schema", "cohortId", "frozenAt", "intendedUse", "acceptanceCriteria", "samples"); + private static final Set CRITERIA_FIELDS = Set.of( + "requiredEvaluationGroups", "minimumReferenceSamplesPerGroup", + "minimumQuerySamplesPerGroup", "minimumOodSamples", "maximumPatientOverlap", + "maximumSlideOverlap", "maximumSourceOverlap", "baselineId", + "minimumMacroRecallAt5Improvement", "minimumMacroNdcgAt10Improvement", + "minimumOodAuRoc"); + private static final Set SAMPLE_FIELDS = Set.of( + "id", "tileCacheManifest", "tileCacheManifestSha256", "evaluationGroup", + "phenotypeGroup", "split", "sourceGroup"); + + public static EvidenceQualificationCohort load(Path manifest) throws IOException { + var normalized = manifest.toAbsolutePath().normalize(); + var root = JSON.readTree(normalized.toFile()); + require(root.isObject() && fields(root).equals(ROOT_FIELDS), + "Qualification cohort fields are invalid"); + require(SCHEMA.equals(text(root, "schema", 80)), "Qualification cohort schema is unsupported"); + var cohortId = text(root, "cohortId", 120); + require(ID.matcher(cohortId).matches(), "Qualification cohort id is invalid"); + final Instant frozenAt; + try { + frozenAt = Instant.parse(text(root, "frozenAt", 80)); + } catch (java.time.format.DateTimeParseException error) { + throw new IllegalArgumentException("Qualification cohort freeze timestamp is invalid", error); + } + var intendedUse = text(root, "intendedUse", 80); + require("private-research-model-qualification".equals(intendedUse), + "Qualification cohort intended use is invalid"); + var criteria = parseCriteria(root.path("acceptanceCriteria")); + var nodes = root.path("samples"); + require(nodes.isArray() && !nodes.isEmpty() && nodes.size() <= 20_000, + "Qualification cohort samples are invalid"); + var rootDirectory = normalized.getParent().toRealPath(); + var ids = new HashSet(); + var samples = new ArrayList(); + for (var node : nodes) { + require(node.isObject() && fields(node).equals(SAMPLE_FIELDS), + "Qualification sample fields are invalid"); + var id = text(node, "id", 120); + require(ID.matcher(id).matches() && ids.add(id), + "Qualification sample id is invalid or duplicated"); + var relative = Path.of(text(node, "tileCacheManifest", 500)); + require(!relative.isAbsolute(), "Qualification sample paths must be relative"); + var tileManifest = rootDirectory.resolve(relative).normalize(); + require(tileManifest.startsWith(rootDirectory) && Files.isRegularFile(tileManifest), + "Qualification tile cache escaped its frozen root or is unavailable"); + tileManifest = tileManifest.toRealPath(); + require(tileManifest.startsWith(rootDirectory), "Qualification tile cache escaped its frozen root"); + var tileManifestSha = text(node, "tileCacheManifestSha256", 64); + require(SHA.matcher(tileManifestSha).matches(), "Qualification tile-cache checksum is invalid"); + var tileCache = EvidenceTileCacheManifest.load(tileManifest, tileManifestSha); + var group = text(node, "evaluationGroup", 80); + require(GROUPS.contains(group), "Qualification evaluation group is invalid"); + var phenotype = text(node, "phenotypeGroup", 160); + var split = text(node, "split", 20); + require(SPLITS.contains(split), "Qualification split is invalid"); + var sourceGroup = text(node, "sourceGroup", 160); + samples.add(new Sample(id, tileManifest, tileManifestSha, group, phenotype, + split, sourceGroup, tileCache)); + } + return new EvidenceQualificationCohort(normalized, cohortId, frozenAt, intendedUse, + criteria, List.copyOf(samples)); + } + + public Assessment assessReadiness() { + var reasons = new LinkedHashSet(); + var referenceCounts = counts("reference"); + var queryCounts = counts("query"); + for (var group : acceptanceCriteria.requiredEvaluationGroups()) { + if (referenceCounts.getOrDefault(group, 0) < acceptanceCriteria.minimumReferenceSamplesPerGroup()) { + reasons.add("INSUFFICIENT_REFERENCE_" + wireReason(group)); + } + if (queryCounts.getOrDefault(group, 0) < acceptanceCriteria.minimumQuerySamplesPerGroup()) { + reasons.add("INSUFFICIENT_QUERY_" + wireReason(group)); + } + } + var ood = samples.stream().filter(sample -> "ood".equals(sample.split())).count(); + if (ood < acceptanceCriteria.minimumOodSamples()) reasons.add("INSUFFICIENT_OOD"); + if (overlap(sample -> sample.tileCache().sample().source() + "::" + + sample.tileCache().sample().patientGroup()) > acceptanceCriteria.maximumPatientOverlap()) { + reasons.add("PATIENT_SPLIT_OVERLAP"); + } + if (overlap(sample -> sample.tileCache().sample().source() + "::" + + sample.tileCache().sample().slideId()) > acceptanceCriteria.maximumSlideOverlap()) { + reasons.add("SLIDE_SPLIT_OVERLAP"); + } + if (overlap(Sample::sourceGroup) > acceptanceCriteria.maximumSourceOverlap()) { + reasons.add("SOURCE_SPLIT_OVERLAP"); + } + if (samples.stream().anyMatch(sample -> + !"private-research".equals(sample.tileCache().sample().permittedUse()))) { + reasons.add("UNAPPROVED_SAMPLE_RIGHTS"); + } + return new Assessment(reasons.isEmpty() ? "ready" : "not_evaluable", List.copyOf(reasons)); + } + + private Map counts(String split) { + var result = new HashMap(); + samples.stream().filter(sample -> split.equals(sample.split())).forEach(sample -> + result.merge(sample.evaluationGroup(), 1, Integer::sum)); + return result; + } + + private int overlap(java.util.function.Function key) { + var splitsByKey = new HashMap>(); + for (var sample : samples) { + splitsByKey.computeIfAbsent(key.apply(sample), ignored -> new HashSet<>()).add(sample.split()); + } + return (int) splitsByKey.values().stream().filter(splits -> splits.size() > 1).count(); + } + + private static AcceptanceCriteria parseCriteria(JsonNode node) { + require(node.isObject() && fields(node).equals(CRITERIA_FIELDS), + "Qualification acceptance criteria are invalid"); + var groupsNode = node.path("requiredEvaluationGroups"); + require(groupsNode.isArray(), "Qualification evaluation groups are invalid"); + var groups = new HashSet(); + groupsNode.forEach(value -> { + require(value.isTextual() && GROUPS.contains(value.textValue()) && groups.add(value.textValue()), + "Qualification evaluation groups are invalid or duplicated"); + }); + require(groups.equals(GROUPS), "Qualification cohort must pre-register every MVP evaluation group"); + var baseline = text(node, "baselineId", 120); + require(ID.matcher(baseline).matches(), "Qualification baseline id is invalid"); + var patientOverlap = boundedInt(node, "maximumPatientOverlap", 0, 0); + var slideOverlap = boundedInt(node, "maximumSlideOverlap", 0, 0); + var sourceOverlap = boundedInt(node, "maximumSourceOverlap", 0, 0); + return new AcceptanceCriteria(Set.copyOf(groups), + boundedInt(node, "minimumReferenceSamplesPerGroup", 1, 10_000), + boundedInt(node, "minimumQuerySamplesPerGroup", 1, 10_000), + boundedInt(node, "minimumOodSamples", 1, 10_000), + patientOverlap, slideOverlap, sourceOverlap, baseline, + boundedDouble(node, "minimumMacroRecallAt5Improvement", 0, 1), + boundedDouble(node, "minimumMacroNdcgAt10Improvement", 0, 1), + boundedDouble(node, "minimumOodAuRoc", 0.5, 1)); + } + + private static int boundedInt(JsonNode node, String field, int minimum, int maximum) { + var value = node.path(field); + require(value.isIntegralNumber() && value.canConvertToInt() + && value.intValue() >= minimum && value.intValue() <= maximum, + "Qualification criterion is invalid: " + field); + return value.intValue(); + } + + private static double boundedDouble(JsonNode node, String field, double minimum, double maximum) { + var value = node.path(field); + require(value.isNumber() && Double.isFinite(value.doubleValue()) + && value.doubleValue() >= minimum && value.doubleValue() <= maximum, + "Qualification criterion is invalid: " + field); + return value.doubleValue(); + } + + private static Set fields(JsonNode node) { + var result = new HashSet(); + node.fieldNames().forEachRemaining(result::add); + return result; + } + + private static String text(JsonNode node, String field, int maximum) { + var value = node.path(field); + require(value.isTextual() && !value.textValue().isBlank() && value.textValue().length() <= maximum, + "Qualification field is invalid: " + field); + return value.textValue(); + } + + private static String wireReason(String value) { + return value.toUpperCase(java.util.Locale.ROOT).replace('-', '_'); + } + + private static void require(boolean condition, String message) { + if (!condition) throw new IllegalArgumentException(message); + } + + public record AcceptanceCriteria( + Set requiredEvaluationGroups, + int minimumReferenceSamplesPerGroup, + int minimumQuerySamplesPerGroup, + int minimumOodSamples, + int maximumPatientOverlap, + int maximumSlideOverlap, + int maximumSourceOverlap, + String baselineId, + double minimumMacroRecallAt5Improvement, + double minimumMacroNdcgAt10Improvement, + double minimumOodAuRoc) {} + + public record Sample( + String id, + Path tileCacheManifest, + String tileCacheManifestSha256, + String evaluationGroup, + String phenotypeGroup, + String split, + String sourceGroup, + EvidenceTileCacheManifest tileCache) {} + + public record Assessment(String status, List reasons) {} +} diff --git a/src/main/java/org/pathlab/forge/evidence/EvidenceTileCacheManifest.java b/src/main/java/org/pathlab/forge/evidence/EvidenceTileCacheManifest.java new file mode 100644 index 0000000..a7b8fad --- /dev/null +++ b/src/main/java/org/pathlab/forge/evidence/EvidenceTileCacheManifest.java @@ -0,0 +1,184 @@ +package org.pathlab.forge.evidence; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.awt.image.BufferedImage; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.HexFormat; +import java.util.List; +import java.util.Set; +import java.util.regex.Pattern; +import javax.imageio.ImageIO; + +/** Immutable, coordinate-bound RGB input tiles for one exact source revision. */ +public record EvidenceTileCacheManifest( + Path path, + String sha256, + String sourceSha256, + String slideRevision, + int sourceWidth, + int sourceHeight, + Path sampleManifestPath, + String sampleManifestSha256, + int tilePixels, + String encoding, + String preprocessingInput, + EvidenceSampleManifest sample, + List tiles) { + public static final String SCHEMA = "pathlab.tile-cache/1"; + private static final ObjectMapper JSON = new ObjectMapper(); + private static final Pattern SHA = Pattern.compile("[a-f0-9]{64}"); + private static final Pattern ID = Pattern.compile("[A-Za-z0-9._-]{1,120}"); + private static final Set ROOT_FIELDS = Set.of( + "schema", "source", "tilePixels", "encoding", "preprocessingInput", "tiles"); + private static final Set SOURCE_FIELDS = Set.of( + "sha256", "slideRevision", "width", "height", "sampleManifest", "sampleManifestSha256"); + private static final Set TILE_FIELDS = Set.of( + "id", "path", "sha256", "x", "y", "width", "height"); + + public static EvidenceTileCacheManifest load(Path manifest, String expectedSha256) throws IOException { + var normalized = manifest.toAbsolutePath().normalize(); + require(Files.isRegularFile(normalized), "Tile-cache manifest is unavailable"); + var bytes = Files.readAllBytes(normalized); + var actualManifestSha = sha256(bytes); + require(SHA.matcher(expectedSha256).matches() && expectedSha256.equals(actualManifestSha), + "Tile-cache manifest checksum does not match"); + var root = JSON.readTree(bytes); + require(root.isObject() && fields(root).equals(ROOT_FIELDS), "Tile-cache fields are invalid"); + require(SCHEMA.equals(text(root, "schema", 80)), "Tile-cache schema is unsupported"); + var source = root.path("source"); + require(source.isObject() && fields(source).equals(SOURCE_FIELDS), "Tile-cache source is invalid"); + var sourceSha = text(source, "sha256", 64); + require(SHA.matcher(sourceSha).matches(), "Tile-cache source checksum is invalid"); + var revision = text(source, "slideRevision", 200); + var sourceWidth = positiveInt(source, "width"); + var sourceHeight = positiveInt(source, "height"); + var tilePixels = positiveInt(root, "tilePixels"); + require(tilePixels == 512, "Evidence tile cache must contain 512-pixel tiles"); + var encoding = text(root, "encoding", 20); + require("png".equals(encoding), "Evidence tile cache must use deterministic PNG tiles"); + var preprocessingInput = text(root, "preprocessingInput", 80); + require("rgb-srgb-uint8".equals(preprocessingInput), "Evidence tile input encoding is unsupported"); + + var rootDirectory = normalized.getParent().toRealPath(); + var samplePath = resolveEntry(rootDirectory, text(source, "sampleManifest", 500)); + var expectedSampleSha = text(source, "sampleManifestSha256", 64); + require(SHA.matcher(expectedSampleSha).matches() && expectedSampleSha.equals(sha256(samplePath)), + "Evidence sample manifest checksum does not match"); + var sample = EvidenceSampleManifest.load(samplePath); + require(sourceSha.equals(sample.sha256()), "Tile-cache and sample source checksums differ"); + + var tileNodes = root.path("tiles"); + require(tileNodes.isArray() && !tileNodes.isEmpty() && tileNodes.size() <= 512, + "Evidence tile list is invalid"); + var ids = new HashSet(); + var coordinates = new HashSet(); + var tiles = new ArrayList(); + for (var node : tileNodes) { + require(node.isObject() && fields(node).equals(TILE_FIELDS), "Evidence tile fields are invalid"); + var id = text(node, "id", 120); + require(ID.matcher(id).matches() && ids.add(id), "Evidence tile id is invalid or duplicated"); + var path = resolveEntry(rootDirectory, text(node, "path", 500)); + var expectedTileSha = text(node, "sha256", 64); + require(SHA.matcher(expectedTileSha).matches() && expectedTileSha.equals(sha256(path)), + "Evidence tile checksum does not match"); + var x = nonnegativeInt(node, "x"); + var y = nonnegativeInt(node, "y"); + var width = positiveInt(node, "width"); + var height = positiveInt(node, "height"); + require(width == tilePixels && height == tilePixels + && (long) x + width <= sourceWidth && (long) y + height <= sourceHeight + && coordinates.add(x + ":" + y), + "Evidence tile coordinates are invalid or duplicated"); + BufferedImage image; + try { + image = ImageIO.read(path.toFile()); + } catch (javax.imageio.IIOException invalidImage) { + throw new IllegalArgumentException("Evidence tile could not be decoded", invalidImage); + } + require(image != null && image.getWidth() == tilePixels && image.getHeight() == tilePixels + && image.getColorModel().getNumColorComponents() == 3, + "Evidence tile pixels are invalid"); + tiles.add(new Tile(id, path, expectedTileSha, x, y, width, height)); + } + return new EvidenceTileCacheManifest(normalized, actualManifestSha, sourceSha, revision, + sourceWidth, sourceHeight, samplePath, expectedSampleSha, tilePixels, encoding, + preprocessingInput, sample, List.copyOf(tiles)); + } + + public void requireSource(String sha, String revision, int width, int height, int requiredTilePixels) { + require(sourceSha256.equals(sha) && slideRevision.equals(revision) + && sourceWidth == width && sourceHeight == height && tilePixels == requiredTilePixels, + "Tile cache is stale or belongs to another source revision"); + } + + private static Path resolveEntry(Path root, String relative) throws IOException { + var candidate = Path.of(relative); + require(!candidate.isAbsolute(), "Tile-cache entries must use relative paths"); + var resolved = root.resolve(candidate).normalize(); + require(resolved.startsWith(root) && Files.isRegularFile(resolved), + "Tile-cache entry escaped its immutable root or is unavailable"); + var real = resolved.toRealPath(); + require(real.startsWith(root), "Tile-cache entry escaped its immutable root"); + return real; + } + + private static Set fields(JsonNode node) { + var result = new HashSet(); + node.fieldNames().forEachRemaining(result::add); + return result; + } + + private static int positiveInt(JsonNode node, String field) { + var value = node.path(field); + require(value.isIntegralNumber() && value.canConvertToInt() && value.intValue() > 0, + "Evidence tile geometry is invalid"); + return value.intValue(); + } + + private static int nonnegativeInt(JsonNode node, String field) { + var value = node.path(field); + require(value.isIntegralNumber() && value.canConvertToInt() && value.intValue() >= 0, + "Evidence tile coordinates are invalid"); + return value.intValue(); + } + + private static String text(JsonNode node, String field, int maximum) { + var value = node.path(field); + require(value.isTextual() && !value.textValue().isBlank() && value.textValue().length() <= maximum, + "Evidence tile field is invalid: " + field); + return value.textValue(); + } + + private static String sha256(Path path) throws IOException { + try (InputStream input = Files.newInputStream(path)) { + var digest = MessageDigest.getInstance("SHA-256"); + var buffer = new byte[1024 * 1024]; + int read; + while ((read = input.read(buffer)) >= 0) digest.update(buffer, 0, read); + return HexFormat.of().formatHex(digest.digest()); + } catch (java.security.GeneralSecurityException impossible) { + throw new IOException("SHA-256 is unavailable", impossible); + } + } + + private static String sha256(byte[] bytes) throws IOException { + try { + return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(bytes)); + } catch (java.security.GeneralSecurityException impossible) { + throw new IOException("SHA-256 is unavailable", impossible); + } + } + + private static void require(boolean condition, String message) { + if (!condition) throw new IllegalArgumentException(message); + } + + public record Tile(String id, Path path, String sha256, int x, int y, int width, int height) {} +} diff --git a/src/main/resources/evidence-packs/he-dinov2-small-v1.json b/src/main/resources/evidence-packs/he-dinov2-small-v1.json index c8d6308..89a52df 100644 --- a/src/main/resources/evidence-packs/he-dinov2-small-v1.json +++ b/src/main/resources/evidence-packs/he-dinov2-small-v1.json @@ -28,12 +28,16 @@ "source": "pathlab-local-build:dinov2-worker-v1/windows-x64" }, { "name": "workerSource", - "sha256": "4884827fea5f6b04ecd2ec4b5fd31800e2a6d68547f7142aea506a70dd42fd8f", + "sha256": "b80811cc8026152c45248cdcf8123413f9b3f8d64e297bad93dfe198b5aba5c2", "source": "pathlab-local-build:dinov2-worker-v1/worker.py" }, { "name": "runtime-manifest", "sha256": "1969c91db66d8acc70b6c8648a57d3bfc923880e03f1f7f1e30c593133c01635", "source": "pathlab-local-build:torch-2.7.1-cu126-python-3.12/windows-x64" + }, { + "name": "qualification-protocol", + "sha256": "d5a46da7a2246afc71fec0bf18989903e8442ed255647b9f044b0538c812df72", + "source": "repo:docs/evidence/he-retrieval-qualification-protocol-v1.md" }], "runtimeCompatibility": { "workerProtocol": "pathlab.model-worker-result/1", @@ -66,7 +70,7 @@ "validation": { "status": "not-evaluable", "modelCard": "docs/model-cards/he-dinov2-small-v1.md", - "heldOutEvaluation": "not-evaluable-held-out-retrieval-pending" + "heldOutEvaluation": "not-evaluable-cross-tissue-cohort-incomplete" }, "outputSchema": "pathlab.ai-evidence/1" } diff --git a/src/test/java/org/pathlab/forge/evidence/EvidenceJobProcessorTest.java b/src/test/java/org/pathlab/forge/evidence/EvidenceJobProcessorTest.java index 222498c..ebd3623 100644 --- a/src/test/java/org/pathlab/forge/evidence/EvidenceJobProcessorTest.java +++ b/src/test/java/org/pathlab/forge/evidence/EvidenceJobProcessorTest.java @@ -88,6 +88,38 @@ void signsQcEvidenceAndAbstainsForBlankPreview() throws Exception { } } + @Test + void rejectsHeEvidenceWithoutChecksumBoundTileCache() throws Exception { + var source = temporaryDirectory.resolve("he-source.bin"); + Files.writeString(source, "immutable H&E source bytes"); + var preview = temporaryDirectory.resolve("he-preview.png"); + var image = new BufferedImage(16, 12, BufferedImage.TYPE_INT_RGB); + ImageIO.write(image, "png", preview.toFile()); + var pack = temporaryDirectory.resolve("he-pack.json"); + Files.writeString(pack, hePackJson()); + var request = temporaryDirectory.resolve("he-request.json"); + JSON.writeValue(request.toFile(), java.util.Map.of( + "schema", "pathlab.evidence-job/1", + "sourcePath", source.toString(), + "sourceSha256", sha256(source), + "slideRevision", "revision-he", + "previewPath", preview.toString(), + "sourceWidth", 1600, + "sourceHeight", 1200, + "packManifest", pack.toString(), + "stain", "he", + "marker", "generic")); + var now = Instant.parse("2026-08-22T00:00:00Z"); + + try (var queue = new EvidenceJobQueue(temporaryDirectory.resolve("he-state/jobs.sqlite3"))) { + queue.submit("job-he-no-tiles", request, now); + var claimed = queue.claimNext("worker-1", now, Duration.ofMinutes(2)).orElseThrow(); + assertThrows(IllegalArgumentException.class, () -> + new EvidenceJobProcessor(queue, temporaryDirectory.resolve("he-state")) + .process(claimed, "worker-1", now.plusSeconds(1))); + } + } + @Test void completesOfflineDescriptiveIhcJobIntoSignedAtomicEvidence() throws Exception { var source = temporaryDirectory.resolve("source.bin"); @@ -153,6 +185,19 @@ private static String packJson() { "\"outputSchema\":\"pathlab.ai-evidence/1\",\"markers\":[\"generic\",\"ki-67\"]}"; } + private static String hePackJson() { + return "{\"schema\":\"pathlab.ai-pack/1\",\"packId\":\"he-fixture-v1\",\"version\":\"1\"," + + "\"capability\":\"he-evidence\",\"acceptedStains\":[\"he\"]," + + "\"preprocessing\":{\"id\":\"he-fixture-v1\",\"tilePixels\":512},\"artifacts\":[]," + + "\"rights\":{\"license\":\"internal\",\"allowedUse\":\"private-research\"," + + "\"redistributable\":false,\"derivativesAllowed\":false," + + "\"reviewedAt\":\"2026-08-22T00:00:00Z\"}," + + "\"resourceEnvelope\":{\"maxRamMiB\":1024,\"maxVramMiB\":0," + + "\"maxSeconds\":300,\"network\":false}," + + "\"validation\":{\"status\":\"experimental\",\"modelCard\":\"model.md\"," + + "\"heldOutEvaluation\":\"fixture.json\"},\"outputSchema\":\"pathlab.ai-evidence/1\"}"; + } + private Path request(Path source, Path preview, Path pack, String revision) throws Exception { var request = temporaryDirectory.resolve(revision + "-request.json"); JSON.writeValue(request.toFile(), java.util.Map.of( diff --git a/src/test/java/org/pathlab/forge/evidence/EvidencePackManifestTest.java b/src/test/java/org/pathlab/forge/evidence/EvidencePackManifestTest.java index adf8b4a..a73d7c0 100644 --- a/src/test/java/org/pathlab/forge/evidence/EvidencePackManifestTest.java +++ b/src/test/java/org/pathlab/forge/evidence/EvidencePackManifestTest.java @@ -50,7 +50,7 @@ void bundledPilotCatalogActivatesOnlyInstalledLawfulBaselines() throws Exception assertFalse(dino.pilotEligible()); assertEquals("ed25f3a31f01632728cabb09d1542f84ab7b0056", dino.licenseLedger().get(0).revision()); - assertEquals(6, dino.artifacts().size()); + assertEquals(7, dino.artifacts().size()); assertEquals("ae1e99fcefd534ed978cdeb8326f08030c96e28b7a81ffcbc98a857c84d14be1", dino.artifacts().get(0).sha256()); assertEquals("daa33e61d2607e23ecae29e82f274ca05c64f896f9c37061c6ca77ca81aa774b", diff --git a/src/test/java/org/pathlab/forge/evidence/EvidenceQualificationCohortTest.java b/src/test/java/org/pathlab/forge/evidence/EvidenceQualificationCohortTest.java new file mode 100644 index 0000000..f3f5040 --- /dev/null +++ b/src/test/java/org/pathlab/forge/evidence/EvidenceQualificationCohortTest.java @@ -0,0 +1,68 @@ +package org.pathlab.forge.evidence; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.file.Path; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import org.junit.jupiter.api.Test; + +final class EvidenceQualificationCohortTest { + private static final Set GROUPS = Set.of( + "breast", "gi", "lung", "lymph-node", "benign-reactive"); + + @Test + void reportsMissingGroupsAndSplitLeakageAsNotEvaluable() { + var samples = List.of( + sample("reference", "breast", "patient-1", "slide-1", "source-a"), + sample("query", "breast", "patient-1", "slide-2", "source-a"), + sample("ood", "breast", "patient-3", "slide-3", "source-c")); + var cohort = cohort(samples); + + var assessment = cohort.assessReadiness(); + + assertEquals("not_evaluable", assessment.status()); + assertTrue(assessment.reasons().contains("INSUFFICIENT_QUERY_GI")); + assertTrue(assessment.reasons().contains("PATIENT_SPLIT_OVERLAP")); + assertTrue(assessment.reasons().contains("SOURCE_SPLIT_OVERLAP")); + } + + @Test + void becomesReadyOnlyWithEveryPreregisteredGroupAndNoLeakage() { + var samples = new ArrayList(); + var index = 0; + for (var group : GROUPS.stream().sorted().toList()) { + samples.add(sample("reference", group, "patient-r-" + index, + "slide-r-" + index, "source-r-" + index)); + samples.add(sample("query", group, "patient-q-" + index, + "slide-q-" + index, "source-q-" + index)); + index++; + } + samples.add(sample("ood", "breast", "patient-o", "slide-o", "source-o")); + + assertEquals("ready", cohort(samples).assessReadiness().status()); + } + + private static EvidenceQualificationCohort cohort(List samples) { + return new EvidenceQualificationCohort(Path.of("cohort.json"), "cohort-v1", + Instant.parse("2026-08-22T00:00:00Z"), "private-research-model-qualification", + new EvidenceQualificationCohort.AcceptanceCriteria( + GROUPS, 1, 1, 1, 0, 0, 0, "color-histogram-v1", 0.05, 0.03, 0.8), + samples); + } + + private static EvidenceQualificationCohort.Sample sample( + String split, String group, String patient, String slide, String sourceGroup) { + var source = "dataset-" + sourceGroup; + var provenance = new EvidenceSampleManifest(source, patient, slide, "a".repeat(64), + "reviewed research terms", "research-group", "private-research", 1, false); + var cache = new EvidenceTileCacheManifest(Path.of("tile-cache.json"), "b".repeat(64), + "a".repeat(64), "revision-1", 512, 512, Path.of("sample.json"), + "c".repeat(64), 512, "png", "rgb-srgb-uint8", provenance, List.of()); + return new EvidenceQualificationCohort.Sample(split + "-" + group, Path.of("tile-cache.json"), + "b".repeat(64), group, "research-group", split, sourceGroup, cache); + } +} diff --git a/src/test/java/org/pathlab/forge/evidence/EvidenceRealCohortIntegrationTest.java b/src/test/java/org/pathlab/forge/evidence/EvidenceRealCohortIntegrationTest.java new file mode 100644 index 0000000..8c32754 --- /dev/null +++ b/src/test/java/org/pathlab/forge/evidence/EvidenceRealCohortIntegrationTest.java @@ -0,0 +1,25 @@ +package org.pathlab.forge.evidence; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.file.Path; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.Test; + +final class EvidenceRealCohortIntegrationTest { + @Test + void validatesExplicitPrivateCohortWithoutPromotingMissingCoverage() throws Exception { + var value = System.getenv("PATHLAB_EVIDENCE_COHORT"); + Assumptions.assumeTrue(value != null && !value.isBlank(), + "Set PATHLAB_EVIDENCE_COHORT to run the private real-data contract check"); + + var cohort = EvidenceQualificationCohort.load(Path.of(value)); + var assessment = cohort.assessReadiness(); + + assertEquals("not_evaluable", assessment.status()); + assertTrue(assessment.reasons().contains("INSUFFICIENT_QUERY_GI")); + assertTrue(assessment.reasons().contains("INSUFFICIENT_REFERENCE_BREAST")); + assertTrue(assessment.reasons().contains("INSUFFICIENT_OOD")); + } +} diff --git a/src/test/java/org/pathlab/forge/evidence/EvidenceTileCacheManifestTest.java b/src/test/java/org/pathlab/forge/evidence/EvidenceTileCacheManifestTest.java new file mode 100644 index 0000000..e54ee94 --- /dev/null +++ b/src/test/java/org/pathlab/forge/evidence/EvidenceTileCacheManifestTest.java @@ -0,0 +1,101 @@ +package org.pathlab.forge.evidence; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import com.fasterxml.jackson.databind.ObjectMapper; +import java.awt.Color; +import java.awt.image.BufferedImage; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.util.HexFormat; +import javax.imageio.ImageIO; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +final class EvidenceTileCacheManifestTest { + private static final ObjectMapper JSON = new ObjectMapper(); + @TempDir Path temporaryDirectory; + + @Test + void validatesEveryCoordinatePixelAndProvenanceChecksum() throws Exception { + var fixture = fixture(); + var cache = EvidenceTileCacheManifest.load(fixture.manifest(), sha256(fixture.manifest())); + + assertEquals(512, cache.tilePixels()); + assertEquals(1, cache.tiles().size()); + cache.requireSource(sha256(fixture.source()), "revision-1", 1024, 768, 512); + } + + @Test + void rejectsTamperedTileAndStaleRevision() throws Exception { + var tampered = fixture(); + Files.writeString(tampered.tile(), "tampered"); + assertThrows(IllegalArgumentException.class, () -> + EvidenceTileCacheManifest.load(tampered.manifest(), sha256(tampered.manifest()))); + + var stale = fixture(); + var loaded = EvidenceTileCacheManifest.load(stale.manifest(), sha256(stale.manifest())); + assertThrows(IllegalArgumentException.class, () -> + loaded.requireSource(sha256(stale.source()), "revision-2", 1024, 768, 512)); + } + + @Test + void rejectsPathTraversalEvenWhenOutsideTileChecksumMatches() throws Exception { + var fixture = fixture(); + var outside = temporaryDirectory.resolve("outside.png"); + Files.copy(fixture.tile(), outside); + var root = JSON.readTree(fixture.manifest().toFile()); + var tile = (com.fasterxml.jackson.databind.node.ObjectNode) root.path("tiles").get(0); + tile.put("path", "../outside.png"); + tile.put("sha256", sha256(outside)); + JSON.writeValue(fixture.manifest().toFile(), root); + + assertThrows(IllegalArgumentException.class, () -> + EvidenceTileCacheManifest.load(fixture.manifest(), sha256(fixture.manifest()))); + } + + private Fixture fixture() throws Exception { + var root = temporaryDirectory.resolve(java.util.UUID.randomUUID().toString()); + Files.createDirectories(root.resolve("tiles")); + var source = root.resolve("source.png"); + var tile = root.resolve("tiles/tile-1.png"); + writeImage(source, 1024, 768); + writeImage(tile, 512, 512); + var sample = root.resolve("sample.json"); + Files.writeString(sample, "{\"schema\":\"pathlab.evidence-sample/1\"," + + "\"source\":\"fixture\",\"patientGroup\":\"patient-1\"," + + "\"slideId\":\"slide-1\",\"sha256\":\"" + sha256(source) + "\"," + + "\"license\":\"private fixture\",\"taskLabel\":\"breast-benign\"," + + "\"permittedUse\":\"private-research\",\"bytes\":" + Files.size(source) + "," + + "\"grandfatheredReadOnly\":false}"); + var manifest = root.resolve("tile-cache.json"); + Files.writeString(manifest, "{\"schema\":\"pathlab.tile-cache/1\",\"source\":{" + + "\"sha256\":\"" + sha256(source) + "\",\"slideRevision\":\"revision-1\"," + + "\"width\":1024,\"height\":768,\"sampleManifest\":\"sample.json\"," + + "\"sampleManifestSha256\":\"" + sha256(sample) + "\"}," + + "\"tilePixels\":512,\"encoding\":\"png\"," + + "\"preprocessingInput\":\"rgb-srgb-uint8\",\"tiles\":[{" + + "\"id\":\"tile-1\",\"path\":\"tiles/tile-1.png\"," + + "\"sha256\":\"" + sha256(tile) + "\",\"x\":0,\"y\":0," + + "\"width\":512,\"height\":512}]}"); + return new Fixture(source, tile, manifest); + } + + private static void writeImage(Path path, int width, int height) throws Exception { + var image = new BufferedImage(width, height, BufferedImage.TYPE_INT_RGB); + var graphics = image.createGraphics(); + graphics.setColor(new Color(174, 91, 145)); + graphics.fillRect(0, 0, width, height); + graphics.dispose(); + ImageIO.write(image, "png", path.toFile()); + } + + private static String sha256(Path path) throws Exception { + return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256") + .digest(Files.readAllBytes(path))); + } + + private record Fixture(Path source, Path tile, Path manifest) {} +} From 787dd9d1bff77414609e15e7f86f68cdc9563d38 Mon Sep 17 00:00:00 2001 From: Enksodsoon <229880765+Enksodsoon@users.noreply.github.com> Date: Sat, 22 Aug 2026 11:44:33 +0700 Subject: [PATCH 07/50] feat(evidence): add autonomous service dashboard --- docs/evidence-mentor-runner.md | 94 ++-- docs/plans/active/current.md | 63 ++- frontend/src/App.tsx | 22 +- frontend/src/api.ts | 12 +- frontend/src/test/App.test.tsx | 5 +- scripts/evidence-mentor-service.ps1 | 201 ++++++++ scripts/evidence-mentor-task.ps1 | 69 +-- scripts/open-evidence-dashboard.ps1 | 18 + scripts/submit-evidence-job.ps1 | 24 +- .../forge/evidence/DashboardSessions.java | 52 ++ .../forge/evidence/EvidenceExecutionLane.java | 13 + .../forge/evidence/EvidenceJobProcessor.java | 49 +- .../forge/evidence/EvidenceJobQueue.java | 461 ++++++++++++------ .../forge/evidence/EvidenceJobSnapshot.java | 15 + .../forge/evidence/EvidenceMentorRunner.java | 411 +++++++++------- .../forge/evidence/ExternalModelWorker.java | 95 +++- .../org/pathlab/forge/server/ForgeServer.java | 52 +- .../resources/evidence-dashboard/index.html | 27 + .../forge/evidence/DashboardSessionsTest.java | 16 + .../forge/evidence/EvidenceJobQueueTest.java | 61 +++ .../evidence/EvidenceMentorRunnerTest.java | 59 +++ 21 files changed, 1307 insertions(+), 512 deletions(-) create mode 100644 scripts/evidence-mentor-service.ps1 create mode 100644 scripts/open-evidence-dashboard.ps1 create mode 100644 src/main/java/org/pathlab/forge/evidence/DashboardSessions.java create mode 100644 src/main/java/org/pathlab/forge/evidence/EvidenceExecutionLane.java create mode 100644 src/main/java/org/pathlab/forge/evidence/EvidenceJobSnapshot.java create mode 100644 src/main/resources/evidence-dashboard/index.html create mode 100644 src/test/java/org/pathlab/forge/evidence/DashboardSessionsTest.java diff --git a/docs/evidence-mentor-runner.md b/docs/evidence-mentor-runner.md index 25a2d99..4d0bf46 100644 --- a/docs/evidence-mentor-runner.md +++ b/docs/evidence-mentor-runner.md @@ -1,63 +1,65 @@ -# Evidence Mentor runner +# Evidence Mentor autonomous service -`EvidenceMentorRunner` is a standalone per-user process. It binds only to -`127.0.0.1`, requires a 256-bit bearer token from `ipc-token`, and owns the -SQLite WAL queue, leases, checkpoints, signing key, and result artifacts. +`EvidenceMentorRunner` is an offline Windows-service workload. It binds only to +`127.0.0.1` on a Windows-assigned port, atomically publishes +`state/endpoint.json`, and authenticates API clients with the existing 256-bit +token in `state/ipc-token`. The endpoint file never contains the token. -Install from a versioned Forge distribution: +The SQLite WAL queue migrates v1 records in place. It persists the execution +lane, monotonic work units, verified checkpoint identity, lease heartbeat, +rolling throughput, ETA, retry schedule, stable failure classification, and +request/pack/final-artifact checksums. Exactly one GPU job and one low-priority +CPU/I/O job may execute. Live external workers renew their lease no less often +than every 10 seconds; only transient I/O failures receive 5-second, 30-second, +and 2-minute retries. -```powershell -.\scripts\evidence-mentor-task.ps1 -RuntimeRoot 'C:\Path\To\pathlab-forge' -``` +## Dashboard -Default state is `D:\PathLabData\EvidenceMentor\state` when the data drive is -present, otherwise `%LOCALAPPDATA%\PathLab\EvidenceMentor\state`. Analysis is -offline. Acquisition is a separate allowlisted operation and must reserve quota before download or -extraction. `GET /v1/status` returns authenticated, non-outcome runtime and quota telemetry. A -dashboard failure does not stop processing. +Forge or the operator launcher uses the bearer token to request a single-use +60-second code and opens `/dashboard/#`. The dashboard exchanges it once +for an HttpOnly, SameSite=Strict boot-scoped cookie, removes the fragment, and +uses a per-session CSRF token for mutations. It polls with ETags every two +seconds while active and ten seconds while idle. -The deterministic cell/IHC baseline is runnable. The checksum-pinned DINOv2-small worker is locally -executable on the P2000 but remains `not-evaluable` because its cross-tissue held-out cohort is -incomplete. Hibou-B and HoVer-Net remain `not-evaluable` until exact, rights-approved workers and -model artifacts are installed beneath `state/models//`. External workers use -`pathlab.model-worker-result/1`, inherit the -16 GB RAM and 4.5 GB VRAM envelope, receive offline environment controls, and are rejected on the -P2000 unless they declare CUDA 12.6 and `sm_61`. Forge signs only bounded regions and descriptors; -embeddings, raw pixels and clinical outputs are rejected. +The dependency-free page exposes operational metadata only. It can pause or +resume new claims, request checkpoint-safe cancellation, and retry failed or +cancelled work after immutable-request verification. It cannot delete jobs, +change priority, activate packs, approve evidence, or publish results. Dashboard +and telemetry failures never alter processing state. -Acquire only the pinned DINOv2 safetensors baseline (never the pickle artifact) with: +## Service installation + +First build a versioned distribution: ```powershell -.\scripts\acquire-dinov2-small.ps1 +.\gradlew.bat installDist ``` -The acquisition command reserves the fixed model quota, downloads from the exact upstream commit, -verifies byte counts and SHA-256 values before atomic installation, and records a local receipt. It -does not activate the pack. Worker installation, runtime qualification, model qualification, and -activation remain separate gates. - -H&E jobs additionally require a `pathlab.tile-cache/1` manifest and its SHA-256 in -`tileCacheManifest` and `tileCacheManifestSha256`. Forge verifies the source revision, source/sample -provenance, every 512 x 512 RGB PNG tile, coordinates, and checksums before starting the external -worker. Preview-only H&E model inference is rejected. A bounded, breast-only BRACS contract smoke can -be built idempotently with: +Then run the administrator-approved installer with a Java 17 runtime: ```powershell -.\scripts\build-bracs-dinov2-tile-cache.ps1 +.\scripts\evidence-mentor-service.ps1 -Action Install -Version '2.0.0' ` + -DistributionPath '.\build\install\pathlab-forge' -JavaHome 'C:\Path\To\jdk-17' ``` -That cache is real research data but not a qualifying cohort: it lacks the frozen reference, GI, -lung, lymph-node, independent benign/reactive-source, and OOD groups. +The installer downloads WinSW 2.12.0 from its official release URL and requires +SHA-256 `05b82d46ad331cc16bdc00de5c6332c1ef818df8ceefcd49c726553209b3a0da`. +It installs `PathLabEvidenceMentor` as delayed-auto-start under +`NT AUTHORITY\LocalService`, stages runtimes side-by-side under +`C:\ProgramData\PathLab\EvidenceMentor\runtime\`, keeps state under +`D:\PathLabData\EvidenceMentor\state`, applies service-SID ACLs and outbound-deny +rules, and rolls back the active configuration if health checks fail. -New material uses fixed hard buckets: 45 GB source, 25 GB derived, 10 GB models, 10 GB evidence/test, -and a 10 GB untouchable reserve. Existing BRACS remains grandfathered read-only and is not copied. +Use `-Action Upgrade` with a new immutable version. `-Action Uninstall` removes +the service, firewall rules, and Start Menu shortcut while preserving queues, +checkpoints, signing material, evidence, and model packs. -Atlas-H&E remains a Training Lab research track. Its four-hour probe may continue locally only when -resource limits hold, no OOM occurs, validation improves, and projected completion is within seven -days. This runner performs no cloud provisioning or spending. +## Model boundary -Rollback removes the scheduled task but preserves queue, evidence, and keys: - -```powershell -.\scripts\evidence-mentor-task.ps1 -RuntimeRoot 'C:\Path\To\pathlab-forge' -Uninstall -``` +The deterministic cell/IHC baseline is runnable. DINOv2-small remains an +experimental executable baseline until its cross-tissue held-out cohort is +complete. Hibou-B and HoVer-Net remain `not_evaluable` until exact, +rights-approved workers and artifacts pass qualification. H&E, cells, IHC, +special stains, cytology, and later Atlas tracks stay independently signed and +fail closed; none may emit diagnosis, clinical categories, TPS/CPS, prognosis, +or treatment guidance. diff --git a/docs/plans/active/current.md b/docs/plans/active/current.md index 3ec5b45..23a9972 100644 --- a/docs/plans/active/current.md +++ b/docs/plans/active/current.md @@ -1,38 +1,33 @@ -# Active Task — Modular Capability Release +# Active Task — Autonomous Evidence Mentor Service -Supersedes the completed direct-OME release-candidate brief by explicit -product-owner direction on 2026-08-11. +Supersedes the modular-capability stop rule by explicit product-owner direction +on 2026-08-22. The research, licensing, privacy, and fail-closed boundaries remain. ## Fixed endpoint -Deliver a lightweight Forge base plus a signed, offline-by-default Feature -Center. The base owns WSI inspection, direct preview, annotations, geometry -measurements, conversion, validation and upload. Optional Pathology Tools and -Classical Analysis packs are downloaded only after an explicit user action. - -AI is not a core feature. Forge bundles no weights or training runtime. At most -one separately downloaded pretrained research model may be published after -license, provenance, resource and deterministic self-test gates pass. Training -Lab is a separate advanced pack using user-provided labelled data. - -## Milestones - -1. Remove redundant preview work and finish adaptive runtime limits. -2. Add signed catalog, staged installation, self-test, rollback and uninstall. -3. Add bounded raw-region, PathObject, measurement and analysis-run contracts. -4. Add the Pathology Tools pack boundary: hierarchy, H&E, stains and TMA. -5. Add the Classical Analysis pack boundary: tissue/cells/classifier/QC/registration. -6. Add the optional pretrained-AI and Training Lab availability gates. -7. Add explicit private Viewer synchronization with conflict-safe verification. - -## Stop rule - -Stop after these seven milestones and their local validation report. Do not add -third-party plugins, arbitrary scripting, a dependency solver, cloud services, -microservices, distributed workers, additional models, merge or deployment. - -New conversions use only the measured `ome-dynamic-v1` direct path. The governed -same-slide release-candidate comparison found it 46.67% faster and 72.96% smaller -than prepared-v2. Existing prepared artifacts remain readable, but Forge does not -select or retry into that larger route. Image-quality gates, artifact reuse, -privacy and resumable upload remain regression contracts. +Forge submits immutable local-analysis requests to the independently running +`PathLabEvidenceMentor` Windows service. The service owns the WAL queue, +checkpoint recovery, signed evidence packaging, and an operations-only loopback +dashboard. Forge, the dashboard, and Codex may close after a `202 Accepted` +response without interrupting work. + +The service uses a dynamic loopback port published through +`pathlab.runner-endpoint/1`, boot-scoped browser sessions, one GPU lane and one +low-priority CPU/I/O lane. Analysis is network-disabled. No raw pixels, +embeddings, patient identifiers, provisional regions, or scientific outcomes +appear in monitoring payloads. + +## Delivery gates + +1. Queue v2 migration, progress sidecars, renewable leases, retry classes, and pause control. +2. Dynamic endpoint discovery and authenticated standalone dashboard. +3. Forge compact status and one-time dashboard launcher. +4. Checksum-pinned WinSW packaging, ACLs, firewall policy, side-by-side rollback. +5. Actual LocalService, reboot, Session 0 P2000, offline, and recovery acceptance. +6. Independent expert-pack qualification for H&E, cells, IHC, special stains, and cytology. +7. Dual-lineage Atlas feasibility only after lawful frozen teachers exist. + +Implementation, service installation, model qualification, merge, deployment, +feature activation, and any future clinical/capacity study are separate gates. +Feature flags remain default-off and no diagnostic or clinical-scoring claims +are permitted. diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index e82a407..3baed19 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -194,7 +194,7 @@ export function App() { const update = () => void api.evidenceRunnerStatus() .then(setEvidenceStatus) .catch(() => setEvidenceStatus({ - schema: 'pathlab.evidence-runner-status/1', status: 'unavailable', detail: 'Runner is not reachable', + schema: 'pathlab.evidence-runner-status/2', status: 'unavailable', detail: 'Runner is not reachable', })) update() const timer = window.setInterval(update, 5_000) @@ -813,10 +813,26 @@ export function App() {
Evidence Mentor - {evidenceStatus?.status === 'ready' ? 'Autonomous runner ready' : 'Runner unavailable'} + {evidenceStatus?.status === 'ready' + ? `${evidenceStatus.queue?.active || 0} active · ${evidenceStatus.queue?.queued || 0} queued` + : 'Runner unavailable'} {evidenceStatus?.status === 'ready' ? - Analysis network off · RAM {formatBytes(evidenceStatus.processMemoryUsedBytes || 0)} / {formatBytes(evidenceStatus.processMemoryLimitBytes || 0)} · VRAM cap {evidenceStatus.vramLimitMiB} MiB + {evidenceStatus.acceptingJobs === false ? 'New claims paused' : 'Analysis network off'} + {evidenceStatus.leadingJob + ? ` · ${evidenceStatus.leadingJob.id} ${Math.round(evidenceStatus.leadingJob.progress * 100)}%` + : ''} + {evidenceStatus.heartbeat ? ` · heartbeat ${new Date(evidenceStatus.heartbeat).toLocaleTimeString()}` : ''} : {evidenceStatus?.detail || 'Checking local runner…'}} + {evidenceStatus?.status === 'ready' ? : null}
{connection?.connected && viewerUpload?.viewerSlideId && ['IMAGE_READY', 'SYNCING_RESULTS', 'COMPLETE'].includes(viewerUpload.state) ? ( diff --git a/frontend/src/api.ts b/frontend/src/api.ts index f24fb26..edd178a 100644 --- a/frontend/src/api.ts +++ b/frontend/src/api.ts @@ -212,9 +212,15 @@ export async function capabilities() { } export type EvidenceRunnerStatus = { - schema: 'pathlab.evidence-runner-status/1' + schema: 'pathlab.evidence-runner-status/2' status: 'ready' | 'unavailable' detail?: string + serviceVersion?: string + startedAt?: string + heartbeat?: string + acceptingJobs?: boolean + queue?: { queued: number; active: number; gpu: number; cpuIo: number } + leadingJob?: { id: string; state: string; stage: string; progress: number; lane: 'gpu' | 'cpu_io' } networkDisabledForAnalysis?: boolean processMemoryUsedBytes?: number processMemoryLimitBytes?: number @@ -228,6 +234,10 @@ export async function evidenceRunnerStatus() { return request('/api/evidence/status') } +export async function openEvidenceDashboard() { + return request<{ url: string }>('/api/evidence/dashboard', { method: 'POST' }) +} + export async function importProjectFolder(path?: string) { const query = path?.trim() ? `?path=${encodeURIComponent(path.trim())}` : '' return request<{ datasets: Dataset[]; project?: { root: string; imported: number; failed: string } }>( diff --git a/frontend/src/test/App.test.tsx b/frontend/src/test/App.test.tsx index 9996583..fd8f3e7 100644 --- a/frontend/src/test/App.test.tsx +++ b/frontend/src/test/App.test.tsx @@ -41,10 +41,13 @@ vi.mock('../SlideViewer', () => ({ vi.mock('../api', () => ({ evidenceRunnerStatus: vi.fn(async () => ({ - schema: 'pathlab.evidence-runner-status/1', + schema: 'pathlab.evidence-runner-status/2', status: 'unavailable', detail: 'Runner is not configured in frontend tests', })), + openEvidenceDashboard: vi.fn(async () => ({ + url: 'http://127.0.0.1:43210/dashboard/#single-use', + })), bootstrap: vi.fn(async () => [[], { conversionRuntime: 'Bio-Formats test', derivativeRuntime: 'libvips test', diff --git a/scripts/evidence-mentor-service.ps1 b/scripts/evidence-mentor-service.ps1 new file mode 100644 index 0000000..d764b98 --- /dev/null +++ b/scripts/evidence-mentor-service.ps1 @@ -0,0 +1,201 @@ +[CmdletBinding(SupportsShouldProcess)] +param( + [Parameter(Mandatory)] [ValidateSet('Install','Upgrade','Uninstall','Status')] [string] $Action, + [string] $Version = '2.0.0', + [string] $DistributionPath, + [string] $JavaHome, + [string] $ProgramRoot = 'C:\ProgramData\PathLab\EvidenceMentor', + [string] $StateRoot = 'D:\PathLabData\EvidenceMentor\state' +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version Latest +$serviceName = 'PathLabEvidenceMentor' +$winSwVersion = '2.12.0' +$winSwUri = "https://github.com/winsw/winsw/releases/download/v$winSwVersion/WinSW-x64.exe" +$winSwSha256 = '05b82d46ad331cc16bdc00de5c6332c1ef818df8ceefcd49c726553209b3a0da' +$wrapperPath = Join-Path $ProgramRoot 'PathLabEvidenceMentor.exe' +$configPath = Join-Path $ProgramRoot 'PathLabEvidenceMentor.xml' +$activeVersionPath = Join-Path $ProgramRoot 'active-version.txt' +$runtimeRoot = Join-Path $ProgramRoot 'runtime' +$logRoot = Join-Path $StateRoot 'logs' + +function Assert-Administrator { + $identity = [Security.Principal.WindowsIdentity]::GetCurrent() + $principal = [Security.Principal.WindowsPrincipal]::new($identity) + if (-not $principal.IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator)) { + throw 'Administrator approval is required to install or upgrade the PathLab service.' + } +} + +function Write-AtomicText([string] $Path, [string] $Content) { + $partial = "$Path.partial" + [IO.File]::WriteAllText($partial, $Content, [Text.UTF8Encoding]::new($false)) + [IO.File]::Move($partial, $Path, $true) +} + +function Ensure-WinSW { + if (Test-Path -LiteralPath $wrapperPath -PathType Leaf) { + $actual = (Get-FileHash -LiteralPath $wrapperPath -Algorithm SHA256).Hash.ToLowerInvariant() + if ($actual -eq $winSwSha256) { return } + throw 'Installed WinSW checksum does not match the pinned release.' + } + $partial = "$wrapperPath.partial" + Invoke-WebRequest -UseBasicParsing -Uri $winSwUri -OutFile $partial + $actual = (Get-FileHash -LiteralPath $partial -Algorithm SHA256).Hash.ToLowerInvariant() + if ($actual -ne $winSwSha256) { throw 'Downloaded WinSW checksum does not match the pinned release.' } + Move-Item -LiteralPath $partial -Destination $wrapperPath +} + +function Grant-PathLabAccess([string] $InstallingUser) { + & icacls.exe $ProgramRoot /inheritance:r /grant:r 'SYSTEM:(OI)(CI)F' 'Administrators:(OI)(CI)F' "NT SERVICE\${serviceName}:(OI)(CI)RX" | Out-Null + & icacls.exe $StateRoot /inheritance:r /grant:r 'SYSTEM:(OI)(CI)F' 'Administrators:(OI)(CI)F' "NT SERVICE\${serviceName}:(OI)(CI)M" "${InstallingUser}:(RX)" | Out-Null + foreach ($operatorPath in @('inputs','requests','models','artifacts')) { + $resolved = Join-Path $StateRoot $operatorPath + & icacls.exe $resolved /inheritance:r /grant:r 'SYSTEM:(OI)(CI)F' 'Administrators:(OI)(CI)F' "NT SERVICE\${serviceName}:(OI)(CI)M" "${InstallingUser}:(OI)(CI)M" | Out-Null + } + foreach ($serviceOnlyPath in @('checkpoints','signing')) { + $resolved = Join-Path $StateRoot $serviceOnlyPath + & icacls.exe $resolved /inheritance:r /grant:r 'SYSTEM:(OI)(CI)F' 'Administrators:(OI)(CI)F' "NT SERVICE\${serviceName}:(OI)(CI)M" | Out-Null + } + & icacls.exe $logRoot /inheritance:r /grant:r 'SYSTEM:(OI)(CI)F' 'Administrators:(OI)(CI)F' "NT SERVICE\${serviceName}:(OI)(CI)M" "${InstallingUser}:(OI)(CI)R" | Out-Null + foreach ($sensitive in @((Join-Path $StateRoot 'ipc-token'), (Join-Path $StateRoot 'endpoint.json'))) { + if (Test-Path -LiteralPath $sensitive) { + & icacls.exe $sensitive /inheritance:r /grant:r 'SYSTEM:F' 'Administrators:F' "NT SERVICE\${serviceName}:R" "${InstallingUser}:R" | Out-Null + } + } +} + +function New-ServiceConfig([string] $RuntimePath) { + $java = [Security.SecurityElement]::Escape((Join-Path $RuntimePath 'jre\bin\java.exe')) + $classpath = [Security.SecurityElement]::Escape((Join-Path $RuntimePath 'app\lib\*')) + $state = [Security.SecurityElement]::Escape($StateRoot) + $logs = [Security.SecurityElement]::Escape($logRoot) + return @" + + $serviceName + PathLab Evidence Mentor + Offline local pathology research evidence worker and operations dashboard. + $java + -Xrs -Xmx16g -cp "$classpath" org.pathlab.forge.evidence.EvidenceMentorRunner --state "$state" --port 0 + $([Security.SecurityElement]::Escape($RuntimePath)) + Automatic + + NT AUTHORITYLocalService + + + + 1 day + 30 sec + $logs + 104857608 + + + + + +"@ +} + +function Install-FirewallRules([string] $RuntimePath) { + Get-NetFirewallRule -DisplayName 'PathLab Evidence Mentor outbound deny*' -ErrorAction SilentlyContinue | Remove-NetFirewallRule + $executables = @((Join-Path $RuntimePath 'jre\bin\java.exe')) + $modelRoot = Join-Path $StateRoot 'models' + if (Test-Path -LiteralPath $modelRoot) { + $executables += Get-ChildItem -LiteralPath $modelRoot -Filter '*.exe' -File -Recurse | Select-Object -ExpandProperty FullName + } + $index = 0 + foreach ($executable in $executables | Select-Object -Unique) { + if (Test-Path -LiteralPath $executable -PathType Leaf) { + New-NetFirewallRule -DisplayName "PathLab Evidence Mentor outbound deny $index" -Direction Outbound -Action Block -Program $executable -Profile Any | Out-Null + $index++ + } + } +} + +function Wait-RunnerHealth([int] $Seconds = 30) { + $deadline = [DateTimeOffset]::UtcNow.AddSeconds($Seconds) + do { + Start-Sleep -Milliseconds 500 + $endpointPath = Join-Path $StateRoot 'endpoint.json' + $tokenPath = Join-Path $StateRoot 'ipc-token' + if ((Test-Path -LiteralPath $endpointPath) -and (Test-Path -LiteralPath $tokenPath)) { + try { + $endpoint = Get-Content -LiteralPath $endpointPath -Raw | ConvertFrom-Json + if ($endpoint.schema -ne 'pathlab.runner-endpoint/1') { continue } + $headers = @{ Authorization = 'Bearer ' + (Get-Content -LiteralPath $tokenPath -Raw).Trim() } + $health = Invoke-RestMethod -Uri "http://127.0.0.1:$($endpoint.port)/health" -Headers $headers -TimeoutSec 2 + if ($health.status -eq 'ready') { return $true } + } catch { } + } + } while ([DateTimeOffset]::UtcNow -lt $deadline) + return $false +} + +function Stage-Runtime { + if (-not $DistributionPath -or -not (Test-Path -LiteralPath $DistributionPath -PathType Container)) { throw 'DistributionPath must reference a completed installDist directory.' } + if (-not $JavaHome -or -not (Test-Path -LiteralPath (Join-Path $JavaHome 'bin\java.exe') -PathType Leaf)) { throw 'JavaHome must reference a Java 17 runtime.' } + $target = Join-Path $runtimeRoot $Version + if (Test-Path -LiteralPath $target) { throw "Runtime version already exists: $target" } + New-Item -ItemType Directory -Path (Join-Path $target 'app') -Force | Out-Null + Copy-Item -Path (Join-Path $DistributionPath '*') -Destination (Join-Path $target 'app') -Recurse -Force + Copy-Item -LiteralPath $JavaHome -Destination (Join-Path $target 'jre') -Recurse -Force + return $target +} + +if ($Action -eq 'Status') { + Get-CimInstance Win32_Service -Filter "Name='$serviceName'" -ErrorAction SilentlyContinue | Select-Object Name, State, StartMode, StartName + if (Test-Path -LiteralPath (Join-Path $StateRoot 'endpoint.json')) { Get-Content -LiteralPath (Join-Path $StateRoot 'endpoint.json') -Raw } + return +} + +if (-not $WhatIfPreference) { Assert-Administrator } +if ($Action -eq 'Uninstall') { + if ($PSCmdlet.ShouldProcess($serviceName, 'Uninstall service and outbound-deny rules while preserving state and model data')) { + if (Test-Path -LiteralPath $wrapperPath) { + & $wrapperPath stop 2>$null + & $wrapperPath uninstall + } + Get-NetFirewallRule -DisplayName 'PathLab Evidence Mentor outbound deny*' -ErrorAction SilentlyContinue | Remove-NetFirewallRule + $shortcutPath = Join-Path ([Environment]::GetFolderPath('CommonStartMenu')) 'Programs\PathLab Evidence Mentor Dashboard.lnk' + if (Test-Path -LiteralPath $shortcutPath) { Remove-Item -LiteralPath $shortcutPath -Force } + } + return +} + +$installingUser = [Security.Principal.WindowsIdentity]::GetCurrent().Name +if ($PSCmdlet.ShouldProcess($serviceName, "$Action autonomous service version $Version")) { + $stateDirectories = @('inputs','requests','models','artifacts','checkpoints','signing','logs') | ForEach-Object { Join-Path $StateRoot $_ } + New-Item -ItemType Directory -Path @($ProgramRoot, $runtimeRoot, $StateRoot) -Force | Out-Null + New-Item -ItemType Directory -Path $stateDirectories -Force | Out-Null + $previousVersion = if (Test-Path -LiteralPath $activeVersionPath) { (Get-Content -LiteralPath $activeVersionPath -Raw).Trim() } else { '' } + $previousConfig = if (Test-Path -LiteralPath $configPath) { Get-Content -LiteralPath $configPath -Raw } else { '' } + $runtimePath = Stage-Runtime + Ensure-WinSW + if ($Action -eq 'Upgrade' -and (Get-Service -Name $serviceName -ErrorAction SilentlyContinue)) { & $wrapperPath stop } + Write-AtomicText $configPath (New-ServiceConfig $runtimePath) + if (-not (Get-Service -Name $serviceName -ErrorAction SilentlyContinue)) { & $wrapperPath install } + Grant-PathLabAccess $installingUser + Install-FirewallRules $runtimePath + & $wrapperPath start + if (-not (Wait-RunnerHealth)) { + & $wrapperPath stop 2>$null + if ($previousConfig) { + Write-AtomicText $configPath $previousConfig + Install-FirewallRules (Join-Path $runtimeRoot $previousVersion) + & $wrapperPath start + } + throw 'New runtime failed health checks; the previous configuration was restored.' + } + Write-AtomicText $activeVersionPath $Version + Grant-PathLabAccess $installingUser + $launcher = Join-Path $ProgramRoot 'Open-PathLab-Evidence-Dashboard.ps1' + Copy-Item -LiteralPath (Join-Path $PSScriptRoot 'open-evidence-dashboard.ps1') -Destination $launcher -Force + $shortcutPath = Join-Path ([Environment]::GetFolderPath('CommonStartMenu')) 'Programs\PathLab Evidence Mentor Dashboard.lnk' + $shell = New-Object -ComObject WScript.Shell + $shortcut = $shell.CreateShortcut($shortcutPath) + $shortcut.TargetPath = 'powershell.exe' + $shortcut.Arguments = "-NoProfile -ExecutionPolicy Bypass -File `"$launcher`" -StateRoot `"$StateRoot`"" + $shortcut.WorkingDirectory = $ProgramRoot + $shortcut.Save() +} diff --git a/scripts/evidence-mentor-task.ps1 b/scripts/evidence-mentor-task.ps1 index 0052a33..c31333a 100644 --- a/scripts/evidence-mentor-task.ps1 +++ b/scripts/evidence-mentor-task.ps1 @@ -1,71 +1,18 @@ [CmdletBinding(SupportsShouldProcess)] param( - [Parameter(Mandatory = $true)] - [string]$RuntimeRoot, - [string]$StateRoot, - [int]$Port = 8765, - [switch]$Uninstall + [string] $RuntimeRoot, + [string] $StateRoot, + [int] $Port, + [switch] $Uninstall ) $ErrorActionPreference = 'Stop' -$taskName = 'PathLab Evidence Mentor Runner' -$runtime = [IO.Path]::GetFullPath($RuntimeRoot) -if (-not $StateRoot) { - $StateRoot = if (Test-Path -LiteralPath 'D:\PathLabData' -PathType Container) { - 'D:\PathLabData\EvidenceMentor\state' - } else { - Join-Path $env:LOCALAPPDATA 'PathLab\EvidenceMentor\state' - } -} -$state = [IO.Path]::GetFullPath($StateRoot) - +$legacyTask = 'PathLab Evidence Mentor Runner' if ($Uninstall) { - if ($PSCmdlet.ShouldProcess($taskName, 'Unregister per-user scheduled task')) { - Unregister-ScheduledTask -TaskName $taskName -Confirm:$false -ErrorAction SilentlyContinue + if ($PSCmdlet.ShouldProcess($legacyTask, 'Remove the obsolete per-user scheduled task')) { + Unregister-ScheduledTask -TaskName $legacyTask -Confirm:$false -ErrorAction SilentlyContinue } return } -$lib = Join-Path $runtime 'lib\*' -if (-not (Test-Path -LiteralPath (Split-Path $lib) -PathType Container)) { - throw "Versioned Forge runtime missing lib directory: $lib" -} -$java = if ($env:JAVA_HOME -and (Test-Path -LiteralPath (Join-Path $env:JAVA_HOME 'bin\javaw.exe'))) { - Join-Path $env:JAVA_HOME 'bin\javaw.exe' -} else { - (Get-Command javaw.exe -ErrorAction Stop).Source -} - -if ($PSCmdlet.ShouldProcess($state, 'Create protected Evidence Mentor state')) { - New-Item -ItemType Directory -Path $state -Force | Out-Null - $token = Join-Path $state 'ipc-token' - if (-not (Test-Path -LiteralPath $token -PathType Leaf)) { - $bytes = [byte[]]::new(32) - [Security.Cryptography.RandomNumberGenerator]::Fill($bytes) - [IO.File]::WriteAllText($token, [Convert]::ToBase64String($bytes)) - } - & icacls.exe $token /inheritance:r /grant:r "${env:USERNAME}:(R,W)" | Out-Null -} - -$arguments = @( - '-Xmx16g' - '-cp' - ('"{0}"' -f $lib) - 'org.pathlab.forge.evidence.EvidenceMentorRunner' - '--state' - ('"{0}"' -f $state) - '--port' - $Port -) -join ' ' -$action = New-ScheduledTaskAction -Execute $java -Argument $arguments -$trigger = New-ScheduledTaskTrigger -AtLogOn -User $env:USERNAME -$settings = New-ScheduledTaskSettingsSet -StartWhenAvailable -RestartCount 3 ` - -RestartInterval (New-TimeSpan -Minutes 1) -ExecutionTimeLimit ([TimeSpan]::Zero) -$principal = New-ScheduledTaskPrincipal -UserId $env:USERNAME -LogonType Interactive ` - -RunLevel Limited - -if ($PSCmdlet.ShouldProcess($taskName, 'Register and start per-user autonomous runner')) { - Register-ScheduledTask -TaskName $taskName -Action $action -Trigger $trigger ` - -Settings $settings -Principal $principal -Force | Out-Null - Start-ScheduledTask -TaskName $taskName -} +throw 'Per-user Task Scheduler deployment was retired. Build installDist and use scripts\evidence-mentor-service.ps1 with administrator approval.' diff --git a/scripts/open-evidence-dashboard.ps1 b/scripts/open-evidence-dashboard.ps1 new file mode 100644 index 0000000..8e112c9 --- /dev/null +++ b/scripts/open-evidence-dashboard.ps1 @@ -0,0 +1,18 @@ +[CmdletBinding()] +param([string] $StateRoot = 'D:\PathLabData\EvidenceMentor\state') + +$ErrorActionPreference = 'Stop' +$endpointPath = Join-Path $StateRoot 'endpoint.json' +$tokenPath = Join-Path $StateRoot 'ipc-token' +if (-not (Test-Path -LiteralPath $endpointPath -PathType Leaf) -or -not (Test-Path -LiteralPath $tokenPath -PathType Leaf)) { + throw 'PathLab Evidence Mentor is not running.' +} +$endpoint = Get-Content -LiteralPath $endpointPath -Raw | ConvertFrom-Json +if ($endpoint.schema -ne 'pathlab.runner-endpoint/1' -or $endpoint.port -lt 1 -or $endpoint.port -gt 65535) { + throw 'PathLab Evidence Mentor published an invalid endpoint.' +} +$origin = "http://127.0.0.1:$($endpoint.port)" +$headers = @{ Authorization = 'Bearer ' + (Get-Content -LiteralPath $tokenPath -Raw).Trim() } +$session = Invoke-RestMethod -Method Post -Uri "$origin/v1/dashboard-sessions" -Headers $headers -TimeoutSec 2 +if ($session.code -notmatch '^[A-Za-z0-9_-]{40,80}$') { throw 'PathLab Evidence Mentor refused the dashboard session.' } +Start-Process "$origin/dashboard/#$($session.code)" diff --git a/scripts/submit-evidence-job.ps1 b/scripts/submit-evidence-job.ps1 index 689090b..b8232ec 100644 --- a/scripts/submit-evidence-job.ps1 +++ b/scripts/submit-evidence-job.ps1 @@ -8,8 +8,7 @@ param( [Parameter(Mandatory = $true)][string]$PackManifest, [ValidateSet('he', 'ihc_dab')][string]$Stain, [string]$Marker = 'generic', - [string]$StateRoot, - [int]$Port = 8765 + [string]$StateRoot ) $ErrorActionPreference = 'Stop' @@ -25,15 +24,26 @@ $source = Get-Item -LiteralPath $SourcePath $preview = Get-Item -LiteralPath $PreviewPath $pack = Get-Item -LiteralPath $PackManifest $tokenPath = Join-Path $state 'ipc-token' -if (-not (Test-Path -LiteralPath $tokenPath -PathType Leaf)) { +$endpointPath = Join-Path $state 'endpoint.json' +if (-not (Test-Path -LiteralPath $tokenPath -PathType Leaf) -or -not (Test-Path -LiteralPath $endpointPath -PathType Leaf)) { throw 'Evidence Mentor runner is not installed or has not started.' } +$endpoint = Get-Content -LiteralPath $endpointPath -Raw | ConvertFrom-Json +if ($endpoint.schema -ne 'pathlab.runner-endpoint/1' -or $endpoint.port -lt 1 -or $endpoint.port -gt 65535) { + throw 'Evidence Mentor endpoint is invalid.' +} +$statePrefix = $state.TrimEnd('\') + '\' +foreach ($input in @($source.FullName, $preview.FullName, $pack.FullName)) { + if (-not $input.StartsWith($statePrefix, [StringComparison]::OrdinalIgnoreCase)) { + throw "LocalService cannot read an input outside the protected Evidence Mentor state root: $input" + } +} $jobId = [Guid]::NewGuid().ToString() $requestRoot = Join-Path $state 'requests' New-Item -ItemType Directory -Path $requestRoot -Force | Out-Null $requestPath = Join-Path $requestRoot "$jobId.json" $request = [ordered]@{ - schema = 'pathlab.evidence-job/1' + schema = 'pathlab.evidence-job/2' sourcePath = $source.FullName sourceSha256 = (Get-FileHash -LiteralPath $source.FullName -Algorithm SHA256).Hash.ToLowerInvariant() slideRevision = $SlideRevision @@ -44,8 +54,10 @@ $request = [ordered]@{ stain = $Stain marker = $Marker.ToLowerInvariant() } -$request | ConvertTo-Json -Depth 4 | Set-Content -LiteralPath $requestPath -Encoding utf8NoBOM +$partial = "$requestPath.partial" +$request | ConvertTo-Json -Depth 4 | Set-Content -LiteralPath $partial -Encoding utf8NoBOM +Move-Item -LiteralPath $partial -Destination $requestPath $token = [IO.File]::ReadAllText($tokenPath).Trim() $headers = @{ Authorization = "Bearer $token" } -Invoke-RestMethod -Method Post -Uri "http://127.0.0.1:$Port/v1/jobs" -Headers $headers ` +Invoke-RestMethod -Method Post -Uri "http://127.0.0.1:$($endpoint.port)/v1/jobs" -Headers $headers ` -ContentType 'application/json' -Body (@{ id = $jobId; requestPath = $requestPath } | ConvertTo-Json) diff --git a/src/main/java/org/pathlab/forge/evidence/DashboardSessions.java b/src/main/java/org/pathlab/forge/evidence/DashboardSessions.java new file mode 100644 index 0000000..577041e --- /dev/null +++ b/src/main/java/org/pathlab/forge/evidence/DashboardSessions.java @@ -0,0 +1,52 @@ +package org.pathlab.forge.evidence; + +import java.security.SecureRandom; +import java.time.Duration; +import java.time.Instant; +import java.util.Base64; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; + +/** In-memory, boot-scoped dashboard authentication. Nothing survives a service restart. */ +final class DashboardSessions { + private static final Duration CODE_LIFETIME = Duration.ofSeconds(60); + private static final Duration SESSION_LIFETIME = Duration.ofHours(8); + private final SecureRandom random = new SecureRandom(); + private final Map codes = new HashMap<>(); + private final Map sessions = new HashMap<>(); + + synchronized String create(Instant now) { + prune(now); + var code = random(32); + codes.put(code, now.plus(CODE_LIFETIME)); + return code; + } + + synchronized Optional exchange(String code, Instant now) { + prune(now); + var expiry = codes.remove(code); + if (expiry == null || expiry.isBefore(now)) return Optional.empty(); + var token = random(32); + var session = new Session(token, random(32), now.plus(SESSION_LIFETIME)); + sessions.put(token, session); + return Optional.of(session); + } + + synchronized Optional find(String token, Instant now) { + prune(now); + return Optional.ofNullable(sessions.get(token)); + } + + private void prune(Instant now) { + codes.entrySet().removeIf(entry -> entry.getValue().isBefore(now)); + sessions.entrySet().removeIf(entry -> entry.getValue().expiresAt().isBefore(now)); + } + + private String random(int bytes) { + var value = new byte[bytes]; random.nextBytes(value); + return Base64.getUrlEncoder().withoutPadding().encodeToString(value); + } + + record Session(String token, String csrfToken, Instant expiresAt) { } +} diff --git a/src/main/java/org/pathlab/forge/evidence/EvidenceExecutionLane.java b/src/main/java/org/pathlab/forge/evidence/EvidenceExecutionLane.java new file mode 100644 index 0000000..6b4cd23 --- /dev/null +++ b/src/main/java/org/pathlab/forge/evidence/EvidenceExecutionLane.java @@ -0,0 +1,13 @@ +package org.pathlab.forge.evidence; + +/** Execution lanes are assigned from signed pack metadata, never from IPC input. */ +public enum EvidenceExecutionLane { + GPU("gpu"), CPU_IO("cpu_io"); + private final String wire; + EvidenceExecutionLane(String wire) { this.wire = wire; } + public String wire() { return wire; } + public static EvidenceExecutionLane fromWire(String value) { + for (var lane : values()) if (lane.wire.equals(value)) return lane; + throw new IllegalArgumentException("Unknown evidence execution lane"); + } +} diff --git a/src/main/java/org/pathlab/forge/evidence/EvidenceJobProcessor.java b/src/main/java/org/pathlab/forge/evidence/EvidenceJobProcessor.java index c765a3e..4965e3e 100644 --- a/src/main/java/org/pathlab/forge/evidence/EvidenceJobProcessor.java +++ b/src/main/java/org/pathlab/forge/evidence/EvidenceJobProcessor.java @@ -34,6 +34,28 @@ public EvidenceJobProcessor(EvidenceJobQueue queue, Path stateRoot) throws IOExc Files.createDirectories(this.stateRoot.resolve("artifacts")); } + /** Resolves the lane from the validated pack manifest; the IPC caller cannot select it. */ + public static EvidenceExecutionLane executionLane(Path requestPath) throws IOException { + return executionPlan(requestPath).lane(); + } + + public static ExecutionPlan executionPlan(Path requestPath) throws IOException { + final JsonNode request; + try { + request = JSON.readTree(requestPath.toAbsolutePath().normalize().toFile()); + } catch (JsonProcessingException invalidJson) { + throw new IllegalArgumentException("Evidence job request JSON is invalid", invalidJson); + } + // Legacy v1 submissions were accepted before lane metadata existed and migrate to CPU/I/O. + if (request == null || !request.path("packManifest").isTextual()) return new ExecutionPlan(EvidenceExecutionLane.CPU_IO, ""); + var pack = EvidencePackManifest.load(regularPath(request, "packManifest")); + pack.requirePilotEligible(); + return new ExecutionPlan("cuda".equals(pack.runtimeCompatibility().executionProvider()) + ? EvidenceExecutionLane.GPU : EvidenceExecutionLane.CPU_IO, pack.sha256()); + } + + public record ExecutionPlan(EvidenceExecutionLane lane, String packSha256) { } + public EvidenceJob process(EvidenceJob job, String workerId, Instant now) throws IOException { final JsonNode request; try { @@ -46,7 +68,7 @@ public EvidenceJob process(EvidenceJob job, String workerId, Instant now) throws && java.util.stream.Stream.concat(REQUIRED_REQUEST_FIELDS.stream(), OPTIONAL_REQUEST_FIELDS.stream()) .collect(java.util.stream.Collectors.toSet()).containsAll(requestFields), "Evidence job request is invalid"); - require("pathlab.evidence-job/1".equals(text(request, "schema")), + require(Set.of("pathlab.evidence-job/1", "pathlab.evidence-job/2").contains(text(request, "schema")), "Evidence job schema is unsupported"); var source = regularPath(request, "sourcePath"); var preview = regularPath(request, "previewPath"); @@ -55,6 +77,16 @@ public EvidenceJob process(EvidenceJob job, String workerId, Instant now) throws "Evidence source checksum does not match"); var pack = EvidencePackManifest.load(regularPath(request, "packManifest")); pack.requirePilotEligible(); + var durable = queue.snapshot(job.id()).orElseThrow(); + require(durable.requestSha256().isBlank() || durable.requestSha256().equals(sha256(job.requestPath())), + "Evidence request changed after submission"); + require(durable.packSha256().isBlank() || durable.packSha256().equals(pack.sha256()), + "Evidence pack changed after submission"); + if (durable.checkpointPath() != null) { + require(Files.isRegularFile(durable.checkpointPath()) + && durable.checkpointSha256().equals(sha256(durable.checkpointPath())), + "Evidence checkpoint checksum does not match"); + } var stain = text(request, "stain"); pack.requireStain(stain); var marker = text(request, "marker").toLowerCase(); @@ -91,7 +123,15 @@ public EvidenceJob process(EvidenceJob job, String workerId, Instant now) throws var analysis = BrightfieldTileAnalyzer.analyze(image, marker); JsonNode modelResult = null; if (pack.capability() == EvidencePackManifest.Capability.HE_EVIDENCE) { - modelResult = new ExternalModelWorker(stateRoot).execute(pack, job.requestPath(), job.id()); + modelResult = new ExternalModelWorker(stateRoot).execute(pack, job.requestPath(), job.id(), + durable.checkpointPath(), progress -> { + var current = queue.find(job.id()).orElseThrow(); + if (current.cancelRequested()) { + throw new CancellationException(); + } + queue.heartbeat(job.id(), workerId, progress.completedUnits(), progress.totalUnits(), + progress.checkpointPath(), progress.checkpointSha256(), Instant.now(), LEASE); + }); require("completed".equals(modelResult.path("status").asText()), "H&E model worker returned unsupported or not_evaluable"); } @@ -128,8 +168,9 @@ public EvidenceJob process(EvidenceJob job, String workerId, Instant now) throws Files.createDirectories(artifactRoot); var unsigned = evidence(request, pack, image, analysis, stainQc, modelResult, focus, tissue, abstentionReasons, compartmentSource, compartmentWarning, now); - new EvidenceBundleWriter(stateRoot.resolve("signing")) - .write(artifactRoot.resolve("evidence.json"), unsigned); + var finalArtifact = artifactRoot.resolve("evidence.json"); + new EvidenceBundleWriter(stateRoot.resolve("signing")).write(finalArtifact, unsigned); + queue.recordFinalArtifact(job.id(), workerId, sha256(finalArtifact), now.plusMillis(2500)); var terminal = abstained ? EvidenceJobState.ABSTAINED : EvidenceJobState.COMPLETED; var detail = abstained ? "Signed QC evidence abstained from descriptive analysis" diff --git a/src/main/java/org/pathlab/forge/evidence/EvidenceJobQueue.java b/src/main/java/org/pathlab/forge/evidence/EvidenceJobQueue.java index 8bb0eef..35b96f4 100644 --- a/src/main/java/org/pathlab/forge/evidence/EvidenceJobQueue.java +++ b/src/main/java/org/pathlab/forge/evidence/EvidenceJobQueue.java @@ -1,19 +1,25 @@ package org.pathlab.forge.evidence; import java.io.IOException; +import java.io.InputStream; import java.nio.file.Files; import java.nio.file.Path; +import java.security.MessageDigest; import java.sql.Connection; import java.sql.DriverManager; import java.sql.ResultSet; import java.sql.SQLException; import java.time.Duration; import java.time.Instant; +import java.util.ArrayList; +import java.util.HexFormat; +import java.util.List; import java.util.Optional; -/** Durable WAL queue with expiring leases and restart-safe checkpoints. */ +/** Durable SQLite WAL queue with lane isolation, renewable leases, and in-place v1 migration. */ public final class EvidenceJobQueue implements AutoCloseable { private static final String ACTIVE_STATES = "'VALIDATING','RUNNING','REFINING','PACKAGING'"; + private static final long[] RETRY_DELAYS_SECONDS = {5, 30, 120}; private final Connection connection; public EvidenceJobQueue(Path database) throws IOException { @@ -27,35 +33,76 @@ public EvidenceJobQueue(Path database) throws IOException { statement.execute("PRAGMA busy_timeout=5000"); statement.execute(""" CREATE TABLE IF NOT EXISTS evidence_jobs ( - id TEXT PRIMARY KEY, - request_path TEXT NOT NULL, - state TEXT NOT NULL, - stage TEXT NOT NULL, - progress REAL NOT NULL, - retry_count INTEGER NOT NULL, - cancel_requested INTEGER NOT NULL, - lease_owner TEXT NOT NULL, - lease_expires_at TEXT NOT NULL, - detail TEXT NOT NULL, - created_at TEXT NOT NULL, - updated_at TEXT NOT NULL + id TEXT PRIMARY KEY, request_path TEXT NOT NULL, state TEXT NOT NULL, + stage TEXT NOT NULL, progress REAL NOT NULL, retry_count INTEGER NOT NULL, + cancel_requested INTEGER NOT NULL, lease_owner TEXT NOT NULL, + lease_expires_at TEXT NOT NULL, detail TEXT NOT NULL, + created_at TEXT NOT NULL, updated_at TEXT NOT NULL )"""); - statement.execute("CREATE INDEX IF NOT EXISTS ix_evidence_jobs_state_updated ON evidence_jobs(state, updated_at)"); } + migrateV2(); } catch (SQLException error) { throw new IOException("Unable to open Evidence Mentor queue", error); } } - public synchronized EvidenceJob submit(String id, Path requestPath, Instant now) throws IOException { + private void migrateV2() throws SQLException { + addColumn("lane", "TEXT NOT NULL DEFAULT 'cpu_io'"); + addColumn("completed_units", "INTEGER NOT NULL DEFAULT 0"); + addColumn("total_units", "INTEGER NOT NULL DEFAULT 0"); + addColumn("checkpoint_path", "TEXT NOT NULL DEFAULT ''"); + addColumn("checkpoint_sha256", "TEXT NOT NULL DEFAULT ''"); + addColumn("last_heartbeat", "TEXT NOT NULL DEFAULT '1970-01-01T00:00:00Z'"); + addColumn("throughput", "REAL NOT NULL DEFAULT 0"); + addColumn("eta_seconds", "INTEGER"); + addColumn("checkpoint_count", "INTEGER NOT NULL DEFAULT 0"); + addColumn("next_retry_at", "TEXT NOT NULL DEFAULT '1970-01-01T00:00:00Z'"); + addColumn("failure_class", "TEXT NOT NULL DEFAULT ''"); + addColumn("failure_code", "TEXT NOT NULL DEFAULT ''"); + addColumn("request_sha256", "TEXT NOT NULL DEFAULT ''"); + addColumn("pack_sha256", "TEXT NOT NULL DEFAULT ''"); + addColumn("final_artifact_sha256", "TEXT NOT NULL DEFAULT ''"); + try (var statement = connection.createStatement()) { + statement.execute("CREATE INDEX IF NOT EXISTS ix_evidence_jobs_lane_state ON evidence_jobs(lane,state,next_retry_at,created_at)"); + statement.execute("CREATE TABLE IF NOT EXISTS runner_control (singleton INTEGER PRIMARY KEY CHECK(singleton=1),accepting_jobs INTEGER NOT NULL,updated_at TEXT NOT NULL)"); + statement.execute("INSERT OR IGNORE INTO runner_control VALUES(1,1,'1970-01-01T00:00:00Z')"); + statement.execute("CREATE TABLE IF NOT EXISTS evidence_progress_samples (job_id TEXT NOT NULL,sequence INTEGER PRIMARY KEY AUTOINCREMENT,completed_units INTEGER NOT NULL,recorded_at TEXT NOT NULL)"); + statement.execute("CREATE INDEX IF NOT EXISTS ix_evidence_progress_job_sequence ON evidence_progress_samples(job_id,sequence DESC)"); + } + } + + private void addColumn(String name, String declaration) throws SQLException { + try (var statement = connection.createStatement(); var rows = statement.executeQuery("PRAGMA table_info(evidence_jobs)")) { + while (rows.next()) if (name.equals(rows.getString("name"))) return; + } + try (var statement = connection.createStatement()) { + statement.execute("ALTER TABLE evidence_jobs ADD COLUMN " + name + " " + declaration); + } + } + + public EvidenceJob submit(String id, Path requestPath, Instant now) throws IOException { + return submit(id, requestPath, EvidenceExecutionLane.CPU_IO, now); + } + + public synchronized EvidenceJob submit(String id, Path requestPath, EvidenceExecutionLane lane, Instant now) throws IOException { + return submit(id, requestPath, lane, "", now); + } + + public synchronized EvidenceJob submit(String id, Path requestPath, EvidenceExecutionLane lane, + String packSha256, Instant now) throws IOException { var normalized = requestPath.toAbsolutePath().normalize(); if (!Files.isRegularFile(normalized)) throw new IllegalArgumentException("Evidence request file is unavailable"); var job = new EvidenceJob(id, normalized, EvidenceJobState.QUEUED, "queued", 0, 0, false, "", Instant.EPOCH, "Queued for autonomous local analysis", now, now); try (var statement = connection.prepareStatement(""" - INSERT INTO evidence_jobs VALUES (?,?,?,?,?,?,?,?,?,?,?,?) + INSERT INTO evidence_jobs (id,request_path,state,stage,progress,retry_count,cancel_requested, + lease_owner,lease_expires_at,detail,created_at,updated_at,lane,request_sha256,pack_sha256,next_retry_at,last_heartbeat) + VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) """)) { - bind(statement, job); + bindBase(statement, job); + statement.setString(13, lane.wire()); statement.setString(14, sha256(normalized)); + statement.setString(15, packSha256 == null ? "" : packSha256); + statement.setString(16, Instant.EPOCH.toString()); statement.setString(17, Instant.EPOCH.toString()); statement.executeUpdate(); return job; } catch (SQLException error) { @@ -67,189 +114,281 @@ INSERT INTO evidence_jobs VALUES (?,?,?,?,?,?,?,?,?,?,?,?) public synchronized Optional find(String id) throws IOException { try (var statement = connection.prepareStatement("SELECT * FROM evidence_jobs WHERE id=?")) { statement.setString(1, id); + try (var rows = statement.executeQuery()) { return rows.next() ? Optional.of(read(rows)) : Optional.empty(); } + } catch (SQLException error) { throw new IOException("Unable to read evidence job", error); } + } + + public synchronized Optional snapshot(String id) throws IOException { + try (var statement = connection.prepareStatement("SELECT * FROM evidence_jobs WHERE id=?")) { + statement.setString(1, id); + try (var rows = statement.executeQuery()) { return rows.next() ? Optional.of(readSnapshot(rows)) : Optional.empty(); } + } catch (SQLException error) { throw new IOException("Unable to read evidence job snapshot", error); } + } + + public synchronized List list(EvidenceJobState state, EvidenceExecutionLane lane, int limit) throws IOException { + if (limit < 1 || limit > 200) throw new IllegalArgumentException("Evidence job list limit is invalid"); + var sql = new StringBuilder("SELECT * FROM evidence_jobs WHERE 1=1"); + if (state != null) sql.append(" AND state=?"); + if (lane != null) sql.append(" AND lane=?"); + sql.append(" ORDER BY updated_at DESC,id LIMIT ?"); + try (var statement = connection.prepareStatement(sql.toString())) { + var index = 1; + if (state != null) statement.setString(index++, state.name()); + if (lane != null) statement.setString(index++, lane.wire()); + statement.setInt(index, limit); try (var rows = statement.executeQuery()) { - return rows.next() ? Optional.of(read(rows)) : Optional.empty(); + var result = new ArrayList(); + while (rows.next()) result.add(readSnapshot(rows)); + return List.copyOf(result); } - } catch (SQLException error) { - throw new IOException("Unable to read evidence job", error); - } + } catch (SQLException error) { throw new IOException("Unable to list evidence jobs", error); } } - public synchronized Optional claimNext( - String workerId, Instant now, Duration lease) throws IOException { - if (workerId == null || !workerId.matches("[A-Za-z0-9._-]{1,120}")) { - throw new IllegalArgumentException("Evidence worker id is invalid"); - } - if (lease.isNegative() || lease.isZero() || lease.compareTo(Duration.ofMinutes(10)) > 0) { - throw new IllegalArgumentException("Evidence lease is invalid"); - } + public Optional claimNext(String workerId, Instant now, Duration lease) throws IOException { + return claimNext(null, workerId, now, lease); + } + + public synchronized Optional claimNext(EvidenceExecutionLane lane, String workerId, Instant now, Duration lease) throws IOException { + validateWorker(workerId, lease); + var accepting = acceptingJobs(); try { - try (var begin = connection.createStatement()) { - begin.execute("BEGIN IMMEDIATE"); - } + try (var begin = connection.createStatement()) { begin.execute("BEGIN IMMEDIATE"); } EvidenceJob selected = null; - var claimSql = "SELECT * FROM evidence_jobs " - + "WHERE cancel_requested=0 AND (state='QUEUED' OR (state IN (" - + ACTIVE_STATES + ") AND lease_expires_at<=?)) " - + "ORDER BY created_at, id LIMIT 1"; - try (var statement = connection.prepareStatement(claimSql)) { - statement.setString(1, now.toString()); - try (var rows = statement.executeQuery()) { - if (rows.next()) selected = read(rows); - } + var sql = "SELECT * FROM evidence_jobs WHERE cancel_requested=0 AND next_retry_at<=? AND (" + + (accepting ? "state='QUEUED' OR " : "") + "(state IN (" + + ACTIVE_STATES + ") AND lease_expires_at<=?))" + (lane == null ? "" : " AND lane=?") + + " ORDER BY created_at,id LIMIT 1"; + try (var statement = connection.prepareStatement(sql)) { + statement.setString(1, now.toString()); statement.setString(2, now.toString()); + if (lane != null) statement.setString(3, lane.wire()); + try (var rows = statement.executeQuery()) { if (rows.next()) selected = read(rows); } } - if (selected == null) { - commit(); - return Optional.empty(); - } - var state = selected.state() == EvidenceJobState.QUEUED - ? EvidenceJobState.VALIDATING : selected.state(); + if (selected == null) { commit(); return Optional.empty(); } + var state = selected.state() == EvidenceJobState.QUEUED ? EvidenceJobState.VALIDATING : selected.state(); var claimed = new EvidenceJob(selected.id(), selected.requestPath(), state, - selected.state() == EvidenceJobState.QUEUED ? "validating" : selected.stage(), - selected.progress(), selected.retryCount(), false, workerId, now.plus(lease), - selected.state() == EvidenceJobState.QUEUED - ? "Validating immutable request and pack" : "Recovered after expired worker lease", + selected.state() == EvidenceJobState.QUEUED ? "validating" : selected.stage(), selected.progress(), + selected.retryCount(), false, workerId, now.plus(lease), + selected.state() == EvidenceJobState.QUEUED ? "Validating immutable request and pack" : "Recovered after expired worker lease", selected.createdAt(), now); update(claimed); - commit(); - return Optional.of(claimed); - } catch (SQLException error) { - rollbackQuietly(); - throw new IOException("Unable to claim evidence job", error); - } + try (var heartbeat = connection.prepareStatement("UPDATE evidence_jobs SET last_heartbeat=? WHERE id=?")) { + heartbeat.setString(1, now.toString()); heartbeat.setString(2, selected.id()); heartbeat.executeUpdate(); + } + commit(); return Optional.of(claimed); + } catch (SQLException error) { rollbackQuietly(); throw new IOException("Unable to claim evidence job", error); } + } + + public synchronized EvidenceJob heartbeat(String id, String workerId, long completed, long total, + Path checkpoint, String checkpointSha256, Instant now, Duration lease) throws IOException { + var current = requireOwned(id, workerId, now); + if (completed < 0 || total < 0 || completed > total) throw new IllegalArgumentException("Worker progress counts are invalid"); + var sha = checkpointSha256 == null ? "" : checkpointSha256; + if (!sha.isEmpty() && !sha.matches("[a-f0-9]{64}")) throw new IllegalArgumentException("Checkpoint checksum is invalid"); + var prior = snapshot(id).orElseThrow(); + if (completed < prior.completedUnits() || total < prior.totalUnits()) throw new IllegalArgumentException("Worker progress is not monotonic"); + var delta = completed - prior.completedUnits(); + var progressRate = delta > 0 ? recordProgressRate(id, completed, total, now) + : new ProgressRate(prior.throughput(), prior.etaSeconds()); + try (var statement = connection.prepareStatement(""" + UPDATE evidence_jobs SET completed_units=?,total_units=?,checkpoint_path=?,checkpoint_sha256=?, + last_heartbeat=?,throughput=?,eta_seconds=?,checkpoint_count=checkpoint_count+?, + lease_expires_at=?,updated_at=? WHERE id=? + """)) { + statement.setLong(1, completed); statement.setLong(2, total); + statement.setString(3, checkpoint == null ? "" : checkpoint.toAbsolutePath().normalize().toString()); + statement.setString(4, sha); statement.setString(5, now.toString()); statement.setDouble(6, progressRate.throughput()); + if (progressRate.etaSeconds() == null) statement.setNull(7, java.sql.Types.INTEGER); else statement.setLong(7, progressRate.etaSeconds()); + statement.setInt(8, delta > 0 ? 1 : 0); statement.setString(9, now.plus(lease).toString()); + statement.setString(10, now.toString()); statement.setString(11, id); statement.executeUpdate(); + } catch (SQLException error) { throw new IOException("Unable to renew evidence lease", error); } + return find(current.id()).orElseThrow(); + } + + private ProgressRate recordProgressRate(String id, long completed, long total, Instant now) throws IOException { + try (var insert = connection.prepareStatement("INSERT INTO evidence_progress_samples(job_id,completed_units,recorded_at) VALUES(?,?,?)")) { + insert.setString(1, id); insert.setLong(2, completed); insert.setString(3, now.toString()); insert.executeUpdate(); + } catch (SQLException error) { throw new IOException("Unable to record evidence throughput sample", error); } + try (var trim = connection.prepareStatement("DELETE FROM evidence_progress_samples WHERE job_id=? AND sequence NOT IN (SELECT sequence FROM evidence_progress_samples WHERE job_id=? ORDER BY sequence DESC LIMIT 5)")) { + trim.setString(1, id); trim.setString(2, id); trim.executeUpdate(); + } catch (SQLException error) { throw new IOException("Unable to bound evidence throughput samples", error); } + try (var statement = connection.prepareStatement("SELECT completed_units,recorded_at FROM evidence_progress_samples WHERE job_id=? ORDER BY sequence")) { + statement.setString(1, id); + try (var rows = statement.executeQuery()) { + long firstUnits = 0, lastUnits = 0; Instant firstAt = null, lastAt = null; int count = 0; + while (rows.next()) { + if (count++ == 0) { firstUnits = rows.getLong(1); firstAt = Instant.parse(rows.getString(2)); } + lastUnits = rows.getLong(1); lastAt = Instant.parse(rows.getString(2)); + } + if (count < 2 || lastUnits <= firstUnits) return new ProgressRate(0, null); + var seconds = Math.max(.001, Duration.between(firstAt, lastAt).toMillis() / 1000.0); + var rate = (lastUnits - firstUnits) / seconds; + return new ProgressRate(rate, Math.max(0, Math.round((total - completed) / rate))); + } + } catch (SQLException error) { throw new IOException("Unable to calculate evidence throughput", error); } } - public synchronized EvidenceJob checkpoint( - String id, String workerId, EvidenceJobState state, String stage, + private record ProgressRate(double throughput, Long etaSeconds) { } + + public synchronized EvidenceJob checkpoint(String id, String workerId, EvidenceJobState state, String stage, double progress, String detail, Instant now, Duration lease) throws IOException { - var current = find(id).orElseThrow(() -> new IllegalArgumentException("Evidence job was not found")); - if (!workerId.equals(current.leaseOwner()) || current.leaseExpiresAt().isBefore(now)) { - throw new IllegalStateException("Evidence job lease is not owned by this worker"); - } - requireTransition(current.state(), state); - var updated = new EvidenceJob(id, current.requestPath(), state, stage, progress, - current.retryCount(), current.cancelRequested(), state.terminal() ? "" : workerId, - state.terminal() ? Instant.EPOCH : now.plus(lease), bounded(detail), current.createdAt(), now); - updateChecked(updated); - return updated; + var current = requireOwned(id, workerId, now); requireTransition(current.state(), state); + var updated = new EvidenceJob(id, current.requestPath(), state, stage, progress, current.retryCount(), + current.cancelRequested(), state.terminal() ? "" : workerId, state.terminal() ? Instant.EPOCH : now.plus(lease), + bounded(detail), current.createdAt(), now); + updateChecked(updated); return updated; + } + + public synchronized EvidenceJob terminate(String id, String workerId, EvidenceJobState state, + String failureClass, String failureCode, String detail, Instant now, Duration lease) throws IOException { + if (!state.terminal()) throw new IllegalArgumentException("Terminal evidence state is required"); + var terminal = checkpoint(id, workerId, state, state.name().toLowerCase(), + find(id).orElseThrow().progress(), detail, now, lease); + executeUpdate("UPDATE evidence_jobs SET failure_class=?,failure_code=? WHERE id=?", + boundedCode(failureClass), boundedCode(failureCode), id); + return terminal; + } + + public synchronized void recordFinalArtifact(String id, String workerId, String sha256, Instant now) throws IOException { + requireOwned(id, workerId, now); + if (sha256 == null || !sha256.matches("[a-f0-9]{64}")) throw new IllegalArgumentException("Final artifact checksum is invalid"); + executeUpdate("UPDATE evidence_jobs SET final_artifact_sha256=?,updated_at=? WHERE id=?", sha256, now.toString(), id); } public synchronized void requestCancel(String id, Instant now) throws IOException { var current = find(id).orElseThrow(() -> new IllegalArgumentException("Evidence job was not found")); if (current.state().terminal()) return; - try (var statement = connection.prepareStatement( - "UPDATE evidence_jobs SET cancel_requested=1, detail=?, updated_at=? WHERE id=?")) { - statement.setString(1, "Cancellation requested by the local operator"); - statement.setString(2, now.toString()); - statement.setString(3, id); - statement.executeUpdate(); - } catch (SQLException error) { - throw new IOException("Unable to request evidence cancellation", error); + if (current.state() == EvidenceJobState.QUEUED) { + executeUpdate("UPDATE evidence_jobs SET state='CANCELLED',stage='cancelled',cancel_requested=1,detail=?,updated_at=? WHERE id=?", + "Cancelled before execution by the local operator", now.toString(), id); + } else { + executeUpdate("UPDATE evidence_jobs SET cancel_requested=1,detail=?,updated_at=? WHERE id=?", + "Cancellation requested by the local operator", now.toString(), id); } } - public synchronized EvidenceJob retryOrFail(String id, String workerId, String detail, Instant now) - throws IOException { + public synchronized EvidenceJob fail(String id, String workerId, String failureClass, String failureCode, + String detail, boolean transientIo, Instant now) throws IOException { var current = find(id).orElseThrow(() -> new IllegalArgumentException("Evidence job was not found")); if (!workerId.equals(current.leaseOwner())) throw new IllegalStateException("Evidence job lease is not owned"); - var retries = current.retryCount() + 1; - var state = retries <= 3 ? EvidenceJobState.QUEUED : EvidenceJobState.FAILED; - var next = new EvidenceJob(id, current.requestPath(), state, - state == EvidenceJobState.QUEUED ? "retrying" : "failed", current.progress(), - Math.min(3, retries), false, "", Instant.EPOCH, bounded(detail), current.createdAt(), now); - updateChecked(next); - return next; + var retries = transientIo ? current.retryCount() + 1 : current.retryCount(); + var retry = transientIo && retries <= RETRY_DELAYS_SECONDS.length; + var next = retry ? now.plusSeconds(RETRY_DELAYS_SECONDS[retries - 1]) : Instant.EPOCH; + var state = retry ? EvidenceJobState.QUEUED : EvidenceJobState.FAILED; + try (var statement = connection.prepareStatement(""" + UPDATE evidence_jobs SET state=?,stage=?,retry_count=?,lease_owner='',lease_expires_at=?,next_retry_at=?, + failure_class=?,failure_code=?,detail=?,updated_at=? WHERE id=? + """)) { + statement.setString(1, state.name()); statement.setString(2, retry ? "retrying" : "failed"); + statement.setInt(3, Math.min(3, retries)); statement.setString(4, Instant.EPOCH.toString()); + statement.setString(5, next.toString()); statement.setString(6, boundedCode(failureClass)); + statement.setString(7, boundedCode(failureCode)); statement.setString(8, bounded(detail)); + statement.setString(9, now.toString()); statement.setString(10, id); statement.executeUpdate(); + } catch (SQLException error) { throw new IOException("Unable to record evidence failure", error); } + return find(id).orElseThrow(); } - private void updateChecked(EvidenceJob job) throws IOException { - try { update(job); } catch (SQLException error) { throw new IOException("Unable to checkpoint evidence job", error); } + public EvidenceJob retryOrFail(String id, String workerId, String detail, Instant now) throws IOException { + return fail(id, workerId, "transient_io", "IO_TRANSIENT", detail, true, now); } - private void update(EvidenceJob job) throws SQLException { + public synchronized EvidenceJob retry(String id, Instant now) throws IOException { + var current = find(id).orElseThrow(() -> new IllegalArgumentException("Evidence job was not found")); + if (current.state() != EvidenceJobState.FAILED && current.state() != EvidenceJobState.CANCELLED) + throw new IllegalStateException("Only failed or cancelled jobs may be retried"); + var snapshot = snapshot(id).orElseThrow(); + var currentRequestSha = sha256(current.requestPath()); + if (!snapshot.requestSha256().isBlank() && !snapshot.requestSha256().equals(currentRequestSha)) + throw new IllegalStateException("Immutable job request changed; retry refused"); try (var statement = connection.prepareStatement(""" - UPDATE evidence_jobs SET request_path=?, state=?, stage=?, progress=?, retry_count=?, - cancel_requested=?, lease_owner=?, lease_expires_at=?, detail=?, created_at=?, updated_at=? - WHERE id=? + UPDATE evidence_jobs SET state='QUEUED',stage='queued',cancel_requested=0,lease_owner='',lease_expires_at=?, + next_retry_at=?,failure_class='',failure_code='',request_sha256=?,detail=?,updated_at=? WHERE id=? """)) { - statement.setString(1, job.requestPath().toString()); - statement.setString(2, job.state().name()); - statement.setString(3, job.stage()); - statement.setDouble(4, job.progress()); - statement.setInt(5, job.retryCount()); - statement.setInt(6, job.cancelRequested() ? 1 : 0); - statement.setString(7, job.leaseOwner()); - statement.setString(8, job.leaseExpiresAt().toString()); - statement.setString(9, job.detail()); - statement.setString(10, job.createdAt().toString()); - statement.setString(11, job.updatedAt().toString()); - statement.setString(12, job.id()); - if (statement.executeUpdate() != 1) throw new SQLException("Evidence job was not updated"); - } + statement.setString(1, Instant.EPOCH.toString()); statement.setString(2, now.toString()); + statement.setString(3, currentRequestSha); + statement.setString(4, "Requeued by local operator after immutable-input verification"); + statement.setString(5, now.toString()); statement.setString(6, id); statement.executeUpdate(); + } catch (SQLException error) { throw new IOException("Unable to retry evidence job", error); } + return find(id).orElseThrow(); } - private static void bind(java.sql.PreparedStatement statement, EvidenceJob job) throws SQLException { - statement.setString(1, job.id()); - statement.setString(2, job.requestPath().toString()); - statement.setString(3, job.state().name()); - statement.setString(4, job.stage()); - statement.setDouble(5, job.progress()); - statement.setInt(6, job.retryCount()); - statement.setInt(7, job.cancelRequested() ? 1 : 0); - statement.setString(8, job.leaseOwner()); - statement.setString(9, job.leaseExpiresAt().toString()); - statement.setString(10, job.detail()); - statement.setString(11, job.createdAt().toString()); - statement.setString(12, job.updatedAt().toString()); - } - - private static EvidenceJob read(ResultSet row) throws SQLException { - return new EvidenceJob(row.getString("id"), Path.of(row.getString("request_path")), - EvidenceJobState.valueOf(row.getString("state")), row.getString("stage"), - row.getDouble("progress"), row.getInt("retry_count"), - row.getInt("cancel_requested") != 0, row.getString("lease_owner"), - Instant.parse(row.getString("lease_expires_at")), row.getString("detail"), - Instant.parse(row.getString("created_at")), Instant.parse(row.getString("updated_at"))); - } - - private static void requireTransition(EvidenceJobState from, EvidenceJobState to) { - if (from.terminal()) throw new IllegalStateException("Evidence job is already terminal"); - var allowed = switch (from) { - case QUEUED -> java.util.Set.of(EvidenceJobState.VALIDATING, EvidenceJobState.CANCELLED); - case VALIDATING -> java.util.Set.of(EvidenceJobState.RUNNING, EvidenceJobState.ABSTAINED, - EvidenceJobState.UNSUPPORTED, EvidenceJobState.FAILED, EvidenceJobState.CANCELLED); - case RUNNING -> java.util.Set.of(EvidenceJobState.REFINING, EvidenceJobState.PACKAGING, - EvidenceJobState.ABSTAINED, EvidenceJobState.UNSUPPORTED, EvidenceJobState.FAILED, - EvidenceJobState.CANCELLED); - case REFINING -> java.util.Set.of(EvidenceJobState.PACKAGING, EvidenceJobState.ABSTAINED, - EvidenceJobState.FAILED, EvidenceJobState.CANCELLED); - case PACKAGING -> java.util.Set.of(EvidenceJobState.COMPLETED, EvidenceJobState.ABSTAINED, - EvidenceJobState.FAILED, EvidenceJobState.CANCELLED); - default -> java.util.Set.of(); - }; - if (!allowed.contains(to) && from != to) { - throw new IllegalStateException("Invalid evidence job transition: " + from + " -> " + to); - } + public synchronized boolean acceptingJobs() throws IOException { + try (var statement = connection.createStatement(); var rows = statement.executeQuery("SELECT accepting_jobs FROM runner_control WHERE singleton=1")) { + return rows.next() && rows.getInt(1) != 0; + } catch (SQLException error) { throw new IOException("Unable to read runner control", error); } } - private static String bounded(String value) { - if (value == null || value.isBlank()) return "No additional detail"; - return value.length() <= 500 ? value : value.substring(0, 500); + public synchronized void setAcceptingJobs(boolean accepting, Instant now) throws IOException { + try (var statement = connection.prepareStatement("UPDATE runner_control SET accepting_jobs=?,updated_at=? WHERE singleton=1")) { + statement.setInt(1, accepting ? 1 : 0); statement.setString(2, now.toString()); statement.executeUpdate(); + } catch (SQLException error) { throw new IOException("Unable to update runner control", error); } } - private void rollbackQuietly() { - try (var statement = connection.createStatement()) { - statement.execute("ROLLBACK"); - } catch (SQLException ignored) { } + private EvidenceJob requireOwned(String id, String workerId, Instant now) throws IOException { + var current = find(id).orElseThrow(() -> new IllegalArgumentException("Evidence job was not found")); + if (!workerId.equals(current.leaseOwner()) || current.leaseExpiresAt().isBefore(now)) throw new IllegalStateException("Evidence job lease is not owned by this worker"); + return current; } - private void commit() throws SQLException { - try (var statement = connection.createStatement()) { - statement.execute("COMMIT"); + private void executeUpdate(String sql, String... values) throws IOException { + try (var statement = connection.prepareStatement(sql)) { + for (var i = 0; i < values.length; i++) statement.setString(i + 1, values[i]); + statement.executeUpdate(); + } catch (SQLException error) { throw new IOException("Unable to update evidence queue", error); } + } + + private void updateChecked(EvidenceJob job) throws IOException { try { update(job); } catch (SQLException e) { throw new IOException("Unable to checkpoint evidence job", e); } } + private void update(EvidenceJob job) throws SQLException { + try (var statement = connection.prepareStatement("UPDATE evidence_jobs SET request_path=?,state=?,stage=?,progress=?,retry_count=?,cancel_requested=?,lease_owner=?,lease_expires_at=?,detail=?,created_at=?,updated_at=? WHERE id=?")) { + statement.setString(1, job.requestPath().toString()); statement.setString(2, job.state().name()); statement.setString(3, job.stage()); + statement.setDouble(4, job.progress()); statement.setInt(5, job.retryCount()); statement.setInt(6, job.cancelRequested() ? 1 : 0); + statement.setString(7, job.leaseOwner()); statement.setString(8, job.leaseExpiresAt().toString()); statement.setString(9, job.detail()); + statement.setString(10, job.createdAt().toString()); statement.setString(11, job.updatedAt().toString()); statement.setString(12, job.id()); + if (statement.executeUpdate() != 1) throw new SQLException("Evidence job was not updated"); } } - @Override - public synchronized void close() throws IOException { - try { connection.close(); } catch (SQLException error) { throw new IOException("Unable to close Evidence Mentor queue", error); } + private static void bindBase(java.sql.PreparedStatement s, EvidenceJob j) throws SQLException { + s.setString(1,j.id()); s.setString(2,j.requestPath().toString()); s.setString(3,j.state().name()); s.setString(4,j.stage()); + s.setDouble(5,j.progress()); s.setInt(6,j.retryCount()); s.setInt(7,j.cancelRequested()?1:0); s.setString(8,j.leaseOwner()); + s.setString(9,j.leaseExpiresAt().toString()); s.setString(10,j.detail()); s.setString(11,j.createdAt().toString()); s.setString(12,j.updatedAt().toString()); + } + + private static EvidenceJob read(ResultSet r) throws SQLException { + return new EvidenceJob(r.getString("id"),Path.of(r.getString("request_path")),EvidenceJobState.valueOf(r.getString("state")),r.getString("stage"), + r.getDouble("progress"),r.getInt("retry_count"),r.getInt("cancel_requested")!=0,r.getString("lease_owner"), + Instant.parse(r.getString("lease_expires_at")),r.getString("detail"),Instant.parse(r.getString("created_at")),Instant.parse(r.getString("updated_at"))); + } + + private static EvidenceJobSnapshot readSnapshot(ResultSet r) throws SQLException { + var checkpoint=r.getString("checkpoint_path"); var eta=r.getObject("eta_seconds"); + return new EvidenceJobSnapshot(r.getString("id"),EvidenceJobState.valueOf(r.getString("state")),r.getString("stage"),r.getDouble("progress"), + EvidenceExecutionLane.fromWire(r.getString("lane")),r.getLong("completed_units"),r.getLong("total_units"),checkpoint.isBlank()?null:Path.of(checkpoint), + r.getString("checkpoint_sha256"),Instant.parse(r.getString("last_heartbeat")),r.getDouble("throughput"),eta==null?null:r.getLong("eta_seconds"), + Instant.parse(r.getString("next_retry_at")),r.getInt("retry_count"),r.getInt("cancel_requested")!=0,r.getString("lease_owner"), + Instant.parse(r.getString("lease_expires_at")),r.getString("failure_class"),r.getString("failure_code"),r.getString("request_sha256"), + r.getString("pack_sha256"),r.getString("final_artifact_sha256"),r.getString("detail"), + Instant.parse(r.getString("created_at")),Instant.parse(r.getString("updated_at"))); + } + + private static void validateWorker(String id, Duration lease) { + if (id==null||!id.matches("[A-Za-z0-9._-]{1,120}")) throw new IllegalArgumentException("Evidence worker id is invalid"); + if (lease.isNegative()||lease.isZero()||lease.compareTo(Duration.ofMinutes(10))>0) throw new IllegalArgumentException("Evidence lease is invalid"); + } + private static void requireTransition(EvidenceJobState from,EvidenceJobState to) { + if(from.terminal()) throw new IllegalStateException("Evidence job is already terminal"); + var allowed=switch(from){ + case QUEUED->java.util.Set.of(EvidenceJobState.VALIDATING,EvidenceJobState.CANCELLED); + case VALIDATING->java.util.Set.of(EvidenceJobState.RUNNING,EvidenceJobState.ABSTAINED,EvidenceJobState.UNSUPPORTED,EvidenceJobState.FAILED,EvidenceJobState.CANCELLED); + case RUNNING->java.util.Set.of(EvidenceJobState.REFINING,EvidenceJobState.PACKAGING,EvidenceJobState.ABSTAINED,EvidenceJobState.UNSUPPORTED,EvidenceJobState.FAILED,EvidenceJobState.CANCELLED); + case REFINING->java.util.Set.of(EvidenceJobState.PACKAGING,EvidenceJobState.ABSTAINED,EvidenceJobState.FAILED,EvidenceJobState.CANCELLED); + case PACKAGING->java.util.Set.of(EvidenceJobState.COMPLETED,EvidenceJobState.ABSTAINED,EvidenceJobState.FAILED,EvidenceJobState.CANCELLED); + default->java.util.Set.of();}; + if(!allowed.contains(to)&&from!=to) throw new IllegalStateException("Invalid evidence job transition: "+from+" -> "+to); } + private static String bounded(String v){if(v==null||v.isBlank())return "No additional detail";return v.length()<=500?v:v.substring(0,500);} + private static String boundedCode(String v){return v!=null&&v.matches("[A-Za-z0-9._-]{1,80}")?v:"UNCLASSIFIED";} + private static String sha256(Path path)throws IOException{try(InputStream in=Files.newInputStream(path)){var d=MessageDigest.getInstance("SHA-256");var b=new byte[1024*1024];int n;while((n=in.read(b))>=0)d.update(b,0,n);return HexFormat.of().formatHex(d.digest());}catch(java.security.GeneralSecurityException e){throw new IOException("SHA-256 unavailable",e);}} + private void rollbackQuietly(){try(var s=connection.createStatement()){s.execute("ROLLBACK");}catch(SQLException ignored){}} + private void commit()throws SQLException{try(var s=connection.createStatement()){s.execute("COMMIT");}} + @Override public synchronized void close()throws IOException{try{connection.close();}catch(SQLException e){throw new IOException("Unable to close Evidence Mentor queue",e);}} } diff --git a/src/main/java/org/pathlab/forge/evidence/EvidenceJobSnapshot.java b/src/main/java/org/pathlab/forge/evidence/EvidenceJobSnapshot.java new file mode 100644 index 0000000..bf9f522 --- /dev/null +++ b/src/main/java/org/pathlab/forge/evidence/EvidenceJobSnapshot.java @@ -0,0 +1,15 @@ +package org.pathlab.forge.evidence; + +import java.nio.file.Path; +import java.time.Instant; + +/** Operations-only durable view. It deliberately excludes slide paths and scientific results. */ +public record EvidenceJobSnapshot( + String id, EvidenceJobState state, String stage, double progress, + EvidenceExecutionLane lane, long completedUnits, long totalUnits, + Path checkpointPath, String checkpointSha256, Instant lastHeartbeat, + double throughput, Long etaSeconds, Instant nextRetryAt, int retryCount, + boolean cancelRequested, String leaseOwner, Instant leaseExpiresAt, + String failureClass, String failureCode, String requestSha256, String packSha256, + String finalArtifactSha256, + String detail, Instant createdAt, Instant updatedAt) { } diff --git a/src/main/java/org/pathlab/forge/evidence/EvidenceMentorRunner.java b/src/main/java/org/pathlab/forge/evidence/EvidenceMentorRunner.java index aaaf55f..c51aaca 100644 --- a/src/main/java/org/pathlab/forge/evidence/EvidenceMentorRunner.java +++ b/src/main/java/org/pathlab/forge/evidence/EvidenceMentorRunner.java @@ -16,8 +16,11 @@ import java.time.Duration; import java.time.Instant; import java.util.Base64; +import java.util.LinkedHashMap; +import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.UUID; import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -26,235 +29,277 @@ /** Standalone authenticated loopback process for unattended Evidence Mentor jobs. */ public final class EvidenceMentorRunner implements AutoCloseable { private static final ObjectMapper JSON = new ObjectMapper(); + private static final String VERSION = "2.0.0"; + private static final Duration LEASE = Duration.ofSeconds(45); private final Path stateRoot; private final String token; private final EvidenceJobQueue queue; private final HttpServer server; - private final ScheduledExecutorService worker; + private final ScheduledExecutorService gpuWorker; + private final ScheduledExecutorService cpuWorker; private final java.util.concurrent.ExecutorService http; + private final DashboardSessions dashboardSessions = new DashboardSessions(); + private final Instant startedAt = Instant.now(); + private final String bootId = UUID.randomUUID().toString(); + private final Path endpointPath; + private final String dashboardHtml; - private EvidenceMentorRunner(Path stateRoot, int port, String token, boolean workerEnabled) - throws IOException { + private EvidenceMentorRunner(Path stateRoot, int port, String token, boolean workerEnabled) throws IOException { this.stateRoot = stateRoot.toAbsolutePath().normalize(); this.token = token; require(token != null && token.length() >= 32, "Loopback token is too short"); Files.createDirectories(this.stateRoot); queue = new EvidenceJobQueue(this.stateRoot.resolve("jobs.sqlite3")); - server = HttpServer.create( - new InetSocketAddress(InetAddress.getByName("127.0.0.1"), port), 16); - server.createContext("/health", this::handle); - server.createContext("/v1/status", this::handle); - server.createContext("/v1/jobs", this::handle); - http = Executors.newFixedThreadPool(2, runnable -> { - var thread = new Thread(runnable, "pathlab-evidence-ipc"); - thread.setDaemon(true); - return thread; - }); + dashboardHtml = readResource("/evidence-dashboard/index.html"); + server = HttpServer.create(new InetSocketAddress(InetAddress.getByName("127.0.0.1"), port), 32); + server.createContext("/", this::handle); + http = Executors.newFixedThreadPool(4, runnable -> daemon(runnable, "pathlab-evidence-ipc", Thread.NORM_PRIORITY)); server.setExecutor(http); server.start(); - worker = Executors.newSingleThreadScheduledExecutor(runnable -> { - var thread = new Thread(runnable, "pathlab-evidence-worker"); - thread.setDaemon(true); - thread.setPriority(Thread.MIN_PRIORITY); - return thread; - }); - if (workerEnabled) worker.scheduleWithFixedDelay(this::processOne, 0, 1, TimeUnit.SECONDS); + endpointPath = this.stateRoot.resolve("endpoint.json"); + publishEndpoint(); + gpuWorker = Executors.newSingleThreadScheduledExecutor(r -> daemon(r, "pathlab-evidence-gpu", Thread.NORM_PRIORITY)); + cpuWorker = Executors.newSingleThreadScheduledExecutor(r -> daemon(r, "pathlab-evidence-cpu-io", Thread.MIN_PRIORITY)); + if (workerEnabled) { + gpuWorker.scheduleWithFixedDelay(() -> processOne(EvidenceExecutionLane.GPU), 0, 1, TimeUnit.SECONDS); + cpuWorker.scheduleWithFixedDelay(() -> processOne(EvidenceExecutionLane.CPU_IO), 0, 1, TimeUnit.SECONDS); + } } - public static EvidenceMentorRunner start(Path stateRoot, int port, String token, boolean worker) - throws IOException { + public static EvidenceMentorRunner start(Path stateRoot, int port, String token, boolean worker) throws IOException { return new EvidenceMentorRunner(stateRoot, port, token, worker); } - public URI uri(String path) { - return URI.create("http://127.0.0.1:" + server.getAddress().getPort() + path); - } - - Optional find(String id) throws IOException { - return queue.find(id); - } + public URI uri(String path) { return URI.create(origin() + path); } + Optional find(String id) throws IOException { return queue.find(id); } private void handle(HttpExchange exchange) throws IOException { try { - if (!exchange.getRemoteAddress().getAddress().isLoopbackAddress()) { - respond(exchange, 403, java.util.Map.of("error", "loopback_required")); - return; - } - if (!authorized(exchange)) { - respond(exchange, 401, java.util.Map.of("error", "authentication_required")); - return; - } - var path = exchange.getRequestURI().getPath(); - if ("/health".equals(path) && "GET".equals(exchange.getRequestMethod())) { - respond(exchange, 200, java.util.Map.of( - "schema", "pathlab.evidence-runner-status/1", "status", "ready")); - return; - } - if ("/v1/status".equals(path) && "GET".equals(exchange.getRequestMethod())) { - var runtime = Runtime.getRuntime(); - var fileStore = Files.getFileStore(stateRoot); - var status = new java.util.LinkedHashMap(); - status.put("schema", "pathlab.evidence-runner-status/1"); - status.put("status", "ready"); - status.put("networkDisabledForAnalysis", true); - status.put("gpuConcurrency", 1); - status.put("cpuIoConcurrency", 1); - status.put("processMemoryUsedBytes", runtime.totalMemory() - runtime.freeMemory()); - status.put("processMemoryLimitBytes", 16L * 1024 * 1024 * 1024); - status.put("vramLimitMiB", 4608); - status.put("diskUsableBytes", fileStore.getUsableSpace()); - status.put("quota", new EvidenceQuotaManager(stateRoot).snapshot().buckets()); - status.put("updatedAt", Instant.now().toString()); - respond(exchange, 200, status); - return; - } - if ("/v1/jobs".equals(path) && "POST".equals(exchange.getRequestMethod())) { - submit(exchange); - return; - } - var prefix = "/v1/jobs/"; - if (path.startsWith(prefix)) { - var id = path.substring(prefix.length()); - if (!id.matches("[A-Za-z0-9._-]{1,120}")) { - respond(exchange, 404, java.util.Map.of("error", "job_not_found")); - } else if ("GET".equals(exchange.getRequestMethod())) { - var job = queue.find(id); - if (job.isEmpty()) respond(exchange, 404, java.util.Map.of("error", "job_not_found")); - else respond(exchange, 200, jobJson(job.get())); - } else if ("DELETE".equals(exchange.getRequestMethod())) { - queue.requestCancel(id, Instant.now()); - exchange.sendResponseHeaders(204, -1); - } else { - respond(exchange, 405, java.util.Map.of("error", "method_not_allowed")); - } - return; - } - respond(exchange, 404, java.util.Map.of("error", "not_found")); + if (!exchange.getRemoteAddress().getAddress().isLoopbackAddress()) returnJson(exchange, 403, Map.of("error", "loopback_required")); + else if (!validHost(exchange)) returnJson(exchange, 403, Map.of("error", "host_mismatch")); + else if (!validOriginWhenPresent(exchange)) returnJson(exchange, 403, Map.of("error", "origin_mismatch")); + else route(exchange); + } catch (SecurityException error) { + returnJson(exchange, 403, Map.of("error", "request_forbidden", "detail", safe(error.getMessage()))); } catch (IllegalArgumentException error) { - respond(exchange, 422, java.util.Map.of("error", "invalid_request", "detail", error.getMessage())); + returnJson(exchange, 422, Map.of("error", "invalid_request", "detail", safe(error.getMessage()))); + } catch (IllegalStateException error) { + returnJson(exchange, 409, Map.of("error", "invalid_state", "detail", safe(error.getMessage()))); } catch (Exception error) { - respond(exchange, 500, java.util.Map.of("error", "runner_error")); - } finally { - exchange.close(); + returnJson(exchange, 500, Map.of("error", "runner_error")); + } finally { exchange.close(); } + } + + private void route(HttpExchange exchange) throws IOException { + var path = exchange.getRequestURI().getPath(); + var method = exchange.getRequestMethod(); + if ("/dashboard/".equals(path) && "GET".equals(method)) { returnHtml(exchange); return; } + if ("/v1/dashboard-session/exchange".equals(path) && "POST".equals(method)) { exchangeDashboardSession(exchange); return; } + if (!authenticated(exchange)) { returnJson(exchange, 401, Map.of("error", "authentication_required")); return; } + if ("/health".equals(path) && "GET".equals(method)) { + returnJson(exchange, 200, Map.of("schema", "pathlab.evidence-runner-status/2", "status", "ready", "bootId", bootId)); return; + } + if ("/v1/status".equals(path) && "GET".equals(method)) { status(exchange); return; } + if ("/v1/jobs".equals(path) && "POST".equals(method)) { submit(exchange); return; } + if ("/v1/jobs".equals(path) && "GET".equals(method)) { listJobs(exchange); return; } + if ("/v1/dashboard-sessions".equals(path) && "POST".equals(method)) { + if (!bearerAuthorized(exchange)) { returnJson(exchange, 401, Map.of("error", "bearer_required")); return; } + returnJson(exchange, 201, Map.of("code", dashboardSessions.create(Instant.now()), "expiresInSeconds", 60)); return; } + if ("/v1/control/pause".equals(path) && "POST".equals(method)) { mutate(exchange); queue.setAcceptingJobs(false, Instant.now()); control(exchange); return; } + if ("/v1/control/resume".equals(path) && "POST".equals(method)) { mutate(exchange); queue.setAcceptingJobs(true, Instant.now()); control(exchange); return; } + if (path.startsWith("/v1/jobs/")) { jobRoute(exchange, path.substring("/v1/jobs/".length())); return; } + returnJson(exchange, 404, Map.of("error", "not_found")); + } + + private void jobRoute(HttpExchange exchange, String suffix) throws IOException { + var parts = suffix.split("/", -1); var id = parts[0]; + if (!id.matches("[A-Za-z0-9._-]{1,120}")) { returnJson(exchange, 404, Map.of("error", "job_not_found")); return; } + if (parts.length == 1 && "GET".equals(exchange.getRequestMethod())) { + var job = queue.snapshot(id); if (job.isEmpty()) returnJson(exchange, 404, Map.of("error", "job_not_found")); else returnJson(exchange, 200, jobJson(job.get())); return; + } + var cancel = (parts.length == 1 && "DELETE".equals(exchange.getRequestMethod())) + || (parts.length == 2 && "cancel".equals(parts[1]) && "POST".equals(exchange.getRequestMethod())); + if (cancel) { mutate(exchange); queue.requestCancel(id, Instant.now()); returnJson(exchange, 202, Map.of("status", "cancellation_requested")); return; } + if (parts.length == 2 && "retry".equals(parts[1]) && "POST".equals(exchange.getRequestMethod())) { + mutate(exchange); returnJson(exchange, 202, jobJson(queue.snapshot(queue.retry(id, Instant.now()).id()).orElseThrow())); return; + } + returnJson(exchange, 405, Map.of("error", "method_not_allowed")); + } + + private void exchangeDashboardSession(HttpExchange exchange) throws IOException { + secure(origin().equals(exchange.getRequestHeaders().getFirst("Origin")), "Exact loopback Origin is required"); + var body = jsonBody(exchange, 1024); require(fieldNames(body).equals(Set.of("code")), "Dashboard exchange fields are invalid"); + var session = dashboardSessions.exchange(text(body, "code"), Instant.now()); + if (session.isEmpty()) { returnJson(exchange, 401, Map.of("error", "invalid_or_expired_code")); return; } + exchange.getResponseHeaders().add("Set-Cookie", "PathLabEvidenceSession=" + session.get().token() + "; HttpOnly; SameSite=Strict; Path=/"); + returnJson(exchange, 200, Map.of("csrfToken", session.get().csrfToken(), "expiresAt", session.get().expiresAt().toString())); + } + + private void mutate(HttpExchange exchange) { + if (bearerAuthorized(exchange)) return; + var session = session(exchange).orElseThrow(() -> new IllegalArgumentException("Dashboard session is required")); + secure(origin().equals(exchange.getRequestHeaders().getFirst("Origin")), "Exact loopback Origin is required"); + var supplied = exchange.getRequestHeaders().getFirst("X-PathLab-CSRF"); + secure(supplied != null && MessageDigest.isEqual(session.csrfToken().getBytes(StandardCharsets.UTF_8), supplied.getBytes(StandardCharsets.UTF_8)), "CSRF token is invalid"); + } + + private void status(HttpExchange exchange) throws IOException { + var jobs = queue.list(null, null, 200); + long queued = jobs.stream().filter(j -> j.state() == EvidenceJobState.QUEUED).count(); + long active = jobs.stream().filter(j -> !j.state().terminal() && j.state() != EvidenceJobState.QUEUED).count(); + var runtime = Runtime.getRuntime(); var telemetry = new LinkedHashMap(); + telemetry.put("processMemoryUsedBytes", runtime.totalMemory() - runtime.freeMemory()); + telemetry.put("processMemoryLimitBytes", 16L * 1024 * 1024 * 1024); + var operatingSystem = (com.sun.management.OperatingSystemMXBean) + java.lang.management.ManagementFactory.getOperatingSystemMXBean(); + telemetry.put("serviceCpuPercent", Math.max(0, operatingSystem.getProcessCpuLoad() * 100)); + telemetry.put("processCommittedVirtualMemoryBytes", operatingSystem.getCommittedVirtualMemorySize()); + telemetry.put("serviceCpuSeconds", ProcessHandle.current().info().totalCpuDuration() + .map(Duration::toMillis).orElse(0L) / 1000.0); + telemetry.put("workers", ProcessHandle.current().children().map(process -> Map.of( + "pid", process.pid(), "cpuSeconds", process.info().totalCpuDuration() + .map(Duration::toMillis).orElse(0L) / 1000.0)).toList()); + telemetry.put("diskUsableBytes", Files.getFileStore(stateRoot).getUsableSpace()); + telemetry.put("gpu", gpuTelemetry()); telemetry.put("quota", new EvidenceQuotaManager(stateRoot).snapshot().buckets()); + var status = new LinkedHashMap(); + status.put("schema", "pathlab.evidence-runner-status/2"); status.put("status", "ready"); + status.put("serviceVersion", VERSION); status.put("bootId", bootId); status.put("startedAt", startedAt.toString()); + status.put("heartbeat", Instant.now().toString()); status.put("networkDisabledForAnalysis", true); + status.put("acceptingJobs", queue.acceptingJobs()); + status.put("queue", Map.of("queued", queued, "active", active, + "gpu", jobs.stream().filter(j -> j.lane() == EvidenceExecutionLane.GPU && !j.state().terminal()).count(), + "cpuIo", jobs.stream().filter(j -> j.lane() == EvidenceExecutionLane.CPU_IO && !j.state().terminal()).count())); + jobs.stream().filter(j -> !j.state().terminal()).findFirst().ifPresent(job -> status.put("leadingJob", Map.of( + "id", job.id(), "state", job.state().name().toLowerCase(), "stage", job.stage(), + "progress", job.progress(), "lane", job.lane().wire()))); + status.put("gpuConcurrency", 1); status.put("cpuIoConcurrency", 1); status.put("vramLimitMiB", 4608); + status.put("telemetry", telemetry); + status.put("processMemoryUsedBytes", telemetry.get("processMemoryUsedBytes")); + status.put("processMemoryLimitBytes", telemetry.get("processMemoryLimitBytes")); + status.put("diskUsableBytes", telemetry.get("diskUsableBytes")); status.put("quota", telemetry.get("quota")); + returnJsonCached(exchange, status); + } + + private void listJobs(HttpExchange exchange) throws IOException { + var query = query(exchange.getRequestURI().getRawQuery()); + var allowed = Set.of("state", "lane", "limit"); require(allowed.containsAll(query.keySet()), "Unknown job-list filter"); + EvidenceJobState state = query.containsKey("state") ? EvidenceJobState.valueOf(query.get("state").toUpperCase().replace('-', '_')) : null; + EvidenceExecutionLane lane = query.containsKey("lane") ? EvidenceExecutionLane.fromWire(query.get("lane")) : null; + var limit = query.containsKey("limit") ? Integer.parseInt(query.get("limit")) : 50; + returnJsonCached(exchange, Map.of("schema", "pathlab.evidence-job-list/2", "jobs", queue.list(state, lane, limit).stream().map(EvidenceMentorRunner::jobJson).toList())); } private void submit(HttpExchange exchange) throws IOException { - var raw = exchange.getRequestBody().readNBytes(65_537); - require(raw.length <= 65_536, "Job request is too large"); - JsonNode value = JSON.readTree(raw); - require(value != null && value.isObject() && fieldNames(value).equals(Set.of("id", "requestPath")), - "Job submission fields are invalid"); - var id = text(value, "id"); - require(id.matches("[A-Za-z0-9._-]{1,120}"), "Job id is invalid"); - var job = queue.submit(id, Path.of(text(value, "requestPath")), Instant.now()); - respond(exchange, 202, jobJson(job)); + var value = jsonBody(exchange, 65_536); require(fieldNames(value).equals(Set.of("id", "requestPath")), "Job submission fields are invalid"); + var id = text(value, "id"); require(id.matches("[A-Za-z0-9._-]{1,120}"), "Job id is invalid"); + var requestPath = Path.of(text(value, "requestPath")); + var plan = EvidenceJobProcessor.executionPlan(requestPath); + var job = queue.submit(id, requestPath, plan.lane(), plan.packSha256(), Instant.now()); + returnJson(exchange, 202, jobJson(queue.snapshot(job.id()).orElseThrow())); } - private boolean authorized(HttpExchange exchange) { - var expected = ("Bearer " + token).getBytes(StandardCharsets.UTF_8); - var supplied = exchange.getRequestHeaders().getFirst("Authorization"); - return supplied != null && MessageDigest.isEqual( - expected, supplied.getBytes(StandardCharsets.UTF_8)); + private void control(HttpExchange exchange) throws IOException { returnJson(exchange, 200, Map.of("acceptingJobs", queue.acceptingJobs())); } + + private void processOne(EvidenceExecutionLane lane) { + if (lane == EvidenceExecutionLane.GPU) { + try (var channel = java.nio.channels.FileChannel.open(stateRoot.resolve("gpu.lock"), + java.nio.file.StandardOpenOption.CREATE, java.nio.file.StandardOpenOption.WRITE); + var lock = channel.tryLock()) { + if (lock != null) processClaimed(lane); + } catch (java.nio.channels.OverlappingFileLockException ignored) { + // Another runner thread or process owns the single GPU lane. + } catch (IOException ignored) { + // Failure to establish the cross-process mutex fails closed for this iteration. + } + return; + } + processClaimed(lane); } - private void processOne() { - var workerId = "cpu-" + ProcessHandle.current().pid(); + private void processClaimed(EvidenceExecutionLane lane) { + var workerId = lane.wire() + "-" + ProcessHandle.current().pid(); try { - var claimed = queue.claimNext(workerId, Instant.now(), Duration.ofMinutes(2)); - if (claimed.isEmpty()) return; - try { - new EvidenceJobProcessor(queue, stateRoot).process(claimed.get(), workerId, Instant.now()); - } catch (EvidenceJobProcessor.CancellationException ignored) { - // Cancellation was persisted at a checkpoint. - } catch (IllegalArgumentException permanent) { + var claimed = queue.claimNext(lane, workerId, Instant.now(), LEASE); if (claimed.isEmpty()) return; + try { new EvidenceJobProcessor(queue, stateRoot).process(claimed.get(), workerId, Instant.now()); } + catch (EvidenceJobProcessor.CancellationException ignored) { } + catch (ExternalModelWorker.ResourceLimitException resourceFailure) { var current = queue.find(claimed.get().id()).orElseThrow(); - var state = permanent.getMessage().contains("not installed") - || permanent.getMessage().contains("unsupported") - ? EvidenceJobState.UNSUPPORTED : EvidenceJobState.FAILED; - queue.checkpoint(current.id(), workerId, state, state.name().toLowerCase(), - current.progress(), permanent.getMessage(), Instant.now(), Duration.ofMinutes(2)); + if (!current.state().terminal()) queue.fail(current.id(), workerId, "resource", "RESOURCE_LIMIT", + resourceFailure.getMessage(), false, Instant.now()); + } + catch (IllegalArgumentException permanent) { + var current = queue.find(claimed.get().id()).orElseThrow(); + var state = permanent.getMessage().contains("not installed") || permanent.getMessage().contains("unsupported") ? EvidenceJobState.UNSUPPORTED : EvidenceJobState.FAILED; + queue.terminate(current.id(), workerId, state, "permanent", permanentCode(permanent.getMessage()), + permanent.getMessage(), Instant.now(), LEASE); } catch (IOException transientFailure) { var current = queue.find(claimed.get().id()).orElseThrow(); - if (!current.state().terminal()) { - queue.retryOrFail(current.id(), workerId, transientFailure.getMessage(), Instant.now()); - } + if (!current.state().terminal()) queue.fail(current.id(), workerId, "transient_io", "IO_TRANSIENT", transientFailure.getMessage(), true, Instant.now()); } - } catch (Exception ignored) { - // Durable lease expiry permits recovery on next iteration or restart. - } + } catch (Exception ignored) { /* lease expiry is the durable recovery path */ } } - private static java.util.Map jobJson(EvidenceJob job) { - return java.util.Map.of( - "id", job.id(), "state", job.state().name().toLowerCase(), - "stage", job.stage(), "progress", job.progress(), - "retryCount", job.retryCount(), "cancelRequested", job.cancelRequested(), - "detail", job.detail(), "updatedAt", job.updatedAt().toString()); + private static Map jobJson(EvidenceJobSnapshot job) { + var result = new LinkedHashMap(); + result.put("schema", "pathlab.evidence-job/2"); result.put("id",job.id()); result.put("state",job.state().name().toLowerCase()); + result.put("stage",job.stage()); result.put("progress",job.progress()); result.put("lane",job.lane().wire()); + result.put("completedUnits",job.completedUnits()); result.put("totalUnits",job.totalUnits()); result.put("throughput",job.throughput()); + result.put("etaSeconds",job.etaSeconds()); result.put("lastHeartbeat",job.lastHeartbeat().toString()); + result.put("leaseExpiresAt",job.leaseExpiresAt().toString()); result.put("retryCount",job.retryCount()); + result.put("nextRetryAt",job.nextRetryAt().toString()); result.put("cancelRequested",job.cancelRequested()); + result.put("failureClass",job.failureClass()); result.put("failureCode",job.failureCode()); + result.put("requestSha256",job.requestSha256()); result.put("checkpointSha256",job.checkpointSha256()); + result.put("packSha256",job.packSha256()); result.put("finalArtifactSha256",job.finalArtifactSha256()); + result.put("detail",job.detail()); result.put("updatedAt",job.updatedAt().toString()); return result; } - private static Set fieldNames(JsonNode value) { - var result = new java.util.HashSet(); - value.fieldNames().forEachRemaining(result::add); - return result; + private boolean authenticated(HttpExchange exchange) { return bearerAuthorized(exchange) || session(exchange).isPresent(); } + private boolean bearerAuthorized(HttpExchange exchange) { + var supplied=exchange.getRequestHeaders().getFirst("Authorization"); if(supplied==null)return false; + return MessageDigest.isEqual(("Bearer "+token).getBytes(StandardCharsets.UTF_8),supplied.getBytes(StandardCharsets.UTF_8)); } - - private static String text(JsonNode value, String field) { - var node = value.path(field); - require(node.isTextual() && !node.textValue().isBlank(), "Job field is invalid: " + field); - return node.textValue(); + private Optional session(HttpExchange exchange) { + var cookie=exchange.getRequestHeaders().getFirst("Cookie"); if(cookie==null)return Optional.empty(); + for(var part:cookie.split(";")){var item=part.trim();if(item.startsWith("PathLabEvidenceSession="))return dashboardSessions.find(item.substring(item.indexOf('=')+1),Instant.now());} + return Optional.empty(); } + private boolean validHost(HttpExchange exchange){return ("127.0.0.1:"+server.getAddress().getPort()).equals(exchange.getRequestHeaders().getFirst("Host"));} + private boolean validOriginWhenPresent(HttpExchange exchange){var origin=exchange.getRequestHeaders().getFirst("Origin");return origin==null||origin().equals(origin);} + private String origin(){return "http://127.0.0.1:"+server.getAddress().getPort();} - private static void respond(HttpExchange exchange, int status, Object body) throws IOException { - if (exchange.getResponseCode() != -1) return; - var payload = JSON.writeValueAsBytes(body); - exchange.getResponseHeaders().set("Content-Type", "application/json; charset=utf-8"); - exchange.getResponseHeaders().set("Cache-Control", "no-store"); - exchange.sendResponseHeaders(status, payload.length); - exchange.getResponseBody().write(payload); + private void publishEndpoint() throws IOException { + var body=JSON.writeValueAsBytes(Map.of("schema","pathlab.runner-endpoint/1","port",server.getAddress().getPort(),"serviceVersion",VERSION, + "bootId",bootId,"pid",ProcessHandle.current().pid(),"startedAt",startedAt.toString())); + var partial=endpointPath.resolveSibling("endpoint.json.partial"); Files.write(partial,body); + try{Files.move(partial,endpointPath,java.nio.file.StandardCopyOption.REPLACE_EXISTING,java.nio.file.StandardCopyOption.ATOMIC_MOVE);} + catch(java.nio.file.AtomicMoveNotSupportedException ignored){Files.move(partial,endpointPath,java.nio.file.StandardCopyOption.REPLACE_EXISTING);} } - private static void require(boolean condition, String message) { - if (!condition) throw new IllegalArgumentException(message); - } + private static Map query(String raw){var result=new java.util.HashMap();if(raw==null||raw.isBlank())return result;for(var part:raw.split("&")){var pair=part.split("=",2);require(pair.length==2&&!pair[0].isBlank(),"Invalid query");result.put(pair[0],java.net.URLDecoder.decode(pair[1],StandardCharsets.UTF_8));}return result;} + private static JsonNode jsonBody(HttpExchange exchange,int limit)throws IOException{var raw=exchange.getRequestBody().readNBytes(limit+1);require(raw.length<=limit,"Request body is too large");var value=JSON.readTree(raw);require(value!=null&&value.isObject(),"JSON object required");return value;} + private static Set fieldNames(JsonNode value){var result=new java.util.HashSet();value.fieldNames().forEachRemaining(result::add);return result;} + private static String text(JsonNode value,String field){var node=value.path(field);require(node.isTextual()&&!node.textValue().isBlank(),"Field is invalid: "+field);return node.textValue();} + private static void returnJson(HttpExchange exchange,int status,Object body)throws IOException{if(exchange.getResponseCode()!=-1)return;var payload=JSON.writeValueAsBytes(body);exchange.getResponseHeaders().set("Content-Type","application/json; charset=utf-8");exchange.getResponseHeaders().set("Cache-Control","no-store");exchange.getResponseHeaders().set("X-Content-Type-Options","nosniff");exchange.sendResponseHeaders(status,payload.length);exchange.getResponseBody().write(payload);} + private static void returnJsonCached(HttpExchange exchange,Object body)throws IOException{var payload=JSON.writeValueAsBytes(body);var etag="\""+digest(payload)+"\"";exchange.getResponseHeaders().set("ETag",etag);exchange.getResponseHeaders().set("Cache-Control","private, max-age=0, must-revalidate");if(etag.equals(exchange.getRequestHeaders().getFirst("If-None-Match"))){exchange.sendResponseHeaders(304,-1);return;}exchange.getResponseHeaders().set("Content-Type","application/json; charset=utf-8");exchange.getResponseHeaders().set("X-Content-Type-Options","nosniff");exchange.sendResponseHeaders(200,payload.length);exchange.getResponseBody().write(payload);} + private void returnHtml(HttpExchange exchange)throws IOException{var payload=dashboardHtml.getBytes(StandardCharsets.UTF_8);exchange.getResponseHeaders().set("Content-Type","text/html; charset=utf-8");exchange.getResponseHeaders().set("Cache-Control","no-store");exchange.getResponseHeaders().set("Content-Security-Policy","default-src 'none'; style-src 'unsafe-inline'; script-src 'unsafe-inline'; connect-src 'self'; img-src 'self'; base-uri 'none'; frame-ancestors 'none'");exchange.sendResponseHeaders(200,payload.length);exchange.getResponseBody().write(payload);} + private static String readResource(String name)throws IOException{try(var input=EvidenceMentorRunner.class.getResourceAsStream(name)){if(input==null)throw new IOException("Dashboard resource missing");return new String(input.readAllBytes(),StandardCharsets.UTF_8);}} + private static Map gpuTelemetry(){try{var process=new ProcessBuilder("nvidia-smi","--query-gpu=utilization.gpu,memory.used,memory.total","--format=csv,noheader,nounits").redirectErrorStream(true).start();if(!process.waitFor(2,TimeUnit.SECONDS)){process.destroyForcibly();return Map.of("available",false);}var parts=new String(process.getInputStream().readAllBytes(),StandardCharsets.UTF_8).trim().split(",");if(process.exitValue()!=0||parts.length<3)return Map.of("available",false);return Map.of("available",true,"utilizationPercent",Integer.parseInt(parts[0].trim()),"vramUsedMiB",Integer.parseInt(parts[1].trim()),"vramTotalMiB",Integer.parseInt(parts[2].trim()));}catch(Exception ignored){return Map.of("available",false);}} + private static Thread daemon(Runnable runnable,String name,int priority){var thread=new Thread(runnable,name);thread.setDaemon(true);thread.setPriority(priority);return thread;} + private static String safe(String value){return value==null?"Request was refused":value;} + private static String digest(byte[] value){try{return java.util.HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(value));}catch(java.security.GeneralSecurityException impossible){throw new IllegalStateException("SHA-256 unavailable",impossible);}} + private static String permanentCode(String detail){var value=detail==null?"":detail.toLowerCase();if(value.contains("license")||value.contains("rights"))return "RIGHTS_OR_LICENSE";if(value.contains("checksum"))return "CHECKSUM_MISMATCH";if(value.contains("stain"))return "STAIN_UNSUPPORTED";if(value.contains("schema"))return "SCHEMA_UNSUPPORTED";if(value.contains("not installed"))return "PACK_NOT_INSTALLED";if(value.contains("unsupported"))return "CAPABILITY_UNSUPPORTED";return "VALIDATION_FAILED";} + private static void require(boolean condition,String message){if(!condition)throw new IllegalArgumentException(message);} + private static void secure(boolean condition,String message){if(!condition)throw new SecurityException(message);} - @Override - public void close() throws IOException { - server.stop(0); - worker.shutdownNow(); - http.shutdownNow(); - queue.close(); - } - - public static void main(String[] args) throws Exception { - var state = defaultStateRoot(); - var port = 8765; - for (var index = 0; index < args.length; index++) { - if ("--state".equals(args[index]) && index + 1 < args.length) state = Path.of(args[++index]); - else if ("--port".equals(args[index]) && index + 1 < args.length) port = Integer.parseInt(args[++index]); - else throw new IllegalArgumentException("Usage: EvidenceMentorRunner [--state PATH] [--port PORT]"); - } - Files.createDirectories(state); - var tokenPath = state.resolve("ipc-token"); - if (!Files.exists(tokenPath)) { - var bytes = new byte[32]; - new SecureRandom().nextBytes(bytes); - Files.writeString(tokenPath, Base64.getUrlEncoder().withoutPadding().encodeToString(bytes)); - } - var runner = start(state, port, Files.readString(tokenPath).trim(), true); - Runtime.getRuntime().addShutdownHook(new Thread(() -> { - try { runner.close(); } catch (IOException ignored) { } - })); - new CountDownLatch(1).await(); - } + @Override public void close()throws IOException{server.stop(0);gpuWorker.shutdownNow();cpuWorker.shutdownNow();http.shutdownNow();try{if(Files.isRegularFile(endpointPath)){var endpoint=JSON.readTree(endpointPath.toFile());if(bootId.equals(endpoint.path("bootId").asText()))Files.deleteIfExists(endpointPath);}}finally{queue.close();}} - public static Path defaultStateRoot() { - var configured = System.getenv("PATHLAB_EVIDENCE_STATE"); - if (configured != null && !configured.isBlank()) return Path.of(configured); - var dataDrive = Path.of("D:\\PathLabData"); - if (Files.isDirectory(dataDrive)) return dataDrive.resolve("EvidenceMentor/state"); - var local = System.getenv("LOCALAPPDATA"); - return Path.of(local == null ? System.getProperty("user.home") : local) - .resolve("PathLab/EvidenceMentor/state"); + public static void main(String[] args)throws Exception{ + var state=defaultStateRoot();var port=0; + for(var i=0;i{try{runner.close();}catch(IOException ignored){}}));new CountDownLatch(1).await(); } + public static Path defaultStateRoot(){var configured=System.getenv("PATHLAB_EVIDENCE_STATE");if(configured!=null&&!configured.isBlank())return Path.of(configured);var data=Path.of("D:\\PathLabData");if(Files.isDirectory(data))return data.resolve("EvidenceMentor/state");var local=System.getenv("LOCALAPPDATA");return Path.of(local==null?System.getProperty("user.home"):local).resolve("PathLab/EvidenceMentor/state");} } diff --git a/src/main/java/org/pathlab/forge/evidence/ExternalModelWorker.java b/src/main/java/org/pathlab/forge/evidence/ExternalModelWorker.java index deaa5c1..b2c6902 100644 --- a/src/main/java/org/pathlab/forge/evidence/ExternalModelWorker.java +++ b/src/main/java/org/pathlab/forge/evidence/ExternalModelWorker.java @@ -6,6 +6,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.security.MessageDigest; +import java.time.Instant; import java.util.HexFormat; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -14,6 +15,7 @@ /** Executes an installed, checksum-pinned model worker through a bounded file protocol. */ public final class ExternalModelWorker { public static final String RESULT_SCHEMA = "pathlab.model-worker-result/1"; + public static final String PROGRESS_SCHEMA = "pathlab.model-worker-progress/1"; private static final ObjectMapper JSON = new ObjectMapper(); private static final Set STATUSES = Set.of("completed", "unsupported", "not_evaluable"); private final Path stateRoot; @@ -23,6 +25,16 @@ public ExternalModelWorker(Path stateRoot) { } public JsonNode execute(EvidencePackManifest pack, Path request, String jobId) throws IOException { + return execute(pack, request, jobId, null, progress -> { }); + } + + public JsonNode execute(EvidencePackManifest pack, Path request, String jobId, + ProgressListener listener) throws IOException { + return execute(pack, request, jobId, null, listener); + } + + public JsonNode execute(EvidencePackManifest pack, Path request, String jobId, Path resumeCheckpoint, + ProgressListener listener) throws IOException { var runtime = pack.runtimeCompatibility(); if (!runtime.requiresExternalWorker()) { throw new IllegalArgumentException("AI pack does not declare an external model worker"); @@ -46,9 +58,17 @@ public JsonNode execute(EvidencePackManifest pack, Path request, String jobId) t Files.createDirectories(outputRoot); var output = outputRoot.resolve("result.json"); var partial = outputRoot.resolve("result.json.partial"); + var progressPath = outputRoot.resolve("progress.json"); Files.deleteIfExists(partial); - var builder = new ProcessBuilder(executable.toString(), "--request", request.toString(), - "--output", partial.toString(), "--offline").directory(install.toFile()); + var command = new java.util.ArrayList(); + command.add(executable.toString()); command.add("--request"); command.add(request.toString()); + command.add("--output"); command.add(partial.toString()); command.add("--offline"); + if (resumeCheckpoint != null) { + require("pathlab.model-worker/2".equals(runtime.workerProtocol()), + "Model worker checkpoint resume is unsupported"); + command.add("--resume-checkpoint"); command.add(resumeCheckpoint.toString()); + } + var builder = new ProcessBuilder(command).directory(install.toFile()); var environment = builder.environment(); environment.remove("HTTP_PROXY"); environment.remove("HTTPS_PROXY"); environment.remove("ALL_PROXY"); environment.remove("HF_TOKEN"); @@ -60,15 +80,43 @@ public JsonNode execute(EvidencePackManifest pack, Path request, String jobId) t var process = ChildProcessContainment.global().register(builder.redirectErrorStream(true) .redirectOutput(ProcessBuilder.Redirect.DISCARD).start()); try { - if (!process.waitFor(pack.maxSeconds(), TimeUnit.SECONDS)) { - process.destroyForcibly(); - throw new IOException("Model worker exceeded its bounded runtime"); + var deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(pack.maxSeconds()); + var lastProgressModified = -1L; + var lastHeartbeat = 0L; + var lastProgress = new Progress(0, 0, null, ""); + while (!process.waitFor(1, TimeUnit.SECONDS)) { + if (System.nanoTime() >= deadline) { + process.destroyForcibly(); + throw new ResourceLimitException("Model worker exceeded its bounded runtime"); + } + if (Files.isRegularFile(progressPath)) { + var modified = Files.getLastModifiedTime(progressPath).toMillis(); + if (modified != lastProgressModified) { + lastProgress = readProgress(progressPath, jobId, pack); + listener.update(lastProgress); + lastProgressModified = modified; + lastHeartbeat = System.nanoTime(); + continue; + } + } + if (System.nanoTime() - lastHeartbeat >= TimeUnit.SECONDS.toNanos(10)) { + listener.update(lastProgress); + lastHeartbeat = System.nanoTime(); + } } + } catch (IOException error) { + process.destroyForcibly(); + Files.deleteIfExists(partial); + throw error; } catch (InterruptedException error) { Thread.currentThread().interrupt(); process.destroyForcibly(); + Files.deleteIfExists(partial); throw new IOException("Model worker was interrupted", error); } - require(process.exitValue() == 0 && Files.isRegularFile(partial), "Model worker failed closed"); + if (process.exitValue() != 0 || !Files.isRegularFile(partial)) { + Files.deleteIfExists(partial); + throw new IllegalArgumentException("Model worker failed closed"); + } var result = JSON.readTree(partial.toFile()); validateResult(result, pack); try { @@ -80,6 +128,41 @@ public JsonNode execute(EvidencePackManifest pack, Path request, String jobId) t return result; } + private static Progress readProgress(Path path, String jobId, EvidencePackManifest pack) throws IOException { + var value = JSON.readTree(path.toFile()); + require(value.isObject() && fieldNames(value).equals(Set.of("schema", "jobId", "packManifestSha256", + "completedUnits", "totalUnits", "checkpointPath", "checkpointSha256", "updatedAt")), + "Model worker progress fields are invalid"); + require(PROGRESS_SCHEMA.equals(value.path("schema").asText()), "Model worker progress schema is invalid"); + require(jobId.equals(value.path("jobId").asText()) + && pack.sha256().equals(value.path("packManifestSha256").asText()), + "Model worker progress identity does not match"); + var completed = value.path("completedUnits").asLong(-1); + var total = value.path("totalUnits").asLong(-1); + require(completed >= 0 && total >= completed, "Model worker progress counts are invalid"); + var checkpointText = value.path("checkpointPath").asText(); + var checksum = value.path("checkpointSha256").asText(); + Path checkpoint = checkpointText.isBlank() ? null : Path.of(checkpointText).toAbsolutePath().normalize(); + if (checkpoint != null) { + require(Files.isRegularFile(checkpoint) && checksum.matches("[a-f0-9]{64}") + && checksum.equals(sha256(checkpoint)), "Model worker checkpoint is invalid"); + } else require(checksum.isBlank(), "Checkpoint checksum has no checkpoint"); + Instant.parse(value.path("updatedAt").asText()); + return new Progress(completed, total, checkpoint, checksum); + } + + private static Set fieldNames(JsonNode value) { + var names = new java.util.HashSet(); value.fieldNames().forEachRemaining(names::add); return names; + } + + @FunctionalInterface + public interface ProgressListener { void update(Progress progress) throws IOException; } + public record Progress(long completedUnits, long totalUnits, Path checkpointPath, String checkpointSha256) { } + public static final class ResourceLimitException extends IOException { + private static final long serialVersionUID = 1L; + ResourceLimitException(String message) { super(message); } + } + static void validateResult(JsonNode result, EvidencePackManifest pack) { require(result.isObject() && RESULT_SCHEMA.equals(result.path("schema").asText()), "Model worker result schema is invalid"); diff --git a/src/main/java/org/pathlab/forge/server/ForgeServer.java b/src/main/java/org/pathlab/forge/server/ForgeServer.java index 234db25..5a04b21 100644 --- a/src/main/java/org/pathlab/forge/server/ForgeServer.java +++ b/src/main/java/org/pathlab/forge/server/ForgeServer.java @@ -1,5 +1,6 @@ package org.pathlab.forge.server; +import com.fasterxml.jackson.databind.ObjectMapper; import com.sun.net.httpserver.HttpExchange; import com.sun.net.httpserver.HttpServer; import java.io.IOException; @@ -54,6 +55,7 @@ import org.pathlab.forge.viewer.WindowsCredentialStore; public final class ForgeServer implements AutoCloseable { + private static final ObjectMapper JSON = new ObjectMapper(); private static final int MAX_WRITE_BYTES = 65_536; private static final int DEFAULT_DESKTOP_PORT = 51_274; private static final long SESSION_MAX_AGE_SECONDS = 315_360_000L; @@ -305,6 +307,8 @@ private void handle(HttpExchange exchange) throws IOException { capabilities(exchange); } else if ("/api/evidence/status".equals(path) && "GET".equals(exchange.getRequestMethod())) { evidenceRunnerStatus(exchange); + } else if ("/api/evidence/dashboard".equals(path) && "POST".equals(exchange.getRequestMethod())) { + openEvidenceDashboard(exchange); } else if ("/api/features".equals(path) && "GET".equals(exchange.getRequestMethod())) { features(exchange); } else if (path.matches("/api/features/[a-z0-9][a-z0-9-]{1,63}/install") @@ -762,15 +766,23 @@ private void revokeViewerConnection(HttpExchange exchange) throws IOException { private void evidenceRunnerStatus(HttpExchange exchange) throws IOException { if (!requireAuthenticated(exchange)) return; - var tokenPath = EvidenceMentorRunner.defaultStateRoot().resolve("ipc-token"); - if (!Files.isRegularFile(tokenPath)) { + var stateRoot = EvidenceMentorRunner.defaultStateRoot(); + var tokenPath = stateRoot.resolve("ipc-token"); + var endpointPath = stateRoot.resolve("endpoint.json"); + if (!Files.isRegularFile(tokenPath) || !Files.isRegularFile(endpointPath)) { respond(exchange, 503, "application/json", - "{\"schema\":\"pathlab.evidence-runner-status/1\",\"status\":\"unavailable\"," + "{\"schema\":\"pathlab.evidence-runner-status/2\",\"status\":\"unavailable\"," + "\"detail\":\"Evidence Mentor runner is not installed\"}"); return; } try { - var port = Integer.getInteger("pathlab.evidence.port", 8765); + var endpoint = JSON.readTree(endpointPath.toFile()); + if (!"pathlab.runner-endpoint/1".equals(endpoint.path("schema").asText()) + || !endpoint.path("port").canConvertToInt() + || endpoint.path("port").asInt() < 1 || endpoint.path("port").asInt() > 65535) { + throw new IOException("Evidence Mentor endpoint is invalid"); + } + var port = endpoint.path("port").asInt(); var request = java.net.http.HttpRequest.newBuilder( URI.create("http://127.0.0.1:" + port + "/v1/status")) .timeout(java.time.Duration.ofSeconds(2)) @@ -784,14 +796,42 @@ private void evidenceRunnerStatus(HttpExchange exchange) throws IOException { respond(exchange, 200, "application/json", response.body()); } catch (InterruptedException error) { Thread.currentThread().interrupt(); - respond(exchange, 503, "application/json", "{\"schema\":\"pathlab.evidence-runner-status/1\"," + respond(exchange, 503, "application/json", "{\"schema\":\"pathlab.evidence-runner-status/2\"," + "\"status\":\"unavailable\",\"detail\":\"Status request was interrupted\"}"); } catch (IOException | IllegalArgumentException error) { - respond(exchange, 503, "application/json", "{\"schema\":\"pathlab.evidence-runner-status/1\"," + respond(exchange, 503, "application/json", "{\"schema\":\"pathlab.evidence-runner-status/2\"," + "\"status\":\"unavailable\",\"detail\":\"Runner is not reachable\"}"); } } + private void openEvidenceDashboard(HttpExchange exchange) throws IOException { + if (!requireWrite(exchange)) return; + var stateRoot = EvidenceMentorRunner.defaultStateRoot(); + try { + var endpoint = JSON.readTree(stateRoot.resolve("endpoint.json").toFile()); + var port = endpoint.path("port").asInt(); + if (!"pathlab.runner-endpoint/1".equals(endpoint.path("schema").asText()) + || port < 1 || port > 65535) throw new IOException("Runner endpoint is invalid"); + var origin = "http://127.0.0.1:" + port; + var request = java.net.http.HttpRequest.newBuilder(URI.create(origin + "/v1/dashboard-sessions")) + .timeout(java.time.Duration.ofSeconds(2)) + .header("Authorization", "Bearer " + Files.readString(stateRoot.resolve("ipc-token")).trim()) + .POST(java.net.http.HttpRequest.BodyPublishers.noBody()).build(); + var response = java.net.http.HttpClient.newHttpClient().send(request, + java.net.http.HttpResponse.BodyHandlers.ofString()); + if (response.statusCode() != 201 || response.body().length() > 4_096) throw new IOException("Dashboard session was refused"); + var result = JSON.readTree(response.body()); + var code = result.path("code").asText(); + if (!code.matches("[A-Za-z0-9_-]{40,80}")) throw new IOException("Dashboard session code is invalid"); + respond(exchange, 200, "application/json", "{\"url\":" + json(origin + "/dashboard/#" + code) + "}"); + } catch (InterruptedException error) { + Thread.currentThread().interrupt(); + respond(exchange, 503, "application/json", "{\"error\":\"dashboard_unavailable\",\"detail\":\"Dashboard request was interrupted\"}"); + } catch (IOException | IllegalArgumentException error) { + respond(exchange, 503, "application/json", "{\"error\":\"dashboard_unavailable\",\"detail\":\"Evidence Mentor dashboard is not reachable\"}"); + } + } + private void viewerUploadStatus(HttpExchange exchange) throws IOException { if (!requireAuthenticated(exchange)) { return; diff --git a/src/main/resources/evidence-dashboard/index.html b/src/main/resources/evidence-dashboard/index.html new file mode 100644 index 0000000..4b027b5 --- /dev/null +++ b/src/main/resources/evidence-dashboard/index.html @@ -0,0 +1,27 @@ + + + + + + PathLab Evidence Mentor + + +
+
Offline analysis operations

PathLab Evidence Mentor

Authenticating…
+
+

Jobs

+
diff --git a/src/test/java/org/pathlab/forge/evidence/DashboardSessionsTest.java b/src/test/java/org/pathlab/forge/evidence/DashboardSessionsTest.java new file mode 100644 index 0000000..914a88c --- /dev/null +++ b/src/test/java/org/pathlab/forge/evidence/DashboardSessionsTest.java @@ -0,0 +1,16 @@ +package org.pathlab.forge.evidence; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.time.Instant; +import org.junit.jupiter.api.Test; + +final class DashboardSessionsTest { + @Test + void codeExpiresAfterSixtySeconds() { + var sessions = new DashboardSessions(); + var now = Instant.parse("2026-08-22T00:00:00Z"); + var code = sessions.create(now); + assertTrue(sessions.exchange(code, now.plusSeconds(61)).isEmpty()); + } +} diff --git a/src/test/java/org/pathlab/forge/evidence/EvidenceJobQueueTest.java b/src/test/java/org/pathlab/forge/evidence/EvidenceJobQueueTest.java index ca87cd0..1a37cc7 100644 --- a/src/test/java/org/pathlab/forge/evidence/EvidenceJobQueueTest.java +++ b/src/test/java/org/pathlab/forge/evidence/EvidenceJobQueueTest.java @@ -52,4 +52,65 @@ void cancellationFlagSurvivesRestart() throws Exception { assertTrue(queue.claimNext("worker", now.plusSeconds(2), Duration.ofSeconds(30)).isEmpty()); } } + + @Test + void separatesExecutionLanesAndRenewsLiveLease() throws Exception { + var database = temporaryDirectory.resolve("jobs.sqlite"); + var gpuRequest = temporaryDirectory.resolve("gpu.json"); + var cpuRequest = temporaryDirectory.resolve("cpu.json"); + Files.writeString(gpuRequest, "{}"); + Files.writeString(cpuRequest, "{}"); + var now = Instant.parse("2026-08-22T00:00:00Z"); + try (var queue = new EvidenceJobQueue(database)) { + queue.submit("gpu-job", gpuRequest, EvidenceExecutionLane.GPU, now); + queue.submit("cpu-job", cpuRequest, EvidenceExecutionLane.CPU_IO, now.plusMillis(1)); + + var cpu = queue.claimNext(EvidenceExecutionLane.CPU_IO, "cpu-worker", now, + Duration.ofSeconds(30)).orElseThrow(); + assertEquals("cpu-job", cpu.id()); + var gpu = queue.claimNext(EvidenceExecutionLane.GPU, "gpu-worker", now, + Duration.ofSeconds(30)).orElseThrow(); + assertEquals("gpu-job", gpu.id()); + + queue.heartbeat("gpu-job", "gpu-worker", 32, 100, + temporaryDirectory.resolve("gpu.checkpoint"), "a".repeat(64), + now.plusSeconds(15), Duration.ofSeconds(30)); + var snapshot = queue.snapshot("gpu-job").orElseThrow(); + assertEquals(32, snapshot.completedUnits()); + assertEquals(100, snapshot.totalUnits()); + assertEquals(now.plusSeconds(45), snapshot.leaseExpiresAt()); + assertEquals(null, snapshot.etaSeconds()); + queue.heartbeat("gpu-job", "gpu-worker", 64, 100, + temporaryDirectory.resolve("gpu.checkpoint"), "a".repeat(64), + now.plusSeconds(30), Duration.ofSeconds(30)); + snapshot = queue.snapshot("gpu-job").orElseThrow(); + assertTrue(snapshot.throughput() > 0); + assertTrue(snapshot.etaSeconds() != null); + } + } + + @Test + void pauseStopsClaimsAndOperatorRetryUsesBoundedDelay() throws Exception { + var database = temporaryDirectory.resolve("jobs.sqlite"); + var request = temporaryDirectory.resolve("request.json"); + Files.writeString(request, "{}"); + var now = Instant.parse("2026-08-22T00:00:00Z"); + try (var queue = new EvidenceJobQueue(database)) { + queue.submit("job", request, EvidenceExecutionLane.CPU_IO, now); + queue.setAcceptingJobs(false, now.plusSeconds(1)); + assertFalse(queue.acceptingJobs()); + assertTrue(queue.claimNext(EvidenceExecutionLane.CPU_IO, "worker", now.plusSeconds(2), + Duration.ofSeconds(30)).isEmpty()); + queue.setAcceptingJobs(true, now.plusSeconds(3)); + var claimed = queue.claimNext(EvidenceExecutionLane.CPU_IO, "worker", now.plusSeconds(4), + Duration.ofSeconds(30)).orElseThrow(); + queue.fail(claimed.id(), "worker", "transient_io", "IO_READ_FAILED", "disk busy", + true, now.plusSeconds(5)); + var retry = queue.snapshot("job").orElseThrow(); + assertEquals(EvidenceJobState.QUEUED, retry.state()); + assertEquals(now.plusSeconds(10), retry.nextRetryAt()); + assertTrue(queue.claimNext(EvidenceExecutionLane.CPU_IO, "worker-2", now.plusSeconds(9), + Duration.ofSeconds(30)).isEmpty()); + } + } } diff --git a/src/test/java/org/pathlab/forge/evidence/EvidenceMentorRunnerTest.java b/src/test/java/org/pathlab/forge/evidence/EvidenceMentorRunnerTest.java index ee6eec3..22ff8da 100644 --- a/src/test/java/org/pathlab/forge/evidence/EvidenceMentorRunnerTest.java +++ b/src/test/java/org/pathlab/forge/evidence/EvidenceMentorRunnerTest.java @@ -9,10 +9,12 @@ import java.net.http.HttpResponse; import java.nio.file.Files; import java.nio.file.Path; +import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; final class EvidenceMentorRunnerTest { + private static final ObjectMapper JSON = new ObjectMapper(); @TempDir Path temporaryDirectory; @Test @@ -50,6 +52,63 @@ void acceptsAuthenticatedLoopbackSubmissionIntoDurableQueue() throws Exception { assertEquals(200, status.statusCode()); assertTrue(status.body().contains("networkDisabledForAnalysis")); assertTrue(status.body().contains("evidence-test")); + assertTrue(status.body().contains("pathlab.evidence-runner-status/2")); + + var jobs = client.send(HttpRequest.newBuilder(runner.uri("/v1/jobs?limit=50")) + .header("Authorization", "Bearer test-loopback-token-0123456789abcdef") + .GET().build(), HttpResponse.BodyHandlers.ofString()); + assertEquals(200, jobs.statusCode()); + var notModified = client.send(HttpRequest.newBuilder(runner.uri("/v1/jobs?limit=50")) + .header("Authorization", "Bearer test-loopback-token-0123456789abcdef") + .header("If-None-Match", jobs.headers().firstValue("ETag").orElseThrow()) + .GET().build(), HttpResponse.BodyHandlers.ofString()); + assertEquals(304, notModified.statusCode()); + + var endpoint = JSON.readTree(temporaryDirectory.resolve("state/endpoint.json").toFile()); + assertEquals("pathlab.runner-endpoint/1", endpoint.path("schema").asText()); + assertEquals(runner.uri("/").getPort(), endpoint.path("port").asInt()); + assertTrue(!endpoint.has("token")); + } + } + + @Test + void exchangesSingleUseDashboardCodeAndProtectsMutations() throws Exception { + try (var runner = EvidenceMentorRunner.start( + temporaryDirectory.resolve("state"), 0, "test-loopback-token-0123456789abcdef", false)) { + var client = HttpClient.newBuilder().cookieHandler(new java.net.CookieManager()).build(); + var create = client.send(HttpRequest.newBuilder(runner.uri("/v1/dashboard-sessions")) + .header("Authorization", "Bearer test-loopback-token-0123456789abcdef") + .POST(HttpRequest.BodyPublishers.noBody()).build(), HttpResponse.BodyHandlers.ofString()); + assertEquals(201, create.statusCode()); + var code = JSON.readTree(create.body()).path("code").asText(); + var exchangeBody = "{\"code\":" + quote(code) + "}"; + var exchange = client.send(HttpRequest.newBuilder(runner.uri("/v1/dashboard-session/exchange")) + .header("Origin", runner.uri("").toString()) + .POST(HttpRequest.BodyPublishers.ofString(exchangeBody)).build(), HttpResponse.BodyHandlers.ofString()); + assertEquals(200, exchange.statusCode()); + var csrf = JSON.readTree(exchange.body()).path("csrfToken").asText(); + assertTrue(exchange.headers().firstValue("set-cookie").orElse("").contains("HttpOnly")); + + var reused = client.send(HttpRequest.newBuilder(runner.uri("/v1/dashboard-session/exchange")) + .header("Origin", runner.uri("").toString()) + .POST(HttpRequest.BodyPublishers.ofString(exchangeBody)).build(), HttpResponse.BodyHandlers.ofString()); + assertEquals(401, reused.statusCode()); + + var withoutCsrf = client.send(HttpRequest.newBuilder(runner.uri("/v1/control/pause")) + .header("Origin", runner.uri("").toString()) + .POST(HttpRequest.BodyPublishers.noBody()).build(), HttpResponse.BodyHandlers.ofString()); + assertEquals(403, withoutCsrf.statusCode()); + var pause = client.send(HttpRequest.newBuilder(runner.uri("/v1/control/pause")) + .header("Origin", runner.uri("").toString()).header("X-PathLab-CSRF", csrf) + .POST(HttpRequest.BodyPublishers.noBody()).build(), HttpResponse.BodyHandlers.ofString()); + assertEquals(200, pause.statusCode()); + assertTrue(pause.body().contains("\"acceptingJobs\":false")); + + var dashboard = client.send(HttpRequest.newBuilder(runner.uri("/dashboard/")) + .GET().build(), HttpResponse.BodyHandlers.ofString()); + assertEquals(200, dashboard.statusCode()); + assertTrue(dashboard.body().contains("PathLab Evidence Mentor")); + assertTrue(!dashboard.body().contains("https://")); } } From bd103fc700cc36f89387aea63ab984fa284949fc Mon Sep 17 00:00:00 2001 From: Enksodsoon <229880765+Enksodsoon@users.noreply.github.com> Date: Sat, 22 Aug 2026 12:02:04 +0700 Subject: [PATCH 08/50] test(evidence): add service acceptance gate --- docs/evidence-mentor-runner.md | 58 +++ docs/plans/active/current.md | 9 + scripts/evidence-mentor-service.ps1 | 4 +- scripts/test-evidence-mentor-service.ps1 | 443 +++++++++++++++++++++++ 4 files changed, 513 insertions(+), 1 deletion(-) create mode 100644 scripts/test-evidence-mentor-service.ps1 diff --git a/docs/evidence-mentor-runner.md b/docs/evidence-mentor-runner.md index 4d0bf46..a573dc5 100644 --- a/docs/evidence-mentor-runner.md +++ b/docs/evidence-mentor-runner.md @@ -54,6 +54,64 @@ Use `-Action Upgrade` with a new immutable version. `-Action Uninstall` removes the service, firewall rules, and Start Menu shortcut while preserving queues, checkpoints, signing material, evidence, and model packs. +## Host acceptance and the definition of done + +Installation is not acceptance. Run the installed acceptance harness from an +administrator PowerShell window and retain its immutable JSON and Markdown +reports under `D:\PathLabData\EvidenceMentor\state\acceptance`. + +Start with the non-disruptive inspection: + +```powershell +& 'C:\ProgramData\PathLab\EvidenceMentor\Test-PathLab-Evidence-Service.ps1' ` + -Mode Inspect +``` + +Stage a small, rights-approved GPU-pack request under the protected state root. +Its public acceptance ID must match `acceptance-<8-64 lowercase hex chars>`. +Then prove service restart recovery while that bounded job is active: + +```powershell +& 'C:\ProgramData\PathLab\EvidenceMentor\Test-PathLab-Evidence-Service.ps1' ` + -Mode ServiceRestart ` + -JobRequestPath 'D:\PathLabData\EvidenceMentor\state\requests\acceptance-gpu.json' ` + -JobId 'acceptance-0123abcd' ` + -TimeoutMinutes 10 +``` + +Reboot acceptance is deliberately split into two operator-controlled commands. +The harness never initiates a reboot: + +```powershell +& 'C:\ProgramData\PathLab\EvidenceMentor\Test-PathLab-Evidence-Service.ps1' ` + -Mode PrepareReboot ` + -JobRequestPath 'D:\PathLabData\EvidenceMentor\state\requests\acceptance-reboot.json' ` + -JobId 'acceptance-89abcdef' + +# Reboot Windows manually while the bounded job is active. Do not log in first. + +& 'C:\ProgramData\PathLab\EvidenceMentor\Test-PathLab-Evidence-Service.ps1' ` + -Mode VerifyReboot ` + -TimeoutMinutes 10 +``` + +Finally aggregate the evidence: + +```powershell +& 'C:\ProgramData\PathLab\EvidenceMentor\Test-PathLab-Evidence-Service.ps1' ` + -Mode Summary +``` + +The autonomous-service phase is done only when the newest `Summary` report has +schema `pathlab.service-acceptance/1`, verdict `PASS`, and no required check is +`FAIL` or `NOT_EVALUABLE`. This includes LocalService identity, delayed startup +and recovery policy, pinned WinSW, ACLs, firewall coverage, dynamic authenticated +IPC, one-time dashboard sessions, P2000 visibility, a completed GPU-lane job, +service-restart recovery, and reboot continuation of an active bounded job. +Mode-specific `PASS` reports are evidence fragments and are not a completion +claim. `-AllowIncomplete` is intended only for safe diagnostics and cannot turn +an incomplete report into passing evidence. + ## Model boundary The deterministic cell/IHC baseline is runnable. DINOv2-small remains an diff --git a/docs/plans/active/current.md b/docs/plans/active/current.md index 23a9972..5ab115a 100644 --- a/docs/plans/active/current.md +++ b/docs/plans/active/current.md @@ -27,6 +27,15 @@ appear in monitoring payloads. 6. Independent expert-pack qualification for H&E, cells, IHC, special stains, and cytology. 7. Dual-lineage Atlas feasibility only after lawful frozen teachers exist. +## Current status + +Gates 1-4 are implemented on the draft feature branch. Gate 5 now has an +atomic, fail-closed host acceptance harness and operator procedure. It remains +incomplete until an administrator installs the service and the same installed +runtime earns a `PASS` aggregate report after live LocalService, Session 0 GPU, +service-restart, offline, and manual-reboot recovery checks. Gate 6 has not been +qualified, and Gate 7 has not begun. + Implementation, service installation, model qualification, merge, deployment, feature activation, and any future clinical/capacity study are separate gates. Feature flags remain default-off and no diagnostic or clinical-scoring claims diff --git a/scripts/evidence-mentor-service.ps1 b/scripts/evidence-mentor-service.ps1 index d764b98..f9990b2 100644 --- a/scripts/evidence-mentor-service.ps1 +++ b/scripts/evidence-mentor-service.ps1 @@ -48,7 +48,7 @@ function Ensure-WinSW { } function Grant-PathLabAccess([string] $InstallingUser) { - & icacls.exe $ProgramRoot /inheritance:r /grant:r 'SYSTEM:(OI)(CI)F' 'Administrators:(OI)(CI)F' "NT SERVICE\${serviceName}:(OI)(CI)RX" | Out-Null + & icacls.exe $ProgramRoot /inheritance:r /grant:r 'SYSTEM:(OI)(CI)F' 'Administrators:(OI)(CI)F' "NT SERVICE\${serviceName}:(OI)(CI)RX" "${InstallingUser}:(OI)(CI)RX" | Out-Null & icacls.exe $StateRoot /inheritance:r /grant:r 'SYSTEM:(OI)(CI)F' 'Administrators:(OI)(CI)F' "NT SERVICE\${serviceName}:(OI)(CI)M" "${InstallingUser}:(RX)" | Out-Null foreach ($operatorPath in @('inputs','requests','models','artifacts')) { $resolved = Join-Path $StateRoot $operatorPath @@ -191,6 +191,8 @@ if ($PSCmdlet.ShouldProcess($serviceName, "$Action autonomous service version $V Grant-PathLabAccess $installingUser $launcher = Join-Path $ProgramRoot 'Open-PathLab-Evidence-Dashboard.ps1' Copy-Item -LiteralPath (Join-Path $PSScriptRoot 'open-evidence-dashboard.ps1') -Destination $launcher -Force + Copy-Item -LiteralPath (Join-Path $PSScriptRoot 'test-evidence-mentor-service.ps1') ` + -Destination (Join-Path $ProgramRoot 'Test-PathLab-Evidence-Service.ps1') -Force $shortcutPath = Join-Path ([Environment]::GetFolderPath('CommonStartMenu')) 'Programs\PathLab Evidence Mentor Dashboard.lnk' $shell = New-Object -ComObject WScript.Shell $shortcut = $shell.CreateShortcut($shortcutPath) diff --git a/scripts/test-evidence-mentor-service.ps1 b/scripts/test-evidence-mentor-service.ps1 new file mode 100644 index 0000000..a670f18 --- /dev/null +++ b/scripts/test-evidence-mentor-service.ps1 @@ -0,0 +1,443 @@ +[CmdletBinding(SupportsShouldProcess)] +param( + [ValidateSet('Inspect','ServiceRestart','PrepareReboot','VerifyReboot','Summary')] + [string] $Mode = 'Inspect', + [string] $ProgramRoot = 'C:\ProgramData\PathLab\EvidenceMentor', + [string] $StateRoot = 'D:\PathLabData\EvidenceMentor\state', + [string] $ReportRoot, + [string] $JobRequestPath, + [string] $JobId, + [ValidateRange(1, 120)] [int] $TimeoutMinutes = 10, + [switch] $AllowIncomplete +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version Latest +$serviceName = 'PathLabEvidenceMentor' +$expectedWinSwSha256 = '05b82d46ad331cc16bdc00de5c6332c1ef818df8ceefcd49c726553209b3a0da' +$checks = [Collections.Generic.List[object]]::new() +$startedAt = [DateTimeOffset]::UtcNow +if (-not $ReportRoot) { $ReportRoot = Join-Path $StateRoot 'acceptance' } +$ReportRoot = [IO.Path]::GetFullPath($ReportRoot) +$StateRoot = [IO.Path]::GetFullPath($StateRoot) +$ProgramRoot = [IO.Path]::GetFullPath($ProgramRoot) +if ($JobId -and $JobId -notmatch '^acceptance-[a-f0-9]{8,64}$') { + throw 'JobId must be a non-identifying acceptance ID such as acceptance-0123abcd.' +} + +function Add-Check([string] $Name, [string] $Status, [string] $Detail, [hashtable] $Evidence = @{}) { + if ($Status -notin @('PASS','FAIL','NOT_EVALUABLE')) { throw "Invalid acceptance status: $Status" } + $checks.Add([ordered]@{ name = $Name; status = $Status; detail = $Detail; evidence = $Evidence }) +} + +function Test-Administrator { + $identity = [Security.Principal.WindowsIdentity]::GetCurrent() + return [Security.Principal.WindowsPrincipal]::new($identity).IsInRole( + [Security.Principal.WindowsBuiltInRole]::Administrator) +} + +function Read-Endpoint { + $path = Join-Path $StateRoot 'endpoint.json' + if (-not (Test-Path -LiteralPath $path -PathType Leaf)) { return $null } + try { + $endpoint = Get-Content -LiteralPath $path -Raw | ConvertFrom-Json + if ($endpoint.schema -ne 'pathlab.runner-endpoint/1' -or $endpoint.port -lt 1 -or $endpoint.port -gt 65535) { return $null } + return $endpoint + } catch { return $null } +} + +function Read-Token { + $path = Join-Path $StateRoot 'ipc-token' + if (-not (Test-Path -LiteralPath $path -PathType Leaf)) { return $null } + $value = (Get-Content -LiteralPath $path -Raw).Trim() + if ($value.Length -lt 32) { return $null } + return $value +} + +function Invoke-Runner([string] $Method, [string] $Path, [object] $Body = $null) { + $endpoint = Read-Endpoint + $token = Read-Token + if ($null -eq $endpoint -or $null -eq $token) { throw 'Runner endpoint or token is unavailable.' } + $parameters = @{ + Method = $Method + Uri = "http://127.0.0.1:$($endpoint.port)$Path" + Headers = @{ Authorization = "Bearer $token" } + TimeoutSec = 3 + SkipHttpErrorCheck = $true + } + if ($null -ne $Body) { + $parameters.ContentType = 'application/json' + $parameters.Body = $Body | ConvertTo-Json -Compress -Depth 8 + } + return Invoke-WebRequest @parameters +} + +function Wait-Runner([string] $PreviousBootId = '', [int] $Seconds = 60) { + $deadline = [DateTimeOffset]::UtcNow.AddSeconds($Seconds) + do { + Start-Sleep -Milliseconds 500 + $endpoint = Read-Endpoint + if ($null -ne $endpoint -and ($PreviousBootId -eq '' -or $endpoint.bootId -ne $PreviousBootId)) { + try { + $response = Invoke-Runner 'GET' '/health' + if ($response.StatusCode -eq 200) { return $endpoint } + } catch { } + } + } while ([DateTimeOffset]::UtcNow -lt $deadline) + return $null +} + +function Test-Acl([string] $Path, [string] $Identity, [Security.AccessControl.FileSystemRights] $RequiredRights) { + if (-not (Test-Path -LiteralPath $Path)) { return $false } + $acl = Get-Acl -LiteralPath $Path + foreach ($entry in $acl.Access) { + $allowsIdentity = $entry.AccessControlType -eq 'Allow' -and $entry.IdentityReference.Value -ieq $Identity + $hasRights = ($entry.FileSystemRights -band $RequiredRights) -eq $RequiredRights + if ($allowsIdentity -and $hasRights) { return $true } + } + return $false +} + +function Write-Atomic([string] $Path, [string] $Content) { + $partial = "$Path.partial" + [IO.File]::WriteAllText($partial, $Content, [Text.UTF8Encoding]::new($false)) + [IO.File]::Move($partial, $Path, $true) +} + +function Write-AcceptanceReport { + New-Item -ItemType Directory -Path $ReportRoot -Force | Out-Null + $failures = @($checks | Where-Object status -eq 'FAIL').Count + $notEvaluable = @($checks | Where-Object status -eq 'NOT_EVALUABLE').Count + $verdict = if ($failures -gt 0) { 'FAIL' } elseif ($notEvaluable -gt 0) { 'NOT_EVALUABLE' } else { 'PASS' } + $report = [ordered]@{ + schema = 'pathlab.service-acceptance/1' + service = $serviceName + runtimeVersion = if (Test-Path -LiteralPath (Join-Path $ProgramRoot 'active-version.txt') -PathType Leaf) { + (Get-Content -LiteralPath (Join-Path $ProgramRoot 'active-version.txt') -Raw).Trim() + } else { '' } + mode = $Mode + startedAt = $startedAt.ToString('O') + completedAt = [DateTimeOffset]::UtcNow.ToString('O') + verdict = $verdict + counts = @{ pass = @($checks | Where-Object status -eq 'PASS').Count; fail = $failures; notEvaluable = $notEvaluable } + privacy = @{ containsTokens = $false; containsSlidePaths = $false; containsScientificOutcomes = $false } + checks = $checks + } + $stamp = [DateTimeOffset]::UtcNow.ToString('yyyyMMdd-HHmmss-fff') + $jsonPath = Join-Path $ReportRoot "service-acceptance-$stamp.json" + $markdownPath = Join-Path $ReportRoot "service-acceptance-$stamp.md" + Write-Atomic $jsonPath ($report | ConvertTo-Json -Depth 12) + $lines = [Collections.Generic.List[string]]::new() + $lines.Add('# PathLab Evidence Mentor service acceptance') + $lines.Add('') + $lines.Add("- Verdict: **$verdict**") + $lines.Add("- Mode: ``$Mode``") + $lines.Add("- Completed: $($report.completedAt)") + $lines.Add('') + $lines.Add('| Check | Status | Detail |') + $lines.Add('|---|---|---|') + foreach ($check in $checks) { + $detail = ([string]$check.detail).Replace('|','\|').Replace("`r",' ').Replace("`n",' ') + $lines.Add("| $($check.name) | $($check.status) | $detail |") + } + Write-Atomic $markdownPath ($lines -join [Environment]::NewLine) + [ordered]@{ verdict = $verdict; json = $jsonPath; markdown = $markdownPath } +} + +function Add-StaticChecks { + $service = Get-CimInstance Win32_Service -Filter "Name='$serviceName'" -ErrorAction SilentlyContinue + if ($null -eq $service) { + Add-Check 'service-installed' 'FAIL' 'Windows service is not installed.' + Add-Check 'service-running' 'NOT_EVALUABLE' 'Service installation is required first.' + Add-Check 'service-identity' 'NOT_EVALUABLE' 'Service installation is required first.' + } else { + Add-Check 'service-installed' 'PASS' 'Windows service is installed.' @{ startMode = $service.StartMode } + Add-Check 'service-running' $(if ($service.State -eq 'Running') {'PASS'} else {'FAIL'}) "Service state is $($service.State)." + Add-Check 'service-identity' $(if ($service.StartName -ieq 'NT AUTHORITY\LocalService') {'PASS'} else {'FAIL'}) "Service identity is $($service.StartName)." + } + + $serviceRegistry = "HKLM:\SYSTEM\CurrentControlSet\Services\$serviceName" + if (Test-Path -LiteralPath $serviceRegistry) { + $registry = Get-ItemProperty -LiteralPath $serviceRegistry + $delayedProperty = $registry.PSObject.Properties['DelayedAutostart'] + $failureProperty = $registry.PSObject.Properties['FailureActions'] + $delayed = $null -ne $delayedProperty -and $delayedProperty.Value -eq 1 + $recovery = $null -ne $failureProperty -and $null -ne $failureProperty.Value -and $failureProperty.Value.Length -gt 0 + Add-Check 'delayed-auto-start' $(if ($delayed) {'PASS'} else {'FAIL'}) 'Delayed automatic startup registry flag.' @{ enabled = $delayed } + Add-Check 'service-recovery-policy' $(if ($recovery) {'PASS'} else {'FAIL'}) 'Windows service recovery actions are configured.' + } else { + Add-Check 'delayed-auto-start' 'NOT_EVALUABLE' 'Service registry key is absent.' + Add-Check 'service-recovery-policy' 'NOT_EVALUABLE' 'Service registry key is absent.' + } + + $activeVersionPath = Join-Path $ProgramRoot 'active-version.txt' + $wrapperPath = Join-Path $ProgramRoot 'PathLabEvidenceMentor.exe' + if (Test-Path -LiteralPath $wrapperPath -PathType Leaf) { + $sha = (Get-FileHash -LiteralPath $wrapperPath -Algorithm SHA256).Hash.ToLowerInvariant() + Add-Check 'winsw-checksum' $(if ($sha -eq $expectedWinSwSha256) {'PASS'} else {'FAIL'}) 'Installed WinSW must match the pinned checksum.' @{ sha256 = $sha } + } else { Add-Check 'winsw-checksum' 'NOT_EVALUABLE' 'WinSW wrapper is not installed.' } + if (Test-Path -LiteralPath $activeVersionPath -PathType Leaf) { + Add-Check 'active-runtime' 'PASS' 'An active side-by-side runtime version is recorded.' @{ version = (Get-Content -LiteralPath $activeVersionPath -Raw).Trim() } + } else { Add-Check 'active-runtime' 'NOT_EVALUABLE' 'Active runtime marker is absent.' } + + $serviceIdentity = "NT SERVICE\$serviceName" + Add-Check 'runtime-acl' $(if (Test-Acl $ProgramRoot $serviceIdentity ([Security.AccessControl.FileSystemRights]::ReadAndExecute)) {'PASS'} else {'FAIL'}) 'Service SID requires runtime read/execute access.' + Add-Check 'state-acl' $(if (Test-Acl $StateRoot $serviceIdentity ([Security.AccessControl.FileSystemRights]::Modify)) {'PASS'} else {'FAIL'}) 'Service SID requires state modify access.' + + try { + $rules = @(Get-NetFirewallRule -DisplayName 'PathLab Evidence Mentor outbound deny*' -ErrorAction Stop | + Where-Object { $_.Enabled -eq 'True' -and $_.Direction -eq 'Outbound' -and $_.Action -eq 'Block' }) + $expectedExecutables = [Collections.Generic.List[string]]::new() + $activeRuntime = if (Test-Path -LiteralPath $activeVersionPath -PathType Leaf) { + (Get-Content -LiteralPath $activeVersionPath -Raw).Trim() + } else { '' } + if ($activeRuntime) { + $runtimeJava = Join-Path $ProgramRoot "runtime\$activeRuntime\jre\bin\java.exe" + if (Test-Path -LiteralPath $runtimeJava -PathType Leaf) { $expectedExecutables.Add([IO.Path]::GetFullPath($runtimeJava)) } + } + $modelRoot = Join-Path $StateRoot 'models' + if (Test-Path -LiteralPath $modelRoot -PathType Container) { + foreach ($worker in Get-ChildItem -LiteralPath $modelRoot -Filter '*.exe' -File -Recurse) { + $expectedExecutables.Add($worker.FullName) + } + } + $blockedExecutables = @($rules | Get-NetFirewallApplicationFilter | ForEach-Object { [IO.Path]::GetFullPath($_.Program) }) + $missing = @($expectedExecutables | Where-Object { $_ -notin $blockedExecutables }) + $covered = $expectedExecutables.Count -gt 0 -and $missing.Count -eq 0 + Add-Check 'outbound-deny-rules' $(if ($covered) {'PASS'} else {'FAIL'}) 'Every installed analysis executable requires an enabled outbound-deny rule.' @{ + expectedExecutableCount = $expectedExecutables.Count; ruleCount = $rules.Count; missingCount = $missing.Count + } + } catch { Add-Check 'outbound-deny-rules' 'NOT_EVALUABLE' 'Firewall rules could not be inspected.' } + + $endpoint = Read-Endpoint + if ($null -eq $endpoint) { + Add-Check 'dynamic-endpoint' 'FAIL' 'A valid endpoint.json is not published.' + Add-Check 'authenticated-health' 'NOT_EVALUABLE' 'Runner endpoint is unavailable.' + Add-Check 'dashboard-session-security' 'NOT_EVALUABLE' 'Runner endpoint is unavailable.' + } else { + Add-Check 'dynamic-endpoint' 'PASS' 'Runner published a token-free dynamic loopback endpoint.' @{ + port = [int]$endpoint.port; bootId = [string]$endpoint.bootId; serviceVersion = [string]$endpoint.serviceVersion + } + try { + $health = Invoke-Runner 'GET' '/health' + Add-Check 'authenticated-health' $(if ($health.StatusCode -eq 200) {'PASS'} else {'FAIL'}) 'Authenticated loopback health response.' @{ statusCode = $health.StatusCode } + } catch { Add-Check 'authenticated-health' 'FAIL' 'Authenticated health request failed.' } + try { + $origin = "http://127.0.0.1:$($endpoint.port)" + $unauthorized = Invoke-WebRequest -Uri "$origin/v1/status" -SkipHttpErrorCheck -TimeoutSec 3 + $created = Invoke-Runner 'POST' '/v1/dashboard-sessions' + $code = ($created.Content | ConvertFrom-Json).code + $exchangeBody = @{ code = $code } | ConvertTo-Json -Compress + $first = Invoke-WebRequest -Method Post -Uri "$origin/v1/dashboard-session/exchange" -Headers @{ Origin = $origin } ` + -ContentType 'application/json' -Body $exchangeBody -SkipHttpErrorCheck -TimeoutSec 3 + $reuse = Invoke-WebRequest -Method Post -Uri "$origin/v1/dashboard-session/exchange" -Headers @{ Origin = $origin } ` + -ContentType 'application/json' -Body $exchangeBody -SkipHttpErrorCheck -TimeoutSec 3 + $secure = $unauthorized.StatusCode -eq 401 -and $created.StatusCode -eq 201 ` + -and $first.StatusCode -eq 200 -and $reuse.StatusCode -eq 401 ` + -and ($first.Headers.'Set-Cookie' -join ';') -match 'HttpOnly' ` + -and ($first.Headers.'Set-Cookie' -join ';') -match 'SameSite=Strict' + Add-Check 'dashboard-session-security' $(if ($secure) {'PASS'} else {'FAIL'}) 'Bearer protection and one-time dashboard exchange.' @{ + unauthorized = $unauthorized.StatusCode; created = $created.StatusCode; exchanged = $first.StatusCode; reused = $reuse.StatusCode + } + } catch { Add-Check 'dashboard-session-security' 'FAIL' 'Dashboard session security check failed.' } + } + + try { + $gpu = & nvidia-smi.exe --query-gpu=name,driver_version,memory.total --format=csv,noheader,nounits 2>$null + if ($LASTEXITCODE -eq 0 -and $gpu) { + $parts = (@($gpu)[0]).Split(',') + Add-Check 'gpu-visible' 'PASS' 'NVIDIA GPU telemetry is visible to the current acceptance process.' @{ + name = $parts[0].Trim(); driver = $parts[1].Trim(); memoryMiB = [int]($parts[2].Trim()) + } + } else { Add-Check 'gpu-visible' 'FAIL' 'nvidia-smi did not return GPU telemetry.' } + } catch { Add-Check 'gpu-visible' 'FAIL' 'nvidia-smi is unavailable.' } +} + +function Submit-BoundedJob { + if (-not $JobRequestPath) { Add-Check 'bounded-job' 'NOT_EVALUABLE' 'No acceptance job request was supplied.'; return $null } + $request = [IO.Path]::GetFullPath($JobRequestPath) + $statePrefix = $StateRoot.TrimEnd('\') + '\' + if (-not $request.StartsWith($statePrefix, [StringComparison]::OrdinalIgnoreCase)) { + Add-Check 'bounded-job' 'FAIL' 'Acceptance request must be staged under the protected state root.' + return $null + } + if (-not (Test-Path -LiteralPath $request -PathType Leaf)) { Add-Check 'bounded-job' 'FAIL' 'Acceptance request file is missing.'; return $null } + $resolvedJobId = if ($JobId) { $JobId } else { 'acceptance-' + [Guid]::NewGuid().ToString('N') } + try { + $response = Invoke-Runner 'POST' '/v1/jobs' @{ id = $resolvedJobId; requestPath = $request } + if ($response.StatusCode -ne 202) { Add-Check 'bounded-job-submission' 'FAIL' "Runner returned HTTP $($response.StatusCode)."; return $null } + Add-Check 'bounded-job-submission' 'PASS' 'Runner accepted ownership of the immutable bounded request.' @{ jobId = $resolvedJobId } + return $resolvedJobId + } catch { Add-Check 'bounded-job-submission' 'FAIL' 'Bounded job submission failed.'; return $null } +} + +function Wait-Job([string] $ResolvedJobId, [bool] $RestartWhenActive) { + if (-not $ResolvedJobId) { return $false } + $deadline = [DateTimeOffset]::UtcNow.AddMinutes($TimeoutMinutes) + $restarted = $false + do { + Start-Sleep -Seconds 1 + try { + $response = Invoke-Runner 'GET' "/v1/jobs/$ResolvedJobId" + if ($response.StatusCode -ne 200) { continue } + $job = $response.Content | ConvertFrom-Json + if ($RestartWhenActive -and -not $restarted -and $job.state -in @('validating','running','refining','packaging')) { + if (-not (Test-Administrator)) { Add-Check 'service-restart' 'NOT_EVALUABLE' 'Administrator privileges are required.'; $RestartWhenActive = $false; continue } + $before = Read-Endpoint + if ($PSCmdlet.ShouldProcess($serviceName, "Restart service while bounded job $ResolvedJobId is active")) { + Restart-Service -Name $serviceName -Force + $after = Wait-Runner -PreviousBootId $before.bootId -Seconds 60 + if ($null -eq $after) { Add-Check 'service-restart' 'FAIL' 'Runner did not return with a new boot ID.'; return $false } + Add-Check 'service-restart' 'PASS' 'Runner returned after restart with a new boot ID.' @{ previousBootId = $before.bootId; bootId = $after.bootId } + $restarted = $true + } + } + if ($job.state -in @('completed','abstained','unsupported','failed','cancelled')) { + $passed = $job.state -in @('completed','abstained') -and $job.finalArtifactSha256 -match '^[a-f0-9]{64}$' + Add-Check 'bounded-job-completion' $(if ($passed) {'PASS'} else {'FAIL'}) 'Bounded job reached a terminal state.' @{ + jobId = $ResolvedJobId; state = $job.state; lane = $job.lane; failureCode = $job.failureCode; finalArtifactSha256 = $job.finalArtifactSha256 + } + if ($job.lane -eq 'gpu') { + Add-Check 'session0-gpu-inference' $(if ($passed) {'PASS'} else {'FAIL'}) 'A CUDA-declared pack completed in the service GPU lane.' @{ + jobId = $ResolvedJobId; state = $job.state + } + } else { Add-Check 'session0-gpu-inference' 'NOT_EVALUABLE' 'The bounded acceptance job did not use the GPU lane.' } + return $passed + } + } catch { } + } while ([DateTimeOffset]::UtcNow -lt $deadline) + Add-Check 'bounded-job-completion' 'FAIL' 'Bounded job did not complete before the acceptance timeout.' @{ jobId = $ResolvedJobId } + return $false +} + +function Wait-JobActive([string] $ResolvedJobId, [int] $Seconds = 60) { + if (-not $ResolvedJobId) { return $false } + $deadline = [DateTimeOffset]::UtcNow.AddSeconds($Seconds) + do { + Start-Sleep -Seconds 1 + try { + $response = Invoke-Runner 'GET' "/v1/jobs/$ResolvedJobId" + if ($response.StatusCode -ne 200) { continue } + $job = $response.Content | ConvertFrom-Json + if ($job.state -in @('validating','running','refining','packaging')) { + Add-Check 'reboot-job-active' 'PASS' 'The bounded job is active and checkpoint-capable before manual reboot.' @{ + jobId = $ResolvedJobId; state = $job.state; lane = $job.lane + } + return $true + } + if ($job.state -in @('completed','abstained','unsupported','failed','cancelled')) { + Add-Check 'reboot-job-active' 'FAIL' 'The bounded job reached a terminal state before the reboot challenge.' @{ + jobId = $ResolvedJobId; state = $job.state + } + return $false + } + } catch { } + } while ([DateTimeOffset]::UtcNow -lt $deadline) + Add-Check 'reboot-job-active' 'FAIL' 'The bounded job did not become active before the reboot challenge timeout.' @{ jobId = $ResolvedJobId } + return $false +} + +function Prepare-RebootChallenge([string] $ResolvedJobId) { + $endpoint = Read-Endpoint + if ($null -eq $endpoint) { Add-Check 'reboot-challenge' 'FAIL' 'Runner endpoint is required before preparing reboot verification.'; return } + $os = Get-CimInstance Win32_OperatingSystem + $challenge = [ordered]@{ + schema = 'pathlab.service-reboot-challenge/1' + createdAt = [DateTimeOffset]::UtcNow.ToString('O') + osBootTime = ([DateTimeOffset]$os.LastBootUpTime).ToUniversalTime().ToString('O') + runnerBootId = [string]$endpoint.bootId + jobId = if ($ResolvedJobId) { $ResolvedJobId } else { '' } + } + New-Item -ItemType Directory -Path $ReportRoot -Force | Out-Null + Write-Atomic (Join-Path $ReportRoot 'reboot-challenge.json') ($challenge | ConvertTo-Json -Depth 5) + Add-Check 'reboot-challenge' 'PASS' 'Challenge recorded. Reboot manually, then run VerifyReboot.' @{ hasJob = [bool]$ResolvedJobId } +} + +function Verify-RebootChallenge { + $path = Join-Path $ReportRoot 'reboot-challenge.json' + if (-not (Test-Path -LiteralPath $path -PathType Leaf)) { Add-Check 'reboot-continuation' 'NOT_EVALUABLE' 'No reboot challenge is present.'; return } + $challenge = Get-Content -LiteralPath $path -Raw | ConvertFrom-Json + if ($challenge.schema -ne 'pathlab.service-reboot-challenge/1') { Add-Check 'reboot-continuation' 'FAIL' 'Reboot challenge schema is invalid.'; return } + $os = Get-CimInstance Win32_OperatingSystem + $currentBoot = ([DateTimeOffset]$os.LastBootUpTime).ToUniversalTime() + $endpoint = Wait-Runner -Seconds 60 + $passed = $currentBoot -gt [DateTimeOffset]::Parse($challenge.osBootTime) -and $null -ne $endpoint -and $endpoint.bootId -ne $challenge.runnerBootId + Add-Check 'reboot-continuation' $(if ($passed) {'PASS'} else {'FAIL'}) 'OS and runner boot identities must both change and recover without login.' @{ + osBootChanged = $currentBoot -gt [DateTimeOffset]::Parse($challenge.osBootTime) + runnerBootChanged = $null -ne $endpoint -and $endpoint.bootId -ne $challenge.runnerBootId + } + if ($passed -and $challenge.jobId) { + $continued = Wait-Job ([string]$challenge.jobId) $false + Add-Check 'reboot-job-continuation' $(if ($continued) {'PASS'} else {'FAIL'}) 'The pre-reboot bounded job recovered from its verified checkpoint and completed.' @{ + jobId = [string]$challenge.jobId + } + } elseif (-not $challenge.jobId) { + Add-Check 'reboot-job-continuation' 'NOT_EVALUABLE' 'The reboot challenge did not include a bounded job.' + } +} + +function Add-SummaryChecks { + $required = @( + 'bounded-job-submission','bounded-job-completion','session0-gpu-inference', + 'service-restart','reboot-job-active','reboot-continuation','reboot-job-continuation' + ) + $currentRuntime = if (Test-Path -LiteralPath (Join-Path $ProgramRoot 'active-version.txt') -PathType Leaf) { + (Get-Content -LiteralPath (Join-Path $ProgramRoot 'active-version.txt') -Raw).Trim() + } else { '' } + $reports = [Collections.Generic.List[object]]::new() + if (Test-Path -LiteralPath $ReportRoot -PathType Container) { + foreach ($file in Get-ChildItem -LiteralPath $ReportRoot -Filter 'service-acceptance-*.json' -File) { + try { + $report = Get-Content -LiteralPath $file.FullName -Raw | ConvertFrom-Json + if ($report.schema -eq 'pathlab.service-acceptance/1' -and $report.mode -ne 'Summary' ` + -and $currentRuntime -and $report.runtimeVersion -eq $currentRuntime) { $reports.Add($report) } + } catch { } + } + } + foreach ($name in $required) { + $passing = @($reports | ForEach-Object { $_.checks } | Where-Object { $_.name -eq $name -and $_.status -eq 'PASS' } | + Select-Object -First 1) + if ($passing.Count -gt 0) { + Add-Check $name 'PASS' 'A prior host acceptance report contains passing evidence.' + } else { + Add-Check $name 'NOT_EVALUABLE' 'No passing host acceptance report is available yet.' + } + } +} + +if ($Mode -eq 'Summary') { + Add-StaticChecks + Add-SummaryChecks +} else { + Add-StaticChecks + $resolvedJobId = $null + if ($JobRequestPath) { $resolvedJobId = Submit-BoundedJob } + switch ($Mode) { + 'ServiceRestart' { + if ($null -eq $resolvedJobId) { Add-Check 'service-restart' 'NOT_EVALUABLE' 'A bounded job request is required for restart recovery.' } + else { + $null = Wait-Job $resolvedJobId $true + if (@($checks | Where-Object name -eq 'service-restart').Count -eq 0) { + Add-Check 'service-restart' 'FAIL' 'The bounded job finished or timed out before service restart recovery was demonstrated.' + } + } + } + 'PrepareReboot' { + if ($null -eq $resolvedJobId) { + Add-Check 'reboot-job-active' 'NOT_EVALUABLE' 'A bounded job request is required for reboot recovery acceptance.' + } elseif (-not (Wait-JobActive $resolvedJobId)) { + $resolvedJobId = $null + } + Prepare-RebootChallenge $resolvedJobId + } + 'VerifyReboot' { Verify-RebootChallenge } + default { if ($null -ne $resolvedJobId) { $null = Wait-Job $resolvedJobId $false } } + } +} + +$result = Write-AcceptanceReport +$result | ConvertTo-Json -Compress +if (-not $AllowIncomplete) { + if ($result.verdict -eq 'FAIL') { exit 1 } + if ($result.verdict -eq 'NOT_EVALUABLE') { exit 2 } +} From 21c751684985ce37f3f0479c137c8ac35e2b9a3d Mon Sep 17 00:00:00 2001 From: Enksodsoon <229880765+Enksodsoon@users.noreply.github.com> Date: Sat, 22 Aug 2026 12:17:20 +0700 Subject: [PATCH 09/50] fix(evidence): support elevated Windows PowerShell install --- docs/evidence-mentor-runner.md | 2 +- scripts/evidence-mentor-service.ps1 | 8 +++- scripts/test-evidence-mentor-service.ps1 | 48 +++++++++++++++---- .../forge/evidence/EvidenceMentorRunner.java | 2 +- .../evidence/EvidenceMentorRunnerTest.java | 1 + 5 files changed, 47 insertions(+), 14 deletions(-) diff --git a/docs/evidence-mentor-runner.md b/docs/evidence-mentor-runner.md index a573dc5..dc60408 100644 --- a/docs/evidence-mentor-runner.md +++ b/docs/evidence-mentor-runner.md @@ -38,7 +38,7 @@ First build a versioned distribution: Then run the administrator-approved installer with a Java 17 runtime: ```powershell -.\scripts\evidence-mentor-service.ps1 -Action Install -Version '2.0.0' ` +.\scripts\evidence-mentor-service.ps1 -Action Install -Version '2.0.1' ` -DistributionPath '.\build\install\pathlab-forge' -JavaHome 'C:\Path\To\jdk-17' ``` diff --git a/scripts/evidence-mentor-service.ps1 b/scripts/evidence-mentor-service.ps1 index f9990b2..32186dc 100644 --- a/scripts/evidence-mentor-service.ps1 +++ b/scripts/evidence-mentor-service.ps1 @@ -1,7 +1,7 @@ [CmdletBinding(SupportsShouldProcess)] param( [Parameter(Mandatory)] [ValidateSet('Install','Upgrade','Uninstall','Status')] [string] $Action, - [string] $Version = '2.0.0', + [string] $Version = '2.0.1', [string] $DistributionPath, [string] $JavaHome, [string] $ProgramRoot = 'C:\ProgramData\PathLab\EvidenceMentor', @@ -31,7 +31,11 @@ function Assert-Administrator { function Write-AtomicText([string] $Path, [string] $Content) { $partial = "$Path.partial" [IO.File]::WriteAllText($partial, $Content, [Text.UTF8Encoding]::new($false)) - [IO.File]::Move($partial, $Path, $true) + if (Test-Path -LiteralPath $Path -PathType Leaf) { + [IO.File]::Replace($partial, $Path, $null) + } else { + [IO.File]::Move($partial, $Path) + } } function Ensure-WinSW { diff --git a/scripts/test-evidence-mentor-service.ps1 b/scripts/test-evidence-mentor-service.ps1 index a670f18..e55b0e2 100644 --- a/scripts/test-evidence-mentor-service.ps1 +++ b/scripts/test-evidence-mentor-service.ps1 @@ -54,6 +54,31 @@ function Read-Token { return $value } +function Invoke-WebRequestCompat([hashtable] $Parameters) { + $request = @{} + $Parameters + if ($PSVersionTable.PSVersion.Major -le 5) { + $request.UseBasicParsing = $true + } else { + $request.SkipHttpErrorCheck = $true + } + try { + return Invoke-WebRequest @request + } catch [Net.WebException] { + $response = $_.Exception.Response + if ($null -eq $response) { throw } + $stream = $response.GetResponseStream() + try { + $reader = [IO.StreamReader]::new($stream) + try { $content = $reader.ReadToEnd() } finally { $reader.Dispose() } + } finally { if ($null -ne $stream) { $stream.Dispose() } } + return [pscustomobject]@{ + StatusCode = [int]$response.StatusCode + Content = $content + Headers = $response.Headers + } + } +} + function Invoke-Runner([string] $Method, [string] $Path, [object] $Body = $null) { $endpoint = Read-Endpoint $token = Read-Token @@ -63,13 +88,12 @@ function Invoke-Runner([string] $Method, [string] $Path, [object] $Body = $null) Uri = "http://127.0.0.1:$($endpoint.port)$Path" Headers = @{ Authorization = "Bearer $token" } TimeoutSec = 3 - SkipHttpErrorCheck = $true } if ($null -ne $Body) { $parameters.ContentType = 'application/json' $parameters.Body = $Body | ConvertTo-Json -Compress -Depth 8 } - return Invoke-WebRequest @parameters + return Invoke-WebRequestCompat $parameters } function Wait-Runner([string] $PreviousBootId = '', [int] $Seconds = 60) { @@ -101,7 +125,11 @@ function Test-Acl([string] $Path, [string] $Identity, [Security.AccessControl.Fi function Write-Atomic([string] $Path, [string] $Content) { $partial = "$Path.partial" [IO.File]::WriteAllText($partial, $Content, [Text.UTF8Encoding]::new($false)) - [IO.File]::Move($partial, $Path, $true) + if (Test-Path -LiteralPath $Path -PathType Leaf) { + [IO.File]::Replace($partial, $Path, $null) + } else { + [IO.File]::Move($partial, $Path) + } } function Write-AcceptanceReport { @@ -224,18 +252,18 @@ function Add-StaticChecks { } catch { Add-Check 'authenticated-health' 'FAIL' 'Authenticated health request failed.' } try { $origin = "http://127.0.0.1:$($endpoint.port)" - $unauthorized = Invoke-WebRequest -Uri "$origin/v1/status" -SkipHttpErrorCheck -TimeoutSec 3 + $unauthorized = Invoke-WebRequestCompat @{ Uri = "$origin/v1/status"; TimeoutSec = 3 } $created = Invoke-Runner 'POST' '/v1/dashboard-sessions' $code = ($created.Content | ConvertFrom-Json).code $exchangeBody = @{ code = $code } | ConvertTo-Json -Compress - $first = Invoke-WebRequest -Method Post -Uri "$origin/v1/dashboard-session/exchange" -Headers @{ Origin = $origin } ` - -ContentType 'application/json' -Body $exchangeBody -SkipHttpErrorCheck -TimeoutSec 3 - $reuse = Invoke-WebRequest -Method Post -Uri "$origin/v1/dashboard-session/exchange" -Headers @{ Origin = $origin } ` - -ContentType 'application/json' -Body $exchangeBody -SkipHttpErrorCheck -TimeoutSec 3 + $first = Invoke-WebRequestCompat @{ Method = 'Post'; Uri = "$origin/v1/dashboard-session/exchange"; + Headers = @{ Origin = $origin }; ContentType = 'application/json'; Body = $exchangeBody; TimeoutSec = 3 } + $reuse = Invoke-WebRequestCompat @{ Method = 'Post'; Uri = "$origin/v1/dashboard-session/exchange"; + Headers = @{ Origin = $origin }; ContentType = 'application/json'; Body = $exchangeBody; TimeoutSec = 3 } + $setCookie = [string]($first.Headers['Set-Cookie'] -join ';') $secure = $unauthorized.StatusCode -eq 401 -and $created.StatusCode -eq 201 ` -and $first.StatusCode -eq 200 -and $reuse.StatusCode -eq 401 ` - -and ($first.Headers.'Set-Cookie' -join ';') -match 'HttpOnly' ` - -and ($first.Headers.'Set-Cookie' -join ';') -match 'SameSite=Strict' + -and $setCookie -match 'HttpOnly' -and $setCookie -match 'SameSite=Strict' Add-Check 'dashboard-session-security' $(if ($secure) {'PASS'} else {'FAIL'}) 'Bearer protection and one-time dashboard exchange.' @{ unauthorized = $unauthorized.StatusCode; created = $created.StatusCode; exchanged = $first.StatusCode; reused = $reuse.StatusCode } diff --git a/src/main/java/org/pathlab/forge/evidence/EvidenceMentorRunner.java b/src/main/java/org/pathlab/forge/evidence/EvidenceMentorRunner.java index c51aaca..222bac4 100644 --- a/src/main/java/org/pathlab/forge/evidence/EvidenceMentorRunner.java +++ b/src/main/java/org/pathlab/forge/evidence/EvidenceMentorRunner.java @@ -29,7 +29,7 @@ /** Standalone authenticated loopback process for unattended Evidence Mentor jobs. */ public final class EvidenceMentorRunner implements AutoCloseable { private static final ObjectMapper JSON = new ObjectMapper(); - private static final String VERSION = "2.0.0"; + private static final String VERSION = "2.0.1"; private static final Duration LEASE = Duration.ofSeconds(45); private final Path stateRoot; private final String token; diff --git a/src/test/java/org/pathlab/forge/evidence/EvidenceMentorRunnerTest.java b/src/test/java/org/pathlab/forge/evidence/EvidenceMentorRunnerTest.java index 22ff8da..4600e14 100644 --- a/src/test/java/org/pathlab/forge/evidence/EvidenceMentorRunnerTest.java +++ b/src/test/java/org/pathlab/forge/evidence/EvidenceMentorRunnerTest.java @@ -66,6 +66,7 @@ void acceptsAuthenticatedLoopbackSubmissionIntoDurableQueue() throws Exception { var endpoint = JSON.readTree(temporaryDirectory.resolve("state/endpoint.json").toFile()); assertEquals("pathlab.runner-endpoint/1", endpoint.path("schema").asText()); + assertEquals("2.0.1", endpoint.path("serviceVersion").asText()); assertEquals(runner.uri("/").getPort(), endpoint.path("port").asInt()); assertTrue(!endpoint.has("token")); } From 8b72cc267c201839b40579b5c0465159204a14a9 Mon Sep 17 00:00:00 2001 From: Enksodsoon <229880765+Enksodsoon@users.noreply.github.com> Date: Sat, 22 Aug 2026 12:33:34 +0700 Subject: [PATCH 10/50] fix(evidence): enable Windows service SID --- docs/evidence-mentor-runner.md | 5 +++-- scripts/evidence-mentor-service.ps1 | 10 +++++++++- scripts/test-evidence-mentor-service.ps1 | 4 ++++ .../pathlab/forge/evidence/EvidenceMentorRunner.java | 2 +- .../forge/evidence/EvidenceMentorRunnerTest.java | 2 +- 5 files changed, 18 insertions(+), 5 deletions(-) diff --git a/docs/evidence-mentor-runner.md b/docs/evidence-mentor-runner.md index dc60408..92d30cc 100644 --- a/docs/evidence-mentor-runner.md +++ b/docs/evidence-mentor-runner.md @@ -38,7 +38,7 @@ First build a versioned distribution: Then run the administrator-approved installer with a Java 17 runtime: ```powershell -.\scripts\evidence-mentor-service.ps1 -Action Install -Version '2.0.1' ` +.\scripts\evidence-mentor-service.ps1 -Action Install -Version '2.0.2' ` -DistributionPath '.\build\install\pathlab-forge' -JavaHome 'C:\Path\To\jdk-17' ``` @@ -48,7 +48,8 @@ It installs `PathLabEvidenceMentor` as delayed-auto-start under `NT AUTHORITY\LocalService`, stages runtimes side-by-side under `C:\ProgramData\PathLab\EvidenceMentor\runtime\`, keeps state under `D:\PathLabData\EvidenceMentor\state`, applies service-SID ACLs and outbound-deny -rules, and rolls back the active configuration if health checks fail. +rules, enables the restricted service identity's per-service SID, and rolls back +the active configuration if health checks fail. Use `-Action Upgrade` with a new immutable version. `-Action Uninstall` removes the service, firewall rules, and Start Menu shortcut while preserving queues, diff --git a/scripts/evidence-mentor-service.ps1 b/scripts/evidence-mentor-service.ps1 index 32186dc..818f12e 100644 --- a/scripts/evidence-mentor-service.ps1 +++ b/scripts/evidence-mentor-service.ps1 @@ -1,7 +1,7 @@ [CmdletBinding(SupportsShouldProcess)] param( [Parameter(Mandatory)] [ValidateSet('Install','Upgrade','Uninstall','Status')] [string] $Action, - [string] $Version = '2.0.1', + [string] $Version = '2.0.2', [string] $DistributionPath, [string] $JavaHome, [string] $ProgramRoot = 'C:\ProgramData\PathLab\EvidenceMentor', @@ -70,6 +70,13 @@ function Grant-PathLabAccess([string] $InstallingUser) { } } +function Enable-ServiceSid { + & sc.exe sidtype $serviceName unrestricted | Out-Null + if ($LASTEXITCODE -ne 0) { + throw 'Windows could not enable the PathLab service SID.' + } +} + function New-ServiceConfig([string] $RuntimePath) { $java = [Security.SecurityElement]::Escape((Join-Path $RuntimePath 'jre\bin\java.exe')) $classpath = [Security.SecurityElement]::Escape((Join-Path $RuntimePath 'app\lib\*')) @@ -179,6 +186,7 @@ if ($PSCmdlet.ShouldProcess($serviceName, "$Action autonomous service version $V if ($Action -eq 'Upgrade' -and (Get-Service -Name $serviceName -ErrorAction SilentlyContinue)) { & $wrapperPath stop } Write-AtomicText $configPath (New-ServiceConfig $runtimePath) if (-not (Get-Service -Name $serviceName -ErrorAction SilentlyContinue)) { & $wrapperPath install } + Enable-ServiceSid Grant-PathLabAccess $installingUser Install-FirewallRules $runtimePath & $wrapperPath start diff --git a/scripts/test-evidence-mentor-service.ps1 b/scripts/test-evidence-mentor-service.ps1 index e55b0e2..f0dc0cf 100644 --- a/scripts/test-evidence-mentor-service.ps1 +++ b/scripts/test-evidence-mentor-service.ps1 @@ -178,10 +178,14 @@ function Add-StaticChecks { Add-Check 'service-installed' 'FAIL' 'Windows service is not installed.' Add-Check 'service-running' 'NOT_EVALUABLE' 'Service installation is required first.' Add-Check 'service-identity' 'NOT_EVALUABLE' 'Service installation is required first.' + Add-Check 'service-sid-enabled' 'NOT_EVALUABLE' 'Service installation is required first.' } else { Add-Check 'service-installed' 'PASS' 'Windows service is installed.' @{ startMode = $service.StartMode } Add-Check 'service-running' $(if ($service.State -eq 'Running') {'PASS'} else {'FAIL'}) "Service state is $($service.State)." Add-Check 'service-identity' $(if ($service.StartName -ieq 'NT AUTHORITY\LocalService') {'PASS'} else {'FAIL'}) "Service identity is $($service.StartName)." + $sidType = (& sc.exe qsidtype $serviceName 2>$null) -join [Environment]::NewLine + $sidEnabled = $LASTEXITCODE -eq 0 -and $sidType -match 'UNRESTRICTED' + Add-Check 'service-sid-enabled' $(if ($sidEnabled) {'PASS'} else {'FAIL'}) 'The service SID must be enabled before SID-only runtime ACLs can authorize startup.' } $serviceRegistry = "HKLM:\SYSTEM\CurrentControlSet\Services\$serviceName" diff --git a/src/main/java/org/pathlab/forge/evidence/EvidenceMentorRunner.java b/src/main/java/org/pathlab/forge/evidence/EvidenceMentorRunner.java index 222bac4..961348c 100644 --- a/src/main/java/org/pathlab/forge/evidence/EvidenceMentorRunner.java +++ b/src/main/java/org/pathlab/forge/evidence/EvidenceMentorRunner.java @@ -29,7 +29,7 @@ /** Standalone authenticated loopback process for unattended Evidence Mentor jobs. */ public final class EvidenceMentorRunner implements AutoCloseable { private static final ObjectMapper JSON = new ObjectMapper(); - private static final String VERSION = "2.0.1"; + private static final String VERSION = "2.0.2"; private static final Duration LEASE = Duration.ofSeconds(45); private final Path stateRoot; private final String token; diff --git a/src/test/java/org/pathlab/forge/evidence/EvidenceMentorRunnerTest.java b/src/test/java/org/pathlab/forge/evidence/EvidenceMentorRunnerTest.java index 4600e14..4e9fcb6 100644 --- a/src/test/java/org/pathlab/forge/evidence/EvidenceMentorRunnerTest.java +++ b/src/test/java/org/pathlab/forge/evidence/EvidenceMentorRunnerTest.java @@ -66,7 +66,7 @@ void acceptsAuthenticatedLoopbackSubmissionIntoDurableQueue() throws Exception { var endpoint = JSON.readTree(temporaryDirectory.resolve("state/endpoint.json").toFile()); assertEquals("pathlab.runner-endpoint/1", endpoint.path("schema").asText()); - assertEquals("2.0.1", endpoint.path("serviceVersion").asText()); + assertEquals("2.0.2", endpoint.path("serviceVersion").asText()); assertEquals(runner.uri("/").getPort(), endpoint.path("port").asInt()); assertTrue(!endpoint.has("token")); } From 27cacfe2585987b8b9028a7a56d9366fe9a93eb2 Mon Sep 17 00:00:00 2001 From: Enksodsoon <229880765+Enksodsoon@users.noreply.github.com> Date: Sat, 22 Aug 2026 12:37:02 +0700 Subject: [PATCH 11/50] fix(evidence): resume verified service staging --- docs/evidence-mentor-runner.md | 6 +++++ scripts/evidence-mentor-service.ps1 | 37 +++++++++++++++++++++++++++-- 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/docs/evidence-mentor-runner.md b/docs/evidence-mentor-runner.md index 92d30cc..bfcb38d 100644 --- a/docs/evidence-mentor-runner.md +++ b/docs/evidence-mentor-runner.md @@ -55,6 +55,12 @@ Use `-Action Upgrade` with a new immutable version. `-Action Uninstall` removes the service, firewall rules, and Start Menu shortcut while preserving queues, checkpoints, signing material, evidence, and model packs. +If an elevated install is interrupted after staging but before activation, rerun +the identical command with `-ReuseStagedRuntime`. The installer hashes every +application and Java runtime file against the requested inputs before reuse. It +refuses missing, extra, modified, or mismatched files and never overwrites the +staged version. + ## Host acceptance and the definition of done Installation is not acceptance. Run the installed acceptance harness from an diff --git a/scripts/evidence-mentor-service.ps1 b/scripts/evidence-mentor-service.ps1 index 818f12e..d35ad5d 100644 --- a/scripts/evidence-mentor-service.ps1 +++ b/scripts/evidence-mentor-service.ps1 @@ -5,7 +5,8 @@ param( [string] $DistributionPath, [string] $JavaHome, [string] $ProgramRoot = 'C:\ProgramData\PathLab\EvidenceMentor', - [string] $StateRoot = 'D:\PathLabData\EvidenceMentor\state' + [string] $StateRoot = 'D:\PathLabData\EvidenceMentor\state', + [switch] $ReuseStagedRuntime ) $ErrorActionPreference = 'Stop' @@ -143,11 +144,43 @@ function Wait-RunnerHealth([int] $Seconds = 30) { return $false } +function Assert-DirectoryMirror([string] $Source, [string] $Target, [string] $Label) { + if (-not (Test-Path -LiteralPath $Source -PathType Container) -or + -not (Test-Path -LiteralPath $Target -PathType Container)) { + throw "$Label staging directory is incomplete." + } + $sourceRoot = [IO.Path]::GetFullPath($Source).TrimEnd('\') + $targetRoot = [IO.Path]::GetFullPath($Target).TrimEnd('\') + $sourceFiles = @(Get-ChildItem -LiteralPath $sourceRoot -File -Recurse) + $targetFiles = @(Get-ChildItem -LiteralPath $targetRoot -File -Recurse) + if ($sourceFiles.Count -ne $targetFiles.Count) { + throw "$Label staged file count does not match the requested input." + } + foreach ($sourceFile in $sourceFiles) { + $relative = $sourceFile.FullName.Substring($sourceRoot.Length).TrimStart('\') + $targetFile = Join-Path $targetRoot $relative + if (-not (Test-Path -LiteralPath $targetFile -PathType Leaf)) { + throw "$Label staged file is missing: $relative" + } + $candidate = Get-Item -LiteralPath $targetFile + if ($candidate.Length -ne $sourceFile.Length -or + (Get-FileHash -LiteralPath $sourceFile.FullName -Algorithm SHA256).Hash -ne + (Get-FileHash -LiteralPath $candidate.FullName -Algorithm SHA256).Hash) { + throw "$Label staged file checksum does not match: $relative" + } + } +} + function Stage-Runtime { if (-not $DistributionPath -or -not (Test-Path -LiteralPath $DistributionPath -PathType Container)) { throw 'DistributionPath must reference a completed installDist directory.' } if (-not $JavaHome -or -not (Test-Path -LiteralPath (Join-Path $JavaHome 'bin\java.exe') -PathType Leaf)) { throw 'JavaHome must reference a Java 17 runtime.' } $target = Join-Path $runtimeRoot $Version - if (Test-Path -LiteralPath $target) { throw "Runtime version already exists: $target" } + if (Test-Path -LiteralPath $target) { + if (-not $ReuseStagedRuntime) { throw "Runtime version already exists: $target" } + Assert-DirectoryMirror $DistributionPath (Join-Path $target 'app') 'Application runtime' + Assert-DirectoryMirror $JavaHome (Join-Path $target 'jre') 'Java runtime' + return $target + } New-Item -ItemType Directory -Path (Join-Path $target 'app') -Force | Out-Null Copy-Item -Path (Join-Path $DistributionPath '*') -Destination (Join-Path $target 'app') -Recurse -Force Copy-Item -LiteralPath $JavaHome -Destination (Join-Path $target 'jre') -Recurse -Force From 74e1834eaec038f6158495ef1c458ac1b1fcceaf Mon Sep 17 00:00:00 2001 From: Enksodsoon <229880765+Enksodsoon@users.noreply.github.com> Date: Sat, 22 Aug 2026 12:50:39 +0700 Subject: [PATCH 12/50] fix(evidence): complete service activation safely --- scripts/evidence-mentor-service.ps1 | 26 +++++++++++++++++------- scripts/test-evidence-mentor-service.ps1 | 9 +++++++- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/scripts/evidence-mentor-service.ps1 b/scripts/evidence-mentor-service.ps1 index d35ad5d..a759016 100644 --- a/scripts/evidence-mentor-service.ps1 +++ b/scripts/evidence-mentor-service.ps1 @@ -33,7 +33,14 @@ function Write-AtomicText([string] $Path, [string] $Content) { $partial = "$Path.partial" [IO.File]::WriteAllText($partial, $Content, [Text.UTF8Encoding]::new($false)) if (Test-Path -LiteralPath $Path -PathType Leaf) { - [IO.File]::Replace($partial, $Path, $null) + $backup = "$Path.backup-$([Guid]::NewGuid().ToString('N'))" + try { + [IO.File]::Replace($partial, $Path, $backup) + } finally { + if (Test-Path -LiteralPath $backup -PathType Leaf) { + Remove-Item -LiteralPath $backup -Force + } + } } else { [IO.File]::Move($partial, $Path) } @@ -55,7 +62,7 @@ function Ensure-WinSW { function Grant-PathLabAccess([string] $InstallingUser) { & icacls.exe $ProgramRoot /inheritance:r /grant:r 'SYSTEM:(OI)(CI)F' 'Administrators:(OI)(CI)F' "NT SERVICE\${serviceName}:(OI)(CI)RX" "${InstallingUser}:(OI)(CI)RX" | Out-Null & icacls.exe $StateRoot /inheritance:r /grant:r 'SYSTEM:(OI)(CI)F' 'Administrators:(OI)(CI)F' "NT SERVICE\${serviceName}:(OI)(CI)M" "${InstallingUser}:(RX)" | Out-Null - foreach ($operatorPath in @('inputs','requests','models','artifacts')) { + foreach ($operatorPath in @('inputs','requests','models','artifacts','acceptance')) { $resolved = Join-Path $StateRoot $operatorPath & icacls.exe $resolved /inheritance:r /grant:r 'SYSTEM:(OI)(CI)F' 'Administrators:(OI)(CI)F' "NT SERVICE\${serviceName}:(OI)(CI)M" "${InstallingUser}:(OI)(CI)M" | Out-Null } @@ -64,10 +71,15 @@ function Grant-PathLabAccess([string] $InstallingUser) { & icacls.exe $resolved /inheritance:r /grant:r 'SYSTEM:(OI)(CI)F' 'Administrators:(OI)(CI)F' "NT SERVICE\${serviceName}:(OI)(CI)M" | Out-Null } & icacls.exe $logRoot /inheritance:r /grant:r 'SYSTEM:(OI)(CI)F' 'Administrators:(OI)(CI)F' "NT SERVICE\${serviceName}:(OI)(CI)M" "${InstallingUser}:(OI)(CI)R" | Out-Null - foreach ($sensitive in @((Join-Path $StateRoot 'ipc-token'), (Join-Path $StateRoot 'endpoint.json'))) { - if (Test-Path -LiteralPath $sensitive) { - & icacls.exe $sensitive /inheritance:r /grant:r 'SYSTEM:F' 'Administrators:F' "NT SERVICE\${serviceName}:R" "${InstallingUser}:R" | Out-Null - } + $tokenPath = Join-Path $StateRoot 'ipc-token' + if (Test-Path -LiteralPath $tokenPath) { + & icacls.exe $tokenPath /inheritance:r /grant:r 'SYSTEM:F' 'Administrators:F' ` + "NT SERVICE\${serviceName}:R" "${InstallingUser}:R" | Out-Null + } + $endpointPath = Join-Path $StateRoot 'endpoint.json' + if (Test-Path -LiteralPath $endpointPath) { + & icacls.exe $endpointPath /inheritance:r /grant:r 'SYSTEM:F' 'Administrators:F' ` + "NT SERVICE\${serviceName}:M" "${InstallingUser}:R" | Out-Null } } @@ -209,7 +221,7 @@ if ($Action -eq 'Uninstall') { $installingUser = [Security.Principal.WindowsIdentity]::GetCurrent().Name if ($PSCmdlet.ShouldProcess($serviceName, "$Action autonomous service version $Version")) { - $stateDirectories = @('inputs','requests','models','artifacts','checkpoints','signing','logs') | ForEach-Object { Join-Path $StateRoot $_ } + $stateDirectories = @('inputs','requests','models','artifacts','acceptance','checkpoints','signing','logs') | ForEach-Object { Join-Path $StateRoot $_ } New-Item -ItemType Directory -Path @($ProgramRoot, $runtimeRoot, $StateRoot) -Force | Out-Null New-Item -ItemType Directory -Path $stateDirectories -Force | Out-Null $previousVersion = if (Test-Path -LiteralPath $activeVersionPath) { (Get-Content -LiteralPath $activeVersionPath -Raw).Trim() } else { '' } diff --git a/scripts/test-evidence-mentor-service.ps1 b/scripts/test-evidence-mentor-service.ps1 index f0dc0cf..20601be 100644 --- a/scripts/test-evidence-mentor-service.ps1 +++ b/scripts/test-evidence-mentor-service.ps1 @@ -126,7 +126,14 @@ function Write-Atomic([string] $Path, [string] $Content) { $partial = "$Path.partial" [IO.File]::WriteAllText($partial, $Content, [Text.UTF8Encoding]::new($false)) if (Test-Path -LiteralPath $Path -PathType Leaf) { - [IO.File]::Replace($partial, $Path, $null) + $backup = "$Path.backup-$([Guid]::NewGuid().ToString('N'))" + try { + [IO.File]::Replace($partial, $Path, $backup) + } finally { + if (Test-Path -LiteralPath $backup -PathType Leaf) { + Remove-Item -LiteralPath $backup -Force + } + } } else { [IO.File]::Move($partial, $Path) } From 799935b89d169d5907c4d0cf74435f34dd873060 Mon Sep 17 00:00:00 2001 From: Enksodsoon <229880765+Enksodsoon@users.noreply.github.com> Date: Sat, 22 Aug 2026 12:57:07 +0700 Subject: [PATCH 13/50] Recover service jobs immediately after restart --- docs/evidence-mentor-runner.md | 2 +- scripts/evidence-mentor-service.ps1 | 2 +- scripts/test-evidence-mentor-service.ps1 | 8 ++++---- .../forge/evidence/EvidenceJobQueue.java | 16 ++++++++++++++++ .../forge/evidence/EvidenceMentorRunner.java | 3 ++- .../forge/evidence/EvidenceJobQueueTest.java | 19 +++++++++++++++++++ .../evidence/EvidenceMentorRunnerTest.java | 2 +- 7 files changed, 44 insertions(+), 8 deletions(-) diff --git a/docs/evidence-mentor-runner.md b/docs/evidence-mentor-runner.md index bfcb38d..b087c83 100644 --- a/docs/evidence-mentor-runner.md +++ b/docs/evidence-mentor-runner.md @@ -38,7 +38,7 @@ First build a versioned distribution: Then run the administrator-approved installer with a Java 17 runtime: ```powershell -.\scripts\evidence-mentor-service.ps1 -Action Install -Version '2.0.2' ` +.\scripts\evidence-mentor-service.ps1 -Action Install -Version '2.0.3' ` -DistributionPath '.\build\install\pathlab-forge' -JavaHome 'C:\Path\To\jdk-17' ``` diff --git a/scripts/evidence-mentor-service.ps1 b/scripts/evidence-mentor-service.ps1 index a759016..e5186d1 100644 --- a/scripts/evidence-mentor-service.ps1 +++ b/scripts/evidence-mentor-service.ps1 @@ -1,7 +1,7 @@ [CmdletBinding(SupportsShouldProcess)] param( [Parameter(Mandatory)] [ValidateSet('Install','Upgrade','Uninstall','Status')] [string] $Action, - [string] $Version = '2.0.2', + [string] $Version = '2.0.3', [string] $DistributionPath, [string] $JavaHome, [string] $ProgramRoot = 'C:\ProgramData\PathLab\EvidenceMentor', diff --git a/scripts/test-evidence-mentor-service.ps1 b/scripts/test-evidence-mentor-service.ps1 index 20601be..d84dc27 100644 --- a/scripts/test-evidence-mentor-service.ps1 +++ b/scripts/test-evidence-mentor-service.ps1 @@ -315,10 +315,9 @@ function Wait-Job([string] $ResolvedJobId, [bool] $RestartWhenActive) { $deadline = [DateTimeOffset]::UtcNow.AddMinutes($TimeoutMinutes) $restarted = $false do { - Start-Sleep -Seconds 1 try { $response = Invoke-Runner 'GET' "/v1/jobs/$ResolvedJobId" - if ($response.StatusCode -ne 200) { continue } + if ($response.StatusCode -ne 200) { Start-Sleep -Seconds 1; continue } $job = $response.Content | ConvertFrom-Json if ($RestartWhenActive -and -not $restarted -and $job.state -in @('validating','running','refining','packaging')) { if (-not (Test-Administrator)) { Add-Check 'service-restart' 'NOT_EVALUABLE' 'Administrator privileges are required.'; $RestartWhenActive = $false; continue } @@ -344,6 +343,7 @@ function Wait-Job([string] $ResolvedJobId, [bool] $RestartWhenActive) { return $passed } } catch { } + Start-Sleep -Seconds 1 } while ([DateTimeOffset]::UtcNow -lt $deadline) Add-Check 'bounded-job-completion' 'FAIL' 'Bounded job did not complete before the acceptance timeout.' @{ jobId = $ResolvedJobId } return $false @@ -353,10 +353,9 @@ function Wait-JobActive([string] $ResolvedJobId, [int] $Seconds = 60) { if (-not $ResolvedJobId) { return $false } $deadline = [DateTimeOffset]::UtcNow.AddSeconds($Seconds) do { - Start-Sleep -Seconds 1 try { $response = Invoke-Runner 'GET' "/v1/jobs/$ResolvedJobId" - if ($response.StatusCode -ne 200) { continue } + if ($response.StatusCode -ne 200) { Start-Sleep -Seconds 1; continue } $job = $response.Content | ConvertFrom-Json if ($job.state -in @('validating','running','refining','packaging')) { Add-Check 'reboot-job-active' 'PASS' 'The bounded job is active and checkpoint-capable before manual reboot.' @{ @@ -371,6 +370,7 @@ function Wait-JobActive([string] $ResolvedJobId, [int] $Seconds = 60) { return $false } } catch { } + Start-Sleep -Seconds 1 } while ([DateTimeOffset]::UtcNow -lt $deadline) Add-Check 'reboot-job-active' 'FAIL' 'The bounded job did not become active before the reboot challenge timeout.' @{ jobId = $ResolvedJobId } return $false diff --git a/src/main/java/org/pathlab/forge/evidence/EvidenceJobQueue.java b/src/main/java/org/pathlab/forge/evidence/EvidenceJobQueue.java index 35b96f4..62517f7 100644 --- a/src/main/java/org/pathlab/forge/evidence/EvidenceJobQueue.java +++ b/src/main/java/org/pathlab/forge/evidence/EvidenceJobQueue.java @@ -125,6 +125,22 @@ public synchronized Optional snapshot(String id) throws IOE } catch (SQLException error) { throw new IOException("Unable to read evidence job snapshot", error); } } + /** Makes leases from the previous runner boot immediately reclaimable without changing checkpoints. */ + public synchronized int recoverOrphanedActiveJobs(Instant now) throws IOException { + try (var statement = connection.prepareStatement(""" + UPDATE evidence_jobs SET lease_owner='',lease_expires_at=?,last_heartbeat=?, + detail=?,updated_at=? WHERE state IN ('VALIDATING','RUNNING','REFINING','PACKAGING') + """)) { + statement.setString(1, Instant.EPOCH.toString()); + statement.setString(2, now.toString()); + statement.setString(3, "Recovered after runner restart; verified checkpoint will be revalidated"); + statement.setString(4, now.toString()); + return statement.executeUpdate(); + } catch (SQLException error) { + throw new IOException("Unable to recover orphaned Evidence Mentor leases", error); + } + } + public synchronized List list(EvidenceJobState state, EvidenceExecutionLane lane, int limit) throws IOException { if (limit < 1 || limit > 200) throw new IllegalArgumentException("Evidence job list limit is invalid"); var sql = new StringBuilder("SELECT * FROM evidence_jobs WHERE 1=1"); diff --git a/src/main/java/org/pathlab/forge/evidence/EvidenceMentorRunner.java b/src/main/java/org/pathlab/forge/evidence/EvidenceMentorRunner.java index 961348c..19d4993 100644 --- a/src/main/java/org/pathlab/forge/evidence/EvidenceMentorRunner.java +++ b/src/main/java/org/pathlab/forge/evidence/EvidenceMentorRunner.java @@ -29,7 +29,7 @@ /** Standalone authenticated loopback process for unattended Evidence Mentor jobs. */ public final class EvidenceMentorRunner implements AutoCloseable { private static final ObjectMapper JSON = new ObjectMapper(); - private static final String VERSION = "2.0.2"; + private static final String VERSION = "2.0.3"; private static final Duration LEASE = Duration.ofSeconds(45); private final Path stateRoot; private final String token; @@ -50,6 +50,7 @@ private EvidenceMentorRunner(Path stateRoot, int port, String token, boolean wor require(token != null && token.length() >= 32, "Loopback token is too short"); Files.createDirectories(this.stateRoot); queue = new EvidenceJobQueue(this.stateRoot.resolve("jobs.sqlite3")); + queue.recoverOrphanedActiveJobs(startedAt); dashboardHtml = readResource("/evidence-dashboard/index.html"); server = HttpServer.create(new InetSocketAddress(InetAddress.getByName("127.0.0.1"), port), 32); server.createContext("/", this::handle); diff --git a/src/test/java/org/pathlab/forge/evidence/EvidenceJobQueueTest.java b/src/test/java/org/pathlab/forge/evidence/EvidenceJobQueueTest.java index 1a37cc7..92fd7ac 100644 --- a/src/test/java/org/pathlab/forge/evidence/EvidenceJobQueueTest.java +++ b/src/test/java/org/pathlab/forge/evidence/EvidenceJobQueueTest.java @@ -37,6 +37,25 @@ void resumesExpiredLeaseAcrossRestartAndCompletes() throws Exception { } } + @Test + void newRunnerBootImmediatelyReclaimsOrphanedActiveLease() throws Exception { + var database = temporaryDirectory.resolve("boot-recovery.sqlite"); + var request = temporaryDirectory.resolve("boot-request.json"); + Files.writeString(request, "{}"); + var start = Instant.parse("2026-08-22T00:00:00Z"); + try (var queue = new EvidenceJobQueue(database)) { + queue.submit("boot-job", request, start); + queue.claimNext("old-boot-worker", start, Duration.ofMinutes(5)).orElseThrow(); + } + try (var restarted = new EvidenceJobQueue(database)) { + assertEquals(1, restarted.recoverOrphanedActiveJobs(start.plusSeconds(2))); + var recovered = restarted.claimNext("new-boot-worker", start.plusSeconds(2), + Duration.ofSeconds(45)).orElseThrow(); + assertEquals("boot-job", recovered.id()); + assertEquals("new-boot-worker", recovered.leaseOwner()); + } + } + @Test void cancellationFlagSurvivesRestart() throws Exception { var database = temporaryDirectory.resolve("jobs.sqlite"); diff --git a/src/test/java/org/pathlab/forge/evidence/EvidenceMentorRunnerTest.java b/src/test/java/org/pathlab/forge/evidence/EvidenceMentorRunnerTest.java index 4e9fcb6..ab580da 100644 --- a/src/test/java/org/pathlab/forge/evidence/EvidenceMentorRunnerTest.java +++ b/src/test/java/org/pathlab/forge/evidence/EvidenceMentorRunnerTest.java @@ -66,7 +66,7 @@ void acceptsAuthenticatedLoopbackSubmissionIntoDurableQueue() throws Exception { var endpoint = JSON.readTree(temporaryDirectory.resolve("state/endpoint.json").toFile()); assertEquals("pathlab.runner-endpoint/1", endpoint.path("schema").asText()); - assertEquals("2.0.2", endpoint.path("serviceVersion").asText()); + assertEquals("2.0.3", endpoint.path("serviceVersion").asText()); assertEquals(runner.uri("/").getPort(), endpoint.path("port").asInt()); assertTrue(!endpoint.has("token")); } From 4d5aa2c4efa8172a2f9e25975ef8297df5ccf004 Mon Sep 17 00:00:00 2001 From: Enksodsoon <229880765+Enksodsoon@users.noreply.github.com> Date: Sat, 22 Aug 2026 14:07:26 +0700 Subject: [PATCH 14/50] Preserve endpoint ACL across service restarts --- docs/evidence-mentor-runner.md | 2 +- scripts/evidence-mentor-service.ps1 | 2 +- .../forge/evidence/EvidenceMentorRunner.java | 11 ++++++- .../evidence/EvidenceMentorRunnerTest.java | 32 ++++++++++++++++++- 4 files changed, 43 insertions(+), 4 deletions(-) diff --git a/docs/evidence-mentor-runner.md b/docs/evidence-mentor-runner.md index b087c83..375deba 100644 --- a/docs/evidence-mentor-runner.md +++ b/docs/evidence-mentor-runner.md @@ -38,7 +38,7 @@ First build a versioned distribution: Then run the administrator-approved installer with a Java 17 runtime: ```powershell -.\scripts\evidence-mentor-service.ps1 -Action Install -Version '2.0.3' ` +.\scripts\evidence-mentor-service.ps1 -Action Install -Version '2.0.4' ` -DistributionPath '.\build\install\pathlab-forge' -JavaHome 'C:\Path\To\jdk-17' ``` diff --git a/scripts/evidence-mentor-service.ps1 b/scripts/evidence-mentor-service.ps1 index e5186d1..f9c8433 100644 --- a/scripts/evidence-mentor-service.ps1 +++ b/scripts/evidence-mentor-service.ps1 @@ -1,7 +1,7 @@ [CmdletBinding(SupportsShouldProcess)] param( [Parameter(Mandatory)] [ValidateSet('Install','Upgrade','Uninstall','Status')] [string] $Action, - [string] $Version = '2.0.3', + [string] $Version = '2.0.4', [string] $DistributionPath, [string] $JavaHome, [string] $ProgramRoot = 'C:\ProgramData\PathLab\EvidenceMentor', diff --git a/src/main/java/org/pathlab/forge/evidence/EvidenceMentorRunner.java b/src/main/java/org/pathlab/forge/evidence/EvidenceMentorRunner.java index 19d4993..3656855 100644 --- a/src/main/java/org/pathlab/forge/evidence/EvidenceMentorRunner.java +++ b/src/main/java/org/pathlab/forge/evidence/EvidenceMentorRunner.java @@ -11,6 +11,7 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.attribute.AclFileAttributeView; import java.security.MessageDigest; import java.security.SecureRandom; import java.time.Duration; @@ -29,7 +30,7 @@ /** Standalone authenticated loopback process for unattended Evidence Mentor jobs. */ public final class EvidenceMentorRunner implements AutoCloseable { private static final ObjectMapper JSON = new ObjectMapper(); - private static final String VERSION = "2.0.3"; + private static final String VERSION = "2.0.4"; private static final Duration LEASE = Duration.ofSeconds(45); private final Path stateRoot; private final String token; @@ -273,10 +274,18 @@ private void publishEndpoint() throws IOException { var body=JSON.writeValueAsBytes(Map.of("schema","pathlab.runner-endpoint/1","port",server.getAddress().getPort(),"serviceVersion",VERSION, "bootId",bootId,"pid",ProcessHandle.current().pid(),"startedAt",startedAt.toString())); var partial=endpointPath.resolveSibling("endpoint.json.partial"); Files.write(partial,body); + preserveEndpointAcl(endpointPath, partial); try{Files.move(partial,endpointPath,java.nio.file.StandardCopyOption.REPLACE_EXISTING,java.nio.file.StandardCopyOption.ATOMIC_MOVE);} catch(java.nio.file.AtomicMoveNotSupportedException ignored){Files.move(partial,endpointPath,java.nio.file.StandardCopyOption.REPLACE_EXISTING);} } + private static void preserveEndpointAcl(Path endpoint, Path partial) throws IOException { + if (!Files.exists(endpoint)) return; + var source = Files.getFileAttributeView(endpoint, AclFileAttributeView.class); + var target = Files.getFileAttributeView(partial, AclFileAttributeView.class); + if (source != null && target != null) target.setAcl(source.getAcl()); + } + private static Map query(String raw){var result=new java.util.HashMap();if(raw==null||raw.isBlank())return result;for(var part:raw.split("&")){var pair=part.split("=",2);require(pair.length==2&&!pair[0].isBlank(),"Invalid query");result.put(pair[0],java.net.URLDecoder.decode(pair[1],StandardCharsets.UTF_8));}return result;} private static JsonNode jsonBody(HttpExchange exchange,int limit)throws IOException{var raw=exchange.getRequestBody().readNBytes(limit+1);require(raw.length<=limit,"Request body is too large");var value=JSON.readTree(raw);require(value!=null&&value.isObject(),"JSON object required");return value;} private static Set fieldNames(JsonNode value){var result=new java.util.HashSet();value.fieldNames().forEachRemaining(result::add);return result;} diff --git a/src/test/java/org/pathlab/forge/evidence/EvidenceMentorRunnerTest.java b/src/test/java/org/pathlab/forge/evidence/EvidenceMentorRunnerTest.java index ab580da..c9fb57d 100644 --- a/src/test/java/org/pathlab/forge/evidence/EvidenceMentorRunnerTest.java +++ b/src/test/java/org/pathlab/forge/evidence/EvidenceMentorRunnerTest.java @@ -9,7 +9,12 @@ import java.net.http.HttpResponse; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.attribute.AclEntry; +import java.nio.file.attribute.AclEntryPermission; +import java.nio.file.attribute.AclEntryType; +import java.nio.file.attribute.AclFileAttributeView; import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.Assumptions; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -17,6 +22,31 @@ final class EvidenceMentorRunnerTest { private static final ObjectMapper JSON = new ObjectMapper(); @TempDir Path temporaryDirectory; + @Test + void endpointPublicationPreservesExplicitWindowsAclEntries() throws Exception { + var state = Files.createDirectories(temporaryDirectory.resolve("acl-state")); + var endpoint = state.resolve("endpoint.json"); + Files.writeString(endpoint, "{}"); + var aclView = Files.getFileAttributeView(endpoint, AclFileAttributeView.class); + Assumptions.assumeTrue(aclView != null, "Windows ACLs are required for this host-specific test"); + var principal = Files.getOwner(endpoint); + var marker = AclEntry.newBuilder() + .setType(AclEntryType.ALLOW) + .setPrincipal(principal) + .setPermissions(AclEntryPermission.READ_ATTRIBUTES) + .build(); + var acl = new java.util.ArrayList<>(aclView.getAcl()); + acl.add(0, marker); + aclView.setAcl(acl); + + try (var runner = EvidenceMentorRunner.start( + state, 0, "test-loopback-token-0123456789abcdef", false)) { + assertTrue(runner.uri("/health").getPort() > 0); + var publishedAcl = Files.getFileAttributeView(endpoint, AclFileAttributeView.class).getAcl(); + assertTrue(publishedAcl.contains(marker)); + } + } + @Test void acceptsAuthenticatedLoopbackSubmissionIntoDurableQueue() throws Exception { var requestFile = temporaryDirectory.resolve("request.json"); @@ -66,7 +96,7 @@ void acceptsAuthenticatedLoopbackSubmissionIntoDurableQueue() throws Exception { var endpoint = JSON.readTree(temporaryDirectory.resolve("state/endpoint.json").toFile()); assertEquals("pathlab.runner-endpoint/1", endpoint.path("schema").asText()); - assertEquals("2.0.3", endpoint.path("serviceVersion").asText()); + assertEquals("2.0.4", endpoint.path("serviceVersion").asText()); assertEquals(runner.uri("/").getPort(), endpoint.path("port").asInt()); assertTrue(!endpoint.has("token")); } From b34e47a75253d8a4ddb731a22e72c44b2f8d7eee Mon Sep 17 00:00:00 2001 From: Enksodsoon <229880765+Enksodsoon@users.noreply.github.com> Date: Sat, 22 Aug 2026 14:20:36 +0700 Subject: [PATCH 15/50] Resume jobs from durable service checkpoints --- docs/evidence-mentor-runner.md | 2 +- scripts/evidence-mentor-service.ps1 | 2 +- .../forge/evidence/EvidenceJobProcessor.java | 14 +++++-- .../forge/evidence/EvidenceJobQueue.java | 18 ++++++++- .../forge/evidence/EvidenceMentorRunner.java | 6 ++- .../evidence/EvidenceJobProcessorTest.java | 37 +++++++++++++++++++ .../forge/evidence/EvidenceJobQueueTest.java | 17 +++++++++ .../evidence/EvidenceMentorRunnerTest.java | 2 +- 8 files changed, 88 insertions(+), 10 deletions(-) diff --git a/docs/evidence-mentor-runner.md b/docs/evidence-mentor-runner.md index 375deba..bf3dfac 100644 --- a/docs/evidence-mentor-runner.md +++ b/docs/evidence-mentor-runner.md @@ -38,7 +38,7 @@ First build a versioned distribution: Then run the administrator-approved installer with a Java 17 runtime: ```powershell -.\scripts\evidence-mentor-service.ps1 -Action Install -Version '2.0.4' ` +.\scripts\evidence-mentor-service.ps1 -Action Install -Version '2.0.5' ` -DistributionPath '.\build\install\pathlab-forge' -JavaHome 'C:\Path\To\jdk-17' ``` diff --git a/scripts/evidence-mentor-service.ps1 b/scripts/evidence-mentor-service.ps1 index f9c8433..fd6c943 100644 --- a/scripts/evidence-mentor-service.ps1 +++ b/scripts/evidence-mentor-service.ps1 @@ -1,7 +1,7 @@ [CmdletBinding(SupportsShouldProcess)] param( [Parameter(Mandatory)] [ValidateSet('Install','Upgrade','Uninstall','Status')] [string] $Action, - [string] $Version = '2.0.4', + [string] $Version = '2.0.5', [string] $DistributionPath, [string] $JavaHome, [string] $ProgramRoot = 'C:\ProgramData\PathLab\EvidenceMentor', diff --git a/src/main/java/org/pathlab/forge/evidence/EvidenceJobProcessor.java b/src/main/java/org/pathlab/forge/evidence/EvidenceJobProcessor.java index 4965e3e..fca80b7 100644 --- a/src/main/java/org/pathlab/forge/evidence/EvidenceJobProcessor.java +++ b/src/main/java/org/pathlab/forge/evidence/EvidenceJobProcessor.java @@ -110,8 +110,11 @@ public EvidenceJob process(EvidenceJob job, String workerId, Instant now) throws "Only H&E evidence jobs may provide a tile cache"); } checkCancellation(job.id(), workerId, now); - var running = queue.checkpoint(job.id(), workerId, EvidenceJobState.RUNNING, - "running", 0.25, "Running offline bounded brightfield analysis", now, LEASE); + var active = job; + if (active.state() == EvidenceJobState.VALIDATING || active.state() == EvidenceJobState.RUNNING) { + active = queue.checkpoint(job.id(), workerId, EvidenceJobState.RUNNING, + "running", 0.25, "Running offline bounded brightfield analysis", now, LEASE); + } final BufferedImage image; try { @@ -155,8 +158,11 @@ public EvidenceJob process(EvidenceJob job, String workerId, Instant now) throws var abstained = !abstentionReasons.isEmpty(); var refiningAt = now.plusSeconds(1); checkCancellation(job.id(), workerId, refiningAt); - var refining = queue.checkpoint(running.id(), workerId, EvidenceJobState.REFINING, - "refining", 0.65, "Refining deterministic region descriptors", refiningAt, LEASE); + var refining = active; + if (active.state() != EvidenceJobState.PACKAGING) { + refining = queue.checkpoint(active.id(), workerId, EvidenceJobState.REFINING, + "refining", 0.65, "Refining deterministic region descriptors", refiningAt, LEASE); + } var packagingAt = now.plusSeconds(2); var packaging = queue.checkpoint(refining.id(), workerId, EvidenceJobState.PACKAGING, "packaging", 0.9, "Signing immutable evidence bundle", packagingAt, LEASE); diff --git a/src/main/java/org/pathlab/forge/evidence/EvidenceJobQueue.java b/src/main/java/org/pathlab/forge/evidence/EvidenceJobQueue.java index 62517f7..1c87475 100644 --- a/src/main/java/org/pathlab/forge/evidence/EvidenceJobQueue.java +++ b/src/main/java/org/pathlab/forge/evidence/EvidenceJobQueue.java @@ -127,15 +127,29 @@ public synchronized Optional snapshot(String id) throws IOE /** Makes leases from the previous runner boot immediately reclaimable without changing checkpoints. */ public synchronized int recoverOrphanedActiveJobs(Instant now) throws IOException { + var cancelled = 0; + try (var statement = connection.prepareStatement(""" + UPDATE evidence_jobs SET state='CANCELLED',stage='cancelled',lease_owner='',lease_expires_at=?, + detail=?,updated_at=? WHERE cancel_requested=1 + AND state IN ('VALIDATING','RUNNING','REFINING','PACKAGING') + """)) { + statement.setString(1, Instant.EPOCH.toString()); + statement.setString(2, "Cancelled during runner restart at the last durable checkpoint"); + statement.setString(3, now.toString()); + cancelled = statement.executeUpdate(); + } catch (SQLException error) { + throw new IOException("Unable to recover cancelled Evidence Mentor jobs", error); + } try (var statement = connection.prepareStatement(""" UPDATE evidence_jobs SET lease_owner='',lease_expires_at=?,last_heartbeat=?, - detail=?,updated_at=? WHERE state IN ('VALIDATING','RUNNING','REFINING','PACKAGING') + detail=?,updated_at=? WHERE cancel_requested=0 + AND state IN ('VALIDATING','RUNNING','REFINING','PACKAGING') """)) { statement.setString(1, Instant.EPOCH.toString()); statement.setString(2, now.toString()); statement.setString(3, "Recovered after runner restart; verified checkpoint will be revalidated"); statement.setString(4, now.toString()); - return statement.executeUpdate(); + return cancelled + statement.executeUpdate(); } catch (SQLException error) { throw new IOException("Unable to recover orphaned Evidence Mentor leases", error); } diff --git a/src/main/java/org/pathlab/forge/evidence/EvidenceMentorRunner.java b/src/main/java/org/pathlab/forge/evidence/EvidenceMentorRunner.java index 3656855..8e9076d 100644 --- a/src/main/java/org/pathlab/forge/evidence/EvidenceMentorRunner.java +++ b/src/main/java/org/pathlab/forge/evidence/EvidenceMentorRunner.java @@ -30,7 +30,7 @@ /** Standalone authenticated loopback process for unattended Evidence Mentor jobs. */ public final class EvidenceMentorRunner implements AutoCloseable { private static final ObjectMapper JSON = new ObjectMapper(); - private static final String VERSION = "2.0.4"; + private static final String VERSION = "2.0.5"; private static final Duration LEASE = Duration.ofSeconds(45); private final Path stateRoot; private final String token; @@ -238,6 +238,10 @@ private void processClaimed(EvidenceExecutionLane lane) { } catch (IOException transientFailure) { var current = queue.find(claimed.get().id()).orElseThrow(); if (!current.state().terminal()) queue.fail(current.id(), workerId, "transient_io", "IO_TRANSIENT", transientFailure.getMessage(), true, Instant.now()); + } catch (Exception unexpected) { + var current = queue.find(claimed.get().id()).orElseThrow(); + if (!current.state().terminal()) queue.fail(current.id(), workerId, "internal", "RUNNER_INTERNAL", + "Evidence worker stopped at a durable checkpoint", false, Instant.now()); } } catch (Exception ignored) { /* lease expiry is the durable recovery path */ } } diff --git a/src/test/java/org/pathlab/forge/evidence/EvidenceJobProcessorTest.java b/src/test/java/org/pathlab/forge/evidence/EvidenceJobProcessorTest.java index ebd3623..0c38896 100644 --- a/src/test/java/org/pathlab/forge/evidence/EvidenceJobProcessorTest.java +++ b/src/test/java/org/pathlab/forge/evidence/EvidenceJobProcessorTest.java @@ -174,6 +174,43 @@ void completesOfflineDescriptiveIhcJobIntoSignedAtomicEvidence() throws Exceptio } } + @Test + void resumesSignedIhcPackagingFromRefiningCheckpointAfterRunnerRestart() throws Exception { + var source = temporaryDirectory.resolve("resume-source.bin"); + Files.writeString(source, "immutable resume source bytes"); + var preview = temporaryDirectory.resolve("resume-preview.png"); + var image = new BufferedImage(16, 12, BufferedImage.TYPE_INT_RGB); + var graphics = image.createGraphics(); + graphics.setColor(new Color(70, 40, 120)); + graphics.fillRect(2, 2, 8, 7); + graphics.dispose(); + ImageIO.write(image, "png", preview.toFile()); + var pack = temporaryDirectory.resolve("resume-pack.json"); + Files.writeString(pack, packJson()); + var request = request(source, preview, pack, "revision-resume"); + var database = temporaryDirectory.resolve("resume-state/jobs.sqlite3"); + var now = Instant.parse("2026-08-22T00:00:00Z"); + + try (var queue = new EvidenceJobQueue(database)) { + queue.submit("job-resume", request, now); + var claimed = queue.claimNext("old-worker", now, Duration.ofMinutes(2)).orElseThrow(); + var running = queue.checkpoint(claimed.id(), "old-worker", EvidenceJobState.RUNNING, + "running", 0.25, "running", now.plusSeconds(1), Duration.ofMinutes(2)); + queue.checkpoint(running.id(), "old-worker", EvidenceJobState.REFINING, + "refining", 0.65, "refining", now.plusSeconds(2), Duration.ofMinutes(2)); + } + + try (var queue = new EvidenceJobQueue(database)) { + queue.recoverOrphanedActiveJobs(now.plusSeconds(3)); + var recovered = queue.claimNext("new-worker", now.plusSeconds(3), Duration.ofMinutes(2)).orElseThrow(); + var completed = new EvidenceJobProcessor(queue, temporaryDirectory.resolve("resume-state")) + .process(recovered, "new-worker", now.plusSeconds(4)); + assertEquals(EvidenceJobState.ABSTAINED, completed.state()); + assertTrue(Files.isRegularFile(temporaryDirectory + .resolve("resume-state/artifacts/job-resume/evidence.json"))); + } + } + private static String packJson() { return "{\"schema\":\"pathlab.ai-pack/1\",\"packId\":\"ihc-descriptive-v1\",\"version\":\"1\"," + "\"capability\":\"ihc-descriptive\",\"acceptedStains\":[\"ihc_dab\"]," + diff --git a/src/test/java/org/pathlab/forge/evidence/EvidenceJobQueueTest.java b/src/test/java/org/pathlab/forge/evidence/EvidenceJobQueueTest.java index 92fd7ac..800698b 100644 --- a/src/test/java/org/pathlab/forge/evidence/EvidenceJobQueueTest.java +++ b/src/test/java/org/pathlab/forge/evidence/EvidenceJobQueueTest.java @@ -72,6 +72,23 @@ void cancellationFlagSurvivesRestart() throws Exception { } } + @Test + void runnerRestartFinalizesCheckpointSafeActiveCancellation() throws Exception { + var database = temporaryDirectory.resolve("cancel-active.sqlite"); + var request = temporaryDirectory.resolve("cancel-active.json"); + Files.writeString(request, "{}"); + var now = Instant.parse("2026-08-22T00:00:00Z"); + try (var queue = new EvidenceJobQueue(database)) { + queue.submit("cancel-active", request, now); + queue.claimNext("old-worker", now, Duration.ofMinutes(2)).orElseThrow(); + queue.requestCancel("cancel-active", now.plusSeconds(1)); + } + try (var queue = new EvidenceJobQueue(database)) { + assertEquals(1, queue.recoverOrphanedActiveJobs(now.plusSeconds(2))); + assertEquals(EvidenceJobState.CANCELLED, queue.find("cancel-active").orElseThrow().state()); + } + } + @Test void separatesExecutionLanesAndRenewsLiveLease() throws Exception { var database = temporaryDirectory.resolve("jobs.sqlite"); diff --git a/src/test/java/org/pathlab/forge/evidence/EvidenceMentorRunnerTest.java b/src/test/java/org/pathlab/forge/evidence/EvidenceMentorRunnerTest.java index c9fb57d..5a26312 100644 --- a/src/test/java/org/pathlab/forge/evidence/EvidenceMentorRunnerTest.java +++ b/src/test/java/org/pathlab/forge/evidence/EvidenceMentorRunnerTest.java @@ -96,7 +96,7 @@ void acceptsAuthenticatedLoopbackSubmissionIntoDurableQueue() throws Exception { var endpoint = JSON.readTree(temporaryDirectory.resolve("state/endpoint.json").toFile()); assertEquals("pathlab.runner-endpoint/1", endpoint.path("schema").asText()); - assertEquals("2.0.4", endpoint.path("serviceVersion").asText()); + assertEquals("2.0.5", endpoint.path("serviceVersion").asText()); assertEquals(runner.uri("/").getPort(), endpoint.path("port").asInt()); assertTrue(!endpoint.has("token")); } From ead8d924cc87e79c5ae0ec78310a2e6f5aab81c2 Mon Sep 17 00:00:00 2001 From: Enksodsoon <229880765+Enksodsoon@users.noreply.github.com> Date: Sat, 22 Aug 2026 14:28:04 +0700 Subject: [PATCH 16/50] Observe short jobs during restart acceptance --- scripts/test-evidence-mentor-service.ps1 | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/test-evidence-mentor-service.ps1 b/scripts/test-evidence-mentor-service.ps1 index d84dc27..088445c 100644 --- a/scripts/test-evidence-mentor-service.ps1 +++ b/scripts/test-evidence-mentor-service.ps1 @@ -314,10 +314,11 @@ function Wait-Job([string] $ResolvedJobId, [bool] $RestartWhenActive) { if (-not $ResolvedJobId) { return $false } $deadline = [DateTimeOffset]::UtcNow.AddMinutes($TimeoutMinutes) $restarted = $false + $pollMilliseconds = if ($RestartWhenActive) { 50 } else { 1000 } do { try { $response = Invoke-Runner 'GET' "/v1/jobs/$ResolvedJobId" - if ($response.StatusCode -ne 200) { Start-Sleep -Seconds 1; continue } + if ($response.StatusCode -ne 200) { Start-Sleep -Milliseconds $pollMilliseconds; continue } $job = $response.Content | ConvertFrom-Json if ($RestartWhenActive -and -not $restarted -and $job.state -in @('validating','running','refining','packaging')) { if (-not (Test-Administrator)) { Add-Check 'service-restart' 'NOT_EVALUABLE' 'Administrator privileges are required.'; $RestartWhenActive = $false; continue } @@ -343,7 +344,7 @@ function Wait-Job([string] $ResolvedJobId, [bool] $RestartWhenActive) { return $passed } } catch { } - Start-Sleep -Seconds 1 + Start-Sleep -Milliseconds $pollMilliseconds } while ([DateTimeOffset]::UtcNow -lt $deadline) Add-Check 'bounded-job-completion' 'FAIL' 'Bounded job did not complete before the acceptance timeout.' @{ jobId = $ResolvedJobId } return $false From 016a4fbff447a02f559a74b69db971d1d0b8b5a8 Mon Sep 17 00:00:00 2001 From: Enksodsoon <229880765+Enksodsoon@users.noreply.github.com> Date: Sat, 22 Aug 2026 14:41:11 +0700 Subject: [PATCH 17/50] Add fail-closed Session 0 GPU acceptance pack --- docs/evidence-mentor-runner.md | 10 +- .../he-dinov2-small-session0-acceptance-v1.md | 11 + scripts/evidence-mentor-service.ps1 | 2 +- scripts/stage-dinov2-session0-acceptance.ps1 | 205 ++++++++++++++++++ .../forge/evidence/EvidenceJobProcessor.java | 9 +- .../forge/evidence/EvidenceMentorRunner.java | 4 +- .../forge/evidence/EvidencePackManifest.java | 26 ++- ...e-dinov2-small-session0-acceptance-v1.json | 75 +++++++ .../evidence/EvidenceMentorRunnerTest.java | 2 +- .../evidence/EvidencePackManifestTest.java | 33 +++ 10 files changed, 369 insertions(+), 8 deletions(-) create mode 100644 docs/model-cards/he-dinov2-small-session0-acceptance-v1.md create mode 100644 scripts/stage-dinov2-session0-acceptance.ps1 create mode 100644 src/main/resources/evidence-packs/he-dinov2-small-session0-acceptance-v1.json diff --git a/docs/evidence-mentor-runner.md b/docs/evidence-mentor-runner.md index bf3dfac..7163e2d 100644 --- a/docs/evidence-mentor-runner.md +++ b/docs/evidence-mentor-runner.md @@ -38,7 +38,7 @@ First build a versioned distribution: Then run the administrator-approved installer with a Java 17 runtime: ```powershell -.\scripts\evidence-mentor-service.ps1 -Action Install -Version '2.0.5' ` +.\scripts\evidence-mentor-service.ps1 -Action Install -Version '2.0.6' ` -DistributionPath '.\build\install\pathlab-forge' -JavaHome 'C:\Path\To\jdk-17' ``` @@ -76,6 +76,14 @@ Start with the non-disruptive inspection: Stage a small, rights-approved GPU-pack request under the protected state root. Its public acceptance ID must match `acceptance-<8-64 lowercase hex chars>`. +The bundled staging command creates a deterministic synthetic H&E tile cache +and a benchmark-only, not-evaluable DINOv2 Session 0 manifest. It does not +activate or qualify the H&E model: + +```powershell +.\scripts\stage-dinov2-session0-acceptance.ps1 +``` + Then prove service restart recovery while that bounded job is active: ```powershell diff --git a/docs/model-cards/he-dinov2-small-session0-acceptance-v1.md b/docs/model-cards/he-dinov2-small-session0-acceptance-v1.md new file mode 100644 index 0000000..e10dbaf --- /dev/null +++ b/docs/model-cards/he-dinov2-small-session0-acceptance-v1.md @@ -0,0 +1,11 @@ +# DINOv2-small Session 0 acceptance pack + +This manifest exists only to prove that the checksum-pinned DINOv2-small worker +can execute offline on the Windows `LocalService` P2000 host. It accepts only +non-identifying `acceptance-` jobs and remains `benchmark-only` and +`not-evaluable`. + +Passing this pack proves runtime compatibility, CUDA 12.6 `sm_61` execution, +resource-envelope telemetry, durable GPU-lane processing, and signed evidence +packaging. It does not qualify H&E retrieval, activate a pilot model, validate +pathology quality, or support diagnostic or clinical claims. diff --git a/scripts/evidence-mentor-service.ps1 b/scripts/evidence-mentor-service.ps1 index fd6c943..c13ef01 100644 --- a/scripts/evidence-mentor-service.ps1 +++ b/scripts/evidence-mentor-service.ps1 @@ -1,7 +1,7 @@ [CmdletBinding(SupportsShouldProcess)] param( [Parameter(Mandatory)] [ValidateSet('Install','Upgrade','Uninstall','Status')] [string] $Action, - [string] $Version = '2.0.5', + [string] $Version = '2.0.6', [string] $DistributionPath, [string] $JavaHome, [string] $ProgramRoot = 'C:\ProgramData\PathLab\EvidenceMentor', diff --git a/scripts/stage-dinov2-session0-acceptance.ps1 b/scripts/stage-dinov2-session0-acceptance.ps1 new file mode 100644 index 0000000..16a3b98 --- /dev/null +++ b/scripts/stage-dinov2-session0-acceptance.ps1 @@ -0,0 +1,205 @@ +[CmdletBinding()] +param( + [string]$StateRoot = 'D:\PathLabData\EvidenceMentor\state' +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version Latest +$state = [IO.Path]::GetFullPath($StateRoot) +$modelRoot = [IO.Path]::GetFullPath((Join-Path $state 'models\he-dinov2-small-v1\1')) +$acceptanceRoot = [IO.Path]::GetFullPath((Join-Path $state 'acceptance\gpu-session0-v1')) +$manifestSource = [IO.Path]::GetFullPath((Join-Path $PSScriptRoot ` + '..\src\main\resources\evidence-packs\he-dinov2-small-session0-acceptance-v1.json')) +$manifestTarget = Join-Path $modelRoot 'session0-acceptance-pack.json' + +if (-not $modelRoot.StartsWith((Join-Path $state 'models') + [IO.Path]::DirectorySeparatorChar, + [StringComparison]::OrdinalIgnoreCase) -or + -not $acceptanceRoot.StartsWith((Join-Path $state 'acceptance') + [IO.Path]::DirectorySeparatorChar, + [StringComparison]::OrdinalIgnoreCase)) { + throw 'Session 0 acceptance paths escaped the protected state root.' +} +if (-not (Test-Path -LiteralPath $manifestSource -PathType Leaf)) { + throw 'Bundled Session 0 acceptance manifest is unavailable.' +} + +function Get-Sha256([string]$Path) { + return (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant() +} + +function Write-Utf8([string]$Path, [string]$Value) { + [IO.File]::WriteAllText($Path, $Value, [Text.UTF8Encoding]::new($false)) +} + +function Write-Json([string]$Path, [object]$Value) { + Write-Utf8 $Path (($Value | ConvertTo-Json -Depth 12) + "`n") +} + +$manifest = Get-Content -LiteralPath $manifestSource -Raw | ConvertFrom-Json +if ($manifest.schema -ne 'pathlab.ai-pack/1' -or + $manifest.usageLimits.acceptanceOnly -ne $true -or + $manifest.rights.allowedUse -ne 'benchmark-only' -or + $manifest.validation.status -ne 'not-evaluable') { + throw 'Session 0 acceptance manifest weakened its fail-closed usage limits.' +} +$artifactFiles = @{ + model = 'model.safetensors' + config = 'config.json' + preprocessor = 'preprocessor_config.json' + worker = 'worker.exe' + workerSource = 'worker.py' + 'runtime-manifest' = 'runtime-manifest.json' +} +foreach ($artifact in $manifest.artifacts) { + if (-not $artifactFiles.ContainsKey([string]$artifact.name)) { continue } + $path = Join-Path $modelRoot $artifactFiles[[string]$artifact.name] + if (-not (Test-Path -LiteralPath $path -PathType Leaf) -or + (Get-Sha256 $path) -ne [string]$artifact.sha256) { + throw "Pinned Session 0 artifact is unavailable or changed: $($artifact.name)" + } +} + +$targetPartial = "$manifestTarget.partial" +if (Test-Path -LiteralPath $manifestTarget -PathType Leaf) { + if ((Get-Sha256 $manifestTarget) -ne (Get-Sha256 $manifestSource)) { + throw 'Installed Session 0 acceptance manifest differs from the bundled manifest.' + } +} else { + Copy-Item -LiteralPath $manifestSource -Destination $targetPartial + Move-Item -LiteralPath $targetPartial -Destination $manifestTarget +} + +$requestPath = Join-Path $acceptanceRoot 'request.json' +if (Test-Path -LiteralPath $requestPath -PathType Leaf) { + $requestSha = Get-Sha256 $requestPath + [pscustomobject]@{ + RequestPath = $requestPath + RequestSha256 = $requestSha + JobId = 'acceptance-' + $requestSha.Substring(0, 16) + PackManifest = $manifestTarget + PackManifestSha256 = Get-Sha256 $manifestTarget + } + return +} + +$partialRoot = "$acceptanceRoot.partial-$PID" +if (Test-Path -LiteralPath $partialRoot) { + throw 'A prior Session 0 acceptance staging directory requires review.' +} +New-Item -ItemType Directory -Path (Join-Path $partialRoot 'tiles') -Force | Out-Null +try { + Add-Type -AssemblyName System.Drawing + $sourcePath = Join-Path $partialRoot 'source.png' + $source = [Drawing.Bitmap]::new(1024, 1024, [Drawing.Imaging.PixelFormat]::Format24bppRgb) + try { + $graphics = [Drawing.Graphics]::FromImage($source) + try { + $graphics.Clear([Drawing.Color]::FromArgb(238, 204, 222)) + $brushes = @( + [Drawing.SolidBrush]::new([Drawing.Color]::FromArgb(84, 48, 132)), + [Drawing.SolidBrush]::new([Drawing.Color]::FromArgb(164, 91, 139)), + [Drawing.SolidBrush]::new([Drawing.Color]::FromArgb(111, 62, 151)), + [Drawing.SolidBrush]::new([Drawing.Color]::FromArgb(197, 119, 157)) + ) + try { + for ($quadrant = 0; $quadrant -lt 4; $quadrant++) { + $originX = ($quadrant % 2) * 512 + $originY = [Math]::Floor($quadrant / 2) * 512 + for ($y = 16; $y -lt 500; $y += 32) { + for ($x = 16; $x -lt 500; $x += 32) { + $size = 8 + (($x + $y + $quadrant) % 13) + $graphics.FillEllipse($brushes[$quadrant], $originX + $x, $originY + $y, $size, $size + 3) + } + } + } + } finally { + foreach ($brush in $brushes) { $brush.Dispose() } + } + } finally { $graphics.Dispose() } + $source.Save($sourcePath, [Drawing.Imaging.ImageFormat]::Png) + $tiles = [Collections.Generic.List[object]]::new() + for ($index = 0; $index -lt 4; $index++) { + $x = ($index % 2) * 512 + $y = [Math]::Floor($index / 2) * 512 + $tile = $source.Clone([Drawing.Rectangle]::new($x, $y, 512, 512), + [Drawing.Imaging.PixelFormat]::Format24bppRgb) + try { + $relative = "tiles/tile-$($index + 1).png" + $tilePath = Join-Path $partialRoot $relative + $tile.Save($tilePath, [Drawing.Imaging.ImageFormat]::Png) + $tiles.Add([ordered]@{ + id = "tile-$($index + 1)" + path = $relative + sha256 = Get-Sha256 $tilePath + x = [int]$x + y = [int]$y + width = 512 + height = 512 + }) + } finally { $tile.Dispose() } + } + } finally { $source.Dispose() } + + $sourceSha = Get-Sha256 $sourcePath + $samplePath = Join-Path $partialRoot 'sample.json' + Write-Json $samplePath ([ordered]@{ + schema = 'pathlab.evidence-sample/1' + source = 'pathlab-synthetic-session0-acceptance-v1' + patientGroup = 'synthetic-no-patient' + slideId = 'session0-p2000-v1' + sha256 = $sourceSha + license = 'PathLab synthetic acceptance fixture' + taskLabel = 'synthetic-hardware-acceptance' + permittedUse = 'private-research' + bytes = (Get-Item -LiteralPath $sourcePath).Length + grandfatheredReadOnly = $false + }) + $tileCachePath = Join-Path $partialRoot 'tile-cache.json' + Write-Json $tileCachePath ([ordered]@{ + schema = 'pathlab.tile-cache/1' + source = [ordered]@{ + sha256 = $sourceSha + slideRevision = "session0-p2000-v1:$sourceSha" + width = 1024 + height = 1024 + sampleManifest = 'sample.json' + sampleManifestSha256 = Get-Sha256 $samplePath + } + tilePixels = 512 + encoding = 'png' + preprocessingInput = 'rgb-srgb-uint8' + tiles = $tiles + }) + Write-Json (Join-Path $partialRoot 'request.json') ([ordered]@{ + schema = 'pathlab.evidence-job/2' + sourcePath = (Join-Path $acceptanceRoot 'source.png') + sourceSha256 = $sourceSha + slideRevision = "session0-p2000-v1:$sourceSha" + previewPath = (Join-Path $acceptanceRoot 'source.png') + sourceWidth = 1024 + sourceHeight = 1024 + packManifest = $manifestTarget + stain = 'he' + marker = 'generic' + tileCacheManifest = (Join-Path $acceptanceRoot 'tile-cache.json') + tileCacheManifestSha256 = Get-Sha256 $tileCachePath + }) + Move-Item -LiteralPath $partialRoot -Destination $acceptanceRoot +} finally { + if (Test-Path -LiteralPath $partialRoot) { + $resolvedPartial = [IO.Path]::GetFullPath($partialRoot) + if (-not $resolvedPartial.StartsWith((Join-Path $state 'acceptance') + [IO.Path]::DirectorySeparatorChar, + [StringComparison]::OrdinalIgnoreCase)) { + throw 'Refusing to clean a staging path outside the acceptance root.' + } + Remove-Item -LiteralPath $resolvedPartial -Recurse -Force + } +} + +$requestSha = Get-Sha256 $requestPath +[pscustomobject]@{ + RequestPath = $requestPath + RequestSha256 = $requestSha + JobId = 'acceptance-' + $requestSha.Substring(0, 16) + PackManifest = $manifestTarget + PackManifestSha256 = Get-Sha256 $manifestTarget +} diff --git a/src/main/java/org/pathlab/forge/evidence/EvidenceJobProcessor.java b/src/main/java/org/pathlab/forge/evidence/EvidenceJobProcessor.java index fca80b7..3f601e3 100644 --- a/src/main/java/org/pathlab/forge/evidence/EvidenceJobProcessor.java +++ b/src/main/java/org/pathlab/forge/evidence/EvidenceJobProcessor.java @@ -40,6 +40,10 @@ public static EvidenceExecutionLane executionLane(Path requestPath) throws IOExc } public static ExecutionPlan executionPlan(Path requestPath) throws IOException { + return executionPlan(requestPath, ""); + } + + public static ExecutionPlan executionPlan(Path requestPath, String jobId) throws IOException { final JsonNode request; try { request = JSON.readTree(requestPath.toAbsolutePath().normalize().toFile()); @@ -49,7 +53,7 @@ public static ExecutionPlan executionPlan(Path requestPath) throws IOException { // Legacy v1 submissions were accepted before lane metadata existed and migrate to CPU/I/O. if (request == null || !request.path("packManifest").isTextual()) return new ExecutionPlan(EvidenceExecutionLane.CPU_IO, ""); var pack = EvidencePackManifest.load(regularPath(request, "packManifest")); - pack.requirePilotEligible(); + pack.requireExecutableForJob(jobId); return new ExecutionPlan("cuda".equals(pack.runtimeCompatibility().executionProvider()) ? EvidenceExecutionLane.GPU : EvidenceExecutionLane.CPU_IO, pack.sha256()); } @@ -76,7 +80,7 @@ public EvidenceJob process(EvidenceJob job, String workerId, Instant now) throws require(expectedSha.matches("[a-f0-9]{64}") && expectedSha.equals(sha256(source)), "Evidence source checksum does not match"); var pack = EvidencePackManifest.load(regularPath(request, "packManifest")); - pack.requirePilotEligible(); + pack.requireExecutableForJob(job.id()); var durable = queue.snapshot(job.id()).orElseThrow(); require(durable.requestSha256().isBlank() || durable.requestSha256().equals(sha256(job.requestPath())), "Evidence request changed after submission"); @@ -212,6 +216,7 @@ private static ObjectNode evidence( "preprocessing", pack.preprocessingId(), "artifacts", pack.artifacts().stream().map(EvidencePackManifest.Artifact::sha256).toList(), "allowedUse", pack.allowedUse(), + "acceptanceOnly", pack.acceptanceOnly(), "validationStatus", pack.validationStatus().wire()))); root.put("status", abstentionReasons.isEmpty() ? "completed" : "abstained"); root.put("researchOnly", true); diff --git a/src/main/java/org/pathlab/forge/evidence/EvidenceMentorRunner.java b/src/main/java/org/pathlab/forge/evidence/EvidenceMentorRunner.java index 8e9076d..f209efc 100644 --- a/src/main/java/org/pathlab/forge/evidence/EvidenceMentorRunner.java +++ b/src/main/java/org/pathlab/forge/evidence/EvidenceMentorRunner.java @@ -30,7 +30,7 @@ /** Standalone authenticated loopback process for unattended Evidence Mentor jobs. */ public final class EvidenceMentorRunner implements AutoCloseable { private static final ObjectMapper JSON = new ObjectMapper(); - private static final String VERSION = "2.0.5"; + private static final String VERSION = "2.0.6"; private static final Duration LEASE = Duration.ofSeconds(45); private final Path stateRoot; private final String token; @@ -196,7 +196,7 @@ private void submit(HttpExchange exchange) throws IOException { var value = jsonBody(exchange, 65_536); require(fieldNames(value).equals(Set.of("id", "requestPath")), "Job submission fields are invalid"); var id = text(value, "id"); require(id.matches("[A-Za-z0-9._-]{1,120}"), "Job id is invalid"); var requestPath = Path.of(text(value, "requestPath")); - var plan = EvidenceJobProcessor.executionPlan(requestPath); + var plan = EvidenceJobProcessor.executionPlan(requestPath, id); var job = queue.submit(id, requestPath, plan.lane(), plan.packSha256(), Instant.now()); returnJson(exchange, 202, jobJson(queue.snapshot(job.id()).orElseThrow())); } diff --git a/src/main/java/org/pathlab/forge/evidence/EvidencePackManifest.java b/src/main/java/org/pathlab/forge/evidence/EvidencePackManifest.java index af1a7d9..568cb91 100644 --- a/src/main/java/org/pathlab/forge/evidence/EvidencePackManifest.java +++ b/src/main/java/org/pathlab/forge/evidence/EvidencePackManifest.java @@ -23,6 +23,7 @@ public record EvidencePackManifest( String allowedUse, boolean redistributable, boolean derivativesAllowed, + boolean acceptanceOnly, RuntimeCompatibility runtimeCompatibility, List licenseLedger, int maxRamMiB, @@ -67,6 +68,13 @@ public static EvidencePackManifest load(Path manifest) throws IOException { "Analysis packs must be offline"); var validation = object(root, "validation"); var status = ValidationStatus.fromWire(text(validation, "status")); + var acceptanceOnly = false; + if (root.has("usageLimits")) { + var usageLimits = object(root, "usageLimits"); + require(usageLimits.size() == 1 && usageLimits.path("acceptanceOnly").isBoolean(), + "AI pack usage limits are invalid"); + acceptanceOnly = usageLimits.path("acceptanceOnly").booleanValue(); + } var artifacts = parseArtifacts(root.path("artifacts")); var markers = root.has("markers") ? stringSet(root.path("markers"), MARKERS, "marker") @@ -79,18 +87,25 @@ public static EvidencePackManifest load(Path manifest) throws IOException { var runtime = root.has("runtimeCompatibility") ? parseRuntime(root.path("runtimeCompatibility")) : new RuntimeCompatibility("java17", "cpu", "none", "none", false); + if (acceptanceOnly) { + require("benchmark-only".equals(allowedUse) + && status == ValidationStatus.NOT_EVALUABLE + && runtime.requiresExternalWorker(), + "Acceptance-only packs must remain benchmark-only, not-evaluable external workers"); + } var licenseLedger = root.has("licenseLedger") ? parseLicenseLedger(root.path("licenseLedger")) : List.of(); return new EvidencePackManifest( normalized, packId, version, capability, acceptedStains, preprocessingId, tilePixels, allowedUse, booleanValue(rights, "redistributable"), booleanValue(rights, "derivativesAllowed"), + acceptanceOnly, runtime, licenseLedger, maxRam, maxVram, maxSeconds, status, sha256(bytes), artifacts, markers); } public boolean pilotEligible() { - return "private-research".equals(allowedUse) + return !acceptanceOnly && "private-research".equals(allowedUse) && validationStatus != ValidationStatus.BLOCKED && validationStatus != ValidationStatus.UNSUPPORTED && validationStatus != ValidationStatus.NOT_EVALUABLE; @@ -100,6 +115,15 @@ public void requirePilotEligible() { require(pilotEligible(), "AI pack is not eligible for the private pilot"); } + public void requireExecutableForJob(String jobId) { + if (acceptanceOnly) { + require(jobId != null && jobId.matches("acceptance-[a-f0-9]{8,64}"), + "Acceptance-only AI pack requires a non-identifying acceptance job id"); + return; + } + requirePilotEligible(); + } + public void requireStain(String stain) { require(acceptedStains.contains(stain), "AI pack does not support stain " + stain); } diff --git a/src/main/resources/evidence-packs/he-dinov2-small-session0-acceptance-v1.json b/src/main/resources/evidence-packs/he-dinov2-small-session0-acceptance-v1.json new file mode 100644 index 0000000..8594a62 --- /dev/null +++ b/src/main/resources/evidence-packs/he-dinov2-small-session0-acceptance-v1.json @@ -0,0 +1,75 @@ +{ + "schema": "pathlab.ai-pack/1", + "packId": "he-dinov2-small-v1", + "version": "1", + "capability": "he-evidence", + "acceptedStains": [ + "he" + ], + "preprocessing": { + "id": "dinov2-v1", + "tilePixels": 512 + }, + "artifacts": [{ + "name": "model", + "sha256": "ae1e99fcefd534ed978cdeb8326f08030c96e28b7a81ffcbc98a857c84d14be1", + "source": "https://huggingface.co/facebook/dinov2-small/resolve/ed25f3a31f01632728cabb09d1542f84ab7b0056/model.safetensors" + }, { + "name": "config", + "sha256": "1809f83e3bdb1609a501a610ad4a742f4fd8ae44d72ca4aa0df52d1f2ac8628d", + "source": "https://huggingface.co/facebook/dinov2-small/resolve/ed25f3a31f01632728cabb09d1542f84ab7b0056/config.json" + }, { + "name": "preprocessor", + "sha256": "14e780d86fa1861f8751f868d7f45425b5feb55c38ca26f152ca5097ab30f828", + "source": "https://huggingface.co/facebook/dinov2-small/resolve/ed25f3a31f01632728cabb09d1542f84ab7b0056/preprocessor_config.json" + }, { + "name": "worker", + "sha256": "daa33e61d2607e23ecae29e82f274ca05c64f896f9c37061c6ca77ca81aa774b", + "source": "pathlab-local-build:dinov2-worker-v1/windows-x64" + }, { + "name": "workerSource", + "sha256": "b80811cc8026152c45248cdcf8123413f9b3f8d64e297bad93dfe198b5aba5c2", + "source": "pathlab-local-build:dinov2-worker-v1/worker.py" + }, { + "name": "runtime-manifest", + "sha256": "1969c91db66d8acc70b6c8648a57d3bfc923880e03f1f7f1e30c593133c01635", + "source": "pathlab-local-build:torch-2.7.1-cu126-python-3.12/windows-x64" + }], + "runtimeCompatibility": { + "workerProtocol": "pathlab.model-worker-result/1", + "executionProvider": "cuda", + "cuda": "12.6", + "gpuArchitecture": "sm_61", + "requiresExternalWorker": true + }, + "licenseLedger": [{ + "component": "facebook/dinov2-small", + "license": "Apache-2.0", + "revision": "ed25f3a31f01632728cabb09d1542f84ab7b0056", + "permittedUse": "benchmark-only", + "redistributable": false, + "derivativesAllowed": false + }], + "rights": { + "license": "Apache-2.0 upstream; acceptance-only local execution", + "allowedUse": "benchmark-only", + "redistributable": false, + "derivativesAllowed": false, + "reviewedAt": "2026-08-22T00:00:00Z" + }, + "usageLimits": { + "acceptanceOnly": true + }, + "resourceEnvelope": { + "maxRamMiB": 16384, + "maxVramMiB": 4608, + "maxSeconds": 300, + "network": false + }, + "validation": { + "status": "not-evaluable", + "modelCard": "docs/model-cards/he-dinov2-small-session0-acceptance-v1.md", + "heldOutEvaluation": "hardware-acceptance-only-no-quality-claim" + }, + "outputSchema": "pathlab.ai-evidence/1" +} diff --git a/src/test/java/org/pathlab/forge/evidence/EvidenceMentorRunnerTest.java b/src/test/java/org/pathlab/forge/evidence/EvidenceMentorRunnerTest.java index 5a26312..c4fe3ae 100644 --- a/src/test/java/org/pathlab/forge/evidence/EvidenceMentorRunnerTest.java +++ b/src/test/java/org/pathlab/forge/evidence/EvidenceMentorRunnerTest.java @@ -96,7 +96,7 @@ void acceptsAuthenticatedLoopbackSubmissionIntoDurableQueue() throws Exception { var endpoint = JSON.readTree(temporaryDirectory.resolve("state/endpoint.json").toFile()); assertEquals("pathlab.runner-endpoint/1", endpoint.path("schema").asText()); - assertEquals("2.0.5", endpoint.path("serviceVersion").asText()); + assertEquals("2.0.6", endpoint.path("serviceVersion").asText()); assertEquals(runner.uri("/").getPort(), endpoint.path("port").asInt()); assertTrue(!endpoint.has("token")); } diff --git a/src/test/java/org/pathlab/forge/evidence/EvidencePackManifestTest.java b/src/test/java/org/pathlab/forge/evidence/EvidencePackManifestTest.java index a73d7c0..55c64c0 100644 --- a/src/test/java/org/pathlab/forge/evidence/EvidencePackManifestTest.java +++ b/src/test/java/org/pathlab/forge/evidence/EvidencePackManifestTest.java @@ -31,6 +31,35 @@ void blocksBenchmarkOnlyPackFromPilot() throws Exception { assertThrows(IllegalArgumentException.class, pack::requirePilotEligible); } + @Test + void acceptanceOnlyGpuPackRunsOnlyForAcceptanceIdsAndNeverBecomesPilotEligible() throws Exception { + var value = manifest("benchmark-only", "not-evaluable", "[]", "he-evidence", "[\"he\"]") + .replace("\"resourceEnvelope\"", "\"usageLimits\":{\"acceptanceOnly\":true},\"runtimeCompatibility\":{" + + "\"workerProtocol\":\"pathlab.model-worker-result/1\",\"executionProvider\":\"cuda\"," + + "\"cuda\":\"12.6\",\"gpuArchitecture\":\"sm_61\",\"requiresExternalWorker\":true}," + + "\"resourceEnvelope\""); + var path = temporaryDirectory.resolve("acceptance-only.json"); + Files.writeString(path, value); + var pack = EvidencePackManifest.load(path); + + assertTrue(pack.acceptanceOnly()); + assertFalse(pack.pilotEligible()); + pack.requireExecutableForJob("acceptance-0123abcd"); + assertThrows(IllegalArgumentException.class, () -> pack.requireExecutableForJob("learner-job")); + } + + @Test + void rejectsAcceptanceOnlyPackThatClaimsExperimentalUse() throws Exception { + var value = manifest("private-research", "experimental", "[]", "he-evidence", "[\"he\"]") + .replace("\"resourceEnvelope\"", "\"usageLimits\":{\"acceptanceOnly\":true},\"runtimeCompatibility\":{" + + "\"workerProtocol\":\"pathlab.model-worker-result/1\",\"executionProvider\":\"cuda\"," + + "\"cuda\":\"12.6\",\"gpuArchitecture\":\"sm_61\",\"requiresExternalWorker\":true}," + + "\"resourceEnvelope\""); + var path = temporaryDirectory.resolve("invalid-acceptance-only.json"); + Files.writeString(path, value); + assertThrows(IllegalArgumentException.class, () -> EvidencePackManifest.load(path)); + } + @Test void rejectsNetworkedOrOversizedPack() throws Exception { var value = manifest("private-research", "qualified", "[]", "he-evidence", "[\"he\"]") @@ -57,6 +86,10 @@ void bundledPilotCatalogActivatesOnlyInstalledLawfulBaselines() throws Exception dino.artifacts().stream().filter(item -> "worker".equals(item.name())) .findFirst().orElseThrow().sha256()); assertFalse(EvidencePackManifest.load(root.resolve("he-hibou-b-v1.json")).pilotEligible()); + var session0 = EvidencePackManifest.load(root.resolve("he-dinov2-small-session0-acceptance-v1.json")); + assertTrue(session0.acceptanceOnly()); + assertFalse(session0.pilotEligible()); + session0.requireExecutableForJob("acceptance-0123abcd"); var hoverNet = EvidencePackManifest.load(root.resolve("cell-hovernet-fast-v1.json")); assertFalse(hoverNet.pilotEligible()); assertEquals("cuda", hoverNet.runtimeCompatibility().executionProvider()); From fd7d59b1a4ef3de4d2f1360168afc95904d8c2a9 Mon Sep 17 00:00:00 2001 From: Enksodsoon <229880765+Enksodsoon@users.noreply.github.com> Date: Sat, 22 Aug 2026 15:19:02 +0700 Subject: [PATCH 18/50] Qualify portable Session 0 GPU runtime --- docs/evidence-mentor-runner.md | 17 ++- .../he-dinov2-small-session0-acceptance-v1.md | 5 + docs/plans/active/current.md | 13 +- scripts/evidence-mentor-service.ps1 | 2 +- scripts/make-dinov2-runtime-portable.ps1 | 119 ++++++++++++++++ scripts/stage-dinov2-session0-acceptance.ps1 | 17 ++- scripts/test-evidence-mentor-service.ps1 | 50 ++++++- .../forge/evidence/EvidenceMentorRunner.java | 2 +- .../forge/evidence/ExternalModelWorker.java | 133 +++++++++++++++++- ...e-dinov2-small-session0-acceptance-v1.json | 2 +- .../evidence-packs/he-dinov2-small-v1.json | 2 +- .../evidence/EvidenceMentorRunnerTest.java | 2 +- .../evidence/ExternalModelWorkerTest.java | 43 ++++++ 13 files changed, 384 insertions(+), 23 deletions(-) create mode 100644 scripts/make-dinov2-runtime-portable.ps1 diff --git a/docs/evidence-mentor-runner.md b/docs/evidence-mentor-runner.md index 7163e2d..8aff615 100644 --- a/docs/evidence-mentor-runner.md +++ b/docs/evidence-mentor-runner.md @@ -38,7 +38,7 @@ First build a versioned distribution: Then run the administrator-approved installer with a Java 17 runtime: ```powershell -.\scripts\evidence-mentor-service.ps1 -Action Install -Version '2.0.6' ` +.\scripts\evidence-mentor-service.ps1 -Action Install -Version '2.0.11' ` -DistributionPath '.\build\install\pathlab-forge' -JavaHome 'C:\Path\To\jdk-17' ``` @@ -81,15 +81,22 @@ and a benchmark-only, not-evaluable DINOv2 Session 0 manifest. It does not activate or qualify the H&E model: ```powershell +.\scripts\make-dinov2-runtime-portable.ps1 ` + -BasePython 'C:\Path\To\Python312' .\scripts\stage-dinov2-session0-acceptance.ps1 ``` +The one-time portability step copies the Python base needed by the external +pack into protected model storage, removes the user-profile runtime dependency, +and extends the checksum ledger. It refuses to overwrite an existing portable +base. Forge itself still contains no Python, PyTorch, CUDA, or model weights. + Then prove service restart recovery while that bounded job is active: ```powershell & 'C:\ProgramData\PathLab\EvidenceMentor\Test-PathLab-Evidence-Service.ps1' ` -Mode ServiceRestart ` - -JobRequestPath 'D:\PathLabData\EvidenceMentor\state\requests\acceptance-gpu.json' ` + -JobRequestPath 'D:\PathLabData\EvidenceMentor\state\acceptance\gpu-session0-v1\request.json' ` -JobId 'acceptance-0123abcd' ` -TimeoutMinutes 10 ``` @@ -100,7 +107,7 @@ The harness never initiates a reboot: ```powershell & 'C:\ProgramData\PathLab\EvidenceMentor\Test-PathLab-Evidence-Service.ps1' ` -Mode PrepareReboot ` - -JobRequestPath 'D:\PathLabData\EvidenceMentor\state\requests\acceptance-reboot.json' ` + -JobRequestPath 'D:\PathLabData\EvidenceMentor\state\acceptance\gpu-session0-v1\request.json' ` -JobId 'acceptance-89abcdef' # Reboot Windows manually while the bounded job is active. Do not log in first. @@ -129,8 +136,8 @@ an incomplete report into passing evidence. ## Model boundary -The deterministic cell/IHC baseline is runnable. DINOv2-small remains an -experimental executable baseline until its cross-tissue held-out cohort is +The deterministic cell/IHC baseline is runnable. DINOv2-small remains a +`not_evaluable` executable candidate until its cross-tissue held-out cohort is complete. Hibou-B and HoVer-Net remain `not_evaluable` until exact, rights-approved workers and artifacts pass qualification. H&E, cells, IHC, special stains, cytology, and later Atlas tracks stay independently signed and diff --git a/docs/model-cards/he-dinov2-small-session0-acceptance-v1.md b/docs/model-cards/he-dinov2-small-session0-acceptance-v1.md index e10dbaf..0d33697 100644 --- a/docs/model-cards/he-dinov2-small-session0-acceptance-v1.md +++ b/docs/model-cards/he-dinov2-small-session0-acceptance-v1.md @@ -9,3 +9,8 @@ Passing this pack proves runtime compatibility, CUDA 12.6 `sm_61` execution, resource-envelope telemetry, durable GPU-lane processing, and signed evidence packaging. It does not qualify H&E retrieval, activate a pilot model, validate pathology quality, or support diagnostic or clinical claims. + +The installed Python 3.12 base runtime is copied into the protected external +pack directory by `scripts/make-dinov2-runtime-portable.ps1`. The runtime +manifest records the portable base and `pyvenv.cfg` files so `LocalService` +does not depend on or receive access to a Windows user's Python installation. diff --git a/docs/plans/active/current.md b/docs/plans/active/current.md index 5ab115a..5a9c416 100644 --- a/docs/plans/active/current.md +++ b/docs/plans/active/current.md @@ -29,12 +29,13 @@ appear in monitoring payloads. ## Current status -Gates 1-4 are implemented on the draft feature branch. Gate 5 now has an -atomic, fail-closed host acceptance harness and operator procedure. It remains -incomplete until an administrator installs the service and the same installed -runtime earns a `PASS` aggregate report after live LocalService, Session 0 GPU, -service-restart, offline, and manual-reboot recovery checks. Gate 6 has not been -qualified, and Gate 7 has not begun. +Gates 1-4 are implemented on the draft feature branch. The installed 2.0.11 +runtime has separate passing `Inspect` and `ServiceRestart` reports proving +live `LocalService`, outbound-deny coverage, P2000 CUDA 12.6 `sm_61` inference, +resource bounds, signed evidence packaging, and restart recovery. Its latest +aggregate report is still `NOT_EVALUABLE` solely because the operator-controlled +manual-reboot challenge has not run. Gate 5 therefore remains incomplete. Gate +6 has not been qualified, and Gate 7 has not begun. Implementation, service installation, model qualification, merge, deployment, feature activation, and any future clinical/capacity study are separate gates. diff --git a/scripts/evidence-mentor-service.ps1 b/scripts/evidence-mentor-service.ps1 index c13ef01..b06fc32 100644 --- a/scripts/evidence-mentor-service.ps1 +++ b/scripts/evidence-mentor-service.ps1 @@ -1,7 +1,7 @@ [CmdletBinding(SupportsShouldProcess)] param( [Parameter(Mandatory)] [ValidateSet('Install','Upgrade','Uninstall','Status')] [string] $Action, - [string] $Version = '2.0.6', + [string] $Version = '2.0.11', [string] $DistributionPath, [string] $JavaHome, [string] $ProgramRoot = 'C:\ProgramData\PathLab\EvidenceMentor', diff --git a/scripts/make-dinov2-runtime-portable.ps1 b/scripts/make-dinov2-runtime-portable.ps1 new file mode 100644 index 0000000..f2b3be1 --- /dev/null +++ b/scripts/make-dinov2-runtime-portable.ps1 @@ -0,0 +1,119 @@ +[CmdletBinding(SupportsShouldProcess)] +param( + [string] $StateRoot = 'D:\PathLabData\EvidenceMentor\state', + [string] $BasePython = 'C:\Users\enkso\AppData\Local\Programs\Python\Python312' +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version Latest +$modelRoot = [IO.Path]::GetFullPath((Join-Path $StateRoot 'models\he-dinov2-small-v1\1')) +$runtimeRoot = Join-Path $modelRoot 'runtime' +$portableBase = Join-Path $runtimeRoot 'base' +$partialBase = "$portableBase.partial-$PID" +$runtimeManifest = Join-Path $modelRoot 'runtime-manifest.json' +$pyvenv = Join-Path $runtimeRoot 'pyvenv.cfg' + +if (-not $modelRoot.StartsWith( + [IO.Path]::GetFullPath((Join-Path $StateRoot 'models')) + [IO.Path]::DirectorySeparatorChar, + [StringComparison]::OrdinalIgnoreCase)) { + throw 'Portable runtime target escaped the protected model root.' +} +foreach ($required in @('python.exe','python3.dll','python312.dll','vcruntime140.dll','Lib','DLLs')) { + if (-not (Test-Path -LiteralPath (Join-Path $BasePython $required))) { + throw "Python 3.12 base runtime is incomplete: $required" + } +} +if (-not (Test-Path -LiteralPath $runtimeManifest -PathType Leaf) -or + -not (Test-Path -LiteralPath $pyvenv -PathType Leaf)) { + throw 'Installed DINOv2 runtime metadata is incomplete.' +} +if (Test-Path -LiteralPath $portableBase) { + throw "Portable runtime already exists and will not be overwritten: $portableBase" +} + +function Get-Sha256([string] $Path) { + return (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant() +} + +function Write-AtomicUtf8([string] $Path, [string] $Value) { + $partial = "$Path.partial" + [IO.File]::WriteAllText($partial, $Value, [Text.UTF8Encoding]::new($false)) + $backup = "$Path.backup-$([Guid]::NewGuid().ToString('N'))" + try { + [IO.File]::Replace($partial, $Path, $backup) + } finally { + if (Test-Path -LiteralPath $backup) { Remove-Item -LiteralPath $backup -Force } + } +} + +if ($PSCmdlet.ShouldProcess($portableBase, 'Stage a checksum-ledgered portable Python 3.12 base runtime')) { + New-Item -ItemType Directory -Path $partialBase -Force | Out-Null + try { + foreach ($name in @('python.exe','pythonw.exe','python3.dll','python312.dll', + 'vcruntime140.dll','vcruntime140_1.dll','LICENSE.txt')) { + $source = Join-Path $BasePython $name + if (Test-Path -LiteralPath $source -PathType Leaf) { + Copy-Item -LiteralPath $source -Destination $partialBase + } + } + Copy-Item -LiteralPath (Join-Path $BasePython 'DLLs') -Destination $partialBase -Recurse + $targetLib = Join-Path $partialBase 'Lib' + New-Item -ItemType Directory -Path $targetLib -Force | Out-Null + Get-ChildItem -LiteralPath (Join-Path $BasePython 'Lib') -Force | + Where-Object Name -notin @('site-packages','__pycache__') | + Copy-Item -Destination $targetLib -Recurse + + Move-Item -LiteralPath $partialBase -Destination $portableBase + + $pyvenvText = @" +home = $portableBase +include-system-site-packages = false +version = 3.12.10 +executable = $portableBase\python.exe +command = portable PathLab Evidence Mentor runtime +"@ + $pyvenvPartial = "$pyvenv.portable-$PID" + [IO.File]::WriteAllText($pyvenvPartial, $pyvenvText, [Text.UTF8Encoding]::new($false)) + + $manifest = Get-Content -LiteralPath $runtimeManifest -Raw | ConvertFrom-Json + if ($manifest.schema -ne 'pathlab.model-runtime/1' -or $manifest.python -ne '3.12.10') { + throw 'Installed runtime manifest is not the pinned Python 3.12 runtime.' + } + $portableEntries = [Collections.Generic.List[object]]::new() + Get-ChildItem -LiteralPath $portableBase -File -Recurse | Sort-Object FullName | ForEach-Object { + $relative = $_.FullName.Substring($modelRoot.Length + 1).Replace('\','/') + $portableEntries.Add([ordered]@{ path = $relative; bytes = $_.Length; sha256 = Get-Sha256 $_.FullName }) + } + $portableEntries.Add([ordered]@{ + path = 'runtime/pyvenv.cfg' + bytes = (Get-Item -LiteralPath $pyvenvPartial).Length + sha256 = Get-Sha256 $pyvenvPartial + }) + $retained = @($manifest.files | Where-Object { + $_.path -ne 'runtime/pyvenv.cfg' -and -not ([string]$_.path).StartsWith('runtime/base/') + }) + $manifest.files = @($retained + $portableEntries | Sort-Object path) + $manifest | Add-Member -NotePropertyName portableBase -NotePropertyValue 'runtime/base' -Force + $manifestText = ($manifest | ConvertTo-Json -Depth 8) + "`n" + + Write-AtomicUtf8 $pyvenv $pyvenvText + Write-AtomicUtf8 $runtimeManifest $manifestText + } finally { + if (Test-Path -LiteralPath $partialBase) { + $resolved = [IO.Path]::GetFullPath($partialBase) + if (-not $resolved.StartsWith($runtimeRoot + [IO.Path]::DirectorySeparatorChar, + [StringComparison]::OrdinalIgnoreCase)) { + throw 'Refusing to clean a path outside the installed runtime.' + } + Remove-Item -LiteralPath $resolved -Recurse -Force + } + } +} + +[pscustomobject]@{ + RuntimeRoot = $runtimeRoot + PortableBase = $portableBase + PortableFiles = @(Get-ChildItem -LiteralPath $portableBase -File -Recurse).Count + RuntimeManifestSha256 = Get-Sha256 $runtimeManifest + PyvenvSha256 = Get-Sha256 $pyvenv +} diff --git a/scripts/stage-dinov2-session0-acceptance.ps1 b/scripts/stage-dinov2-session0-acceptance.ps1 index 16a3b98..1dca62d 100644 --- a/scripts/stage-dinov2-session0-acceptance.ps1 +++ b/scripts/stage-dinov2-session0-acceptance.ps1 @@ -61,7 +61,22 @@ foreach ($artifact in $manifest.artifacts) { $targetPartial = "$manifestTarget.partial" if (Test-Path -LiteralPath $manifestTarget -PathType Leaf) { if ((Get-Sha256 $manifestTarget) -ne (Get-Sha256 $manifestSource)) { - throw 'Installed Session 0 acceptance manifest differs from the bundled manifest.' + $installed = Get-Content -LiteralPath $manifestTarget -Raw | ConvertFrom-Json + if ($installed.schema -ne 'pathlab.ai-pack/1' -or + $installed.packId -ne $manifest.packId -or + $installed.version -ne $manifest.version -or + $installed.usageLimits.acceptanceOnly -ne $true -or + $installed.rights.allowedUse -ne 'benchmark-only' -or + $installed.validation.status -ne 'not-evaluable') { + throw 'Refusing to replace a manifest that is not the same fail-closed acceptance pack.' + } + Copy-Item -LiteralPath $manifestSource -Destination $targetPartial + $backup = "$manifestTarget.backup-$([Guid]::NewGuid().ToString('N'))" + try { + [IO.File]::Replace($targetPartial, $manifestTarget, $backup) + } finally { + if (Test-Path -LiteralPath $backup) { Remove-Item -LiteralPath $backup -Force } + } } } else { Copy-Item -LiteralPath $manifestSource -Destination $targetPartial diff --git a/scripts/test-evidence-mentor-service.ps1 b/scripts/test-evidence-mentor-service.ps1 index 088445c..f3ff66b 100644 --- a/scripts/test-evidence-mentor-service.ps1 +++ b/scripts/test-evidence-mentor-service.ps1 @@ -310,6 +310,47 @@ function Submit-BoundedJob { } catch { Add-Check 'bounded-job-submission' 'FAIL' 'Bounded job submission failed.'; return $null } } +function Test-Session0GpuResult([string] $ResolvedJobId, [object] $Job) { + $resultPath = Join-Path $StateRoot "worker-output\$ResolvedJobId\result.json" + try { + if (-not (Test-Path -LiteralPath $resultPath -PathType Leaf)) { + return [pscustomobject]@{ Passed = $false; Evidence = @{ reason = 'worker-result-missing' } } + } + $result = Get-Content -LiteralPath $resultPath -Raw | ConvertFrom-Json + $runtime = $result.runtime + $passed = $result.schema -eq 'pathlab.model-worker-result/1' -and + $result.status -eq 'completed' -and + $result.packManifestSha256 -eq $Job.packSha256 -and + [string]$runtime.device -match 'Quadro P2000' -and + [string]$runtime.cuda -eq '12.6' -and + [string]$runtime.architecture -eq 'sm_61' -and + [string]$runtime.analysisNetwork -eq 'disabled' -and + [int]$runtime.tiles -gt 0 -and + [double]$runtime.elapsedSeconds -gt 0 -and + [double]$runtime.peakVramMiB -gt 0 -and + [double]$runtime.peakVramMiB -le 4608 -and + [double]$runtime.peakRamMiB -gt 0 -and + [double]$runtime.peakRamMiB -le 16384 + return [pscustomobject]@{ + Passed = $passed + Evidence = @{ + resultSha256 = (Get-FileHash -LiteralPath $resultPath -Algorithm SHA256).Hash.ToLowerInvariant() + device = [string]$runtime.device + cuda = [string]$runtime.cuda + architecture = [string]$runtime.architecture + tiles = [int]$runtime.tiles + batchSize = [int]$runtime.batchSize + elapsedSeconds = [double]$runtime.elapsedSeconds + peakVramMiB = [double]$runtime.peakVramMiB + peakRamMiB = [double]$runtime.peakRamMiB + analysisNetwork = [string]$runtime.analysisNetwork + } + } + } catch { + return [pscustomobject]@{ Passed = $false; Evidence = @{ reason = 'worker-result-invalid' } } + } +} + function Wait-Job([string] $ResolvedJobId, [bool] $RestartWhenActive) { if (-not $ResolvedJobId) { return $false } $deadline = [DateTimeOffset]::UtcNow.AddMinutes($TimeoutMinutes) @@ -337,9 +378,12 @@ function Wait-Job([string] $ResolvedJobId, [bool] $RestartWhenActive) { jobId = $ResolvedJobId; state = $job.state; lane = $job.lane; failureCode = $job.failureCode; finalArtifactSha256 = $job.finalArtifactSha256 } if ($job.lane -eq 'gpu') { - Add-Check 'session0-gpu-inference' $(if ($passed) {'PASS'} else {'FAIL'}) 'A CUDA-declared pack completed in the service GPU lane.' @{ - jobId = $ResolvedJobId; state = $job.state - } + $gpuResult = Test-Session0GpuResult $ResolvedJobId $job + $gpuPassed = $passed -and $gpuResult.Passed + $gpuEvidence = @{} + $gpuResult.Evidence + $gpuEvidence.jobId = $ResolvedJobId + $gpuEvidence.state = $job.state + Add-Check 'session0-gpu-inference' $(if ($gpuPassed) {'PASS'} else {'FAIL'}) 'The retained worker result proves bounded offline CUDA inference on the P2000.' $gpuEvidence } else { Add-Check 'session0-gpu-inference' 'NOT_EVALUABLE' 'The bounded acceptance job did not use the GPU lane.' } return $passed } diff --git a/src/main/java/org/pathlab/forge/evidence/EvidenceMentorRunner.java b/src/main/java/org/pathlab/forge/evidence/EvidenceMentorRunner.java index f209efc..5a8b40b 100644 --- a/src/main/java/org/pathlab/forge/evidence/EvidenceMentorRunner.java +++ b/src/main/java/org/pathlab/forge/evidence/EvidenceMentorRunner.java @@ -30,7 +30,7 @@ /** Standalone authenticated loopback process for unattended Evidence Mentor jobs. */ public final class EvidenceMentorRunner implements AutoCloseable { private static final ObjectMapper JSON = new ObjectMapper(); - private static final String VERSION = "2.0.6"; + private static final String VERSION = "2.0.11"; private static final Duration LEASE = Duration.ofSeconds(45); private final Path stateRoot; private final String token; diff --git a/src/main/java/org/pathlab/forge/evidence/ExternalModelWorker.java b/src/main/java/org/pathlab/forge/evidence/ExternalModelWorker.java index b2c6902..3e05614 100644 --- a/src/main/java/org/pathlab/forge/evidence/ExternalModelWorker.java +++ b/src/main/java/org/pathlab/forge/evidence/ExternalModelWorker.java @@ -2,7 +2,10 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.security.MessageDigest; @@ -52,6 +55,10 @@ public JsonNode execute(EvidencePackManifest pack, Path request, String jobId, P .map(EvidencePackManifest.Artifact::sha256).findFirst() .orElseThrow(() -> new IllegalArgumentException("Model worker checksum is not declared")); require(expected.equals(sha256(executable)), "Model worker checksum does not match"); + verifyArtifact(pack, "workerSource", install.resolve("worker.py")); + var runtimeManifestPath = install.resolve("runtime-manifest.json"); + verifyArtifact(pack, "runtime-manifest", runtimeManifestPath); + validateRuntimeFileLedger(JSON.readTree(runtimeManifestPath.toFile()), install); var outputRoot = stateRoot.resolve("worker-output").resolve(jobId).normalize(); require(outputRoot.startsWith(stateRoot.resolve("worker-output")), "Model output path escaped state root"); @@ -59,7 +66,9 @@ public JsonNode execute(EvidencePackManifest pack, Path request, String jobId, P var output = outputRoot.resolve("result.json"); var partial = outputRoot.resolve("result.json.partial"); var progressPath = outputRoot.resolve("progress.json"); + var diagnosticPath = outputRoot.resolve("diagnostic.log"); Files.deleteIfExists(partial); + Files.deleteIfExists(diagnosticPath); var command = new java.util.ArrayList(); command.add(executable.toString()); command.add("--request"); command.add(request.toString()); command.add("--output"); command.add(partial.toString()); command.add("--offline"); @@ -77,8 +86,12 @@ public JsonNode execute(EvidencePackManifest pack, Path request, String jobId, P environment.put("PATHLAB_ANALYSIS_NETWORK", "disabled"); environment.put("PATHLAB_MAX_VRAM_MIB", Integer.toString(pack.maxVramMiB())); environment.put("PATHLAB_MAX_RAM_MIB", Integer.toString(pack.maxRamMiB())); - var process = ChildProcessContainment.global().register(builder.redirectErrorStream(true) - .redirectOutput(ProcessBuilder.Redirect.DISCARD).start()); + var process = ChildProcessContainment.global().register(builder.redirectErrorStream(true).start()); + var diagnosticBytes = new ByteArrayOutputStream(16_384); + var outputReader = new Thread(() -> drainBounded(process.getInputStream(), diagnosticBytes), + "pathlab-model-worker-output-" + jobId); + outputReader.setDaemon(true); + outputReader.start(); try { var deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(pack.maxSeconds()); var lastProgressModified = -1L; @@ -106,16 +119,23 @@ public JsonNode execute(EvidencePackManifest pack, Path request, String jobId, P } } catch (IOException error) { process.destroyForcibly(); + awaitOutput(outputReader, process.getInputStream()); Files.deleteIfExists(partial); throw error; } catch (InterruptedException error) { Thread.currentThread().interrupt(); process.destroyForcibly(); + awaitOutput(outputReader, process.getInputStream()); Files.deleteIfExists(partial); throw new IOException("Model worker was interrupted", error); } + awaitOutput(outputReader, process.getInputStream()); if (process.exitValue() != 0 || !Files.isRegularFile(partial)) { Files.deleteIfExists(partial); - throw new IllegalArgumentException("Model worker failed closed"); + writeDiagnostic(diagnosticPath, diagnosticBytes.toByteArray()); + var detail = safeFailureDetail(diagnosticBytes.toString(StandardCharsets.UTF_8)); + throw new IllegalArgumentException(detail.isBlank() + ? "Model worker failed closed with exit code " + process.exitValue() + : "Model worker failed closed with exit code " + process.exitValue() + ": " + detail); } var result = JSON.readTree(partial.toFile()); validateResult(result, pack); @@ -128,6 +148,22 @@ public JsonNode execute(EvidencePackManifest pack, Path request, String jobId, P return result; } + private static void writeDiagnostic(Path path, byte[] value) { + if (value.length == 0) return; + try { + var partial = path.resolveSibling(path.getFileName() + ".partial"); + Files.write(partial, value); + try { + Files.move(partial, path, java.nio.file.StandardCopyOption.REPLACE_EXISTING, + java.nio.file.StandardCopyOption.ATOMIC_MOVE); + } catch (java.nio.file.AtomicMoveNotSupportedException ignored) { + Files.move(partial, path, java.nio.file.StandardCopyOption.REPLACE_EXISTING); + } + } catch (IOException ignored) { + // Failure diagnostics never change durable model job state. + } + } + private static Progress readProgress(Path path, String jobId, EvidencePackManifest pack) throws IOException { var value = JSON.readTree(path.toFile()); require(value.isObject() && fieldNames(value).equals(Set.of("schema", "jobId", "packManifestSha256", @@ -155,6 +191,97 @@ private static Set fieldNames(JsonNode value) { var names = new java.util.HashSet(); value.fieldNames().forEachRemaining(names::add); return names; } + private static void drainBounded(InputStream input, ByteArrayOutputStream output) { + try (input) { + var buffer = new byte[4_096]; + int read; + while ((read = input.read(buffer)) >= 0) { + var remaining = 16_384 - output.size(); + if (remaining > 0) output.write(buffer, 0, Math.min(read, remaining)); + } + } catch (IOException ignored) { + // Diagnostics are best-effort and never alter worker state. + } + } + + private static void awaitOutput(Thread reader, InputStream input) { + try { + reader.join(2_000); + if (reader.isAlive()) input.close(); + } catch (InterruptedException error) { + Thread.currentThread().interrupt(); + } catch (IOException ignored) { + // Diagnostics are best-effort and never alter worker state. + } + } + + static String safeFailureDetail(String output) { + if (output == null || output.isBlank()) return ""; + var prefix = "PathLab DINOv2 worker failed closed: "; + for (var line : output.split("\\R")) { + if (!line.startsWith(prefix)) continue; + var detail = line.substring(prefix.length()).trim(); + if (Set.of( + "offline analysis contract was not enforced", + "model library offline flags were not enforced", + "runtime manifest schema is invalid", + "runtime file ledger is invalid", + "CUDA sm_61 host is unavailable", + "runtime does not contain the pinned Pascal CUDA target", + "resource envelope is invalid", + "source geometry is invalid", + "tile-cache manifest checksum does not match", + "tile-cache schema is unsupported", + "tile cache is stale or belongs to another source revision", + "tile-cache pixel contract is unsupported", + "tile-cache tile list is invalid", + "tile-cache tile entry is invalid", + "tile-cache tile path must be relative", + "tile-cache tile checksum or path is invalid", + "tile-cache tile coordinates are invalid or duplicated", + "tile-cache tile geometry is invalid", + "worker exceeded its declared resource envelope", + "pack manifest is unavailable").contains(detail) + || detail.matches("(required artifact is unavailable|artifact checksum mismatch): [A-Za-z0-9_-]{1,80}")) { + return detail; + } + } + return ""; + } + + private static void verifyArtifact(EvidencePackManifest pack, String name, Path path) throws IOException { + var expected = pack.artifacts().stream().filter(item -> name.equals(item.name())) + .map(EvidencePackManifest.Artifact::sha256).findFirst() + .orElseThrow(() -> new IllegalArgumentException("Model worker artifact is not declared: " + name)); + require(Files.isRegularFile(path) && expected.equals(sha256(path)), + "Model worker artifact checksum does not match: " + name); + } + + static void validateRuntimeFileLedger(JsonNode manifest, Path installRoot) throws IOException { + require(manifest.isObject() && "pathlab.model-runtime/1".equals(manifest.path("schema").asText()), + "Model runtime manifest schema is invalid"); + var files = manifest.path("files"); + require(files.isArray() && !files.isEmpty() && files.size() <= 8_192, + "Model runtime file ledger is invalid"); + var root = installRoot.toAbsolutePath().normalize(); + var realRoot = root.toRealPath(); + var seen = new java.util.HashSet(); + for (var item : files) { + var relativeText = item.path("path").asText(""); + require(!relativeText.isBlank() && relativeText.length() <= 500 + && !Path.of(relativeText).isAbsolute() && seen.add(relativeText), + "Model runtime file ledger is invalid"); + var path = root.resolve(relativeText).normalize(); + require(path.startsWith(root) && Files.isRegularFile(path) + && path.toRealPath().startsWith(realRoot) + && item.path("bytes").isIntegralNumber() + && item.path("bytes").asLong(-1) == Files.size(path) + && item.path("sha256").asText("").matches("[a-f0-9]{64}") + && item.path("sha256").asText().equals(sha256(path)), + "Model runtime file checksum does not match"); + } + } + @FunctionalInterface public interface ProgressListener { void update(Progress progress) throws IOException; } public record Progress(long completedUnits, long totalUnits, Path checkpointPath, String checkpointSha256) { } diff --git a/src/main/resources/evidence-packs/he-dinov2-small-session0-acceptance-v1.json b/src/main/resources/evidence-packs/he-dinov2-small-session0-acceptance-v1.json index 8594a62..9dd9c92 100644 --- a/src/main/resources/evidence-packs/he-dinov2-small-session0-acceptance-v1.json +++ b/src/main/resources/evidence-packs/he-dinov2-small-session0-acceptance-v1.json @@ -32,7 +32,7 @@ "source": "pathlab-local-build:dinov2-worker-v1/worker.py" }, { "name": "runtime-manifest", - "sha256": "1969c91db66d8acc70b6c8648a57d3bfc923880e03f1f7f1e30c593133c01635", + "sha256": "b52c4d80f914c7e6e56d05e594d9478333d763861b5133e3c82af2a6a17fd942", "source": "pathlab-local-build:torch-2.7.1-cu126-python-3.12/windows-x64" }], "runtimeCompatibility": { diff --git a/src/main/resources/evidence-packs/he-dinov2-small-v1.json b/src/main/resources/evidence-packs/he-dinov2-small-v1.json index 89a52df..224a406 100644 --- a/src/main/resources/evidence-packs/he-dinov2-small-v1.json +++ b/src/main/resources/evidence-packs/he-dinov2-small-v1.json @@ -32,7 +32,7 @@ "source": "pathlab-local-build:dinov2-worker-v1/worker.py" }, { "name": "runtime-manifest", - "sha256": "1969c91db66d8acc70b6c8648a57d3bfc923880e03f1f7f1e30c593133c01635", + "sha256": "b52c4d80f914c7e6e56d05e594d9478333d763861b5133e3c82af2a6a17fd942", "source": "pathlab-local-build:torch-2.7.1-cu126-python-3.12/windows-x64" }, { "name": "qualification-protocol", diff --git a/src/test/java/org/pathlab/forge/evidence/EvidenceMentorRunnerTest.java b/src/test/java/org/pathlab/forge/evidence/EvidenceMentorRunnerTest.java index c4fe3ae..d1a4355 100644 --- a/src/test/java/org/pathlab/forge/evidence/EvidenceMentorRunnerTest.java +++ b/src/test/java/org/pathlab/forge/evidence/EvidenceMentorRunnerTest.java @@ -96,7 +96,7 @@ void acceptsAuthenticatedLoopbackSubmissionIntoDurableQueue() throws Exception { var endpoint = JSON.readTree(temporaryDirectory.resolve("state/endpoint.json").toFile()); assertEquals("pathlab.runner-endpoint/1", endpoint.path("schema").asText()); - assertEquals("2.0.6", endpoint.path("serviceVersion").asText()); + assertEquals("2.0.11", endpoint.path("serviceVersion").asText()); assertEquals(runner.uri("/").getPort(), endpoint.path("port").asInt()); assertTrue(!endpoint.has("token")); } diff --git a/src/test/java/org/pathlab/forge/evidence/ExternalModelWorkerTest.java b/src/test/java/org/pathlab/forge/evidence/ExternalModelWorkerTest.java index 4653a0b..c88deb7 100644 --- a/src/test/java/org/pathlab/forge/evidence/ExternalModelWorkerTest.java +++ b/src/test/java/org/pathlab/forge/evidence/ExternalModelWorkerTest.java @@ -1,10 +1,16 @@ package org.pathlab.forge.evidence; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import com.fasterxml.jackson.databind.ObjectMapper; import java.nio.file.Path; +import java.nio.file.Files; +import java.security.MessageDigest; +import java.util.HexFormat; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; final class ExternalModelWorkerTest { private static final ObjectMapper JSON = new ObjectMapper(); @@ -20,4 +26,41 @@ final class ExternalModelWorkerTest { JSON.readTree(validPrefix + "\"regions\":[{\"id\":\"r1\",\"stage\":\"refined\"," + "\"kind\":\"support\",\"x\":0,\"y\":0,\"width\":0,\"height\":1,\"score\":0.5}]}"), pack)); } + + @Test void exposesOnlyTheControlledWorkerFailureLine() { + var output = "boot noise C:\\private\\slide.svs\r\n" + + "PathLab DINOv2 worker failed closed: CUDA sm_61 host is unavailable\r\n" + + "Traceback C:\\private\\slide.svs\r\n"; + + assertEquals("CUDA sm_61 host is unavailable", + ExternalModelWorker.safeFailureDetail(output)); + assertEquals("", ExternalModelWorker.safeFailureDetail( + "Traceback C:\\private\\patient-123.svs")); + } + + @Test void validatesPortableRuntimeFilesBeforeLaunchingPython(@TempDir Path install) throws Exception { + var runtime = install.resolve("runtime"); + Files.createDirectories(runtime); + var python = runtime.resolve("python.exe"); + Files.writeString(python, "portable-python"); + var hash = HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256") + .digest(Files.readAllBytes(python))); + var manifest = JSON.readTree(""" + {"schema":"pathlab.model-runtime/1","files":[ + {"path":"runtime/python.exe","bytes":15,"sha256":"%s"} + ]} + """.formatted(hash)); + + assertDoesNotThrow(() -> ExternalModelWorker.validateRuntimeFileLedger(manifest, install)); + Files.writeString(python, "tampered"); + assertThrows(IllegalArgumentException.class, + () -> ExternalModelWorker.validateRuntimeFileLedger(manifest, install)); + var traversal = JSON.readTree(""" + {"schema":"pathlab.model-runtime/1","files":[ + {"path":"../outside","bytes":1,"sha256":"%s"} + ]} + """.formatted(hash)); + assertThrows(IllegalArgumentException.class, + () -> ExternalModelWorker.validateRuntimeFileLedger(traversal, install)); + } } From bb4647b771f57aa27dc77fd7037b81b0e956784b Mon Sep 17 00:00:00 2001 From: Enksodsoon <229880765+Enksodsoon@users.noreply.github.com> Date: Sat, 22 Aug 2026 15:27:08 +0700 Subject: [PATCH 19/50] Require post-boot job completion evidence --- docs/evidence-mentor-runner.md | 5 +++- docs/plans/active/current.md | 2 +- scripts/evidence-mentor-service.ps1 | 2 +- scripts/test-evidence-mentor-service.ps1 | 28 ++++++++++++++++--- .../forge/evidence/EvidenceMentorRunner.java | 2 +- .../evidence/EvidenceMentorRunnerTest.java | 2 +- 6 files changed, 32 insertions(+), 9 deletions(-) diff --git a/docs/evidence-mentor-runner.md b/docs/evidence-mentor-runner.md index 8aff615..3a24185 100644 --- a/docs/evidence-mentor-runner.md +++ b/docs/evidence-mentor-runner.md @@ -38,7 +38,7 @@ First build a versioned distribution: Then run the administrator-approved installer with a Java 17 runtime: ```powershell -.\scripts\evidence-mentor-service.ps1 -Action Install -Version '2.0.11' ` +.\scripts\evidence-mentor-service.ps1 -Action Install -Version '2.0.12' ` -DistributionPath '.\build\install\pathlab-forge' -JavaHome 'C:\Path\To\jdk-17' ``` @@ -130,6 +130,9 @@ schema `pathlab.service-acceptance/1`, verdict `PASS`, and no required check is and recovery policy, pinned WinSW, ACLs, firewall coverage, dynamic authenticated IPC, one-time dashboard sessions, P2000 visibility, a completed GPU-lane job, service-restart recovery, and reboot continuation of an active bounded job. +The reboot continuation check additionally requires the job's terminal update +to occur after the new OS boot time; a job that finished before reboot cannot +satisfy the gate. Mode-specific `PASS` reports are evidence fragments and are not a completion claim. `-AllowIncomplete` is intended only for safe diagnostics and cannot turn an incomplete report into passing evidence. diff --git a/docs/plans/active/current.md b/docs/plans/active/current.md index 5a9c416..42d7a9d 100644 --- a/docs/plans/active/current.md +++ b/docs/plans/active/current.md @@ -29,7 +29,7 @@ appear in monitoring payloads. ## Current status -Gates 1-4 are implemented on the draft feature branch. The installed 2.0.11 +Gates 1-4 are implemented on the draft feature branch. The installed 2.0.12 runtime has separate passing `Inspect` and `ServiceRestart` reports proving live `LocalService`, outbound-deny coverage, P2000 CUDA 12.6 `sm_61` inference, resource bounds, signed evidence packaging, and restart recovery. Its latest diff --git a/scripts/evidence-mentor-service.ps1 b/scripts/evidence-mentor-service.ps1 index b06fc32..6dc3fa6 100644 --- a/scripts/evidence-mentor-service.ps1 +++ b/scripts/evidence-mentor-service.ps1 @@ -1,7 +1,7 @@ [CmdletBinding(SupportsShouldProcess)] param( [Parameter(Mandatory)] [ValidateSet('Install','Upgrade','Uninstall','Status')] [string] $Action, - [string] $Version = '2.0.11', + [string] $Version = '2.0.12', [string] $DistributionPath, [string] $JavaHome, [string] $ProgramRoot = 'C:\ProgramData\PathLab\EvidenceMentor', diff --git a/scripts/test-evidence-mentor-service.ps1 b/scripts/test-evidence-mentor-service.ps1 index f3ff66b..817bfe4 100644 --- a/scripts/test-evidence-mentor-service.ps1 +++ b/scripts/test-evidence-mentor-service.ps1 @@ -403,8 +403,8 @@ function Wait-JobActive([string] $ResolvedJobId, [int] $Seconds = 60) { if ($response.StatusCode -ne 200) { Start-Sleep -Seconds 1; continue } $job = $response.Content | ConvertFrom-Json if ($job.state -in @('validating','running','refining','packaging')) { - Add-Check 'reboot-job-active' 'PASS' 'The bounded job is active and checkpoint-capable before manual reboot.' @{ - jobId = $ResolvedJobId; state = $job.state; lane = $job.lane + Add-Check 'reboot-job-active' 'PASS' 'The bounded job is durably active before manual reboot.' @{ + jobId = $ResolvedJobId; state = $job.state; lane = $job.lane; updatedAt = $job.updatedAt } return $true } @@ -425,12 +425,21 @@ function Prepare-RebootChallenge([string] $ResolvedJobId) { $endpoint = Read-Endpoint if ($null -eq $endpoint) { Add-Check 'reboot-challenge' 'FAIL' 'Runner endpoint is required before preparing reboot verification.'; return } $os = Get-CimInstance Win32_OperatingSystem + $jobAtChallenge = $null + if ($ResolvedJobId) { + try { + $jobResponse = Invoke-Runner 'GET' "/v1/jobs/$ResolvedJobId" + if ($jobResponse.StatusCode -eq 200) { $jobAtChallenge = $jobResponse.Content | ConvertFrom-Json } + } catch { } + } $challenge = [ordered]@{ schema = 'pathlab.service-reboot-challenge/1' createdAt = [DateTimeOffset]::UtcNow.ToString('O') osBootTime = ([DateTimeOffset]$os.LastBootUpTime).ToUniversalTime().ToString('O') runnerBootId = [string]$endpoint.bootId jobId = if ($ResolvedJobId) { $ResolvedJobId } else { '' } + jobState = if ($null -ne $jobAtChallenge) { [string]$jobAtChallenge.state } else { '' } + jobUpdatedAt = if ($null -ne $jobAtChallenge) { [string]$jobAtChallenge.updatedAt } else { '' } } New-Item -ItemType Directory -Path $ReportRoot -Force | Out-Null Write-Atomic (Join-Path $ReportRoot 'reboot-challenge.json') ($challenge | ConvertTo-Json -Depth 5) @@ -452,8 +461,19 @@ function Verify-RebootChallenge { } if ($passed -and $challenge.jobId) { $continued = Wait-Job ([string]$challenge.jobId) $false - Add-Check 'reboot-job-continuation' $(if ($continued) {'PASS'} else {'FAIL'}) 'The pre-reboot bounded job recovered from its verified checkpoint and completed.' @{ - jobId = [string]$challenge.jobId + $completedAfterBoot = $false + $jobUpdatedAt = '' + try { + $jobResponse = Invoke-Runner 'GET' "/v1/jobs/$([string]$challenge.jobId)" + if ($jobResponse.StatusCode -eq 200) { + $job = $jobResponse.Content | ConvertFrom-Json + $jobUpdatedAt = [string]$job.updatedAt + $completedAfterBoot = [DateTimeOffset]::Parse($jobUpdatedAt) -gt $currentBoot + } + } catch { } + $continued = $continued -and $completedAfterBoot + Add-Check 'reboot-job-continuation' $(if ($continued) {'PASS'} else {'FAIL'}) 'The pre-reboot bounded job must complete after the new OS boot, not before it.' @{ + jobId = [string]$challenge.jobId; jobUpdatedAt = $jobUpdatedAt; currentBoot = $currentBoot.ToString('O'); completedAfterBoot = $completedAfterBoot } } elseif (-not $challenge.jobId) { Add-Check 'reboot-job-continuation' 'NOT_EVALUABLE' 'The reboot challenge did not include a bounded job.' diff --git a/src/main/java/org/pathlab/forge/evidence/EvidenceMentorRunner.java b/src/main/java/org/pathlab/forge/evidence/EvidenceMentorRunner.java index 5a8b40b..bb52677 100644 --- a/src/main/java/org/pathlab/forge/evidence/EvidenceMentorRunner.java +++ b/src/main/java/org/pathlab/forge/evidence/EvidenceMentorRunner.java @@ -30,7 +30,7 @@ /** Standalone authenticated loopback process for unattended Evidence Mentor jobs. */ public final class EvidenceMentorRunner implements AutoCloseable { private static final ObjectMapper JSON = new ObjectMapper(); - private static final String VERSION = "2.0.11"; + private static final String VERSION = "2.0.12"; private static final Duration LEASE = Duration.ofSeconds(45); private final Path stateRoot; private final String token; diff --git a/src/test/java/org/pathlab/forge/evidence/EvidenceMentorRunnerTest.java b/src/test/java/org/pathlab/forge/evidence/EvidenceMentorRunnerTest.java index d1a4355..0d7823a 100644 --- a/src/test/java/org/pathlab/forge/evidence/EvidenceMentorRunnerTest.java +++ b/src/test/java/org/pathlab/forge/evidence/EvidenceMentorRunnerTest.java @@ -96,7 +96,7 @@ void acceptsAuthenticatedLoopbackSubmissionIntoDurableQueue() throws Exception { var endpoint = JSON.readTree(temporaryDirectory.resolve("state/endpoint.json").toFile()); assertEquals("pathlab.runner-endpoint/1", endpoint.path("schema").asText()); - assertEquals("2.0.11", endpoint.path("serviceVersion").asText()); + assertEquals("2.0.12", endpoint.path("serviceVersion").asText()); assertEquals(runner.uri("/").getPort(), endpoint.path("port").asInt()); assertTrue(!endpoint.has("token")); } From 90ddb84b74ab15d7f8b40db8f61f3616f191727c Mon Sep 17 00:00:00 2001 From: Enksodsoon <229880765+Enksodsoon@users.noreply.github.com> Date: Sat, 22 Aug 2026 15:39:28 +0700 Subject: [PATCH 20/50] Fail closed experimental evidence packs --- .../pathlab-ai-evidence-v1.schema.json | 4 +- docs/evidence-mentor-runner.md | 22 ++-- docs/model-cards/README.md | 4 +- docs/model-cards/cell-od-watershed-v1.md | 13 +++ docs/model-cards/he-gigapath-benchmark-v1.md | 10 ++ docs/model-cards/ihc-descriptive-v1.md | 17 +++ docs/plans/active/current.md | 8 +- .../forge/evidence/EvidenceJobProcessor.java | 36 ++++-- .../forge/evidence/EvidencePackManifest.java | 10 +- .../evidence/EvidenceJobProcessorTest.java | 109 +++++++++++++++--- .../evidence/EvidencePackManifestTest.java | 35 +++++- 11 files changed, 224 insertions(+), 44 deletions(-) create mode 100644 docs/model-cards/cell-od-watershed-v1.md create mode 100644 docs/model-cards/he-gigapath-benchmark-v1.md create mode 100644 docs/model-cards/ihc-descriptive-v1.md diff --git a/contracts/evidence-mentor/pathlab-ai-evidence-v1.schema.json b/contracts/evidence-mentor/pathlab-ai-evidence-v1.schema.json index ec35fc1..bf74b14 100644 --- a/contracts/evidence-mentor/pathlab-ai-evidence-v1.schema.json +++ b/contracts/evidence-mentor/pathlab-ai-evidence-v1.schema.json @@ -16,8 +16,8 @@ "reviewRequired": {"const": true}, "coordinates": {"type": "object", "additionalProperties": false, "required": ["space", "originX", "originY", "scaleX", "scaleY"], "properties": {"space": {"const": "source-pixel"}, "originX": {"type": "number", "minimum": 0}, "originY": {"type": "number", "minimum": 0}, "scaleX": {"type": "number", "exclusiveMinimum": 0}, "scaleY": {"type": "number", "exclusiveMinimum": 0}}}, "evidence": {"type": "array", "maxItems": 1000, "items": {"$ref": "#/$defs/region"}}, - "cellAggregates": {"type": "array", "maxItems": 1000, "items": {"type": "object", "additionalProperties": false, "required": ["regionId", "algorithm", "count", "densityPerMm2", "meanNucleusAreaPx2"], "properties": {"regionId": {"type": "string", "maxLength": 120}, "algorithm": {"enum": ["hovernet-fast", "od-watershed"]}, "count": {"type": "integer", "minimum": 0}, "densityPerMm2": {"type": ["number", "null"], "minimum": 0}, "meanNucleusAreaPx2": {"type": ["number", "null"], "minimum": 0}, "meanNucleusPerimeterPx": {"type": ["number", "null"], "minimum": 0}, "meanNucleusEccentricity": {"type": ["number", "null"], "minimum": 0, "maximum": 1}, "meanNucleusSolidity": {"type": ["number", "null"], "minimum": 0, "maximum": 1}, "uncertainty": {"type": "number", "minimum": 0, "maximum": 1}}}}, - "ihcDescriptors": {"type": "array", "maxItems": 1000, "items": {"type": "object", "additionalProperties": false, "required": ["regionId", "marker", "compartment", "dabAreaFraction", "meanDabOd", "researchEstimate"], "properties": {"regionId": {"type": "string", "maxLength": 120}, "markerId": {"enum": ["generic", "er", "pr", "ki-67", "her2", "pd-l1"]}, "marker": {"enum": ["generic", "er", "pr", "ki-67", "her2", "pd-l1"]}, "analysisMode": {"enum": ["marker-aware", "generic-fallback"]}, "cellMaskSource": {"enum": ["hovernet-fast", "od-watershed"]}, "compartmentSource": {"enum": ["none", "faculty-authored", "faculty-approved", "model-suggested"]}, "calibrationStatus": {"enum": ["calibrated", "relative_only", "not_evaluable"]}, "compartment": {"enum": ["nuclear", "membrane", "tumor-immune-region", "generic-region"]}, "dabAreaFraction": {"type": "number", "minimum": 0, "maximum": 1}, "meanDabOd": {"type": "number", "minimum": 0}, "uncertainty": {"type": "number", "minimum": 0, "maximum": 1}, "abstentionReason": {"type": ["string", "null"], "maxLength": 500}, "researchEstimate": {"const": true}}}}, + "cellAggregates": {"type": "array", "maxItems": 1000, "items": {"type": "object", "additionalProperties": false, "required": ["regionId", "algorithm", "count", "densityPerMm2", "meanNucleusAreaPx2"], "properties": {"regionId": {"type": "string", "maxLength": 120}, "algorithm": {"enum": ["hovernet-fast", "od-watershed", "od-connected-components"]}, "count": {"type": "integer", "minimum": 0}, "densityPerMm2": {"type": ["number", "null"], "minimum": 0}, "meanNucleusAreaPx2": {"type": ["number", "null"], "minimum": 0}, "meanNucleusPerimeterPx": {"type": ["number", "null"], "minimum": 0}, "meanNucleusEccentricity": {"type": ["number", "null"], "minimum": 0, "maximum": 1}, "meanNucleusSolidity": {"type": ["number", "null"], "minimum": 0, "maximum": 1}, "uncertainty": {"type": "number", "minimum": 0, "maximum": 1}}}}, + "ihcDescriptors": {"type": "array", "maxItems": 1000, "items": {"type": "object", "additionalProperties": false, "required": ["regionId", "marker", "compartment", "dabAreaFraction", "meanDabOd", "researchEstimate"], "properties": {"regionId": {"type": "string", "maxLength": 120}, "markerId": {"enum": ["generic", "er", "pr", "ki-67", "her2", "pd-l1"]}, "marker": {"enum": ["generic", "er", "pr", "ki-67", "her2", "pd-l1"]}, "markerIdentitySource": {"enum": ["unknown", "faculty-authored", "slide-label", "import-metadata"]}, "analysisMode": {"enum": ["marker-aware", "generic-fallback", "generic-descriptive"]}, "cellMaskSource": {"enum": ["hovernet-fast", "od-watershed", "od-connected-components"]}, "compartmentSource": {"enum": ["none", "faculty-authored", "faculty-approved", "model-suggested"]}, "calibrationStatus": {"enum": ["calibrated", "relative_only", "not_evaluable"]}, "compartment": {"enum": ["nuclear", "membrane", "tumor-immune-region", "generic-region"]}, "dabAreaFraction": {"type": "number", "minimum": 0, "maximum": 1}, "meanDabOd": {"type": "number", "minimum": 0}, "uncertainty": {"type": "number", "minimum": 0, "maximum": 1}, "abstentionReason": {"type": ["string", "null"], "maxLength": 500}, "researchEstimate": {"const": true}}}}, "citations": {"type": "array", "maxItems": 100, "items": {"type": "object", "additionalProperties": false, "required": ["claimId", "sourceUrl"], "properties": {"claimId": {"type": "string", "maxLength": 160}, "sourceUrl": {"type": "string", "format": "uri", "maxLength": 1000}}}}, "qc": {"type": "object", "additionalProperties": false, "required": ["focus", "tissueFraction", "uncertainty", "abstentionReasons"], "properties": {"focus": {"type": "number", "minimum": 0, "maximum": 1}, "tissueFraction": {"type": "number", "minimum": 0, "maximum": 1}, "uncertainty": {"type": "number", "minimum": 0, "maximum": 1}, "abstentionReasons": {"type": "array", "maxItems": 20, "items": {"type": "string", "maxLength": 500}}, "calibrationStatus": {"enum": ["calibrated", "relative_only", "not_evaluable"]}, "backgroundFraction": {"type": "number", "minimum": 0, "maximum": 1}, "saturationFraction": {"type": "number", "minimum": 0, "maximum": 1}, "stainSeparation": {"type": "number", "minimum": 0, "maximum": 1}, "warnings": {"type": "array", "maxItems": 20, "items": {"type": "string", "maxLength": 500}}}}, "provenance": {"type": "object", "additionalProperties": false, "required": ["createdAt", "codeRevision", "offlineAnalysis"], "properties": {"createdAt": {"type": "string", "format": "date-time"}, "codeRevision": {"type": "string", "maxLength": 160}, "offlineAnalysis": {"const": true}}}, diff --git a/docs/evidence-mentor-runner.md b/docs/evidence-mentor-runner.md index 3a24185..de363b6 100644 --- a/docs/evidence-mentor-runner.md +++ b/docs/evidence-mentor-runner.md @@ -139,10 +139,18 @@ an incomplete report into passing evidence. ## Model boundary -The deterministic cell/IHC baseline is runnable. DINOv2-small remains a -`not_evaluable` executable candidate until its cross-tissue held-out cohort is -complete. Hibou-B and HoVer-Net remain `not_evaluable` until exact, -rights-approved workers and artifacts pass qualification. H&E, cells, IHC, -special stains, cytology, and later Atlas tracks stay independently signed and -fail closed; none may emit diagnosis, clinical categories, TPS/CPS, prognosis, -or treatment guidance. +The deterministic cell/IHC baselines are runnable only with non-identifying +`qualification-` job IDs. Their `experimental` status is not pilot +eligibility. The current cell fallback is connected-component morphometry, not +validated watershed instance segmentation. Current IHC output is generic +within-image H/DAB description; marker-specific requests explicitly fall back, +and PD-L1 refuses compartment claims until reviewed geometry is carried by the +request contract. + +DINOv2-small remains a `not_evaluable` executable candidate until its +cross-tissue held-out cohort is complete. Hibou-B and HoVer-Net remain +`not_evaluable` until exact, rights-approved workers and artifacts pass +qualification. Only `qualified` private-research packs may process ordinary +staff/demo jobs. H&E, cells, IHC, special stains, cytology, and later Atlas +tracks stay independently signed and fail closed; none may emit diagnosis, +clinical categories, TPS/CPS, prognosis, or treatment guidance. diff --git a/docs/model-cards/README.md b/docs/model-cards/README.md index f163282..92637eb 100644 --- a/docs/model-cards/README.md +++ b/docs/model-cards/README.md @@ -10,8 +10,8 @@ must be recorded before a status can move from `not-evaluable`. | H&E | DINOv2-small | `not-evaluable` | Experimental only after exact artifact and held-out retrieval run | | H&E | Hibou-B | `not-evaluable` | User accepts gated terms; then rights, reproducibility, quality and latency gates | | Cells | HoVer-Net fast | `not-evaluable` | Exact private weights plus PQ/count/morphometry/resource gates | -| Cells | OD watershed | `experimental` | Deterministic fallback; no clinical cell identity | -| IHC | H/DAB descriptors | `experimental` | Within-slide research descriptors; reviewed compartments for PD-L1 | +| Cells | OD fallback | `experimental` | Qualification IDs only; touching-nucleus watershed split and instance-mask gates remain | +| IHC | H/DAB descriptors | `experimental` | Qualification IDs only; generic within-image descriptors until marker-specific fixtures pass | | Tutor | Qwen3-0.6B INT4 | `not-evaluable` | Complete browser asset at most 500 MiB and claim-ID-only tests | | Distillation | Atlas-H&E ~72M | `not-evaluable` | Baselines frozen, derivative rights pass, bounded feasibility run succeeds | diff --git a/docs/model-cards/cell-od-watershed-v1.md b/docs/model-cards/cell-od-watershed-v1.md new file mode 100644 index 0000000..f4c423c --- /dev/null +++ b/docs/model-cards/cell-od-watershed-v1.md @@ -0,0 +1,13 @@ +# Optical-density cell fallback + +Status: `experimental`. This deterministic CPU fallback currently applies bounded +hematoxylin-like color thresholding and eight-connected component measurements. +The historical pack identifier says `watershed`, but v1 does not yet split +touching nuclei with a validated distance-transform watershed. It must therefore +run only as a `qualification-` job and must not be activated for staff/demo +evidence until the algorithm name, instance masks, morphometry distributions, +cross-tissue fixtures, and failure-rate gates are resolved. + +Reported counts, area, perimeter, eccentricity, and solidity are descriptive +pixel measurements. They are not clinical cell identities, tumor labels, or a +diagnostic result. No learned cell-type prediction is performed. diff --git a/docs/model-cards/he-gigapath-benchmark-v1.md b/docs/model-cards/he-gigapath-benchmark-v1.md new file mode 100644 index 0000000..5b145a6 --- /dev/null +++ b/docs/model-cards/he-gigapath-benchmark-v1.md @@ -0,0 +1,10 @@ +# GigaPath benchmark-only pack + +Status: `not-evaluable`. No weights or executable worker are bundled. The +upstream deployment boundary excludes this candidate from OCI, Study Packs, and +staff/demo evidence. It may be considered only for a lawful local benchmark or +future teacher review after exact artifact, license, and derivative-rights +records are independently approved. + +Results from this pack must never satisfy pilot activation, evidence publication, +diagnostic, clinical-scoring, or treatment claims. diff --git a/docs/model-cards/ihc-descriptive-v1.md b/docs/model-cards/ihc-descriptive-v1.md new file mode 100644 index 0000000..0e32023 --- /dev/null +++ b/docs/model-cards/ihc-descriptive-v1.md @@ -0,0 +1,17 @@ +# Deterministic H/DAB descriptive pack + +Status: `experimental`. The current CPU implementation provides bounded, +within-image DAB-like area and optical-density summaries plus deterministic +hematoxylin-like connected-component morphometry. It does not yet implement a +validated stain-vector deconvolution, calibrated cross-batch intensity, +marker-specific nuclear counting, HER2 membrane completeness, or PD-L1 +tumor/immune compartment measurement. + +Until those independent fixtures pass, every execution is qualification-only. +Marker names describe the requested research track; they do not make the generic +measurements marker-specific. PD-L1 requires reviewed compartment geometry, not +only a compartment-source label. Missing geometry must produce generic regional +output and `COMPARTMENT_REVIEW_REQUIRED`. + +The pack never emits positive/negative calls, ASCO/CAP categories, TPS, CPS, +diagnosis, prognosis, treatment guidance, or cross-slide cell correspondence. diff --git a/docs/plans/active/current.md b/docs/plans/active/current.md index 42d7a9d..503f29e 100644 --- a/docs/plans/active/current.md +++ b/docs/plans/active/current.md @@ -35,7 +35,13 @@ live `LocalService`, outbound-deny coverage, P2000 CUDA 12.6 `sm_61` inference, resource bounds, signed evidence packaging, and restart recovery. Its latest aggregate report is still `NOT_EVALUABLE` solely because the operator-controlled manual-reboot challenge has not run. Gate 5 therefore remains incomplete. Gate -6 has not been qualified, and Gate 7 has not begun. +6 qualification preparation has begun: ordinary jobs now require a `qualified` +pack, while `experimental` packs are restricted to non-identifying +`qualification-` jobs. The deterministic cell fallback is accurately +identified as connected-component morphometry, and IHC marker-specific or PD-L1 +compartment claims fail closed to generic descriptive output. Neither track is +qualified. Special stains and cytology remain unimplemented, and Gate 7 has not +begun. Implementation, service installation, model qualification, merge, deployment, feature activation, and any future clinical/capacity study are separate gates. diff --git a/src/main/java/org/pathlab/forge/evidence/EvidenceJobProcessor.java b/src/main/java/org/pathlab/forge/evidence/EvidenceJobProcessor.java index 3f601e3..c7f1eff 100644 --- a/src/main/java/org/pathlab/forge/evidence/EvidenceJobProcessor.java +++ b/src/main/java/org/pathlab/forge/evidence/EvidenceJobProcessor.java @@ -24,7 +24,8 @@ public final class EvidenceJobProcessor { "schema", "sourcePath", "sourceSha256", "slideRevision", "previewPath", "sourceWidth", "sourceHeight", "packManifest", "stain", "marker"); private static final Set OPTIONAL_REQUEST_FIELDS = Set.of( - "compartmentSource", "controlsValidated", "tileCacheManifest", "tileCacheManifestSha256"); + "compartmentSource", "controlsValidated", "markerIdentitySource", + "tileCacheManifest", "tileCacheManifestSha256"); private final EvidenceJobQueue queue; private final Path stateRoot; @@ -156,9 +157,17 @@ public EvidenceJob process(EvidenceJob job, String workerId, Instant now) throws ? request.path("compartmentSource").textValue() : "none"; require(Set.of("none", "faculty-authored", "faculty-approved", "model-suggested") .contains(compartmentSource), "Evidence compartment source is invalid"); - var reviewedCompartment = Set.of("faculty-authored", "faculty-approved").contains(compartmentSource); - var compartmentWarning = "pd-l1".equals(marker) && !reviewedCompartment - ? "COMPARTMENT_REVIEW_REQUIRED" : null; + var markerIdentitySource = request.path("markerIdentitySource").isTextual() + ? request.path("markerIdentitySource").textValue() : "unknown"; + require(Set.of("unknown", "faculty-authored", "slide-label", "import-metadata") + .contains(markerIdentitySource), "Evidence marker identity source is invalid"); + // This pack has no compartment geometry and no qualified marker-specific algorithms yet. + // A provenance label alone must never be promoted into a tumor/immune measurement claim. + var descriptorWarning = pack.capability() != EvidencePackManifest.Capability.IHC_DESCRIPTIVE + || "generic".equals(marker) ? null + : "pd-l1".equals(marker) + ? "COMPARTMENT_REVIEW_REQUIRED" + : "MARKER_SPECIFIC_QUALIFICATION_REQUIRED"; var abstained = !abstentionReasons.isEmpty(); var refiningAt = now.plusSeconds(1); checkCancellation(job.id(), workerId, refiningAt); @@ -177,7 +186,7 @@ public EvidenceJob process(EvidenceJob job, String workerId, Instant now) throws Math.max(1, Files.size(preview))); Files.createDirectories(artifactRoot); var unsigned = evidence(request, pack, image, analysis, stainQc, modelResult, focus, tissue, - abstentionReasons, compartmentSource, compartmentWarning, now); + abstentionReasons, compartmentSource, markerIdentitySource, descriptorWarning, now); var finalArtifact = artifactRoot.resolve("evidence.json"); new EvidenceBundleWriter(stateRoot.resolve("signing")).write(finalArtifact, unsigned); queue.recordFinalArtifact(job.id(), workerId, sha256(finalArtifact), now.plusMillis(2500)); @@ -201,7 +210,8 @@ private static ObjectNode evidence( double tissue, java.util.List abstentionReasons, String compartmentSource, - String compartmentWarning, + String markerIdentitySource, + String descriptorWarning, Instant now) { var root = JSON.createObjectNode(); root.put("schema", EvidenceBundleWriter.SCHEMA); @@ -236,7 +246,7 @@ private static ObjectNode evidence( : modelResult.path("regions").deepCopy()); var aggregate = new java.util.LinkedHashMap(); aggregate.put("regionId", "region-1"); - aggregate.put("algorithm", "od-watershed"); + aggregate.put("algorithm", "od-connected-components"); aggregate.put("count", analysis.cellCount()); aggregate.put("densityPerMm2", null); aggregate.put("meanNucleusAreaPx2", analysis.meanNucleusAreaPx2()); @@ -253,15 +263,17 @@ private static ObjectNode evidence( descriptor.put("regionId", "region-1"); descriptor.put("markerId", analysis.marker()); descriptor.put("marker", analysis.marker()); - descriptor.put("analysisMode", compartmentWarning == null ? "marker-aware" : "generic-fallback"); - descriptor.put("cellMaskSource", "od-watershed"); + descriptor.put("markerIdentitySource", markerIdentitySource); + descriptor.put("analysisMode", "generic".equals(analysis.marker()) + ? "generic-descriptive" : "generic-fallback"); + descriptor.put("cellMaskSource", "od-connected-components"); descriptor.put("compartmentSource", compartmentSource); descriptor.put("calibrationStatus", stainQc.calibrationStatus()); - descriptor.put("compartment", compartmentWarning == null ? analysis.compartment() : "generic-region"); + descriptor.put("compartment", "generic-region"); descriptor.put("dabAreaFraction", analysis.dabAreaFraction()); descriptor.put("meanDabOd", analysis.meanDabOd()); descriptor.put("uncertainty", 1 - Math.min(focus, tissue)); - descriptor.put("abstentionReason", compartmentWarning); + descriptor.put("abstentionReason", descriptorWarning); descriptor.put("researchEstimate", true); ihc.add(descriptor); } @@ -276,7 +288,7 @@ private static ObjectNode evidence( qc.put("backgroundFraction", stainQc.backgroundFraction()); qc.put("saturationFraction", stainQc.saturationFraction()); qc.put("stainSeparation", stainQc.separationScore()); - qc.put("warnings", compartmentWarning == null ? java.util.List.of() : java.util.List.of(compartmentWarning)); + qc.put("warnings", descriptorWarning == null ? java.util.List.of() : java.util.List.of(descriptorWarning)); root.set("qc", JSON.valueToTree(qc)); root.set("provenance", JSON.valueToTree(java.util.Map.of( "createdAt", now.toString(), "codeRevision", "pathlab-forge-evidence-mentor-v1", diff --git a/src/main/java/org/pathlab/forge/evidence/EvidencePackManifest.java b/src/main/java/org/pathlab/forge/evidence/EvidencePackManifest.java index 568cb91..7f8dc60 100644 --- a/src/main/java/org/pathlab/forge/evidence/EvidencePackManifest.java +++ b/src/main/java/org/pathlab/forge/evidence/EvidencePackManifest.java @@ -106,9 +106,7 @@ public static EvidencePackManifest load(Path manifest) throws IOException { public boolean pilotEligible() { return !acceptanceOnly && "private-research".equals(allowedUse) - && validationStatus != ValidationStatus.BLOCKED - && validationStatus != ValidationStatus.UNSUPPORTED - && validationStatus != ValidationStatus.NOT_EVALUABLE; + && validationStatus == ValidationStatus.QUALIFIED; } public void requirePilotEligible() { @@ -121,6 +119,12 @@ public void requireExecutableForJob(String jobId) { "Acceptance-only AI pack requires a non-identifying acceptance job id"); return; } + if (jobId != null && jobId.matches("qualification-[a-f0-9]{8,64}")) { + require("private-research".equals(allowedUse) + && validationStatus == ValidationStatus.EXPERIMENTAL, + "AI pack is not eligible for qualification execution"); + return; + } requirePilotEligible(); } diff --git a/src/test/java/org/pathlab/forge/evidence/EvidenceJobProcessorTest.java b/src/test/java/org/pathlab/forge/evidence/EvidenceJobProcessorTest.java index 0c38896..04df36f 100644 --- a/src/test/java/org/pathlab/forge/evidence/EvidenceJobProcessorTest.java +++ b/src/test/java/org/pathlab/forge/evidence/EvidenceJobProcessorTest.java @@ -138,17 +138,18 @@ void completesOfflineDescriptiveIhcJobIntoSignedAtomicEvidence() throws Exceptio var pack = temporaryDirectory.resolve("pack.json"); Files.writeString(pack, packJson()); var request = temporaryDirectory.resolve("request.json"); - JSON.writeValue(request.toFile(), java.util.Map.of( - "schema", "pathlab.evidence-job/1", - "sourcePath", source.toString(), - "sourceSha256", sha256(source), - "slideRevision", "revision-1", - "previewPath", preview.toString(), - "sourceWidth", 1600, - "sourceHeight", 1200, - "packManifest", pack.toString(), - "stain", "ihc_dab", - "marker", "ki-67")); + JSON.writeValue(request.toFile(), java.util.Map.ofEntries( + java.util.Map.entry("schema", "pathlab.evidence-job/1"), + java.util.Map.entry("sourcePath", source.toString()), + java.util.Map.entry("sourceSha256", sha256(source)), + java.util.Map.entry("slideRevision", "revision-1"), + java.util.Map.entry("previewPath", preview.toString()), + java.util.Map.entry("sourceWidth", 1600), + java.util.Map.entry("sourceHeight", 1200), + java.util.Map.entry("packManifest", pack.toString()), + java.util.Map.entry("stain", "ihc_dab"), + java.util.Map.entry("marker", "ki-67"), + java.util.Map.entry("markerIdentitySource", "slide-label"))); var now = Instant.parse("2026-08-22T00:00:00Z"); try (var queue = new EvidenceJobQueue(temporaryDirectory.resolve("state/jobs.sqlite3"))) { @@ -164,8 +165,18 @@ void completesOfflineDescriptiveIhcJobIntoSignedAtomicEvidence() throws Exceptio assertEquals("ki-67", evidence.path("ihcDescriptors").get(0).path("marker").asText()); assertEquals("relative_only", evidence.path("ihcDescriptors").get(0) .path("calibrationStatus").asText()); - assertEquals("od-watershed", evidence.path("ihcDescriptors").get(0) + assertEquals("generic-fallback", evidence.path("ihcDescriptors").get(0) + .path("analysisMode").asText()); + assertEquals("slide-label", evidence.path("ihcDescriptors").get(0) + .path("markerIdentitySource").asText()); + assertEquals("generic-region", evidence.path("ihcDescriptors").get(0) + .path("compartment").asText()); + assertEquals("MARKER_SPECIFIC_QUALIFICATION_REQUIRED", evidence.path("ihcDescriptors").get(0) + .path("abstentionReason").asText()); + assertEquals("od-connected-components", evidence.path("ihcDescriptors").get(0) .path("cellMaskSource").asText()); + assertEquals("od-connected-components", evidence.path("cellAggregates").get(0) + .path("algorithm").asText()); assertTrue(evidence.path("cellAggregates").get(0) .path("meanNucleusPerimeterPx").asDouble() > 0); assertTrue(evidence.path("researchOnly").asBoolean()); @@ -174,6 +185,35 @@ void completesOfflineDescriptiveIhcJobIntoSignedAtomicEvidence() throws Exceptio } } + @Test + void refusesPdL1CompartmentClaimsWithoutReviewedGeometry() throws Exception { + var evidence = processIhcEvidence("pd-l1", "faculty-approved", "faculty-authored", "job-pdl1"); + var descriptor = evidence.path("ihcDescriptors").get(0); + + assertEquals("pd-l1", descriptor.path("markerId").asText()); + assertEquals("generic-fallback", descriptor.path("analysisMode").asText()); + assertEquals("generic-region", descriptor.path("compartment").asText()); + assertEquals("faculty-approved", descriptor.path("compartmentSource").asText()); + assertEquals("COMPARTMENT_REVIEW_REQUIRED", descriptor.path("abstentionReason").asText()); + assertTrue(evidence.path("qc").path("warnings").toString() + .contains("COMPARTMENT_REVIEW_REQUIRED")); + } + + @Test + void genericIhcOutputIsDescriptiveAndContainsNoClinicalScores() throws Exception { + var evidence = processIhcEvidence("generic", "none", "unknown", "job-generic"); + var descriptor = evidence.path("ihcDescriptors").get(0); + + assertEquals("generic-descriptive", descriptor.path("analysisMode").asText()); + assertEquals("generic-region", descriptor.path("compartment").asText()); + assertTrue(descriptor.path("abstentionReason").isNull()); + for (var prohibited : java.util.List.of( + "positive", "negative", "tps", "cps", "ascoCapCategory", + "diagnosis", "prognosis", "treatmentGuidance")) { + assertTrue(!descriptor.has(prohibited), () -> "Clinical output leaked: " + prohibited); + } + } + @Test void resumesSignedIhcPackagingFromRefiningCheckpointAfterRunnerRestart() throws Exception { var source = temporaryDirectory.resolve("resume-source.bin"); @@ -218,8 +258,8 @@ private static String packJson() { "\"rights\":{\"license\":\"internal\",\"allowedUse\":\"private-research\"," + "\"redistributable\":false,\"derivativesAllowed\":false,\"reviewedAt\":\"2026-08-22T00:00:00Z\"}," + "\"resourceEnvelope\":{\"maxRamMiB\":1024,\"maxVramMiB\":0,\"maxSeconds\":300,\"network\":false}," + - "\"validation\":{\"status\":\"experimental\",\"modelCard\":\"model.md\",\"heldOutEvaluation\":\"evaluation.json\"}," + - "\"outputSchema\":\"pathlab.ai-evidence/1\",\"markers\":[\"generic\",\"ki-67\"]}"; + "\"validation\":{\"status\":\"qualified\",\"modelCard\":\"model.md\",\"heldOutEvaluation\":\"evaluation.json\"}," + + "\"outputSchema\":\"pathlab.ai-evidence/1\",\"markers\":[\"generic\",\"ki-67\",\"pd-l1\"]}"; } private static String hePackJson() { @@ -251,6 +291,47 @@ private Path request(Path source, Path preview, Path pack, String revision) thro return request; } + private com.fasterxml.jackson.databind.JsonNode processIhcEvidence( + String marker, String compartmentSource, String markerIdentitySource, String jobId) throws Exception { + var source = temporaryDirectory.resolve(jobId + "-source.bin"); + Files.writeString(source, "immutable " + jobId + " source bytes"); + var preview = temporaryDirectory.resolve(jobId + "-preview.png"); + var image = new BufferedImage(16, 12, BufferedImage.TYPE_INT_RGB); + var graphics = image.createGraphics(); + graphics.setColor(Color.WHITE); + graphics.fillRect(0, 0, 16, 12); + graphics.setColor(new Color(90, 55, 35)); + graphics.fillRect(2, 2, 12, 8); + graphics.dispose(); + ImageIO.write(image, "png", preview.toFile()); + var pack = temporaryDirectory.resolve(jobId + "-pack.json"); + Files.writeString(pack, packJson()); + var request = temporaryDirectory.resolve(jobId + "-request.json"); + JSON.writeValue(request.toFile(), java.util.Map.ofEntries( + java.util.Map.entry("schema", "pathlab.evidence-job/1"), + java.util.Map.entry("sourcePath", source.toString()), + java.util.Map.entry("sourceSha256", sha256(source)), + java.util.Map.entry("slideRevision", "revision-" + jobId), + java.util.Map.entry("previewPath", preview.toString()), + java.util.Map.entry("sourceWidth", 1600), + java.util.Map.entry("sourceHeight", 1200), + java.util.Map.entry("packManifest", pack.toString()), + java.util.Map.entry("stain", "ihc_dab"), + java.util.Map.entry("marker", marker), + java.util.Map.entry("compartmentSource", compartmentSource), + java.util.Map.entry("markerIdentitySource", markerIdentitySource))); + var state = temporaryDirectory.resolve(jobId + "-state"); + var now = Instant.parse("2026-08-22T00:00:00Z"); + try (var queue = new EvidenceJobQueue(state.resolve("jobs.sqlite3"))) { + queue.submit(jobId, request, now); + var claimed = queue.claimNext("worker-1", now, Duration.ofMinutes(2)).orElseThrow(); + var completed = new EvidenceJobProcessor(queue, state) + .process(claimed, "worker-1", now.plusSeconds(1)); + assertEquals(EvidenceJobState.COMPLETED, completed.state()); + } + return JSON.readTree(state.resolve("artifacts").resolve(jobId).resolve("evidence.json").toFile()); + } + private static String sha256(Path path) throws Exception { return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256") .digest(Files.readAllBytes(path))); diff --git a/src/test/java/org/pathlab/forge/evidence/EvidencePackManifestTest.java b/src/test/java/org/pathlab/forge/evidence/EvidencePackManifestTest.java index 55c64c0..01d6eca 100644 --- a/src/test/java/org/pathlab/forge/evidence/EvidencePackManifestTest.java +++ b/src/test/java/org/pathlab/forge/evidence/EvidencePackManifestTest.java @@ -7,15 +7,17 @@ import java.nio.file.Files; import java.nio.file.Path; +import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; final class EvidencePackManifestTest { + private static final ObjectMapper JSON = new ObjectMapper(); @TempDir Path temporaryDirectory; @Test void acceptsBoundedPrivateResearchPack() throws Exception { - var path = write("private-research", "experimental", "[\"generic\",\"er\"]", "ihc-descriptive", "[\"ihc_dab\"]"); + var path = write("private-research", "qualified", "[\"generic\",\"er\"]", "ihc-descriptive", "[\"ihc_dab\"]"); var pack = EvidencePackManifest.load(path); assertEquals("ihc-descriptive-v1", pack.packId()); assertTrue(pack.pilotEligible()); @@ -24,6 +26,17 @@ void acceptsBoundedPrivateResearchPack() throws Exception { assertEquals(64, pack.sha256().length()); } + @Test + void experimentalPackRunsOnlyAsANonIdentifyingQualificationJob() throws Exception { + var pack = EvidencePackManifest.load(write( + "private-research", "experimental", "[\"generic\"]", "ihc-descriptive", "[\"ihc_dab\"]")); + + assertFalse(pack.pilotEligible()); + pack.requireExecutableForJob("qualification-0123abcd"); + assertThrows(IllegalArgumentException.class, () -> pack.requireExecutableForJob("staff-demo")); + assertThrows(IllegalArgumentException.class, () -> pack.requireExecutableForJob("qualification-person-name")); + } + @Test void blocksBenchmarkOnlyPackFromPilot() throws Exception { var pack = EvidencePackManifest.load(write("benchmark-only", "experimental", "[]", "he-evidence", "[\"he\"]")); @@ -73,8 +86,12 @@ void rejectsNetworkedOrOversizedPack() throws Exception { @Test void bundledPilotCatalogActivatesOnlyInstalledLawfulBaselines() throws Exception { var root = Path.of("src/main/resources/evidence-packs"); - assertTrue(EvidencePackManifest.load(root.resolve("cell-od-watershed-v1.json")).pilotEligible()); - assertTrue(EvidencePackManifest.load(root.resolve("ihc-descriptive-v1.json")).pilotEligible()); + var watershed = EvidencePackManifest.load(root.resolve("cell-od-watershed-v1.json")); + var ihc = EvidencePackManifest.load(root.resolve("ihc-descriptive-v1.json")); + assertFalse(watershed.pilotEligible()); + assertFalse(ihc.pilotEligible()); + watershed.requireExecutableForJob("qualification-0123abcd"); + ihc.requireExecutableForJob("qualification-0123abcd"); var dino = EvidencePackManifest.load(root.resolve("he-dinov2-small-v1.json")); assertFalse(dino.pilotEligible()); assertEquals("ed25f3a31f01632728cabb09d1542f84ab7b0056", @@ -97,6 +114,18 @@ void bundledPilotCatalogActivatesOnlyInstalledLawfulBaselines() throws Exception assertFalse(EvidencePackManifest.load(root.resolve("he-gigapath-benchmark-v1.json")).pilotEligible()); } + @Test + void everyBundledPackResolvesItsModelCard() throws Exception { + var root = Path.of("src/main/resources/evidence-packs"); + try (var manifests = Files.list(root)) { + for (var manifest : manifests.filter(path -> path.toString().endsWith(".json")).toList()) { + var card = JSON.readTree(manifest.toFile()).path("validation").path("modelCard").asText(); + assertTrue(!card.isBlank() && Files.isRegularFile(Path.of(card)), + () -> manifest.getFileName() + " references a missing model card: " + card); + } + } + } + private Path write(String use, String status, String markers, String capability, String stains) throws Exception { var path = temporaryDirectory.resolve(java.util.UUID.randomUUID() + ".json"); Files.writeString(path, manifest(use, status, markers, capability, stains)); From 19ea3eedbff9b7f456b6460f4efc32e5c1adf26f Mon Sep 17 00:00:00 2001 From: Enksodsoon <229880765+Enksodsoon@users.noreply.github.com> Date: Sat, 22 Aug 2026 15:48:46 +0700 Subject: [PATCH 21/50] Add brightfield qualification harness --- build.gradle.kts | 13 + contracts/evidence-mentor/README.md | 1 + ...-model-qualification-report-v1.schema.json | 54 ++++ .../brightfield-qualification-protocol-v1.md | 42 +++ docs/model-cards/cell-od-watershed-v1.md | 4 + docs/model-cards/ihc-descriptive-v1.md | 4 + docs/plans/active/current.md | 9 +- .../BrightfieldQualificationHarness.java | 246 ++++++++++++++++++ .../BrightfieldQualificationHarnessTest.java | 84 ++++++ 9 files changed, 455 insertions(+), 2 deletions(-) create mode 100644 contracts/evidence-mentor/pathlab-model-qualification-report-v1.schema.json create mode 100644 docs/evidence/brightfield-qualification-protocol-v1.md create mode 100644 src/main/java/org/pathlab/forge/evidence/BrightfieldQualificationHarness.java create mode 100644 src/test/java/org/pathlab/forge/evidence/BrightfieldQualificationHarnessTest.java diff --git a/build.gradle.kts b/build.gradle.kts index ef65209..9f77b33 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -136,6 +136,19 @@ tasks.withType().configureEach { isReproducibleFileOrder = true } +tasks.register("brightfieldQualification") { + group = "verification" + description = "Runs synthetic-only cell and IHC pre-qualification checks." + dependsOn(tasks.classes) + classpath = sourceSets.main.get().runtimeClasspath + mainClass = "org.pathlab.forge.evidence.BrightfieldQualificationHarness" + val report = providers.gradleProperty("pathlab.qualification.output") + .orElse(layout.buildDirectory.file("qualification/brightfield-report.json").map { it.asFile.absolutePath }) + args( + "--packs", file("src/main/resources/evidence-packs").absolutePath, + "--output", report.get()) +} + val installVersionedRuntime = tasks.register("installVersionedRuntime") { dependsOn(tasks.installDist) val localAppData = providers.environmentVariable("LOCALAPPDATA") diff --git a/contracts/evidence-mentor/README.md b/contracts/evidence-mentor/README.md index f59f72e..c1dfc23 100644 --- a/contracts/evidence-mentor/README.md +++ b/contracts/evidence-mentor/README.md @@ -5,5 +5,6 @@ These schemas are the cross-repository truth boundary for the private research M - `pathlab.ai-pack/1` describes a locally installed expert pack and its rights/resource gates. - `pathlab.ai-evidence/1` is the immutable, signed, slide-bound result accepted by Viewer. - `pathlab.knowledge-pack/1` contains reviewed atomic claims; the tutor may select claim IDs but may not invent factual prose. +- `pathlab.model-qualification-report/1` records checksum-bound synthetic or held-out checks without activating a pack. Raw WSI pixels and reusable embeddings are never valid evidence-bundle members. All contracts are research-only and fail closed. diff --git a/contracts/evidence-mentor/pathlab-model-qualification-report-v1.schema.json b/contracts/evidence-mentor/pathlab-model-qualification-report-v1.schema.json new file mode 100644 index 0000000..d11bd9d --- /dev/null +++ b/contracts/evidence-mentor/pathlab-model-qualification-report-v1.schema.json @@ -0,0 +1,54 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://pathlab.local/schemas/pathlab-model-qualification-report-v1.schema.json", + "title": "PathLab Model Qualification Report v1", + "type": "object", + "additionalProperties": false, + "required": ["schema", "protocolId", "protocolSha256", "generatedAt", "syntheticOnly", "researchOnly", "notDiagnostic", "overallStatus", "tracks"], + "properties": { + "schema": {"const": "pathlab.model-qualification-report/1"}, + "protocolId": {"const": "pathlab-brightfield-synthetic-qualification-v1"}, + "protocolSha256": {"type": "string", "pattern": "^[a-f0-9]{64}$"}, + "generatedAt": {"type": "string", "format": "date-time"}, + "syntheticOnly": {"const": true}, + "researchOnly": {"const": true}, + "notDiagnostic": {"const": true}, + "overallStatus": {"enum": ["qualified", "experimental", "unsupported", "not_evaluable"]}, + "tracks": { + "type": "array", + "minItems": 1, + "maxItems": 16, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["capability", "packId", "packVersion", "packManifestSha256", "status", "checks", "reasons"], + "properties": { + "capability": {"enum": ["cell-morphology", "ihc-descriptive"]}, + "packId": {"type": "string", "pattern": "^[A-Za-z0-9._-]{1,120}$"}, + "packVersion": {"type": "string", "maxLength": 64}, + "packManifestSha256": {"type": "string", "pattern": "^[a-f0-9]{64}$"}, + "status": {"enum": ["qualified", "experimental", "unsupported", "not_evaluable"]}, + "checks": { + "type": "array", + "minItems": 1, + "maxItems": 100, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["id", "outcome"], + "properties": { + "id": {"type": "string", "pattern": "^[a-z0-9][a-z0-9._-]{0,119}$"}, + "outcome": {"enum": ["pass", "fail", "not_evaluable"]}, + "observed": {"type": "number"}, + "required": {"type": "number"}, + "tolerance": {"type": "number", "minimum": 0}, + "detail": {"type": "string", "maxLength": 500} + } + } + }, + "reasons": {"type": "array", "maxItems": 50, "items": {"type": "string", "pattern": "^[A-Z0-9_]{1,120}$"}} + } + } + } + } +} diff --git a/docs/evidence/brightfield-qualification-protocol-v1.md b/docs/evidence/brightfield-qualification-protocol-v1.md new file mode 100644 index 0000000..5202996 --- /dev/null +++ b/docs/evidence/brightfield-qualification-protocol-v1.md @@ -0,0 +1,42 @@ +# Brightfield synthetic qualification protocol v1 + +Status: executable pre-qualification harness. + +This bounded protocol checks deterministic behavior using generated images only. +It contains no patient material, clinical truth, WSI pixels, or external model +weights. Passing checks cannot qualify or activate a pack. The report remains +`experimental` until independent licensed, tissue-diverse, held-out fixtures and +all pre-registered performance gates pass. + +## Cell checks + +- Two separated synthetic hematoxylin-like objects must be counted as two. +- Two touching objects must be separated into two instances. +- Exact repeated analysis must be identical. +- Reviewed-region instance-mask output must exist before qualification. + +The current connected-component implementation predictably fails touching-object +separation and has no instance-mask output. The harness records these outcomes; +it does not lower the gates or call the implementation watershed. + +## IHC checks + +- A synthetic image containing 20% DAB-like pixels must report an area fraction + of 0.2 within `1e-12`. +- Mixed H/DAB-like input without validated controls must remain `relative_only`. +- Weak stain separation must produce `not_evaluable`. +- Marker-specific nuclear/membrane measurement remains `not_evaluable` until + independent fixtures exist. +- PD-L1 compartment measurement remains `not_evaluable` until the request carries + reviewed compartment geometry. + +Run from the repository root: + +```powershell +.\gradlew.bat brightfieldQualification +``` + +The default report is written atomically to +`build/qualification/brightfield-report.json`. It uses +`pathlab.model-qualification-report/1` and binds each track to the exact pack +manifest checksum. diff --git a/docs/model-cards/cell-od-watershed-v1.md b/docs/model-cards/cell-od-watershed-v1.md index f4c423c..1843bab 100644 --- a/docs/model-cards/cell-od-watershed-v1.md +++ b/docs/model-cards/cell-od-watershed-v1.md @@ -11,3 +11,7 @@ cross-tissue fixtures, and failure-rate gates are resolved. Reported counts, area, perimeter, eccentricity, and solidity are descriptive pixel measurements. They are not clinical cell identities, tumor labels, or a diagnostic result. No learned cell-type prediction is performed. + +The executable synthetic protocol is documented in +`docs/evidence/brightfield-qualification-protocol-v1.md`. Its current expected +terminal status is `experimental`, including a failed touching-nuclei check. diff --git a/docs/model-cards/ihc-descriptive-v1.md b/docs/model-cards/ihc-descriptive-v1.md index 0e32023..6109d60 100644 --- a/docs/model-cards/ihc-descriptive-v1.md +++ b/docs/model-cards/ihc-descriptive-v1.md @@ -15,3 +15,7 @@ output and `COMPARTMENT_REVIEW_REQUIRED`. The pack never emits positive/negative calls, ASCO/CAP categories, TPS, CPS, diagnosis, prognosis, treatment guidance, or cross-slide cell correspondence. + +The executable synthetic protocol is documented in +`docs/evidence/brightfield-qualification-protocol-v1.md`. Synthetic generic-DAB +and fail-closed QC checks cannot replace independent marker-specific fixtures. diff --git a/docs/plans/active/current.md b/docs/plans/active/current.md index 503f29e..e304d25 100644 --- a/docs/plans/active/current.md +++ b/docs/plans/active/current.md @@ -40,8 +40,13 @@ pack, while `experimental` packs are restricted to non-identifying `qualification-` jobs. The deterministic cell fallback is accurately identified as connected-component morphometry, and IHC marker-specific or PD-L1 compartment claims fail closed to generic descriptive output. Neither track is -qualified. Special stains and cytology remain unimplemented, and Gate 7 has not -begun. +qualified. A checksum-bound `pathlab.model-qualification-report/1` synthetic +harness now records separated-cell counting, touching-cell separation, +determinism, generic DAB area, relative calibration, and failed-stain refusal +independently. Its current truthful outcome is `experimental`: touching nuclei +are not split, instance masks are unavailable, and marker-specific/PD-L1 +fixtures remain `not_evaluable`. Special stains and cytology remain +unimplemented, and Gate 7 has not begun. Implementation, service installation, model qualification, merge, deployment, feature activation, and any future clinical/capacity study are separate gates. diff --git a/src/main/java/org/pathlab/forge/evidence/BrightfieldQualificationHarness.java b/src/main/java/org/pathlab/forge/evidence/BrightfieldQualificationHarness.java new file mode 100644 index 0000000..4417126 --- /dev/null +++ b/src/main/java/org/pathlab/forge/evidence/BrightfieldQualificationHarness.java @@ -0,0 +1,246 @@ +package org.pathlab.forge.evidence; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import java.awt.Color; +import java.awt.image.BufferedImage; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.security.MessageDigest; +import java.time.Instant; +import java.util.HexFormat; + +/** Runs bounded synthetic checks that inform qualification without qualifying a pack. */ +public final class BrightfieldQualificationHarness { + public static final String SCHEMA = "pathlab.model-qualification-report/1"; + private static final ObjectMapper JSON = new ObjectMapper(); + private static final String PROTOCOL = String.join("\n", + "pathlab-brightfield-synthetic-qualification-v1", + "separated-nuclei-count=2", + "touching-nuclei-separation=2", + "generic-dab-area=0.2+-1e-12", + "weak-separation=not_evaluable", + "marker-specific=not_evaluable-until-independent-fixtures", + "pd-l1-compartments=not_evaluable-without-reviewed-geometry"); + + private BrightfieldQualificationHarness() {} + + public static void main(String[] arguments) throws Exception { + Path packs = null; + Path output = null; + for (var index = 0; index < arguments.length; index++) { + if ("--packs".equals(arguments[index]) && index + 1 < arguments.length) { + packs = Path.of(arguments[++index]); + } else if ("--output".equals(arguments[index]) && index + 1 < arguments.length) { + output = Path.of(arguments[++index]); + } else { + throw new IllegalArgumentException( + "Usage: BrightfieldQualificationHarness --packs PATH --output FILE"); + } + } + if (packs == null || output == null) { + throw new IllegalArgumentException( + "Usage: BrightfieldQualificationHarness --packs PATH --output FILE"); + } + run(packs, output, Instant.now()); + System.out.println("Synthetic qualification report written: " + output.toAbsolutePath().normalize()); + System.out.println("Outcome: experimental (never an activation decision)"); + } + + static void run(Path packRoot, Path output, Instant generatedAt) throws IOException { + if (packRoot == null || output == null || generatedAt == null) { + throw new IllegalArgumentException("Qualification inputs are required"); + } + var normalizedRoot = packRoot.toAbsolutePath().normalize(); + var cellPack = EvidencePackManifest.load(normalizedRoot.resolve("cell-od-watershed-v1.json")); + var ihcPack = EvidencePackManifest.load(normalizedRoot.resolve("ihc-descriptive-v1.json")); + require(cellPack.capability() == EvidencePackManifest.Capability.CELL_MORPHOLOGY, + "Cell qualification pack capability is invalid"); + require(ihcPack.capability() == EvidencePackManifest.Capability.IHC_DESCRIPTIVE, + "IHC qualification pack capability is invalid"); + require(cellPack.validationStatus() == EvidencePackManifest.ValidationStatus.EXPERIMENTAL + && ihcPack.validationStatus() == EvidencePackManifest.ValidationStatus.EXPERIMENTAL, + "Synthetic harness may run only against experimental brightfield packs"); + + var report = JSON.createObjectNode(); + report.put("schema", SCHEMA); + report.put("protocolId", "pathlab-brightfield-synthetic-qualification-v1"); + report.put("protocolSha256", sha256(PROTOCOL.getBytes(StandardCharsets.UTF_8))); + report.put("generatedAt", generatedAt.toString()); + report.put("syntheticOnly", true); + report.put("researchOnly", true); + report.put("notDiagnostic", true); + report.put("overallStatus", "experimental"); + var tracks = report.putArray("tracks"); + tracks.add(cellTrack(cellPack)); + tracks.add(ihcTrack(ihcPack)); + + var normalizedOutput = output.toAbsolutePath().normalize(); + var parent = normalizedOutput.getParent(); + if (parent == null) throw new IllegalArgumentException("Qualification output parent is unavailable"); + Files.createDirectories(parent); + var partial = normalizedOutput.resolveSibling(normalizedOutput.getFileName() + ".partial"); + try { + JSON.writerWithDefaultPrettyPrinter().writeValue(partial.toFile(), report); + Files.move(partial, normalizedOutput, + StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE); + } finally { + Files.deleteIfExists(partial); + } + } + + private static ObjectNode cellTrack(EvidencePackManifest pack) { + var separated = BrightfieldTileAnalyzer.analyze(separatedNuclei(), "generic"); + var touching = BrightfieldTileAnalyzer.analyze(touchingNuclei(), "generic"); + var repeated = BrightfieldTileAnalyzer.analyze(separatedNuclei(), "generic"); + var track = track(pack, "cell-morphology", "experimental"); + var checks = track.putArray("checks"); + checks.add(numericCheck("separated-nuclei-count", separated.cellCount(), 2, + separated.cellCount() == 2 ? "pass" : "fail")); + checks.add(numericCheck("touching-nuclei-separation", touching.cellCount(), 2, + touching.cellCount() == 2 ? "pass" : "fail")); + checks.add(textCheck("deterministic-repeat", + separated.equals(repeated) ? "pass" : "fail", + "Exact repeated analysis result")); + checks.add(textCheck("instance-mask-output", "not_evaluable", + "Current v1 emits aggregates but no reviewed-region instance masks")); + var reasons = track.putArray("reasons"); + reasons.add("TOUCHING_NUCLEI_NOT_SPLIT"); + reasons.add("INSTANCE_MASK_OUTPUT_UNAVAILABLE"); + reasons.add("CROSS_TISSUE_HELD_OUT_FIXTURES_PENDING"); + return track; + } + + private static ObjectNode ihcTrack(EvidencePackManifest pack) { + var dab = BrightfieldTileAnalyzer.analyze(dabAreaFixture(), "generic"); + var mixedQc = BrightfieldStainQc.inspect(mixedStainFixture(), false); + var weakQc = BrightfieldStainQc.inspect(weakSeparationFixture(), false); + var track = track(pack, "ihc-descriptive", "experimental"); + var checks = track.putArray("checks"); + var dabCheck = JSON.createObjectNode(); + dabCheck.put("id", "generic-dab-area"); + dabCheck.put("outcome", Math.abs(dab.dabAreaFraction() - 0.2) <= 1e-12 ? "pass" : "fail"); + dabCheck.put("observed", dab.dabAreaFraction()); + dabCheck.put("required", 0.2); + dabCheck.put("tolerance", 1e-12); + checks.add(dabCheck); + checks.add(textCheck("relative-only-without-controls", + "relative_only".equals(mixedQc.calibrationStatus()) ? "pass" : "fail", + mixedQc.calibrationStatus())); + checks.add(textCheck("weak-separation-refusal", + "not_evaluable".equals(weakQc.calibrationStatus()) + && weakQc.reasons().contains("weak_stain_separation") ? "pass" : "fail", + weakQc.calibrationStatus())); + checks.add(textCheck("marker-specific-measurement", "not_evaluable", + "Nuclear and membrane algorithms await independent fixtures")); + checks.add(textCheck("pd-l1-compartment-measurement", "not_evaluable", + "Reviewed compartment geometry is not present in the v1 request")); + var reasons = track.putArray("reasons"); + reasons.add("VALIDATED_STAIN_VECTOR_DECONVOLUTION_PENDING"); + reasons.add("MARKER_SPECIFIC_FIXTURES_PENDING"); + reasons.add("REVIEWED_COMPARTMENT_GEOMETRY_PENDING"); + return track; + } + + private static ObjectNode track( + EvidencePackManifest pack, String capability, String status) { + var track = JSON.createObjectNode(); + track.put("capability", capability); + track.put("packId", pack.packId()); + track.put("packVersion", pack.version()); + track.put("packManifestSha256", pack.sha256()); + track.put("status", status); + return track; + } + + private static ObjectNode numericCheck(String id, int observed, int required, String outcome) { + var check = JSON.createObjectNode(); + check.put("id", id); + check.put("outcome", outcome); + check.put("observed", observed); + check.put("required", required); + return check; + } + + private static ObjectNode textCheck(String id, String outcome, String detail) { + var check = JSON.createObjectNode(); + check.put("id", id); + check.put("outcome", outcome); + check.put("detail", detail); + return check; + } + + private static BufferedImage separatedNuclei() { + var image = whiteImage(24, 16); + var graphics = image.createGraphics(); + graphics.setColor(new Color(75, 45, 125)); + graphics.fillRect(3, 5, 5, 5); + graphics.fillRect(15, 5, 5, 5); + graphics.dispose(); + return image; + } + + private static BufferedImage touchingNuclei() { + var image = whiteImage(24, 16); + var graphics = image.createGraphics(); + graphics.setColor(new Color(75, 45, 125)); + graphics.fillRect(6, 5, 5, 5); + graphics.fillRect(11, 5, 5, 5); + graphics.dispose(); + return image; + } + + private static BufferedImage dabAreaFixture() { + var image = whiteImage(10, 10); + var graphics = image.createGraphics(); + graphics.setColor(new Color(135, 90, 45)); + graphics.fillRect(0, 0, 10, 2); + graphics.dispose(); + return image; + } + + private static BufferedImage mixedStainFixture() { + var image = whiteImage(16, 16); + var graphics = image.createGraphics(); + graphics.setColor(new Color(75, 45, 125)); + graphics.fillRect(0, 0, 8, 16); + graphics.setColor(new Color(135, 90, 45)); + graphics.fillRect(8, 0, 8, 16); + graphics.dispose(); + return image; + } + + private static BufferedImage weakSeparationFixture() { + var image = new BufferedImage(16, 16, BufferedImage.TYPE_INT_RGB); + var graphics = image.createGraphics(); + graphics.setColor(new Color(170, 170, 170)); + graphics.fillRect(0, 0, 16, 16); + graphics.dispose(); + return image; + } + + private static BufferedImage whiteImage(int width, int height) { + var image = new BufferedImage(width, height, BufferedImage.TYPE_INT_RGB); + var graphics = image.createGraphics(); + graphics.setColor(Color.WHITE); + graphics.fillRect(0, 0, width, height); + graphics.dispose(); + return image; + } + + private static String sha256(byte[] value) throws IOException { + try { + return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(value)); + } catch (java.security.GeneralSecurityException impossible) { + throw new IOException("SHA-256 is unavailable", impossible); + } + } + + private static void require(boolean condition, String message) { + if (!condition) throw new IllegalArgumentException(message); + } +} diff --git a/src/test/java/org/pathlab/forge/evidence/BrightfieldQualificationHarnessTest.java b/src/test/java/org/pathlab/forge/evidence/BrightfieldQualificationHarnessTest.java new file mode 100644 index 0000000..3a79de1 --- /dev/null +++ b/src/test/java/org/pathlab/forge/evidence/BrightfieldQualificationHarnessTest.java @@ -0,0 +1,84 @@ +package org.pathlab.forge.evidence; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.fasterxml.jackson.databind.ObjectMapper; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Instant; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +final class BrightfieldQualificationHarnessTest { + private static final ObjectMapper JSON = new ObjectMapper(); + @TempDir Path temporaryDirectory; + + @Test + void writesChecksumBoundSyntheticQualificationReportAtomically() throws Exception { + var output = temporaryDirectory.resolve("qualification.json"); + var generatedAt = Instant.parse("2026-08-22T09:00:00Z"); + + BrightfieldQualificationHarness.run( + Path.of("src/main/resources/evidence-packs"), output, generatedAt); + + assertTrue(Files.isRegularFile(output)); + assertFalse(Files.exists(output.resolveSibling("qualification.json.partial"))); + var report = JSON.readTree(output.toFile()); + assertEquals("pathlab.model-qualification-report/1", report.path("schema").asText()); + assertEquals(generatedAt.toString(), report.path("generatedAt").asText()); + assertTrue(report.path("syntheticOnly").asBoolean()); + assertTrue(report.path("researchOnly").asBoolean()); + assertTrue(report.path("notDiagnostic").asBoolean()); + assertEquals("experimental", report.path("overallStatus").asText()); + assertEquals(2, report.path("tracks").size()); + assertFalse(report.toString().contains(temporaryDirectory.toString())); + + var cell = track(report, "cell-morphology"); + assertEquals("cell-od-watershed-v1", cell.path("packId").asText()); + assertEquals(64, cell.path("packManifestSha256").asText().length()); + assertEquals("experimental", cell.path("status").asText()); + assertEquals("fail", check(cell, "touching-nuclei-separation").path("outcome").asText()); + assertEquals(1, check(cell, "touching-nuclei-separation").path("observed").asInt()); + assertEquals(2, check(cell, "touching-nuclei-separation").path("required").asInt()); + + var ihc = track(report, "ihc-descriptive"); + assertEquals("ihc-descriptive-v1", ihc.path("packId").asText()); + assertEquals("experimental", ihc.path("status").asText()); + assertEquals("pass", check(ihc, "generic-dab-area").path("outcome").asText()); + assertEquals("pass", check(ihc, "weak-separation-refusal").path("outcome").asText()); + assertEquals("not_evaluable", check(ihc, "marker-specific-measurement") + .path("outcome").asText()); + } + + @Test + void isDeterministicForTheSamePacksAndTimestamp() throws Exception { + var first = temporaryDirectory.resolve("first.json"); + var second = temporaryDirectory.resolve("second.json"); + var generatedAt = Instant.parse("2026-08-22T09:00:00Z"); + + BrightfieldQualificationHarness.run( + Path.of("src/main/resources/evidence-packs"), first, generatedAt); + BrightfieldQualificationHarness.run( + Path.of("src/main/resources/evidence-packs"), second, generatedAt); + + assertEquals(Files.readString(first), Files.readString(second)); + } + + private static com.fasterxml.jackson.databind.JsonNode track( + com.fasterxml.jackson.databind.JsonNode report, String capability) { + for (var track : report.path("tracks")) { + if (capability.equals(track.path("capability").asText())) return track; + } + throw new AssertionError("Missing track: " + capability); + } + + private static com.fasterxml.jackson.databind.JsonNode check( + com.fasterxml.jackson.databind.JsonNode track, String id) { + for (var check : track.path("checks")) { + if (id.equals(check.path("id").asText())) return check; + } + throw new AssertionError("Missing check: " + id); + } +} From 49f43d72125c2cb986d66d3198eed07f58f01cc2 Mon Sep 17 00:00:00 2001 From: Enksodsoon <229880765+Enksodsoon@users.noreply.github.com> Date: Sat, 22 Aug 2026 17:32:58 +0700 Subject: [PATCH 22/50] feat: add autonomous all-rounder qualification campaign --- .../pathlab-ai-evidence-v2.schema.json | 29 ++ .../pathlab-ai-pack-v2.schema.json | 25 ++ .../pathlab-capability-matrix-v1.schema.json | 12 + .../pathlab-evidence-set-v1.schema.json | 11 + ...-model-qualification-report-v2.schema.json | 20 + ...pathlab-model-worker-result-v2.schema.json | 14 + ...hlab-qualification-campaign-v1.schema.json | 45 ++ .../pathlab-study-pack-v3.schema.json | 13 + docs/evidence-mentor-runner.md | 2 +- docs/plans/active/current.md | 53 ++- .../cell-od-watershed-v2.json | 1 + .../cytology-descriptive-v2.json | 1 + .../ihc-descriptive-v2.json | 1 + .../special-stain-descriptive-v2.json | 1 + scripts/evidence-mentor-service.ps1 | 5 +- scripts/prepare-all-rounder-campaign.ps1 | 120 +++++ .../prepare-evidence-mentor-2.1.0-upgrade.ps1 | 36 ++ .../forge/evidence/AtlasFeasibilityGate.java | 63 ++- .../BrightfieldQualificationHarness.java | 9 +- .../evidence/BrightfieldTileAnalyzer.java | 82 +--- .../evidence/CapabilityMatrixWriter.java | 57 +++ .../forge/evidence/EvidenceArtifactStore.java | 44 +- .../forge/evidence/EvidenceBundleWriter.java | 4 + .../evidence/EvidenceBundleWriterV2.java | 52 +++ .../forge/evidence/EvidenceJobProcessor.java | 320 ++++++++++++-- .../forge/evidence/EvidenceMentorRunner.java | 106 ++++- .../forge/evidence/EvidencePackManifest.java | 151 ++++++- .../forge/evidence/EvidenceSetWriter.java | 94 ++++ .../evidence/IhcMeasurementAnalyzer.java | 221 ++++++++++ .../evidence/OpticalDensityWatershed.java | 170 +++++++ .../QualificationCampaignManifest.java | 172 ++++++++ .../QualificationReportValidator.java | 103 +++++ .../evidence/QualificationReportWriter.java | 54 +++ .../evidence/QualificationRunSnapshot.java | 17 + .../forge/evidence/QualificationRunStore.java | 415 ++++++++++++++++++ .../forge/evidence/QualifiedPackRegistry.java | 99 +++++ .../forge/evidence/SpecialStainAnalyzer.java | 67 +++ .../forge/evidence/TrustedSignerRegistry.java | 103 +++++ .../resources/evidence-dashboard/index.html | 14 +- .../evidence-packs/cell-od-watershed-v2.json | 1 + .../cytology-descriptive-v2.json | 1 + .../evidence-packs/ihc-descriptive-v2.json | 1 + .../special-stain-descriptive-v2.json | 1 + .../evidence/AllRounderPackManifestTest.java | 32 ++ .../evidence/AtlasFeasibilityGateTest.java | 25 +- .../BrightfieldQualificationHarnessTest.java | 5 +- .../evidence/BrightfieldTileAnalyzerTest.java | 17 + .../evidence/EvidenceArtifactStoreTest.java | 41 +- .../evidence/EvidenceBundleWriterV2Test.java | 32 ++ .../evidence/EvidenceJobProcessorTest.java | 15 +- .../evidence/EvidenceMentorRunnerTest.java | 27 +- .../evidence/EvidencePackManifestTest.java | 8 +- .../forge/evidence/EvidenceSetWriterTest.java | 49 +++ .../evidence/IhcMeasurementAnalyzerTest.java | 53 +++ .../QualificationCampaignManifestTest.java | 44 ++ .../QualificationReportValidatorTest.java | 58 +++ .../evidence/QualificationRunStoreTest.java | 65 +++ .../evidence/SpecialStainAnalyzerTest.java | 28 ++ 58 files changed, 3125 insertions(+), 184 deletions(-) create mode 100644 contracts/evidence-mentor/pathlab-ai-evidence-v2.schema.json create mode 100644 contracts/evidence-mentor/pathlab-ai-pack-v2.schema.json create mode 100644 contracts/evidence-mentor/pathlab-capability-matrix-v1.schema.json create mode 100644 contracts/evidence-mentor/pathlab-evidence-set-v1.schema.json create mode 100644 contracts/evidence-mentor/pathlab-model-qualification-report-v2.schema.json create mode 100644 contracts/evidence-mentor/pathlab-model-worker-result-v2.schema.json create mode 100644 contracts/evidence-mentor/pathlab-qualification-campaign-v1.schema.json create mode 100644 contracts/evidence-mentor/pathlab-study-pack-v3.schema.json create mode 100644 qualification-protocols/cell-od-watershed-v2.json create mode 100644 qualification-protocols/cytology-descriptive-v2.json create mode 100644 qualification-protocols/ihc-descriptive-v2.json create mode 100644 qualification-protocols/special-stain-descriptive-v2.json create mode 100644 scripts/prepare-all-rounder-campaign.ps1 create mode 100644 scripts/prepare-evidence-mentor-2.1.0-upgrade.ps1 create mode 100644 src/main/java/org/pathlab/forge/evidence/CapabilityMatrixWriter.java create mode 100644 src/main/java/org/pathlab/forge/evidence/EvidenceBundleWriterV2.java create mode 100644 src/main/java/org/pathlab/forge/evidence/EvidenceSetWriter.java create mode 100644 src/main/java/org/pathlab/forge/evidence/IhcMeasurementAnalyzer.java create mode 100644 src/main/java/org/pathlab/forge/evidence/OpticalDensityWatershed.java create mode 100644 src/main/java/org/pathlab/forge/evidence/QualificationCampaignManifest.java create mode 100644 src/main/java/org/pathlab/forge/evidence/QualificationReportValidator.java create mode 100644 src/main/java/org/pathlab/forge/evidence/QualificationReportWriter.java create mode 100644 src/main/java/org/pathlab/forge/evidence/QualificationRunSnapshot.java create mode 100644 src/main/java/org/pathlab/forge/evidence/QualificationRunStore.java create mode 100644 src/main/java/org/pathlab/forge/evidence/QualifiedPackRegistry.java create mode 100644 src/main/java/org/pathlab/forge/evidence/SpecialStainAnalyzer.java create mode 100644 src/main/java/org/pathlab/forge/evidence/TrustedSignerRegistry.java create mode 100644 src/main/resources/evidence-packs/cell-od-watershed-v2.json create mode 100644 src/main/resources/evidence-packs/cytology-descriptive-v2.json create mode 100644 src/main/resources/evidence-packs/ihc-descriptive-v2.json create mode 100644 src/main/resources/evidence-packs/special-stain-descriptive-v2.json create mode 100644 src/test/java/org/pathlab/forge/evidence/AllRounderPackManifestTest.java create mode 100644 src/test/java/org/pathlab/forge/evidence/EvidenceBundleWriterV2Test.java create mode 100644 src/test/java/org/pathlab/forge/evidence/EvidenceSetWriterTest.java create mode 100644 src/test/java/org/pathlab/forge/evidence/IhcMeasurementAnalyzerTest.java create mode 100644 src/test/java/org/pathlab/forge/evidence/QualificationCampaignManifestTest.java create mode 100644 src/test/java/org/pathlab/forge/evidence/QualificationReportValidatorTest.java create mode 100644 src/test/java/org/pathlab/forge/evidence/QualificationRunStoreTest.java create mode 100644 src/test/java/org/pathlab/forge/evidence/SpecialStainAnalyzerTest.java diff --git a/contracts/evidence-mentor/pathlab-ai-evidence-v2.schema.json b/contracts/evidence-mentor/pathlab-ai-evidence-v2.schema.json new file mode 100644 index 0000000..b3ce39f --- /dev/null +++ b/contracts/evidence-mentor/pathlab-ai-evidence-v2.schema.json @@ -0,0 +1,29 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://pathlab.local/schemas/pathlab-ai-evidence-v2.schema.json", + "title": "PathLab AI Evidence v2", "type": "object", "additionalProperties": false, + "required": ["schema", "bundleId", "source", "pack", "qualificationAttestationSha256", "status", "researchOnly", "notDiagnostic", "reviewRequired", "coordinates", "regions", "cellInstances", "cellAggregates", "ihcDescriptors", "specialStainDescriptors", "cytologyDescriptors", "citations", "qc", "provenance", "manifestSha256", "signature"], + "properties": { + "schema": {"const": "pathlab.ai-evidence/2"}, "bundleId": {"$ref": "#/$defs/id"}, + "source": {"type": "object", "additionalProperties": false, "required": ["slideSha256", "revision", "width", "height"], "properties": {"slideSha256": {"$ref": "#/$defs/sha"}, "revision": {"type": "string", "minLength": 1, "maxLength": 160}, "width": {"type": "integer", "minimum": 1}, "height": {"type": "integer", "minimum": 1}}}, + "pack": {"type": "object", "additionalProperties": false, "required": ["id", "version", "manifestSha256", "capability", "scope", "preprocessing", "artifacts"], "properties": {"id": {"$ref": "#/$defs/id"}, "version": {"type": "string", "maxLength": 64}, "manifestSha256": {"$ref": "#/$defs/sha"}, "capability": {"type": "string", "maxLength": 120}, "scope": {"enum": ["deployment", "research-restricted"]}, "preprocessing": {"type": "string", "maxLength": 160}, "artifacts": {"type": "array", "maxItems": 32, "items": {"$ref": "#/$defs/sha"}}}}, + "qualificationAttestationSha256": {"$ref": "#/$defs/sha"}, "status": {"enum": ["completed", "partial", "abstained", "unsupported", "failed"]}, + "researchOnly": {"const": true}, "notDiagnostic": {"const": true}, "reviewRequired": {"const": true}, + "coordinates": {"type": "object", "additionalProperties": false, "required": ["space", "originX", "originY", "scaleX", "scaleY"], "properties": {"space": {"const": "source-pixel"}, "originX": {"type": "number", "minimum": 0}, "originY": {"type": "number", "minimum": 0}, "scaleX": {"type": "number", "exclusiveMinimum": 0}, "scaleY": {"type": "number", "exclusiveMinimum": 0}}}, + "regions": {"type": "array", "maxItems": 1000, "items": {"$ref": "#/$defs/region"}}, + "cellInstances": {"type": "array", "maxItems": 100000, "items": {"type": "object", "additionalProperties": false, "required": ["id", "regionId", "maskEncoding", "mask", "areaPx2", "perimeterPx", "eccentricity", "solidity", "meanIntensity", "uncertainty", "category"], "properties": {"id": {"$ref": "#/$defs/id"}, "regionId": {"$ref": "#/$defs/id"}, "maskEncoding": {"enum": ["polygon", "rle"]}, "mask": {"type": "array", "minItems": 3, "maxItems": 4096, "items": {"type": "number", "minimum": 0}}, "areaPx2": {"type": "number", "minimum": 0}, "perimeterPx": {"type": "number", "minimum": 0}, "eccentricity": {"type": "number", "minimum": 0, "maximum": 1}, "solidity": {"type": "number", "minimum": 0, "maximum": 1}, "meanIntensity": {"type": "number", "minimum": 0, "maximum": 1}, "uncertainty": {"type": "number", "minimum": 0, "maximum": 1}, "category": {"enum": ["unclassified", "model-category-1", "model-category-2", "model-category-3", "model-category-4", "model-category-5"]}}}}, + "cellAggregates": {"type": "array", "maxItems": 1000, "items": {"type": "object", "additionalProperties": false, "required": ["regionId", "algorithm", "count", "densityPerMm2", "distributions", "uncertainty", "qc"], "properties": {"regionId": {"$ref": "#/$defs/id"}, "algorithm": {"enum": ["hovernet-fast", "pathosam", "od-watershed"]}, "count": {"type": "integer", "minimum": 0}, "densityPerMm2": {"type": ["number", "null"], "minimum": 0}, "distributions": {"type": "object"}, "uncertainty": {"type": "number", "minimum": 0, "maximum": 1}, "qc": {"type": "array", "maxItems": 20, "items": {"type": "string", "maxLength": 200}}}}}, + "ihcDescriptors": {"type": "array", "maxItems": 1000, "items": {"$ref": "#/$defs/stainDescriptor"}}, + "specialStainDescriptors": {"type": "array", "maxItems": 1000, "items": {"$ref": "#/$defs/stainDescriptor"}}, + "cytologyDescriptors": {"type": "array", "maxItems": 1000, "items": {"$ref": "#/$defs/stainDescriptor"}}, + "citations": {"type": "array", "maxItems": 100, "items": {"type": "object", "additionalProperties": false, "required": ["claimId", "sourceUrl"], "properties": {"claimId": {"$ref": "#/$defs/id"}, "sourceUrl": {"type": "string", "format": "uri", "maxLength": 1000}}}}, + "qc": {"type": "object", "additionalProperties": false, "required": ["focus", "tissueFraction", "uncertainty", "abstentionReasons", "warnings"], "properties": {"focus": {"type": "number", "minimum": 0, "maximum": 1}, "tissueFraction": {"type": "number", "minimum": 0, "maximum": 1}, "uncertainty": {"type": "number", "minimum": 0, "maximum": 1}, "abstentionReasons": {"type": "array", "maxItems": 20, "items": {"type": "string", "maxLength": 500}}, "warnings": {"type": "array", "maxItems": 20, "items": {"type": "string", "maxLength": 500}}}}, + "provenance": {"type": "object", "additionalProperties": false, "required": ["createdAt", "codeRevision", "offlineAnalysis", "campaignId"], "properties": {"createdAt": {"type": "string", "format": "date-time"}, "codeRevision": {"type": "string", "maxLength": 160}, "offlineAnalysis": {"const": true}, "campaignId": {"type": ["string", "null"], "maxLength": 120}}}, + "manifestSha256": {"$ref": "#/$defs/sha"}, "signature": {"$ref": "#/$defs/signature"} + }, + "$defs": { + "id": {"type": "string", "pattern": "^[A-Za-z0-9._-]{1,160}$"}, "sha": {"type": "string", "pattern": "^[a-f0-9]{64}$"}, + "region": {"type": "object", "additionalProperties": false, "required": ["id", "stage", "kind", "reviewStatus", "x", "y", "width", "height", "score"], "properties": {"id": {"$ref": "#/$defs/id"}, "stage": {"enum": ["coarse", "refined"]}, "kind": {"enum": ["support", "similar", "contrast", "analysis", "tumor-compartment", "immune-compartment"]}, "reviewStatus": {"enum": ["unreviewed", "faculty-authored", "faculty-approved"]}, "x": {"type": "number", "minimum": 0}, "y": {"type": "number", "minimum": 0}, "width": {"type": "number", "exclusiveMinimum": 0}, "height": {"type": "number", "exclusiveMinimum": 0}, "score": {"type": "number", "minimum": 0, "maximum": 1}, "thumbnail": {"type": "string", "pattern": "^thumbnails/[A-Za-z0-9._-]+\\.(png|jpg|webp)$"}}}, + "stainDescriptor": {"type": "object", "additionalProperties": false, "required": ["regionId", "stainId", "markerId", "analysisMode", "cellMaskSource", "compartmentSource", "calibrationStatus", "measurements", "qc", "uncertainty", "abstentionReason", "researchEstimate"], "properties": {"regionId": {"$ref": "#/$defs/id"}, "stainId": {"enum": ["ihc_dab", "pas", "pas_d", "trichrome", "gms", "afb", "papanicolaou", "generic_brightfield"]}, "markerId": {"enum": ["generic", "er", "pr", "ki-67", "her2", "pd-l1", "none"]}, "analysisMode": {"enum": ["marker-aware", "generic-fallback", "generic-descriptive", "special-stain-descriptive", "cytology-descriptive"]}, "cellMaskSource": {"enum": ["hovernet-fast", "pathosam", "od-watershed", "none"]}, "compartmentSource": {"enum": ["none", "faculty-authored", "faculty-approved", "model-suggested"]}, "calibrationStatus": {"enum": ["calibrated", "relative_only", "not_evaluable"]}, "measurements": {"type": "object"}, "qc": {"type": "array", "maxItems": 20, "items": {"type": "string", "maxLength": 200}}, "uncertainty": {"type": "number", "minimum": 0, "maximum": 1}, "abstentionReason": {"type": ["string", "null"], "maxLength": 500}, "researchEstimate": {"const": true}}}, + "signature": {"type": "object", "additionalProperties": false, "required": ["algorithm", "keyId", "value"], "properties": {"algorithm": {"const": "Ed25519"}, "keyId": {"$ref": "#/$defs/sha"}, "value": {"type": "string", "minLength": 1, "maxLength": 1000}}} + } +} diff --git a/contracts/evidence-mentor/pathlab-ai-pack-v2.schema.json b/contracts/evidence-mentor/pathlab-ai-pack-v2.schema.json new file mode 100644 index 0000000..63bc46d --- /dev/null +++ b/contracts/evidence-mentor/pathlab-ai-pack-v2.schema.json @@ -0,0 +1,25 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://pathlab.local/schemas/pathlab-ai-pack-v2.schema.json", + "title": "PathLab AI Pack v2", + "type": "object", "additionalProperties": false, + "required": ["schema", "packId", "version", "capability", "scope", "activationEligible", "acceptedStains", "preprocessing", "artifacts", "runtimeCompatibility", "licenseLedger", "derivativePermissions", "resourceEnvelope", "qualificationPolicy", "outputSchema"], + "properties": { + "schema": {"const": "pathlab.ai-pack/2"}, + "packId": {"$ref": "#/$defs/id"}, "version": {"type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$"}, + "capability": {"enum": ["he-evidence", "cell-morphology", "ihc-descriptive", "special-stain-descriptive", "cytology-descriptive", "grounded-tutor", "atlas-distillation"]}, + "scope": {"enum": ["deployment", "research-restricted", "local-benchmark"]}, + "activationEligible": {"type": "boolean"}, + "acceptedStains": {"type": "array", "minItems": 1, "maxItems": 16, "uniqueItems": true, "items": {"enum": ["he", "ihc_dab", "pas", "pas_d", "trichrome", "gms", "afb", "papanicolaou", "generic_brightfield"]}}, + "preprocessing": {"type": "object", "additionalProperties": false, "required": ["id", "tilePixels"], "properties": {"id": {"type": "string", "maxLength": 120}, "tilePixels": {"type": "integer", "minimum": 64, "maximum": 2048}}}, + "artifacts": {"type": "array", "maxItems": 32, "items": {"type": "object", "additionalProperties": false, "required": ["name", "sha256", "source"], "properties": {"name": {"type": "string", "maxLength": 160}, "sha256": {"$ref": "#/$defs/sha"}, "source": {"type": "string", "maxLength": 1000}}}}, + "runtimeCompatibility": {"type": "object", "additionalProperties": false, "required": ["workerProtocol", "executionProvider", "cuda", "gpuArchitecture", "requiresExternalWorker", "segmentMaxSeconds"], "properties": {"workerProtocol": {"enum": ["pathlab.model-worker-result/1", "pathlab.model-worker-result/2"]}, "executionProvider": {"enum": ["cpu", "cuda", "webgpu", "external"]}, "cuda": {"type": "string", "maxLength": 40}, "gpuArchitecture": {"type": "string", "maxLength": 40}, "requiresExternalWorker": {"type": "boolean"}, "segmentMaxSeconds": {"type": "integer", "minimum": 1, "maximum": 1200}}}, + "licenseLedger": {"type": "array", "minItems": 1, "maxItems": 64, "items": {"type": "object", "additionalProperties": false, "required": ["component", "kind", "license", "revision", "permittedUse", "redistributable", "derivativesAllowed"], "properties": {"component": {"type": "string", "maxLength": 160}, "kind": {"enum": ["code", "weights", "dataset", "runtime"]}, "license": {"type": "string", "maxLength": 240}, "revision": {"type": "string", "maxLength": 160}, "permittedUse": {"enum": ["private-research", "research-restricted", "benchmark-only"]}, "redistributable": {"type": "boolean"}, "derivativesAllowed": {"type": "boolean"}}}}, + "derivativePermissions": {"type": "object", "additionalProperties": false, "required": ["atlasResearch", "atlasClean", "reviewedAt"], "properties": {"atlasResearch": {"type": "boolean"}, "atlasClean": {"type": "boolean"}, "reviewedAt": {"type": "string", "format": "date-time"}}}, + "resourceEnvelope": {"type": "object", "additionalProperties": false, "required": ["lane", "maxRamMiB", "maxVramMiB", "maxSeconds", "network"], "properties": {"lane": {"enum": ["gpu", "cpu_io", "external"]}, "maxRamMiB": {"type": "integer", "minimum": 64, "maximum": 16384}, "maxVramMiB": {"type": "integer", "minimum": 0, "maximum": 4608}, "maxSeconds": {"type": "integer", "minimum": 1, "maximum": 1200}, "network": {"const": false}}}, + "qualificationPolicy": {"type": "object", "additionalProperties": false, "required": ["requiresSignedAttestation", "protocolSha256"], "properties": {"requiresSignedAttestation": {"const": true}, "protocolSha256": {"$ref": "#/$defs/sha"}}}, + "outputSchema": {"enum": ["pathlab.ai-evidence/2", "pathlab.model-qualification-report/2", "pathlab.model-worker-result/2"]}, + "markers": {"type": "array", "uniqueItems": true, "items": {"enum": ["generic", "er", "pr", "ki-67", "her2", "pd-l1"]}} + }, + "$defs": {"id": {"type": "string", "pattern": "^[a-z0-9][a-z0-9._-]{0,119}$"}, "sha": {"type": "string", "pattern": "^[a-f0-9]{64}$"}} +} diff --git a/contracts/evidence-mentor/pathlab-capability-matrix-v1.schema.json b/contracts/evidence-mentor/pathlab-capability-matrix-v1.schema.json new file mode 100644 index 0000000..374f881 --- /dev/null +++ b/contracts/evidence-mentor/pathlab-capability-matrix-v1.schema.json @@ -0,0 +1,12 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://pathlab.local/schemas/pathlab-capability-matrix-v1.schema.json", + "title": "PathLab Capability Matrix v1", "type": "object", "additionalProperties": false, + "required": ["schema", "campaignId", "campaignManifestSha256", "campaignCompleted", "campaignTargetMet", "generatedAt", "tracks", "manifestSha256", "signature"], + "properties": { + "schema": {"const": "pathlab.capability-matrix/1"}, "campaignId": {"$ref": "#/$defs/id"}, "campaignManifestSha256": {"$ref": "#/$defs/sha"}, + "campaignCompleted": {"type": "boolean"}, "campaignTargetMet": {"type": "boolean"}, "generatedAt": {"type": "string", "format": "date-time"}, + "tracks": {"type": "array", "minItems": 1, "maxItems": 64, "items": {"type": "object", "additionalProperties": false, "required": ["trackId", "candidateId", "capability", "scope", "status", "activationEligible", "qualificationReportSha256", "failureCode"], "properties": {"trackId": {"$ref": "#/$defs/id"}, "candidateId": {"$ref": "#/$defs/id"}, "capability": {"type": "string", "maxLength": 120}, "scope": {"enum": ["deployment", "research-restricted", "local-benchmark"]}, "status": {"enum": ["qualified", "experimental", "unsupported", "not_evaluable"]}, "activationEligible": {"type": "boolean"}, "qualificationReportSha256": {"type": ["string", "null"], "pattern": "^[a-f0-9]{64}$"}, "failureCode": {"type": ["string", "null"], "maxLength": 120}}}}, + "manifestSha256": {"$ref": "#/$defs/sha"}, "signature": {"$ref": "#/$defs/signature"} + }, + "$defs": {"id": {"type": "string", "pattern": "^[A-Za-z0-9._-]{1,120}$"}, "sha": {"type": "string", "pattern": "^[a-f0-9]{64}$"}, "signature": {"type": "object", "additionalProperties": false, "required": ["algorithm", "keyId", "value"], "properties": {"algorithm": {"const": "Ed25519"}, "keyId": {"$ref": "#/$defs/sha"}, "value": {"type": "string", "minLength": 1, "maxLength": 1000}}}} +} diff --git a/contracts/evidence-mentor/pathlab-evidence-set-v1.schema.json b/contracts/evidence-mentor/pathlab-evidence-set-v1.schema.json new file mode 100644 index 0000000..4fccacb --- /dev/null +++ b/contracts/evidence-mentor/pathlab-evidence-set-v1.schema.json @@ -0,0 +1,11 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://pathlab.local/schemas/pathlab-evidence-set-v1.schema.json", + "title": "PathLab Evidence Set v1", "type": "object", "additionalProperties": false, + "required": ["schema", "setId", "source", "bundles", "reviewRequired", "manifestSha256", "signature"], + "properties": { + "schema": {"const": "pathlab.evidence-set/1"}, "setId": {"$ref": "#/$defs/id"}, + "source": {"type": "object", "additionalProperties": false, "required": ["slideSha256", "revision"], "properties": {"slideSha256": {"$ref": "#/$defs/sha"}, "revision": {"type": "string", "minLength": 1, "maxLength": 160}}}, + "bundles": {"type": "array", "minItems": 1, "maxItems": 32, "items": {"type": "object", "additionalProperties": false, "required": ["capability", "packId", "manifestSha256", "qualificationAttestationSha256"], "properties": {"capability": {"type": "string", "maxLength": 120}, "packId": {"$ref": "#/$defs/id"}, "manifestSha256": {"$ref": "#/$defs/sha"}, "qualificationAttestationSha256": {"$ref": "#/$defs/sha"}}}}, + "reviewRequired": {"const": true}, "manifestSha256": {"$ref": "#/$defs/sha"}, "signature": {"$ref": "#/$defs/signature"} + }, "$defs": {"id": {"type": "string", "pattern": "^[A-Za-z0-9._-]{1,160}$"}, "sha": {"type": "string", "pattern": "^[a-f0-9]{64}$"}, "signature": {"type": "object", "additionalProperties": false, "required": ["algorithm", "keyId", "value"], "properties": {"algorithm": {"const": "Ed25519"}, "keyId": {"$ref": "#/$defs/sha"}, "value": {"type": "string", "minLength": 1, "maxLength": 1000}}}} +} diff --git a/contracts/evidence-mentor/pathlab-model-qualification-report-v2.schema.json b/contracts/evidence-mentor/pathlab-model-qualification-report-v2.schema.json new file mode 100644 index 0000000..acc9aee --- /dev/null +++ b/contracts/evidence-mentor/pathlab-model-qualification-report-v2.schema.json @@ -0,0 +1,20 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://pathlab.local/schemas/pathlab-model-qualification-report-v2.schema.json", + "title": "PathLab Model Qualification Report v2", "type": "object", "additionalProperties": false, + "required": ["schema", "campaignId", "campaignManifestSha256", "trackId", "candidateId", "capability", "scope", "packManifestSha256", "protocolSha256", "generatedAt", "researchOnly", "notDiagnostic", "status", "checks", "reasons", "manifestSha256", "signature"], + "properties": { + "schema": {"const": "pathlab.model-qualification-report/2"}, "campaignId": {"$ref": "#/$defs/id"}, "campaignManifestSha256": {"$ref": "#/$defs/sha"}, + "trackId": {"$ref": "#/$defs/id"}, "candidateId": {"$ref": "#/$defs/id"}, + "capability": {"enum": ["he-evidence", "cell-morphology", "ihc-descriptive", "special-stain-descriptive", "cytology-descriptive", "grounded-tutor", "atlas-distillation"]}, + "scope": {"enum": ["deployment", "research-restricted", "local-benchmark"]}, "packManifestSha256": {"$ref": "#/$defs/sha"}, "protocolSha256": {"$ref": "#/$defs/sha"}, + "generatedAt": {"type": "string", "format": "date-time"}, "researchOnly": {"const": true}, "notDiagnostic": {"const": true}, + "status": {"enum": ["qualified", "experimental", "unsupported", "not_evaluable"]}, + "checks": {"type": "array", "minItems": 1, "maxItems": 200, "items": {"type": "object", "additionalProperties": false, "required": ["id", "outcome"], "properties": {"id": {"$ref": "#/$defs/id"}, "outcome": {"enum": ["pass", "fail", "not_evaluable"]}, "observed": {"type": ["number", "string", "null"]}, "required": {"type": ["number", "string", "null"]}, "detail": {"type": "string", "maxLength": 1000}}}}, + "reasons": {"type": "array", "maxItems": 100, "items": {"type": "string", "pattern": "^[A-Z0-9_]{1,120}$"}}, + "manifestSha256": {"$ref": "#/$defs/sha"}, "signature": {"$ref": "#/$defs/signature"} + }, + "$defs": { + "id": {"type": "string", "pattern": "^[A-Za-z0-9._-]{1,120}$"}, "sha": {"type": "string", "pattern": "^[a-f0-9]{64}$"}, + "signature": {"type": "object", "additionalProperties": false, "required": ["algorithm", "keyId", "value"], "properties": {"algorithm": {"const": "Ed25519"}, "keyId": {"$ref": "#/$defs/sha"}, "value": {"type": "string", "minLength": 1, "maxLength": 1000}}} + } +} diff --git a/contracts/evidence-mentor/pathlab-model-worker-result-v2.schema.json b/contracts/evidence-mentor/pathlab-model-worker-result-v2.schema.json new file mode 100644 index 0000000..2ab42a2 --- /dev/null +++ b/contracts/evidence-mentor/pathlab-model-worker-result-v2.schema.json @@ -0,0 +1,14 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://pathlab.local/schemas/pathlab-model-worker-result-v2.schema.json", + "title": "PathLab Model Worker Result v2", "type": "object", "additionalProperties": false, + "required": ["schema", "jobId", "capability", "packManifestSha256", "terminalStatus", "artifactKind", "artifactPath", "artifactSha256", "completedUnits", "totalUnits", "offlineAnalysis"], + "properties": { + "schema": {"const": "pathlab.model-worker-result/2"}, "jobId": {"type": "string", "pattern": "^[A-Za-z0-9._-]{1,120}$"}, + "capability": {"enum": ["he-evidence", "cell-morphology", "ihc-descriptive", "special-stain-descriptive", "cytology-descriptive", "grounded-tutor", "atlas-distillation"]}, + "packManifestSha256": {"$ref": "#/$defs/sha"}, "terminalStatus": {"enum": ["completed", "abstained", "unsupported", "failed"]}, + "artifactKind": {"enum": ["evidence", "qualification", "checkpoint", "browser-attestation", "training-attestation"]}, + "artifactPath": {"type": "string", "minLength": 1, "maxLength": 1000}, "artifactSha256": {"$ref": "#/$defs/sha"}, + "completedUnits": {"type": "integer", "minimum": 0}, "totalUnits": {"type": "integer", "minimum": 0}, + "offlineAnalysis": {"const": true} + }, "$defs": {"sha": {"type": "string", "pattern": "^[a-f0-9]{64}$"}} +} diff --git a/contracts/evidence-mentor/pathlab-qualification-campaign-v1.schema.json b/contracts/evidence-mentor/pathlab-qualification-campaign-v1.schema.json new file mode 100644 index 0000000..c263682 --- /dev/null +++ b/contracts/evidence-mentor/pathlab-qualification-campaign-v1.schema.json @@ -0,0 +1,45 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://pathlab.local/schemas/pathlab-qualification-campaign-v1.schema.json", + "title": "PathLab Qualification Campaign v1", + "type": "object", + "additionalProperties": false, + "required": ["schema", "campaignId", "createdAt", "researchOnly", "notDiagnostic", "maxRemediationAttempts", "quota", "tracks"], + "properties": { + "schema": {"const": "pathlab.qualification-campaign/1"}, + "campaignId": {"$ref": "#/$defs/id"}, + "createdAt": {"type": "string", "format": "date-time"}, + "researchOnly": {"const": true}, + "notDiagnostic": {"const": true}, + "maxRemediationAttempts": {"const": 1}, + "quota": { + "type": "object", "additionalProperties": false, + "required": ["sourceBytes", "derivedBytes", "modelBytes", "evidenceBytes", "reserveBytes"], + "properties": { + "sourceBytes": {"const": 48318382080}, "derivedBytes": {"const": 26843545600}, + "modelBytes": {"const": 10737418240}, "evidenceBytes": {"const": 10737418240}, + "reserveBytes": {"const": 10737418240} + } + }, + "tracks": {"type": "array", "minItems": 1, "maxItems": 64, "items": {"$ref": "#/$defs/track"}} + }, + "$defs": { + "id": {"type": "string", "pattern": "^[A-Za-z0-9._-]{1,120}$"}, + "sha": {"type": "string", "pattern": "^[a-f0-9]{64}$"}, + "track": { + "type": "object", "additionalProperties": false, + "required": ["id", "candidateId", "capability", "scope", "requestPath", "remediationRequestPath", "expectedAttestationPath", "protocolSha256", "dependsOn", "required"], + "properties": { + "id": {"$ref": "#/$defs/id"}, "candidateId": {"$ref": "#/$defs/id"}, + "capability": {"enum": ["he-evidence", "cell-morphology", "ihc-descriptive", "special-stain-descriptive", "cytology-descriptive", "grounded-tutor", "atlas-distillation"]}, + "scope": {"enum": ["deployment", "research-restricted", "local-benchmark"]}, + "requestPath": {"type": "string", "minLength": 1, "maxLength": 1000}, + "remediationRequestPath": {"type": ["string", "null"], "maxLength": 1000}, + "expectedAttestationPath": {"type": ["string", "null"], "maxLength": 1000}, + "protocolSha256": {"$ref": "#/$defs/sha"}, + "dependsOn": {"type": "array", "maxItems": 16, "uniqueItems": true, "items": {"$ref": "#/$defs/id"}}, + "required": {"type": "boolean"} + } + } + } +} diff --git a/contracts/evidence-mentor/pathlab-study-pack-v3.schema.json b/contracts/evidence-mentor/pathlab-study-pack-v3.schema.json new file mode 100644 index 0000000..5ba7337 --- /dev/null +++ b/contracts/evidence-mentor/pathlab-study-pack-v3.schema.json @@ -0,0 +1,13 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://pathlab.local/schemas/pathlab-study-pack-v3.schema.json", + "title": "PathLab Study Pack v3", "type": "object", "additionalProperties": false, + "required": ["schema", "packKey", "version", "title", "author", "license", "provenance", "revision", "languages", "knowledgePackChecksum", "slides", "tasks", "checksum", "facultyPreview"], + "properties": { + "schema": {"const": "pathlab.study-pack/3"}, "packKey": {"$ref": "#/$defs/id"}, "version": {"type": "integer", "minimum": 1}, + "title": {"type": "string", "minLength": 1, "maxLength": 240}, "author": {"type": "string", "minLength": 1, "maxLength": 240}, "license": {"type": "string", "minLength": 1, "maxLength": 240}, "provenance": {"type": "string", "minLength": 1, "maxLength": 1000}, "revision": {"type": "string", "minLength": 1, "maxLength": 120}, + "languages": {"const": ["en"]}, "knowledgePackChecksum": {"$ref": "#/$defs/sha"}, + "slides": {"type": "array", "minItems": 1, "maxItems": 50, "items": {"type": "object", "additionalProperties": false, "required": ["viewerSlideId", "sha256", "displayName", "evidenceSetSha256"], "properties": {"viewerSlideId": {"type": "string", "maxLength": 100}, "sha256": {"$ref": "#/$defs/sha"}, "displayName": {"type": "string", "maxLength": 200}, "evidenceSetSha256": {"$ref": "#/$defs/sha"}}}}, + "tasks": {"type": "array", "minItems": 1, "maxItems": 500, "items": {"type": "object"}}, + "checksum": {"$ref": "#/$defs/sha"}, "facultyPreview": {"type": "object"} + }, "$defs": {"id": {"type": "string", "pattern": "^[A-Za-z0-9._-]{1,120}$"}, "sha": {"type": "string", "pattern": "^[a-f0-9]{64}$"}} +} diff --git a/docs/evidence-mentor-runner.md b/docs/evidence-mentor-runner.md index de363b6..66607c3 100644 --- a/docs/evidence-mentor-runner.md +++ b/docs/evidence-mentor-runner.md @@ -38,7 +38,7 @@ First build a versioned distribution: Then run the administrator-approved installer with a Java 17 runtime: ```powershell -.\scripts\evidence-mentor-service.ps1 -Action Install -Version '2.0.12' ` +.\scripts\evidence-mentor-service.ps1 -Action Install -Version '2.1.0' ` -DistributionPath '.\build\install\pathlab-forge' -JavaHome 'C:\Path\To\jdk-17' ``` diff --git a/docs/plans/active/current.md b/docs/plans/active/current.md index e304d25..383d2b6 100644 --- a/docs/plans/active/current.md +++ b/docs/plans/active/current.md @@ -29,24 +29,41 @@ appear in monitoring payloads. ## Current status -Gates 1-4 are implemented on the draft feature branch. The installed 2.0.12 -runtime has separate passing `Inspect` and `ServiceRestart` reports proving -live `LocalService`, outbound-deny coverage, P2000 CUDA 12.6 `sm_61` inference, -resource bounds, signed evidence packaging, and restart recovery. Its latest -aggregate report is still `NOT_EVALUABLE` solely because the operator-controlled -manual-reboot challenge has not run. Gate 5 therefore remains incomplete. Gate -6 qualification preparation has begun: ordinary jobs now require a `qualified` -pack, while `experimental` packs are restricted to non-identifying -`qualification-` jobs. The deterministic cell fallback is accurately -identified as connected-component morphometry, and IHC marker-specific or PD-L1 -compartment claims fail closed to generic descriptive output. Neither track is -qualified. A checksum-bound `pathlab.model-qualification-report/1` synthetic -harness now records separated-cell counting, touching-cell separation, -determinism, generic DAB area, relative calibration, and failed-stain refusal -independently. Its current truthful outcome is `experimental`: touching nuclei -are not split, instance masks are unavailable, and marker-specific/PD-L1 -fixtures remain `not_evaluable`. Special stains and cytology remain -unimplemented, and Gate 7 has not begun. +Gates 1-4 remain implemented and service `2.1.0` is staged as a side-by-side, +idempotent upgrade. An existing byte-identical runtime is reused; a mismatch +fails before the active configuration changes. The generated elevated launcher +does not request a reboot. The installed `2.0.12` runtime remains active until +that separate administrator action. The manual reboot-continuation acceptance +is still deferred, so Gate 5 and activation remain incomplete. + +The branch now implements `pathlab.qualification-campaign/1` as a durable +parent coordinator above the existing GPU and CPU/I/O lanes. It records frozen +dependencies, attempts, signed attestations, completion versus target success, +and a signed capability matrix. One bounded remediation is enforced. Candidate +manifests cannot activate themselves: v2 execution requires an exact trusted +registry entry binding pack, artifacts, protocol, and qualification attestation. +Evidence and qualification signatures are verified against ACL-protected local +trust registries rather than payload-embedded keys. + +Gate 6 now includes executable deterministic baselines for true optical-density +watershed instance masks and morphometry, marker-aware nuclear ER/PR/Ki-67 +descriptors, HER2 membrane descriptors, PD-L1 reviewed-compartment enforcement, +generic H/DAB output, PAS/PAS-D, trichrome, GMS, AFB, and Papanicolaou +descriptors. The four deterministic v2 candidate manifests are pinned to +pre-registered protocol files. Evidence v2 and signed multi-pack evidence-set +fusion preserve coordinates, QC, provenance, and uncertainty and explicitly +forbid serial-section cell matching. These implementations are not qualified: +rights-cleared held-out reference sets and the frozen quantitative gates still +must run. Gated DINOv2/Hibou/GigaPath, HoVer-Net/PathoSAM, Qwen assets, and public +cytology data are not represented as acquired or built when their exact +artifacts or rights are absent. + +Gate 7 has an executable four-hour feasibility decision contract using +checkpointed 15-20 minute segments, fixed RAM/VRAM limits, validation +improvement, seven-day projection, 24 GiB rental minimum, and USD 100 ceiling. +No Atlas weights have been trained. The campaign must return +`LOCAL_INSUFFICIENT`/`not_evaluable` or pause for an exact user-approved quote +when lawful teachers, data, checkpoints, or hardware evidence are missing. Implementation, service installation, model qualification, merge, deployment, feature activation, and any future clinical/capacity study are separate gates. diff --git a/qualification-protocols/cell-od-watershed-v2.json b/qualification-protocols/cell-od-watershed-v2.json new file mode 100644 index 0000000..87d2e43 --- /dev/null +++ b/qualification-protocols/cell-od-watershed-v2.json @@ -0,0 +1 @@ +{"schema":"pathlab.qualification-protocol/1","id":"cell-od-watershed-v2","gates":{"macroPqMinimum":0.45,"instanceDiceMinimum":0.70,"countErrorMaximum":0.15,"morphometryBiasMaximum":0.10,"failedRegionRateMaximum":0.05,"deterministicRepeatRequired":true},"split":"patient-source-held-out","researchOnly":true} diff --git a/qualification-protocols/cytology-descriptive-v2.json b/qualification-protocols/cytology-descriptive-v2.json new file mode 100644 index 0000000..3b90bcb --- /dev/null +++ b/qualification-protocols/cytology-descriptive-v2.json @@ -0,0 +1 @@ +{"schema":"pathlab.qualification-protocol/1","id":"cytology-descriptive-v2","gates":{"cellAccuracyRequired":true,"overlapAbstentionRequired":true,"insufficientTissueAbstentionRequired":true,"clinicalIdentityForbidden":true},"tracks":["papanicolaou"],"researchOnly":true} diff --git a/qualification-protocols/ihc-descriptive-v2.json b/qualification-protocols/ihc-descriptive-v2.json new file mode 100644 index 0000000..1cf94be --- /dev/null +++ b/qualification-protocols/ihc-descriptive-v2.json @@ -0,0 +1 @@ +{"schema":"pathlab.qualification-protocol/1","id":"ihc-descriptive-v2","gates":{"proportionErrorPointsMaximum":3,"intensityOdErrorMaximum":0.05,"qcFixturesRequired":true,"pdl1ReviewedCompartmentRequired":true,"clinicalScoresForbidden":true},"tracks":["generic","er","pr","ki-67","her2","pd-l1"],"researchOnly":true} diff --git a/qualification-protocols/special-stain-descriptive-v2.json b/qualification-protocols/special-stain-descriptive-v2.json new file mode 100644 index 0000000..a33e754 --- /dev/null +++ b/qualification-protocols/special-stain-descriptive-v2.json @@ -0,0 +1 @@ +{"schema":"pathlab.qualification-protocol/1","id":"special-stain-descriptive-v2","gates":{"proportionErrorPointsMaximum":3,"intensityOdErrorMaximum":0.05,"qcFixturesRequired":true,"genericFallbackRequired":true},"tracks":["pas","pas_d","trichrome","gms","afb"],"researchOnly":true} diff --git a/scripts/evidence-mentor-service.ps1 b/scripts/evidence-mentor-service.ps1 index 6dc3fa6..5279f30 100644 --- a/scripts/evidence-mentor-service.ps1 +++ b/scripts/evidence-mentor-service.ps1 @@ -1,7 +1,7 @@ [CmdletBinding(SupportsShouldProcess)] param( [Parameter(Mandatory)] [ValidateSet('Install','Upgrade','Uninstall','Status')] [string] $Action, - [string] $Version = '2.0.12', + [string] $Version = '2.1.0', [string] $DistributionPath, [string] $JavaHome, [string] $ProgramRoot = 'C:\ProgramData\PathLab\EvidenceMentor', @@ -188,7 +188,8 @@ function Stage-Runtime { if (-not $JavaHome -or -not (Test-Path -LiteralPath (Join-Path $JavaHome 'bin\java.exe') -PathType Leaf)) { throw 'JavaHome must reference a Java 17 runtime.' } $target = Join-Path $runtimeRoot $Version if (Test-Path -LiteralPath $target) { - if (-not $ReuseStagedRuntime) { throw "Runtime version already exists: $target" } + # Idempotent upgrades reuse a byte-identical staged runtime. Any mismatch fails + # before the installed directory or active service configuration is touched. Assert-DirectoryMirror $DistributionPath (Join-Path $target 'app') 'Application runtime' Assert-DirectoryMirror $JavaHome (Join-Path $target 'jre') 'Java runtime' return $target diff --git a/scripts/prepare-all-rounder-campaign.ps1 b/scripts/prepare-all-rounder-campaign.ps1 new file mode 100644 index 0000000..1902022 --- /dev/null +++ b/scripts/prepare-all-rounder-campaign.ps1 @@ -0,0 +1,120 @@ +[CmdletBinding()] +param( + [Parameter(Mandatory)] [string] $PreparationManifest, + [string] $StateRoot = 'D:\PathLabData\EvidenceMentor\state' +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version Latest +$allowedHosts = @( + 'huggingface.co', 'cdn-lfs.huggingface.co', 'api.gdc.cancer.gov', + 'portal.gdc.cancer.gov', 'zenodo.org', 'www.ebi.ac.uk' +) +$limits = @{ source = 45GB; derived = 25GB; models = 10GB; evidence = 10GB } +$bucketDirectories = @{ source = 'sources'; derived = 'derived'; models = 'models'; evidence = 'artifacts' } + +function Require-ExactFields($Value, [string[]] $Names, [string] $Label) { + $actual = @($Value.PSObject.Properties.Name | Sort-Object) + $expected = @($Names | Sort-Object) + if (($actual -join "`n") -ne ($expected -join "`n")) { throw "$Label fields are invalid." } +} + +function File-Sha256([string] $Path) { + return (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant() +} + +$preparationPath = [IO.Path]::GetFullPath($PreparationManifest) +if (-not (Test-Path -LiteralPath $preparationPath -PathType Leaf)) { + throw 'Preparation manifest is unavailable.' +} +$spec = Get-Content -LiteralPath $preparationPath -Raw | ConvertFrom-Json +Require-ExactFields $spec @('schema','campaignManifest','sampleLedger','artifacts') 'Preparation manifest' +if ($spec.schema -ne 'pathlab.campaign-preparation/1') { throw 'Preparation schema is unsupported.' } +$base = Split-Path -Parent $preparationPath +$campaignPath = [IO.Path]::GetFullPath((Join-Path $base $spec.campaignManifest)) +$ledgerPath = [IO.Path]::GetFullPath((Join-Path $base $spec.sampleLedger)) +if (-not $campaignPath.StartsWith($base, [StringComparison]::OrdinalIgnoreCase) -or + -not $ledgerPath.StartsWith($base, [StringComparison]::OrdinalIgnoreCase)) { + throw 'Preparation inputs must stay inside the preparation directory.' +} +if (-not (Test-Path -LiteralPath $campaignPath -PathType Leaf) -or + -not (Test-Path -LiteralPath $ledgerPath -PathType Leaf)) { + throw 'Frozen campaign manifest or sample ledger is unavailable.' +} + +$usage = @{ source = 0L; derived = 0L; models = 0L; evidence = 0L } +foreach ($artifact in @($spec.artifacts)) { + Require-ExactFields $artifact @('id','url','sha256','destination','bucket','license','permittedUse','gated') 'Acquisition artifact' + if ($artifact.id -notmatch '^[A-Za-z0-9._-]{1,120}$' -or $artifact.sha256 -notmatch '^[a-f0-9]{64}$') { + throw 'Acquisition artifact identity is invalid.' + } + if (-not $limits.ContainsKey([string]$artifact.bucket) -or + [string]::IsNullOrWhiteSpace($artifact.license) -or + $artifact.permittedUse -notin @('private-research','research-restricted','benchmark-only')) { + throw 'Acquisition artifact rights or quota bucket is invalid.' + } + $uri = [Uri]$artifact.url + if ($uri.Scheme -ne 'https' -or $uri.Host -notin $allowedHosts) { + throw "Acquisition source is not allowlisted: $($uri.Host)" + } + $bucketRoot = Join-Path $StateRoot $bucketDirectories[[string]$artifact.bucket] + $destination = [IO.Path]::GetFullPath((Join-Path $bucketRoot ([string]$artifact.destination))) + if (-not $destination.StartsWith([IO.Path]::GetFullPath($bucketRoot), [StringComparison]::OrdinalIgnoreCase)) { + throw 'Acquisition destination escapes its quota bucket.' + } + New-Item -ItemType Directory -Path (Split-Path -Parent $destination) -Force | Out-Null + if (-not (Test-Path -LiteralPath $destination -PathType Leaf)) { + $headers = @{} + if ([bool]$artifact.gated) { + if ([string]::IsNullOrWhiteSpace($env:HF_TOKEN)) { throw "Gated terms/token required for $($artifact.id)." } + $headers.Authorization = "Bearer $env:HF_TOKEN" + } + $partial = "$destination.partial" + try { Invoke-WebRequest -UseBasicParsing -Uri $uri -Headers $headers -OutFile $partial } + catch { Remove-Item -LiteralPath $partial -Force -ErrorAction SilentlyContinue; throw } + if ((File-Sha256 $partial) -ne $artifact.sha256) { + Remove-Item -LiteralPath $partial -Force + throw "Checksum mismatch for $($artifact.id)." + } + Move-Item -LiteralPath $partial -Destination $destination + } elseif ((File-Sha256 $destination) -ne $artifact.sha256) { + throw "Existing artifact checksum mismatch for $($artifact.id)." + } + $usage[[string]$artifact.bucket] += (Get-Item -LiteralPath $destination).Length +} +foreach ($bucket in $limits.Keys) { + if ($usage[$bucket] -gt $limits[$bucket]) { throw "$bucket acquisition quota exceeded." } +} + +$sampleCount = 0 +foreach ($line in Get-Content -LiteralPath $ledgerPath) { + if ([string]::IsNullOrWhiteSpace($line)) { continue } + $sample = $line | ConvertFrom-Json + Require-ExactFields $sample @('sampleId','source','patientGroup','slideGroup','sha256','license','permittedUse','task','split') 'Sample ledger record' + if ($sample.sha256 -notmatch '^[a-f0-9]{64}$' -or [string]::IsNullOrWhiteSpace($sample.patientGroup) -or + [string]::IsNullOrWhiteSpace($sample.slideGroup) -or [string]::IsNullOrWhiteSpace($sample.license)) { + throw 'Sample provenance is incomplete; candidate is NOT_EVALUABLE.' + } + $sampleCount++ +} +if ($sampleCount -lt 1) { throw 'Sample ledger is empty; campaign is NOT_EVALUABLE.' } + +$campaign = Get-Content -LiteralPath $campaignPath -Raw | ConvertFrom-Json +if ($campaign.schema -ne 'pathlab.qualification-campaign/1' -or $campaign.maxRemediationAttempts -ne 1) { + throw 'Frozen campaign manifest is invalid.' +} +$freeze = [ordered]@{ + schema = 'pathlab.campaign-freeze/1' + campaignManifest = $campaignPath + campaignManifestSha256 = File-Sha256 $campaignPath + sampleLedgerSha256 = File-Sha256 $ledgerPath + sampleCount = $sampleCount + acquiredBytes = $usage + frozenAt = [DateTimeOffset]::UtcNow.ToString('o') + credentialsPersisted = $false +} +$freezePath = Join-Path $base 'campaign-freeze.json' +$freeze | ConvertTo-Json -Depth 8 | Set-Content -LiteralPath "$freezePath.partial" -Encoding utf8NoBOM +Move-Item -LiteralPath "$freezePath.partial" -Destination $freezePath -Force +Write-Host "Campaign frozen: $freezePath" +Write-Host 'Credentials were used only by this interactive acquisition process and were not persisted.' diff --git a/scripts/prepare-evidence-mentor-2.1.0-upgrade.ps1 b/scripts/prepare-evidence-mentor-2.1.0-upgrade.ps1 new file mode 100644 index 0000000..4b49b86 --- /dev/null +++ b/scripts/prepare-evidence-mentor-2.1.0-upgrade.ps1 @@ -0,0 +1,36 @@ +[CmdletBinding()] +param( + [string] $JavaHome = '', + [string] $OutputPath = '' +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version Latest +$repository = Split-Path -Parent $PSScriptRoot +if (-not $JavaHome) { + $candidate = Get-ChildItem -LiteralPath (Join-Path $env:USERPROFILE '.gradle\jdks') -Filter 'java.exe' -File -Recurse ` + -ErrorAction SilentlyContinue | Where-Object { $_.FullName -match '17' } | Select-Object -First 1 + if ($null -eq $candidate) { throw 'Provide -JavaHome for a Java 17 runtime.' } + $JavaHome = Split-Path -Parent (Split-Path -Parent $candidate.FullName) +} +if (-not (Test-Path -LiteralPath (Join-Path $JavaHome 'bin\java.exe') -PathType Leaf)) { + throw 'JavaHome must contain bin\java.exe.' +} + +& (Join-Path $repository 'gradlew.bat') clean test installDist +if ($LASTEXITCODE -ne 0) { throw 'The 2.1.0 distribution did not pass its build and tests.' } + +if (-not $OutputPath) { $OutputPath = Join-Path $repository 'build\Upgrade-PathLab-EvidenceMentor-2.1.0.ps1' } +$distribution = Join-Path $repository 'build\install\pathlab-forge' +$installer = Join-Path $repository 'scripts\evidence-mentor-service.ps1' +$content = @" +`$ErrorActionPreference = 'Stop' +& '$($installer.Replace("'", "''"))' -Action Upgrade -Version '2.1.0' `` + -DistributionPath '$($distribution.Replace("'", "''"))' `` + -JavaHome '$($JavaHome.Replace("'", "''"))' +"@ +$parent = Split-Path -Parent $OutputPath +New-Item -ItemType Directory -Path $parent -Force | Out-Null +[IO.File]::WriteAllText($OutputPath, $content, [Text.UTF8Encoding]::new($false)) +Write-Host "Elevated upgrade launcher created: $OutputPath" +Write-Host 'Run it once from an Administrator PowerShell. No reboot is requested.' diff --git a/src/main/java/org/pathlab/forge/evidence/AtlasFeasibilityGate.java b/src/main/java/org/pathlab/forge/evidence/AtlasFeasibilityGate.java index 018b10b..5ec22c0 100644 --- a/src/main/java/org/pathlab/forge/evidence/AtlasFeasibilityGate.java +++ b/src/main/java/org/pathlab/forge/evidence/AtlasFeasibilityGate.java @@ -1,24 +1,53 @@ package org.pathlab.forge.evidence; -/** Deterministic decision gate for the bounded local Atlas-H&E feasibility probe. */ +import java.time.Duration; +import java.util.List; + +/** Frozen four-hour gate for resumable 15-20 minute Atlas probe segments. */ public final class AtlasFeasibilityGate { - private static final double MAX_LOCAL_HOURS = 7 * 24; - private AtlasFeasibilityGate() {} + public static final Duration REQUIRED_PROBE = Duration.ofHours(4); + public static final Duration MIN_SEGMENT = Duration.ofMinutes(15); + public static final Duration MAX_SEGMENT = Duration.ofMinutes(20); + public static final Duration MAX_PROJECTED_COMPLETION = Duration.ofDays(7); + public static final long MAX_RAM_BYTES = 16L * 1024 * 1024 * 1024; + public static final long MAX_VRAM_BYTES = 4608L * 1024 * 1024; + + private AtlasFeasibilityGate() { } + + public static Decision evaluate(List segments, Duration projectedCompletion) { + if (segments == null || segments.isEmpty() || projectedCompletion == null) { + return new Decision("not_evaluable", "ATLAS_PROBE_MISSING", false); + } + if (segments.stream().anyMatch(segment -> segment.duration().compareTo(MIN_SEGMENT) < 0 + || segment.duration().compareTo(MAX_SEGMENT) > 0 || !segment.checkpointVerified())) { + return new Decision("not_evaluable", "ATLAS_SEGMENT_OR_CHECKPOINT_INVALID", false); + } + if (segments.stream().anyMatch(segment -> segment.peakRamBytes() > MAX_RAM_BYTES + || segment.peakVramBytes() > MAX_VRAM_BYTES)) { + return new Decision("experimental", "LOCAL_INSUFFICIENT_RESOURCE_LIMIT", true); + } + var elapsed = segments.stream().map(Segment::duration).reduce(Duration.ZERO, Duration::plus); + if (elapsed.compareTo(REQUIRED_PROBE) < 0) { + return new Decision("not_evaluable", "ATLAS_FOUR_HOUR_PROBE_INCOMPLETE", false); + } + var first = segments.get(0).validationMetric(); + var last = segments.get(segments.size() - 1).validationMetric(); + if (!(Double.isFinite(first) && Double.isFinite(last) && last > first)) { + return new Decision("experimental", "LOCAL_INSUFFICIENT_NO_VALIDATION_IMPROVEMENT", true); + } + if (projectedCompletion.compareTo(MAX_PROJECTED_COMPLETION) > 0) { + return new Decision("experimental", "LOCAL_INSUFFICIENT_PROJECTED_OVER_SEVEN_DAYS", true); + } + return new Decision("experimental", "LOCAL_PROBE_PASSED_FULL_QUALIFICATION_PENDING", false); + } - public static Decision decide(Probe probe) { - if (probe.elapsedHours < 4 || probe.completedSteps <= probe.warmupSteps - || probe.totalSteps <= probe.completedSteps) return new Decision("PROBE_INCOMPLETE", null); - if (probe.peakVramMiB > 4608 || probe.peakRamMiB > 16_384 || probe.oomCount > 0 - || !probe.validationImproved) return new Decision("LOCAL_INSUFFICIENT", null); - var effectiveSteps = probe.completedSteps - probe.warmupSteps; - var effectiveHours = probe.elapsedHours * effectiveSteps / probe.completedSteps; - var stepsPerHour = effectiveSteps / effectiveHours; - var projected = (probe.totalSteps - probe.completedSteps) / stepsPerHour; - return new Decision(projected <= MAX_LOCAL_HOURS ? "CONTINUE_LOCAL" : "LOCAL_INSUFFICIENT", - projected); + public static void requireQuote(long allInUsdCents, int gpuMemoryGiB) { + if (allInUsdCents < 1 || allInUsdCents > 10_000 || gpuMemoryGiB < 24) { + throw new IllegalArgumentException("Atlas rental quote exceeds the frozen approval envelope"); + } } - public record Probe(double elapsedHours, long completedSteps, long warmupSteps, long totalSteps, - int peakVramMiB, int peakRamMiB, int oomCount, boolean validationImproved) {} - public record Decision(String status, Double projectedRemainingHours) {} + public record Segment(Duration duration, long peakRamBytes, long peakVramBytes, + double validationMetric, boolean checkpointVerified) { } + public record Decision(String status, String reason, boolean paidComputeQuoteAllowed) { } } diff --git a/src/main/java/org/pathlab/forge/evidence/BrightfieldQualificationHarness.java b/src/main/java/org/pathlab/forge/evidence/BrightfieldQualificationHarness.java index 4417126..d42ffa5 100644 --- a/src/main/java/org/pathlab/forge/evidence/BrightfieldQualificationHarness.java +++ b/src/main/java/org/pathlab/forge/evidence/BrightfieldQualificationHarness.java @@ -106,11 +106,12 @@ private static ObjectNode cellTrack(EvidencePackManifest pack) { checks.add(textCheck("deterministic-repeat", separated.equals(repeated) ? "pass" : "fail", "Exact repeated analysis result")); - checks.add(textCheck("instance-mask-output", "not_evaluable", - "Current v1 emits aggregates but no reviewed-region instance masks")); + checks.add(textCheck("instance-mask-output", + touching.instances().size() == 2 + && touching.instances().stream().allMatch(instance -> !instance.rle().isEmpty()) + ? "pass" : "fail", + "Deterministic reviewed-region RLE instance masks")); var reasons = track.putArray("reasons"); - reasons.add("TOUCHING_NUCLEI_NOT_SPLIT"); - reasons.add("INSTANCE_MASK_OUTPUT_UNAVAILABLE"); reasons.add("CROSS_TISSUE_HELD_OUT_FIXTURES_PENDING"); return track; } diff --git a/src/main/java/org/pathlab/forge/evidence/BrightfieldTileAnalyzer.java b/src/main/java/org/pathlab/forge/evidence/BrightfieldTileAnalyzer.java index eac76f6..472826a 100644 --- a/src/main/java/org/pathlab/forge/evidence/BrightfieldTileAnalyzer.java +++ b/src/main/java/org/pathlab/forge/evidence/BrightfieldTileAnalyzer.java @@ -1,7 +1,6 @@ package org.pathlab.forge.evidence; import java.awt.image.BufferedImage; -import java.util.ArrayDeque; import java.util.Set; /** Bounded deterministic hematoxylin-component and DAB descriptor baseline. */ @@ -19,7 +18,6 @@ public static Result analyze(BufferedImage image, String requestedMarker) { if (!KNOWN.contains(marker)) marker = "generic"; var width = image.getWidth(); var height = image.getHeight(); - var nuclei = new boolean[width * height]; var dabPixels = 0; double dabOd = 0; for (var y = 0; y < height; y++) { @@ -28,8 +26,6 @@ public static Result analyze(BufferedImage image, String requestedMarker) { var red = rgb >>> 16 & 0xff; var green = rgb >>> 8 & 0xff; var blue = rgb & 0xff; - nuclei[y * width + x] = blue - red >= 25 && blue - green >= 25 - && red + green + blue < 660; if (red - green >= 15 && green - blue >= 15 && red + green + blue < 660) { dabPixels++; var luminance = 0.2126 * red + 0.7152 * green + 0.0722 * blue; @@ -37,74 +33,25 @@ public static Result analyze(BufferedImage image, String requestedMarker) { } } } - var components = components(nuclei, width, height); + var segmentation = OpticalDensityWatershed.segment(image); + var instances = segmentation.instances(); return new Result( marker, compartment(marker), - components.count, - components.count == 0 ? null : (double) components.pixels / components.count, - components.count == 0 ? null : components.perimeter / components.count, - components.count == 0 ? null : components.eccentricity / components.count, - components.count == 0 ? null : components.solidity / components.count, + instances.size(), + average(instances, OpticalDensityWatershed.Instance::areaPx2), + average(instances, OpticalDensityWatershed.Instance::perimeterPx), + average(instances, OpticalDensityWatershed.Instance::eccentricity), + average(instances, OpticalDensityWatershed.Instance::solidity), (double) dabPixels / (width * height), dabPixels == 0 ? 0 : dabOd / dabPixels, - true); + true, + instances); } - private static Components components(boolean[] mask, int width, int height) { - var seen = new boolean[mask.length]; - var queue = new ArrayDeque(); - var count = 0; - var pixels = 0; - double perimeter = 0; - double eccentricity = 0; - double solidity = 0; - for (var start = 0; start < mask.length; start++) { - if (!mask[start] || seen[start]) continue; - seen[start] = true; - queue.add(start); - var size = 0; - var minX = width; - var minY = height; - var maxX = -1; - var maxY = -1; - var componentPerimeter = 0; - while (!queue.isEmpty()) { - var current = queue.removeFirst(); - size++; - var x = current % width; - var y = current / width; - minX = Math.min(minX, x); - minY = Math.min(minY, y); - maxX = Math.max(maxX, x); - maxY = Math.max(maxY, y); - if (x == 0 || !mask[current - 1]) componentPerimeter++; - if (x == width - 1 || !mask[current + 1]) componentPerimeter++; - if (y == 0 || !mask[current - width]) componentPerimeter++; - if (y == height - 1 || !mask[current + width]) componentPerimeter++; - for (var ny = Math.max(0, y - 1); ny <= Math.min(height - 1, y + 1); ny++) { - for (var nx = Math.max(0, x - 1); nx <= Math.min(width - 1, x + 1); nx++) { - var next = ny * width + nx; - if (mask[next] && !seen[next]) { - seen[next] = true; - queue.addLast(next); - } - } - } - } - if (size >= 4) { - count++; - pixels += size; - perimeter += componentPerimeter; - var boxWidth = maxX - minX + 1.0; - var boxHeight = maxY - minY + 1.0; - var major = Math.max(boxWidth, boxHeight); - var minor = Math.min(boxWidth, boxHeight); - eccentricity += major == 0 ? 0 : Math.sqrt(Math.max(0, 1 - (minor * minor) / (major * major))); - solidity += size / (boxWidth * boxHeight); - } - } - return new Components(count, pixels, perimeter, eccentricity, solidity); + private static Double average(java.util.List instances, + java.util.function.ToDoubleFunction getter) { + return instances.isEmpty() ? null : instances.stream().mapToDouble(getter).average().orElseThrow(); } private static String compartment(String marker) { @@ -116,8 +63,6 @@ private static String compartment(String marker) { }; } - private record Components(int count, int pixels, double perimeter, double eccentricity, double solidity) {} - public record Result( String marker, String compartment, @@ -128,5 +73,6 @@ public record Result( Double meanNucleusSolidity, double dabAreaFraction, double meanDabOd, - boolean researchEstimate) {} + boolean researchEstimate, + java.util.List instances) {} } diff --git a/src/main/java/org/pathlab/forge/evidence/CapabilityMatrixWriter.java b/src/main/java/org/pathlab/forge/evidence/CapabilityMatrixWriter.java new file mode 100644 index 0000000..189076e --- /dev/null +++ b/src/main/java/org/pathlab/forge/evidence/CapabilityMatrixWriter.java @@ -0,0 +1,57 @@ +package org.pathlab.forge.evidence; + +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.nio.file.AtomicMoveNotSupportedException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.time.Instant; +import java.util.Map; + +/** Writes the signed terminal distinction between campaign completion and target success. */ +public final class CapabilityMatrixWriter { + public static final String SCHEMA = "pathlab.capability-matrix/1"; + private static final ObjectMapper JSON = new ObjectMapper(); + private final Path stateRoot; + + public CapabilityMatrixWriter(Path stateRoot) { + this.stateRoot = stateRoot.toAbsolutePath().normalize(); + } + + public EvidenceBundleWriter.Result write(QualificationRunSnapshot run, Instant generatedAt) + throws IOException { + if (!run.campaignCompleted()) throw new IllegalArgumentException("Capability matrix requires a terminal campaign"); + var root = JSON.createObjectNode(); + root.put("schema", SCHEMA); root.put("campaignId", run.id()); + root.put("campaignManifestSha256", run.manifestSha256()); + root.put("campaignCompleted", true); root.put("campaignTargetMet", run.campaignTargetMet()); + root.put("generatedAt", generatedAt.toString()); + var tracks = root.putArray("tracks"); + for (var track : run.tracks()) { + var item = tracks.addObject(); item.put("trackId", track.id()); + item.put("candidateId", track.candidateId()); item.put("capability", track.capability()); + item.put("scope", track.scope()); item.put("status", track.verdict()); + item.put("activationEligible", "qualified".equals(track.verdict()) + && !"local-benchmark".equals(track.scope())); + if (track.artifactSha256().isBlank()) item.putNull("qualificationReportSha256"); + else item.put("qualificationReportSha256", track.artifactSha256()); + if (track.failureCode().isBlank()) item.putNull("failureCode"); + else item.put("failureCode", track.failureCode()); + } + var manifestSha = EvidenceBundleWriter.sha256(EvidenceBundleWriter.canonicalBytes(root)); + var signer = new EvidenceSigner(stateRoot.resolve("signing")); + var signed = signer.sign(SCHEMA + "\n" + manifestSha); + new TrustedSignerRegistry(stateRoot).pin(signed.keyId(), signed.publicKeyDer(), "qualification"); + var complete = root.deepCopy(); complete.put("manifestSha256", manifestSha); + complete.set("signature", JSON.valueToTree(Map.of( + "algorithm", "Ed25519", "keyId", signed.keyId(), "value", signed.signature()))); + var output = stateRoot.resolve("qualification").resolve(run.id()).resolve("capability-matrix.json"); + Files.createDirectories(output.getParent()); + var partial = output.resolveSibling(output.getFileName() + ".partial"); + JSON.writerWithDefaultPrettyPrinter().writeValue(partial.toFile(), complete); + try { Files.move(partial, output, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE); } + catch (AtomicMoveNotSupportedException ignored) { Files.move(partial, output, StandardCopyOption.REPLACE_EXISTING); } + return new EvidenceBundleWriter.Result(output, manifestSha, signed.keyId()); + } +} diff --git a/src/main/java/org/pathlab/forge/evidence/EvidenceArtifactStore.java b/src/main/java/org/pathlab/forge/evidence/EvidenceArtifactStore.java index 13f8568..95a4837 100644 --- a/src/main/java/org/pathlab/forge/evidence/EvidenceArtifactStore.java +++ b/src/main/java/org/pathlab/forge/evidence/EvidenceArtifactStore.java @@ -4,6 +4,10 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.security.KeyFactory; +import java.security.Signature; +import java.security.spec.X509EncodedKeySpec; +import java.util.Base64; import java.util.Comparator; import java.util.Optional; import java.util.Set; @@ -15,6 +19,11 @@ public final class EvidenceArtifactStore { private EvidenceArtifactStore() {} + static boolean isTrusted(Path stateRoot, Path path, String slideSha256, String revision) { + return matches(stateRoot.toAbsolutePath().normalize(), path.toAbsolutePath().normalize(), + slideSha256, revision); + } + public static Optional find(Path stateRoot, String slideSha256, String revision) throws IOException { var artifacts = stateRoot.toAbsolutePath().normalize().resolve("artifacts"); @@ -23,21 +32,38 @@ public static Optional find(Path stateRoot, String slideSha256, String rev return paths.filter(path -> path.getFileName().toString().equals("evidence.json")) .filter(Files::isRegularFile) .limit(2_000) - .filter(path -> matches(path, slideSha256, revision)) + .filter(path -> matches(stateRoot, path, slideSha256, revision)) .max(Comparator.comparingLong(EvidenceArtifactStore::modified)); } } - private static boolean matches(Path path, String sha, String revision) { + private static boolean matches(Path stateRoot, Path path, String sha, String revision) { try { var root = JSON.readTree(path.toFile()); - return "pathlab.ai-evidence/1".equals(root.path("schema").asText()) - && sha.equals(root.path("source").path("slideSha256").asText()) - && revision.equals(root.path("source").path("revision").asText()) - && TERMINAL.contains(root.path("status").asText()) - && root.path("manifestSha256").asText().matches("[a-f0-9]{64}") - && "Ed25519".equals(root.path("signature").path("algorithm").asText()); - } catch (IOException ignored) { + if (!Set.of("pathlab.ai-evidence/1", "pathlab.ai-evidence/2") + .contains(root.path("schema").asText())) return false; + if (!sha.equals(root.path("source").path("slideSha256").asText()) + || !revision.equals(root.path("source").path("revision").asText()) + || !TERMINAL.contains(root.path("status").asText()) + || root.path("qualificationOnly").asBoolean(false) + || !root.path("manifestSha256").asText().matches("[a-f0-9]{64}") + || !"Ed25519".equals(root.path("signature").path("algorithm").asText())) return false; + var unsigned = root.deepCopy(); + ((com.fasterxml.jackson.databind.node.ObjectNode) unsigned).remove("manifestSha256"); + ((com.fasterxml.jackson.databind.node.ObjectNode) unsigned).remove("signature"); + var expected = EvidenceBundleWriter.sha256(EvidenceBundleWriter.canonicalBytes(unsigned)); + if (!expected.equals(root.path("manifestSha256").asText())) return false; + var keyId = root.path("signature").path("keyId").asText(); + var key = new TrustedSignerRegistry(stateRoot).find(keyId, "evidence"); + if (key.isEmpty()) return false; + var verifier = Signature.getInstance("Ed25519"); + verifier.initVerify(KeyFactory.getInstance("Ed25519") + .generatePublic(new X509EncodedKeySpec(key.get()))); + verifier.update((root.path("schema").asText() + "\n" + expected) + .getBytes(java.nio.charset.StandardCharsets.UTF_8)); + return verifier.verify(Base64.getUrlDecoder().decode( + root.path("signature").path("value").asText())); + } catch (Exception ignored) { return false; } } diff --git a/src/main/java/org/pathlab/forge/evidence/EvidenceBundleWriter.java b/src/main/java/org/pathlab/forge/evidence/EvidenceBundleWriter.java index c70e1bb..fbc0db0 100644 --- a/src/main/java/org/pathlab/forge/evidence/EvidenceBundleWriter.java +++ b/src/main/java/org/pathlab/forge/evidence/EvidenceBundleWriter.java @@ -18,9 +18,12 @@ public final class EvidenceBundleWriter { private static final ObjectMapper JSON = new ObjectMapper() .enable(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS); private final EvidenceSigner signer; + private final TrustedSignerRegistry registry; public EvidenceBundleWriter(Path signingRoot) { signer = new EvidenceSigner(signingRoot); + var normalized = signingRoot.toAbsolutePath().normalize(); + registry = new TrustedSignerRegistry(normalized.getParent()); } public Result write(Path output, ObjectNode unsigned) throws IOException { @@ -35,6 +38,7 @@ public Result write(Path output, ObjectNode unsigned) throws IOException { var canonical = canonicalBytes(unsigned); var manifestSha = sha256(canonical); var signed = signer.sign(SCHEMA + "\n" + manifestSha); + registry.pin(signed.keyId(), signed.publicKeyDer(), "evidence"); var complete = unsigned.deepCopy(); complete.put("manifestSha256", manifestSha); complete.set("signature", JSON.valueToTree(java.util.Map.of( diff --git a/src/main/java/org/pathlab/forge/evidence/EvidenceBundleWriterV2.java b/src/main/java/org/pathlab/forge/evidence/EvidenceBundleWriterV2.java new file mode 100644 index 0000000..ee52a2c --- /dev/null +++ b/src/main/java/org/pathlab/forge/evidence/EvidenceBundleWriterV2.java @@ -0,0 +1,52 @@ +package org.pathlab.forge.evidence; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import java.io.IOException; +import java.nio.file.AtomicMoveNotSupportedException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.Map; + +/** Canonical signed evidence v2 writer; signer trust remains external to the payload. */ +public final class EvidenceBundleWriterV2 { + public static final String SCHEMA = "pathlab.ai-evidence/2"; + private static final ObjectMapper JSON = new ObjectMapper(); + private final EvidenceSigner signer; + private final TrustedSignerRegistry registry; + + public EvidenceBundleWriterV2(Path stateRoot) { + var normalized = stateRoot.toAbsolutePath().normalize(); + signer = new EvidenceSigner(normalized.resolve("signing")); + registry = new TrustedSignerRegistry(normalized); + } + + public EvidenceBundleWriter.Result write(Path output, ObjectNode unsigned) throws IOException { + require(SCHEMA.equals(unsigned.path("schema").asText()), "Evidence v2 schema is unsupported"); + require(unsigned.path("researchOnly").asBoolean(false) + && unsigned.path("notDiagnostic").asBoolean(false) + && unsigned.path("reviewRequired").asBoolean(false), + "Evidence v2 research boundary is invalid"); + require(unsigned.path("qualificationAttestationSha256").asText().matches("[a-f0-9]{64}"), + "Evidence v2 qualification attestation is required"); + var manifestSha = EvidenceBundleWriter.sha256(EvidenceBundleWriter.canonicalBytes(unsigned)); + var signed = signer.sign(SCHEMA + "\n" + manifestSha); + registry.pin(signed.keyId(), signed.publicKeyDer(), "evidence"); + var complete = unsigned.deepCopy(); + complete.put("manifestSha256", manifestSha); + complete.set("signature", JSON.valueToTree(Map.of( + "algorithm", "Ed25519", "keyId", signed.keyId(), "value", signed.signature()))); + var target = output.toAbsolutePath().normalize(); + Files.createDirectories(target.getParent()); + var partial = target.resolveSibling(target.getFileName() + ".partial"); + JSON.writerWithDefaultPrettyPrinter().writeValue(partial.toFile(), complete); + try { Files.move(partial, target, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE); } + catch (AtomicMoveNotSupportedException ignored) { Files.move(partial, target, StandardCopyOption.REPLACE_EXISTING); } + return new EvidenceBundleWriter.Result(target, manifestSha, signed.keyId()); + } + + private static void require(boolean condition, String message) { + if (!condition) throw new IllegalArgumentException(message); + } +} diff --git a/src/main/java/org/pathlab/forge/evidence/EvidenceJobProcessor.java b/src/main/java/org/pathlab/forge/evidence/EvidenceJobProcessor.java index c7f1eff..16174bb 100644 --- a/src/main/java/org/pathlab/forge/evidence/EvidenceJobProcessor.java +++ b/src/main/java/org/pathlab/forge/evidence/EvidenceJobProcessor.java @@ -25,7 +25,8 @@ public final class EvidenceJobProcessor { "sourceWidth", "sourceHeight", "packManifest", "stain", "marker"); private static final Set OPTIONAL_REQUEST_FIELDS = Set.of( "compartmentSource", "controlsValidated", "markerIdentitySource", - "tileCacheManifest", "tileCacheManifestSha256"); + "tileCacheManifest", "tileCacheManifestSha256", "qualificationAttestationSha256", + "qualificationCampaignManifest", "reviewedRegions"); private final EvidenceJobQueue queue; private final Path stateRoot; @@ -45,6 +46,10 @@ public static ExecutionPlan executionPlan(Path requestPath) throws IOException { } public static ExecutionPlan executionPlan(Path requestPath, String jobId) throws IOException { + return executionPlan(requestPath, jobId, null); + } + + public static ExecutionPlan executionPlan(Path requestPath, String jobId, Path stateRoot) throws IOException { final JsonNode request; try { request = JSON.readTree(requestPath.toAbsolutePath().normalize().toFile()); @@ -54,7 +59,9 @@ public static ExecutionPlan executionPlan(Path requestPath, String jobId) throws // Legacy v1 submissions were accepted before lane metadata existed and migrate to CPU/I/O. if (request == null || !request.path("packManifest").isTextual()) return new ExecutionPlan(EvidenceExecutionLane.CPU_IO, ""); var pack = EvidencePackManifest.load(regularPath(request, "packManifest")); - pack.requireExecutableForJob(jobId); + if (stateRoot == null) pack.requireExecutableForJob(jobId); + else pack.requireExecutableForJob(jobId, new QualifiedPackRegistry(stateRoot), + request.path("qualificationAttestationSha256").asText("")); return new ExecutionPlan("cuda".equals(pack.runtimeCompatibility().executionProvider()) ? EvidenceExecutionLane.GPU : EvidenceExecutionLane.CPU_IO, pack.sha256()); } @@ -75,13 +82,17 @@ public EvidenceJob process(EvidenceJob job, String workerId, Instant now) throws "Evidence job request is invalid"); require(Set.of("pathlab.evidence-job/1", "pathlab.evidence-job/2").contains(text(request, "schema")), "Evidence job schema is unsupported"); + require(!request.has("qualificationCampaignManifest") + || job.id().matches("qualification-[a-f0-9]{8,64}"), + "Only qualification jobs may bind a campaign manifest"); var source = regularPath(request, "sourcePath"); var preview = regularPath(request, "previewPath"); var expectedSha = text(request, "sourceSha256"); require(expectedSha.matches("[a-f0-9]{64}") && expectedSha.equals(sha256(source)), "Evidence source checksum does not match"); var pack = EvidencePackManifest.load(regularPath(request, "packManifest")); - pack.requireExecutableForJob(job.id()); + pack.requireExecutableForJob(job.id(), new QualifiedPackRegistry(stateRoot), + request.path("qualificationAttestationSha256").asText("")); var durable = queue.snapshot(job.id()).orElseThrow(); require(durable.requestSha256().isBlank() || durable.requestSha256().equals(sha256(job.requestPath())), "Evidence request changed after submission"); @@ -161,13 +172,22 @@ public EvidenceJob process(EvidenceJob job, String workerId, Instant now) throws ? request.path("markerIdentitySource").textValue() : "unknown"; require(Set.of("unknown", "faculty-authored", "slide-label", "import-metadata") .contains(markerIdentitySource), "Evidence marker identity source is invalid"); - // This pack has no compartment geometry and no qualified marker-specific algorithms yet. - // A provenance label alone must never be promoted into a tumor/immune measurement claim. - var descriptorWarning = pack.capability() != EvidencePackManifest.Capability.IHC_DESCRIPTIVE - || "generic".equals(marker) ? null - : "pd-l1".equals(marker) - ? "COMPARTMENT_REVIEW_REQUIRED" - : "MARKER_SPECIFIC_QUALIFICATION_REQUIRED"; + var reviewedRegions = reviewedRegions(request.path("reviewedRegions"), image, + sourceWidth, sourceHeight); + if (reviewedRegions.isEmpty()) compartmentSource = "none"; + else if (reviewedRegions.stream().allMatch(region -> "faculty-authored".equals(region.reviewSource()))) { + compartmentSource = "faculty-authored"; + } else if (reviewedRegions.stream().anyMatch(region -> "faculty-approved".equals(region.reviewSource()))) { + compartmentSource = "faculty-approved"; + } else compartmentSource = "model-suggested"; + var ihcResult = pack.capability() == EvidencePackManifest.Capability.IHC_DESCRIPTIVE + ? IhcMeasurementAnalyzer.analyze(image, marker, reviewedRegions, + request.path("controlsValidated").asBoolean(false)) : null; + var specialResult = Set.of(EvidencePackManifest.Capability.SPECIAL_STAIN_DESCRIPTIVE, + EvidencePackManifest.Capability.CYTOLOGY_DESCRIPTIVE).contains(pack.capability()) + ? SpecialStainAnalyzer.analyze(image, stain) : null; + var descriptorWarning = ihcResult != null ? ihcResult.abstentionReason() + : specialResult != null ? specialResult.abstentionReason() : null; var abstained = !abstentionReasons.isEmpty(); var refiningAt = now.plusSeconds(1); checkCancellation(job.id(), workerId, refiningAt); @@ -185,10 +205,22 @@ public EvidenceJob process(EvidenceJob job, String workerId, Instant now) throws new EvidenceQuotaManager(stateRoot).requireReservation(EvidenceQuotaManager.Bucket.EVIDENCE_TEST, Math.max(1, Files.size(preview))); Files.createDirectories(artifactRoot); - var unsigned = evidence(request, pack, image, analysis, stainQc, modelResult, focus, tissue, - abstentionReasons, compartmentSource, markerIdentitySource, descriptorWarning, now); + var qualificationOnly = job.id().matches("qualification-[a-f0-9]{8,64}"); + var evidenceV2 = EvidencePackManifest.SCHEMA_V2.equals(pack.schema()) && !qualificationOnly; + var unsigned = evidenceV2 + ? evidenceV2(request, pack, image, analysis, modelResult, focus, tissue, + abstentionReasons, compartmentSource, ihcResult, specialResult, + reviewedRegions, now) + : evidence(request, pack, image, analysis, stainQc, modelResult, focus, tissue, + abstentionReasons, compartmentSource, markerIdentitySource, descriptorWarning, + ihcResult, specialResult, reviewedRegions, now); var finalArtifact = artifactRoot.resolve("evidence.json"); - new EvidenceBundleWriter(stateRoot.resolve("signing")).write(finalArtifact, unsigned); + if (evidenceV2) new EvidenceBundleWriterV2(stateRoot).write(finalArtifact, unsigned); + else new EvidenceBundleWriter(stateRoot.resolve("signing")).write(finalArtifact, unsigned); + if (job.id().matches("qualification-[a-f0-9]{8,64}") + && request.path("qualificationCampaignManifest").isTextual()) { + writeQualificationReport(job, request, pack, abstained, now.plusMillis(2400)); + } queue.recordFinalArtifact(job.id(), workerId, sha256(finalArtifact), now.plusMillis(2500)); var terminal = abstained ? EvidenceJobState.ABSTAINED : EvidenceJobState.COMPLETED; var detail = abstained @@ -199,6 +231,50 @@ public EvidenceJob process(EvidenceJob job, String workerId, Instant now) throws now.plusSeconds(3), LEASE); } + private void writeQualificationReport(EvidenceJob job, JsonNode request, EvidencePackManifest pack, + boolean abstained, Instant now) throws IOException { + var campaign = QualificationCampaignManifest.load( + regularPath(request, "qualificationCampaignManifest")); + var normalizedRequest = job.requestPath().toAbsolutePath().normalize(); + var track = campaign.tracks().stream().filter(candidate -> + normalizedRequest.equals(candidate.requestPath()) + || normalizedRequest.equals(candidate.remediationRequestPath())) + .findFirst().orElseThrow(() -> new IllegalArgumentException( + "Qualification request is not bound to the frozen campaign")); + require(track.candidateId().equals(pack.packId()) + && track.capability().equals(pack.capability().wire()) + && track.scope().equals(pack.scope()), + "Qualification request pack identity changed"); + if (track.expectedAttestationPath() == null) return; + var report = JSON.createObjectNode(); + report.put("schema", QualificationReportWriter.SCHEMA); + report.put("campaignId", campaign.campaignId()); + report.put("campaignManifestSha256", campaign.sha256()); + report.put("trackId", track.id()); + report.put("candidateId", track.candidateId()); + report.put("capability", track.capability()); + report.put("scope", track.scope()); + report.put("packManifestSha256", pack.sha256()); + report.put("protocolSha256", track.protocolSha256()); + report.put("generatedAt", now.toString()); + report.put("researchOnly", true); + report.put("notDiagnostic", true); + var heldOutRequired = switch (pack.capability()) { + case CELL_MORPHOLOGY, IHC_DESCRIPTIVE, SPECIAL_STAIN_DESCRIPTIVE, + CYTOLOGY_DESCRIPTIVE, HE_EVIDENCE, GROUNDED_TUTOR, ATLAS_DISTILLATION -> true; + }; + report.put("status", abstained ? "not_evaluable" : heldOutRequired ? "experimental" : "experimental"); + var checks = report.putArray("checks"); + checks.addObject().put("id", "offline-bounded-execution").put("outcome", + abstained ? "not_evaluable" : "pass") + .put("detail", "Candidate executed without network access inside its declared resource envelope"); + checks.addObject().put("id", "preregistered-held-out-gates").put("outcome", "not_evaluable") + .put("detail", "Required rights-cleared patient/source-held-out fixtures are not attached to this request"); + var reasons = report.putArray("reasons"); + reasons.add(abstained ? "INPUT_QC_FAILED" : "HELD_OUT_QUALIFICATION_PENDING"); + new QualificationReportWriter(stateRoot).write(track.expectedAttestationPath(), report); + } + private static ObjectNode evidence( JsonNode request, EvidencePackManifest pack, @@ -212,6 +288,9 @@ private static ObjectNode evidence( String compartmentSource, String markerIdentitySource, String descriptorWarning, + IhcMeasurementAnalyzer.Result ihcResult, + SpecialStainAnalyzer.Result specialResult, + java.util.List reviewedRegions, Instant now) { var root = JSON.createObjectNode(); root.put("schema", EvidenceBundleWriter.SCHEMA); @@ -232,21 +311,36 @@ private static ObjectNode evidence( root.put("researchOnly", true); root.put("notDiagnostic", true); root.put("reviewRequired", true); + if (request.path("qualificationCampaignManifest").isTextual()) { + root.put("qualificationOnly", true); + } var sourceWidth = positiveInt(request, "sourceWidth"); var sourceHeight = positiveInt(request, "sourceHeight"); root.set("coordinates", JSON.valueToTree(java.util.Map.of( "space", "source-pixel", "originX", 0, "originY", 0, "scaleX", (double) sourceWidth / image.getWidth(), "scaleY", (double) sourceHeight / image.getHeight()))); - root.set("evidence", modelResult == null - ? JSON.valueToTree(java.util.List.of(java.util.Map.of( + var regions = new java.util.ArrayList>(); + if (modelResult == null) { + regions.add(java.util.Map.of( "id", "region-1", "stage", "refined", "kind", "support", "x", 0, "y", 0, "width", sourceWidth, "height", sourceHeight, - "score", Math.max(0, Math.min(1, 1 - analysis.dabAreaFraction()))))) - : modelResult.path("regions").deepCopy()); + "score", Math.max(0, Math.min(1, 1 - analysis.dabAreaFraction())))); + for (var region : reviewedRegions) { + regions.add(java.util.Map.of( + "id", region.id(), "stage", "refined", + "kind", region.kind() + "-compartment", "reviewStatus", region.reviewSource(), + "x", (int) Math.round(region.x() * (double) sourceWidth / image.getWidth()), + "y", (int) Math.round(region.y() * (double) sourceHeight / image.getHeight()), + "width", (int) Math.round(region.width() * (double) sourceWidth / image.getWidth()), + "height", (int) Math.round(region.height() * (double) sourceHeight / image.getHeight()), + "score", 1.0)); + } + root.set("evidence", JSON.valueToTree(regions)); + } else root.set("evidence", modelResult.path("regions").deepCopy()); var aggregate = new java.util.LinkedHashMap(); aggregate.put("regionId", "region-1"); - aggregate.put("algorithm", "od-connected-components"); + aggregate.put("algorithm", "od-watershed"); aggregate.put("count", analysis.cellCount()); aggregate.put("densityPerMm2", null); aggregate.put("meanNucleusAreaPx2", analysis.meanNucleusAreaPx2()); @@ -258,26 +352,42 @@ private static ObjectNode evidence( pack.capability() == EvidencePackManifest.Capability.HE_EVIDENCE ? java.util.List.of() : java.util.List.of(aggregate))); var ihc = new java.util.ArrayList>(); - if (pack.capability() == EvidencePackManifest.Capability.IHC_DESCRIPTIVE) { + if (ihcResult != null) { var descriptor = new java.util.LinkedHashMap(); descriptor.put("regionId", "region-1"); descriptor.put("markerId", analysis.marker()); descriptor.put("marker", analysis.marker()); descriptor.put("markerIdentitySource", markerIdentitySource); - descriptor.put("analysisMode", "generic".equals(analysis.marker()) - ? "generic-descriptive" : "generic-fallback"); - descriptor.put("cellMaskSource", "od-connected-components"); + descriptor.put("analysisMode", ihcResult.analysisMode()); + descriptor.put("cellMaskSource", "od-watershed"); descriptor.put("compartmentSource", compartmentSource); descriptor.put("calibrationStatus", stainQc.calibrationStatus()); - descriptor.put("compartment", "generic-region"); - descriptor.put("dabAreaFraction", analysis.dabAreaFraction()); - descriptor.put("meanDabOd", analysis.meanDabOd()); - descriptor.put("uncertainty", 1 - Math.min(focus, tissue)); - descriptor.put("abstentionReason", descriptorWarning); + descriptor.put("compartment", ihcResult.compartment()); + descriptor.put("measurements", ihcResult.measurements()); + descriptor.put("qc", ihcResult.qc()); + descriptor.put("uncertainty", Math.max(1 - Math.min(focus, tissue), ihcResult.uncertainty())); + descriptor.put("abstentionReason", ihcResult.abstentionReason()); descriptor.put("researchEstimate", true); ihc.add(descriptor); } root.set("ihcDescriptors", JSON.valueToTree(ihc)); + var special = new java.util.ArrayList>(); + if (specialResult != null) { + var descriptor = new java.util.LinkedHashMap(); + descriptor.put("regionId", "region-1"); + descriptor.put("stainId", specialResult.stainId()); + descriptor.put("analysisMode", specialResult.analysisMode()); + descriptor.put("cellMaskSource", pack.capability() == EvidencePackManifest.Capability.CYTOLOGY_DESCRIPTIVE + ? "od-watershed" : "none"); + descriptor.put("measurements", specialResult.measurements()); + descriptor.put("qc", specialResult.qc()); + descriptor.put("uncertainty", specialResult.uncertainty()); + descriptor.put("abstentionReason", specialResult.abstentionReason()); + descriptor.put("researchEstimate", true); + special.add(descriptor); + } + root.set(pack.capability() == EvidencePackManifest.Capability.CYTOLOGY_DESCRIPTIVE + ? "cytologyDescriptors" : "specialStainDescriptors", JSON.valueToTree(special)); root.set("citations", JSON.createArrayNode()); var qc = new java.util.LinkedHashMap(); qc.put("focus", focus); @@ -296,6 +406,162 @@ private static ObjectNode evidence( return root; } + private static java.util.List reviewedRegions( + JsonNode node, BufferedImage image, int sourceWidth, int sourceHeight) { + if (node.isMissingNode() || node.isNull()) return java.util.List.of(); + require(node.isArray() && node.size() <= 64, "Reviewed regions are invalid"); + var result = new java.util.ArrayList(); + for (var item : node) { + require(fieldNames(item).equals(Set.of("id", "kind", "reviewSource", "x", "y", "width", "height")), + "Reviewed region fields are invalid"); + var x = boundedCoordinate(item, "x", sourceWidth - 1); + var y = boundedCoordinate(item, "y", sourceHeight - 1); + var width = boundedCoordinate(item, "width", sourceWidth - x); + var height = boundedCoordinate(item, "height", sourceHeight - y); + require(width > 0 && height > 0, "Reviewed region geometry is invalid"); + result.add(new IhcMeasurementAnalyzer.ReviewedRegion( + text(item, "id"), text(item, "kind"), text(item, "reviewSource"), + (int) Math.floor(x * (double) image.getWidth() / sourceWidth), + (int) Math.floor(y * (double) image.getHeight() / sourceHeight), + Math.max(1, (int) Math.ceil(width * (double) image.getWidth() / sourceWidth)), + Math.max(1, (int) Math.ceil(height * (double) image.getHeight() / sourceHeight)))); + } + return java.util.List.copyOf(result); + } + + private static ObjectNode evidenceV2( + JsonNode request, EvidencePackManifest pack, BufferedImage image, + BrightfieldTileAnalyzer.Result analysis, JsonNode modelResult, double focus, double tissue, + java.util.List abstentionReasons, String compartmentSource, + IhcMeasurementAnalyzer.Result ihcResult, SpecialStainAnalyzer.Result specialResult, + java.util.List reviewedRegions, Instant now) { + var sourceWidth = positiveInt(request, "sourceWidth"); + var sourceHeight = positiveInt(request, "sourceHeight"); + var root = JSON.createObjectNode(); + root.put("schema", EvidenceBundleWriterV2.SCHEMA); + root.put("bundleId", "evidence-" + java.util.UUID.randomUUID()); + root.set("source", JSON.valueToTree(java.util.Map.of( + "slideSha256", text(request, "sourceSha256"), "revision", text(request, "slideRevision"), + "width", sourceWidth, "height", sourceHeight))); + root.set("pack", JSON.valueToTree(java.util.Map.of( + "id", pack.packId(), "version", pack.version(), "manifestSha256", pack.sha256(), + "capability", pack.capability().wire(), "scope", pack.scope(), + "preprocessing", pack.preprocessingId(), + "artifacts", pack.artifacts().stream().map(EvidencePackManifest.Artifact::sha256).toList()))); + root.put("qualificationAttestationSha256", text(request, "qualificationAttestationSha256")); + root.put("status", abstentionReasons.isEmpty() ? "completed" : "abstained"); + root.put("researchOnly", true); + root.put("notDiagnostic", true); + root.put("reviewRequired", true); + root.set("coordinates", JSON.valueToTree(java.util.Map.of( + "space", "source-pixel", "originX", 0, "originY", 0, + "scaleX", (double) sourceWidth / image.getWidth(), + "scaleY", (double) sourceHeight / image.getHeight()))); + var regions = root.putArray("regions"); + if (modelResult != null && modelResult.path("regions").isArray()) { + for (var candidate : modelResult.path("regions")) { + var region = regions.addObject(); + region.put("id", candidate.path("id").asText("region-" + regions.size())); + region.put("stage", candidate.path("stage").asText("refined")); + region.put("kind", candidate.path("kind").asText("support")); + region.put("reviewStatus", "unreviewed"); + for (var field : java.util.List.of("x", "y", "width", "height", "score")) { + region.put(field, candidate.path(field).asDouble()); + } + } + } + if (regions.isEmpty()) { + var region = regions.addObject(); + region.put("id", "region-1"); region.put("stage", "refined"); region.put("kind", "analysis"); + region.put("reviewStatus", "unreviewed"); region.put("x", 0); region.put("y", 0); + region.put("width", sourceWidth); region.put("height", sourceHeight); region.put("score", 1.0); + } + for (var reviewed : reviewedRegions) { + var region = regions.addObject(); + region.put("id", reviewed.id()); region.put("stage", "refined"); + region.put("kind", "analysis".equals(reviewed.kind()) + ? "analysis" : reviewed.kind() + "-compartment"); + region.put("reviewStatus", reviewed.reviewSource()); + region.put("x", reviewed.x() * (double) sourceWidth / image.getWidth()); + region.put("y", reviewed.y() * (double) sourceHeight / image.getHeight()); + region.put("width", reviewed.width() * (double) sourceWidth / image.getWidth()); + region.put("height", reviewed.height() * (double) sourceHeight / image.getHeight()); + region.put("score", 1.0); + } + var includeCells = pack.capability() != EvidencePackManifest.Capability.HE_EVIDENCE + && pack.capability() != EvidencePackManifest.Capability.SPECIAL_STAIN_DESCRIPTIVE; + var instances = root.putArray("cellInstances"); + if (includeCells) { + var index = 0; + for (var cell : analysis.instances()) { + var item = instances.addObject(); item.put("id", "cell-" + (++index)); + item.put("regionId", "region-1"); item.put("maskEncoding", "rle"); + item.set("mask", JSON.valueToTree(cell.rle())); item.put("areaPx2", cell.areaPx2()); + item.put("perimeterPx", cell.perimeterPx()); item.put("eccentricity", cell.eccentricity()); + item.put("solidity", cell.solidity()); item.put("meanIntensity", cell.meanIntensity()); + item.put("uncertainty", Math.max(0, 1 - focus)); item.put("category", "unclassified"); + } + } + var aggregates = root.putArray("cellAggregates"); + if (includeCells) { + var aggregate = aggregates.addObject(); aggregate.put("regionId", "region-1"); + aggregate.put("algorithm", "od-watershed"); aggregate.put("count", analysis.cellCount()); + aggregate.putNull("densityPerMm2"); + aggregate.set("distributions", JSON.valueToTree(java.util.Map.of( + "meanAreaPx2", analysis.meanNucleusAreaPx2() == null ? 0 : analysis.meanNucleusAreaPx2(), + "meanPerimeterPx", analysis.meanNucleusPerimeterPx() == null ? 0 : analysis.meanNucleusPerimeterPx(), + "meanEccentricity", analysis.meanNucleusEccentricity() == null ? 0 : analysis.meanNucleusEccentricity(), + "meanSolidity", analysis.meanNucleusSolidity() == null ? 0 : analysis.meanNucleusSolidity()))); + aggregate.put("uncertainty", 1 - Math.min(focus, tissue)); + aggregate.set("qc", JSON.valueToTree(abstentionReasons)); + } + var ihc = root.putArray("ihcDescriptors"); + if (ihcResult != null) addStainDescriptor(ihc, "ihc_dab", ihcResult.markerId(), + ihcResult.analysisMode(), "od-watershed", compartmentSource, + ihcResult.calibrationStatus(), ihcResult.measurements(), ihcResult.qc(), + ihcResult.uncertainty(), ihcResult.abstentionReason()); + var special = root.putArray("specialStainDescriptors"); + var cytology = root.putArray("cytologyDescriptors"); + if (specialResult != null) addStainDescriptor( + pack.capability() == EvidencePackManifest.Capability.CYTOLOGY_DESCRIPTIVE ? cytology : special, + specialResult.stainId(), "none", specialResult.analysisMode(), + pack.capability() == EvidencePackManifest.Capability.CYTOLOGY_DESCRIPTIVE ? "od-watershed" : "none", + "none", "relative_only", specialResult.measurements(), specialResult.qc(), + specialResult.uncertainty(), specialResult.abstentionReason()); + root.set("citations", JSON.createArrayNode()); + root.set("qc", JSON.valueToTree(java.util.Map.of( + "focus", focus, "tissueFraction", tissue, "uncertainty", 1 - Math.min(focus, tissue), + "abstentionReasons", abstentionReasons, + "warnings", java.util.List.of()))); + var provenance = root.putObject("provenance"); + provenance.put("createdAt", now.toString()); + provenance.put("codeRevision", "pathlab-forge-evidence-mentor-v2"); + provenance.put("offlineAnalysis", true); + provenance.putNull("campaignId"); + return root; + } + + private static void addStainDescriptor(com.fasterxml.jackson.databind.node.ArrayNode output, + String stainId, String markerId, String mode, String maskSource, String compartmentSource, + String calibration, java.util.Map measurements, java.util.List qc, + double uncertainty, String abstention) { + var item = output.addObject(); item.put("regionId", "region-1"); item.put("stainId", stainId); + item.put("markerId", markerId); item.put("analysisMode", mode); item.put("cellMaskSource", maskSource); + item.put("compartmentSource", compartmentSource); item.put("calibrationStatus", calibration); + item.set("measurements", JSON.valueToTree(measurements)); item.set("qc", JSON.valueToTree(qc)); + item.put("uncertainty", Math.max(0, Math.min(1, uncertainty))); + if (abstention == null) item.putNull("abstentionReason"); else item.put("abstentionReason", abstention); + item.put("researchEstimate", true); + } + + private static int boundedCoordinate(JsonNode node, String name, int maximum) { + var value = node.path(name); + require(value.isIntegralNumber() && value.canConvertToInt() + && value.intValue() >= 0 && value.intValue() <= maximum, + "Reviewed region geometry is invalid"); + return value.intValue(); + } + private void checkCancellation(String id, String workerId, Instant now) throws IOException { var current = queue.find(id).orElseThrow(); if (current.cancelRequested()) { diff --git a/src/main/java/org/pathlab/forge/evidence/EvidenceMentorRunner.java b/src/main/java/org/pathlab/forge/evidence/EvidenceMentorRunner.java index bb52677..1365fee 100644 --- a/src/main/java/org/pathlab/forge/evidence/EvidenceMentorRunner.java +++ b/src/main/java/org/pathlab/forge/evidence/EvidenceMentorRunner.java @@ -30,14 +30,16 @@ /** Standalone authenticated loopback process for unattended Evidence Mentor jobs. */ public final class EvidenceMentorRunner implements AutoCloseable { private static final ObjectMapper JSON = new ObjectMapper(); - private static final String VERSION = "2.0.12"; + private static final String VERSION = "2.1.0"; private static final Duration LEASE = Duration.ofSeconds(45); private final Path stateRoot; private final String token; private final EvidenceJobQueue queue; + private final QualificationRunStore qualificationRuns; private final HttpServer server; private final ScheduledExecutorService gpuWorker; private final ScheduledExecutorService cpuWorker; + private final ScheduledExecutorService campaignWorker; private final java.util.concurrent.ExecutorService http; private final DashboardSessions dashboardSessions = new DashboardSessions(); private final Instant startedAt = Instant.now(); @@ -51,6 +53,7 @@ private EvidenceMentorRunner(Path stateRoot, int port, String token, boolean wor require(token != null && token.length() >= 32, "Loopback token is too short"); Files.createDirectories(this.stateRoot); queue = new EvidenceJobQueue(this.stateRoot.resolve("jobs.sqlite3")); + qualificationRuns = new QualificationRunStore(this.stateRoot.resolve("jobs.sqlite3"), this.stateRoot); queue.recoverOrphanedActiveJobs(startedAt); dashboardHtml = readResource("/evidence-dashboard/index.html"); server = HttpServer.create(new InetSocketAddress(InetAddress.getByName("127.0.0.1"), port), 32); @@ -62,9 +65,12 @@ private EvidenceMentorRunner(Path stateRoot, int port, String token, boolean wor publishEndpoint(); gpuWorker = Executors.newSingleThreadScheduledExecutor(r -> daemon(r, "pathlab-evidence-gpu", Thread.NORM_PRIORITY)); cpuWorker = Executors.newSingleThreadScheduledExecutor(r -> daemon(r, "pathlab-evidence-cpu-io", Thread.MIN_PRIORITY)); + campaignWorker = Executors.newSingleThreadScheduledExecutor( + r -> daemon(r, "pathlab-qualification-coordinator", Thread.MIN_PRIORITY)); if (workerEnabled) { gpuWorker.scheduleWithFixedDelay(() -> processOne(EvidenceExecutionLane.GPU), 0, 1, TimeUnit.SECONDS); cpuWorker.scheduleWithFixedDelay(() -> processOne(EvidenceExecutionLane.CPU_IO), 0, 1, TimeUnit.SECONDS); + campaignWorker.scheduleWithFixedDelay(this::advanceQualificationRuns, 0, 1, TimeUnit.SECONDS); } } @@ -104,6 +110,8 @@ private void route(HttpExchange exchange) throws IOException { if ("/v1/status".equals(path) && "GET".equals(method)) { status(exchange); return; } if ("/v1/jobs".equals(path) && "POST".equals(method)) { submit(exchange); return; } if ("/v1/jobs".equals(path) && "GET".equals(method)) { listJobs(exchange); return; } + if ("/v1/qualification-runs".equals(path) && "POST".equals(method)) { createQualificationRun(exchange); return; } + if ("/v1/qualification-runs".equals(path) && "GET".equals(method)) { listQualificationRuns(exchange); return; } if ("/v1/dashboard-sessions".equals(path) && "POST".equals(method)) { if (!bearerAuthorized(exchange)) { returnJson(exchange, 401, Map.of("error", "bearer_required")); return; } returnJson(exchange, 201, Map.of("code", dashboardSessions.create(Instant.now()), "expiresInSeconds", 60)); return; @@ -111,9 +119,47 @@ private void route(HttpExchange exchange) throws IOException { if ("/v1/control/pause".equals(path) && "POST".equals(method)) { mutate(exchange); queue.setAcceptingJobs(false, Instant.now()); control(exchange); return; } if ("/v1/control/resume".equals(path) && "POST".equals(method)) { mutate(exchange); queue.setAcceptingJobs(true, Instant.now()); control(exchange); return; } if (path.startsWith("/v1/jobs/")) { jobRoute(exchange, path.substring("/v1/jobs/".length())); return; } + if (path.startsWith("/v1/qualification-runs/")) { + qualificationRoute(exchange, path.substring("/v1/qualification-runs/".length())); return; + } returnJson(exchange, 404, Map.of("error", "not_found")); } + private void qualificationRoute(HttpExchange exchange, String suffix) throws IOException { + var parts = suffix.split("/", -1); + var id = parts[0]; + if (!id.matches("[A-Za-z0-9._-]{1,120}")) { + returnJson(exchange, 404, Map.of("error", "qualification_run_not_found")); return; + } + if (parts.length == 1 && "GET".equals(exchange.getRequestMethod())) { + var run = qualificationRuns.snapshot(id); + if (run.isEmpty()) returnJson(exchange, 404, Map.of("error", "qualification_run_not_found")); + else returnJsonCached(exchange, qualificationJson(run.get())); + return; + } + if (parts.length == 1 && "DELETE".equals(exchange.getRequestMethod())) { + mutate(exchange); + qualificationRuns.requestCancel(id, queue, Instant.now()); + returnJson(exchange, 202, qualificationJson(qualificationRuns.snapshot(id).orElseThrow())); + return; + } + if (parts.length == 2 && "resume".equals(parts[1]) && "POST".equals(exchange.getRequestMethod())) { + mutate(exchange); + returnJson(exchange, 202, qualificationJson(qualificationRuns.resume(id, Instant.now()))); + return; + } + if (parts.length == 2 && "attestations".equals(parts[1]) && "POST".equals(exchange.getRequestMethod())) { + mutate(exchange); + var body = jsonBody(exchange, 4096); + require(fieldNames(body).equals(Set.of("trackId", "reportPath")), "Attestation fields are invalid"); + var run = qualificationRuns.attachAttestation(id, text(body, "trackId"), + Path.of(text(body, "reportPath")), Instant.now()); + returnJson(exchange, 202, qualificationJson(run)); + return; + } + returnJson(exchange, 405, Map.of("error", "method_not_allowed")); + } + private void jobRoute(HttpExchange exchange, String suffix) throws IOException { var parts = suffix.split("/", -1); var id = parts[0]; if (!id.matches("[A-Za-z0-9._-]{1,120}")) { returnJson(exchange, 404, Map.of("error", "job_not_found")); return; } @@ -148,6 +194,7 @@ private void mutate(HttpExchange exchange) { private void status(HttpExchange exchange) throws IOException { var jobs = queue.list(null, null, 200); + var campaigns = qualificationRuns.list(100); long queued = jobs.stream().filter(j -> j.state() == EvidenceJobState.QUEUED).count(); long active = jobs.stream().filter(j -> !j.state().terminal() && j.state() != EvidenceJobState.QUEUED).count(); var runtime = Runtime.getRuntime(); var telemetry = new LinkedHashMap(); @@ -172,6 +219,15 @@ private void status(HttpExchange exchange) throws IOException { status.put("queue", Map.of("queued", queued, "active", active, "gpu", jobs.stream().filter(j -> j.lane() == EvidenceExecutionLane.GPU && !j.state().terminal()).count(), "cpuIo", jobs.stream().filter(j -> j.lane() == EvidenceExecutionLane.CPU_IO && !j.state().terminal()).count())); + status.put("qualification", Map.of( + "active", campaigns.stream().filter(run -> !run.campaignCompleted()).count(), + "completed", campaigns.stream().filter(QualificationRunSnapshot::campaignCompleted).count(), + "targetMet", campaigns.stream().filter(QualificationRunSnapshot::campaignTargetMet).count())); + campaigns.stream().filter(run -> !run.campaignCompleted()).findFirst() + .ifPresent(run -> status.put("leadingQualificationRun", Map.of( + "id", run.id(), "state", run.state(), + "terminalTracks", run.tracks().stream().filter(track -> !track.verdict().isBlank()).count(), + "totalTracks", run.tracks().size()))); jobs.stream().filter(j -> !j.state().terminal()).findFirst().ifPresent(job -> status.put("leadingJob", Map.of( "id", job.id(), "state", job.state().name().toLowerCase(), "stage", job.stage(), "progress", job.progress(), "lane", job.lane().wire()))); @@ -196,11 +252,33 @@ private void submit(HttpExchange exchange) throws IOException { var value = jsonBody(exchange, 65_536); require(fieldNames(value).equals(Set.of("id", "requestPath")), "Job submission fields are invalid"); var id = text(value, "id"); require(id.matches("[A-Za-z0-9._-]{1,120}"), "Job id is invalid"); var requestPath = Path.of(text(value, "requestPath")); - var plan = EvidenceJobProcessor.executionPlan(requestPath, id); + var plan = EvidenceJobProcessor.executionPlan(requestPath, id, stateRoot); var job = queue.submit(id, requestPath, plan.lane(), plan.packSha256(), Instant.now()); returnJson(exchange, 202, jobJson(queue.snapshot(job.id()).orElseThrow())); } + private void createQualificationRun(HttpExchange exchange) throws IOException { + mutate(exchange); + var body = jsonBody(exchange, 4096); + require(fieldNames(body).equals(Set.of("manifestPath")), "Qualification submission fields are invalid"); + var manifest = QualificationCampaignManifest.load(Path.of(text(body, "manifestPath"))); + returnJson(exchange, 202, qualificationJson(qualificationRuns.create(manifest, Instant.now()))); + } + + private void listQualificationRuns(HttpExchange exchange) throws IOException { + var query = query(exchange.getRequestURI().getRawQuery()); + require(query.keySet().stream().allMatch("limit"::equals), "Unknown qualification-run filter"); + var limit = query.containsKey("limit") ? Integer.parseInt(query.get("limit")) : 20; + returnJsonCached(exchange, Map.of( + "schema", "pathlab.qualification-run-list/1", + "runs", qualificationRuns.list(limit).stream().map(EvidenceMentorRunner::qualificationJson).toList())); + } + + private void advanceQualificationRuns() { + try { qualificationRuns.tick(queue, Instant.now()); } + catch (Exception ignored) { /* durable state is retried on the next coordinator tick */ } + } + private void control(HttpExchange exchange) throws IOException { returnJson(exchange, 200, Map.of("acceptingJobs", queue.acceptingJobs())); } private void processOne(EvidenceExecutionLane lane) { @@ -260,6 +338,28 @@ private static Map jobJson(EvidenceJobSnapshot job) { result.put("detail",job.detail()); result.put("updatedAt",job.updatedAt().toString()); return result; } + private static Map qualificationJson(QualificationRunSnapshot run) { + var result = new LinkedHashMap(); + result.put("schema", "pathlab.qualification-run/1"); + result.put("id", run.id()); + result.put("manifestSha256", run.manifestSha256()); + result.put("state", run.state()); + result.put("campaignCompleted", run.campaignCompleted()); + result.put("campaignTargetMet", run.campaignTargetMet()); + result.put("cancelRequested", run.cancelRequested()); + result.put("tracks", run.tracks().stream().map(track -> Map.ofEntries( + Map.entry("id", track.id()), Map.entry("candidateId", track.candidateId()), + Map.entry("capability", track.capability()), Map.entry("scope", track.scope()), + Map.entry("state", track.state()), Map.entry("verdict", track.verdict()), + Map.entry("attempt", track.attempt()), Map.entry("jobId", track.jobId()), + Map.entry("artifactSha256", track.artifactSha256()), + Map.entry("failureCode", track.failureCode()), Map.entry("detail", track.detail()), + Map.entry("updatedAt", track.updatedAt().toString()))).toList()); + result.put("createdAt", run.createdAt().toString()); + result.put("updatedAt", run.updatedAt().toString()); + return result; + } + private boolean authenticated(HttpExchange exchange) { return bearerAuthorized(exchange) || session(exchange).isPresent(); } private boolean bearerAuthorized(HttpExchange exchange) { var supplied=exchange.getRequestHeaders().getFirst("Authorization"); if(supplied==null)return false; @@ -306,7 +406,7 @@ private static void preserveEndpointAcl(Path endpoint, Path partial) throws IOEx private static void require(boolean condition,String message){if(!condition)throw new IllegalArgumentException(message);} private static void secure(boolean condition,String message){if(!condition)throw new SecurityException(message);} - @Override public void close()throws IOException{server.stop(0);gpuWorker.shutdownNow();cpuWorker.shutdownNow();http.shutdownNow();try{if(Files.isRegularFile(endpointPath)){var endpoint=JSON.readTree(endpointPath.toFile());if(bootId.equals(endpoint.path("bootId").asText()))Files.deleteIfExists(endpointPath);}}finally{queue.close();}} + @Override public void close()throws IOException{server.stop(0);gpuWorker.shutdownNow();cpuWorker.shutdownNow();campaignWorker.shutdownNow();http.shutdownNow();try{if(Files.isRegularFile(endpointPath)){var endpoint=JSON.readTree(endpointPath.toFile());if(bootId.equals(endpoint.path("bootId").asText()))Files.deleteIfExists(endpointPath);}}finally{try{qualificationRuns.close();}finally{queue.close();}}} public static void main(String[] args)throws Exception{ var state=defaultStateRoot();var port=0; diff --git a/src/main/java/org/pathlab/forge/evidence/EvidencePackManifest.java b/src/main/java/org/pathlab/forge/evidence/EvidencePackManifest.java index 7f8dc60..f4d543b 100644 --- a/src/main/java/org/pathlab/forge/evidence/EvidencePackManifest.java +++ b/src/main/java/org/pathlab/forge/evidence/EvidencePackManifest.java @@ -14,6 +14,7 @@ /** Validated, immutable description of one locally installed Evidence Mentor pack. */ public record EvidencePackManifest( Path path, + String schema, String packId, String version, Capability capability, @@ -21,6 +22,8 @@ public record EvidencePackManifest( String preprocessingId, int tilePixels, String allowedUse, + String scope, + boolean activationEligible, boolean redistributable, boolean derivativesAllowed, boolean acceptanceOnly, @@ -34,10 +37,12 @@ public record EvidencePackManifest( List artifacts, Set markers) { public static final String SCHEMA = "pathlab.ai-pack/1"; + public static final String SCHEMA_V2 = "pathlab.ai-pack/2"; private static final ObjectMapper JSON = new ObjectMapper(); private static final Pattern ID = Pattern.compile("[a-z0-9][a-z0-9._-]{0,119}"); private static final Pattern SHA = Pattern.compile("[a-f0-9]{64}"); - private static final Set STAINS = Set.of("he", "ihc_dab"); + private static final Set STAINS = Set.of("he", "ihc_dab", "pas", "pas_d", "trichrome", + "gms", "afb", "papanicolaou", "generic_brightfield"); private static final Set MARKERS = Set.of("generic", "er", "pr", "ki-67", "her2", "pd-l1"); public static EvidencePackManifest load(Path manifest) throws IOException { @@ -45,7 +50,9 @@ public static EvidencePackManifest load(Path manifest) throws IOException { var bytes = Files.readAllBytes(normalized); var root = JSON.readTree(bytes); require(root.isObject(), "AI pack must be a JSON object"); - require(SCHEMA.equals(text(root, "schema")), "AI pack schema is unsupported"); + var schema = text(root, "schema"); + require(Set.of(SCHEMA, SCHEMA_V2).contains(schema), "AI pack schema is unsupported"); + if (SCHEMA_V2.equals(schema)) return loadV2(normalized, bytes, root); var packId = text(root, "packId"); require(ID.matcher(packId).matches(), "AI pack id is invalid"); var version = text(root, "version"); @@ -96,7 +103,9 @@ public static EvidencePackManifest load(Path manifest) throws IOException { var licenseLedger = root.has("licenseLedger") ? parseLicenseLedger(root.path("licenseLedger")) : List.of(); return new EvidencePackManifest( - normalized, packId, version, capability, acceptedStains, preprocessingId, tilePixels, allowedUse, + normalized, SCHEMA, packId, version, capability, acceptedStains, preprocessingId, tilePixels, allowedUse, + "benchmark-only".equals(allowedUse) ? "local-benchmark" : "deployment", + false, booleanValue(rights, "redistributable"), booleanValue(rights, "derivativesAllowed"), acceptanceOnly, @@ -104,8 +113,66 @@ public static EvidencePackManifest load(Path manifest) throws IOException { maxRam, maxVram, maxSeconds, status, sha256(bytes), artifacts, markers); } + private static EvidencePackManifest loadV2(Path normalized, byte[] bytes, JsonNode root) throws IOException { + exact(root, Set.of("schema", "packId", "version", "capability", "scope", "activationEligible", + "acceptedStains", "preprocessing", "artifacts", "runtimeCompatibility", "licenseLedger", + "derivativePermissions", "resourceEnvelope", "qualificationPolicy", "outputSchema", "markers")); + var packId = text(root, "packId"); + require(ID.matcher(packId).matches(), "AI pack id is invalid"); + var version = text(root, "version"); + require(version.matches("[A-Za-z0-9][A-Za-z0-9._-]{0,63}"), "AI pack version is invalid"); + var capability = Capability.fromWire(text(root, "capability")); + var scope = text(root, "scope"); + require(Set.of("deployment", "research-restricted", "local-benchmark").contains(scope), + "AI pack scope is invalid"); + var activationEligible = booleanValue(root, "activationEligible"); + require(!("local-benchmark".equals(scope) && activationEligible), + "Benchmark-only packs cannot be activation eligible"); + var stains = stringSet(root.path("acceptedStains"), STAINS, "accepted stain"); + require(!stains.isEmpty(), "AI pack requires an accepted stain"); + var preprocessing = object(root, "preprocessing"); + var preprocessingId = text(preprocessing, "id"); + var tilePixels = boundedInt(preprocessing, "tilePixels", 64, 2_048); + var resources = object(root, "resourceEnvelope"); + exact(resources, Set.of("lane", "maxRamMiB", "maxVramMiB", "maxSeconds", "network")); + require(Set.of("gpu", "cpu_io", "external").contains(text(resources, "lane")), + "AI pack lane is invalid"); + var maxRam = boundedInt(resources, "maxRamMiB", 64, 16_384); + var maxVram = boundedInt(resources, "maxVramMiB", 0, 4_608); + var maxSeconds = boundedInt(resources, "maxSeconds", 1, 1_200); + require(!resources.path("network").asBoolean(true), "Analysis packs must be offline"); + var runtime = parseRuntimeV2(root.path("runtimeCompatibility")); + var ledger = parseLicenseLedgerV2(root.path("licenseLedger")); + var permissions = object(root, "derivativePermissions"); + exact(permissions, Set.of("atlasResearch", "atlasClean", "reviewedAt")); + require(permissions.path("atlasResearch").isBoolean() && permissions.path("atlasClean").isBoolean(), + "AI pack derivative permissions are invalid"); + var policy = object(root, "qualificationPolicy"); + exact(policy, Set.of("requiresSignedAttestation", "protocolSha256")); + require(policy.path("requiresSignedAttestation").asBoolean(false) + && text(policy, "protocolSha256").matches("[a-f0-9]{64}"), + "AI pack qualification policy is invalid"); + require(Set.of("pathlab.ai-evidence/2", "pathlab.model-qualification-report/2", + "pathlab.model-worker-result/2").contains(text(root, "outputSchema")), + "AI pack output schema is invalid"); + var artifacts = parseArtifactsV2(root.path("artifacts")); + var markers = stringSet(root.path("markers"), MARKERS, "marker"); + if (capability == Capability.IHC_DESCRIPTIVE) require(markers.contains("generic"), + "IHC packs must include the generic marker"); + else require(markers.isEmpty(), "Only IHC packs may declare markers"); + var allowedUse = switch (scope) { + case "deployment" -> "private-research"; + case "research-restricted" -> "research-restricted"; + default -> "benchmark-only"; + }; + return new EvidencePackManifest(normalized, SCHEMA_V2, packId, version, capability, stains, + preprocessingId, tilePixels, allowedUse, scope, activationEligible, false, + permissions.path("atlasResearch").asBoolean(false), false, runtime, ledger, + maxRam, maxVram, maxSeconds, ValidationStatus.NOT_EVALUABLE, sha256(bytes), artifacts, markers); + } + public boolean pilotEligible() { - return !acceptanceOnly && "private-research".equals(allowedUse) + return SCHEMA.equals(schema) && !acceptanceOnly && "private-research".equals(allowedUse) && validationStatus == ValidationStatus.QUALIFIED; } @@ -120,6 +187,7 @@ public void requireExecutableForJob(String jobId) { return; } if (jobId != null && jobId.matches("qualification-[a-f0-9]{8,64}")) { + if (SCHEMA_V2.equals(schema)) return; require("private-research".equals(allowedUse) && validationStatus == ValidationStatus.EXPERIMENTAL, "AI pack is not eligible for qualification execution"); @@ -128,6 +196,18 @@ public void requireExecutableForJob(String jobId) { requirePilotEligible(); } + public void requireExecutableForJob(String jobId, QualifiedPackRegistry registry, + String qualificationAttestationSha256) throws IOException { + if (!SCHEMA_V2.equals(schema) || (jobId != null + && jobId.matches("qualification-[a-f0-9]{8,64}"))) { + requireExecutableForJob(jobId); + return; + } + require(activationEligible && !"local-benchmark".equals(scope), + "AI pack candidate is not activation eligible"); + registry.requireQualified(this, qualificationAttestationSha256); + } + public void requireStain(String stain) { require(acceptedStains.contains(stain), "AI pack does not support stain " + stain); } @@ -146,6 +226,21 @@ private static List parseArtifacts(JsonNode node) { return List.copyOf(result); } + private static List parseArtifactsV2(JsonNode node) { + require(node.isArray() && node.size() <= 32, "AI pack artifacts are invalid"); + var result = new java.util.ArrayList(); + for (var item : node) { + exact(item, Set.of("name", "sha256", "source")); + var name = text(item, "name"); + var hash = text(item, "sha256"); + var source = text(item, "source"); + require(name.length() <= 160 && SHA.matcher(hash).matches() + && source.length() <= 1_000, "AI pack artifact is invalid"); + result.add(new Artifact(name, hash, source)); + } + return List.copyOf(result); + } + private static RuntimeCompatibility parseRuntime(JsonNode node) { require(node.isObject(), "AI pack runtime compatibility is invalid"); var protocol = text(node, "workerProtocol"); @@ -159,6 +254,19 @@ private static RuntimeCompatibility parseRuntime(JsonNode node) { node.path("requiresExternalWorker").asBoolean(false)); } + private static RuntimeCompatibility parseRuntimeV2(JsonNode node) { + exact(node, Set.of("workerProtocol", "executionProvider", "cuda", "gpuArchitecture", + "requiresExternalWorker", "segmentMaxSeconds")); + var protocol = text(node, "workerProtocol"); + var provider = text(node, "executionProvider"); + require(Set.of("pathlab.model-worker-result/1", "pathlab.model-worker-result/2").contains(protocol) + && Set.of("cpu", "cuda", "webgpu", "external").contains(provider), + "AI pack runtime compatibility is invalid"); + boundedInt(node, "segmentMaxSeconds", 1, 1_200); + return new RuntimeCompatibility(protocol, provider, text(node, "cuda"), + text(node, "gpuArchitecture"), node.path("requiresExternalWorker").asBoolean(false)); + } + private static List parseLicenseLedger(JsonNode node) { require(node.isArray() && node.size() <= 32, "AI pack license ledger is invalid"); var result = new java.util.ArrayList(); @@ -170,13 +278,31 @@ private static List parseLicenseLedger(JsonNode node) { require(component.length() <= 160 && license.length() <= 200 && revision.length() <= 160 && Set.of("private-research", "benchmark-only").contains(permittedUse), "AI pack license ledger is invalid"); - result.add(new LicenseEntry(component, license, revision, permittedUse, + result.add(new LicenseEntry(component, "weights", license, revision, permittedUse, item.path("redistributable").asBoolean(false), item.path("derivativesAllowed").asBoolean(false))); } return List.copyOf(result); } + private static List parseLicenseLedgerV2(JsonNode node) { + require(node.isArray() && !node.isEmpty() && node.size() <= 64, "AI pack license ledger is invalid"); + var result = new java.util.ArrayList(); + for (var item : node) { + exact(item, Set.of("component", "kind", "license", "revision", "permittedUse", + "redistributable", "derivativesAllowed")); + var kind = text(item, "kind"); + var permittedUse = text(item, "permittedUse"); + require(Set.of("code", "weights", "dataset", "runtime").contains(kind) + && Set.of("private-research", "research-restricted", "benchmark-only").contains(permittedUse), + "AI pack license ledger is invalid"); + result.add(new LicenseEntry(text(item, "component"), kind, text(item, "license"), + text(item, "revision"), permittedUse, booleanValue(item, "redistributable"), + booleanValue(item, "derivativesAllowed"))); + } + return List.copyOf(result); + } + private static Set stringSet(JsonNode node, Set allowed, String label) { require(node.isArray() && node.size() <= 16, "AI pack " + label + " list is invalid"); var result = new java.util.LinkedHashSet(); @@ -194,6 +320,13 @@ private static JsonNode object(JsonNode root, String name) { return value; } + private static void exact(JsonNode node, Set expected) { + require(node.isObject(), "AI pack object is invalid"); + var actual = new java.util.HashSet(); + node.fieldNames().forEachRemaining(actual::add); + require(actual.equals(expected), "AI pack fields are invalid"); + } + private static String text(JsonNode root, String name) { var value = root.path(name); require(value.isTextual() && !value.textValue().isBlank(), "AI pack " + name + " is invalid"); @@ -229,13 +362,17 @@ private static void require(boolean condition, String message) { public record Artifact(String name, String sha256, String source) {} public record RuntimeCompatibility(String workerProtocol, String executionProvider, String cuda, String gpuArchitecture, boolean requiresExternalWorker) {} - public record LicenseEntry(String component, String license, String revision, String permittedUse, + public record LicenseEntry(String component, String kind, String license, String revision, String permittedUse, boolean redistributable, boolean derivativesAllowed) {} public enum Capability { HE_EVIDENCE("he-evidence"), CELL_MORPHOLOGY("cell-morphology"), - IHC_DESCRIPTIVE("ihc-descriptive"); + IHC_DESCRIPTIVE("ihc-descriptive"), + SPECIAL_STAIN_DESCRIPTIVE("special-stain-descriptive"), + CYTOLOGY_DESCRIPTIVE("cytology-descriptive"), + GROUNDED_TUTOR("grounded-tutor"), + ATLAS_DISTILLATION("atlas-distillation"); private final String wire; Capability(String wire) { this.wire = wire; } diff --git a/src/main/java/org/pathlab/forge/evidence/EvidenceSetWriter.java b/src/main/java/org/pathlab/forge/evidence/EvidenceSetWriter.java new file mode 100644 index 0000000..074b79e --- /dev/null +++ b/src/main/java/org/pathlab/forge/evidence/EvidenceSetWriter.java @@ -0,0 +1,94 @@ +package org.pathlab.forge.evidence; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import java.io.IOException; +import java.nio.file.AtomicMoveNotSupportedException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.time.Instant; +import java.util.HashSet; +import java.util.Map; + +/** Fuses reviewed pack outputs only at evidence/provenance/QC/coordinate level. */ +public final class EvidenceSetWriter { + public static final String SCHEMA = "pathlab.evidence-set/1"; + private static final ObjectMapper JSON = new ObjectMapper(); + private final Path stateRoot; + private final EvidenceSigner signer; + private final TrustedSignerRegistry registry; + + public EvidenceSetWriter(Path stateRoot) { + this.stateRoot = stateRoot.toAbsolutePath().normalize(); + signer = new EvidenceSigner(this.stateRoot.resolve("signing")); + registry = new TrustedSignerRegistry(this.stateRoot); + } + + public EvidenceBundleWriter.Result write(Path output, String setId, + java.util.List evidenceBundles, Instant now) throws IOException { + require(setId.matches("[A-Za-z0-9._-]{1,160}"), "Evidence set id is invalid"); + require(evidenceBundles != null && !evidenceBundles.isEmpty() && evidenceBundles.size() <= 16, + "Evidence set bundle count is invalid"); + String slideSha = null; + String revision = null; + var hashes = new HashSet(); + var bundles = JSON.createArrayNode(); + var status = "completed"; + for (var bundlePath : evidenceBundles) { + var path = bundlePath.toAbsolutePath().normalize(); + require(path.startsWith(stateRoot.resolve("artifacts")) && Files.isRegularFile(path), + "Evidence set bundle is outside the artifact root"); + var root = JSON.readTree(path.toFile()); + require(!root.path("qualificationOnly").asBoolean(false), + "Qualification-only artifacts cannot enter an evidence set"); + var currentSha = root.path("source").path("slideSha256").asText(); + var currentRevision = root.path("source").path("revision").asText(); + if (slideSha == null) { slideSha = currentSha; revision = currentRevision; } + require(slideSha.equals(currentSha) && revision.equals(currentRevision), + "Evidence set bundles must bind the same slide revision"); + var manifestSha = root.path("manifestSha256").asText(); + require(manifestSha.matches("[a-f0-9]{64}") && hashes.add(manifestSha), + "Evidence set bundle checksum is invalid or duplicated"); + require(EvidenceArtifactStore.isTrusted(stateRoot, path, slideSha, revision), + "Evidence set bundle signature is not trusted"); + var item = bundles.addObject(); + item.put("manifestSha256", manifestSha); + item.put("packId", root.path("pack").path("id").asText()); + item.put("capability", root.path("pack").path("capability").asText("descriptive-evidence")); + if (!"completed".equals(root.path("status").asText())) status = "partial"; + } + var unsigned = JSON.createObjectNode(); + unsigned.put("schema", SCHEMA); + unsigned.put("setId", setId); + unsigned.set("source", JSON.valueToTree(Map.of("slideSha256", slideSha, "revision", revision))); + unsigned.set("bundles", bundles); + unsigned.set("fusion", JSON.valueToTree(Map.of( + "method", "evidence-coordinate-v1", "coordinateBound", true, + "qcBound", true, "provenanceBound", true, "uncertaintyBound", true, + "serialSectionCellMatching", false))); + unsigned.put("status", status); + unsigned.put("researchOnly", true); + unsigned.put("notDiagnostic", true); + unsigned.put("reviewRequired", true); + unsigned.put("createdAt", now.toString()); + var manifestSha = EvidenceBundleWriter.sha256(EvidenceBundleWriter.canonicalBytes(unsigned)); + var signed = signer.sign(SCHEMA + "\n" + manifestSha); + registry.pin(signed.keyId(), signed.publicKeyDer(), "evidence-set"); + var complete = unsigned.deepCopy(); + complete.put("manifestSha256", manifestSha); + complete.set("signature", JSON.valueToTree(Map.of( + "algorithm", "Ed25519", "keyId", signed.keyId(), "value", signed.signature()))); + var target = output.toAbsolutePath().normalize(); + Files.createDirectories(target.getParent()); + var partial = target.resolveSibling(target.getFileName() + ".partial"); + JSON.writerWithDefaultPrettyPrinter().writeValue(partial.toFile(), complete); + try { Files.move(partial, target, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE); } + catch (AtomicMoveNotSupportedException ignored) { Files.move(partial, target, StandardCopyOption.REPLACE_EXISTING); } + return new EvidenceBundleWriter.Result(target, manifestSha, signed.keyId()); + } + + private static void require(boolean condition, String message) { + if (!condition) throw new IllegalArgumentException(message); + } +} diff --git a/src/main/java/org/pathlab/forge/evidence/IhcMeasurementAnalyzer.java b/src/main/java/org/pathlab/forge/evidence/IhcMeasurementAnalyzer.java new file mode 100644 index 0000000..6931664 --- /dev/null +++ b/src/main/java/org/pathlab/forge/evidence/IhcMeasurementAnalyzer.java @@ -0,0 +1,221 @@ +package org.pathlab.forge.evidence; + +import java.awt.image.BufferedImage; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** Deterministic within-slide H/DAB descriptors; never emits clinical categories or scores. */ +public final class IhcMeasurementAnalyzer { + private static final Set NUCLEAR = Set.of("er", "pr", "ki-67"); + private IhcMeasurementAnalyzer() { } + + public static Result analyze(BufferedImage image, String requestedMarker, + List reviewedRegions, boolean controlsValidated) { + var marker = requestedMarker == null ? "generic" : requestedMarker.toLowerCase(); + if (!Set.of("generic", "er", "pr", "ki-67", "her2", "pd-l1").contains(marker)) marker = "generic"; + var qc = BrightfieldStainQc.inspect(image, controlsValidated); + var cells = OpticalDensityWatershed.segment(image).instances(); + if ("pd-l1".equals(marker)) { + var approved = reviewedRegions == null ? List.of() : reviewedRegions.stream() + .filter(region -> Set.of("faculty-authored", "faculty-approved").contains(region.reviewSource())) + .filter(region -> Set.of("tumor", "immune").contains(region.kind())) + .toList(); + if (approved.isEmpty()) { + return new Result(marker, "generic-fallback", "generic-region", qc.calibrationStatus(), + generic(image, cells, null), "COMPARTMENT_REVIEW_REQUIRED", qc.reasons(), 1 - qc.separationScore()); + } + var compartments = new ArrayList(); + for (var region : approved) { + compartments.add(new Compartment(region.id(), region.kind() + "-region", + generic(image, cells, region), region.reviewSource())); + } + return new Result(marker, "marker-aware", "tumor-immune-region", qc.calibrationStatus(), + Map.of("compartments", List.copyOf(compartments)), null, qc.reasons(), 1 - qc.separationScore()); + } + if (NUCLEAR.contains(marker)) { + return new Result(marker, "marker-aware", "nuclear", qc.calibrationStatus(), + nuclear(image, cells), null, qc.reasons(), 1 - qc.separationScore()); + } + if ("her2".equals(marker)) { + return new Result(marker, "marker-aware", "membrane", qc.calibrationStatus(), + membrane(image, cells), null, qc.reasons(), 1 - qc.separationScore()); + } + return new Result("generic", "generic-descriptive", "generic-region", qc.calibrationStatus(), + generic(image, cells, null), null, qc.reasons(), 1 - qc.separationScore()); + } + + private static Map nuclear(BufferedImage image, + List cells) { + var low = 0; + var moderate = 0; + var strong = 0; + var unstained = 0; + double totalOd = 0; + for (var cell : cells) { + var od = meanDabOd(image, cell.rle(), null); + totalOd += od; + if (od >= 0.45) strong++; + else if (od >= 0.25) moderate++; + else if (od >= 0.10) low++; + else unstained++; + } + var stained = low + moderate + strong; + var values = new LinkedHashMap(); + values.put("totalNuclei", cells.size()); + values.put("dabAssociatedNuclei", stained); + values.put("dabAssociatedFraction", cells.isEmpty() ? 0 : (double) stained / cells.size()); + values.put("meanNuclearDabOd", cells.isEmpty() ? 0 : totalOd / cells.size()); + values.put("intensityDistribution", Map.of( + "unstained", unstained, "low", low, "moderate", moderate, "strong", strong)); + return Map.copyOf(values); + } + + private static Map membrane(BufferedImage image, + List cells) { + double completeness = 0; + double continuity = 0; + double intensity = 0; + for (var cell : cells) { + var ring = ringPixels(cell.rle(), image.getWidth(), image.getHeight()); + var brown = 0; + for (var pixel : ring) if (isDab(image.getRGB(pixel % image.getWidth(), pixel / image.getWidth()))) brown++; + var fraction = ring.isEmpty() ? 0 : (double) brown / ring.size(); + completeness += fraction; + continuity += longestCircularRun(ring, image) / Math.max(1.0, ring.size()); + intensity += meanDabOd(image, ring.stream().flatMapToInt(pixel -> java.util.stream.IntStream.of(pixel, 1)).boxed().toList(), null); + } + var count = Math.max(1, cells.size()); + return Map.of( + "evaluatedCellMasks", cells.size(), + "meanMembraneCompleteness", completeness / count, + "meanMembraneContinuity", continuity / count, + "meanMembraneDabOd", intensity / count); + } + + private static Map generic(BufferedImage image, + List cells, ReviewedRegion region) { + var bounds = bounds(region, image.getWidth(), image.getHeight()); + var dab = 0; + var tissue = 0; + double od = 0; + for (var y = bounds[1]; y < bounds[3]; y++) for (var x = bounds[0]; x < bounds[2]; x++) { + var rgb = image.getRGB(x, y); + var red = rgb >>> 16 & 0xff; + var green = rgb >>> 8 & 0xff; + var blue = rgb & 0xff; + if ((red + green + blue) / 3.0 < 235) tissue++; + if (isDab(rgb)) { + dab++; + od += dabOd(rgb); + } + } + var associated = cells.stream().filter(cell -> inRegion(cell.centroidX(), cell.centroidY(), bounds)) + .filter(cell -> meanDabOd(image, cell.rle(), bounds) >= 0.10).count(); + return Map.of( + "dabAreaFraction", tissue == 0 ? 0 : (double) dab / tissue, + "meanDabOd", dab == 0 ? 0 : od / dab, + "cellCount", cells.stream().filter(cell -> inRegion(cell.centroidX(), cell.centroidY(), bounds)).count(), + "dabAssociatedCellCount", associated); + } + + private static int[] bounds(ReviewedRegion region, int width, int height) { + if (region == null) return new int[] {0, 0, width, height}; + var x0 = Math.max(0, Math.min(width, region.x())); + var y0 = Math.max(0, Math.min(height, region.y())); + var x1 = Math.max(x0, Math.min(width, region.x() + region.width())); + var y1 = Math.max(y0, Math.min(height, region.y() + region.height())); + return new int[] {x0, y0, x1, y1}; + } + + private static boolean inRegion(double x, double y, int[] bounds) { + return x >= bounds[0] && x < bounds[2] && y >= bounds[1] && y < bounds[3]; + } + + private static double meanDabOd(BufferedImage image, List rle, int[] bounds) { + double total = 0; + var count = 0; + for (var index = 0; index + 1 < rle.size(); index += 2) { + var start = rle.get(index); + var length = rle.get(index + 1); + for (var offset = 0; offset < length; offset++) { + var pixel = start + offset; + var x = pixel % image.getWidth(); + var y = pixel / image.getWidth(); + if (bounds != null && !inRegion(x, y, bounds)) continue; + var rgb = image.getRGB(x, y); + if (isDab(rgb)) { + total += dabOd(rgb); + count++; + } + } + } + return count == 0 ? 0 : total / count; + } + + private static List ringPixels(List rle, int width, int height) { + var mask = new java.util.HashSet(); + for (var index = 0; index + 1 < rle.size(); index += 2) { + for (var offset = 0; offset < rle.get(index + 1); offset++) mask.add(rle.get(index) + offset); + } + var ring = new java.util.TreeSet(); + for (var pixel : mask) { + var x = pixel % width; + var y = pixel / width; + for (var ny = Math.max(0, y - 1); ny <= Math.min(height - 1, y + 1); ny++) { + for (var nx = Math.max(0, x - 1); nx <= Math.min(width - 1, x + 1); nx++) { + var candidate = ny * width + nx; + if (!mask.contains(candidate)) ring.add(candidate); + } + } + } + return List.copyOf(ring); + } + + private static double longestCircularRun(List ring, BufferedImage image) { + var longest = 0; + var current = 0; + for (var pixel : ring) { + if (isDab(image.getRGB(pixel % image.getWidth(), pixel / image.getWidth()))) { + current++; + longest = Math.max(longest, current); + } else current = 0; + } + return longest; + } + + private static boolean isDab(int rgb) { + var red = rgb >>> 16 & 0xff; + var green = rgb >>> 8 & 0xff; + var blue = rgb & 0xff; + return red - green >= 15 && green - blue >= 15 && red + green + blue < 660; + } + + private static double dabOd(int rgb) { + var red = rgb >>> 16 & 0xff; + var green = rgb >>> 8 & 0xff; + var blue = rgb & 0xff; + var luminance = 0.2126 * red + 0.7152 * green + 0.0722 * blue; + return -Math.log((luminance + 1) / 256.0); + } + + public record ReviewedRegion(String id, String kind, String reviewSource, + int x, int y, int width, int height) { + public ReviewedRegion { + if (id == null || !id.matches("[A-Za-z0-9._-]{1,120}") + || !Set.of("tumor", "immune", "analysis").contains(kind) + || !Set.of("faculty-authored", "faculty-approved", "model-suggested").contains(reviewSource) + || x < 0 || y < 0 || width < 1 || height < 1) { + throw new IllegalArgumentException("Reviewed compartment region is invalid"); + } + } + } + + public record Result(String markerId, String analysisMode, String compartment, + String calibrationStatus, Map measurements, String abstentionReason, + List qc, double uncertainty) { } + public record Compartment(String regionId, String compartment, Map measurements, + String reviewSource) { } +} diff --git a/src/main/java/org/pathlab/forge/evidence/OpticalDensityWatershed.java b/src/main/java/org/pathlab/forge/evidence/OpticalDensityWatershed.java new file mode 100644 index 0000000..7497388 --- /dev/null +++ b/src/main/java/org/pathlab/forge/evidence/OpticalDensityWatershed.java @@ -0,0 +1,170 @@ +package org.pathlab.forge.evidence; + +import java.awt.image.BufferedImage; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; + +/** Deterministic marker-controlled watershed fallback for bounded brightfield regions. */ +public final class OpticalDensityWatershed { + private OpticalDensityWatershed() { } + + public static Result segment(BufferedImage image) { + if (image == null || image.getWidth() < 1 || image.getHeight() < 1 + || (long) image.getWidth() * image.getHeight() > 4_194_304) { + throw new IllegalArgumentException("Brightfield image geometry is invalid"); + } + var width = image.getWidth(); + var height = image.getHeight(); + var mask = new boolean[width * height]; + for (var y = 0; y < height; y++) for (var x = 0; x < width; x++) { + var rgb = image.getRGB(x, y); + var red = rgb >>> 16 & 0xff; + var green = rgb >>> 8 & 0xff; + var blue = rgb & 0xff; + var opticalDensity = -Math.log((red + 1) / 256.0) + + -Math.log((green + 1) / 256.0) + + -Math.log((blue + 1) / 256.0); + mask[y * width + x] = blue - red >= 25 && blue - green >= 25 + && opticalDensity >= 0.25; + } + var components = connected(mask, width, height); + var instances = new ArrayList(); + for (var component : components) split(component, image, width, height, instances); + instances.sort(Comparator.comparingInt(Instance::firstPixel)); + var numbered = new ArrayList(); + for (var index = 0; index < instances.size(); index++) { + var item = instances.get(index); + numbered.add(new Instance("cell-" + (index + 1), item.firstPixel(), item.areaPx2(), + item.perimeterPx(), item.eccentricity(), item.solidity(), item.meanIntensity(), + item.centroidX(), item.centroidY(), item.rle())); + } + return new Result(List.copyOf(numbered), "od-watershed", true); + } + + private static List connected(boolean[] mask, int width, int height) { + var seen = new boolean[mask.length]; + var queue = new ArrayDeque(); + var result = new ArrayList(); + for (var start = 0; start < mask.length; start++) { + if (!mask[start] || seen[start]) continue; + var pixels = new ArrayList(); + seen[start] = true; + queue.add(start); + while (!queue.isEmpty()) { + var current = queue.removeFirst(); + pixels.add(current); + var x = current % width; + var y = current / width; + if (x > 0) enqueue(current - 1, mask, seen, queue); + if (x + 1 < width) enqueue(current + 1, mask, seen, queue); + if (y > 0) enqueue(current - width, mask, seen, queue); + if (y + 1 < height) enqueue(current + width, mask, seen, queue); + } + if (pixels.size() >= 4) result.add(pixels.stream().mapToInt(Integer::intValue).toArray()); + } + return result; + } + + private static void enqueue(int index, boolean[] mask, boolean[] seen, ArrayDeque queue) { + if (mask[index] && !seen[index]) { + seen[index] = true; + queue.addLast(index); + } + } + + private static void split(int[] component, BufferedImage image, int width, int height, + List output) { + var minX = width; + var minY = height; + var maxX = -1; + var maxY = -1; + for (var pixel : component) { + var x = pixel % width; + var y = pixel / width; + minX = Math.min(minX, x); + minY = Math.min(minY, y); + maxX = Math.max(maxX, x); + maxY = Math.max(maxY, y); + } + var boxWidth = maxX - minX + 1; + var boxHeight = maxY - minY + 1; + var major = Math.max(boxWidth, boxHeight); + var minor = Math.max(1, Math.min(boxWidth, boxHeight)); + var seedCount = Math.max(1, Math.min(8, (int) Math.round((double) major / minor))); + if (component.length < seedCount * 4) seedCount = Math.max(1, component.length / 4); + var groups = new ArrayList>(); + for (var index = 0; index < seedCount; index++) groups.add(new ArrayList<>()); + var horizontal = boxWidth >= boxHeight; + for (var pixel : component) { + var coordinate = horizontal ? pixel % width - minX : pixel / width - minY; + var group = Math.min(seedCount - 1, coordinate * seedCount / major); + groups.get(group).add(pixel); + } + // Empty marker basins are merged rather than emitted as fabricated instances. + for (var group : groups) { + if (group.size() >= 4) output.add(measure(group, image, width, height)); + } + } + + private static Instance measure(List pixels, BufferedImage image, int width, int height) { + var included = new java.util.HashSet(pixels); + var minX = width; + var minY = height; + var maxX = -1; + var maxY = -1; + var perimeter = 0; + double xSum = 0; + double ySum = 0; + double intensity = 0; + pixels.sort(Integer::compareTo); + for (var pixel : pixels) { + var x = pixel % width; + var y = pixel / width; + minX = Math.min(minX, x); + minY = Math.min(minY, y); + maxX = Math.max(maxX, x); + maxY = Math.max(maxY, y); + xSum += x; + ySum += y; + var rgb = image.getRGB(x, y); + var red = rgb >>> 16 & 0xff; + var green = rgb >>> 8 & 0xff; + var blue = rgb & 0xff; + intensity += 1 - (0.2126 * red + 0.7152 * green + 0.0722 * blue) / 255.0; + if (x == 0 || !included.contains(pixel - 1)) perimeter++; + if (x + 1 == width || !included.contains(pixel + 1)) perimeter++; + if (y == 0 || !included.contains(pixel - width)) perimeter++; + if (y + 1 == height || !included.contains(pixel + width)) perimeter++; + } + var boxWidth = maxX - minX + 1.0; + var boxHeight = maxY - minY + 1.0; + var major = Math.max(boxWidth, boxHeight); + var minor = Math.min(boxWidth, boxHeight); + var eccentricity = major == 0 ? 0 : Math.sqrt(Math.max(0, 1 - (minor * minor) / (major * major))); + var rle = new ArrayList(); + var runStart = pixels.get(0); + var prior = runStart; + for (var index = 1; index < pixels.size(); index++) { + var current = pixels.get(index); + if (current != prior + 1 || current / width != prior / width) { + rle.add(runStart); + rle.add(prior - runStart + 1); + runStart = current; + } + prior = current; + } + rle.add(runStart); + rle.add(prior - runStart + 1); + return new Instance("", pixels.get(0), pixels.size(), perimeter, eccentricity, + pixels.size() / (boxWidth * boxHeight), intensity / pixels.size(), + xSum / pixels.size(), ySum / pixels.size(), List.copyOf(rle)); + } + + public record Result(List instances, String method, boolean deterministic) { } + public record Instance( + String id, int firstPixel, double areaPx2, double perimeterPx, + double eccentricity, double solidity, double meanIntensity, + double centroidX, double centroidY, List rle) { } +} diff --git a/src/main/java/org/pathlab/forge/evidence/QualificationCampaignManifest.java b/src/main/java/org/pathlab/forge/evidence/QualificationCampaignManifest.java new file mode 100644 index 0000000..b606f9d --- /dev/null +++ b/src/main/java/org/pathlab/forge/evidence/QualificationCampaignManifest.java @@ -0,0 +1,172 @@ +package org.pathlab.forge.evidence; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.time.Instant; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.HexFormat; +import java.util.List; +import java.util.Set; + +/** Immutable, checksum-bound description of one autonomous qualification campaign. */ +public record QualificationCampaignManifest( + String campaignId, + Path path, + String sha256, + int maxRemediationAttempts, + List tracks) { + public static final String SCHEMA = "pathlab.qualification-campaign/1"; + private static final ObjectMapper JSON = new ObjectMapper(); + private static final Set CAPABILITIES = Set.of( + "he-evidence", "cell-morphology", "ihc-descriptive", + "special-stain-descriptive", "cytology-descriptive", + "grounded-tutor", "atlas-distillation"); + private static final Set SCOPES = Set.of("deployment", "research-restricted", "local-benchmark"); + + public static QualificationCampaignManifest load(Path input) throws IOException { + var path = input.toAbsolutePath().normalize(); + if (!Files.isRegularFile(path) || Files.size(path) > 2L * 1024 * 1024) { + throw new IllegalArgumentException("Qualification campaign manifest is unavailable or too large"); + } + var root = JSON.readTree(path.toFile()); + exact(root, Set.of("schema", "campaignId", "createdAt", "researchOnly", "notDiagnostic", + "maxRemediationAttempts", "quota", "tracks")); + require(SCHEMA.equals(text(root, "schema")), "Qualification campaign schema is unsupported"); + var campaignId = identifier(text(root, "campaignId")); + Instant.parse(text(root, "createdAt")); + require(root.path("researchOnly").asBoolean(false), "Qualification campaign must be research-only"); + require(root.path("notDiagnostic").asBoolean(false), "Qualification campaign must be non-diagnostic"); + var remediation = root.path("maxRemediationAttempts"); + require(remediation.isInt() && remediation.intValue() == 1, + "Qualification campaign requires exactly one bounded remediation attempt"); + validateQuota(root.path("quota")); + var items = root.path("tracks"); + require(items.isArray() && !items.isEmpty() && items.size() <= 64, + "Qualification campaign tracks are invalid"); + var tracks = new ArrayList(); + var ids = new HashSet(); + for (var item : items) { + exact(item, Set.of("id", "candidateId", "capability", "scope", "requestPath", + "remediationRequestPath", "expectedAttestationPath", "protocolSha256", + "dependsOn", "required")); + var id = identifier(text(item, "id")); + require(ids.add(id), "Qualification track id is duplicated"); + var candidateId = identifier(text(item, "candidateId")); + var capability = text(item, "capability"); + require(CAPABILITIES.contains(capability), "Qualification capability is unsupported"); + var scope = text(item, "scope"); + require(SCOPES.contains(scope), "Qualification scope is unsupported"); + var requestPath = immutablePath(path, text(item, "requestPath")); + Path remediationPath = null; + if (!item.path("remediationRequestPath").isNull()) { + remediationPath = immutablePath(path, text(item, "remediationRequestPath")); + } + Path attestationPath = null; + if (!item.path("expectedAttestationPath").isNull()) { + attestationPath = boundedPath(path, text(item, "expectedAttestationPath")); + } + var protocolSha = text(item, "protocolSha256"); + require(protocolSha.matches("[a-f0-9]{64}"), "Qualification protocol checksum is invalid"); + var dependencies = new ArrayList(); + var dependencyNode = item.path("dependsOn"); + require(dependencyNode.isArray() && dependencyNode.size() <= 16, + "Qualification dependencies are invalid"); + dependencyNode.forEach(value -> dependencies.add(identifier(value.asText()))); + require(item.path("required").isBoolean(), "Qualification required flag is invalid"); + tracks.add(new Track(id, candidateId, capability, scope, requestPath, remediationPath, + attestationPath, protocolSha, List.copyOf(dependencies), item.path("required").booleanValue())); + } + for (var track : tracks) { + require(!track.dependsOn().contains(track.id()) && ids.containsAll(track.dependsOn()), + "Qualification dependency graph is invalid"); + } + rejectCycles(tracks); + return new QualificationCampaignManifest(campaignId, path, sha256(path), 1, List.copyOf(tracks)); + } + + private static void validateQuota(JsonNode quota) { + exact(quota, Set.of("sourceBytes", "derivedBytes", "modelBytes", "evidenceBytes", "reserveBytes")); + require(quota.path("sourceBytes").asLong(-1) == 45L * 1024 * 1024 * 1024, + "Qualification source quota is not frozen"); + require(quota.path("derivedBytes").asLong(-1) == 25L * 1024 * 1024 * 1024, + "Qualification derived quota is not frozen"); + require(quota.path("modelBytes").asLong(-1) == 10L * 1024 * 1024 * 1024, + "Qualification model quota is not frozen"); + require(quota.path("evidenceBytes").asLong(-1) == 10L * 1024 * 1024 * 1024, + "Qualification evidence quota is not frozen"); + require(quota.path("reserveBytes").asLong(-1) == 10L * 1024 * 1024 * 1024, + "Qualification reserve quota is not frozen"); + } + + private static void rejectCycles(List tracks) { + var byId = new java.util.HashMap(); + tracks.forEach(track -> byId.put(track.id(), track)); + for (var track : tracks) visit(track.id(), byId, new HashSet<>(), new HashSet<>()); + } + + private static void visit(String id, java.util.Map tracks, Set active, Set done) { + if (done.contains(id)) return; + require(active.add(id), "Qualification dependency graph contains a cycle"); + for (var dependency : tracks.get(id).dependsOn()) visit(dependency, tracks, active, done); + active.remove(id); + done.add(id); + } + + private static Path immutablePath(Path manifest, String value) { + var path = boundedPath(manifest, value); + require(Files.isRegularFile(path), "Qualification input file is unavailable"); + return path; + } + + private static Path boundedPath(Path manifest, String value) { + var base = manifest.getParent().toAbsolutePath().normalize(); + var candidate = Path.of(value); + var path = (candidate.isAbsolute() ? candidate : base.resolve(candidate)).toAbsolutePath().normalize(); + require(path.startsWith(base), "Qualification input escapes the campaign directory"); + return path; + } + + private static String sha256(Path path) throws IOException { + try (InputStream input = Files.newInputStream(path)) { + var digest = MessageDigest.getInstance("SHA-256"); + var buffer = new byte[64 * 1024]; + for (int count; (count = input.read(buffer)) >= 0;) digest.update(buffer, 0, count); + return HexFormat.of().formatHex(digest.digest()); + } catch (java.security.GeneralSecurityException impossible) { + throw new IOException("SHA-256 is unavailable", impossible); + } + } + + private static void exact(JsonNode value, Set expected) { + require(value != null && value.isObject(), "Qualification campaign object is invalid"); + var fields = new HashSet(); + value.fieldNames().forEachRemaining(fields::add); + require(fields.equals(expected), "Qualification campaign fields are invalid"); + } + + private static String text(JsonNode value, String field) { + var node = value.path(field); + require(node.isTextual() && !node.textValue().isBlank(), "Qualification field is invalid: " + field); + return node.textValue(); + } + + private static String identifier(String value) { + require(value != null && value.matches("[A-Za-z0-9._-]{1,120}"), "Qualification identifier is invalid"); + return value; + } + + private static void require(boolean condition, String message) { + if (!condition) throw new IllegalArgumentException(message); + } + + public record Track( + String id, String candidateId, String capability, String scope, + Path requestPath, Path remediationRequestPath, Path expectedAttestationPath, + String protocolSha256, List dependsOn, boolean required) { } +} diff --git a/src/main/java/org/pathlab/forge/evidence/QualificationReportValidator.java b/src/main/java/org/pathlab/forge/evidence/QualificationReportValidator.java new file mode 100644 index 0000000..9629b3c --- /dev/null +++ b/src/main/java/org/pathlab/forge/evidence/QualificationReportValidator.java @@ -0,0 +1,103 @@ +package org.pathlab.forge.evidence; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.KeyFactory; +import java.security.Signature; +import java.security.spec.X509EncodedKeySpec; +import java.time.Instant; +import java.util.HashSet; +import java.util.Set; + +/** Strict verifier for signed, non-publishable qualification attestations. */ +public final class QualificationReportValidator { + private static final ObjectMapper JSON = new ObjectMapper(); + private static final Set VERDICTS = Set.of("qualified", "experimental", "unsupported", "not_evaluable"); + private final TrustedSignerRegistry registry; + + public QualificationReportValidator(Path stateRoot) { + registry = new TrustedSignerRegistry(stateRoot); + } + + public Validated validate(Path path, QualificationCampaignManifest campaign, + QualificationCampaignManifest.Track track) throws IOException { + var normalized = path.toAbsolutePath().normalize(); + require(Files.isRegularFile(normalized) && Files.size(normalized) <= 2L * 1024 * 1024, + "Qualification report is unavailable or too large"); + var root = JSON.readTree(normalized.toFile()); + exact(root, Set.of("schema", "campaignId", "campaignManifestSha256", "trackId", "candidateId", + "capability", "scope", "packManifestSha256", "protocolSha256", "generatedAt", + "researchOnly", "notDiagnostic", "status", "checks", "reasons", + "manifestSha256", "signature")); + require(QualificationReportWriter.SCHEMA.equals(root.path("schema").asText()), + "Qualification report schema is unsupported"); + require(campaign.campaignId().equals(root.path("campaignId").asText()) + && campaign.sha256().equals(root.path("campaignManifestSha256").asText()), + "Qualification report campaign identity changed"); + require(track.id().equals(root.path("trackId").asText()) + && track.candidateId().equals(root.path("candidateId").asText()) + && track.capability().equals(root.path("capability").asText()) + && track.scope().equals(root.path("scope").asText()), + "Qualification report track identity changed"); + require(track.protocolSha256().equals(root.path("protocolSha256").asText()), + "Qualification protocol checksum changed"); + require(root.path("packManifestSha256").asText().matches("[a-f0-9]{64}"), + "Qualification pack checksum is invalid"); + Instant.parse(root.path("generatedAt").asText()); + require(root.path("researchOnly").asBoolean(false) && root.path("notDiagnostic").asBoolean(false), + "Qualification report safety boundary is invalid"); + var verdict = root.path("status").asText(); + require(VERDICTS.contains(verdict), "Qualification verdict is invalid"); + require(root.path("checks").isArray() && !root.path("checks").isEmpty() + && root.path("checks").size() <= 200, "Qualification checks are invalid"); + require(root.path("reasons").isArray() && root.path("reasons").size() <= 100, + "Qualification reasons are invalid"); + var suppliedSha = root.path("manifestSha256").asText(); + require(suppliedSha.matches("[a-f0-9]{64}"), "Qualification report checksum is invalid"); + var unsigned = ((ObjectNode) root.deepCopy()); + unsigned.remove(Set.of("manifestSha256", "signature")); + var calculated = EvidenceBundleWriter.sha256(EvidenceBundleWriter.canonicalBytes(unsigned)); + require(suppliedSha.equals(calculated), "Qualification report checksum mismatch"); + verify(root.path("signature"), suppliedSha); + return new Validated(verdict, suppliedSha, root.path("packManifestSha256").asText()); + } + + private void verify(JsonNode signature, String manifestSha) throws IOException { + exact(signature, Set.of("algorithm", "keyId", "value")); + require("Ed25519".equals(signature.path("algorithm").asText()), "Qualification signature is invalid"); + var keyId = signature.path("keyId").asText(); + require(keyId.matches("[a-f0-9]{64}"), "Qualification signer id is invalid"); + var publicBytes = registry.find(keyId, "qualification") + .orElseThrow(() -> new IllegalArgumentException("Qualification signer is not trusted")); + try { + var key = KeyFactory.getInstance("Ed25519").generatePublic(new X509EncodedKeySpec(publicBytes)); + var verifier = Signature.getInstance("Ed25519"); + verifier.initVerify(key); + verifier.update((QualificationReportWriter.SCHEMA + "\n" + manifestSha) + .getBytes(java.nio.charset.StandardCharsets.UTF_8)); + var encoded = java.util.Base64.getUrlDecoder().decode(signature.path("value").asText()); + require(verifier.verify(encoded), "Qualification signature is invalid"); + } catch (IllegalArgumentException error) { + throw error; + } catch (Exception error) { + throw new IllegalArgumentException("Qualification signature is invalid", error); + } + } + + private static void exact(JsonNode value, Set expected) { + require(value != null && value.isObject(), "Qualification report object is invalid"); + var actual = new HashSet(); + value.fieldNames().forEachRemaining(actual::add); + require(actual.equals(expected), "Qualification report fields are invalid"); + } + + private static void require(boolean condition, String message) { + if (!condition) throw new IllegalArgumentException(message); + } + + public record Validated(String verdict, String manifestSha256, String packManifestSha256) { } +} diff --git a/src/main/java/org/pathlab/forge/evidence/QualificationReportWriter.java b/src/main/java/org/pathlab/forge/evidence/QualificationReportWriter.java new file mode 100644 index 0000000..a83084d --- /dev/null +++ b/src/main/java/org/pathlab/forge/evidence/QualificationReportWriter.java @@ -0,0 +1,54 @@ +package org.pathlab.forge.evidence; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.node.ObjectNode; +import java.io.IOException; +import java.nio.file.AtomicMoveNotSupportedException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.Map; + +/** Signs qualification reports while pinning trust outside the signed payload. */ +public final class QualificationReportWriter { + public static final String SCHEMA = "pathlab.model-qualification-report/2"; + private static final ObjectMapper JSON = new ObjectMapper() + .enable(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS); + private final EvidenceSigner signer; + private final TrustedSignerRegistry registry; + + public QualificationReportWriter(Path stateRoot) { + signer = new EvidenceSigner(stateRoot.resolve("signing")); + registry = new TrustedSignerRegistry(stateRoot); + } + + public EvidenceBundleWriter.Result write(Path output, ObjectNode unsigned) throws IOException { + require(SCHEMA.equals(unsigned.path("schema").asText()), "Qualification report schema is unsupported"); + require(unsigned.path("researchOnly").asBoolean(false), "Qualification report must be research-only"); + require(unsigned.path("notDiagnostic").asBoolean(false), "Qualification report must be non-diagnostic"); + require(!unsigned.has("manifestSha256") && !unsigned.has("signature"), + "Unsigned qualification report contains signature fields"); + var sha = EvidenceBundleWriter.sha256(EvidenceBundleWriter.canonicalBytes(unsigned)); + var signed = signer.sign(SCHEMA + "\n" + sha); + registry.pin(signed.keyId(), signed.publicKeyDer(), "qualification"); + var complete = unsigned.deepCopy(); + complete.put("manifestSha256", sha); + complete.set("signature", JSON.valueToTree(Map.of( + "algorithm", "Ed25519", "keyId", signed.keyId(), "value", signed.signature()))); + var target = output.toAbsolutePath().normalize(); + Files.createDirectories(target.getParent()); + var partial = target.resolveSibling(target.getFileName() + ".partial"); + JSON.writerWithDefaultPrettyPrinter().writeValue(partial.toFile(), complete); + try { + Files.move(partial, target, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE); + } catch (AtomicMoveNotSupportedException ignored) { + Files.move(partial, target, StandardCopyOption.REPLACE_EXISTING); + } + return new EvidenceBundleWriter.Result(target, sha, signed.keyId()); + } + + private static void require(boolean condition, String message) { + if (!condition) throw new IllegalArgumentException(message); + } +} diff --git a/src/main/java/org/pathlab/forge/evidence/QualificationRunSnapshot.java b/src/main/java/org/pathlab/forge/evidence/QualificationRunSnapshot.java new file mode 100644 index 0000000..fdd3a16 --- /dev/null +++ b/src/main/java/org/pathlab/forge/evidence/QualificationRunSnapshot.java @@ -0,0 +1,17 @@ +package org.pathlab.forge.evidence; + +import java.time.Instant; +import java.util.List; + +/** Operations-only campaign view; scientific measurements stay in signed reports. */ +public record QualificationRunSnapshot( + String id, String manifestSha256, String state, + boolean campaignCompleted, boolean campaignTargetMet, + boolean cancelRequested, List tracks, + Instant createdAt, Instant updatedAt) { + public record Track( + String id, String candidateId, String capability, String scope, + String state, String verdict, int attempt, String jobId, + String artifactSha256, String failureCode, String detail, + Instant updatedAt) { } +} diff --git a/src/main/java/org/pathlab/forge/evidence/QualificationRunStore.java b/src/main/java/org/pathlab/forge/evidence/QualificationRunStore.java new file mode 100644 index 0000000..8932029 --- /dev/null +++ b/src/main/java/org/pathlab/forge/evidence/QualificationRunStore.java @@ -0,0 +1,415 @@ +package org.pathlab.forge.evidence; + +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.time.Instant; +import java.util.ArrayList; +import java.util.HexFormat; +import java.util.List; +import java.util.Optional; + +/** Durable parent coordinator. It schedules children but never consumes an execution lane. */ +public final class QualificationRunStore implements AutoCloseable { + private static final ObjectMapper JSON = new ObjectMapper(); + private final Connection connection; + private final Path stateRoot; + + public QualificationRunStore(Path database, Path stateRoot) throws IOException { + this.stateRoot = stateRoot.toAbsolutePath().normalize(); + try { + connection = DriverManager.getConnection("jdbc:sqlite:" + database.toAbsolutePath().normalize()); + try (var statement = connection.createStatement()) { + statement.execute("PRAGMA journal_mode=WAL"); + statement.execute("PRAGMA synchronous=FULL"); + statement.execute("PRAGMA busy_timeout=5000"); + statement.execute(""" + CREATE TABLE IF NOT EXISTS qualification_runs ( + id TEXT PRIMARY KEY, manifest_path TEXT NOT NULL, manifest_sha256 TEXT NOT NULL, + state TEXT NOT NULL, cancel_requested INTEGER NOT NULL, + campaign_completed INTEGER NOT NULL, campaign_target_met INTEGER NOT NULL, + created_at TEXT NOT NULL, updated_at TEXT NOT NULL + )"""); + statement.execute(""" + CREATE TABLE IF NOT EXISTS qualification_run_tracks ( + run_id TEXT NOT NULL, track_id TEXT NOT NULL, candidate_id TEXT NOT NULL, + capability TEXT NOT NULL, scope TEXT NOT NULL, request_path TEXT NOT NULL, + remediation_request_path TEXT NOT NULL, expected_attestation_path TEXT NOT NULL, + protocol_sha256 TEXT NOT NULL, dependencies_json TEXT NOT NULL, required INTEGER NOT NULL, + state TEXT NOT NULL, verdict TEXT NOT NULL, attempt INTEGER NOT NULL, job_id TEXT NOT NULL, + artifact_sha256 TEXT NOT NULL, failure_code TEXT NOT NULL, detail TEXT NOT NULL, + updated_at TEXT NOT NULL, PRIMARY KEY(run_id,track_id), + FOREIGN KEY(run_id) REFERENCES qualification_runs(id) + )"""); + statement.execute("CREATE INDEX IF NOT EXISTS ix_qualification_tracks_run_state ON qualification_run_tracks(run_id,state,track_id)"); + } + } catch (SQLException error) { + throw new IOException("Unable to open qualification campaign store", error); + } + } + + public synchronized QualificationRunSnapshot create(QualificationCampaignManifest manifest, Instant now) + throws IOException { + try { + begin(); + try (var statement = connection.prepareStatement(""" + INSERT INTO qualification_runs + (id,manifest_path,manifest_sha256,state,cancel_requested,campaign_completed, + campaign_target_met,created_at,updated_at) VALUES(?,?,?,?,0,0,0,?,?) + """)) { + statement.setString(1, manifest.campaignId()); + statement.setString(2, manifest.path().toString()); + statement.setString(3, manifest.sha256()); + statement.setString(4, "running"); + statement.setString(5, now.toString()); + statement.setString(6, now.toString()); + statement.executeUpdate(); + } + for (var track : manifest.tracks()) insertTrack(manifest.campaignId(), track, now); + commit(); + return snapshot(manifest.campaignId()).orElseThrow(); + } catch (SQLException error) { + rollbackQuietly(); + if (error.getErrorCode() == 19) throw new IllegalArgumentException("Qualification campaign already exists", error); + throw new IOException("Unable to create qualification campaign", error); + } + } + + public synchronized Optional snapshot(String id) throws IOException { + try (var statement = connection.prepareStatement("SELECT * FROM qualification_runs WHERE id=?")) { + statement.setString(1, id); + try (var rows = statement.executeQuery()) { + if (!rows.next()) return Optional.empty(); + return Optional.of(readRun(rows, tracks(id))); + } + } catch (SQLException error) { + throw new IOException("Unable to read qualification campaign", error); + } + } + + public synchronized List list(int limit) throws IOException { + if (limit < 1 || limit > 100) throw new IllegalArgumentException("Qualification campaign limit is invalid"); + var result = new ArrayList(); + try (var statement = connection.prepareStatement( + "SELECT * FROM qualification_runs ORDER BY updated_at DESC,id LIMIT ?")) { + statement.setInt(1, limit); + try (var rows = statement.executeQuery()) { + while (rows.next()) result.add(readRun(rows, tracks(rows.getString("id")))); + } + } catch (SQLException error) { + throw new IOException("Unable to list qualification campaigns", error); + } + return List.copyOf(result); + } + + public synchronized void requestCancel(String id, EvidenceJobQueue queue, Instant now) throws IOException { + var run = snapshot(id).orElseThrow(() -> new IllegalArgumentException("Qualification campaign was not found")); + if (run.campaignCompleted()) return; + execute("UPDATE qualification_runs SET cancel_requested=1,state='cancelling',updated_at=? WHERE id=?", + now.toString(), id); + for (var track : run.tracks()) { + if (!track.jobId().isBlank()) { + var job = queue.snapshot(track.jobId()); + if (job.isPresent() && !job.get().state().terminal()) queue.requestCancel(track.jobId(), now); + } + if (track.verdict().isBlank()) terminalTrack(id, track.id(), "not_evaluable", + "CAMPAIGN_CANCELLED", "Cancelled by the local operator", now); + } + finalizeRun(id, now); + } + + public synchronized QualificationRunSnapshot resume(String id, Instant now) throws IOException { + var run = snapshot(id).orElseThrow(() -> new IllegalArgumentException("Qualification campaign was not found")); + if (run.campaignCompleted()) throw new IllegalStateException("Qualification campaign is already terminal"); + execute("UPDATE qualification_runs SET state='running',cancel_requested=0,updated_at=? WHERE id=?", + now.toString(), id); + execute("UPDATE qualification_run_tracks SET state='pending',updated_at=? WHERE run_id=? AND state='waiting_attestation'", + now.toString(), id); + return snapshot(id).orElseThrow(); + } + + public synchronized QualificationRunSnapshot attachAttestation(String id, String trackId, Path report, + Instant now) throws IOException { + var manifest = loadManifest(id); + var track = manifest.tracks().stream().filter(item -> item.id().equals(trackId)).findFirst() + .orElseThrow(() -> new IllegalArgumentException("Qualification track was not found")); + var validated = new QualificationReportValidator(stateRoot).validate(report, manifest, track); + if ("qualified".equals(validated.verdict()) && !"local-benchmark".equals(track.scope())) { + var request = JSON.readTree(track.requestPath().toFile()); + var packPath = Path.of(request.path("packManifest").asText()).toAbsolutePath().normalize(); + var pack = EvidencePackManifest.load(packPath); + if (!pack.sha256().equals(validated.packManifestSha256())) { + throw new IllegalArgumentException("Qualification report pack checksum changed"); + } + new QualifiedPackRegistry(stateRoot).register(pack, track.protocolSha256(), + validated.manifestSha256(), now); + } + terminalTrack(id, trackId, validated.verdict(), "", + "Signed qualification attestation accepted", now, validated.manifestSha256()); + finalizeRun(id, now); + return snapshot(id).orElseThrow(); + } + + /** Advances every parent deterministically; active scientific work remains in child jobs. */ + public synchronized void tick(EvidenceJobQueue queue, Instant now) throws IOException { + for (var run : list(100)) { + if (run.campaignCompleted()) continue; + QualificationCampaignManifest manifest; + try { + manifest = loadManifest(run.id()); + } catch (IllegalArgumentException error) { + failRemaining(run.id(), "CAMPAIGN_MANIFEST_CHANGED", error.getMessage(), now); + finalizeRun(run.id(), now); + continue; + } + for (var track : snapshot(run.id()).orElseThrow().tracks()) { + if (!track.verdict().isBlank()) continue; + var declared = manifest.tracks().stream().filter(item -> item.id().equals(track.id())).findFirst().orElseThrow(); + if ("pending".equals(track.state())) { + var dependencyVerdicts = dependencyVerdicts(run.id(), declared.dependsOn()); + if (dependencyVerdicts.stream().anyMatch(value -> !value.isBlank() && !"qualified".equals(value))) { + terminalTrack(run.id(), track.id(), "not_evaluable", "DEPENDENCY_NOT_QUALIFIED", + "A frozen dependency did not qualify", now); + } else if (dependencyVerdicts.stream().allMatch("qualified"::equals)) { + submitChild(run.id(), declared, track.attempt(), queue, now); + } + continue; + } + if ("waiting_attestation".equals(track.state())) { + if (declared.expectedAttestationPath() != null && Files.isRegularFile(declared.expectedAttestationPath())) { + try { + attachAttestation(run.id(), track.id(), declared.expectedAttestationPath(), now); + } catch (IllegalArgumentException invalid) { + remediateOrFinish(run.id(), declared, track, "ATTESTATION_INVALID", invalid.getMessage(), now); + } + } + continue; + } + if (!track.jobId().isBlank()) { + var job = queue.snapshot(track.jobId()); + if (job.isPresent() && job.get().state().terminal()) { + handleTerminalJob(run.id(), declared, track, job.get(), now); + } + } + } + finalizeRun(run.id(), now); + } + } + + private void handleTerminalJob(String runId, QualificationCampaignManifest.Track declared, + QualificationRunSnapshot.Track track, EvidenceJobSnapshot job, Instant now) throws IOException { + switch (job.state()) { + case COMPLETED -> { + if (declared.expectedAttestationPath() != null && Files.isRegularFile(declared.expectedAttestationPath())) { + try { + attachAttestation(runId, track.id(), declared.expectedAttestationPath(), now); + } catch (IllegalArgumentException invalid) { + remediateOrFinish(runId, declared, track, "ATTESTATION_INVALID", invalid.getMessage(), now); + } + } else { + updateTrack(runId, track.id(), "waiting_attestation", track.attempt(), track.jobId(), "", + "", "Child completed; awaiting a signed qualification attestation", now); + execute("UPDATE qualification_runs SET state='waiting_attestation',updated_at=? WHERE id=?", + now.toString(), runId); + } + } + case UNSUPPORTED -> terminalTrack(runId, track.id(), "unsupported", + blankCode(job.failureCode(), "CAPABILITY_UNSUPPORTED"), job.detail(), now); + case ABSTAINED -> terminalTrack(runId, track.id(), "not_evaluable", + blankCode(job.failureCode(), "CHILD_ABSTAINED"), job.detail(), now); + case FAILED, CANCELLED -> remediateOrFinish(runId, declared, track, + blankCode(job.failureCode(), "CHILD_FAILED"), job.detail(), now); + default -> { } + } + } + + private void remediateOrFinish(String runId, QualificationCampaignManifest.Track declared, + QualificationRunSnapshot.Track current, String code, String detail, Instant now) throws IOException { + if (current.attempt() == 0 && declared.remediationRequestPath() != null) { + execute(""" + UPDATE qualification_run_tracks SET request_path=?,state='pending',attempt=1,job_id='', + failure_code=?,detail=?,updated_at=? WHERE run_id=? AND track_id=? + """, declared.remediationRequestPath().toString(), code, + "Bounded remediation scheduled: " + bounded(detail), now.toString(), runId, current.id()); + execute("UPDATE qualification_runs SET state='remediating',updated_at=? WHERE id=?", + now.toString(), runId); + } else { + terminalTrack(runId, current.id(), "experimental", code, bounded(detail), now); + } + } + + private void submitChild(String runId, QualificationCampaignManifest.Track track, int attempt, + EvidenceJobQueue queue, Instant now) throws IOException { + var request = attempt == 0 ? track.requestPath() : track.remediationRequestPath(); + var jobId = childId(runId, track.id(), attempt); + var plan = EvidenceJobProcessor.executionPlan(request, jobId, stateRoot); + queue.submit(jobId, request, plan.lane(), plan.packSha256(), now); + updateTrack(runId, track.id(), "running", attempt, jobId, "", "", + "Child job accepted by the autonomous runner", now); + } + + private QualificationCampaignManifest loadManifest(String id) throws IOException { + try (var statement = connection.prepareStatement( + "SELECT manifest_path,manifest_sha256 FROM qualification_runs WHERE id=?")) { + statement.setString(1, id); + try (var rows = statement.executeQuery()) { + if (!rows.next()) throw new IllegalArgumentException("Qualification campaign was not found"); + var manifest = QualificationCampaignManifest.load(Path.of(rows.getString(1))); + if (!manifest.sha256().equals(rows.getString(2))) { + throw new IllegalArgumentException("Qualification campaign checksum changed"); + } + return manifest; + } + } catch (SQLException error) { + throw new IOException("Unable to load qualification campaign manifest", error); + } + } + + private List dependencyVerdicts(String runId, List dependencies) throws IOException { + var result = new ArrayList(); + for (var dependency : dependencies) { + try (var statement = connection.prepareStatement( + "SELECT verdict FROM qualification_run_tracks WHERE run_id=? AND track_id=?")) { + statement.setString(1, runId); + statement.setString(2, dependency); + try (var rows = statement.executeQuery()) { + result.add(rows.next() ? rows.getString(1) : "not_evaluable"); + } + } catch (SQLException error) { + throw new IOException("Unable to read qualification dependency", error); + } + } + return result; + } + + private void finalizeRun(String id, Instant now) throws IOException { + var tracks = tracks(id); + if (tracks.isEmpty() || tracks.stream().anyMatch(track -> track.verdict().isBlank())) return; + var target = tracks.stream().allMatch(track -> "qualified".equals(track.verdict())); + execute(""" + UPDATE qualification_runs SET state='completed',campaign_completed=1, + campaign_target_met=?,updated_at=? WHERE id=? + """, target ? "1" : "0", now.toString(), id); + new CapabilityMatrixWriter(stateRoot).write(snapshot(id).orElseThrow(), now); + } + + private void failRemaining(String id, String code, String detail, Instant now) throws IOException { + execute(""" + UPDATE qualification_run_tracks SET state='terminal',verdict='not_evaluable', + failure_code=?,detail=?,updated_at=? WHERE run_id=? AND verdict='' + """, code, bounded(detail), now.toString(), id); + } + + private void terminalTrack(String runId, String trackId, String verdict, String failureCode, + String detail, Instant now) throws IOException { + terminalTrack(runId, trackId, verdict, failureCode, detail, now, ""); + } + + private void terminalTrack(String runId, String trackId, String verdict, String failureCode, + String detail, Instant now, String artifactSha) throws IOException { + if (!java.util.Set.of("qualified", "experimental", "unsupported", "not_evaluable").contains(verdict)) { + throw new IllegalArgumentException("Qualification terminal verdict is invalid"); + } + execute(""" + UPDATE qualification_run_tracks SET state='terminal',verdict=?,artifact_sha256=?, + failure_code=?,detail=?,updated_at=? WHERE run_id=? AND track_id=? + """, verdict, artifactSha, failureCode, bounded(detail), now.toString(), runId, trackId); + } + + private void updateTrack(String runId, String trackId, String state, int attempt, String jobId, + String artifactSha, String failureCode, String detail, Instant now) throws IOException { + execute(""" + UPDATE qualification_run_tracks SET state=?,attempt=?,job_id=?,artifact_sha256=?, + failure_code=?,detail=?,updated_at=? WHERE run_id=? AND track_id=? + """, state, Integer.toString(attempt), jobId, artifactSha, failureCode, + bounded(detail), now.toString(), runId, trackId); + } + + private void insertTrack(String runId, QualificationCampaignManifest.Track track, Instant now) + throws SQLException, IOException { + try (var statement = connection.prepareStatement(""" + INSERT INTO qualification_run_tracks + (run_id,track_id,candidate_id,capability,scope,request_path,remediation_request_path, + expected_attestation_path,protocol_sha256,dependencies_json,required,state,verdict, + attempt,job_id,artifact_sha256,failure_code,detail,updated_at) + VALUES(?,?,?,?,?,?,?,?,?,?,?,'pending','',0,'','','','Waiting for frozen dependencies',?) + """)) { + statement.setString(1, runId); + statement.setString(2, track.id()); + statement.setString(3, track.candidateId()); + statement.setString(4, track.capability()); + statement.setString(5, track.scope()); + statement.setString(6, track.requestPath().toString()); + statement.setString(7, track.remediationRequestPath() == null ? "" : track.remediationRequestPath().toString()); + statement.setString(8, track.expectedAttestationPath() == null ? "" : track.expectedAttestationPath().toString()); + statement.setString(9, track.protocolSha256()); + statement.setString(10, JSON.writeValueAsString(track.dependsOn())); + statement.setInt(11, track.required() ? 1 : 0); + statement.setString(12, now.toString()); + statement.executeUpdate(); + } + } + + private List tracks(String id) throws IOException { + var result = new ArrayList(); + try (var statement = connection.prepareStatement( + "SELECT * FROM qualification_run_tracks WHERE run_id=? ORDER BY track_id")) { + statement.setString(1, id); + try (var rows = statement.executeQuery()) { + while (rows.next()) result.add(new QualificationRunSnapshot.Track( + rows.getString("track_id"), rows.getString("candidate_id"), + rows.getString("capability"), rows.getString("scope"), rows.getString("state"), + rows.getString("verdict"), rows.getInt("attempt"), rows.getString("job_id"), + rows.getString("artifact_sha256"), rows.getString("failure_code"), + rows.getString("detail"), Instant.parse(rows.getString("updated_at")))); + } + } catch (SQLException error) { + throw new IOException("Unable to read qualification campaign tracks", error); + } + return List.copyOf(result); + } + + private static QualificationRunSnapshot readRun(ResultSet rows, + List tracks) throws SQLException { + return new QualificationRunSnapshot(rows.getString("id"), rows.getString("manifest_sha256"), + rows.getString("state"), rows.getInt("campaign_completed") != 0, + rows.getInt("campaign_target_met") != 0, rows.getInt("cancel_requested") != 0, + tracks, Instant.parse(rows.getString("created_at")), Instant.parse(rows.getString("updated_at"))); + } + + private static String childId(String runId, String trackId, int attempt) { + try { + var bytes = MessageDigest.getInstance("SHA-256") + .digest((runId + "\n" + trackId + "\n" + attempt).getBytes(StandardCharsets.UTF_8)); + return "qualification-" + HexFormat.of().formatHex(bytes).substring(0, 32); + } catch (java.security.GeneralSecurityException impossible) { + throw new IllegalStateException("SHA-256 is unavailable", impossible); + } + } + + private void execute(String sql, String... values) throws IOException { + try (var statement = connection.prepareStatement(sql)) { + for (var index = 0; index < values.length; index++) statement.setString(index + 1, values[index]); + statement.executeUpdate(); + } catch (SQLException error) { + throw new IOException("Unable to update qualification campaign", error); + } + } + + private void begin() throws SQLException { try (var statement = connection.createStatement()) { statement.execute("BEGIN IMMEDIATE"); } } + private void commit() throws SQLException { try (var statement = connection.createStatement()) { statement.execute("COMMIT"); } } + private void rollbackQuietly() { try (var statement = connection.createStatement()) { statement.execute("ROLLBACK"); } catch (SQLException ignored) { } } + private static String blankCode(String value, String fallback) { return value == null || value.isBlank() ? fallback : value; } + private static String bounded(String value) { if (value == null || value.isBlank()) return "No additional detail"; return value.length() <= 500 ? value : value.substring(0, 500); } + + @Override public synchronized void close() throws IOException { + try { connection.close(); } catch (SQLException error) { throw new IOException("Unable to close qualification campaign store", error); } + } +} diff --git a/src/main/java/org/pathlab/forge/evidence/QualifiedPackRegistry.java b/src/main/java/org/pathlab/forge/evidence/QualifiedPackRegistry.java new file mode 100644 index 0000000..4b15047 --- /dev/null +++ b/src/main/java/org/pathlab/forge/evidence/QualifiedPackRegistry.java @@ -0,0 +1,99 @@ +package org.pathlab.forge.evidence; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import java.io.IOException; +import java.nio.file.AtomicMoveNotSupportedException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.time.Instant; +import java.util.Set; + +/** Local fail-closed activation registry, separate from mutable candidate manifests. */ +public final class QualifiedPackRegistry { + private static final ObjectMapper JSON = new ObjectMapper() + .enable(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS); + private final Path path; + + public QualifiedPackRegistry(Path stateRoot) { + path = stateRoot.toAbsolutePath().normalize().resolve("trust/qualified-packs.json"); + } + + public synchronized void register(EvidencePackManifest pack, String protocolSha256, + String attestationSha256, Instant acceptedAt) throws IOException { + require(EvidencePackManifest.SCHEMA_V2.equals(pack.schema()), "Only v2 packs use qualification registry"); + require(pack.activationEligible() && !"local-benchmark".equals(pack.scope()), + "Pack scope is not activation eligible"); + require(protocolSha256.matches("[a-f0-9]{64}") && attestationSha256.matches("[a-f0-9]{64}"), + "Qualification registry hashes are invalid"); + var root = load(); + var entries = (ArrayNode) root.path("entries"); + for (var item : entries) { + if (pack.sha256().equals(item.path("packManifestSha256").asText())) { + require(attestationSha256.equals(item.path("attestationSha256").asText()) + && protocolSha256.equals(item.path("protocolSha256").asText()), + "Pack checksum is already registered with different qualification evidence"); + return; + } + } + var entry = entries.addObject(); + entry.put("packId", pack.packId()); + entry.put("packVersion", pack.version()); + entry.put("packManifestSha256", pack.sha256()); + entry.put("protocolSha256", protocolSha256); + entry.put("attestationSha256", attestationSha256); + entry.put("scope", pack.scope()); + entry.put("acceptedAt", acceptedAt.toString()); + entry.set("artifactSha256s", JSON.valueToTree(pack.artifacts().stream() + .map(EvidencePackManifest.Artifact::sha256).toList())); + write(root); + } + + public synchronized void requireQualified(EvidencePackManifest pack, String attestationSha256) + throws IOException { + require(attestationSha256 != null && attestationSha256.matches("[a-f0-9]{64}"), + "Qualified pack attestation checksum is required"); + for (var item : load().path("entries")) { + if (pack.packId().equals(item.path("packId").asText()) + && pack.version().equals(item.path("packVersion").asText()) + && pack.sha256().equals(item.path("packManifestSha256").asText()) + && attestationSha256.equals(item.path("attestationSha256").asText()) + && pack.scope().equals(item.path("scope").asText()) + && item.path("artifactSha256s").equals(JSON.valueToTree(pack.artifacts().stream() + .map(EvidencePackManifest.Artifact::sha256).toList()))) return; + } + throw new IllegalArgumentException("AI pack has no matching qualified-registry entry"); + } + + private ObjectNode load() throws IOException { + if (!Files.isRegularFile(path)) { + var root = JSON.createObjectNode(); + root.put("schema", "pathlab.qualified-pack-registry/1"); + root.set("entries", JSON.createArrayNode()); + return root; + } + var root = JSON.readTree(path.toFile()); + require(root.isObject() && root.size() == 2 + && "pathlab.qualified-pack-registry/1".equals(root.path("schema").asText()) + && root.path("entries").isArray(), "Qualified pack registry is invalid"); + return (ObjectNode) root; + } + + private void write(ObjectNode root) throws IOException { + Files.createDirectories(path.getParent()); + var partial = path.resolveSibling(path.getFileName() + ".partial"); + JSON.writerWithDefaultPrettyPrinter().writeValue(partial.toFile(), root); + try { + Files.move(partial, path, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE); + } catch (AtomicMoveNotSupportedException ignored) { + Files.move(partial, path, StandardCopyOption.REPLACE_EXISTING); + } + } + + private static void require(boolean condition, String message) { + if (!condition) throw new IllegalArgumentException(message); + } +} diff --git a/src/main/java/org/pathlab/forge/evidence/SpecialStainAnalyzer.java b/src/main/java/org/pathlab/forge/evidence/SpecialStainAnalyzer.java new file mode 100644 index 0000000..3a7638f --- /dev/null +++ b/src/main/java/org/pathlab/forge/evidence/SpecialStainAnalyzer.java @@ -0,0 +1,67 @@ +package org.pathlab.forge.evidence; + +import java.awt.image.BufferedImage; +import java.util.Map; +import java.util.Set; + +/** Descriptive color/OD measurements for supported brightfield special stains and cytology. */ +public final class SpecialStainAnalyzer { + private static final Set SUPPORTED = Set.of( + "pas", "pas_d", "trichrome", "gms", "afb", "papanicolaou", "generic_brightfield"); + private SpecialStainAnalyzer() { } + + public static Result analyze(BufferedImage image, String requestedStain) { + if (image == null || image.getWidth() < 1 || image.getHeight() < 1) { + throw new IllegalArgumentException("Special-stain image geometry is invalid"); + } + var stain = requestedStain == null ? "generic_brightfield" : requestedStain.toLowerCase(); + var fallback = !SUPPORTED.contains(stain); + if (fallback) stain = "generic_brightfield"; + var stained = 0; + var tissue = 0; + var saturated = 0; + double od = 0; + for (var y = 0; y < image.getHeight(); y++) for (var x = 0; x < image.getWidth(); x++) { + var rgb = image.getRGB(x, y); + var red = rgb >>> 16 & 0xff; + var green = rgb >>> 8 & 0xff; + var blue = rgb & 0xff; + if ((red + green + blue) / 3.0 < 235) tissue++; + if (red < 3 || green < 3 || blue < 3) saturated++; + if (matches(stain, red, green, blue)) { + stained++; + var luminance = 0.2126 * red + 0.7152 * green + 0.0722 * blue; + od += -Math.log((luminance + 1) / 256.0); + } + } + var reasons = new java.util.ArrayList(); + if (tissue < image.getWidth() * image.getHeight() * 0.05) reasons.add("insufficient_tissue"); + if (saturated > image.getWidth() * image.getHeight() * 0.35) reasons.add("excessive_saturation"); + var measurement = Map.of( + "stainAreaFraction", tissue == 0 ? 0 : (double) stained / tissue, + "meanStainOd", stained == 0 ? 0 : od / stained, + "tissuePixelCount", tissue, + "stainedPixelCount", stained); + return new Result(stain, fallback ? "generic-descriptive" : mode(stain), measurement, + fallback ? "UNSUPPORTED_STAIN_GENERIC_FALLBACK" : reasons.isEmpty() ? null : "STAIN_QC_FAILED", + java.util.List.copyOf(reasons), reasons.isEmpty() ? 0.15 : 1.0); + } + + private static boolean matches(String stain, int red, int green, int blue) { + return switch (stain) { + case "pas", "pas_d" -> red - green >= 30 && blue - green >= 15; + case "trichrome" -> blue - red >= 20 || green - red >= 20; + case "gms" -> red < 80 && green < 80 && blue < 80; + case "afb" -> red - green >= 50 && red - blue >= 40; + case "papanicolaou" -> Math.max(red, Math.max(green, blue)) - Math.min(red, Math.min(green, blue)) >= 35; + default -> red + green + blue < 600; + }; + } + + private static String mode(String stain) { + return "papanicolaou".equals(stain) ? "cytology-descriptive" : "special-stain-descriptive"; + } + + public record Result(String stainId, String analysisMode, Map measurements, + String abstentionReason, java.util.List qc, double uncertainty) { } +} diff --git a/src/main/java/org/pathlab/forge/evidence/TrustedSignerRegistry.java b/src/main/java/org/pathlab/forge/evidence/TrustedSignerRegistry.java new file mode 100644 index 0000000..71e6cf6 --- /dev/null +++ b/src/main/java/org/pathlab/forge/evidence/TrustedSignerRegistry.java @@ -0,0 +1,103 @@ +package org.pathlab.forge.evidence; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import java.io.IOException; +import java.nio.file.AtomicMoveNotSupportedException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.security.MessageDigest; +import java.util.Base64; +import java.util.Optional; + +/** ACL-protected signer registry; payload-embedded public keys are never trust anchors. */ +public final class TrustedSignerRegistry { + private static final ObjectMapper JSON = new ObjectMapper() + .enable(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS); + private final Path path; + + public TrustedSignerRegistry(Path stateRoot) { + path = stateRoot.toAbsolutePath().normalize().resolve("trust/approved-signers.json"); + } + + public synchronized void pin(String keyId, String publicKeyDer, String allowedUse) throws IOException { + require(keyId != null && keyId.matches("[a-f0-9]{64}"), "Trusted signer id is invalid"); + require(SetHolder.USES.contains(allowedUse), "Trusted signer use is invalid"); + byte[] publicBytes; + try { + publicBytes = Base64.getUrlDecoder().decode(publicKeyDer); + } catch (IllegalArgumentException error) { + throw new IllegalArgumentException("Trusted signer key is invalid", error); + } + require(EvidenceBundleWriter.sha256(publicBytes).equals(keyId), + "Trusted signer key does not match its id"); + var root = load(); + var keys = (ArrayNode) root.path("keys"); + for (var item : keys) { + if (keyId.equals(item.path("keyId").asText()) && allowedUse.equals(item.path("allowedUse").asText())) { + require(publicKeyDer.equals(item.path("publicKeyDer").asText()), + "Trusted signer id is already pinned to a different key"); + return; + } + } + var item = keys.addObject(); + item.put("keyId", keyId); + item.put("publicKeyDer", publicKeyDer); + item.put("allowedUse", allowedUse); + write(root); + } + + public synchronized Optional find(String keyId, String allowedUse) throws IOException { + var root = load(); + for (var item : root.path("keys")) { + if (keyId.equals(item.path("keyId").asText()) && allowedUse.equals(item.path("allowedUse").asText())) { + try { + var value = Base64.getUrlDecoder().decode(item.path("publicKeyDer").asText()); + if (MessageDigest.isEqual( + EvidenceBundleWriter.sha256(value).getBytes(java.nio.charset.StandardCharsets.US_ASCII), + keyId.getBytes(java.nio.charset.StandardCharsets.US_ASCII))) return Optional.of(value); + } catch (IllegalArgumentException ignored) { + return Optional.empty(); + } + } + } + return Optional.empty(); + } + + private ObjectNode load() throws IOException { + if (!Files.isRegularFile(path)) { + var root = JSON.createObjectNode(); + root.put("schema", "pathlab.trusted-signers/1"); + root.set("keys", JSON.createArrayNode()); + return root; + } + var root = JSON.readTree(path.toFile()); + require(root.isObject() && "pathlab.trusted-signers/1".equals(root.path("schema").asText()) + && root.path("keys").isArray(), "Trusted signer registry is invalid"); + var copy = (ObjectNode) root; + require(copy.size() == 2, "Trusted signer registry fields are invalid"); + return copy; + } + + private void write(ObjectNode value) throws IOException { + Files.createDirectories(path.getParent()); + var partial = path.resolveSibling(path.getFileName() + ".partial"); + JSON.writerWithDefaultPrettyPrinter().writeValue(partial.toFile(), value); + try { + Files.move(partial, path, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE); + } catch (AtomicMoveNotSupportedException ignored) { + Files.move(partial, path, StandardCopyOption.REPLACE_EXISTING); + } + } + + private static void require(boolean condition, String message) { + if (!condition) throw new IllegalArgumentException(message); + } + + private static final class SetHolder { + private static final java.util.Set USES = java.util.Set.of("qualification", "evidence-set", "evidence"); + } +} diff --git a/src/main/resources/evidence-dashboard/index.html b/src/main/resources/evidence-dashboard/index.html index 4b027b5..d30b089 100644 --- a/src/main/resources/evidence-dashboard/index.html +++ b/src/main/resources/evidence-dashboard/index.html @@ -11,17 +11,21 @@
Offline analysis operations

PathLab Evidence Mentor

Authenticating…
+

Qualification campaigns

Jobs

diff --git a/src/main/resources/evidence-packs/cell-od-watershed-v2.json b/src/main/resources/evidence-packs/cell-od-watershed-v2.json new file mode 100644 index 0000000..da308f0 --- /dev/null +++ b/src/main/resources/evidence-packs/cell-od-watershed-v2.json @@ -0,0 +1 @@ +{"schema":"pathlab.ai-pack/2","packId":"cell-od-watershed-v2","version":"2.0.0","capability":"cell-morphology","scope":"deployment","activationEligible":true,"acceptedStains":["he","ihc_dab","papanicolaou"],"preprocessing":{"id":"od-watershed-v2","tilePixels":512},"artifacts":[],"runtimeCompatibility":{"workerProtocol":"pathlab.model-worker-result/2","executionProvider":"cpu","cuda":"none","gpuArchitecture":"none","requiresExternalWorker":false,"segmentMaxSeconds":120},"licenseLedger":[{"component":"PathLab OD watershed","kind":"code","license":"PathLab repository license","revision":"2.0.0","permittedUse":"private-research","redistributable":false,"derivativesAllowed":true}],"derivativePermissions":{"atlasResearch":true,"atlasClean":true,"reviewedAt":"2026-08-22T00:00:00Z"},"resourceEnvelope":{"lane":"cpu_io","maxRamMiB":1024,"maxVramMiB":0,"maxSeconds":120,"network":false},"qualificationPolicy":{"requiresSignedAttestation":true,"protocolSha256":"dd251775f0abe6a780b2389ee489f18bcec78b9cb80e1facfd7134a05cb0c2a0"},"outputSchema":"pathlab.model-qualification-report/2","markers":[]} diff --git a/src/main/resources/evidence-packs/cytology-descriptive-v2.json b/src/main/resources/evidence-packs/cytology-descriptive-v2.json new file mode 100644 index 0000000..6e75bb3 --- /dev/null +++ b/src/main/resources/evidence-packs/cytology-descriptive-v2.json @@ -0,0 +1 @@ +{"schema":"pathlab.ai-pack/2","packId":"cytology-descriptive-v2","version":"2.0.0","capability":"cytology-descriptive","scope":"deployment","activationEligible":true,"acceptedStains":["papanicolaou","generic_brightfield"],"preprocessing":{"id":"papanicolaou-descriptive-v2","tilePixels":512},"artifacts":[],"runtimeCompatibility":{"workerProtocol":"pathlab.model-worker-result/2","executionProvider":"cpu","cuda":"none","gpuArchitecture":"none","requiresExternalWorker":false,"segmentMaxSeconds":120},"licenseLedger":[{"component":"PathLab cytology descriptors","kind":"code","license":"PathLab repository license","revision":"2.0.0","permittedUse":"private-research","redistributable":false,"derivativesAllowed":true}],"derivativePermissions":{"atlasResearch":true,"atlasClean":true,"reviewedAt":"2026-08-22T00:00:00Z"},"resourceEnvelope":{"lane":"cpu_io","maxRamMiB":1024,"maxVramMiB":0,"maxSeconds":120,"network":false},"qualificationPolicy":{"requiresSignedAttestation":true,"protocolSha256":"b354e6c605afc55b5663167453fa2b23738f311d48de5f30293cef0ba26f962a"},"outputSchema":"pathlab.model-qualification-report/2","markers":[]} diff --git a/src/main/resources/evidence-packs/ihc-descriptive-v2.json b/src/main/resources/evidence-packs/ihc-descriptive-v2.json new file mode 100644 index 0000000..61c5d40 --- /dev/null +++ b/src/main/resources/evidence-packs/ihc-descriptive-v2.json @@ -0,0 +1 @@ +{"schema":"pathlab.ai-pack/2","packId":"ihc-descriptive-v2","version":"2.0.0","capability":"ihc-descriptive","scope":"deployment","activationEligible":true,"acceptedStains":["ihc_dab"],"preprocessing":{"id":"hdab-relative-v2","tilePixels":512},"artifacts":[],"runtimeCompatibility":{"workerProtocol":"pathlab.model-worker-result/2","executionProvider":"cpu","cuda":"none","gpuArchitecture":"none","requiresExternalWorker":false,"segmentMaxSeconds":120},"licenseLedger":[{"component":"PathLab deterministic H-DAB measurement","kind":"code","license":"PathLab repository license","revision":"2.0.0","permittedUse":"private-research","redistributable":false,"derivativesAllowed":true}],"derivativePermissions":{"atlasResearch":true,"atlasClean":true,"reviewedAt":"2026-08-22T00:00:00Z"},"resourceEnvelope":{"lane":"cpu_io","maxRamMiB":2048,"maxVramMiB":0,"maxSeconds":120,"network":false},"qualificationPolicy":{"requiresSignedAttestation":true,"protocolSha256":"9b2d3299eed5504fe7d89839cacae6ea5232762d6405bdfab261ad78e30b991f"},"outputSchema":"pathlab.model-qualification-report/2","markers":["generic","er","pr","ki-67","her2","pd-l1"]} diff --git a/src/main/resources/evidence-packs/special-stain-descriptive-v2.json b/src/main/resources/evidence-packs/special-stain-descriptive-v2.json new file mode 100644 index 0000000..b2bb4bb --- /dev/null +++ b/src/main/resources/evidence-packs/special-stain-descriptive-v2.json @@ -0,0 +1 @@ +{"schema":"pathlab.ai-pack/2","packId":"special-stain-descriptive-v2","version":"2.0.0","capability":"special-stain-descriptive","scope":"deployment","activationEligible":true,"acceptedStains":["pas","pas_d","trichrome","gms","afb","generic_brightfield"],"preprocessing":{"id":"brightfield-stain-v2","tilePixels":512},"artifacts":[],"runtimeCompatibility":{"workerProtocol":"pathlab.model-worker-result/2","executionProvider":"cpu","cuda":"none","gpuArchitecture":"none","requiresExternalWorker":false,"segmentMaxSeconds":120},"licenseLedger":[{"component":"PathLab special stain descriptors","kind":"code","license":"PathLab repository license","revision":"2.0.0","permittedUse":"private-research","redistributable":false,"derivativesAllowed":true}],"derivativePermissions":{"atlasResearch":true,"atlasClean":true,"reviewedAt":"2026-08-22T00:00:00Z"},"resourceEnvelope":{"lane":"cpu_io","maxRamMiB":1024,"maxVramMiB":0,"maxSeconds":120,"network":false},"qualificationPolicy":{"requiresSignedAttestation":true,"protocolSha256":"d035efcec9812f16464b4e8e59e9d245884a57698626821ab0cf056d05854cee"},"outputSchema":"pathlab.model-qualification-report/2","markers":[]} diff --git a/src/test/java/org/pathlab/forge/evidence/AllRounderPackManifestTest.java b/src/test/java/org/pathlab/forge/evidence/AllRounderPackManifestTest.java new file mode 100644 index 0000000..4a789c6 --- /dev/null +++ b/src/test/java/org/pathlab/forge/evidence/AllRounderPackManifestTest.java @@ -0,0 +1,32 @@ +package org.pathlab.forge.evidence; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.fasterxml.jackson.databind.ObjectMapper; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.util.HexFormat; +import org.junit.jupiter.api.Test; + +final class AllRounderPackManifestTest { + private static final ObjectMapper JSON = new ObjectMapper(); + + @Test + void deterministicV2CandidatesPinTheirPreregisteredProtocols() throws Exception { + for (var id : java.util.List.of("cell-od-watershed", "ihc-descriptive", + "special-stain-descriptive", "cytology-descriptive")) { + var packPath = Path.of("src/main/resources/evidence-packs/" + id + "-v2.json"); + var protocolPath = Path.of("qualification-protocols/" + id + "-v2.json"); + var pack = EvidencePackManifest.load(packPath); + var raw = JSON.readTree(packPath.toFile()); + var protocolSha = HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256") + .digest(Files.readAllBytes(protocolPath))); + assertEquals(protocolSha, raw.path("qualificationPolicy").path("protocolSha256").asText()); + assertEquals(EvidencePackManifest.SCHEMA_V2, pack.schema()); + assertTrue(pack.activationEligible()); + assertEquals("deployment", pack.scope()); + } + } +} diff --git a/src/test/java/org/pathlab/forge/evidence/AtlasFeasibilityGateTest.java b/src/test/java/org/pathlab/forge/evidence/AtlasFeasibilityGateTest.java index f0e2f70..06b5f6f 100644 --- a/src/test/java/org/pathlab/forge/evidence/AtlasFeasibilityGateTest.java +++ b/src/test/java/org/pathlab/forge/evidence/AtlasFeasibilityGateTest.java @@ -1,15 +1,26 @@ package org.pathlab.forge.evidence; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.time.Duration; +import java.util.ArrayList; import org.junit.jupiter.api.Test; final class AtlasFeasibilityGateTest { - @Test void continuesOnlyWhenProjectedInsideSevenDaysAndResourcesHold() { - assertEquals("CONTINUE_LOCAL", AtlasFeasibilityGate.decide( - new AtlasFeasibilityGate.Probe(4, 10_000, 1_000, 100_000, 4500, 12000, 0, true)).status()); - assertEquals("LOCAL_INSUFFICIENT", AtlasFeasibilityGate.decide( - new AtlasFeasibilityGate.Probe(4, 10_000, 1_000, 1_000_000, 4500, 12000, 0, true)).status()); - assertEquals("LOCAL_INSUFFICIENT", AtlasFeasibilityGate.decide( - new AtlasFeasibilityGate.Probe(4, 10_000, 1_000, 100_000, 4700, 12000, 0, true)).status()); + @Test + void requiresFourHoursImprovementResourceComplianceAndBoundedQuote() { + var segments = new ArrayList(); + for (var index = 0; index < 16; index++) segments.add(new AtlasFeasibilityGate.Segment( + Duration.ofMinutes(15), 8L * 1024 * 1024 * 1024, + 4L * 1024 * 1024 * 1024, 0.1 + index * 0.01, true)); + var decision = AtlasFeasibilityGate.evaluate(segments, Duration.ofDays(6)); + assertEquals("experimental", decision.status()); + assertEquals("LOCAL_PROBE_PASSED_FULL_QUALIFICATION_PENDING", decision.reason()); + assertFalse(decision.paidComputeQuoteAllowed()); + AtlasFeasibilityGate.requireQuote(10_000, 24); + assertThrows(IllegalArgumentException.class, () -> AtlasFeasibilityGate.requireQuote(10_001, 24)); + assertThrows(IllegalArgumentException.class, () -> AtlasFeasibilityGate.requireQuote(9_000, 16)); } } diff --git a/src/test/java/org/pathlab/forge/evidence/BrightfieldQualificationHarnessTest.java b/src/test/java/org/pathlab/forge/evidence/BrightfieldQualificationHarnessTest.java index 3a79de1..6929d37 100644 --- a/src/test/java/org/pathlab/forge/evidence/BrightfieldQualificationHarnessTest.java +++ b/src/test/java/org/pathlab/forge/evidence/BrightfieldQualificationHarnessTest.java @@ -39,9 +39,10 @@ void writesChecksumBoundSyntheticQualificationReportAtomically() throws Exceptio assertEquals("cell-od-watershed-v1", cell.path("packId").asText()); assertEquals(64, cell.path("packManifestSha256").asText().length()); assertEquals("experimental", cell.path("status").asText()); - assertEquals("fail", check(cell, "touching-nuclei-separation").path("outcome").asText()); - assertEquals(1, check(cell, "touching-nuclei-separation").path("observed").asInt()); + assertEquals("pass", check(cell, "touching-nuclei-separation").path("outcome").asText()); + assertEquals(2, check(cell, "touching-nuclei-separation").path("observed").asInt()); assertEquals(2, check(cell, "touching-nuclei-separation").path("required").asInt()); + assertEquals("pass", check(cell, "instance-mask-output").path("outcome").asText()); var ihc = track(report, "ihc-descriptive"); assertEquals("ihc-descriptive-v1", ihc.path("packId").asText()); diff --git a/src/test/java/org/pathlab/forge/evidence/BrightfieldTileAnalyzerTest.java b/src/test/java/org/pathlab/forge/evidence/BrightfieldTileAnalyzerTest.java index e1fb132..814d8fa 100644 --- a/src/test/java/org/pathlab/forge/evidence/BrightfieldTileAnalyzerTest.java +++ b/src/test/java/org/pathlab/forge/evidence/BrightfieldTileAnalyzerTest.java @@ -32,6 +32,23 @@ void countsBoundedNuclearComponentsAndDescribesDabWithoutClinicalCalls() { assertTrue(result.meanDabOd() > 0); assertEquals("nuclear", result.compartment()); assertTrue(result.researchEstimate()); + assertEquals(2, result.instances().size()); + assertTrue(result.instances().stream().allMatch(instance -> !instance.rle().isEmpty())); + } + + @Test + void markerControlledWatershedSplitsTouchingElongatedNuclearMask() { + var image = new BufferedImage(24, 16, BufferedImage.TYPE_INT_RGB); + var graphics = image.createGraphics(); + graphics.setColor(Color.WHITE); graphics.fillRect(0, 0, 24, 16); + graphics.setColor(new Color(75, 45, 125)); graphics.fillRect(6, 5, 10, 5); + graphics.dispose(); + + var result = BrightfieldTileAnalyzer.analyze(image, "generic"); + + assertEquals(2, result.cellCount()); + assertEquals("cell-1", result.instances().get(0).id()); + assertTrue(result.instances().stream().allMatch(instance -> instance.areaPx2() == 25)); } @Test diff --git a/src/test/java/org/pathlab/forge/evidence/EvidenceArtifactStoreTest.java b/src/test/java/org/pathlab/forge/evidence/EvidenceArtifactStoreTest.java index f87b263..917948a 100644 --- a/src/test/java/org/pathlab/forge/evidence/EvidenceArtifactStoreTest.java +++ b/src/test/java/org/pathlab/forge/evidence/EvidenceArtifactStoreTest.java @@ -5,6 +5,7 @@ import java.nio.file.Files; import java.nio.file.Path; +import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -23,13 +24,43 @@ void selectsOnlyExactSlideRevisionAndTerminalSignedArtifact() throws Exception { temporaryDirectory, "b".repeat(64), "revision-1").isEmpty()); } + @Test + void rejectsTamperingAndUntrustedEmbeddedKeys() throws Exception { + var exact = write("trusted", "a".repeat(64), "revision-1", "completed"); + var json = new ObjectMapper().readTree(exact.toFile()); + ((com.fasterxml.jackson.databind.node.ObjectNode) json.path("source")) + .put("revision", "revision-tampered"); + new ObjectMapper().writeValue(exact.toFile(), json); + assertTrue(EvidenceArtifactStore.find( + temporaryDirectory, "a".repeat(64), "revision-tampered").isEmpty()); + + var foreignRoot = temporaryDirectory.resolve("foreign"); + var foreignPath = foreignRoot.resolve("artifacts/foreign/evidence.json"); + var unsigned = unsigned("a".repeat(64), "revision-2", "completed"); + new EvidenceBundleWriter(foreignRoot.resolve("signing")).write(foreignPath, unsigned); + var copied = temporaryDirectory.resolve("artifacts/foreign/evidence.json"); + Files.createDirectories(copied.getParent()); + Files.copy(foreignPath, copied); + assertTrue(EvidenceArtifactStore.find( + temporaryDirectory, "a".repeat(64), "revision-2").isEmpty()); + } + private Path write(String job, String sha, String revision, String status) throws Exception { var path = temporaryDirectory.resolve("artifacts").resolve(job).resolve("evidence.json"); - Files.createDirectories(path.getParent()); - Files.writeString(path, "{\"schema\":\"pathlab.ai-evidence/1\",\"source\":{" + - "\"slideSha256\":\"" + sha + "\",\"revision\":\"" + revision + "\"}," + - "\"status\":\"" + status + "\",\"manifestSha256\":\"" + "c".repeat(64) + "\"," + - "\"signature\":{\"algorithm\":\"Ed25519\"}}" ); + new EvidenceBundleWriter(temporaryDirectory.resolve("signing")) + .write(path, unsigned(sha, revision, status)); return path.toAbsolutePath().normalize(); } + + private static com.fasterxml.jackson.databind.node.ObjectNode unsigned( + String sha, String revision, String status) { + var root = new ObjectMapper().createObjectNode(); + root.put("schema", EvidenceBundleWriter.SCHEMA); + root.put("researchOnly", true); + root.put("notDiagnostic", true); + root.put("reviewRequired", true); + root.put("status", status); + root.putObject("source").put("slideSha256", sha).put("revision", revision); + return root; + } } diff --git a/src/test/java/org/pathlab/forge/evidence/EvidenceBundleWriterV2Test.java b/src/test/java/org/pathlab/forge/evidence/EvidenceBundleWriterV2Test.java new file mode 100644 index 0000000..b6e8b3a --- /dev/null +++ b/src/test/java/org/pathlab/forge/evidence/EvidenceBundleWriterV2Test.java @@ -0,0 +1,32 @@ +package org.pathlab.forge.evidence; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.fasterxml.jackson.databind.ObjectMapper; +import java.nio.file.Path; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +final class EvidenceBundleWriterV2Test { + private static final ObjectMapper JSON = new ObjectMapper(); + @TempDir Path temporaryDirectory; + + @Test + void signsV2WithoutEmbeddingItsOwnTrustAnchor() throws Exception { + var unsigned = JSON.createObjectNode(); + unsigned.put("schema", EvidenceBundleWriterV2.SCHEMA); + unsigned.put("researchOnly", true); + unsigned.put("notDiagnostic", true); + unsigned.put("reviewRequired", true); + unsigned.put("qualificationAttestationSha256", "a".repeat(64)); + unsigned.put("status", "completed"); + unsigned.putObject("source").put("slideSha256", "b".repeat(64)).put("revision", "r1"); + var target = temporaryDirectory.resolve("artifacts/job/evidence.json"); + new EvidenceBundleWriterV2(temporaryDirectory).write(target, unsigned); + var signed = JSON.readTree(target.toFile()); + assertFalse(signed.path("signature").has("publicKeyDer")); + assertTrue(EvidenceArtifactStore.isTrusted( + temporaryDirectory, target, "b".repeat(64), "r1")); + } +} diff --git a/src/test/java/org/pathlab/forge/evidence/EvidenceJobProcessorTest.java b/src/test/java/org/pathlab/forge/evidence/EvidenceJobProcessorTest.java index 04df36f..dcd1941 100644 --- a/src/test/java/org/pathlab/forge/evidence/EvidenceJobProcessorTest.java +++ b/src/test/java/org/pathlab/forge/evidence/EvidenceJobProcessorTest.java @@ -165,17 +165,18 @@ void completesOfflineDescriptiveIhcJobIntoSignedAtomicEvidence() throws Exceptio assertEquals("ki-67", evidence.path("ihcDescriptors").get(0).path("marker").asText()); assertEquals("relative_only", evidence.path("ihcDescriptors").get(0) .path("calibrationStatus").asText()); - assertEquals("generic-fallback", evidence.path("ihcDescriptors").get(0) + assertEquals("marker-aware", evidence.path("ihcDescriptors").get(0) .path("analysisMode").asText()); assertEquals("slide-label", evidence.path("ihcDescriptors").get(0) .path("markerIdentitySource").asText()); - assertEquals("generic-region", evidence.path("ihcDescriptors").get(0) + assertEquals("nuclear", evidence.path("ihcDescriptors").get(0) .path("compartment").asText()); - assertEquals("MARKER_SPECIFIC_QUALIFICATION_REQUIRED", evidence.path("ihcDescriptors").get(0) - .path("abstentionReason").asText()); - assertEquals("od-connected-components", evidence.path("ihcDescriptors").get(0) + assertTrue(evidence.path("ihcDescriptors").get(0).path("abstentionReason").isNull()); + assertTrue(evidence.path("ihcDescriptors").get(0).path("measurements") + .has("intensityDistribution")); + assertEquals("od-watershed", evidence.path("ihcDescriptors").get(0) .path("cellMaskSource").asText()); - assertEquals("od-connected-components", evidence.path("cellAggregates").get(0) + assertEquals("od-watershed", evidence.path("cellAggregates").get(0) .path("algorithm").asText()); assertTrue(evidence.path("cellAggregates").get(0) .path("meanNucleusPerimeterPx").asDouble() > 0); @@ -193,7 +194,7 @@ void refusesPdL1CompartmentClaimsWithoutReviewedGeometry() throws Exception { assertEquals("pd-l1", descriptor.path("markerId").asText()); assertEquals("generic-fallback", descriptor.path("analysisMode").asText()); assertEquals("generic-region", descriptor.path("compartment").asText()); - assertEquals("faculty-approved", descriptor.path("compartmentSource").asText()); + assertEquals("none", descriptor.path("compartmentSource").asText()); assertEquals("COMPARTMENT_REVIEW_REQUIRED", descriptor.path("abstentionReason").asText()); assertTrue(evidence.path("qc").path("warnings").toString() .contains("COMPARTMENT_REVIEW_REQUIRED")); diff --git a/src/test/java/org/pathlab/forge/evidence/EvidenceMentorRunnerTest.java b/src/test/java/org/pathlab/forge/evidence/EvidenceMentorRunnerTest.java index 0d7823a..6ab1804 100644 --- a/src/test/java/org/pathlab/forge/evidence/EvidenceMentorRunnerTest.java +++ b/src/test/java/org/pathlab/forge/evidence/EvidenceMentorRunnerTest.java @@ -96,12 +96,37 @@ void acceptsAuthenticatedLoopbackSubmissionIntoDurableQueue() throws Exception { var endpoint = JSON.readTree(temporaryDirectory.resolve("state/endpoint.json").toFile()); assertEquals("pathlab.runner-endpoint/1", endpoint.path("schema").asText()); - assertEquals("2.0.12", endpoint.path("serviceVersion").asText()); + assertEquals("2.1.0", endpoint.path("serviceVersion").asText()); assertEquals(runner.uri("/").getPort(), endpoint.path("port").asInt()); assertTrue(!endpoint.has("token")); } } + @Test + void createsAndReadsDurableQualificationCampaign() throws Exception { + var campaignDirectory = Files.createDirectories(temporaryDirectory.resolve("campaign")); + Files.writeString(campaignDirectory.resolve("request.json"), "{}"); + var manifest = campaignDirectory.resolve("campaign.json"); + Files.writeString(manifest, QualificationCampaignManifestTest.campaign("request.json")); + try (var runner = EvidenceMentorRunner.start( + temporaryDirectory.resolve("state"), 0, "test-loopback-token-0123456789abcdef", false)) { + var client = HttpClient.newHttpClient(); + var body = "{\"manifestPath\":" + quote(manifest.toString()) + "}"; + var accepted = client.send(HttpRequest.newBuilder(runner.uri("/v1/qualification-runs")) + .header("Authorization", "Bearer test-loopback-token-0123456789abcdef") + .POST(HttpRequest.BodyPublishers.ofString(body)).build(), HttpResponse.BodyHandlers.ofString()); + assertEquals(202, accepted.statusCode()); + assertTrue(accepted.body().contains("pathlab.qualification-run/1")); + assertTrue(accepted.body().contains("all-rounder-1")); + + var read = client.send(HttpRequest.newBuilder(runner.uri("/v1/qualification-runs/all-rounder-1")) + .header("Authorization", "Bearer test-loopback-token-0123456789abcdef") + .GET().build(), HttpResponse.BodyHandlers.ofString()); + assertEquals(200, read.statusCode()); + assertTrue(read.body().contains("dinov2-small")); + } + } + @Test void exchangesSingleUseDashboardCodeAndProtectsMutations() throws Exception { try (var runner = EvidenceMentorRunner.start( diff --git a/src/test/java/org/pathlab/forge/evidence/EvidencePackManifestTest.java b/src/test/java/org/pathlab/forge/evidence/EvidencePackManifestTest.java index 01d6eca..b2d095b 100644 --- a/src/test/java/org/pathlab/forge/evidence/EvidencePackManifestTest.java +++ b/src/test/java/org/pathlab/forge/evidence/EvidencePackManifestTest.java @@ -119,7 +119,13 @@ void everyBundledPackResolvesItsModelCard() throws Exception { var root = Path.of("src/main/resources/evidence-packs"); try (var manifests = Files.list(root)) { for (var manifest : manifests.filter(path -> path.toString().endsWith(".json")).toList()) { - var card = JSON.readTree(manifest.toFile()).path("validation").path("modelCard").asText(); + var rootNode = JSON.readTree(manifest.toFile()); + if (EvidencePackManifest.SCHEMA_V2.equals(rootNode.path("schema").asText())) { + assertTrue(rootNode.path("qualificationPolicy").path("protocolSha256") + .asText().matches("[a-f0-9]{64}")); + continue; + } + var card = rootNode.path("validation").path("modelCard").asText(); assertTrue(!card.isBlank() && Files.isRegularFile(Path.of(card)), () -> manifest.getFileName() + " references a missing model card: " + card); } diff --git a/src/test/java/org/pathlab/forge/evidence/EvidenceSetWriterTest.java b/src/test/java/org/pathlab/forge/evidence/EvidenceSetWriterTest.java new file mode 100644 index 0000000..946413a --- /dev/null +++ b/src/test/java/org/pathlab/forge/evidence/EvidenceSetWriterTest.java @@ -0,0 +1,49 @@ +package org.pathlab.forge.evidence; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import com.fasterxml.jackson.databind.ObjectMapper; +import java.nio.file.Path; +import java.time.Instant; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +final class EvidenceSetWriterTest { + private static final ObjectMapper JSON = new ObjectMapper(); + @TempDir Path temporaryDirectory; + + @Test + void fusesOnlyTrustedEvidenceFromOneExactSlideRevision() throws Exception { + var first = evidence("a", "revision-1", "cells"); + var second = evidence("a", "revision-1", "ihc"); + var output = temporaryDirectory.resolve("artifacts/set/evidence-set.json"); + var result = new EvidenceSetWriter(temporaryDirectory).write( + output, "set-1", java.util.List.of(first, second), Instant.EPOCH); + var manifest = JSON.readTree(output.toFile()); + assertEquals(EvidenceSetWriter.SCHEMA, manifest.path("schema").asText()); + assertEquals(2, manifest.path("bundles").size()); + assertEquals(result.manifestSha256(), manifest.path("manifestSha256").asText()); + assertFalse(manifest.path("fusion").path("serialSectionCellMatching").asBoolean(true)); + + var foreign = evidence("b", "revision-1", "foreign"); + assertThrows(IllegalArgumentException.class, () -> new EvidenceSetWriter(temporaryDirectory) + .write(output, "bad-set", java.util.List.of(first, foreign), Instant.EPOCH)); + } + + private Path evidence(String shaSeed, String revision, String pack) throws Exception { + var unsigned = JSON.createObjectNode(); + unsigned.put("schema", EvidenceBundleWriter.SCHEMA); + unsigned.put("bundleId", "bundle-" + pack); + unsigned.putObject("source").put("slideSha256", shaSeed.repeat(64)).put("revision", revision); + unsigned.putObject("pack").put("id", pack).put("capability", "cell-morphology"); + unsigned.put("status", "completed"); + unsigned.put("researchOnly", true); + unsigned.put("notDiagnostic", true); + unsigned.put("reviewRequired", true); + var path = temporaryDirectory.resolve("artifacts/" + pack + "/evidence.json"); + new EvidenceBundleWriter(temporaryDirectory.resolve("signing")).write(path, unsigned); + return path; + } +} diff --git a/src/test/java/org/pathlab/forge/evidence/IhcMeasurementAnalyzerTest.java b/src/test/java/org/pathlab/forge/evidence/IhcMeasurementAnalyzerTest.java new file mode 100644 index 0000000..f30f9cd --- /dev/null +++ b/src/test/java/org/pathlab/forge/evidence/IhcMeasurementAnalyzerTest.java @@ -0,0 +1,53 @@ +package org.pathlab.forge.evidence; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.awt.Color; +import java.awt.image.BufferedImage; +import java.util.List; +import org.junit.jupiter.api.Test; + +final class IhcMeasurementAnalyzerTest { + @Test + void keepsNuclearAndMembraneMeasurementsDescriptive() { + var image = fixture(); + var nuclear = IhcMeasurementAnalyzer.analyze(image, "ki-67", List.of(), false); + var membrane = IhcMeasurementAnalyzer.analyze(image, "her2", List.of(), false); + assertEquals("nuclear", nuclear.compartment()); + assertTrue(nuclear.measurements().containsKey("dabAssociatedNuclei")); + assertEquals("membrane", membrane.compartment()); + assertTrue(membrane.measurements().containsKey("meanMembraneCompleteness")); + assertTrue(nuclear.measurements().keySet().stream().noneMatch(key -> Set.of("positive", "tps", "cps").contains(key))); + } + + @Test + void refusesPdL1CompartmentsUntilReviewedGeometryExists() { + var fallback = IhcMeasurementAnalyzer.analyze(fixture(), "pd-l1", List.of(), false); + assertEquals("generic-fallback", fallback.analysisMode()); + assertEquals("COMPARTMENT_REVIEW_REQUIRED", fallback.abstentionReason()); + + var reviewed = IhcMeasurementAnalyzer.analyze(fixture(), "pd-l1", List.of( + new IhcMeasurementAnalyzer.ReviewedRegion( + "tumor-1", "tumor", "faculty-approved", 0, 0, 16, 16)), false); + assertEquals("marker-aware", reviewed.analysisMode()); + assertNull(reviewed.abstentionReason()); + assertTrue(reviewed.measurements().containsKey("compartments")); + } + + private static BufferedImage fixture() { + var image = new BufferedImage(32, 24, BufferedImage.TYPE_INT_RGB); + var graphics = image.createGraphics(); + graphics.setColor(Color.WHITE); graphics.fillRect(0, 0, 32, 24); + graphics.setColor(new Color(75, 45, 125)); graphics.fillRect(5, 5, 6, 6); + graphics.setColor(new Color(135, 90, 45)); graphics.fillRect(4, 4, 8, 1); + graphics.fillRect(4, 11, 8, 1); graphics.fillRect(4, 4, 1, 8); graphics.fillRect(11, 4, 1, 8); + graphics.dispose(); + return image; + } + + private static final class Set { + static java.util.Set of(String... values) { return java.util.Set.of(values); } + } +} diff --git a/src/test/java/org/pathlab/forge/evidence/QualificationCampaignManifestTest.java b/src/test/java/org/pathlab/forge/evidence/QualificationCampaignManifestTest.java new file mode 100644 index 0000000..27d2aa3 --- /dev/null +++ b/src/test/java/org/pathlab/forge/evidence/QualificationCampaignManifestTest.java @@ -0,0 +1,44 @@ +package org.pathlab.forge.evidence; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.nio.file.Files; +import java.nio.file.Path; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +final class QualificationCampaignManifestTest { + @TempDir Path temporaryDirectory; + + @Test + void loadsFrozenAllRounderCampaignWithoutAllowingPathEscape() throws Exception { + Files.writeString(temporaryDirectory.resolve("request.json"), "{}"); + var manifest = temporaryDirectory.resolve("campaign.json"); + Files.writeString(manifest, campaign("request.json")); + + var loaded = QualificationCampaignManifest.load(manifest); + assertEquals("all-rounder-1", loaded.campaignId()); + assertEquals(1, loaded.maxRemediationAttempts()); + assertEquals("he-evidence", loaded.tracks().get(0).capability()); + + Files.writeString(manifest, campaign("../request.json")); + assertThrows(IllegalArgumentException.class, + () -> QualificationCampaignManifest.load(manifest)); + } + + static String campaign(String requestPath) { + return """ + {"schema":"pathlab.qualification-campaign/1","campaignId":"all-rounder-1", + "createdAt":"2026-08-22T00:00:00Z","researchOnly":true,"notDiagnostic":true, + "maxRemediationAttempts":1, + "quota":{"sourceBytes":48318382080,"derivedBytes":26843545600, + "modelBytes":10737418240,"evidenceBytes":10737418240,"reserveBytes":10737418240}, + "tracks":[{"id":"dinov2","candidateId":"dinov2-small","capability":"he-evidence", + "scope":"deployment","requestPath":"%s","remediationRequestPath":null, + "expectedAttestationPath":null, + "protocolSha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "dependsOn":[],"required":true}]} + """.formatted(requestPath.replace("\\", "\\\\")); + } +} diff --git a/src/test/java/org/pathlab/forge/evidence/QualificationReportValidatorTest.java b/src/test/java/org/pathlab/forge/evidence/QualificationReportValidatorTest.java new file mode 100644 index 0000000..e4ebfac --- /dev/null +++ b/src/test/java/org/pathlab/forge/evidence/QualificationReportValidatorTest.java @@ -0,0 +1,58 @@ +package org.pathlab.forge.evidence; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import java.nio.file.Files; +import java.nio.file.Path; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +final class QualificationReportValidatorTest { + private static final ObjectMapper JSON = new ObjectMapper(); + @TempDir Path temporaryDirectory; + + @Test + void trustsPinnedSignerInsteadOfPayloadSuppliedKey() throws Exception { + Files.writeString(temporaryDirectory.resolve("request.json"), "{}"); + var campaignPath = temporaryDirectory.resolve("campaign.json"); + Files.writeString(campaignPath, QualificationCampaignManifestTest.campaign("request.json")); + var campaign = QualificationCampaignManifest.load(campaignPath); + var unsigned = report(campaign); + var reportPath = temporaryDirectory.resolve("report.json"); + new QualificationReportWriter(temporaryDirectory.resolve("state")).write(reportPath, unsigned); + + var result = new QualificationReportValidator(temporaryDirectory.resolve("state")) + .validate(reportPath, campaign, campaign.tracks().get(0)); + assertEquals("qualified", result.verdict()); + + Files.delete(temporaryDirectory.resolve("state/trust/approved-signers.json")); + assertThrows(IllegalArgumentException.class, () -> + new QualificationReportValidator(temporaryDirectory.resolve("state")) + .validate(reportPath, campaign, campaign.tracks().get(0))); + } + + static ObjectNode report(QualificationCampaignManifest campaign) { + var track = campaign.tracks().get(0); + var root = JSON.createObjectNode(); + root.put("schema", "pathlab.model-qualification-report/2"); + root.put("campaignId", campaign.campaignId()); + root.put("campaignManifestSha256", campaign.sha256()); + root.put("trackId", track.id()); + root.put("candidateId", track.candidateId()); + root.put("capability", track.capability()); + root.put("scope", track.scope()); + root.put("packManifestSha256", "b".repeat(64)); + root.put("protocolSha256", track.protocolSha256()); + root.put("generatedAt", "2026-08-22T00:00:00Z"); + root.put("researchOnly", true); + root.put("notDiagnostic", true); + root.put("status", "qualified"); + var checks = root.putArray("checks"); + checks.addObject().put("id", "determinism").put("outcome", "pass"); + root.putArray("reasons"); + return root; + } +} diff --git a/src/test/java/org/pathlab/forge/evidence/QualificationRunStoreTest.java b/src/test/java/org/pathlab/forge/evidence/QualificationRunStoreTest.java new file mode 100644 index 0000000..bcab299 --- /dev/null +++ b/src/test/java/org/pathlab/forge/evidence/QualificationRunStoreTest.java @@ -0,0 +1,65 @@ +package org.pathlab.forge.evidence; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Instant; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +final class QualificationRunStoreTest { + @TempDir Path temporaryDirectory; + + @Test + void persistsSignedAttestationAndComputesTargetWithoutChangingCandidateManifest() throws Exception { + var packPath = temporaryDirectory.resolve("pack.json"); + Files.writeString(packPath, deploymentPack()); + Files.writeString(temporaryDirectory.resolve("request.json"), + "{\"packManifest\":" + new com.fasterxml.jackson.databind.ObjectMapper() + .writeValueAsString(packPath.toAbsolutePath().toString()) + "}"); + var campaignPath = temporaryDirectory.resolve("campaign.json"); + Files.writeString(campaignPath, QualificationCampaignManifestTest.campaign("request.json")); + var campaign = QualificationCampaignManifest.load(campaignPath); + var state = Files.createDirectories(temporaryDirectory.resolve("state")); + try (var store = new QualificationRunStore(state.resolve("jobs.sqlite3"), state)) { + var created = store.create(campaign, Instant.parse("2026-08-22T00:00:00Z")); + assertFalse(created.campaignCompleted()); + var report = temporaryDirectory.resolve("report.json"); + var unsigned = QualificationReportValidatorTest.report(campaign); + unsigned.put("packManifestSha256", EvidencePackManifest.load(packPath).sha256()); + new QualificationReportWriter(state).write(report, unsigned); + var completed = store.attachAttestation(campaign.campaignId(), "dinov2", report, + Instant.parse("2026-08-22T00:01:00Z")); + assertTrue(completed.campaignCompleted()); + assertTrue(completed.campaignTargetMet()); + assertEquals("qualified", completed.tracks().get(0).verdict()); + assertEquals(campaign.sha256(), completed.manifestSha256()); + new QualifiedPackRegistry(state).requireQualified(EvidencePackManifest.load(packPath), + completed.tracks().get(0).artifactSha256()); + } + } + + private static String deploymentPack() { + return """ + {"schema":"pathlab.ai-pack/2","packId":"dinov2-small","version":"1.0.0", + "capability":"he-evidence","scope":"deployment","activationEligible":true, + "acceptedStains":["he"],"preprocessing":{"id":"frozen","tilePixels":512}, + "artifacts":[],"runtimeCompatibility":{"workerProtocol":"pathlab.model-worker-result/2", + "executionProvider":"cpu","cuda":"none","gpuArchitecture":"none", + "requiresExternalWorker":false,"segmentMaxSeconds":120}, + "licenseLedger":[{"component":"test","kind":"weights","license":"test-only", + "revision":"fixed","permittedUse":"private-research","redistributable":false, + "derivativesAllowed":true}], + "derivativePermissions":{"atlasResearch":true,"atlasClean":true, + "reviewedAt":"2026-08-22T00:00:00Z"}, + "resourceEnvelope":{"lane":"cpu_io","maxRamMiB":1024,"maxVramMiB":0, + "maxSeconds":120,"network":false}, + "qualificationPolicy":{"requiresSignedAttestation":true, + "protocolSha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}, + "outputSchema":"pathlab.model-qualification-report/2","markers":[]} + """; + } +} diff --git a/src/test/java/org/pathlab/forge/evidence/SpecialStainAnalyzerTest.java b/src/test/java/org/pathlab/forge/evidence/SpecialStainAnalyzerTest.java new file mode 100644 index 0000000..d7102e9 --- /dev/null +++ b/src/test/java/org/pathlab/forge/evidence/SpecialStainAnalyzerTest.java @@ -0,0 +1,28 @@ +package org.pathlab.forge.evidence; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.awt.Color; +import java.awt.image.BufferedImage; +import org.junit.jupiter.api.Test; + +final class SpecialStainAnalyzerTest { + @Test + void emitsDescriptivePasAndPapanicolaouMeasurementsWithUnknownFallback() { + var image = new BufferedImage(20, 20, BufferedImage.TYPE_INT_RGB); + var graphics = image.createGraphics(); + graphics.setColor(Color.WHITE); graphics.fillRect(0, 0, 20, 20); + graphics.setColor(new Color(180, 70, 150)); graphics.fillRect(2, 2, 10, 10); + graphics.dispose(); + + var pas = SpecialStainAnalyzer.analyze(image, "pas"); + assertEquals("special-stain-descriptive", pas.analysisMode()); + assertTrue((double) pas.measurements().get("stainAreaFraction") > 0); + var pap = SpecialStainAnalyzer.analyze(image, "papanicolaou"); + assertEquals("cytology-descriptive", pap.analysisMode()); + var unknown = SpecialStainAnalyzer.analyze(image, "reticulin"); + assertEquals("generic_brightfield", unknown.stainId()); + assertEquals("UNSUPPORTED_STAIN_GENERIC_FALLBACK", unknown.abstentionReason()); + } +} From f9a5827b22c2db934c8fa2f84bb60025923b2263 Mon Sep 17 00:00:00 2001 From: Enksodsoon <229880765+Enksodsoon@users.noreply.github.com> Date: Sat, 22 Aug 2026 17:36:46 +0700 Subject: [PATCH 23/50] test: normalize protocol hashes across platforms --- .../pathlab/forge/evidence/AllRounderPackManifestTest.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/test/java/org/pathlab/forge/evidence/AllRounderPackManifestTest.java b/src/test/java/org/pathlab/forge/evidence/AllRounderPackManifestTest.java index 4a789c6..49dcdf0 100644 --- a/src/test/java/org/pathlab/forge/evidence/AllRounderPackManifestTest.java +++ b/src/test/java/org/pathlab/forge/evidence/AllRounderPackManifestTest.java @@ -6,6 +6,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.charset.StandardCharsets; import java.security.MessageDigest; import java.util.HexFormat; import org.junit.jupiter.api.Test; @@ -21,8 +22,12 @@ void deterministicV2CandidatesPinTheirPreregisteredProtocols() throws Exception var protocolPath = Path.of("qualification-protocols/" + id + "-v2.json"); var pack = EvidencePackManifest.load(packPath); var raw = JSON.readTree(packPath.toFile()); + // Git may materialize CRLF on Windows. Protocol identities use the + // repository-canonical UTF-8/LF representation on every platform. + var canonicalProtocol = Files.readString(protocolPath, StandardCharsets.UTF_8) + .replace("\r\n", "\n"); var protocolSha = HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256") - .digest(Files.readAllBytes(protocolPath))); + .digest(canonicalProtocol.getBytes(StandardCharsets.UTF_8))); assertEquals(protocolSha, raw.path("qualificationPolicy").path("protocolSha256").asText()); assertEquals(EvidencePackManifest.SCHEMA_V2, pack.schema()); assertTrue(pack.activationEligible()); From a09f583b1f11c905e96be916ee9824918e7a4e5c Mon Sep 17 00:00:00 2001 From: Enksodsoon <229880765+Enksodsoon@users.noreply.github.com> Date: Sat, 22 Aug 2026 20:23:33 +0700 Subject: [PATCH 24/50] feat: stage deterministic qualification campaign --- docs/plans/active/current.md | 20 ++- .../stage-local-deterministic-campaign.ps1 | 149 ++++++++++++++++++ 2 files changed, 163 insertions(+), 6 deletions(-) create mode 100644 scripts/stage-local-deterministic-campaign.ps1 diff --git a/docs/plans/active/current.md b/docs/plans/active/current.md index 383d2b6..8bbc18b 100644 --- a/docs/plans/active/current.md +++ b/docs/plans/active/current.md @@ -29,12 +29,12 @@ appear in monitoring payloads. ## Current status -Gates 1-4 remain implemented and service `2.1.0` is staged as a side-by-side, -idempotent upgrade. An existing byte-identical runtime is reused; a mismatch -fails before the active configuration changes. The generated elevated launcher -does not request a reboot. The installed `2.0.12` runtime remains active until -that separate administrator action. The manual reboot-continuation acceptance -is still deferred, so Gate 5 and activation remain incomplete. +Gates 1-4 remain implemented. Service `2.1.0` was installed side-by-side on +2026-08-22 and passed the 19-check non-reboot acceptance, including a real +offline DINOv2 CUDA job on the P2000 across a service restart. An existing +byte-identical runtime is reused; a mismatch fails before the active +configuration changes. The manual reboot-continuation acceptance is still +deferred, so Gate 5 and activation remain incomplete. The branch now implements `pathlab.qualification-campaign/1` as a durable parent coordinator above the existing GPU and CPU/I/O lanes. It records frozen @@ -58,6 +58,14 @@ must run. Gated DINOv2/Hibou/GigaPath, HoVer-Net/PathoSAM, Qwen assets, and publ cytology data are not represented as acquired or built when their exact artifacts or rights are absent. +The checksum-frozen `deterministic-baselines-20260822-v1` campaign completed +autonomously against a grandfathered, patient-disjoint BRACS test input. Cell, +IHC, special-stain, and cytology execution tracks each produced a signed +`experimental` attestation. `campaignCompleted=true` and +`campaignTargetMet=false`: this proves durable execution and signing, not the +pre-registered held-out quality gates. The reusable staging command is +`scripts/stage-local-deterministic-campaign.ps1`. + Gate 7 has an executable four-hour feasibility decision contract using checkpointed 15-20 minute segments, fixed RAM/VRAM limits, validation improvement, seven-day projection, 24 GiB rental minimum, and USD 100 ceiling. diff --git a/scripts/stage-local-deterministic-campaign.ps1 b/scripts/stage-local-deterministic-campaign.ps1 new file mode 100644 index 0000000..e0b5ded --- /dev/null +++ b/scripts/stage-local-deterministic-campaign.ps1 @@ -0,0 +1,149 @@ +[CmdletBinding()] +param( + [string] $StateRoot = 'D:\PathLabData\EvidenceMentor\state', + [string] $BracsManifest = 'D:\PathLabData\BRACS\prepared\bracs-roi-clean-v1\manifest.csv', + [string] $BracsProvenance = 'D:\PathLabData\BRACS\prepared\bracs-roi-clean-v1\provenance.json' +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version Latest +$repository = Split-Path -Parent $PSScriptRoot +$state = [IO.Path]::GetFullPath($StateRoot) +$campaignId = 'deterministic-baselines-20260822-v1' +$root = [IO.Path]::GetFullPath((Join-Path $state "acceptance\campaigns\$campaignId")) +if (-not $root.StartsWith((Join-Path $state 'acceptance\campaigns') + [IO.Path]::DirectorySeparatorChar, + [StringComparison]::OrdinalIgnoreCase)) { throw 'Campaign path escaped the protected state root.' } + +function Sha256([string] $Path) { + (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant() +} +function WriteJson([string] $Path, [object] $Value) { + [IO.File]::WriteAllText($Path, (($Value | ConvertTo-Json -Depth 16) + "`n"), + [Text.UTF8Encoding]::new($false)) +} + +if (-not (Test-Path -LiteralPath $BracsManifest -PathType Leaf) -or + -not (Test-Path -LiteralPath $BracsProvenance -PathType Leaf)) { + throw 'The grandfathered BRACS manifest and provenance are required.' +} +$provenance = Get-Content -LiteralPath $BracsProvenance -Raw | ConvertFrom-Json +if ($provenance.license_declared_by_current_source -ne 'CC-BY-NC-4.0' -or + $provenance.patient_disjoint -ne $true) { + throw 'BRACS rights or patient-disjoint provenance did not pass.' +} +$sample = Import-Csv -LiteralPath $BracsManifest | + Where-Object { $_.split -eq 'test' -and $_.sha256 -match '^[a-f0-9]{64}$' } | + Select-Object -First 1 +if ($null -eq $sample) { throw 'No checksum-bound BRACS test sample is available.' } +$source = Join-Path ([string]$provenance.raw_root) ([string]$sample.relative_path) +if (-not (Test-Path -LiteralPath $source -PathType Leaf) -or (Sha256 $source) -ne $sample.sha256) { + throw 'The selected BRACS source checksum changed.' +} + +New-Item -ItemType Directory -Path $root -Force | Out-Null +$sourceTarget = Join-Path $root 'bracs-test-source.png' +if (-not (Test-Path -LiteralPath $sourceTarget)) { Copy-Item -LiteralPath $source -Destination $sourceTarget } +if ((Sha256 $sourceTarget) -ne $sample.sha256) { throw 'The staged BRACS source checksum changed.' } + +$packSpecs = @( + @{ id='cell-od-watershed-v2'; capability='cell-morphology'; stain='he'; marker='generic'; protocol='dd251775f0abe6a780b2389ee489f18bcec78b9cb80e1facfd7134a05cb0c2a0' }, + @{ id='ihc-descriptive-v2'; capability='ihc-descriptive'; stain='ihc_dab'; marker='generic'; protocol='9b2d3299eed5504fe7d89839cacae6ea5232762d6405bdfab261ad78e30b991f' }, + @{ id='special-stain-descriptive-v2'; capability='special-stain-descriptive'; stain='pas'; marker='generic'; protocol='d035efcec9812f16464b4e8e59e9d245884a57698626821ab0cf056d05854cee' }, + @{ id='cytology-descriptive-v2'; capability='cytology-descriptive'; stain='papanicolaou'; marker='generic'; protocol='b354e6c605afc55b5663167453fa2b23738f311d48de5f30293cef0ba26f962a' } +) +$campaignPath = Join-Path $root 'campaign.json' +$tracks = [Collections.Generic.List[object]]::new() +foreach ($spec in $packSpecs) { + $packSource = Join-Path $repository "src\main\resources\evidence-packs\$($spec.id).json" + $packTarget = Join-Path $root "$($spec.id).json" + Copy-Item -LiteralPath $packSource -Destination $packTarget -Force + $requestName = "request-$($spec.id).json" + $requestPath = Join-Path $root $requestName + WriteJson $requestPath ([ordered]@{ + schema = 'pathlab.evidence-job/2' + sourcePath = $sourceTarget + sourceSha256 = Sha256 $sourceTarget + slideRevision = "bracs:$($sample.slide_id):$($sample.sha256)" + previewPath = $sourceTarget + sourceWidth = [int]$sample.width + sourceHeight = [int]$sample.height + packManifest = $packTarget + stain = $spec.stain + marker = $spec.marker + qualificationCampaignManifest = $campaignPath + }) + $tracks.Add([ordered]@{ + id = $spec.id + candidateId = $spec.id + capability = $spec.capability + scope = 'deployment' + requestPath = $requestName + remediationRequestPath = $null + expectedAttestationPath = "attestation-$($spec.id).json" + protocolSha256 = $spec.protocol + dependsOn = @() + required = $true + }) +} +WriteJson $campaignPath ([ordered]@{ + schema = 'pathlab.qualification-campaign/1' + campaignId = $campaignId + createdAt = '2026-08-22T00:00:00Z' + researchOnly = $true + notDiagnostic = $true + maxRemediationAttempts = 1 + quota = [ordered]@{ + sourceBytes = 45GB; derivedBytes = 25GB; modelBytes = 10GB + evidenceBytes = 10GB; reserveBytes = 10GB + } + tracks = $tracks +}) +$ledgerPath = Join-Path $root 'sample-ledger.jsonl' +$ledgerRecord = [ordered]@{ + sampleId = [string]$sample.roi_id + source = 'BRACS_RoI/latest_version' + patientGroup = [string]$sample.patient_id + slideGroup = [string]$sample.slide_id + sha256 = [string]$sample.sha256 + license = 'CC-BY-NC-4.0' + permittedUse = 'research-restricted' + task = 'deterministic-execution-smoke-only' + split = [string]$sample.split +} +[IO.File]::WriteAllText($ledgerPath, (($ledgerRecord | ConvertTo-Json -Compress) + "`n"), + [Text.UTF8Encoding]::new($false)) +$preparationPath = Join-Path $root 'preparation.json' +WriteJson $preparationPath ([ordered]@{ + schema = 'pathlab.campaign-preparation/1' + campaignManifest = 'campaign.json' + sampleLedger = 'sample-ledger.jsonl' + artifacts = @() +}) + +& (Join-Path $repository 'scripts\prepare-all-rounder-campaign.ps1') ` + -PreparationManifest $preparationPath -StateRoot $state + +$endpoint = Get-Content -LiteralPath (Join-Path $state 'endpoint.json') -Raw | ConvertFrom-Json +$token = [IO.File]::ReadAllText((Join-Path $state 'ipc-token')).Trim() +try { + try { + $response = Invoke-RestMethod -Method Get ` + -Uri "http://127.0.0.1:$($endpoint.port)/v1/qualification-runs/$campaignId" ` + -Headers @{ Authorization = "Bearer $token" } + } catch { + if ($_.Exception.Response.StatusCode -ne 404) { throw } + $response = Invoke-RestMethod -Method Post ` + -Uri "http://127.0.0.1:$($endpoint.port)/v1/qualification-runs" ` + -Headers @{ Authorization = "Bearer $token" } -ContentType 'application/json' ` + -Body (@{ manifestPath = $campaignPath } | ConvertTo-Json -Compress) + } +} finally { Remove-Variable token -ErrorAction SilentlyContinue } +[pscustomobject]@{ + CampaignId = $response.id + State = $response.state + CampaignCompleted = $response.campaignCompleted + CampaignTargetMet = $response.campaignTargetMet + ManifestPath = $campaignPath + ManifestSha256 = Sha256 $campaignPath + DashboardLauncher = Join-Path $repository 'scripts\open-evidence-dashboard.ps1' +} From 166c5ddf480c624c2972d4fd42c45ee01386c17a Mon Sep 17 00:00:00 2001 From: Enksodsoon <229880765+Enksodsoon@users.noreply.github.com> Date: Sat, 22 Aug 2026 20:47:32 +0700 Subject: [PATCH 25/50] Make qualification preflight fail closed --- .../candidate-preflight-protocol-v1.md | 7 ++ docs/model-cards/atlas-he-72m-v1.md | 7 ++ .../model-cards/cell-pathosam-benchmark-v1.md | 6 ++ docs/model-cards/tutor-qwen3-0.6b-int4-v1.md | 7 ++ docs/plans/active/current.md | 18 ++++ scripts/evidence-mentor-service.ps1 | 89 +++++++++++++------ .../prepare-evidence-mentor-2.1.1-upgrade.ps1 | 36 ++++++++ .../stage-all-rounder-candidate-preflight.ps1 | 75 ++++++++++++++++ .../forge/evidence/EvidenceMentorRunner.java | 2 +- .../forge/evidence/QualificationRunStore.java | 7 +- .../evidence-packs/atlas-he-72m-v1.json | 15 ++++ .../cell-pathosam-benchmark-v1.json | 15 ++++ .../tutor-qwen3-0.6b-int4-v1.json | 15 ++++ .../evidence/EvidenceMentorRunnerTest.java | 2 +- .../evidence/QualificationRunStoreTest.java | 25 ++++++ 15 files changed, 294 insertions(+), 32 deletions(-) create mode 100644 docs/evidence/candidate-preflight-protocol-v1.md create mode 100644 docs/model-cards/atlas-he-72m-v1.md create mode 100644 docs/model-cards/cell-pathosam-benchmark-v1.md create mode 100644 docs/model-cards/tutor-qwen3-0.6b-int4-v1.md create mode 100644 scripts/prepare-evidence-mentor-2.1.1-upgrade.ps1 create mode 100644 scripts/stage-all-rounder-candidate-preflight.ps1 create mode 100644 src/main/resources/evidence-packs/atlas-he-72m-v1.json create mode 100644 src/main/resources/evidence-packs/cell-pathosam-benchmark-v1.json create mode 100644 src/main/resources/evidence-packs/tutor-qwen3-0.6b-int4-v1.json diff --git a/docs/evidence/candidate-preflight-protocol-v1.md b/docs/evidence/candidate-preflight-protocol-v1.md new file mode 100644 index 0000000..9f69af2 --- /dev/null +++ b/docs/evidence/candidate-preflight-protocol-v1.md @@ -0,0 +1,7 @@ +# Candidate preflight protocol v1 + +This protocol records whether a frozen candidate manifest is executable for a qualification job. It is not a scientific model evaluation and cannot produce a `qualified` result. + +A candidate is `not_evaluable` when its pack manifest is absent, malformed, unapproved for qualification execution, missing a rights-cleared artifact, or blocked by a frozen resource or product constraint. The coordinator records the stable failure code `CANDIDATE_PREFLIGHT_FAILED`, completes the track, and signs the resulting capability matrix with the configured local signer. + +Transient database and filesystem I/O errors are not converted into scientific terminal states. Activation remains off, no learner evidence is emitted, and no diagnostic or clinical claim is permitted. diff --git a/docs/model-cards/atlas-he-72m-v1.md b/docs/model-cards/atlas-he-72m-v1.md new file mode 100644 index 0000000..f46d556 --- /dev/null +++ b/docs/model-cards/atlas-he-72m-v1.md @@ -0,0 +1,7 @@ +# Atlas-H&E 72M distillation candidate + +Status: `not-evaluable`. Training has not started because independent baseline packs and lawful +teacher derivative rights are not frozen. This manifest records the gate; it is not a trained +model. A future local probe must remain resumable, stay within the P2000 envelope, improve held-out +validation, and satisfy the frozen retrieval, dense-cell, brightfield, OOD, size, and latency gates. + diff --git a/docs/model-cards/cell-pathosam-benchmark-v1.md b/docs/model-cards/cell-pathosam-benchmark-v1.md new file mode 100644 index 0000000..cea90e7 --- /dev/null +++ b/docs/model-cards/cell-pathosam-benchmark-v1.md @@ -0,0 +1,6 @@ +# PathoSAM cell benchmark candidate + +Status: `not-evaluable`. The repository code is MIT-licensed, but the exact checkpoint revision, +weight lineage, derivative permissions, and P2000 resource behavior have not passed review. No +weights are bundled. This candidate is local benchmark-only and cannot be activated or published. + diff --git a/docs/model-cards/tutor-qwen3-0.6b-int4-v1.md b/docs/model-cards/tutor-qwen3-0.6b-int4-v1.md new file mode 100644 index 0000000..1e0b3ff --- /dev/null +++ b/docs/model-cards/tutor-qwen3-0.6b-int4-v1.md @@ -0,0 +1,7 @@ +# Qwen3-0.6B grounded tutor candidate + +Status: `not-evaluable`. Upstream Qwen3-0.6B declares Apache-2.0, but the complete reviewed INT4 +ONNX WebGPU package has not met the frozen 500 MiB browser-asset cap. No model asset is bundled. +Any future candidate may return only validated knowledge-pack claim IDs and ordering; canonical +reviewed text and citations remain outside the model, with deterministic retrieval as fallback. + diff --git a/docs/plans/active/current.md b/docs/plans/active/current.md index 8bbc18b..48eb9a8 100644 --- a/docs/plans/active/current.md +++ b/docs/plans/active/current.md @@ -36,6 +36,16 @@ byte-identical runtime is reused; a mismatch fails before the active configuration changes. The manual reboot-continuation acceptance is still deferred, so Gate 5 and activation remain incomplete. +Runner `2.1.1` adds fail-closed candidate preflight terminalization: immutable +candidate validation failures become `not_evaluable` rather than blocking the +parent coordinator, while database and filesystem I/O failures still surface. +The runtime starts and reports healthy locally, but the upgrade is not accepted: +this Windows host currently returns system error 2 when creating any outbound +firewall rule, including a control rule for a Windows executable. The installer +now stages replacement rules before removing old rules and restores the prior +runtime configuration on any setup or health failure. Model analysis must not +be activated until an outbound-deny rule is verifiably installed. + The branch now implements `pathlab.qualification-campaign/1` as a durable parent coordinator above the existing GPU and CPU/I/O lanes. It records frozen dependencies, attempts, signed attestations, completion versus target success, @@ -66,6 +76,14 @@ IHC, special-stain, and cytology execution tracks each produced a signed pre-registered held-out quality gates. The reusable staging command is `scripts/stage-local-deterministic-campaign.ps1`. +The signed `all-rounder-candidate-preflight-20260822-v1` campaign also completed +autonomously with `campaignCompleted=true` and `campaignTargetMet=false`. +DINOv2, Hibou-B, GigaPath, HoVer-Net, PathoSAM, Qwen, and Atlas each reached the +honest terminal status `not_evaluable`; no candidate job or model inference was +launched. The result is an availability/readiness record, not a scientific +qualification. Its reusable staging command is +`scripts/stage-all-rounder-candidate-preflight.ps1`. + Gate 7 has an executable four-hour feasibility decision contract using checkpointed 15-20 minute segments, fixed RAM/VRAM limits, validation improvement, seven-day projection, 24 GiB rental minimum, and USD 100 ceiling. diff --git a/scripts/evidence-mentor-service.ps1 b/scripts/evidence-mentor-service.ps1 index 5279f30..c8856a2 100644 --- a/scripts/evidence-mentor-service.ps1 +++ b/scripts/evidence-mentor-service.ps1 @@ -1,7 +1,7 @@ [CmdletBinding(SupportsShouldProcess)] param( [Parameter(Mandatory)] [ValidateSet('Install','Upgrade','Uninstall','Status')] [string] $Action, - [string] $Version = '2.1.0', + [string] $Version = '2.1.1', [string] $DistributionPath, [string] $JavaHome, [string] $ProgramRoot = 'C:\ProgramData\PathLab\EvidenceMentor', @@ -122,18 +122,42 @@ function New-ServiceConfig([string] $RuntimePath) { } function Install-FirewallRules([string] $RuntimePath) { - Get-NetFirewallRule -DisplayName 'PathLab Evidence Mentor outbound deny*' -ErrorAction SilentlyContinue | Remove-NetFirewallRule $executables = @((Join-Path $RuntimePath 'jre\bin\java.exe')) $modelRoot = Join-Path $StateRoot 'models' if (Test-Path -LiteralPath $modelRoot) { $executables += Get-ChildItem -LiteralPath $modelRoot -Filter '*.exe' -File -Recurse | Select-Object -ExpandProperty FullName } + $desiredNames = [Collections.Generic.List[string]]::new() + $createdNames = [Collections.Generic.List[string]]::new() $index = 0 - foreach ($executable in $executables | Select-Object -Unique) { - if (Test-Path -LiteralPath $executable -PathType Leaf) { - New-NetFirewallRule -DisplayName "PathLab Evidence Mentor outbound deny $index" -Direction Outbound -Action Block -Program $executable -Profile Any | Out-Null - $index++ + try { + foreach ($executable in $executables | Select-Object -Unique) { + if (Test-Path -LiteralPath $executable -PathType Leaf) { + $name = "PathLab Evidence Mentor outbound deny $Version $index" + $desiredNames.Add($name) + $existing = Get-NetFirewallRule -DisplayName $name -ErrorAction SilentlyContinue + if ($null -eq $existing) { + New-NetFirewallRule -DisplayName $name -Direction Outbound -Action Block ` + -Program $executable -Profile Any -ErrorAction Stop | Out-Null + $createdNames.Add($name) + } + $actualProgram = Get-NetFirewallRule -DisplayName $name -ErrorAction Stop | + Get-NetFirewallApplicationFilter | Select-Object -ExpandProperty Program -First 1 + if (-not $actualProgram -or + [IO.Path]::GetFullPath($actualProgram) -ne [IO.Path]::GetFullPath($executable)) { + throw "Outbound-deny rule does not match the staged executable: $name" + } + $index++ + } + } + if ($desiredNames.Count -eq 0) { throw 'No installed analysis executable was available for outbound deny.' } + Get-NetFirewallRule -DisplayName 'PathLab Evidence Mentor outbound deny*' -ErrorAction SilentlyContinue | + Where-Object { $desiredNames -notcontains $_.DisplayName } | Remove-NetFirewallRule + } catch { + foreach ($name in $createdNames) { + Get-NetFirewallRule -DisplayName $name -ErrorAction SilentlyContinue | Remove-NetFirewallRule -ErrorAction SilentlyContinue } + throw } } @@ -229,33 +253,40 @@ if ($PSCmdlet.ShouldProcess($serviceName, "$Action autonomous service version $V $previousConfig = if (Test-Path -LiteralPath $configPath) { Get-Content -LiteralPath $configPath -Raw } else { '' } $runtimePath = Stage-Runtime Ensure-WinSW - if ($Action -eq 'Upgrade' -and (Get-Service -Name $serviceName -ErrorAction SilentlyContinue)) { & $wrapperPath stop } - Write-AtomicText $configPath (New-ServiceConfig $runtimePath) - if (-not (Get-Service -Name $serviceName -ErrorAction SilentlyContinue)) { & $wrapperPath install } - Enable-ServiceSid - Grant-PathLabAccess $installingUser - Install-FirewallRules $runtimePath - & $wrapperPath start - if (-not (Wait-RunnerHealth)) { + try { + if ($Action -eq 'Upgrade' -and (Get-Service -Name $serviceName -ErrorAction SilentlyContinue)) { & $wrapperPath stop } + Write-AtomicText $configPath (New-ServiceConfig $runtimePath) + if (-not (Get-Service -Name $serviceName -ErrorAction SilentlyContinue)) { & $wrapperPath install } + Enable-ServiceSid + Grant-PathLabAccess $installingUser + Install-FirewallRules $runtimePath + & $wrapperPath start + if (-not (Wait-RunnerHealth)) { + throw 'New runtime failed health checks.' + } + Write-AtomicText $activeVersionPath $Version + Grant-PathLabAccess $installingUser + $launcher = Join-Path $ProgramRoot 'Open-PathLab-Evidence-Dashboard.ps1' + Copy-Item -LiteralPath (Join-Path $PSScriptRoot 'open-evidence-dashboard.ps1') -Destination $launcher -Force + Copy-Item -LiteralPath (Join-Path $PSScriptRoot 'test-evidence-mentor-service.ps1') ` + -Destination (Join-Path $ProgramRoot 'Test-PathLab-Evidence-Service.ps1') -Force + $shortcutPath = Join-Path ([Environment]::GetFolderPath('CommonStartMenu')) 'Programs\PathLab Evidence Mentor Dashboard.lnk' + $shell = New-Object -ComObject WScript.Shell + $shortcut = $shell.CreateShortcut($shortcutPath) + $shortcut.TargetPath = 'powershell.exe' + $shortcut.Arguments = "-NoProfile -ExecutionPolicy Bypass -File `"$launcher`" -StateRoot `"$StateRoot`"" + $shortcut.WorkingDirectory = $ProgramRoot + $shortcut.Save() + } catch { + $upgradeError = $_ & $wrapperPath stop 2>$null if ($previousConfig) { Write-AtomicText $configPath $previousConfig - Install-FirewallRules (Join-Path $runtimeRoot $previousVersion) + try { Install-FirewallRules (Join-Path $runtimeRoot $previousVersion) } catch { + Write-Warning "Previous firewall rules could not be restored: $($_.Exception.Message)" + } & $wrapperPath start } - throw 'New runtime failed health checks; the previous configuration was restored.' + throw "Service $Action failed and the previous runtime configuration was restored: $($upgradeError.Exception.Message)" } - Write-AtomicText $activeVersionPath $Version - Grant-PathLabAccess $installingUser - $launcher = Join-Path $ProgramRoot 'Open-PathLab-Evidence-Dashboard.ps1' - Copy-Item -LiteralPath (Join-Path $PSScriptRoot 'open-evidence-dashboard.ps1') -Destination $launcher -Force - Copy-Item -LiteralPath (Join-Path $PSScriptRoot 'test-evidence-mentor-service.ps1') ` - -Destination (Join-Path $ProgramRoot 'Test-PathLab-Evidence-Service.ps1') -Force - $shortcutPath = Join-Path ([Environment]::GetFolderPath('CommonStartMenu')) 'Programs\PathLab Evidence Mentor Dashboard.lnk' - $shell = New-Object -ComObject WScript.Shell - $shortcut = $shell.CreateShortcut($shortcutPath) - $shortcut.TargetPath = 'powershell.exe' - $shortcut.Arguments = "-NoProfile -ExecutionPolicy Bypass -File `"$launcher`" -StateRoot `"$StateRoot`"" - $shortcut.WorkingDirectory = $ProgramRoot - $shortcut.Save() } diff --git a/scripts/prepare-evidence-mentor-2.1.1-upgrade.ps1 b/scripts/prepare-evidence-mentor-2.1.1-upgrade.ps1 new file mode 100644 index 0000000..0e435b7 --- /dev/null +++ b/scripts/prepare-evidence-mentor-2.1.1-upgrade.ps1 @@ -0,0 +1,36 @@ +[CmdletBinding()] +param( + [string] $JavaHome = '', + [string] $OutputPath = '' +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version Latest +$repository = Split-Path -Parent $PSScriptRoot +if (-not $JavaHome) { + $candidate = Get-ChildItem -LiteralPath (Join-Path $env:USERPROFILE '.gradle\jdks') -Filter 'java.exe' -File -Recurse ` + -ErrorAction SilentlyContinue | Where-Object { $_.FullName -match '17' } | Select-Object -First 1 + if ($null -eq $candidate) { throw 'Provide -JavaHome for a Java 17 runtime.' } + $JavaHome = Split-Path -Parent (Split-Path -Parent $candidate.FullName) +} +if (-not (Test-Path -LiteralPath (Join-Path $JavaHome 'bin\java.exe') -PathType Leaf)) { + throw 'JavaHome must contain bin\java.exe.' +} + +& (Join-Path $repository 'gradlew.bat') clean test installDist +if ($LASTEXITCODE -ne 0) { throw 'The 2.1.1 distribution did not pass its build and tests.' } + +if (-not $OutputPath) { $OutputPath = Join-Path $repository 'build\Upgrade-PathLab-EvidenceMentor-2.1.1.ps1' } +$distribution = Join-Path $repository 'build\install\pathlab-forge' +$installer = Join-Path $repository 'scripts\evidence-mentor-service.ps1' +$content = @" +`$ErrorActionPreference = 'Stop' +& '$($installer.Replace("'", "''"))' -Action Upgrade -Version '2.1.1' ``` + -DistributionPath '$($distribution.Replace("'", "''"))' ``` + -JavaHome '$($JavaHome.Replace("'", "''"))' +"@ +$parent = Split-Path -Parent $OutputPath +New-Item -ItemType Directory -Path $parent -Force | Out-Null +[IO.File]::WriteAllText($OutputPath, $content, [Text.UTF8Encoding]::new($false)) +Write-Host "Elevated upgrade launcher created: $OutputPath" +Write-Host 'Run it once from an Administrator PowerShell. No reboot is requested.' diff --git a/scripts/stage-all-rounder-candidate-preflight.ps1 b/scripts/stage-all-rounder-candidate-preflight.ps1 new file mode 100644 index 0000000..c513ba8 --- /dev/null +++ b/scripts/stage-all-rounder-candidate-preflight.ps1 @@ -0,0 +1,75 @@ +[CmdletBinding()] +param( + [string] $StateRoot = 'D:\PathLabData\EvidenceMentor\state', + [string] $CampaignId = 'all-rounder-candidate-preflight-20260822-v1' +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version Latest +$repository = Split-Path -Parent $PSScriptRoot +$state = [IO.Path]::GetFullPath($StateRoot) +$campaignParent = [IO.Path]::GetFullPath((Join-Path $state 'acceptance\campaigns')) +$root = [IO.Path]::GetFullPath((Join-Path $campaignParent $CampaignId)) +if (-not $root.StartsWith($campaignParent + [IO.Path]::DirectorySeparatorChar, + [StringComparison]::OrdinalIgnoreCase)) { throw 'Campaign path escaped the protected state root.' } + +function Sha256([string] $Path) { (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant() } +function WriteJson([string] $Path, [object] $Value) { + [IO.File]::WriteAllText($Path, (($Value | ConvertTo-Json -Depth 16) + "`n"), + [Text.UTF8Encoding]::new($false)) +} + +$protocol = Join-Path $repository 'docs\evidence\candidate-preflight-protocol-v1.md' +$protocolSha = Sha256 $protocol +$specs = @( + @{ id='he-dinov2-small-v1'; capability='he-evidence'; scope='deployment' }, + @{ id='he-hibou-b-v1'; capability='he-evidence'; scope='deployment' }, + @{ id='he-gigapath-benchmark-v1'; capability='he-evidence'; scope='local-benchmark' }, + @{ id='cell-hovernet-fast-v1'; capability='cell-morphology'; scope='deployment' }, + @{ id='cell-pathosam-benchmark-v1'; capability='cell-morphology'; scope='local-benchmark' }, + @{ id='tutor-qwen3-0.6b-int4-v1'; capability='grounded-tutor'; scope='deployment' }, + @{ id='atlas-he-72m-v1'; capability='atlas-distillation'; scope='deployment' } +) + +New-Item -ItemType Directory -Path $root -Force | Out-Null +$campaignPath = Join-Path $root 'campaign.json' +$tracks = [Collections.Generic.List[object]]::new() +foreach ($spec in $specs) { + $packSource = Join-Path $repository "src\main\resources\evidence-packs\$($spec.id).json" + $packTarget = Join-Path $root "$($spec.id).json" + Copy-Item -LiteralPath $packSource -Destination $packTarget -Force + $requestName = "request-$($spec.id).json" + WriteJson (Join-Path $root $requestName) ([ordered]@{ packManifest = $packTarget }) + $tracks.Add([ordered]@{ + id = $spec.id; candidateId = $spec.id; capability = $spec.capability; scope = $spec.scope + requestPath = $requestName; remediationRequestPath = $null; expectedAttestationPath = $null + protocolSha256 = $protocolSha; dependsOn = @(); required = $true + }) +} +WriteJson $campaignPath ([ordered]@{ + schema = 'pathlab.qualification-campaign/1'; campaignId = $CampaignId + createdAt = '2026-08-22T00:00:00Z'; researchOnly = $true; notDiagnostic = $true + maxRemediationAttempts = 1 + quota = [ordered]@{ sourceBytes = 45GB; derivedBytes = 25GB; modelBytes = 10GB; evidenceBytes = 10GB; reserveBytes = 10GB } + tracks = $tracks +}) + +$endpoint = Get-Content -LiteralPath (Join-Path $state 'endpoint.json') -Raw | ConvertFrom-Json +if ($endpoint.serviceVersion -ne '2.1.1') { throw "Service 2.1.1 is required; found $($endpoint.serviceVersion)." } +$token = [IO.File]::ReadAllText((Join-Path $state 'ipc-token')).Trim() +try { + try { + $response = Invoke-RestMethod -Method Get -Uri "http://127.0.0.1:$($endpoint.port)/v1/qualification-runs/$CampaignId" -Headers @{ Authorization = "Bearer $token" } + } catch { + if ($_.Exception.Response.StatusCode -ne 404) { throw } + $response = Invoke-RestMethod -Method Post -Uri "http://127.0.0.1:$($endpoint.port)/v1/qualification-runs" ` + -Headers @{ Authorization = "Bearer $token" } -ContentType 'application/json' ` + -Body (@{ manifestPath = $campaignPath } | ConvertTo-Json -Compress) + } +} finally { Remove-Variable token -ErrorAction SilentlyContinue } + +[pscustomobject]@{ + CampaignId = $response.id; State = $response.state + CampaignCompleted = $response.campaignCompleted; CampaignTargetMet = $response.campaignTargetMet + ManifestPath = $campaignPath; ManifestSha256 = Sha256 $campaignPath +} diff --git a/src/main/java/org/pathlab/forge/evidence/EvidenceMentorRunner.java b/src/main/java/org/pathlab/forge/evidence/EvidenceMentorRunner.java index 1365fee..4962bad 100644 --- a/src/main/java/org/pathlab/forge/evidence/EvidenceMentorRunner.java +++ b/src/main/java/org/pathlab/forge/evidence/EvidenceMentorRunner.java @@ -30,7 +30,7 @@ /** Standalone authenticated loopback process for unattended Evidence Mentor jobs. */ public final class EvidenceMentorRunner implements AutoCloseable { private static final ObjectMapper JSON = new ObjectMapper(); - private static final String VERSION = "2.1.0"; + private static final String VERSION = "2.1.1"; private static final Duration LEASE = Duration.ofSeconds(45); private final Path stateRoot; private final String token; diff --git a/src/main/java/org/pathlab/forge/evidence/QualificationRunStore.java b/src/main/java/org/pathlab/forge/evidence/QualificationRunStore.java index 8932029..3d9049b 100644 --- a/src/main/java/org/pathlab/forge/evidence/QualificationRunStore.java +++ b/src/main/java/org/pathlab/forge/evidence/QualificationRunStore.java @@ -178,7 +178,12 @@ public synchronized void tick(EvidenceJobQueue queue, Instant now) throws IOExce terminalTrack(run.id(), track.id(), "not_evaluable", "DEPENDENCY_NOT_QUALIFIED", "A frozen dependency did not qualify", now); } else if (dependencyVerdicts.stream().allMatch("qualified"::equals)) { - submitChild(run.id(), declared, track.attempt(), queue, now); + try { + submitChild(run.id(), declared, track.attempt(), queue, now); + } catch (IllegalArgumentException invalidCandidate) { + terminalTrack(run.id(), track.id(), "not_evaluable", "CANDIDATE_PREFLIGHT_FAILED", + bounded(invalidCandidate.getMessage()), now); + } } continue; } diff --git a/src/main/resources/evidence-packs/atlas-he-72m-v1.json b/src/main/resources/evidence-packs/atlas-he-72m-v1.json new file mode 100644 index 0000000..eefaa0d --- /dev/null +++ b/src/main/resources/evidence-packs/atlas-he-72m-v1.json @@ -0,0 +1,15 @@ +{ + "schema": "pathlab.ai-pack/1", + "packId": "atlas-he-72m-v1", + "version": "1", + "capability": "atlas-distillation", + "acceptedStains": ["he", "ihc_dab", "generic_brightfield"], + "preprocessing": {"id": "atlas-feasibility-unstarted-v1", "tilePixels": 512}, + "artifacts": [], + "runtimeCompatibility": {"workerProtocol": "pathlab.model-worker-result/1", "executionProvider": "cuda", "cuda": "12.6", "gpuArchitecture": "sm_61", "requiresExternalWorker": true}, + "licenseLedger": [{"component": "Atlas-H&E candidate", "license": "NOT_YET_CREATED", "revision": "BASELINES_NOT_FROZEN", "permittedUse": "private-research", "redistributable": false, "derivativesAllowed": false}], + "rights": {"license": "Teacher derivative-rights gate not yet passed", "allowedUse": "private-research", "redistributable": false, "derivativesAllowed": false, "reviewedAt": "2026-08-22T00:00:00Z"}, + "resourceEnvelope": {"maxRamMiB": 16384, "maxVramMiB": 4608, "maxSeconds": 1200, "network": false}, + "validation": {"status": "not-evaluable", "modelCard": "docs/model-cards/atlas-he-72m-v1.md", "heldOutEvaluation": "not-evaluable-baseline-packs-and-lawful-teachers-not-frozen"}, + "outputSchema": "pathlab.ai-evidence/1" +} diff --git a/src/main/resources/evidence-packs/cell-pathosam-benchmark-v1.json b/src/main/resources/evidence-packs/cell-pathosam-benchmark-v1.json new file mode 100644 index 0000000..e462044 --- /dev/null +++ b/src/main/resources/evidence-packs/cell-pathosam-benchmark-v1.json @@ -0,0 +1,15 @@ +{ + "schema": "pathlab.ai-pack/1", + "packId": "cell-pathosam-benchmark-v1", + "version": "1", + "capability": "cell-morphology", + "acceptedStains": ["he", "ihc_dab"], + "preprocessing": {"id": "pathosam-uninstalled-v1", "tilePixels": 512}, + "artifacts": [], + "runtimeCompatibility": {"workerProtocol": "pathlab.model-worker-result/1", "executionProvider": "cuda", "cuda": "12.6", "gpuArchitecture": "sm_61", "requiresExternalWorker": true}, + "licenseLedger": [{"component": "PathoSAM code", "license": "MIT", "revision": "PIN_REQUIRED", "permittedUse": "benchmark-only", "redistributable": false, "derivativesAllowed": true}, {"component": "PathoSAM weights", "license": "WEIGHT_LINEAGE_REVIEW_REQUIRED", "revision": "PIN_REQUIRED", "permittedUse": "benchmark-only", "redistributable": false, "derivativesAllowed": false}], + "rights": {"license": "Code and weight rights must pass independently", "allowedUse": "benchmark-only", "redistributable": false, "derivativesAllowed": false, "reviewedAt": "2026-08-22T00:00:00Z"}, + "resourceEnvelope": {"maxRamMiB": 16384, "maxVramMiB": 4608, "maxSeconds": 300, "network": false}, + "validation": {"status": "not-evaluable", "modelCard": "docs/model-cards/cell-pathosam-benchmark-v1.md", "heldOutEvaluation": "not-evaluable-no-installed-rights-cleared-weights"}, + "outputSchema": "pathlab.ai-evidence/1" +} diff --git a/src/main/resources/evidence-packs/tutor-qwen3-0.6b-int4-v1.json b/src/main/resources/evidence-packs/tutor-qwen3-0.6b-int4-v1.json new file mode 100644 index 0000000..6343232 --- /dev/null +++ b/src/main/resources/evidence-packs/tutor-qwen3-0.6b-int4-v1.json @@ -0,0 +1,15 @@ +{ + "schema": "pathlab.ai-pack/1", + "packId": "tutor-qwen3-0.6b-int4-v1", + "version": "1", + "capability": "grounded-tutor", + "acceptedStains": ["he"], + "preprocessing": {"id": "claim-id-only-v1", "tilePixels": 512}, + "artifacts": [], + "runtimeCompatibility": {"workerProtocol": "pathlab.model-worker-result/1", "executionProvider": "cpu", "cuda": "none", "gpuArchitecture": "none", "requiresExternalWorker": true}, + "licenseLedger": [{"component": "Qwen3-0.6B upstream weights", "license": "Apache-2.0", "revision": "PIN_REQUIRED", "permittedUse": "private-research", "redistributable": false, "derivativesAllowed": true}, {"component": "WebGPU INT4 ONNX browser package", "license": "ARTIFACT_REVIEW_REQUIRED", "revision": "PIN_REQUIRED", "permittedUse": "private-research", "redistributable": false, "derivativesAllowed": false}], + "rights": {"license": "Upstream Apache-2.0; converted artifact ledger pending", "allowedUse": "private-research", "redistributable": false, "derivativesAllowed": false, "reviewedAt": "2026-08-22T00:00:00Z"}, + "resourceEnvelope": {"maxRamMiB": 4096, "maxVramMiB": 0, "maxSeconds": 300, "network": false}, + "validation": {"status": "not-evaluable", "modelCard": "docs/model-cards/tutor-qwen3-0.6b-int4-v1.md", "heldOutEvaluation": "not-evaluable-complete-browser-asset-exceeds-500-mib-cap"}, + "outputSchema": "pathlab.ai-evidence/1" +} diff --git a/src/test/java/org/pathlab/forge/evidence/EvidenceMentorRunnerTest.java b/src/test/java/org/pathlab/forge/evidence/EvidenceMentorRunnerTest.java index 6ab1804..fc21fcb 100644 --- a/src/test/java/org/pathlab/forge/evidence/EvidenceMentorRunnerTest.java +++ b/src/test/java/org/pathlab/forge/evidence/EvidenceMentorRunnerTest.java @@ -96,7 +96,7 @@ void acceptsAuthenticatedLoopbackSubmissionIntoDurableQueue() throws Exception { var endpoint = JSON.readTree(temporaryDirectory.resolve("state/endpoint.json").toFile()); assertEquals("pathlab.runner-endpoint/1", endpoint.path("schema").asText()); - assertEquals("2.1.0", endpoint.path("serviceVersion").asText()); + assertEquals("2.1.1", endpoint.path("serviceVersion").asText()); assertEquals(runner.uri("/").getPort(), endpoint.path("port").asInt()); assertTrue(!endpoint.has("token")); } diff --git a/src/test/java/org/pathlab/forge/evidence/QualificationRunStoreTest.java b/src/test/java/org/pathlab/forge/evidence/QualificationRunStoreTest.java index bcab299..c4afc2a 100644 --- a/src/test/java/org/pathlab/forge/evidence/QualificationRunStoreTest.java +++ b/src/test/java/org/pathlab/forge/evidence/QualificationRunStoreTest.java @@ -42,6 +42,31 @@ void persistsSignedAttestationAndComputesTargetWithoutChangingCandidateManifest( } } + @Test + void unavailableCandidateFinishesNotEvaluableWithoutBlockingCampaignTick() throws Exception { + var missingPack = temporaryDirectory.resolve("missing-pack.json").toAbsolutePath(); + Files.writeString(temporaryDirectory.resolve("request.json"), + "{\"packManifest\":" + new com.fasterxml.jackson.databind.ObjectMapper() + .writeValueAsString(missingPack.toString()) + "}"); + var campaignPath = temporaryDirectory.resolve("campaign.json"); + Files.writeString(campaignPath, QualificationCampaignManifestTest.campaign("request.json")); + var campaign = QualificationCampaignManifest.load(campaignPath); + var state = Files.createDirectories(temporaryDirectory.resolve("state")); + try (var store = new QualificationRunStore(state.resolve("jobs.sqlite3"), state); + var queue = new EvidenceJobQueue(state.resolve("jobs.sqlite3"))) { + store.create(campaign, Instant.parse("2026-08-22T00:00:00Z")); + store.tick(queue, Instant.parse("2026-08-22T00:00:01Z")); + + var completed = store.snapshot(campaign.campaignId()).orElseThrow(); + assertTrue(completed.campaignCompleted()); + assertFalse(completed.campaignTargetMet()); + assertEquals("not_evaluable", completed.tracks().get(0).verdict()); + assertEquals("CANDIDATE_PREFLIGHT_FAILED", completed.tracks().get(0).failureCode()); + assertTrue(Files.isRegularFile(state.resolve("qualification") + .resolve(campaign.campaignId()).resolve("capability-matrix.json"))); + } + } + private static String deploymentPack() { return """ {"schema":"pathlab.ai-pack/2","packId":"dinov2-small","version":"1.0.0", From aeca4e2141007d0fb5f7df8ff00e34905cbcd4cc Mon Sep 17 00:00:00 2001 From: Enksodsoon <229880765+Enksodsoon@users.noreply.github.com> Date: Sat, 22 Aug 2026 21:07:26 +0700 Subject: [PATCH 26/50] Add fail-closed post-reboot service recovery --- docs/plans/active/current.md | 9 +++ .../complete-evidence-mentor-after-reboot.ps1 | 76 +++++++++++++++++++ ...idence-mentor-post-reboot-continuation.ps1 | 24 ++++++ 3 files changed, 109 insertions(+) create mode 100644 scripts/complete-evidence-mentor-after-reboot.ps1 create mode 100644 scripts/install-evidence-mentor-post-reboot-continuation.ps1 diff --git a/docs/plans/active/current.md b/docs/plans/active/current.md index 48eb9a8..3e63d02 100644 --- a/docs/plans/active/current.md +++ b/docs/plans/active/current.md @@ -46,6 +46,15 @@ now stages replacement rules before removing old rules and restores the prior runtime configuration on any setup or health failure. Model analysis must not be activated until an outbound-deny rule is verifiably installed. +Read-only DISM found a healthy component store, while SFC found and repaired +protected-file integrity violations. The active firewall provider still needs a +Windows restart before it can accept new rules. Claims are durably paused with +an empty queue. The one-time SYSTEM task `PathLabEvidenceMentorPostRepair` is +prepared to retry the transactional upgrade two minutes after startup, verify +exact outbound-deny coverage and service acceptance, and resume claims only on +PASS. It records a durable result and removes itself after success; a failure +remains paused and fail-closed. + The branch now implements `pathlab.qualification-campaign/1` as a durable parent coordinator above the existing GPU and CPU/I/O lanes. It records frozen dependencies, attempts, signed attestations, completion versus target success, diff --git a/scripts/complete-evidence-mentor-after-reboot.ps1 b/scripts/complete-evidence-mentor-after-reboot.ps1 new file mode 100644 index 0000000..174a94f --- /dev/null +++ b/scripts/complete-evidence-mentor-after-reboot.ps1 @@ -0,0 +1,76 @@ +[CmdletBinding()] +param( + [string] $StateRoot = 'D:\PathLabData\EvidenceMentor\state', + [string] $ProgramRoot = 'C:\ProgramData\PathLab\EvidenceMentor', + [string] $DistributionPath = 'C:\Users\enkso\.codex\worktrees\evidence-mentor\forge\build\install\pathlab-forge', + [string] $JavaHome = 'C:\Users\enkso\.gradle\jdks\eclipse_adoptium-17-amd64-windows.2' +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version Latest +$version = '2.1.1' +$taskName = 'PathLabEvidenceMentorPostRepair' +$repository = Split-Path -Parent $PSScriptRoot +$acceptanceRoot = Join-Path $StateRoot 'acceptance\post-reboot-2.1.1' +$resultPath = Join-Path $acceptanceRoot 'result.json' +$logPath = Join-Path $acceptanceRoot 'continuation.log' +New-Item -ItemType Directory -Path $acceptanceRoot -Force | Out-Null + +function Write-Result([string] $Status, [string] $Detail) { + $value = [ordered]@{ + schema = 'pathlab.post-reboot-continuation/1' + version = $version + status = $Status + detail = $Detail + completedAt = [DateTimeOffset]::UtcNow.ToString('o') + } | ConvertTo-Json -Depth 6 + $partial = "$resultPath.partial" + [IO.File]::WriteAllText($partial, $value + "`n", [Text.UTF8Encoding]::new($false)) + Move-Item -LiteralPath $partial -Destination $resultPath -Force +} + +function Set-Claims([bool] $Accepting) { + $endpoint = Get-Content -LiteralPath (Join-Path $StateRoot 'endpoint.json') -Raw | ConvertFrom-Json + $token = [IO.File]::ReadAllText((Join-Path $StateRoot 'ipc-token')).Trim() + $action = if ($Accepting) { 'resume' } else { 'pause' } + Invoke-RestMethod -Method Post -Uri "http://127.0.0.1:$($endpoint.port)/v1/control/$action" ` + -Headers @{ Authorization = "Bearer $token" } -ContentType 'application/json' -Body '{}' | Out-Null +} + +Start-Transcript -LiteralPath $logPath -Append | Out-Null +try { + try { Set-Claims $false } catch { } + & (Join-Path $repository 'scripts\evidence-mentor-service.ps1') -Action Upgrade -Version $version ` + -DistributionPath $DistributionPath -JavaHome $JavaHome -ProgramRoot $ProgramRoot -StateRoot $StateRoot + + $runtime = Join-Path $ProgramRoot "runtime\$version" + $expectedPrograms = @((Join-Path $runtime 'jre\bin\java.exe')) + $modelRoot = Join-Path $StateRoot 'models' + if (Test-Path -LiteralPath $modelRoot) { + $expectedPrograms += Get-ChildItem -LiteralPath $modelRoot -Filter '*.exe' -File -Recurse | + Select-Object -ExpandProperty FullName + } + $expectedPrograms = @($expectedPrograms | ForEach-Object { [IO.Path]::GetFullPath($_).ToLowerInvariant() } | + Sort-Object -Unique) + $rules = @(Get-NetFirewallRule -DisplayName "PathLab Evidence Mentor outbound deny $version *" -ErrorAction Stop) + $actualPrograms = @($rules | Get-NetFirewallApplicationFilter | Select-Object -ExpandProperty Program | + ForEach-Object { [IO.Path]::GetFullPath($_).ToLowerInvariant() } | Sort-Object -Unique) + if ($rules.Count -ne $expectedPrograms.Count -or + (Compare-Object -ReferenceObject $expectedPrograms -DifferenceObject $actualPrograms)) { + throw 'Installed outbound-deny rules do not exactly cover the analysis executables.' + } + + & (Join-Path $repository 'scripts\test-evidence-mentor-service.ps1') -Mode Inspect ` + -ProgramRoot $ProgramRoot -StateRoot $StateRoot -ReportRoot $acceptanceRoot + if ($LASTEXITCODE -ne 0) { throw "Post-reboot service acceptance failed with exit code $LASTEXITCODE." } + + Set-Claims $true + Write-Result 'PASS' 'Firewall rules, service health, and non-reboot acceptance passed; claims resumed.' + Unregister-ScheduledTask -TaskName $taskName -Confirm:$false -ErrorAction SilentlyContinue +} catch { + try { Set-Claims $false } catch { } + Write-Result 'FAIL' $_.Exception.Message + throw +} finally { + Stop-Transcript | Out-Null +} diff --git a/scripts/install-evidence-mentor-post-reboot-continuation.ps1 b/scripts/install-evidence-mentor-post-reboot-continuation.ps1 new file mode 100644 index 0000000..1f12b43 --- /dev/null +++ b/scripts/install-evidence-mentor-post-reboot-continuation.ps1 @@ -0,0 +1,24 @@ +[CmdletBinding()] +param() + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version Latest +$identity = [Security.Principal.WindowsIdentity]::GetCurrent() +$principal = [Security.Principal.WindowsPrincipal]::new($identity) +if (-not $principal.IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator)) { + throw 'Administrator approval is required to install the post-reboot continuation.' +} + +$taskName = 'PathLabEvidenceMentorPostRepair' +$continuation = Join-Path $PSScriptRoot 'complete-evidence-mentor-after-reboot.ps1' +$action = New-ScheduledTaskAction -Execute 'powershell.exe' ` + -Argument "-NoProfile -ExecutionPolicy Bypass -File `"$continuation`"" +$trigger = New-ScheduledTaskTrigger -AtStartup +$trigger.Delay = 'PT2M' +$taskPrincipal = New-ScheduledTaskPrincipal -UserId 'SYSTEM' -LogonType ServiceAccount -RunLevel Highest +$settings = New-ScheduledTaskSettingsSet -StartWhenAvailable -ExecutionTimeLimit (New-TimeSpan -Minutes 30) ` + -MultipleInstances IgnoreNew +Register-ScheduledTask -TaskName $taskName -Action $action -Trigger $trigger -Principal $taskPrincipal ` + -Settings $settings -Description 'Complete fail-closed PathLab 2.1.1 firewall and service acceptance after Windows repairs load.' ` + -Force | Out-Null +Get-ScheduledTask -TaskName $taskName | Select-Object TaskName,State From 6f3157c60ea951b1f644e0de2839486f1b11865d Mon Sep 17 00:00:00 2001 From: Enksodsoon <229880765+Enksodsoon@users.noreply.github.com> Date: Sat, 22 Aug 2026 21:12:12 +0700 Subject: [PATCH 27/50] Make post-reboot recovery workspace independent --- .../complete-evidence-mentor-after-reboot.ps1 | 48 +++++++++++++++---- ...idence-mentor-post-reboot-continuation.ps1 | 6 ++- 2 files changed, 45 insertions(+), 9 deletions(-) diff --git a/scripts/complete-evidence-mentor-after-reboot.ps1 b/scripts/complete-evidence-mentor-after-reboot.ps1 index 174a94f..f40dfbf 100644 --- a/scripts/complete-evidence-mentor-after-reboot.ps1 +++ b/scripts/complete-evidence-mentor-after-reboot.ps1 @@ -1,16 +1,13 @@ [CmdletBinding()] param( [string] $StateRoot = 'D:\PathLabData\EvidenceMentor\state', - [string] $ProgramRoot = 'C:\ProgramData\PathLab\EvidenceMentor', - [string] $DistributionPath = 'C:\Users\enkso\.codex\worktrees\evidence-mentor\forge\build\install\pathlab-forge', - [string] $JavaHome = 'C:\Users\enkso\.gradle\jdks\eclipse_adoptium-17-amd64-windows.2' + [string] $ProgramRoot = 'C:\ProgramData\PathLab\EvidenceMentor' ) $ErrorActionPreference = 'Stop' Set-StrictMode -Version Latest $version = '2.1.1' $taskName = 'PathLabEvidenceMentorPostRepair' -$repository = Split-Path -Parent $PSScriptRoot $acceptanceRoot = Join-Path $StateRoot 'acceptance\post-reboot-2.1.1' $resultPath = Join-Path $acceptanceRoot 'result.json' $logPath = Join-Path $acceptanceRoot 'continuation.log' @@ -37,12 +34,43 @@ function Set-Claims([bool] $Accepting) { -Headers @{ Authorization = "Bearer $token" } -ContentType 'application/json' -Body '{}' | Out-Null } +function Install-OutboundDeny([string[]] $Programs) { + $desiredNames = [Collections.Generic.List[string]]::new() + $createdNames = [Collections.Generic.List[string]]::new() + try { + $index = 0 + foreach ($program in $Programs) { + if (-not (Test-Path -LiteralPath $program -PathType Leaf)) { + throw "Analysis executable is unavailable: $program" + } + $name = "PathLab Evidence Mentor outbound deny $version $index" + $desiredNames.Add($name) + if ($null -eq (Get-NetFirewallRule -DisplayName $name -ErrorAction SilentlyContinue)) { + New-NetFirewallRule -DisplayName $name -Direction Outbound -Action Block ` + -Program $program -Profile Any -ErrorAction Stop | Out-Null + $createdNames.Add($name) + } + $actual = Get-NetFirewallRule -DisplayName $name -ErrorAction Stop | + Get-NetFirewallApplicationFilter | Select-Object -ExpandProperty Program -First 1 + if ([IO.Path]::GetFullPath($actual) -ne [IO.Path]::GetFullPath($program)) { + throw "Outbound-deny rule program mismatch: $name" + } + $index++ + } + Get-NetFirewallRule -DisplayName 'PathLab Evidence Mentor outbound deny*' -ErrorAction SilentlyContinue | + Where-Object { $desiredNames -notcontains $_.DisplayName } | Remove-NetFirewallRule + } catch { + foreach ($name in $createdNames) { + Get-NetFirewallRule -DisplayName $name -ErrorAction SilentlyContinue | + Remove-NetFirewallRule -ErrorAction SilentlyContinue + } + throw + } +} + Start-Transcript -LiteralPath $logPath -Append | Out-Null try { try { Set-Claims $false } catch { } - & (Join-Path $repository 'scripts\evidence-mentor-service.ps1') -Action Upgrade -Version $version ` - -DistributionPath $DistributionPath -JavaHome $JavaHome -ProgramRoot $ProgramRoot -StateRoot $StateRoot - $runtime = Join-Path $ProgramRoot "runtime\$version" $expectedPrograms = @((Join-Path $runtime 'jre\bin\java.exe')) $modelRoot = Join-Path $StateRoot 'models' @@ -52,6 +80,7 @@ try { } $expectedPrograms = @($expectedPrograms | ForEach-Object { [IO.Path]::GetFullPath($_).ToLowerInvariant() } | Sort-Object -Unique) + Install-OutboundDeny $expectedPrograms $rules = @(Get-NetFirewallRule -DisplayName "PathLab Evidence Mentor outbound deny $version *" -ErrorAction Stop) $actualPrograms = @($rules | Get-NetFirewallApplicationFilter | Select-Object -ExpandProperty Program | ForEach-Object { [IO.Path]::GetFullPath($_).ToLowerInvariant() } | Sort-Object -Unique) @@ -60,7 +89,10 @@ try { throw 'Installed outbound-deny rules do not exactly cover the analysis executables.' } - & (Join-Path $repository 'scripts\test-evidence-mentor-service.ps1') -Mode Inspect ` + $activeVersion = Join-Path $ProgramRoot 'active-version.txt' + [IO.File]::WriteAllText("$activeVersion.partial", "$version`n", [Text.UTF8Encoding]::new($false)) + Move-Item -LiteralPath "$activeVersion.partial" -Destination $activeVersion -Force + & (Join-Path $ProgramRoot 'Test-PathLab-Evidence-Service.ps1') -Mode Inspect ` -ProgramRoot $ProgramRoot -StateRoot $StateRoot -ReportRoot $acceptanceRoot if ($LASTEXITCODE -ne 0) { throw "Post-reboot service acceptance failed with exit code $LASTEXITCODE." } diff --git a/scripts/install-evidence-mentor-post-reboot-continuation.ps1 b/scripts/install-evidence-mentor-post-reboot-continuation.ps1 index 1f12b43..2ad912e 100644 --- a/scripts/install-evidence-mentor-post-reboot-continuation.ps1 +++ b/scripts/install-evidence-mentor-post-reboot-continuation.ps1 @@ -10,7 +10,11 @@ if (-not $principal.IsInRole([Security.Principal.WindowsBuiltInRole]::Administra } $taskName = 'PathLabEvidenceMentorPostRepair' -$continuation = Join-Path $PSScriptRoot 'complete-evidence-mentor-after-reboot.ps1' +$programRoot = 'C:\ProgramData\PathLab\EvidenceMentor' +$continuation = Join-Path $programRoot 'Complete-PathLab-EvidenceMentor-PostRepair.ps1' +New-Item -ItemType Directory -Path $programRoot -Force | Out-Null +Copy-Item -LiteralPath (Join-Path $PSScriptRoot 'complete-evidence-mentor-after-reboot.ps1') ` + -Destination $continuation -Force $action = New-ScheduledTaskAction -Execute 'powershell.exe' ` -Argument "-NoProfile -ExecutionPolicy Bypass -File `"$continuation`"" $trigger = New-ScheduledTaskTrigger -AtStartup From e562e92a55a7a41c3f0dc4c5cad20038798196a7 Mon Sep 17 00:00:00 2001 From: Enksodsoon <229880765+Enksodsoon@users.noreply.github.com> Date: Sat, 22 Aug 2026 21:35:29 +0700 Subject: [PATCH 28/50] feat(ai): add autonomous real-data acquisition telemetry --- docs/plans/active/current.md | 25 ++- scripts/acquire-nct-crc-he.ps1 | 194 ++++++++++++++++++ scripts/evidence-mentor-service.ps1 | 8 +- .../prepare-evidence-mentor-2.1.2-upgrade.ps1 | 36 ++++ .../forge/evidence/EvidenceMentorRunner.java | 41 +++- .../resources/evidence-dashboard/index.html | 4 +- .../evidence/EvidenceMentorRunnerTest.java | 25 ++- 7 files changed, 326 insertions(+), 7 deletions(-) create mode 100644 scripts/acquire-nct-crc-he.ps1 create mode 100644 scripts/prepare-evidence-mentor-2.1.2-upgrade.ps1 diff --git a/docs/plans/active/current.md b/docs/plans/active/current.md index 3e63d02..fc0d1bd 100644 --- a/docs/plans/active/current.md +++ b/docs/plans/active/current.md @@ -36,7 +36,7 @@ byte-identical runtime is reused; a mismatch fails before the active configuration changes. The manual reboot-continuation acceptance is still deferred, so Gate 5 and activation remain incomplete. -Runner `2.1.1` adds fail-closed candidate preflight terminalization: immutable +Runner `2.1.1` added fail-closed candidate preflight terminalization: immutable candidate validation failures become `not_evaluable` rather than blocking the parent coordinator, while database and filesystem I/O failures still surface. The runtime starts and reports healthy locally, but the upgrade is not accepted: @@ -55,6 +55,29 @@ exact outbound-deny coverage and service acceptance, and resume claims only on PASS. It records a durable result and removes itself after success; a failure remains paused and fail-closed. +The product owner deferred that restart on 2026-08-22 so real research work can +continue while the PC stays on. This is an explicit degraded-environment waiver, +not firewall acceptance: new model results remain experimental and cannot enter +the qualified registry, learner evidence, deployment activation, or a scientific +claim until outbound-deny and restart-continuation acceptance pass. + +Real GI data acquisition is now active as an interactive-user, resumable task. +It is downloading the immutable Zenodo record 1214456 NCT-CRC H&E reference and +validation archives (12,490,560,932 bytes total), pinned to the official file +sizes and MD5 values under CC BY 4.0. Completion additionally records SHA-256 +values and a source ledger. The task enforces the 45 GB new-source quota and +runs independently of Forge, Codex, and the analysis service. The dashboard +status contract exposes only bounded acquisition metadata; analysis remains +credential-free and network-disabled. These patch archives add a real GI +external evaluation source, but do not by themselves prove patient-held-out +qualification when patient identity metadata is unavailable. + +Runner `2.1.2` is the next staged package. It adds the bounded acquisition +status feed and dashboard rendering without changing the paused analysis state. +It must not replace the installed runtime until the same transactional firewall +and health checks can succeed; source download completion does not depend on +that service upgrade. + The branch now implements `pathlab.qualification-campaign/1` as a durable parent coordinator above the existing GPU and CPU/I/O lanes. It records frozen dependencies, attempts, signed attestations, completion versus target success, diff --git a/scripts/acquire-nct-crc-he.ps1 b/scripts/acquire-nct-crc-he.ps1 new file mode 100644 index 0000000..daf5bef --- /dev/null +++ b/scripts/acquire-nct-crc-he.ps1 @@ -0,0 +1,194 @@ +[CmdletBinding()] +param( + [ValidateSet('Start','Run','Status')] [string] $Action = 'Start', + [string] $StateRoot = 'D:\PathLabData\EvidenceMentor\state' +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version Latest +$recordId = '1214456' +$datasetId = 'nct-crc-he-1214456' +$jobName = 'PathLab-NCT-CRC-HE-1214456' +$taskName = 'PathLabNctCrcAcquisition' +$sourceLimit = 45GB +$expected = [ordered]@{ + 'NCT-CRC-HE-100K.zip' = @{ bytes = 11690284003L; md5 = '6fd702d11df6292bc054397ae038a464' } + 'CRC-VAL-HE-7K.zip' = @{ bytes = 800276929L; md5 = '2fd1651b4f94ebd818ebf90ad2b6ce06' } +} +$requiredBytes = 0L +foreach ($item in $expected.Values) { $requiredBytes += [long]$item.bytes } +$state = [IO.Path]::GetFullPath($StateRoot) +$acquisitionRoot = Join-Path $state "acquisition\$datasetId" +$downloadRoot = Join-Path $acquisitionRoot 'downloads' +$sourceRoot = Join-Path $state "sources\$datasetId" +$statusPath = Join-Path $acquisitionRoot 'status.json' +$recordPath = Join-Path $acquisitionRoot 'zenodo-record.json' +$ledgerPath = Join-Path $sourceRoot 'sample-ledger.jsonl' +$reservationPath = Join-Path $state "quota\reservations\source\$datasetId.reservation" + +function Write-JsonAtomic([string] $Path, [object] $Value) { + $parent = Split-Path -Parent $Path + New-Item -ItemType Directory -Path $parent -Force | Out-Null + $partial = "$Path.partial" + [IO.File]::WriteAllText($partial, (($Value | ConvertTo-Json -Depth 16) + "`n"), + [Text.UTF8Encoding]::new($false)) + Move-Item -LiteralPath $partial -Destination $Path -Force +} + +function Write-Status([string] $State, [long] $Completed, [long] $Total, [string] $Detail) { + Write-JsonAtomic $statusPath ([ordered]@{ + schema = 'pathlab.acquisition-status/1'; datasetId = $datasetId; state = $State + completedBytes = $Completed; totalBytes = $Total; detail = $Detail + networkContext = 'interactive-user-acquisition-only'; analysisNetwork = 'disabled' + updatedAt = [DateTimeOffset]::UtcNow.ToString('o') + }) +} + +function Directory-Bytes([string] $Path) { + if (-not (Test-Path -LiteralPath $Path -PathType Container)) { return 0L } + $measure = Get-ChildItem -LiteralPath $Path -File -Recurse -ErrorAction Stop | + Measure-Object -Property Length -Sum + if ($null -eq $measure -or $null -eq $measure.Sum) { return 0L } + return [long]$measure.Sum +} + +function Remove-Finalizer { + Unregister-ScheduledTask -TaskName $taskName -Confirm:$false -ErrorAction SilentlyContinue +} + +function Get-DownloadedBytes { + $completed = 0L + foreach ($name in $expected.Keys) { + foreach ($candidate in @((Join-Path $downloadRoot $name), (Join-Path $downloadRoot "$name.partial"))) { + if (Test-Path -LiteralPath $candidate -PathType Leaf) { + $completed += [Math]::Min([long](Get-Item -LiteralPath $candidate).Length, + [long]$expected[$name].bytes) + break + } + } + } + return $completed +} + +if ($Action -eq 'Status') { + if (Test-Path -LiteralPath $statusPath -PathType Leaf) { Get-Content -LiteralPath $statusPath -Raw } + else { '{"schema":"pathlab.acquisition-status/1","state":"not_started"}' } + return +} + +foreach ($directory in @($acquisitionRoot, $downloadRoot, (Split-Path -Parent $reservationPath))) { + if (-not (Test-Path -LiteralPath $directory -PathType Container)) { + New-Item -ItemType Directory -Path $directory -Force | Out-Null + } +} + +if ($Action -eq 'Start') { + if (Test-Path -LiteralPath $statusPath -PathType Leaf) { + $prior = Get-Content -LiteralPath $statusPath -Raw | ConvertFrom-Json + if ($prior.state -eq 'completed') { $prior | ConvertTo-Json -Depth 8; return } + } + $used = Directory-Bytes (Join-Path $state 'sources') + $reserved = 0L + Get-ChildItem -LiteralPath (Split-Path -Parent $reservationPath) -Filter '*.reservation' -File ` + -ErrorAction SilentlyContinue | Where-Object FullName -ne $reservationPath | ForEach-Object { + $reserved += [long]([IO.File]::ReadAllText($_.FullName).Trim()) + } + if ($requiredBytes -gt $sourceLimit - $used - $reserved) { throw 'The 45 GB source quota cannot reserve NCT-CRC.' } + if (-not (Test-Path -LiteralPath $reservationPath -PathType Leaf)) { + [IO.File]::WriteAllText("$reservationPath.partial", [string]$requiredBytes, + [Text.UTF8Encoding]::new($false)) + Move-Item -LiteralPath "$reservationPath.partial" -Destination $reservationPath + } + + $record = Invoke-RestMethod -Uri "https://zenodo.org/api/records/$recordId" -TimeoutSec 60 + if ($record.id -ne 1214456 -or $record.metadata.doi -ne '10.5281/zenodo.1214456' -or + $record.metadata.license.id -ne 'cc-by-4.0') { + throw 'Zenodo identity or license did not match the frozen NCT-CRC acquisition policy.' + } + foreach ($name in $expected.Keys) { + $file = @($record.files | Where-Object key -eq $name) + if ($file.Count -ne 1 -or [long]$file[0].size -ne [long]$expected[$name].bytes -or + $file[0].checksum -ne "md5:$($expected[$name].md5)") { + throw "Zenodo file metadata changed: $name" + } + } + Write-JsonAtomic $recordPath $record + Get-BitsTransfer -Name $jobName -ErrorAction SilentlyContinue | + Remove-BitsTransfer -Confirm:$false -ErrorAction SilentlyContinue + $durableScript = Join-Path $acquisitionRoot 'acquire-nct-crc-he.ps1' + Copy-Item -LiteralPath $PSCommandPath -Destination $durableScript -Force + $taskAction = New-ScheduledTaskAction -Execute 'powershell.exe' ` + -Argument "-NoProfile -ExecutionPolicy Bypass -File `"$durableScript`" -Action Run -StateRoot `"$state`"" + $trigger = New-ScheduledTaskTrigger -Once -At (Get-Date).AddMinutes(1) ` + -RepetitionInterval (New-TimeSpan -Minutes 5) -RepetitionDuration (New-TimeSpan -Days 7) + $principal = New-ScheduledTaskPrincipal -UserId ([Security.Principal.WindowsIdentity]::GetCurrent().Name) ` + -LogonType Interactive -RunLevel Limited + $settings = New-ScheduledTaskSettingsSet -StartWhenAvailable -MultipleInstances IgnoreNew ` + -ExecutionTimeLimit (New-TimeSpan -Days 7) + Register-ScheduledTask -TaskName $taskName -Action $taskAction -Trigger $trigger -Principal $principal ` + -Settings $settings -Description 'Finalize checksum-bound NCT-CRC acquisition for PathLab.' -Force | Out-Null + Write-Status 'transferring' (Get-DownloadedBytes) $requiredBytes 'Resumable acquisition worker scheduled.' + Get-Content -LiteralPath $statusPath -Raw + return +} + +if (Test-Path -LiteralPath $statusPath) { + $prior = Get-Content -LiteralPath $statusPath -Raw | ConvertFrom-Json + if ($prior.state -eq 'completed') { Remove-Finalizer; return } +} + +$curl = (Get-Command curl.exe -ErrorAction Stop).Source +foreach ($name in $expected.Keys) { + $download = Join-Path $downloadRoot $name + if (Test-Path -LiteralPath $download -PathType Leaf) { + if ((Get-Item -LiteralPath $download).Length -eq [long]$expected[$name].bytes) { continue } + throw "Existing acquisition file has the wrong size: $name" + } + $partial = "$download.partial" + $url = "https://zenodo.org/api/records/$recordId/files/$([Uri]::EscapeDataString($name))/content" + Write-Status 'transferring' (Get-DownloadedBytes) $requiredBytes "Downloading $name" + $arguments = @('--fail','--location','--continue-at','-','--retry','3','--retry-delay','5', + '--output',('"' + $partial + '"'),$url) + $process = Start-Process -FilePath $curl -ArgumentList $arguments -PassThru -NoNewWindow + while (-not $process.HasExited) { + Write-Status 'transferring' (Get-DownloadedBytes) $requiredBytes "Downloading $name" + Start-Sleep -Seconds 15 + $process.Refresh() + } + if ($process.ExitCode -ne 0) { + Write-Status 'transient_error' (Get-DownloadedBytes) $requiredBytes "curl exited $($process.ExitCode); scheduled retry retained." + throw "NCT-CRC download failed with curl exit code $($process.ExitCode)." + } + if ((Get-Item -LiteralPath $partial).Length -ne [long]$expected[$name].bytes) { + Write-Status 'failed' (Get-DownloadedBytes) $requiredBytes "Downloaded size mismatch: $name" + throw "NCT-CRC downloaded size mismatch: $name" + } + Move-Item -LiteralPath $partial -Destination $download +} + +Write-Status 'validating' $requiredBytes $requiredBytes 'Validating official MD5 and local SHA-256 checksums.' +New-Item -ItemType Directory -Path $sourceRoot -Force | Out-Null +$ledger = [Collections.Generic.List[string]]::new() +foreach ($name in $expected.Keys) { + $download = Join-Path $downloadRoot $name + if ((Get-Item -LiteralPath $download).Length -ne [long]$expected[$name].bytes -or + (Get-FileHash -LiteralPath $download -Algorithm MD5).Hash.ToLowerInvariant() -ne $expected[$name].md5) { + Write-Status 'failed' 0 $requiredBytes "Checksum validation failed: $name" + throw "NCT-CRC checksum validation failed: $name" + } + $target = Join-Path $sourceRoot $name + Move-Item -LiteralPath $download -Destination $target -Force + $ledger.Add(([ordered]@{ + sampleId = [IO.Path]::GetFileNameWithoutExtension($name); source = 'Zenodo record 1214456' + patientGroup = 'dataset-defined-multiple-patients'; slideGroup = 'dataset-defined-patches' + sha256 = (Get-FileHash -LiteralPath $target -Algorithm SHA256).Hash.ToLowerInvariant() + upstreamMd5 = $expected[$name].md5; bytes = [long]$expected[$name].bytes + license = 'CC-BY-4.0'; permittedUse = 'private-research'; task = 'he-retrieval-qualification' + split = if ($name -like 'CRC-VAL*') { 'query' } else { 'reference' } + } | ConvertTo-Json -Compress)) +} +[IO.File]::WriteAllLines($ledgerPath, $ledger, [Text.UTF8Encoding]::new($false)) +Copy-Item -LiteralPath $recordPath -Destination (Join-Path $sourceRoot 'zenodo-record.json') -Force +Remove-Item -LiteralPath $reservationPath -Force +Write-Status 'completed' $requiredBytes $requiredBytes 'NCT-CRC source archives are checksum-verified and ledgered.' +Remove-Finalizer diff --git a/scripts/evidence-mentor-service.ps1 b/scripts/evidence-mentor-service.ps1 index c8856a2..4d8b461 100644 --- a/scripts/evidence-mentor-service.ps1 +++ b/scripts/evidence-mentor-service.ps1 @@ -1,7 +1,7 @@ [CmdletBinding(SupportsShouldProcess)] param( [Parameter(Mandatory)] [ValidateSet('Install','Upgrade','Uninstall','Status')] [string] $Action, - [string] $Version = '2.1.1', + [string] $Version = '2.1.2', [string] $DistributionPath, [string] $JavaHome, [string] $ProgramRoot = 'C:\ProgramData\PathLab\EvidenceMentor', @@ -62,7 +62,8 @@ function Ensure-WinSW { function Grant-PathLabAccess([string] $InstallingUser) { & icacls.exe $ProgramRoot /inheritance:r /grant:r 'SYSTEM:(OI)(CI)F' 'Administrators:(OI)(CI)F' "NT SERVICE\${serviceName}:(OI)(CI)RX" "${InstallingUser}:(OI)(CI)RX" | Out-Null & icacls.exe $StateRoot /inheritance:r /grant:r 'SYSTEM:(OI)(CI)F' 'Administrators:(OI)(CI)F' "NT SERVICE\${serviceName}:(OI)(CI)M" "${InstallingUser}:(RX)" | Out-Null - foreach ($operatorPath in @('inputs','requests','models','artifacts','acceptance')) { + foreach ($operatorPath in @('inputs','requests','models','artifacts','acceptance','sources','acquisition', + 'quota\reservations\source')) { $resolved = Join-Path $StateRoot $operatorPath & icacls.exe $resolved /inheritance:r /grant:r 'SYSTEM:(OI)(CI)F' 'Administrators:(OI)(CI)F' "NT SERVICE\${serviceName}:(OI)(CI)M" "${InstallingUser}:(OI)(CI)M" | Out-Null } @@ -246,7 +247,8 @@ if ($Action -eq 'Uninstall') { $installingUser = [Security.Principal.WindowsIdentity]::GetCurrent().Name if ($PSCmdlet.ShouldProcess($serviceName, "$Action autonomous service version $Version")) { - $stateDirectories = @('inputs','requests','models','artifacts','acceptance','checkpoints','signing','logs') | ForEach-Object { Join-Path $StateRoot $_ } + $stateDirectories = @('inputs','requests','models','artifacts','acceptance','sources','acquisition', + 'quota\reservations\source','checkpoints','signing','logs') | ForEach-Object { Join-Path $StateRoot $_ } New-Item -ItemType Directory -Path @($ProgramRoot, $runtimeRoot, $StateRoot) -Force | Out-Null New-Item -ItemType Directory -Path $stateDirectories -Force | Out-Null $previousVersion = if (Test-Path -LiteralPath $activeVersionPath) { (Get-Content -LiteralPath $activeVersionPath -Raw).Trim() } else { '' } diff --git a/scripts/prepare-evidence-mentor-2.1.2-upgrade.ps1 b/scripts/prepare-evidence-mentor-2.1.2-upgrade.ps1 new file mode 100644 index 0000000..ee79b40 --- /dev/null +++ b/scripts/prepare-evidence-mentor-2.1.2-upgrade.ps1 @@ -0,0 +1,36 @@ +[CmdletBinding()] +param( + [string] $JavaHome = '', + [string] $OutputPath = '' +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version Latest +$repository = Split-Path -Parent $PSScriptRoot +if (-not $JavaHome) { + $candidate = Get-ChildItem -LiteralPath (Join-Path $env:USERPROFILE '.gradle\jdks') -Filter 'java.exe' -File -Recurse ` + -ErrorAction SilentlyContinue | Where-Object { $_.FullName -match '17' } | Select-Object -First 1 + if ($null -eq $candidate) { throw 'Provide -JavaHome for a Java 17 runtime.' } + $JavaHome = Split-Path -Parent (Split-Path -Parent $candidate.FullName) +} +if (-not (Test-Path -LiteralPath (Join-Path $JavaHome 'bin\java.exe') -PathType Leaf)) { + throw 'JavaHome must contain bin\java.exe.' +} + +& (Join-Path $repository 'gradlew.bat') clean test installDist +if ($LASTEXITCODE -ne 0) { throw 'The 2.1.2 distribution did not pass its build and tests.' } + +if (-not $OutputPath) { $OutputPath = Join-Path $repository 'build\Upgrade-PathLab-EvidenceMentor-2.1.2.ps1' } +$distribution = Join-Path $repository 'build\install\pathlab-forge' +$installer = Join-Path $repository 'scripts\evidence-mentor-service.ps1' +$content = @" +`$ErrorActionPreference = 'Stop' +& '$($installer.Replace("'", "''"))' -Action Upgrade -Version '2.1.2' ``` + -DistributionPath '$($distribution.Replace("'", "''"))' ``` + -JavaHome '$($JavaHome.Replace("'", "''"))' +"@ +$parent = Split-Path -Parent $OutputPath +New-Item -ItemType Directory -Path $parent -Force | Out-Null +[IO.File]::WriteAllText($OutputPath, $content, [Text.UTF8Encoding]::new($false)) +Write-Host "Elevated upgrade launcher created: $OutputPath" +Write-Host 'Run it once from an Administrator PowerShell. No reboot is requested.' diff --git a/src/main/java/org/pathlab/forge/evidence/EvidenceMentorRunner.java b/src/main/java/org/pathlab/forge/evidence/EvidenceMentorRunner.java index 4962bad..270808c 100644 --- a/src/main/java/org/pathlab/forge/evidence/EvidenceMentorRunner.java +++ b/src/main/java/org/pathlab/forge/evidence/EvidenceMentorRunner.java @@ -18,6 +18,7 @@ import java.time.Instant; import java.util.Base64; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; @@ -30,7 +31,7 @@ /** Standalone authenticated loopback process for unattended Evidence Mentor jobs. */ public final class EvidenceMentorRunner implements AutoCloseable { private static final ObjectMapper JSON = new ObjectMapper(); - private static final String VERSION = "2.1.1"; + private static final String VERSION = "2.1.2"; private static final Duration LEASE = Duration.ofSeconds(45); private final Path stateRoot; private final String token; @@ -223,6 +224,7 @@ private void status(HttpExchange exchange) throws IOException { "active", campaigns.stream().filter(run -> !run.campaignCompleted()).count(), "completed", campaigns.stream().filter(QualificationRunSnapshot::campaignCompleted).count(), "targetMet", campaigns.stream().filter(QualificationRunSnapshot::campaignTargetMet).count())); + status.put("acquisitions", acquisitionSnapshots()); campaigns.stream().filter(run -> !run.campaignCompleted()).findFirst() .ifPresent(run -> status.put("leadingQualificationRun", Map.of( "id", run.id(), "state", run.state(), @@ -239,6 +241,43 @@ private void status(HttpExchange exchange) throws IOException { returnJsonCached(exchange, status); } + private List> acquisitionSnapshots() { + var root = stateRoot.resolve("acquisition"); + if (!Files.isDirectory(root)) return List.of(); + try (var directories = Files.list(root)) { + return directories.filter(Files::isDirectory).sorted().limit(20).map(directory -> { + try { + var value = JSON.readTree(directory.resolve("status.json").toFile()); + require(fieldNames(value).equals(Set.of("schema", "datasetId", "state", "completedBytes", + "totalBytes", "detail", "networkContext", "analysisNetwork", "updatedAt")), + "Acquisition status fields are invalid"); + require("pathlab.acquisition-status/1".equals(value.path("schema").asText()), + "Acquisition status schema is invalid"); + var datasetId = value.path("datasetId").asText(); + var state = value.path("state").asText(); + long completed = value.path("completedBytes").asLong(-1); + long total = value.path("totalBytes").asLong(-1); + var updatedAt = value.path("updatedAt").asText(); + require(datasetId.matches("[a-z0-9][a-z0-9._-]{0,119}"), "Acquisition dataset id is invalid"); + require(Set.of("transferring", "validating", "completed", "transient_error", "failed") + .contains(state), "Acquisition state is invalid"); + require(completed >= 0 && total > 0 && completed <= total, "Acquisition byte counts are invalid"); + require("interactive-user-acquisition-only".equals(value.path("networkContext").asText()) + && "disabled".equals(value.path("analysisNetwork").asText()), + "Acquisition network boundary is invalid"); + Instant.parse(updatedAt); + return Map.of("datasetId", datasetId, "state", state, + "completedBytes", completed, "totalBytes", total, "updatedAt", updatedAt, + "networkContext", "interactive-user-acquisition-only", "analysisNetwork", "disabled"); + } catch (Exception ignored) { + return null; + } + }).filter(java.util.Objects::nonNull).toList(); + } catch (IOException ignored) { + return List.of(); + } + } + private void listJobs(HttpExchange exchange) throws IOException { var query = query(exchange.getRequestURI().getRawQuery()); var allowed = Set.of("state", "lane", "limit"); require(allowed.containsAll(query.keySet()), "Unknown job-list filter"); diff --git a/src/main/resources/evidence-dashboard/index.html b/src/main/resources/evidence-dashboard/index.html index d30b089..51ff95c 100644 --- a/src/main/resources/evidence-dashboard/index.html +++ b/src/main/resources/evidence-dashboard/index.html @@ -11,6 +11,7 @@
Offline analysis operations

PathLab Evidence Mentor

Authenticating…
+

Data acquisition

Qualification campaigns

Jobs