Fix final-audit confirmation replay and prepare beta.31

Directory.Build.props

@@ -6,7 +6,7 @@
   -->
   <PropertyGroup>
     <VersionPrefix>0.1.1</VersionPrefix>
-    <VersionSuffix>beta.30</VersionSuffix>
+    <VersionSuffix>beta.31</VersionSuffix>
     <Version Condition="'$(VersionSuffix)' == ''">$(VersionPrefix)</Version>
     <Version Condition="'$(VersionSuffix)' != ''">$(VersionPrefix)-$(VersionSuffix)</Version>
     <PackageVersion>$(Version)</PackageVersion>

docs/release-notes/0.1.1-beta.31.md

@@ -0,0 +1,53 @@
+# 0.1.1-beta.31 — final-audit confirmation replay corrections
+
+The final beta.29 owner audit found five kinds of successful mutation whose exact
+confirmation retry returned an error. The writes had succeeded once; the caller
+could not reliably reconcile the result. Beta.30 had already been tagged from
+the earlier checkpoint, so this release retains its job-definition and run-action
+repairs and integrates the four additional final-audit findings. The published
+beta.30 tag and artifacts remain immutable.
+
+| Finding | Correction |
+| --- | --- |
+| [#333](https://github.com/BostonTechnologies/netratel/issues/333), retained from beta.30 | Reconcile completed job-definition mutations before current ETag validation; return the original resulting version and added item ID. The final audit expanded the reproduced family to update, parameter add/update/delete and step add/update/reorder/delete in both tested tenants. |
+| [#336](https://github.com/BostonTechnologies/netratel/issues/336) | Reconcile successful script deletion through retained scoped ownership and content integrity, returning the original deletion result without restoring script visibility. |
+| [#337](https://github.com/BostonTechnologies/netratel/issues/337) | Reconcile successful policy replacement, disable and revoke before mutable version/lifecycle validation; retain the original normalized result across later changes. |
+| [#338](https://github.com/BostonTechnologies/netratel/issues/338) | Reconcile successful artifact cleanup before rejecting already-cleaned state. Cleaned artifact status and download remain unavailable. |
+| [#339](https://github.com/BostonTechnologies/netratel/issues/339) | Admit deleted-job replay through the local delegation exchange only with retained ownership and an exact consumed-success confirmation binding, then recheck it at business dispatch. Ordinary deleted or missing object references remain denied. |
+
+The deleted-object boundary also covers the run-action replay repair
+[#334](https://github.com/BostonTechnologies/netratel/issues/334) retained from
+beta.30. That finding came from source review; it is separate from the five
+owner-observed defects.
+
+Replay preserves current caller, credential, client, resource, instance, tenant,
+agent, policy, payload and expiry checks. It does not repeat writes, native
+dispatch, cancellation, cleanup or accepted-operation audits. Fresh invalid
+plans remain unconsumed; fresh stale writes and altered requests remain denied.
+New receipts retain the original successful result. Legacy ID-only receipts
+keep their scoped compatibility behavior and cannot recreate historical fields
+that were never recorded.
+
+The repair extends existing confirmation, ownership and delegation contracts;
+it requires no new database migration. Existing generic CI and release workflows
+remain the build and publication gates. Focused tests cover real persisted
+confirmation lifecycles and the local exchange/business authorization boundary.
+
+## Next owner campaign
+
+Deploy only after this tag's build and distribution verification complete. Verify
+the running API, Web, HTTP MCP and migrations against the publication record;
+prepare independently enrolled fixed-version Windows/Linux executors and prove
+the selected consumer's loaded bridge and native schema identity. Preserve the
+key ring, existing grant scope/expiry, retained evidence and original cleanup
+dependencies.
+
+The [owner repeat campaign](../mcp-http/campaign-retest.md) uses the final beta.29
+audit: all 518 context IDs and histories, five replay defects, 28 admission
+denials and the remaining functional work. It retains the eight actual native
+prompt workflows and the exact isolated 240-call hour. Autofill remains deferred.
+
+Source delivery and distribution integrity do not complete owner functional
+acceptance in [#294](https://github.com/BostonTechnologies/netratel/issues/294)
+or the production-readiness epic
+[#282](https://github.com/BostonTechnologies/netratel/issues/282).

docs/release-notes/README.md

@@ -2,6 +2,7 @@
 
 These notes describe the named release and preserve historical status. [Release engineering](../deployment/RELEASES.md) describes the current publication process; the product version is controlled only by `Directory.Build.props`.
 
+- [0.1.1-beta.31 — final-audit confirmation replay corrections](0.1.1-beta.31.md)
 - [0.1.1-beta.30 — exact job mutation and run-action replay](0.1.1-beta.30.md)
 - [0.1.1-beta.29 candidate — managed automation retry attempts](0.1.1-beta.29.md)
 

src/NetRatel/NetRatel.API/Endpoints/Auth/McpLocalDelegationEndpoints.cs

@@ -88,12 +88,12 @@ public static IEndpointRouteBuilder MapMcpLocalDelegationEndpoints(this IEndpoin
             }
             if (!McpLocalDelegationTargetRequirements.TryGetAuthorizationTenant(pairing, out var authorizationTenantId))
                 return Denied(http, "local_delegation_target_invalid", StatusCodes.Status400BadRequest, pairing);
+            // Ordinary object requests retain their established target-binding
+            // failure before permission evaluation. They cannot see tombstones.
             if (target?.Model is McpOperationTargetModel.ObjectDerived && pairing.ObjectTargetResolutionEnabled &&
-                (objectTargets is null ||
+                pairing.ReplayBinding is null && (objectTargets is null ||
                  !await objectTargets.MatchesDelegationAsync(pairing, cancellationToken).ConfigureAwait(false)))
-            {
                 return Denied(http, "local_delegation_binding_mismatch", StatusCodes.Status403Forbidden, pairing);
-            }
 
             var credentialId = http.User.FindFirstValue(IntegrationCredentialAuthenticationHandler.CredentialIdClaimType);
             var ownerPrincipalId = http.User.FindFirstValue("netratel_principal_id");
@@ -115,6 +115,16 @@ public static IEndpointRouteBuilder MapMcpLocalDelegationEndpoints(this IEndpoin
                 [],
                 McpLocalDelegationPermissionMapper.RolesFor(operationAccess.RequiredScope),
                 [McpOperationAccessScopeNames.Canonical(operationAccess.RequiredScope)]);
+            // The pairing assertion authenticates the gateway, never the
+            // receipt owner. Only the admitted ingress credential supplies
+            // the owner and client used for an exact deleted-object replay.
+            if (target?.Model is McpOperationTargetModel.ObjectDerived && pairing.ObjectTargetResolutionEnabled &&
+                pairing.ReplayBinding is not null &&
+                (objectTargets is null ||
+                 !await objectTargets.MatchesDelegationAsync(pairing with { Identity = identity, IngressCredentialId = credentialId }, cancellationToken).ConfigureAwait(false)))
+            {
+                return Denied(http, "local_delegation_binding_mismatch", StatusCodes.Status403Forbidden, pairing);
+            }
             var assertion = tokens.Create(identity, new McpOperatorDelegationRequest(
                 pairing.Tool,
                 pairing.Operation,
@@ -127,6 +137,7 @@ public static IEndpointRouteBuilder MapMcpLocalDelegationEndpoints(this IEndpoin
             {
                 ObjectReference = pairing.ObjectReference,
                 ObjectTargetResolutionEnabled = pairing.ObjectTargetResolutionEnabled,
+                ReplayBinding = pairing.ReplayBinding,
                 IngressCredentialId = credentialId,
                 IngressPermission = requiredPermission
             });

src/NetRatel/NetRatel.API/Endpoints/Client/McpOperatorClientFileEndpoints.cs

@@ -573,12 +573,29 @@ private static async Task<IResult> CleanupArtifactConfirmAsync(
         var artifact = await GetOwnedArtifactAsync(artifacts, artifactId, context, includeContent: false, cancellationToken).ConfigureAwait(false);
         if (artifact is null)
             return Results.NotFound();
+
+        var confirmationRequest = new McpOperatorConfirmationRequest(
+            request.PlanToken, request.IdempotencyKey, ArtifactCleanupPayloadHash(artifactId), context.Decision);
+        // A completed cleanup keeps its authenticated ownership record after
+        // wiping the bytes. Reconcile that receipt before rejecting deleted state.
+        var replayRequest = confirmationRequest with { ReplayOnly = true };
+        var replay = await confirmations.ConfirmAsync(replayRequest, cancellationToken).ConfigureAwait(false);
+        if (replay.FailureCode is { } replayFailure)
+            return Failure(replayFailure, context);
+        if (replay.IsReplay)
+            return await ReplayArtifactCleanupAsync(replay, artifacts, context, cancellationToken).ConfigureAwait(false);
         if (artifact.DeletedAtUtc is not null)
-            return ArtifactAlreadyCleaned(context);
+        {
+            // Another confirmation may have completed after the first lookup.
+            replay = await confirmations.ConfirmAsync(replayRequest, cancellationToken).ConfigureAwait(false);
+            if (replay.FailureCode is { } concurrentFailure)
+                return Failure(concurrentFailure, context);
+            return replay.IsReplay
+                ? await ReplayArtifactCleanupAsync(replay, artifacts, context, cancellationToken).ConfigureAwait(false)
+                : ArtifactAlreadyCleaned(context);
+        }
 
-        var confirmation = await confirmations.ConfirmAsync(
-            new McpOperatorConfirmationRequest(request.PlanToken, request.IdempotencyKey, ArtifactCleanupPayloadHash(artifactId), context.Decision),
-            cancellationToken).ConfigureAwait(false);
+        var confirmation = await confirmations.ConfirmAsync(confirmationRequest, cancellationToken).ConfigureAwait(false);
         if (confirmation.FailureCode is { } confirmationFailure)
             return Failure(confirmationFailure, context);
         if (confirmation.IsReplay)
@@ -616,14 +633,14 @@ private static async Task<IResult> CleanupArtifactConfirmAsync(
             await confirmations.CompleteAsync(idempotencyId, McpOperatorIdempotencyOutcome.Failed, "artifact_unavailable", cancellationToken).ConfigureAwait(false);
             return Results.NotFound();
         }
+        artifact = NormalizeCleanupTimestamp(artifact);
 
+        // The bytes are already wiped. Preserve that outcome before a cancelled
+        // request or a current-authority recheck can prevent response delivery.
+        await confirmations.CompleteAsync(idempotencyId, McpOperatorIdempotencyOutcome.Succeeded, artifactId.ToString("D"), CancellationToken.None).ConfigureAwait(false);
         if (await RecheckArtifactCleanupAsync(admission, presence, new ClientKey(tenantId, agentId), context, cancellationToken).ConfigureAwait(false) is { } afterCleanup)
-        {
-            await confirmations.CompleteAsync(idempotencyId, McpOperatorIdempotencyOutcome.Succeeded, artifactId.ToString("D"), cancellationToken).ConfigureAwait(false);
             return Failure(afterCleanup, context);
-        }
 
-        await confirmations.CompleteAsync(idempotencyId, McpOperatorIdempotencyOutcome.Succeeded, artifactId.ToString("D"), cancellationToken).ConfigureAwait(false);
         return Results.Ok(ToArtifactMetadata(artifact, replayed: false, context.CorrelationId));
     }
 
@@ -1226,7 +1243,7 @@ private static async Task<IResult> ReplayArtifactCleanupAsync(
         var artifact = await GetOwnedArtifactAsync(artifacts, artifactId, context, includeContent: false, cancellationToken).ConfigureAwait(false);
         return artifact is null
             ? Failure("idempotency_replay_unavailable", context)
-            : Results.Ok(ToArtifactMetadata(artifact, replayed: true, context.CorrelationId));
+            : Results.Ok(ToArtifactMetadata(NormalizeCleanupTimestamp(artifact), replayed: true, context.CorrelationId));
     }
 
     private static Task<McpOperatorFileArtifact?> GetOwnedArtifactAsync(
@@ -1397,6 +1414,13 @@ private static string NormalizeArtifactMimeType(string? mimeType, byte[] content
     private static bool IsArtifactAvailable(McpOperatorFileArtifact artifact, DateTimeOffset now) =>
         artifact.DeletedAtUtc is null && artifact.ExpiresAtUtc > now;
 
+    private static McpOperatorFileArtifact NormalizeCleanupTimestamp(McpOperatorFileArtifact artifact) =>
+        // Use the durable PostgreSQL precision for both the first cleanup response
+        // and replay, including legacy identifier-only receipts.
+        artifact.DeletedAtUtc is { } deletedAtUtc
+            ? artifact with { DeletedAtUtc = new DateTimeOffset(deletedAtUtc.UtcTicks - deletedAtUtc.UtcTicks % TimeSpan.TicksPerMicrosecond, TimeSpan.Zero) }
+            : artifact;
+
     private static McpOperatorFileArtifactMetadata ToArtifactMetadata(
         McpOperatorFileArtifact artifact,
         bool replayed,

src/NetRatel/NetRatel.API/Endpoints/Client/McpOperatorJobEndpoints.cs

@@ -153,6 +153,7 @@ private static async Task<IResult> ValidateAsync(int tenantId, Guid agentId, Mcp
         var result = await TryContextAsync(JobsTool, "create", tenantId, agentId, http, environment, presence, admission, localAgents, dispatcher, false, cancellationToken, delegatedOperation: "create").ConfigureAwait(false);
         if (result.Failure is { } failure) return failure;
         var context = result.Context!;
+        if (http.TryGetMcpOperatorDelegation(out var delegation) && delegation?.ReplayBinding is not null) return Failure("delegated_identity_invalid", context);
         try
         {
             await jobs.ValidateDraftAsync(context.Decision, draft, cancellationToken).ConfigureAwait(false);
@@ -171,6 +172,7 @@ private static async Task<IResult> PreviewMutationAsync(int tenantId, Guid agent
         var result = await TryContextAsync(JobsTool, action, tenantId, agentId, http, environment, presence, admission, localAgents, dispatcher, false, cancellationToken).ConfigureAwait(false);
         if (result.Failure is { } failure) return failure;
         var context = result.Context!;
+        if (http.TryGetMcpOperatorDelegation(out var delegation) && delegation?.ReplayBinding is not null) return Failure("delegated_identity_invalid", context);
         var validation = await ValidateMutationAsync(action, request, jobs, context, cancellationToken).ConfigureAwait(false);
         if (validation.FailureCode is { } code) return Failure(code, context);
         try
@@ -190,6 +192,7 @@ private static async Task<IResult> ConfirmMutationAsync(int tenantId, Guid agent
         if (result.Failure is { } failure) return failure;
         var context = result.Context!;
         if (!HasPlan(request.PlanToken, request.IdempotencyKey)) return Failure("confirmation_plan_invalid", context);
+        if (!MatchesMutationDelegation(http, action, request)) return Failure("delegated_identity_invalid", context);
         var confirmationRequest = new McpOperatorConfirmationRequest(request.PlanToken!, request.IdempotencyKey!, MutationHash(action, request), context.Decision);
         // Successful mutations advance the version or hide the definition. Only
         // an authenticated existing admission may bypass those mutable checks;
@@ -409,6 +412,7 @@ private static async Task<IResult> PreviewRunActionAsync(string action, int tena
         var result = await TryContextAsync(RunsTool, action, tenantId, agentId, http, environment, presence, admission, localAgents, dispatcher, requiresGateway, cancellationToken).ConfigureAwait(false);
         if (result.Failure is { } failure) return failure;
         var context = result.Context!;
+        if (http.TryGetMcpOperatorDelegation(out var delegation) && delegation?.ReplayBinding is not null) return Failure("delegated_identity_invalid", context);
         var run = await GetOwnedRunAsync(runId, jobs, context, cancellationToken).ConfigureAwait(false);
         if (run is null || run.TargetSetDigest != context.Decision.Request.TargetSetDigest || (action == "cancel" && IsTerminal(run.State)) || (action == "delete" && !IsTerminal(run.State))) return Failure("job_run_action_invalid", context);
         try
@@ -427,6 +431,7 @@ private static async Task<IResult> ConfirmRunActionAsync(string action, int tena
         if (result.Failure is { } failure) return failure;
         var context = result.Context!;
         if (!HasPlan(request.PlanToken, request.IdempotencyKey)) return Failure("confirmation_plan_invalid", context);
+        if (!MatchesRunActionDelegation(http, action, runId, request)) return Failure("delegated_identity_invalid", context);
         var confirmationRequest = new McpOperatorConfirmationRequest(request.PlanToken!, request.IdempotencyKey!, RunActionHash(action, runId, request.Reason), context.Decision);
         var replayRequest = confirmationRequest with { ReplayOnly = true };
         var replay = await confirmations.ConfirmAsync(replayRequest, cancellationToken).ConfigureAwait(false);
@@ -679,10 +684,28 @@ private static bool InputsAreValid(IReadOnlyList<McpOperatorJobParameter> schema
         return inputs.Keys.All(key => schema.Any(parameter => parameter.Name == key));
     }
 
-    private static string MutationHash(string action, McpOperatorJobMutationRequest request) => Hash(JsonSerializer.Serialize(new { action, request.JobId, request.ExpectedVersion, request.Job, request.ParameterId, request.Parameter, request.StepId, request.Step, request.Ordinal }));
+    private static string MutationHash(string action, McpOperatorJobMutationRequest request) =>
+        McpOperatorJobConfirmationPayload.MutationHash(action, JsonSerializer.SerializeToElement(request));
     private static string RunHash(McpOperatorJobExecutionSnapshot snapshot, McpOperatorJobRunRequest request) =>
         Hash(JsonSerializer.Serialize(new { jobId = snapshot.Definition.Job.Id, executionSnapshot = snapshot.ComputeDigest(), request.Inputs }));
-    private static string RunActionHash(string action, ulong runId, string? reason) => Hash(JsonSerializer.Serialize(new { action, runId, reason = BoundedReason(reason) }));
+    private static string RunActionHash(string action, ulong runId, string? reason) =>
+        McpOperatorJobConfirmationPayload.RunActionHash(action, runId, reason);
+
+    private static bool MatchesMutationDelegation(HttpContext http, string action, McpOperatorJobMutationRequest request)
+    {
+        if (!http.TryGetMcpOperatorDelegation(out var delegation) || delegation is not { ObjectTargetResolutionEnabled: true }) return true;
+        if (delegation.ObjectReference != request.JobId?.ToString(CultureInfo.InvariantCulture)) return false;
+        return delegation.ReplayBinding is not { } binding ||
+            McpOperatorJobConfirmationPayload.TryCreateBinding(JobsTool, action, JsonSerializer.SerializeToElement(request), out var actual) && actual == binding;
+    }
+
+    private static bool MatchesRunActionDelegation(HttpContext http, string action, ulong runId, McpOperatorJobRunActionRequest request)
+    {
+        if (!http.TryGetMcpOperatorDelegation(out var delegation) || delegation is not { ObjectTargetResolutionEnabled: true }) return true;
+        if (delegation.ObjectReference != runId.ToString(CultureInfo.InvariantCulture)) return false;
+        return delegation.ReplayBinding is not { } binding || binding ==
+            McpOperatorJobConfirmationPayload.RunActionBinding(action, runId, request.Reason, request.PlanToken!, request.IdempotencyKey!);
+    }
     private static string Hash(string value) => Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(value)));
     private static bool HasPlan(string? plan, string? key) => IsOpaque(plan) && IsOpaque(key);
     private static bool IsOpaque(string? value) => value is { Length: >= 32 and <= 128 } && value.All(character => char.IsAsciiLetterOrDigit(character) || character is '-' or '_');