diff --git a/Directory.Build.props b/Directory.Build.props index 8e0b66e4..90aa9032 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -6,7 +6,7 @@ --> 0.1.1 - beta.30 + beta.31 $(VersionPrefix) $(VersionPrefix)-$(VersionSuffix) $(Version) diff --git a/docs/mcp-http/campaign-retest.md b/docs/mcp-http/campaign-retest.md index a667dcc2..a1a72d6a 100644 --- a/docs/mcp-http/campaign-retest.md +++ b/docs/mcp-http/campaign-retest.md @@ -7,100 +7,147 @@ publication integrity and owner functional acceptance are separate stages. [campaign #294](https://github.com/BostonTechnologies/netratel/issues/294) remain open until the owner explicitly disposes the campaign contexts. -## Current authority: beta.29 owner checkpoint and beta.30 continuation - -The owner tested beta.29 at public source -`4b148ebe824b3de445a12d30e95b8f5c30d53655`. Its API, Web, HTTP MCP and migration -deployment digests matched publication; an independent read-only database check -confirmed the required migrations. Retained key-ring storage was observed; -full key-file and backup continuity was not established. - -The checkpoint contains 33/35 fresh bounded functional assertions. Both failures -are manifestations of [#333](https://github.com/BostonTechnologies/netratel/issues/333): -exact successful definition-update and parameter-add confirmations were rejected -by mutable ETag validation before replay reconciliation. Historical remediation -(12/12), metadata-only projections (2/2), access controls (7/7), and new-fixture -cleanup (3/3) are separate populations. - -Beta.30 authenticates recorded definition outcomes before fresh-write validation. -For newly completed mutations, replay must preserve the original job ID, -resulting version and added item ID even after later edits or soft deletion, -without another write. Current authority, ownership, target/policy binding and -plan expiry remain enforced. Legacy ID-only receipts retain their documented -current ownership/tombstone-version fallback; do not invent missing historical -fields or replay expired historical plans. - -The release source review also identified and repairs -[#334](https://github.com/BostonTechnologies/netratel/issues/334): exact successful -cancellation replay after termination and deletion replay after tombstoning. -This is a separate source-review finding, not a second beta.29 owner-observed -defect. Retest both on fresh owned disposable runs within the valid confirmation -window. Compare the original run ID and action-result state, and prove no repeat -cancellation, deletion or audit mutation. Keep current gateway prerequisites -for cancellation and current authority/ownership/expiry checks for both actions. - -### Beta.30 execution order - -1. The owner deploys the verified beta.30 artifacts and confirms actual running - image digests, source/version and migration state. Preserve the existing - Data Protection key ring and backup. Distribution proof alone does not prove - which components are deployed. -2. The owner recovers the original QA container/volume and supplies independently - enrolled beta.30 Windows/Linux disposable executors. The beta.29 campaign's - original executors still ran beta.25 and its original QA gateway remained - offline. Follow the [isolated fixture guidance](../client/CLIENT.md#disposable-executors-for-owner-acceptance); +## Current authority: final beta.29 audit and beta.31 continuation + +The final owner audit tests beta.29 at public source +`4b148ebe824b3de445a12d30e95b8f5c30d53655` and supersedes the earlier checkpoint +counts. It contains 268 bounded assertions, of which the functional population +is 135/161 passed and 26 failed assertions across five P2 replay defects. +The 811 raw native receipts and 283 invoked contexts are separate evidence +populations; none is a count of completed production acceptance. + +Beta.30 was already tagged from the first-checkpoint repair. Beta.31 retains +that work and integrates the four additional final-audit findings. Preserve +published beta.30 history. Use the beta.31 publication record only after its +build and distribution checks complete. + +| Final-audit defect | Required fresh deployed regression | +| --- | --- | +| DEF-B29-01 / [#333](https://github.com/BostonTechnologies/netratel/issues/333) | Job update, parameter add/update/delete and step add/update/reorder/delete in both original tenants: exact successful confirmation returns the original version and added item ID without another mutation. | +| DEF-B29-02 / [#336](https://github.com/BostonTechnologies/netratel/issues/336) | Script deletion: exact successful confirmation remains reconcilable after the script becomes a tombstone, with retained scoped ownership and integrity. | +| DEF-B29-03 / [#337](https://github.com/BostonTechnologies/netratel/issues/337) | Policy replacement, disable and revoke: replay preserves the original normalized result despite later lifecycle/version changes. | +| DEF-B29-04 / [#338](https://github.com/BostonTechnologies/netratel/issues/338) | Artifact cleanup: exact successful confirmation returns the recorded result; cleaned status/download remain unavailable and cleanup is not redispatched. | +| DEF-B29-05 / [#339](https://github.com/BostonTechnologies/netratel/issues/339) | Job deletion: the actual local delegation exchange and business recheck admit only the exact consumed-success retry through retained ownership. Direct endpoint replay alone does not prove this repair. | + +The source-review run cancellation/deletion repair +[#334](https://github.com/BostonTechnologies/netratel/issues/334) is an additional +regression obligation, not a sixth owner-observed defect. Exercise fresh owned +disposable runs, including the deleted-object local exchange boundary. Preserve +current cancellation gateway prerequisites and compare the original run ID and +action-result state. + +Replay checks continue to enforce current identity, credential, client, +resource, instance, tenant, agent, ownership, target/policy binding, payload +and original plan expiry. Altered requests, arbitrary deleted/missing objects, +fresh stale writes, revoked authority and expired plans remain denied. New +invalid confirmations remain unconsumed. Exact success must not repeat writes, +versions, native dispatch or accepted-operation audits. Use fresh reviewed +fixtures; expired historical confirmations are evidence, not runnable fixtures. +Legacy ID-only receipts retain their documented scoped fallback and cannot +reconstruct historical fields that were never recorded. + +### Final ledger baseline + +Preserve all 518 IDs and every attempt: 438 original, 60 earlier additions and +20 beta.27 additions. Current dispositions are 225 bounded operation passes, +3 catalog/schema-only observations, 218 untested, 28 exact admission denials, +25 failed contexts, 14 prohibited, 2 negative-enforcement-only observations, +2 unsupported and 1 deferred. All prohibited contexts remain uninvoked. +Append beta.31 evidence without replacing historical failures or inferring +functional coverage from schema, catalogs, permission denials or neighboring +operations. + +Twenty policy receipt mappings across nine operation contexts were corrected from an apparent control-plane target +to their nested/persisted tenant target. Retain those corrected scopes; the +final audit establishes no positive control-plane mutation coverage. Preserve +the native receipt, bounded assertion, context invocation and disposition +counts as independent populations. + +### Beta.31 execution order + +1. The owner deploys the verified API, Web, HTTP MCP and migrations, then records + the actual running digests/source/version and migration state. Retain the + Data Protection key ring and establish key-file and backup continuity. + The beta.29 server identities and migration state had matching evidence; + that does not establish the next deployment. +2. The owner recovers the original QA container and volume with explicit + postconditions, then supplies independently enrolled beta.31 Windows/Linux + executors. The original native executors remained beta.25. Follow the + [isolated fixture guidance](../client/CLIENT.md#disposable-executors-for-owner-acceptance); an extra executable directory does not isolate a canonical device-key store. 3. The owner replaces/reconnects the selected consumer through supported controls. - Prove its exact selected process, loaded implementation assembly and complete - raw native schema identity using the [bridge identity guide](../integrations/http-mcp-bridge.md#prove-which-bridge-the-consumer-is-running). - The beta.29 checkpoint observed beta.26/beta.27 bridges. Current descriptions, - diagnostic bridges and configuration are separate from selected-process proof. -4. Revalidate exact current target, profile, revision, grant and expiry state. - Preserve the primary and auxiliary scope and original deadlines. Do not - replay consumed reviews. Four positive contexts, `clients/update_metadata` - and `monitoring/permissions` in both original tenants, denied exact admission - despite configured grants. Obtain legitimate review from a distinct eligible - authenticated reviewer if those positive cases are required; neighboring - admitted reads do not establish their admission. -5. Retest #333 on newly reviewed disposable definitions, covering update, - parameter/step operations, deletion, original-result fidelity after later - changes, and no duplicate writes. Retain create/start replay, changed-request, - stale-new-write, current-authority and single-winner controls. Complete the - remaining #305/#306 profile/bootstrap/upsert/classification/tag/revision/race/ - rollback, fresh secret-reader and output-surface, service timing, injection, - monitoring trigger/clear and fresh flow/event/delivery assertions. -6. Complete all eight actual native prompt workflows with real required human - decisions and useful effects. The checkpoint completed 0/8. Prove actual - native progress notifications and in-flight protocol cancellation under #288; - polling output and application-job cancellation do not supply that evidence. - Bounded Linux/Windows browse behavior now has owner regression evidence under - #321, including filtered-entry behavior at the exact eligible limit. Preserve - the two unsupported continuation contexts and deferred autofill. -7. After all other campaign and paired-product traffic ends, run #310's unchanged - exact isolated hour: 240 serial calls at offsets 0, 15 through 3585 seconds, - with observation through 3600 seconds. Repeat tenant A flow list, auth status, - tenant B flow list, system version 60 times. Preserve every failed, missed or - overrun slot without retries or catch-up; separate 120 core samples and 60 - admitted reads per tenant. No qualifying endurance hour ran in beta.29. -8. Reconcile the original receiver incidents and background artifacts without - duplicate delivery, then finish original dependency-ordered cleanup, effective - auxiliary revocation and update-intent restoration. New beta.29 fixtures and - inspection sessions were removed; original cleanup remains open. Preserve - terminal audit and protected state. Infrastructure disposal is owner work. + Prove its exact selected process, loaded implementation and complete raw + native schema identity with the + [bridge identity guide](../integrations/http-mcp-bridge.md#prove-which-bridge-the-consumer-is-running). + Observed beta.26/beta.27 bridges do not establish fixed-consumer acceptance. +4. Read back exact current target, profile, revision, grant and expiry state. + Preserve the primary credential's complete 56 tenant and 28 instance grants, + and the auxiliary's 54 no-reveal grants and original expiry. Do not trim, + extend, substitute administrator credentials or replay consumed reviews. + The 28 admission-denied contexts require their own legitimate current review + by an eligible distinct authenticated reviewer before positive execution. + The absent `mcp.connectivity.read` scope remains a prerequisite, not an + implied grant. Continue unaffected admitted work while a specific prerequisite + blocks other contexts. +5. Retest all five replay defects in both original tenants through the selected + native consumer, then the additional run actions. Compare original results, + persisted versions and audit/dispatch counts; retain changed-payload, + stale-new-write, wrong-owner, current-revocation, expiry and concurrent + single-winner controls. Do not treat endpoint-only tests as local exchange + acceptance for #339. +6. Finish #305/#306 profile bootstrap/upsert/persistence, classification/tag/ + revision changes, race/rollback and untrusted-context controls. Generate fresh + standalone and job secret producers on fixed native executors and inspect the + full supported UI/API/MCP reader and output-surface matrix with exact adjacent + nonsecret values. Complete inert-service Start/Stop with issuance, expiry, + clock-skew, transitions and inventory evidence. Keep the delivered native + timing repair and protected host/service state. +7. Complete live monitoring trigger/clear, event and delivery assertions using a + verified owned synthetic notification sink. Disabled configuration checks do + not prove live acceptance. Establish actual native progress notifications and + in-flight protocol cancellation under #288; polling and application-job + cancellation do not establish those protocol behaviors. +8. Complete all eight actual native prompt workflows with the required genuine + human decisions: `inspect_client`, `diagnose_job_run`, + `controlled_remote_execution`, `create_workflow`, + `manage_operator_access`, `safe_file_operation`, + `safe_terminal_operation` and `onboard_client`. The final beta.29 audit + completed 0/8. Use a consumer with actual native prompt controls; retrieval, + rendering or separate tool replays are not workflow execution. Preserve the + two unsupported continuation contexts and deferred, unexecuted autofill. +9. After all other campaign and paired-product traffic ends, run #310's exact + isolated hour: 240 serial calls at offsets 0, 15 through 3585 seconds, with + observation through 3600 seconds. Repeat tenant A flow list, auth status, + tenant B flow list and system version 60 times. Keep all failed, missed and + overrun slots without retries, catch-up, overlap or unrelated traffic. + Separate 120 core samples and 60 admitted reads per tenant. Do not invent a + latency SLA. The final beta.29 baseline remains 0/240 calls and 0/3600 seconds. +10. Finish original dependency-ordered cleanup, including effective auxiliary + revocation after remaining tests and retained update-intent restoration. + Preserve audits, runs, terminal history and protected state. Both original + receiver incidents are resolved; retain their outbound email failure entries + without another delivery attempt. Knowledge/background cleanup remains + unresolved; an empty conversation UI is not authoritative absence. + Infrastructure disposal and recovery are owner actions. The local test agent performs acceptance, evidence collection and explicitly authorized campaign-fixture operations. Deployment, infrastructure recovery, restarts/upgrades, installation/enrolment and consumer reconnection are owner -actions. Continue unaffected admitted checks when a prerequisite blocks one -area; do not turn a failed acceptance check into source or release work. - -Preserve all 518 context IDs and history: 438 original, 60 earlier additions and -20 beta.27 additions. Beta.29 invocation coverage is 110/518; dispositions are -104 bounded passes, 2 failed contexts, 4 exact admission denials, 391 untested, -14 prohibited, 2 unsupported and 1 deferred. All prohibited contexts remained -uninvoked. Append beta.30 attempts and record each evidence population separately. +actions. A failed acceptance check does not authorize source or release changes. + Keep #282/#294/#288/#304/#305/#306/#308/#310 open for their actual remaining -criteria. Delivered repairs #315–#318 and #321/#322 remain closed. +criteria. Delivered repairs #315–#318 and #321/#322 stay closed; #316's start +admission is not the later definition-retry defect. The original minimal +two-tenant admission under #301 remains completed. Source delivery closure for +#333/#334/#336–#339 is separate from the deployed regression and overall +readiness dispositions. + +### Historical first beta.29 checkpoint + +The earlier checkpoint had 33/35 fresh bounded functional assertions and two +failed contexts under #333. It drove beta.30's source work. Its 110 invoked +contexts and first-checkpoint dispositions remain historical evidence; the +final audit baseline above is the current continuation authority. ## Historical beta.27 owner checkpoint and beta.28 continuation @@ -327,8 +374,9 @@ and pre-existing sessions while these prerequisites are completed. complete requested operation schema with the client's loaded native metadata after its supported reconnect. The server cannot certify or refresh metadata inside the client. -2. Install the [versioned bridge](../integrations/http-mcp-bridge.md) or use native - Streamable HTTP. Record the actual installed version, reconnect and migration. +2. The owner installs the [versioned bridge](../integrations/http-mcp-bridge.md) + or selects native Streamable HTTP, and performs the supported reconnect. + The local tester records the actual installed version and migration. Loopback evidence does not prove owner-machine migration or live streaming. 3. Use [reviewed setup](setup.md) and [local credentials](local-credential-mode.md). Keep protected capabilities and secrets outside model input/output. Inspect the @@ -347,7 +395,7 @@ and pre-existing sessions while these prerequisites are completed. admitted endurance phase; synthetic CI tenants cannot replace them. 5. Record protected API liveness/readiness and HealthRead admission through the saved destination separately from MCP-host health and business admission. - Deploy the alias-capable API before or with MCP, then verify both + The owner deploys the alias-capable API before or with MCP; the tester verifies both `/api/v2/mcp/operator/health/live` and `/api/v2/mcp/operator/health/ready` through the actual ingress using the saved MCP client. Source tests do not resolve deployment routing, and a 404 must remain observable without fallback. @@ -394,9 +442,11 @@ actual native AI-host calls separately. Diagnostic prompt retrieval does not complete a native workflow. Use existing optional integration/browser selections for expensive acceptance; the owner hour does not run in required CI. -Install the [GitHub CLI](https://cli.github.com/manual/installation) and use an -authenticated account with repository Actions access. From that client terminal, -run the bounded disposable +The following optional source/protocol diagnostic belongs to the owner or the +release/implementation agent. The local laptop acceptance tester does not manage +CI and must not dispatch it. When the owner separately requests this diagnostic, +install the [GitHub CLI](https://cli.github.com/manual/installation) and use an +authenticated account with repository Actions access to run the bounded disposable protocol/client check against the exact published tag deployed for this campaign. Replace the placeholder with that tag; it must agree with the retained release, source and manifest-digest receipt: diff --git a/docs/release-notes/0.1.1-beta.31.md b/docs/release-notes/0.1.1-beta.31.md new file mode 100644 index 00000000..4284a096 --- /dev/null +++ b/docs/release-notes/0.1.1-beta.31.md @@ -0,0 +1,53 @@ +# 0.1.1-beta.31 — final-audit confirmation replay corrections + +The final beta.29 owner audit found five kinds of successful mutation whose exact +confirmation retry returned an error. The writes had succeeded once; the caller +could not reliably reconcile the result. Beta.30 had already been tagged from +the earlier checkpoint, so this release retains its job-definition and run-action +repairs and integrates the four additional final-audit findings. The published +beta.30 tag and artifacts remain immutable. + +| Finding | Correction | +| --- | --- | +| [#333](https://github.com/BostonTechnologies/netratel/issues/333), retained from beta.30 | Reconcile completed job-definition mutations before current ETag validation; return the original resulting version and added item ID. The final audit expanded the reproduced family to update, parameter add/update/delete and step add/update/reorder/delete in both tested tenants. | +| [#336](https://github.com/BostonTechnologies/netratel/issues/336) | Reconcile successful script deletion through retained scoped ownership and content integrity, returning the original deletion result without restoring script visibility. | +| [#337](https://github.com/BostonTechnologies/netratel/issues/337) | Reconcile successful policy replacement, disable and revoke before mutable version/lifecycle validation; retain the original normalized result across later changes. | +| [#338](https://github.com/BostonTechnologies/netratel/issues/338) | Reconcile successful artifact cleanup before rejecting already-cleaned state. Cleaned artifact status and download remain unavailable. | +| [#339](https://github.com/BostonTechnologies/netratel/issues/339) | Admit deleted-job replay through the local delegation exchange only with retained ownership and an exact consumed-success confirmation binding, then recheck it at business dispatch. Ordinary deleted or missing object references remain denied. | + +The deleted-object boundary also covers the run-action replay repair +[#334](https://github.com/BostonTechnologies/netratel/issues/334) retained from +beta.30. That finding came from source review; it is separate from the five +owner-observed defects. + +Replay preserves current caller, credential, client, resource, instance, tenant, +agent, policy, payload and expiry checks. It does not repeat writes, native +dispatch, cancellation, cleanup or accepted-operation audits. Fresh invalid +plans remain unconsumed; fresh stale writes and altered requests remain denied. +New receipts retain the original successful result. Legacy ID-only receipts +keep their scoped compatibility behavior and cannot recreate historical fields +that were never recorded. + +The repair extends existing confirmation, ownership and delegation contracts; +it requires no new database migration. Existing generic CI and release workflows +remain the build and publication gates. Focused tests cover real persisted +confirmation lifecycles and the local exchange/business authorization boundary. + +## Next owner campaign + +Deploy only after this tag's build and distribution verification complete. Verify +the running API, Web, HTTP MCP and migrations against the publication record; +prepare independently enrolled fixed-version Windows/Linux executors and prove +the selected consumer's loaded bridge and native schema identity. Preserve the +key ring, existing grant scope/expiry, retained evidence and original cleanup +dependencies. + +The [owner repeat campaign](../mcp-http/campaign-retest.md) uses the final beta.29 +audit: all 518 context IDs and histories, five replay defects, 28 admission +denials and the remaining functional work. It retains the eight actual native +prompt workflows and the exact isolated 240-call hour. Autofill remains deferred. + +Source delivery and distribution integrity do not complete owner functional +acceptance in [#294](https://github.com/BostonTechnologies/netratel/issues/294) +or the production-readiness epic +[#282](https://github.com/BostonTechnologies/netratel/issues/282). diff --git a/docs/release-notes/README.md b/docs/release-notes/README.md index c91cd528..49f8c1e8 100644 --- a/docs/release-notes/README.md +++ b/docs/release-notes/README.md @@ -2,6 +2,7 @@ These notes describe the named release and preserve historical status. [Release engineering](../deployment/RELEASES.md) describes the current publication process; the product version is controlled only by `Directory.Build.props`. +- [0.1.1-beta.31 — final-audit confirmation replay corrections](0.1.1-beta.31.md) - [0.1.1-beta.30 — exact job mutation and run-action replay](0.1.1-beta.30.md) - [0.1.1-beta.29 candidate — managed automation retry attempts](0.1.1-beta.29.md) diff --git a/src/NetRatel/NetRatel.API/Endpoints/Auth/McpLocalDelegationEndpoints.cs b/src/NetRatel/NetRatel.API/Endpoints/Auth/McpLocalDelegationEndpoints.cs index 8b6a1e1b..8f0f96f7 100644 --- a/src/NetRatel/NetRatel.API/Endpoints/Auth/McpLocalDelegationEndpoints.cs +++ b/src/NetRatel/NetRatel.API/Endpoints/Auth/McpLocalDelegationEndpoints.cs @@ -88,12 +88,12 @@ public static IEndpointRouteBuilder MapMcpLocalDelegationEndpoints(this IEndpoin } if (!McpLocalDelegationTargetRequirements.TryGetAuthorizationTenant(pairing, out var authorizationTenantId)) return Denied(http, "local_delegation_target_invalid", StatusCodes.Status400BadRequest, pairing); + // Ordinary object requests retain their established target-binding + // failure before permission evaluation. They cannot see tombstones. if (target?.Model is McpOperationTargetModel.ObjectDerived && pairing.ObjectTargetResolutionEnabled && - (objectTargets is null || + pairing.ReplayBinding is null && (objectTargets is null || !await objectTargets.MatchesDelegationAsync(pairing, cancellationToken).ConfigureAwait(false))) - { return Denied(http, "local_delegation_binding_mismatch", StatusCodes.Status403Forbidden, pairing); - } var credentialId = http.User.FindFirstValue(IntegrationCredentialAuthenticationHandler.CredentialIdClaimType); var ownerPrincipalId = http.User.FindFirstValue("netratel_principal_id"); @@ -115,6 +115,16 @@ public static IEndpointRouteBuilder MapMcpLocalDelegationEndpoints(this IEndpoin [], McpLocalDelegationPermissionMapper.RolesFor(operationAccess.RequiredScope), [McpOperationAccessScopeNames.Canonical(operationAccess.RequiredScope)]); + // The pairing assertion authenticates the gateway, never the + // receipt owner. Only the admitted ingress credential supplies + // the owner and client used for an exact deleted-object replay. + if (target?.Model is McpOperationTargetModel.ObjectDerived && pairing.ObjectTargetResolutionEnabled && + pairing.ReplayBinding is not null && + (objectTargets is null || + !await objectTargets.MatchesDelegationAsync(pairing with { Identity = identity, IngressCredentialId = credentialId }, cancellationToken).ConfigureAwait(false))) + { + return Denied(http, "local_delegation_binding_mismatch", StatusCodes.Status403Forbidden, pairing); + } var assertion = tokens.Create(identity, new McpOperatorDelegationRequest( pairing.Tool, pairing.Operation, @@ -127,6 +137,7 @@ public static IEndpointRouteBuilder MapMcpLocalDelegationEndpoints(this IEndpoin { ObjectReference = pairing.ObjectReference, ObjectTargetResolutionEnabled = pairing.ObjectTargetResolutionEnabled, + ReplayBinding = pairing.ReplayBinding, IngressCredentialId = credentialId, IngressPermission = requiredPermission }); diff --git a/src/NetRatel/NetRatel.API/Endpoints/Client/McpOperatorClientFileEndpoints.cs b/src/NetRatel/NetRatel.API/Endpoints/Client/McpOperatorClientFileEndpoints.cs index 150fd702..fa17bbfc 100644 --- a/src/NetRatel/NetRatel.API/Endpoints/Client/McpOperatorClientFileEndpoints.cs +++ b/src/NetRatel/NetRatel.API/Endpoints/Client/McpOperatorClientFileEndpoints.cs @@ -573,12 +573,29 @@ private static async Task CleanupArtifactConfirmAsync( var artifact = await GetOwnedArtifactAsync(artifacts, artifactId, context, includeContent: false, cancellationToken).ConfigureAwait(false); if (artifact is null) return Results.NotFound(); + + var confirmationRequest = new McpOperatorConfirmationRequest( + request.PlanToken, request.IdempotencyKey, ArtifactCleanupPayloadHash(artifactId), context.Decision); + // A completed cleanup keeps its authenticated ownership record after + // wiping the bytes. Reconcile that receipt before rejecting deleted state. + var replayRequest = confirmationRequest with { ReplayOnly = true }; + var replay = await confirmations.ConfirmAsync(replayRequest, cancellationToken).ConfigureAwait(false); + if (replay.FailureCode is { } replayFailure) + return Failure(replayFailure, context); + if (replay.IsReplay) + return await ReplayArtifactCleanupAsync(replay, artifacts, context, cancellationToken).ConfigureAwait(false); if (artifact.DeletedAtUtc is not null) - return ArtifactAlreadyCleaned(context); + { + // Another confirmation may have completed after the first lookup. + replay = await confirmations.ConfirmAsync(replayRequest, cancellationToken).ConfigureAwait(false); + if (replay.FailureCode is { } concurrentFailure) + return Failure(concurrentFailure, context); + return replay.IsReplay + ? await ReplayArtifactCleanupAsync(replay, artifacts, context, cancellationToken).ConfigureAwait(false) + : ArtifactAlreadyCleaned(context); + } - var confirmation = await confirmations.ConfirmAsync( - new McpOperatorConfirmationRequest(request.PlanToken, request.IdempotencyKey, ArtifactCleanupPayloadHash(artifactId), context.Decision), - cancellationToken).ConfigureAwait(false); + var confirmation = await confirmations.ConfirmAsync(confirmationRequest, cancellationToken).ConfigureAwait(false); if (confirmation.FailureCode is { } confirmationFailure) return Failure(confirmationFailure, context); if (confirmation.IsReplay) @@ -616,14 +633,14 @@ private static async Task CleanupArtifactConfirmAsync( await confirmations.CompleteAsync(idempotencyId, McpOperatorIdempotencyOutcome.Failed, "artifact_unavailable", cancellationToken).ConfigureAwait(false); return Results.NotFound(); } + artifact = NormalizeCleanupTimestamp(artifact); + // The bytes are already wiped. Preserve that outcome before a cancelled + // request or a current-authority recheck can prevent response delivery. + await confirmations.CompleteAsync(idempotencyId, McpOperatorIdempotencyOutcome.Succeeded, artifactId.ToString("D"), CancellationToken.None).ConfigureAwait(false); if (await RecheckArtifactCleanupAsync(admission, presence, new ClientKey(tenantId, agentId), context, cancellationToken).ConfigureAwait(false) is { } afterCleanup) - { - await confirmations.CompleteAsync(idempotencyId, McpOperatorIdempotencyOutcome.Succeeded, artifactId.ToString("D"), cancellationToken).ConfigureAwait(false); return Failure(afterCleanup, context); - } - await confirmations.CompleteAsync(idempotencyId, McpOperatorIdempotencyOutcome.Succeeded, artifactId.ToString("D"), cancellationToken).ConfigureAwait(false); return Results.Ok(ToArtifactMetadata(artifact, replayed: false, context.CorrelationId)); } @@ -1226,7 +1243,7 @@ private static async Task ReplayArtifactCleanupAsync( var artifact = await GetOwnedArtifactAsync(artifacts, artifactId, context, includeContent: false, cancellationToken).ConfigureAwait(false); return artifact is null ? Failure("idempotency_replay_unavailable", context) - : Results.Ok(ToArtifactMetadata(artifact, replayed: true, context.CorrelationId)); + : Results.Ok(ToArtifactMetadata(NormalizeCleanupTimestamp(artifact), replayed: true, context.CorrelationId)); } private static Task GetOwnedArtifactAsync( @@ -1397,6 +1414,13 @@ private static string NormalizeArtifactMimeType(string? mimeType, byte[] content private static bool IsArtifactAvailable(McpOperatorFileArtifact artifact, DateTimeOffset now) => artifact.DeletedAtUtc is null && artifact.ExpiresAtUtc > now; + private static McpOperatorFileArtifact NormalizeCleanupTimestamp(McpOperatorFileArtifact artifact) => + // Use the durable PostgreSQL precision for both the first cleanup response + // and replay, including legacy identifier-only receipts. + artifact.DeletedAtUtc is { } deletedAtUtc + ? artifact with { DeletedAtUtc = new DateTimeOffset(deletedAtUtc.UtcTicks - deletedAtUtc.UtcTicks % TimeSpan.TicksPerMicrosecond, TimeSpan.Zero) } + : artifact; + private static McpOperatorFileArtifactMetadata ToArtifactMetadata( McpOperatorFileArtifact artifact, bool replayed, diff --git a/src/NetRatel/NetRatel.API/Endpoints/Client/McpOperatorJobEndpoints.cs b/src/NetRatel/NetRatel.API/Endpoints/Client/McpOperatorJobEndpoints.cs index fceac89f..45543668 100644 --- a/src/NetRatel/NetRatel.API/Endpoints/Client/McpOperatorJobEndpoints.cs +++ b/src/NetRatel/NetRatel.API/Endpoints/Client/McpOperatorJobEndpoints.cs @@ -153,6 +153,7 @@ private static async Task ValidateAsync(int tenantId, Guid agentId, Mcp var result = await TryContextAsync(JobsTool, "create", tenantId, agentId, http, environment, presence, admission, localAgents, dispatcher, false, cancellationToken, delegatedOperation: "create").ConfigureAwait(false); if (result.Failure is { } failure) return failure; var context = result.Context!; + if (http.TryGetMcpOperatorDelegation(out var delegation) && delegation?.ReplayBinding is not null) return Failure("delegated_identity_invalid", context); try { await jobs.ValidateDraftAsync(context.Decision, draft, cancellationToken).ConfigureAwait(false); @@ -171,6 +172,7 @@ private static async Task PreviewMutationAsync(int tenantId, Guid agent var result = await TryContextAsync(JobsTool, action, tenantId, agentId, http, environment, presence, admission, localAgents, dispatcher, false, cancellationToken).ConfigureAwait(false); if (result.Failure is { } failure) return failure; var context = result.Context!; + if (http.TryGetMcpOperatorDelegation(out var delegation) && delegation?.ReplayBinding is not null) return Failure("delegated_identity_invalid", context); var validation = await ValidateMutationAsync(action, request, jobs, context, cancellationToken).ConfigureAwait(false); if (validation.FailureCode is { } code) return Failure(code, context); try @@ -190,6 +192,7 @@ private static async Task ConfirmMutationAsync(int tenantId, Guid agent if (result.Failure is { } failure) return failure; var context = result.Context!; if (!HasPlan(request.PlanToken, request.IdempotencyKey)) return Failure("confirmation_plan_invalid", context); + if (!MatchesMutationDelegation(http, action, request)) return Failure("delegated_identity_invalid", context); var confirmationRequest = new McpOperatorConfirmationRequest(request.PlanToken!, request.IdempotencyKey!, MutationHash(action, request), context.Decision); // Successful mutations advance the version or hide the definition. Only // an authenticated existing admission may bypass those mutable checks; @@ -409,6 +412,7 @@ private static async Task PreviewRunActionAsync(string action, int tena var result = await TryContextAsync(RunsTool, action, tenantId, agentId, http, environment, presence, admission, localAgents, dispatcher, requiresGateway, cancellationToken).ConfigureAwait(false); if (result.Failure is { } failure) return failure; var context = result.Context!; + if (http.TryGetMcpOperatorDelegation(out var delegation) && delegation?.ReplayBinding is not null) return Failure("delegated_identity_invalid", context); var run = await GetOwnedRunAsync(runId, jobs, context, cancellationToken).ConfigureAwait(false); if (run is null || run.TargetSetDigest != context.Decision.Request.TargetSetDigest || (action == "cancel" && IsTerminal(run.State)) || (action == "delete" && !IsTerminal(run.State))) return Failure("job_run_action_invalid", context); try @@ -427,6 +431,7 @@ private static async Task ConfirmRunActionAsync(string action, int tena if (result.Failure is { } failure) return failure; var context = result.Context!; if (!HasPlan(request.PlanToken, request.IdempotencyKey)) return Failure("confirmation_plan_invalid", context); + if (!MatchesRunActionDelegation(http, action, runId, request)) return Failure("delegated_identity_invalid", context); var confirmationRequest = new McpOperatorConfirmationRequest(request.PlanToken!, request.IdempotencyKey!, RunActionHash(action, runId, request.Reason), context.Decision); var replayRequest = confirmationRequest with { ReplayOnly = true }; var replay = await confirmations.ConfirmAsync(replayRequest, cancellationToken).ConfigureAwait(false); @@ -679,10 +684,28 @@ private static bool InputsAreValid(IReadOnlyList schema return inputs.Keys.All(key => schema.Any(parameter => parameter.Name == key)); } - private static string MutationHash(string action, McpOperatorJobMutationRequest request) => Hash(JsonSerializer.Serialize(new { action, request.JobId, request.ExpectedVersion, request.Job, request.ParameterId, request.Parameter, request.StepId, request.Step, request.Ordinal })); + private static string MutationHash(string action, McpOperatorJobMutationRequest request) => + McpOperatorJobConfirmationPayload.MutationHash(action, JsonSerializer.SerializeToElement(request)); private static string RunHash(McpOperatorJobExecutionSnapshot snapshot, McpOperatorJobRunRequest request) => Hash(JsonSerializer.Serialize(new { jobId = snapshot.Definition.Job.Id, executionSnapshot = snapshot.ComputeDigest(), request.Inputs })); - private static string RunActionHash(string action, ulong runId, string? reason) => Hash(JsonSerializer.Serialize(new { action, runId, reason = BoundedReason(reason) })); + private static string RunActionHash(string action, ulong runId, string? reason) => + McpOperatorJobConfirmationPayload.RunActionHash(action, runId, reason); + + private static bool MatchesMutationDelegation(HttpContext http, string action, McpOperatorJobMutationRequest request) + { + if (!http.TryGetMcpOperatorDelegation(out var delegation) || delegation is not { ObjectTargetResolutionEnabled: true }) return true; + if (delegation.ObjectReference != request.JobId?.ToString(CultureInfo.InvariantCulture)) return false; + return delegation.ReplayBinding is not { } binding || + McpOperatorJobConfirmationPayload.TryCreateBinding(JobsTool, action, JsonSerializer.SerializeToElement(request), out var actual) && actual == binding; + } + + private static bool MatchesRunActionDelegation(HttpContext http, string action, ulong runId, McpOperatorJobRunActionRequest request) + { + if (!http.TryGetMcpOperatorDelegation(out var delegation) || delegation is not { ObjectTargetResolutionEnabled: true }) return true; + if (delegation.ObjectReference != runId.ToString(CultureInfo.InvariantCulture)) return false; + return delegation.ReplayBinding is not { } binding || binding == + McpOperatorJobConfirmationPayload.RunActionBinding(action, runId, request.Reason, request.PlanToken!, request.IdempotencyKey!); + } private static string Hash(string value) => Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(value))); private static bool HasPlan(string? plan, string? key) => IsOpaque(plan) && IsOpaque(key); private static bool IsOpaque(string? value) => value is { Length: >= 32 and <= 128 } && value.All(character => char.IsAsciiLetterOrDigit(character) || character is '-' or '_'); diff --git a/src/NetRatel/NetRatel.API/Endpoints/Client/McpOperatorPolicyMutationEndpoints.cs b/src/NetRatel/NetRatel.API/Endpoints/Client/McpOperatorPolicyMutationEndpoints.cs index fdaf1d99..ad4fa5a7 100644 --- a/src/NetRatel/NetRatel.API/Endpoints/Client/McpOperatorPolicyMutationEndpoints.cs +++ b/src/NetRatel/NetRatel.API/Endpoints/Client/McpOperatorPolicyMutationEndpoints.cs @@ -1,3 +1,4 @@ +using System.Globalization; using System.Security.Cryptography; using System.Text; using System.Text.Json; @@ -145,7 +146,7 @@ public static IEndpointRouteBuilder MapMcpOperatorPolicyMutationEndpoints(this I decision, context, target, payloadHash, CreateOperation, authorization, administration, mutationToken => administration.CreateAsync(request.Policy, context.Principal.Subject, mutationToken), cancellationToken).ConfigureAwait(false); - await confirmations.CompleteAsync(idempotencyId, McpOperatorIdempotencyOutcome.Succeeded, created.PolicyId.ToString("D"), cancellationToken).ConfigureAwait(false); + await confirmations.CompleteAsync(idempotencyId, McpOperatorIdempotencyOutcome.Succeeded, created.PolicyId.ToString("D"), CancellationToken.None).ConfigureAwait(false); return Results.Created( $"/api/v2/mcp/operator/policy/policies/{created.PolicyId:D}", new McpOperatorPolicyCreateResult(created, false, context.CorrelationId)); @@ -319,7 +320,7 @@ private static async Task ConfirmTargetProfileAsync( target.TenantId, agentId, request.Classification, request.Tags, request.ExpectedVersion, context.Principal.Subject, mutationToken), cancellationToken).ConfigureAwait(false); - await confirmations.CompleteAsync(idempotencyId, McpOperatorIdempotencyOutcome.Succeeded, profile.AgentId.ToString("D"), cancellationToken).ConfigureAwait(false); + await confirmations.CompleteAsync(idempotencyId, McpOperatorIdempotencyOutcome.Succeeded, profile.AgentId.ToString("D"), CancellationToken.None).ConfigureAwait(false); return Results.Ok(new McpOperatorTargetProfileMutationResult(profile, false, context.CorrelationId)); } catch (McpOperatorAdmissionRejectedException rejection) @@ -420,13 +421,23 @@ private static async Task ConfirmReplaceAsync( var policy = existing.Policy!; var target = existing.Target!; - if (!CanReplace(policy, request.ExpectedVersion, request.Policy!, context!.Principal, out failure)) - return Failure(failure!, target.TenantId, target.AgentId, context.CorrelationId, mutationOperation: ReplaceOperation); - var payloadHash = PayloadHash(ReplaceOperation, new PolicyReplacePayload(request.PolicyId, request.ExpectedVersion, request.Policy!)); - var decision = await EvaluateAsync(context, target, payloadHash, ReplaceOperation, authorization, administration, cancellationToken).ConfigureAwait(false); + var decision = await EvaluateAsync(context!, target, payloadHash, ReplaceOperation, authorization, administration, cancellationToken).ConfigureAwait(false); if (!decision.IsAllowed) - return Failure(decision.FailureCode ?? "target_policy_missing", target.TenantId, target.AgentId, context.CorrelationId, mutationOperation: ReplaceOperation); + return Failure(decision.FailureCode ?? "target_policy_missing", target.TenantId, target.AgentId, context!.CorrelationId, mutationOperation: ReplaceOperation); + + var confirmationRequest = new McpOperatorConfirmationRequest(request.PlanToken!, request.IdempotencyKey!, payloadHash, decision); + if (await TryReplayMutationAsync(confirmationRequest, confirmations, administration, target, context!, ReplaceOperation, + request.PolicyId, request.ExpectedVersion, request.Policy!, cancellationToken).ConfigureAwait(false) is { } replay) + return replay; + if (!CanReplace(policy, request.ExpectedVersion, request.Policy!, context!.Principal, out failure)) + { + // A concurrent confirmation may finish between the receipt lookup + // and mutable version/state validation. Never consume a stale new plan. + return await TryReplayMutationAsync(confirmationRequest, confirmations, administration, target, context!, ReplaceOperation, + request.PolicyId, request.ExpectedVersion, request.Policy!, cancellationToken).ConfigureAwait(false) + ?? Failure(failure!, target.TenantId, target.AgentId, context!.CorrelationId, mutationOperation: ReplaceOperation); + } try { @@ -445,12 +456,11 @@ private static async Task ConfirmReplaceAsync( return Failure("operator_policy_rejected", target.TenantId, target.AgentId, context.CorrelationId, StatusCodes.Status409Conflict, ReplaceOperation); } - var admission = await confirmations.ConfirmAsync( - new McpOperatorConfirmationRequest(request.PlanToken!, request.IdempotencyKey!, payloadHash, decision), cancellationToken).ConfigureAwait(false); + var admission = await confirmations.ConfirmAsync(confirmationRequest, cancellationToken).ConfigureAwait(false); if (admission.FailureCode is { } confirmationFailure) return Failure(confirmationFailure, target.TenantId, target.AgentId, context.CorrelationId, mutationOperation: ReplaceOperation); if (admission.IsReplay) - return await ReplayMutationAsync(admission, administration, target, context, ReplaceOperation, cancellationToken).ConfigureAwait(false); + return await ReplayMutationAsync(admission, administration, target, context!, ReplaceOperation, request.PolicyId, request.ExpectedVersion, request.Policy!, cancellationToken).ConfigureAwait(false); if (!admission.IsNewDispatch || admission.IdempotencyId is not { } idempotencyId) return Failure("confirmation_plan_invalid", target.TenantId, target.AgentId, context.CorrelationId, mutationOperation: ReplaceOperation); @@ -461,7 +471,8 @@ private static async Task ConfirmReplaceAsync( mutationToken => administration.ReplaceAsync(request.PolicyId, request.ExpectedVersion, request.Policy!, context.Principal.Subject, mutationToken), cancellationToken).ConfigureAwait(false); - await confirmations.CompleteAsync(idempotencyId, McpOperatorIdempotencyOutcome.Succeeded, replaced.PolicyId.ToString("D"), cancellationToken).ConfigureAwait(false); + // The mutation is durable; preserve its receipt even if response delivery is cancelled. + await confirmations.CompleteAsync(idempotencyId, McpOperatorIdempotencyOutcome.Succeeded, PolicyMutationResultReference(replaced), CancellationToken.None).ConfigureAwait(false); return Results.Ok(new McpOperatorPolicyMutationResult(replaced, false, context.CorrelationId)); } catch (McpOperatorAdmissionRejectedException rejection) @@ -547,20 +558,30 @@ private static async Task ConfirmDisableAsync( var target = existing.Target!; if (!SameTarget(policy.TargetSelector, request.Target)) return Failure("policy_target_immutable", target.TenantId, target.AgentId, context!.CorrelationId, mutationOperation: DisableOperation); - if (!CanDisable(policy, request.ExpectedVersion, context!.Principal, out failure)) - return Failure(failure!, target.TenantId, target.AgentId, context.CorrelationId, mutationOperation: DisableOperation); var payloadHash = PayloadHash(DisableOperation, new PolicyDisablePayload(request.PolicyId, request.ExpectedVersion, request.Target)); - var decision = await EvaluateAsync(context, target, payloadHash, DisableOperation, authorization, administration, cancellationToken).ConfigureAwait(false); + var decision = await EvaluateAsync(context!, target, payloadHash, DisableOperation, authorization, administration, cancellationToken).ConfigureAwait(false); if (!decision.IsAllowed) - return Failure(decision.FailureCode ?? "target_policy_missing", target.TenantId, target.AgentId, context.CorrelationId, mutationOperation: DisableOperation); + return Failure(decision.FailureCode ?? "target_policy_missing", target.TenantId, target.AgentId, context!.CorrelationId, mutationOperation: DisableOperation); - var admission = await confirmations.ConfirmAsync( - new McpOperatorConfirmationRequest(request.PlanToken!, request.IdempotencyKey!, payloadHash, decision), cancellationToken).ConfigureAwait(false); + var confirmationRequest = new McpOperatorConfirmationRequest(request.PlanToken!, request.IdempotencyKey!, payloadHash, decision); + if (await TryReplayMutationAsync(confirmationRequest, confirmations, administration, target, context!, DisableOperation, + request.PolicyId, request.ExpectedVersion, null, cancellationToken).ConfigureAwait(false) is { } replay) + return replay; + if (!CanDisable(policy, request.ExpectedVersion, context!.Principal, out failure)) + { + // A concurrent confirmation may finish between the receipt lookup + // and mutable version/state validation. Never consume a stale new plan. + return await TryReplayMutationAsync(confirmationRequest, confirmations, administration, target, context!, DisableOperation, + request.PolicyId, request.ExpectedVersion, null, cancellationToken).ConfigureAwait(false) + ?? Failure(failure!, target.TenantId, target.AgentId, context!.CorrelationId, mutationOperation: DisableOperation); + } + + var admission = await confirmations.ConfirmAsync(confirmationRequest, cancellationToken).ConfigureAwait(false); if (admission.FailureCode is { } confirmationFailure) return Failure(confirmationFailure, target.TenantId, target.AgentId, context.CorrelationId, mutationOperation: DisableOperation); if (admission.IsReplay) - return await ReplayMutationAsync(admission, administration, target, context, DisableOperation, cancellationToken).ConfigureAwait(false); + return await ReplayMutationAsync(admission, administration, target, context!, DisableOperation, request.PolicyId, request.ExpectedVersion, null, cancellationToken).ConfigureAwait(false); if (!admission.IsNewDispatch || admission.IdempotencyId is not { } idempotencyId) return Failure("confirmation_plan_invalid", target.TenantId, target.AgentId, context.CorrelationId, mutationOperation: DisableOperation); @@ -576,7 +597,7 @@ private static async Task ConfirmDisableAsync( return Failure("operator_policy_not_found", target.TenantId, target.AgentId, context.CorrelationId, mutationOperation: DisableOperation); } - await confirmations.CompleteAsync(idempotencyId, McpOperatorIdempotencyOutcome.Succeeded, disabled.PolicyId.ToString("D"), cancellationToken).ConfigureAwait(false); + await confirmations.CompleteAsync(idempotencyId, McpOperatorIdempotencyOutcome.Succeeded, PolicyMutationResultReference(disabled), CancellationToken.None).ConfigureAwait(false); return Results.Ok(new McpOperatorPolicyMutationResult(disabled, false, context.CorrelationId)); } catch (McpOperatorAdmissionRejectedException rejection) @@ -605,7 +626,7 @@ private static async Task PreviewRevokeAsync( IMcpOperatorConfirmationService confirmations, CancellationToken cancellationToken) { - if (!TryContext(http, hostEnvironment, "preview_revoke", out var context, out var failure)) + if (!TryContext(http, hostEnvironment, "preview_revoke", out var context, out var failure) || context is null) return Failure(failure!, null, null, http.TraceIdentifier, mutationOperation: RevokeOperation); if (!IsValidVersionedPolicyRequest(request.PolicyId, request.ExpectedVersion) || request.Target is null) return Failure("validation_error", null, null, context!.CorrelationId, mutationOperation: RevokeOperation); @@ -638,7 +659,7 @@ private static async Task ConfirmRevokeAsync( IMcpOperatorConfirmationService confirmations, CancellationToken cancellationToken) { - if (!TryContext(http, hostEnvironment, "confirm_revoke", out var context, out var failure)) + if (!TryContext(http, hostEnvironment, "confirm_revoke", out var context, out var failure) || context is null) return Failure(failure!, null, null, http.TraceIdentifier, mutationOperation: RevokeOperation); if (!IsValidVersionedPolicyRequest(request.PolicyId, request.ExpectedVersion) || request.Target is null || !HasPlanCredentials(request.PlanToken, request.IdempotencyKey)) @@ -653,20 +674,30 @@ private static async Task ConfirmRevokeAsync( var target = existing.Target!; if (!SameTarget(policy.TargetSelector, request.Target)) return Failure("policy_target_immutable", target.TenantId, target.AgentId, context!.CorrelationId, mutationOperation: RevokeOperation); - if (!CanRevoke(policy, request.ExpectedVersion, context.Principal, out failure)) - return Failure(failure!, target.TenantId, target.AgentId, context.CorrelationId, mutationOperation: RevokeOperation); var payloadHash = PayloadHash(RevokeOperation, new PolicyDisablePayload(request.PolicyId, request.ExpectedVersion, request.Target)); - var decision = await EvaluateAsync(context, target, payloadHash, RevokeOperation, authorization, administration, cancellationToken).ConfigureAwait(false); + var decision = await EvaluateAsync(context!, target, payloadHash, RevokeOperation, authorization, administration, cancellationToken).ConfigureAwait(false); if (!decision.IsAllowed) - return Failure(decision.FailureCode ?? "target_policy_missing", target.TenantId, target.AgentId, context.CorrelationId, mutationOperation: RevokeOperation); + return Failure(decision.FailureCode ?? "target_policy_missing", target.TenantId, target.AgentId, context!.CorrelationId, mutationOperation: RevokeOperation); - var admission = await confirmations.ConfirmAsync( - new McpOperatorConfirmationRequest(request.PlanToken!, request.IdempotencyKey!, payloadHash, decision), cancellationToken).ConfigureAwait(false); + var confirmationRequest = new McpOperatorConfirmationRequest(request.PlanToken!, request.IdempotencyKey!, payloadHash, decision); + if (await TryReplayMutationAsync(confirmationRequest, confirmations, administration, target, context!, RevokeOperation, + request.PolicyId, request.ExpectedVersion, null, cancellationToken).ConfigureAwait(false) is { } replay) + return replay; + if (!CanRevoke(policy, request.ExpectedVersion, context.Principal, out failure)) + { + // A concurrent confirmation may finish between the receipt lookup + // and mutable version/state validation. Never consume a stale new plan. + return await TryReplayMutationAsync(confirmationRequest, confirmations, administration, target, context!, RevokeOperation, + request.PolicyId, request.ExpectedVersion, null, cancellationToken).ConfigureAwait(false) + ?? Failure(failure!, target.TenantId, target.AgentId, context!.CorrelationId, mutationOperation: RevokeOperation); + } + + var admission = await confirmations.ConfirmAsync(confirmationRequest, cancellationToken).ConfigureAwait(false); if (admission.FailureCode is { } confirmationFailure) return Failure(confirmationFailure, target.TenantId, target.AgentId, context.CorrelationId, mutationOperation: RevokeOperation); if (admission.IsReplay) - return await ReplayMutationAsync(admission, administration, target, context, RevokeOperation, cancellationToken).ConfigureAwait(false); + return await ReplayMutationAsync(admission, administration, target, context!, RevokeOperation, request.PolicyId, request.ExpectedVersion, null, cancellationToken).ConfigureAwait(false); if (!admission.IsNewDispatch || admission.IdempotencyId is not { } idempotencyId) return Failure("confirmation_plan_invalid", target.TenantId, target.AgentId, context.CorrelationId, mutationOperation: RevokeOperation); @@ -682,7 +713,7 @@ private static async Task ConfirmRevokeAsync( return Failure("operator_policy_not_found", target.TenantId, target.AgentId, context.CorrelationId, mutationOperation: RevokeOperation); } - await confirmations.CompleteAsync(idempotencyId, McpOperatorIdempotencyOutcome.Succeeded, revoked.PolicyId.ToString("D"), cancellationToken).ConfigureAwait(false); + await confirmations.CompleteAsync(idempotencyId, McpOperatorIdempotencyOutcome.Succeeded, PolicyMutationResultReference(revoked), CancellationToken.None).ConfigureAwait(false); return Results.Ok(new McpOperatorPolicyMutationResult(revoked, false, context.CorrelationId)); } catch (McpOperatorAdmissionRejectedException rejection) @@ -728,26 +759,133 @@ private static async Task ReplayAsync( : Results.Ok(new McpOperatorPolicyCreateResult(created, true, context.CorrelationId)); } + private static async Task TryReplayMutationAsync( + McpOperatorConfirmationRequest request, + IMcpOperatorConfirmationService confirmations, + IMcpOperatorPolicyAdministration administration, + McpOperatorPolicyMutationTarget target, + McpOperatorPolicyMutationContext context, + string mutationOperation, + Guid policyId, + long expectedVersion, + McpOperatorPolicyDraft? replacement, + CancellationToken cancellationToken) + { + // The current decision binds identity, target, policy revision and exact + // payload. A replay-only lookup cannot reserve or consume a new dispatch. + var admission = await confirmations.ConfirmAsync(request with { ReplayOnly = true }, cancellationToken).ConfigureAwait(false); + if (admission.FailureCode is { } failure) + return Failure(failure, target.TenantId, target.AgentId, context.CorrelationId, mutationOperation: mutationOperation); + return admission.IsReplay + ? await ReplayMutationAsync(admission, administration, target, context, mutationOperation, policyId, expectedVersion, replacement, cancellationToken).ConfigureAwait(false) + : null; + } + private static async Task ReplayMutationAsync( McpOperatorConfirmationAdmission admission, IMcpOperatorPolicyAdministration administration, McpOperatorPolicyMutationTarget target, McpOperatorPolicyMutationContext context, string mutationOperation, + Guid policyId, + long expectedVersion, + McpOperatorPolicyDraft? replacement, CancellationToken cancellationToken) { if (admission.Outcome == McpOperatorIdempotencyOutcome.Pending) return Failure("idempotency_pending", target.TenantId, target.AgentId, context.CorrelationId, mutationOperation: mutationOperation); - if (admission.Outcome != McpOperatorIdempotencyOutcome.Succeeded || - !Guid.TryParse(admission.ResultReference, out var policyId)) - { + if (admission.Outcome != McpOperatorIdempotencyOutcome.Succeeded || expectedVersion == long.MaxValue) return Failure("idempotency_failed", target.TenantId, target.AgentId, context.CorrelationId, mutationOperation: mutationOperation); - } var policy = await administration.GetAsync(policyId, cancellationToken).ConfigureAwait(false); - return policy is null - ? Failure("idempotency_replay_unavailable", target.TenantId, target.AgentId, context.CorrelationId, mutationOperation: mutationOperation) - : Results.Ok(new McpOperatorPolicyMutationResult(policy, true, context.CorrelationId)); + if (policy is null || policy.Environment != context.Environment || + !TryBindTarget(context, policy.TargetSelector, out var currentTarget, out _) || currentTarget != target || + !TryReadPolicyMutationResult(admission.ResultReference, policy, expectedVersion + 1, mutationOperation, out var original)) + return Failure("idempotency_replay_unavailable", target.TenantId, target.AgentId, context.CorrelationId, mutationOperation: mutationOperation); + + // A replaced draft is bound by the consumed plan's payload hash. Rebuild + // its normalized fields, retaining immutable creation metadata, so later + // replacements or lifecycle changes cannot rewrite the original result. + if (replacement is not null) + { + original = original with + { + Name = replacement.Name.Trim(), + Environment = replacement.Environment, + Effect = replacement.Effect, + Priority = replacement.Priority, + PrincipalSelector = replacement.PrincipalSelector with { Value = replacement.PrincipalSelector.Value.Trim() }, + TargetSelector = replacement.TargetSelector with { ClientTag = replacement.TargetSelector.ClientTag?.Trim() }, + TargetClassification = replacement.TargetClassification, + OperationFamily = replacement.OperationFamily, + Operation = string.IsNullOrWhiteSpace(replacement.Operation) ? null : replacement.Operation.Trim(), + Constraints = replacement.Constraints, + ExpiresAtUtc = replacement.ExpiresAtUtc, + ReviewByUtc = replacement.ReviewByUtc, + AuditReference = string.IsNullOrWhiteSpace(replacement.AuditReference) ? null : replacement.AuditReference.Trim() + }; + } + + return Results.Ok(new McpOperatorPolicyMutationResult(original, true, context.CorrelationId)); + } + + private static string PolicyMutationResultReference(McpOperatorPolicy policy) => + string.Create(CultureInfo.InvariantCulture, + $"policy-mutation:{policy.PolicyId:D}:{policy.Version}:{(int)policy.LifecycleState}:{policy.DisabledAtUtc?.UtcTicks ?? 0}"); + + private static bool TryReadPolicyMutationResult( + string? resultReference, + McpOperatorPolicy current, + long expectedResultVersion, + string mutationOperation, + out McpOperatorPolicy original) + { + original = current; + var lifecycle = mutationOperation switch + { + ReplaceOperation => McpOperatorPolicyLifecycleState.Active, + DisableOperation => McpOperatorPolicyLifecycleState.Disabled, + _ => McpOperatorPolicyLifecycleState.Revoked + }; + var disabledAtUtc = mutationOperation == ReplaceOperation ? null : current.DisabledAtUtc; + if (!Guid.TryParse(resultReference, out var policyId)) + { + var parts = resultReference?.Split(':'); + if (parts is not { Length: 5 } || parts[0] != "policy-mutation" || + !Guid.TryParse(parts[1], out policyId) || + !long.TryParse(parts[2], NumberStyles.None, CultureInfo.InvariantCulture, out var version) || version != expectedResultVersion || + !int.TryParse(parts[3], NumberStyles.None, CultureInfo.InvariantCulture, out var state) || state != (int)lifecycle || + !long.TryParse(parts[4], NumberStyles.None, CultureInfo.InvariantCulture, out var disabledTicks) || + disabledTicks < 0 || disabledTicks > DateTimeOffset.MaxValue.UtcTicks) + { + return false; + } + disabledAtUtc = disabledTicks == 0 ? null : new DateTimeOffset(disabledTicks, TimeSpan.Zero); + } + // Legacy identifier-only receipts still have an exact versioned payload. + // Disabled/revoked definitions and their disabled actor remain immutable. + // PostgreSQL retains microseconds; compare at that precision while the + // receipt preserves the original response's full timestamp precision. + if (policyId != current.PolicyId || current.Version < expectedResultVersion || + (mutationOperation == ReplaceOperation && disabledAtUtc is not null) || + (mutationOperation == DisableOperation && current.LifecycleState is not (McpOperatorPolicyLifecycleState.Disabled or McpOperatorPolicyLifecycleState.Revoked)) || + (mutationOperation == RevokeOperation && current.LifecycleState != McpOperatorPolicyLifecycleState.Revoked) || + (mutationOperation != ReplaceOperation && + (disabledAtUtc is not { } originalDisabledAtUtc || current.DisabledAtUtc is not { } persistedDisabledAtUtc || + persistedDisabledAtUtc.UtcTicks / TimeSpan.TicksPerMicrosecond != originalDisabledAtUtc.UtcTicks / TimeSpan.TicksPerMicrosecond || + current.DisabledBy is null))) + { + return false; + } + + original = current with + { + Version = expectedResultVersion, + LifecycleState = lifecycle, + DisabledAtUtc = disabledAtUtc, + DisabledBy = mutationOperation == ReplaceOperation ? null : current.DisabledBy + }; + return true; } private static async Task ReplayTargetProfileAsync( @@ -1178,7 +1316,7 @@ private static IResult Failure( "delegated_identity_required" or "delegated_identity_invalid" => StatusCodes.Status401Unauthorized, "target_not_found" or "operator_policy_not_found" => StatusCodes.Status404NotFound, "validation_error" or "environment_mismatch" => StatusCodes.Status400BadRequest, - "operator_policy_self_escalation" or "operator_policy_version_conflict" or "policy_environment_immutable" or "policy_target_immutable" or "confirmation_plan_invalid" or "confirmation_plan_stale" or "confirmation_plan_expired" or "idempotency_conflict" => StatusCodes.Status409Conflict, + "operator_policy_self_escalation" or "operator_policy_version_conflict" or "policy_environment_immutable" or "policy_target_immutable" or "confirmation_plan_invalid" or "confirmation_plan_stale" or "confirmation_plan_expired" or "idempotency_conflict" or "idempotency_pending" => StatusCodes.Status409Conflict, _ => StatusCodes.Status403Forbidden }; return Results.Problem( diff --git a/src/NetRatel/NetRatel.API/Endpoints/Client/McpOperatorScriptEndpoints.cs b/src/NetRatel/NetRatel.API/Endpoints/Client/McpOperatorScriptEndpoints.cs index 8846678d..037ab820 100644 --- a/src/NetRatel/NetRatel.API/Endpoints/Client/McpOperatorScriptEndpoints.cs +++ b/src/NetRatel/NetRatel.API/Endpoints/Client/McpOperatorScriptEndpoints.cs @@ -1,3 +1,4 @@ +using System.Globalization; using System.Security.Cryptography; using System.Text; using System.Text.Json; @@ -218,18 +219,47 @@ private static async Task ConfirmMutationAsync( } if (!HasPlanCredentials(request.PlanToken, request.IdempotencyKey) || - !TryBuildMutationDraft(action, request, tenantId, context, scripts, cancellationToken, out var draftTask)) + !TryBuildMutationDraft(action, request, tenantId, context, scripts, cancellationToken, out var draftTask, includeDeleted: true)) return Failure("confirmation_plan_invalid", context); McpOperatorScriptDraft? draft; try { draft = await draftTask.ConfigureAwait(false); } catch (McpOperatorScriptIntegrityException) { return Failure("script_integrity_invalid", context); } if (draft is null) return Failure("script_not_found", context); - var confirmation = await confirmations.ConfirmAsync(new McpOperatorConfirmationRequest( - request.PlanToken!, request.IdempotencyKey!, MutationPayloadHash(action, request.ScriptId, request.ExpectedVersion, draft), context.Decision), cancellationToken).ConfigureAwait(false); + var confirmationRequest = new McpOperatorConfirmationRequest(request.PlanToken!, request.IdempotencyKey!, + MutationPayloadHash(action, request.ScriptId, request.ExpectedVersion, draft), context.Decision); + if (action == "delete") + { + // Retained owner-bound content preserves the original payload hash + // after deletion. Only a matching consumed confirmation can replay; + // a fresh plan must still name a live revision and stay unconsumed + // when that revision has been changed or deleted. + var replayRequest = confirmationRequest with { ReplayOnly = true }; + var replay = await confirmations.ConfirmAsync(replayRequest, cancellationToken).ConfigureAwait(false); + if (replay.FailureCode is { } replayCode) return Failure(replayCode, context); + if (replay.IsReplay) + return await ReplayMutationAsync(action, replay, request.ScriptId, tenantId, context, http, scripts, cancellationToken).ConfigureAwait(false); + (McpOperatorScriptLease Lease, string Content, string? ManifestJson)? current; + try + { + current = await scripts.GetOwnedAsync(request.ScriptId!.Value, tenantId, context.Request.Principal, + context.Request.McpResource, context.Request.McpInstance, cancellationToken).ConfigureAwait(false); + } + catch (McpOperatorScriptIntegrityException) { return Failure("script_integrity_invalid", context); } + if (current is null || current.Value.Lease.Version != request.ExpectedVersion) + { + // Reconcile a confirmation that completed during the live read. + replay = await confirmations.ConfirmAsync(replayRequest, cancellationToken).ConfigureAwait(false); + if (replay.FailureCode is { } concurrentCode) return Failure(concurrentCode, context); + return replay.IsReplay + ? await ReplayMutationAsync(action, replay, request.ScriptId, tenantId, context, http, scripts, cancellationToken).ConfigureAwait(false) + : Failure(current is null ? "script_not_found" : "script_etag_mismatch", context); + } + } + var confirmation = await confirmations.ConfirmAsync(confirmationRequest, cancellationToken).ConfigureAwait(false); if (confirmation.FailureCode is { } failureCode) return Failure(failureCode, context); if (confirmation.IsReplay) - return await ReplayMutationAsync(action, confirmation, request.ScriptId, tenantId, context, scripts, cancellationToken).ConfigureAwait(false); + return await ReplayMutationAsync(action, confirmation, request.ScriptId, tenantId, context, http, scripts, cancellationToken).ConfigureAwait(false); if (!confirmation.IsNewDispatch || confirmation.IdempotencyId is not { } idempotencyId) return Failure("confirmation_plan_invalid", context); @@ -248,7 +278,10 @@ private static async Task ConfirmMutationAsync( await confirmations.CompleteAsync(idempotencyId, McpOperatorIdempotencyOutcome.Failed, "script_not_found", cancellationToken).ConfigureAwait(false); return Failure("script_not_found", context); } - await confirmations.CompleteAsync(idempotencyId, McpOperatorIdempotencyOutcome.Succeeded, result.ScriptId.ToString(), cancellationToken).ConfigureAwait(false); + var resultReference = string.Create(CultureInfo.InvariantCulture, $"script-mutation:{result.ScriptId}:{result.Version}"); + // The durable mutation has already succeeded. Persist its receipt + // even when the client disconnects before receiving the response. + await confirmations.CompleteAsync(idempotencyId, McpOperatorIdempotencyOutcome.Succeeded, resultReference, CancellationToken.None).ConfigureAwait(false); SetEtag(http, result.Version); return action == "delete" ? Results.Ok(new McpOperatorScriptMutationResult(result.ScriptId, result.Version, true, false, context.CorrelationId)) @@ -426,7 +459,7 @@ constraints.MaxFanOut is not > 0 || constraints.MaxOutputBytes is not > 0 || private static bool TryBuildMutationDraft( string action, McpOperatorScriptMutationRequest request, int tenantId, McpOperatorScriptContext context, - IMcpOperatorScriptStore scripts, CancellationToken cancellationToken, out Task draftTask) + IMcpOperatorScriptStore scripts, CancellationToken cancellationToken, out Task draftTask, bool includeDeleted = false) { if (action is "create" or "update") { @@ -445,12 +478,12 @@ private static bool TryBuildMutationDraft( } if (action == "delete") { - if (request.ScriptId is not > 0 || request.ExpectedVersion is not > 0) + if (request.ScriptId is not > 0 || request.ExpectedVersion is not > 0 || request.Script is not null || request.ManifestJson is not null) { draftTask = Task.FromResult(null); return false; } - draftTask = BuildExistingDraftAsync(request.ScriptId.Value, tenantId, context, scripts, cancellationToken); + draftTask = BuildExistingDraftAsync(request.ScriptId.Value, tenantId, context, scripts, includeDeleted, cancellationToken); return true; } draftTask = Task.FromResult(null); @@ -463,9 +496,11 @@ private static bool TryBuildMutationDraft( return existing is null ? null : ToDraft(existing.Value, request.ManifestJson); } - private static async Task BuildExistingDraftAsync(long scriptId, int tenantId, McpOperatorScriptContext context, IMcpOperatorScriptStore scripts, CancellationToken cancellationToken) + private static async Task BuildExistingDraftAsync(long scriptId, int tenantId, McpOperatorScriptContext context, IMcpOperatorScriptStore scripts, bool includeDeleted, CancellationToken cancellationToken) { - var existing = await scripts.GetOwnedAsync(scriptId, tenantId, context.Request.Principal, context.Request.McpResource, context.Request.McpInstance, cancellationToken).ConfigureAwait(false); + var existing = includeDeleted + ? await scripts.GetOwnedForReplayAsync(scriptId, tenantId, context.Request.Principal, context.Request.McpResource, context.Request.McpInstance, cancellationToken).ConfigureAwait(false) + : await scripts.GetOwnedAsync(scriptId, tenantId, context.Request.Principal, context.Request.McpResource, context.Request.McpInstance, cancellationToken).ConfigureAwait(false); return existing is null ? null : ToDraft(existing.Value, existing.Value.ManifestJson); } @@ -473,26 +508,40 @@ private static bool TryBuildMutationDraft( existing.Lease.Name, existing.Lease.Description, existing.Lease.ShellType, existing.Content, existing.Lease.ContentHash, existing.Lease.Parameters, existing.Lease.TimeoutSeconds, existing.Lease.WorkingDirectory, existing.Lease.DeclaredSideEffects, manifestJson); - private static async Task ReplayMutationAsync(string action, McpOperatorConfirmationAdmission confirmation, long? scriptId, int tenantId, McpOperatorScriptContext context, IMcpOperatorScriptStore scripts, CancellationToken cancellationToken) + private static async Task ReplayMutationAsync(string action, McpOperatorConfirmationAdmission confirmation, long? scriptId, int tenantId, McpOperatorScriptContext context, HttpContext http, IMcpOperatorScriptStore scripts, CancellationToken cancellationToken) { if (confirmation.Outcome == McpOperatorIdempotencyOutcome.Pending) return Failure("idempotency_pending", context); - if (confirmation.Outcome != McpOperatorIdempotencyOutcome.Succeeded || !long.TryParse(confirmation.ResultReference, out var resultId)) + if (confirmation.Outcome != McpOperatorIdempotencyOutcome.Succeeded || ReadMutationReference(confirmation.ResultReference) is not { } recorded || + (action != "create" && scriptId != recorded.ScriptId)) return Failure("idempotency_replay_unavailable", context); - if (action == "delete") return Results.Ok(new McpOperatorScriptMutationResult(resultId, 0, true, true, context.CorrelationId)); (McpOperatorScriptLease Lease, string Content, string? ManifestJson)? existing; try { - existing = await scripts.GetOwnedAsync(resultId, tenantId, context.Request.Principal, context.Request.McpResource, context.Request.McpInstance, cancellationToken).ConfigureAwait(false); + existing = await scripts.GetOwnedForReplayAsync(recorded.ScriptId, tenantId, context.Request.Principal, context.Request.McpResource, context.Request.McpInstance, cancellationToken).ConfigureAwait(false); } catch (McpOperatorScriptIntegrityException) { return Failure("script_integrity_invalid", context); } - return existing is null - ? Failure("idempotency_replay_unavailable", context) - : Results.Ok(new McpOperatorScriptMutationResult(resultId, existing.Value.Lease.Version, false, true, context.CorrelationId)); + if (existing is null) return Failure("idempotency_replay_unavailable", context); + // Legacy id-only receipts can supply only the current owned revision. + var version = recorded.Version ?? existing.Value.Lease.Version; + SetEtag(http, version); + return Results.Ok(new McpOperatorScriptMutationResult(recorded.ScriptId, version, action == "delete", true, context.CorrelationId)); + } + + private static ScriptMutationReference? ReadMutationReference(string? reference) + { + if (PositiveDecimal(reference, out var legacyScriptId)) return new(legacyScriptId, null); + return reference?.Split(':') is ["script-mutation", var scriptIdText, var versionText] && + PositiveDecimal(scriptIdText, out var scriptId) && PositiveDecimal(versionText, out var version) + ? new(scriptId, version) + : null; } + private static bool PositiveDecimal(string? value, out long parsed) => + long.TryParse(value, NumberStyles.None, CultureInfo.InvariantCulture, out parsed) && parsed > 0; + private static async Task TryCreateContextAsync( string operation, int tenantId, Guid agentId, HttpContext http, IHostEnvironment environment, [FromServices] IClientPresenceRouter presence, IMcpOperatorRouteAdmission admission, @@ -640,6 +689,7 @@ private static IResult Failure(string code, McpOperatorScriptContext context) private sealed record McpOperatorScriptContext(McpOperatorRouteAccessRequest Request, McpOperatorDecision Decision, string RequiredScope, int TenantId, Guid AgentId, string Operation, string CorrelationId); private sealed record McpOperatorScriptContextResult(McpOperatorScriptContext? Context, IResult? Failure); private sealed record McpOperatorScriptRunResolution(McpOperatorScriptContext? Context, (McpOperatorScriptLease Lease, string Content, string? ManifestJson)? Script, IResult? Failure); + private sealed record ScriptMutationReference(long ScriptId, long? Version); } public sealed record McpOperatorScriptMutationRequest( diff --git a/src/NetRatel/NetRatel.API/Middleware/McpOperatorDelegationMiddleware.cs b/src/NetRatel/NetRatel.API/Middleware/McpOperatorDelegationMiddleware.cs index bb77deb7..996a21cb 100644 --- a/src/NetRatel/NetRatel.API/Middleware/McpOperatorDelegationMiddleware.cs +++ b/src/NetRatel/NetRatel.API/Middleware/McpOperatorDelegationMiddleware.cs @@ -110,6 +110,12 @@ await context.Response.WriteAsJsonAsync(new if (credential.OwnerPrincipalId != delegation.Identity.Subject || !string.Equals(credential.Resource, delegation.Resource, StringComparison.Ordinal)) return "local_delegation_binding_mismatch"; + // Existing signed relay clients keep their own application client id. + // A receipt proof instead binds the exact plan to the exchanged ingress + // credential identity, so it cannot borrow that compatibility contract. + if (delegation.ReplayBinding is not null && + (credential.CredentialId != delegation.Identity.ClientId || credential.CredentialId != delegation.IngressCredentialId)) + return "local_delegation_binding_mismatch"; if (!McpLocalDelegationTargetRequirements.TryGetAuthorizationTenant(delegation, out var authorizationTenantId)) return "local_delegation_target_invalid"; var target = McpOperationTargetCatalog.Find(delegation.Tool, delegation.Operation); diff --git a/src/NetRatel/NetRatel.Application/Operations/McpOperatorContracts.cs b/src/NetRatel/NetRatel.Application/Operations/McpOperatorContracts.cs index 666c0a3f..68aa025f 100644 --- a/src/NetRatel/NetRatel.Application/Operations/McpOperatorContracts.cs +++ b/src/NetRatel/NetRatel.Application/Operations/McpOperatorContracts.cs @@ -1038,6 +1038,15 @@ Task> ListOwnedAsync( string mcpInstance, CancellationToken cancellationToken); + /// Reads an owned retained revision, including a tombstone, only to reconcile an authenticated confirmation receipt. + Task<(McpOperatorScriptLease Lease, string Content, string? ManifestJson)?> GetOwnedForReplayAsync( + long scriptId, + int tenantId, + McpOperatorPrincipal principal, + string mcpResource, + string mcpInstance, + CancellationToken cancellationToken) => GetOwnedAsync(scriptId, tenantId, principal, mcpResource, mcpInstance, cancellationToken); + Task CreateAsync( McpOperatorScriptCreateRequest request, CancellationToken cancellationToken); diff --git a/src/NetRatel/NetRatel.Infrastructure/Identity/McpOperationObjectTargetResolver.cs b/src/NetRatel/NetRatel.Infrastructure/Identity/McpOperationObjectTargetResolver.cs index 56b3ff34..9b13dc79 100644 --- a/src/NetRatel/NetRatel.Infrastructure/Identity/McpOperationObjectTargetResolver.cs +++ b/src/NetRatel/NetRatel.Infrastructure/Identity/McpOperationObjectTargetResolver.cs @@ -1,6 +1,9 @@ using Microsoft.EntityFrameworkCore; +using NetRatel.Application.Jobs; +using NetRatel.Application.Operations; using NetRatel.Infrastructure.Persistence; using NetRatel.Shared.Operations; +using System.Globalization; namespace NetRatel.Infrastructure.Identity; @@ -54,11 +57,87 @@ public async Task MatchesDelegationAsync( _ => null }; - return owner is { } resolved && - resolved.TenantId == delegation.TenantId && - resolved.AgentId == delegation.AgentId; + if (owner is { } resolved) + return resolved.TenantId == delegation.TenantId && resolved.AgentId == delegation.AgentId; + + return await MatchesDeletedReplayAsync(delegation, identifier, cancellationToken).ConfigureAwait(false); } + private async Task MatchesDeletedReplayAsync(McpOperatorDelegation delegation, long identifier, CancellationToken cancellationToken) + { + if (!delegation.ObjectTargetResolutionEnabled || delegation.ReplayBinding is not { IsValid: true } binding || + string.IsNullOrWhiteSpace(delegation.IngressCredentialId) || + delegation.IngressCredentialId != delegation.Identity.ClientId || + string.IsNullOrWhiteSpace(delegation.Resource) || string.IsNullOrWhiteSpace(delegation.Instance)) return false; + + var jobMutation = delegation.Tool == "netratel_jobs" && McpOperatorJobConfirmationPayload.IsMutation(delegation.Operation); + var runAction = delegation.Tool == "netratel_job_runs" && delegation.Operation is "cancel" or "delete"; + if (!jobMutation && !runAction) return false; + + // A tombstone is an owner-bound receipt source only. Joining a run's + // durable job record prevents another job with the same public id from + // supplying identity after the original definition has been removed. + var owners = jobMutation + ? db.McpOperatorJobs.AsNoTracking().Where(job => job.JobId == identifier && job.DeletedAtUtc != null) + .Select(job => new ReplayOwner(job.TenantId, job.AgentId, job.Subject, job.ClientId, job.McpResource, job.McpInstance, job.TargetSetDigest)) + : from run in db.McpOperatorJobRuns.AsNoTracking() + join job in db.McpOperatorJobs.AsNoTracking() on run.JobRecordId equals job.Id + where run.JobRunId == identifier && run.DeletedAtUtc != null && run.JobId == job.JobId && + run.TenantId == job.TenantId && run.AgentId == job.AgentId + select new ReplayOwner(run.TenantId, run.AgentId, job.Subject, job.ClientId, job.McpResource, job.McpInstance, run.TargetSetDigest); + var matches = await owners.Take(2).ToArrayAsync(cancellationToken).ConfigureAwait(false); + if (matches is not [var owner] || owner.TenantId != delegation.TenantId || owner.AgentId != delegation.AgentId || + owner.Subject != delegation.Identity.Subject || owner.ClientId != delegation.Identity.ClientId || + owner.Resource != delegation.Resource || owner.Instance != delegation.Instance || string.IsNullOrWhiteSpace(owner.TargetSetDigest)) return false; + + var operation = McpOperatorOperationCatalog.Find(delegation.Tool, delegation.Operation); + var plan = await db.McpOperatorConfirmationPlans.AsNoTracking() + .SingleOrDefaultAsync(record => record.TokenHash == binding.PlanTokenHash, cancellationToken).ConfigureAwait(false); + var now = DateTimeOffset.UtcNow; + if (operation is null || plan?.ConsumedIdempotencyId is not { } receiptId || plan.ConsumedAtUtc is null || + plan.ExpiresAtUtc <= now || plan.Subject != owner.Subject || plan.ClientId != owner.ClientId || + plan.McpResource != owner.Resource || plan.McpInstance != owner.Instance || + plan.TenantId != owner.TenantId || plan.AgentId != owner.AgentId || plan.TargetSetDigest != owner.TargetSetDigest || + plan.Operation != $"{delegation.Tool}/{delegation.Operation}" || plan.OperationFamily != operation.OperationFamily || + plan.ConfirmationClass != operation.ConfirmationClass || plan.PayloadHash != binding.PayloadHash || + McpOperatorReplayBinding.Hash(plan.IdempotencyKey) != binding.IdempotencyKeyHash) return false; + + var receipt = await db.McpOperatorIdempotencyRecords.AsNoTracking() + .SingleOrDefaultAsync(record => record.Id == receiptId, cancellationToken).ConfigureAwait(false); + if (receipt is null || receipt.Outcome != McpOperatorIdempotencyOutcome.Succeeded || receipt.CompletedAtUtc is null || + receipt.Environment != plan.Environment || receipt.Subject != plan.Subject || receipt.ClientId != plan.ClientId || + receipt.TenantId != plan.TenantId || receipt.AgentId != plan.AgentId || receipt.TargetSetDigest != plan.TargetSetDigest || + receipt.PolicyId != plan.PolicyId || receipt.PolicyVersion != plan.PolicyVersion || + receipt.OperationFamily != plan.OperationFamily || receipt.Operation != plan.Operation || + receipt.IdempotencyKey != plan.IdempotencyKey || receipt.PayloadHash != plan.PayloadHash || + !ResultMatchesObject(receipt.ResultReference, identifier, jobMutation)) return false; + + // This inexpensive withdrawal check is repeated at execution ingress. + // Route admission and ReplayOnly still evaluate the complete current + // policy, target profile, permission and request before returning data. + return await db.McpOperatorPolicies.AsNoTracking().AnyAsync(policy => policy.Id == plan.PolicyId && + policy.Version == plan.PolicyVersion && policy.Environment == plan.Environment && + policy.Effect == McpOperatorPolicyEffect.Allow && policy.DisabledAtUtc == null && + policy.LifecycleState == McpOperatorPolicyLifecycleState.Active && + (policy.ExpiresAtUtc == null || policy.ExpiresAtUtc > now), cancellationToken).ConfigureAwait(false); + } + + private static bool ResultMatchesObject(string? reference, long identifier, bool jobMutation) + { + if (PositiveDecimal(reference, out var legacyId)) return legacyId == identifier; + var parts = reference?.Split(':'); + if (jobMutation) + return parts is ["job-mutation", var jobIdText, var versionText, var itemIdText] && + PositiveDecimal(jobIdText, out var jobId) && jobId == identifier && PositiveDecimal(versionText, out _) && + (itemIdText.Length == 0 || PositiveDecimal(itemIdText, out _)); + return parts is ["job-run-action", var runIdText, var stateText] && + PositiveDecimal(runIdText, out var runId) && runId == identifier && + Enum.TryParse(stateText, out var state) && Enum.IsDefined(state) && state.ToString() == stateText; + } + + private static bool PositiveDecimal(string? value, out long parsed) => + long.TryParse(value, NumberStyles.None, CultureInfo.InvariantCulture, out parsed) && parsed > 0; + private async Task JobOwnerAsync(long jobId, CancellationToken cancellationToken) => await SingleOwnerAsync( db.McpOperatorJobs.AsNoTracking() @@ -101,4 +180,5 @@ public async Task MatchesDelegationAsync( } private sealed record McpObjectOwner(int TenantId, Guid AgentId); + private sealed record ReplayOwner(int TenantId, Guid AgentId, string Subject, string ClientId, string Resource, string Instance, string TargetSetDigest); } diff --git a/src/NetRatel/NetRatel.Infrastructure/Services/McpOperatorScriptStore.cs b/src/NetRatel/NetRatel.Infrastructure/Services/McpOperatorScriptStore.cs index d8562564..3a956f23 100644 --- a/src/NetRatel/NetRatel.Infrastructure/Services/McpOperatorScriptStore.cs +++ b/src/NetRatel/NetRatel.Infrastructure/Services/McpOperatorScriptStore.cs @@ -66,12 +66,31 @@ public async Task> ListOwnedAsync( .Select(entry => ToLease(entry.record)).ToArray(); } - public async Task<(McpOperatorScriptLease Lease, string Content, string? ManifestJson)?> GetOwnedAsync( + public Task<(McpOperatorScriptLease Lease, string Content, string? ManifestJson)?> GetOwnedAsync( long scriptId, int tenantId, McpOperatorPrincipal principal, string mcpResource, string mcpInstance, + CancellationToken cancellationToken) => + ReadOwnedAsync(scriptId, tenantId, principal, mcpResource, mcpInstance, false, cancellationToken); + + public Task<(McpOperatorScriptLease Lease, string Content, string? ManifestJson)?> GetOwnedForReplayAsync( + long scriptId, + int tenantId, + McpOperatorPrincipal principal, + string mcpResource, + string mcpInstance, + CancellationToken cancellationToken) => + ReadOwnedAsync(scriptId, tenantId, principal, mcpResource, mcpInstance, true, cancellationToken); + + private async Task<(McpOperatorScriptLease Lease, string Content, string? ManifestJson)?> ReadOwnedAsync( + long scriptId, + int tenantId, + McpOperatorPrincipal principal, + string mcpResource, + string mcpInstance, + bool includeDeleted, CancellationToken cancellationToken) { if (scriptId <= 0 || !IsOwnershipValid(tenantId, principal, mcpResource, mcpInstance)) @@ -85,7 +104,7 @@ public async Task> ListOwnedAsync( entry.record.ClientId == ClientId(principal) && entry.record.McpResource == mcpResource && entry.record.McpInstance == mcpInstance && - entry.record.DeletedAtUtc == null, + (includeDeleted || entry.record.DeletedAtUtc == null), cancellationToken) .ConfigureAwait(false); if (result is null) diff --git a/src/NetRatel/NetRatel.Mcp.Http/McpLocalCredentialPairingService.cs b/src/NetRatel/NetRatel.Mcp.Http/McpLocalCredentialPairingService.cs index e2a11ce3..628dff80 100644 --- a/src/NetRatel/NetRatel.Mcp.Http/McpLocalCredentialPairingService.cs +++ b/src/NetRatel/NetRatel.Mcp.Http/McpLocalCredentialPairingService.cs @@ -17,7 +17,7 @@ public sealed class McpLocalCredentialPairingService( "netratel_gateway", "authenticate", Guid.NewGuid().ToString("N"), options.PublicResourceUri.TrimEnd('/'), hostContext.Target.Instance, CorrelationId: Activity.Current?.TraceId.ToString() ?? Guid.NewGuid().ToString("N"))); - public string CreateExecutionProof(string tool, string operation, int? tenantId, Guid? agentId, string? objectReference = null, bool objectTargetResolutionEnabled = false, string? correlationId = null) => tokens.Create( + public string CreateExecutionProof(string tool, string operation, int? tenantId, Guid? agentId, string? objectReference = null, bool objectTargetResolutionEnabled = false, string? correlationId = null, McpOperatorReplayBinding? replayBinding = null) => tokens.Create( GatewayIdentity, new McpOperatorDelegationRequest( tool, @@ -30,6 +30,7 @@ public string CreateExecutionProof(string tool, string operation, int? tenantId, correlationId ?? Activity.Current?.TraceId.ToString() ?? Guid.NewGuid().ToString("N")) { ObjectReference = objectReference, - ObjectTargetResolutionEnabled = objectTargetResolutionEnabled + ObjectTargetResolutionEnabled = objectTargetResolutionEnabled, + ReplayBinding = replayBinding }); } diff --git a/src/NetRatel/NetRatel.Mcp.Http/McpOperatorDelegationPropagation.cs b/src/NetRatel/NetRatel.Mcp.Http/McpOperatorDelegationPropagation.cs index 947883f9..8c475a18 100644 --- a/src/NetRatel/NetRatel.Mcp.Http/McpOperatorDelegationPropagation.cs +++ b/src/NetRatel/NetRatel.Mcp.Http/McpOperatorDelegationPropagation.cs @@ -55,7 +55,8 @@ public IDisposable Begin(string? assertion) Activity.Current?.TraceId.ToString() ?? Guid.NewGuid().ToString("N")) { ObjectReference = hostContext.OperatorSurfaceEnabled ? ObjectReferenceFor(target, arguments) : null, - ObjectTargetResolutionEnabled = hostContext.OperatorSurfaceEnabled + ObjectTargetResolutionEnabled = hostContext.OperatorSurfaceEnabled, + ReplayBinding = ReplayBindingFor(tool, operation, arguments) }, issuedAtUtc: null); } @@ -93,7 +94,8 @@ public IDisposable Begin(string? assertion) AgentIdFor(target?.Model, arguments), hostContext.OperatorSurfaceEnabled ? ObjectReferenceFor(target, arguments) : null, hostContext.OperatorSurfaceEnabled, - correlationId)); + correlationId, + ReplayBindingFor(tool, operation, arguments))); try { using var deadline = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); @@ -179,6 +181,12 @@ private static string Operation(IDictionary? arguments) private static string Token(string? value) => string.IsNullOrWhiteSpace(value) || value.Length > 128 || value.Any(char.IsControl) ? "unknown" : value; + private McpOperatorReplayBinding? ReplayBindingFor(string tool, string operation, IDictionary? arguments) => + hostContext.OperatorSurfaceEnabled && arguments is not null && + arguments.TryGetValue("confirm", out var confirm) && confirm.ValueKind == JsonValueKind.True && + arguments.TryGetValue("request", out var request) && + McpOperatorJobConfirmationPayload.TryCreateBinding(tool, operation, request, out var binding) ? binding : null; + private static bool TryRequestProperty(IDictionary? arguments, string name, out JsonElement value) { if (arguments is not null && arguments.TryGetValue("request", out var request) && request.ValueKind == JsonValueKind.Object && diff --git a/src/NetRatel/NetRatel.Shared/Operations/IMcpOperationObjectTargetResolver.cs b/src/NetRatel/NetRatel.Shared/Operations/IMcpOperationObjectTargetResolver.cs index e8f67cef..be23ec0f 100644 --- a/src/NetRatel/NetRatel.Shared/Operations/IMcpOperationObjectTargetResolver.cs +++ b/src/NetRatel/NetRatel.Shared/Operations/IMcpOperationObjectTargetResolver.cs @@ -2,8 +2,10 @@ namespace NetRatel.Shared.Operations; /// /// Resolves the persisted ownership of a closed-schema MCP object reference. -/// Implementations must fail closed when an object is missing, deleted, or no -/// longer belongs to the tenant and agent pair in the delegation. +/// Implementations fail closed when ownership cannot be proved. A deleted +/// object is admitted only for a signed, exact, successful confirmation replay +/// owned by the currently authenticated caller; it remains hidden to reads and +/// fresh previews. Business endpoints still enforce current policy and receipt. /// public interface IMcpOperationObjectTargetResolver { diff --git a/src/NetRatel/NetRatel.Shared/Operations/McpOperatorDelegation.cs b/src/NetRatel/NetRatel.Shared/Operations/McpOperatorDelegation.cs index 9c1925a0..f3f5b215 100644 --- a/src/NetRatel/NetRatel.Shared/Operations/McpOperatorDelegation.cs +++ b/src/NetRatel/NetRatel.Shared/Operations/McpOperatorDelegation.cs @@ -266,6 +266,9 @@ public sealed record McpOperatorDelegationRequest( /// public bool ObjectTargetResolutionEnabled { get; init; } + /// Signed hash-only binding for an exact confirmed job request. + public McpOperatorReplayBinding? ReplayBinding { get; init; } + /// /// The opaque HTTP-MCP credential identifier after the API has exchanged /// it. This is never the bearer secret and is rechecked by the API before @@ -297,6 +300,9 @@ public sealed record McpOperatorDelegation( /// Whether this assertion requires API-side durable object-owner revalidation. public bool ObjectTargetResolutionEnabled { get; init; } + /// Exact confirmation hashes; never independently authorize execution. + public McpOperatorReplayBinding? ReplayBinding { get; init; } + /// Non-secret source credential identity for local HTTP MCP execution. public string? IngressCredentialId { get; init; } @@ -361,6 +367,12 @@ public string Create( AddOptional(claims, "mcp_object_reference", request.ObjectReference); if (request.ObjectTargetResolutionEnabled) claims.Add(new Claim("mcp_object_target_resolution", "true")); + if (request.ReplayBinding is { } replay) + { + claims.Add(new Claim("mcp_replay_plan_hash", replay.PlanTokenHash)); + claims.Add(new Claim("mcp_replay_key_hash", replay.IdempotencyKeyHash)); + claims.Add(new Claim("mcp_replay_payload_hash", replay.PayloadHash)); + } AddOptional(claims, "mcp_ingress_credential_id", request.IngressCredentialId); AddOptional(claims, "mcp_ingress_permission", request.IngressPermission); if (request.TenantId is { } tenantId) @@ -441,6 +453,7 @@ public bool TryValidate(string? assertion, out McpOperatorDelegation? delegation { ObjectReference = OptionalClaim(principal, "mcp_object_reference"), ObjectTargetResolutionEnabled = OptionalBooleanClaim(principal, "mcp_object_target_resolution"), + ReplayBinding = ReadReplayBinding(principal), IngressCredentialId = OptionalClaim(principal, "mcp_ingress_credential_id"), IngressPermission = OptionalClaim(principal, "mcp_ingress_permission") }; @@ -460,6 +473,7 @@ public bool TryValidate(string? assertion, out McpOperatorDelegation? delegation { ObjectReference = delegationRequest.ObjectReference, ObjectTargetResolutionEnabled = delegationRequest.ObjectTargetResolutionEnabled, + ReplayBinding = delegationRequest.ReplayBinding, IngressCredentialId = delegationRequest.IngressCredentialId, IngressPermission = delegationRequest.IngressPermission }; @@ -532,6 +546,17 @@ private static bool OptionalBooleanClaim(ClaimsPrincipal principal, string type) private static IReadOnlyCollection Values(ClaimsPrincipal principal, string type) => principal.FindAll(type).Select(claim => claim.Value).Distinct(StringComparer.Ordinal).Order(StringComparer.Ordinal).ToArray(); + private static McpOperatorReplayBinding? ReadReplayBinding(ClaimsPrincipal principal) + { + var plan = principal.FindAll("mcp_replay_plan_hash").ToArray(); + var key = principal.FindAll("mcp_replay_key_hash").ToArray(); + var payload = principal.FindAll("mcp_replay_payload_hash").ToArray(); + if (plan.Length == 0 && key.Length == 0 && payload.Length == 0) return null; + if (plan.Length != 1 || key.Length != 1 || payload.Length != 1) + throw new InvalidOperationException("Delegation replay binding must contain exactly three hashes."); + return new(plan[0].Value, key[0].Value, payload[0].Value); + } + private static void ValidateIdentity(McpOperatorDelegationIdentity identity) { ArgumentNullException.ThrowIfNull(identity); @@ -558,6 +583,10 @@ private static void ValidateRequest(McpOperatorDelegationRequest request) ValidateOptional(request.ObjectReference, nameof(request.ObjectReference)); ValidateOptional(request.IngressCredentialId, nameof(request.IngressCredentialId)); ValidateOptional(request.IngressPermission, nameof(request.IngressPermission)); + if (request.ReplayBinding is { } replay && (!replay.IsValid || !request.ObjectTargetResolutionEnabled || + !(request.Tool == "netratel_jobs" && McpOperatorJobConfirmationPayload.IsMutation(request.Operation) || + request.Tool == "netratel_job_runs" && request.Operation is "cancel" or "delete"))) + throw new InvalidOperationException("Delegation replay binding is invalid for this operation."); if (request.TenantId is <= 0) throw new InvalidOperationException("Delegation TenantId must be positive when supplied."); if (request.AgentId == Guid.Empty) diff --git a/src/NetRatel/NetRatel.Shared/Operations/McpOperatorReplayBinding.cs b/src/NetRatel/NetRatel.Shared/Operations/McpOperatorReplayBinding.cs new file mode 100644 index 00000000..062b3a02 --- /dev/null +++ b/src/NetRatel/NetRatel.Shared/Operations/McpOperatorReplayBinding.cs @@ -0,0 +1,139 @@ +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace NetRatel.Shared.Operations; + +/// +/// Content-free proof of an exact confirmed request. Neither an opaque plan +/// token, idempotency capability nor business request content enters the JWT. +/// +public sealed record McpOperatorReplayBinding(string PlanTokenHash, string IdempotencyKeyHash, string PayloadHash) +{ + public bool IsValid => IsHash(PlanTokenHash) && IsHash(IdempotencyKeyHash) && IsHash(PayloadHash); + + public static string Hash(string value) => Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(value))); + + private static bool IsHash(string? value) => value is { Length: 64 } && value.All(char.IsAsciiHexDigit); +} + +/// +/// The existing job confirmation payload serialization, shared by the API and +/// the HTTP MCP proof issuer. Nullable/default fields and their order preserve +/// already-issued canonical plan hashes. This is not an authorization decision. +/// +public static class McpOperatorJobConfirmationPayload +{ + private static readonly JsonSerializerOptions ReadOptions = new() + { + PropertyNameCaseInsensitive = true, + NumberHandling = JsonNumberHandling.AllowReadingFromString, + UnmappedMemberHandling = JsonUnmappedMemberHandling.Disallow + }; + + public static bool IsMutation(string action) => action is "create" or "update" or "delete" or + "param_add" or "param_update" or "param_delete" or "step_add" or "step_update" or "step_reorder" or "step_delete"; + + public static string MutationHash(string action, JsonElement request) + { + if (!IsMutation(action)) throw new ArgumentException("A job mutation is required.", nameof(action)); + if (!HasUnambiguousProperties(request)) throw new JsonException("A job mutation has ambiguous properties."); + var body = request.Deserialize(ReadOptions) ?? throw new JsonException("A job mutation is required."); + return MutationHash(action, body); + } + + public static string RunActionHash(string action, ulong runId, string? reason) + { + if (action is not ("cancel" or "delete")) throw new ArgumentException("A job run action is required.", nameof(action)); + return McpOperatorReplayBinding.Hash(JsonSerializer.Serialize(new { action, runId, reason = BoundedReason(reason) })); + } + + /// + /// Rejects invalid or ambiguous shapes before the MCP host's business + /// adapter runs. The API also uses its already-bound typed request to + /// check that the signed proof belongs to the actual confirmation body. + /// + public static bool TryCreateBinding(string tool, string operation, JsonElement request, out McpOperatorReplayBinding? binding) + { + binding = null; + if (request.ValueKind != JsonValueKind.Object || !HasUnambiguousProperties(request)) return false; + try + { + string? planToken; + string? idempotencyKey; + string payloadHash; + if (tool == "netratel_jobs" && IsMutation(operation)) + { + var body = request.Deserialize(ReadOptions); + if (body is null) return false; + planToken = body.PlanToken; + idempotencyKey = body.IdempotencyKey; + payloadHash = MutationHash(operation, body); + } + else if (tool == "netratel_job_runs" && operation is "cancel" or "delete") + { + var body = request.Deserialize(ReadOptions); + if (body?.JobRunId is not > 0) return false; + planToken = body.PlanToken; + idempotencyKey = body.IdempotencyKey; + payloadHash = RunActionHash(operation, body.JobRunId.Value, body.Reason); + } + else return false; + + if (!IsOpaque(planToken) || !IsOpaque(idempotencyKey)) return false; + binding = new(McpOperatorReplayBinding.Hash(planToken!), McpOperatorReplayBinding.Hash(idempotencyKey!), payloadHash); + return true; + } + catch (JsonException) + { + return false; + } + } + + public static McpOperatorReplayBinding RunActionBinding(string action, ulong runId, string? reason, string planToken, string idempotencyKey) => + new(McpOperatorReplayBinding.Hash(planToken), McpOperatorReplayBinding.Hash(idempotencyKey), RunActionHash(action, runId, reason)); + + private static string MutationHash(string action, MutationBody body) => McpOperatorReplayBinding.Hash(JsonSerializer.Serialize(new + { + action, body.JobId, body.ExpectedVersion, body.Job, body.ParameterId, body.Parameter, body.StepId, body.Step, body.Ordinal + })); + + private static string BoundedReason(string? value) => string.IsNullOrWhiteSpace(value) ? "operator_requested" : + value.Trim().Length > 128 || value.Any(char.IsControl) ? "operator_requested" : value.Trim(); + + private static bool IsOpaque(string? value) => value is { Length: >= 32 and <= 128 } && + value.All(character => char.IsAsciiLetterOrDigit(character) || character is '-' or '_'); + + private static bool HasUnambiguousProperties(JsonElement value) + { + if (value.ValueKind == JsonValueKind.Object) + { + var names = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var property in value.EnumerateObject()) + if (!names.Add(property.Name) || !HasUnambiguousProperties(property.Value)) return false; + } + else if (value.ValueKind == JsonValueKind.Array) + { + foreach (var item in value.EnumerateArray()) + if (!HasUnambiguousProperties(item)) return false; + } + return true; + } + + // The extra root fields belong to the validated MCP envelope. They never + // enter the business hash; tenant/agent are separately signed and checked. + private sealed record MutationBody( + JobDraft? Job = null, long? JobId = null, long? ExpectedVersion = null, + long? ParameterId = null, JobParameter? Parameter = null, + long? StepId = null, JobStep? Step = null, int? Ordinal = null, + string? PlanToken = null, string? IdempotencyKey = null, int? TenantId = null, Guid? AgentId = null); + + private sealed record JobDraft(string Name, string FolderPath, string? Description = null, string? OptionsJson = null); + private sealed record JobParameter(string Name, string Type, bool Required, string? Description = null, + string? DefaultValue = null, IReadOnlyList? Options = null, string? SecretReference = null, long? Id = null); + private sealed record JobStep(int? Ordinal, long ScriptId, long ScriptVersion, string ScriptContentHash, + bool Enabled = true, long? Id = null); + private sealed record RunActionBody(string? Reason = null, string? PlanToken = null, string? IdempotencyKey = null, + ulong? JobRunId = null, int? TenantId = null, Guid? AgentId = null); +} diff --git a/src/NetRatel/NetRatel.Tests/API/McpOperatorClientFileEndpointTests.cs b/src/NetRatel/NetRatel.Tests/API/McpOperatorClientFileEndpointTests.cs index 2f2a363e..2f6ceea2 100644 --- a/src/NetRatel/NetRatel.Tests/API/McpOperatorClientFileEndpointTests.cs +++ b/src/NetRatel/NetRatel.Tests/API/McpOperatorClientFileEndpointTests.cs @@ -6,12 +6,15 @@ using System.Security.Claims; using System.Text; using System.Text.Encodings.Web; +using System.Text.Json.Nodes; using System.Threading.Channels; using AwesomeAssertions; using Microsoft.AspNetCore.Authentication; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.TestHost; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Diagnostics; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; @@ -24,6 +27,8 @@ using NetRatel.API.Security.M2M; using NetRatel.Application.Operations; using NetRatel.Application.Presence; +using NetRatel.Infrastructure.Persistence; +using NetRatel.Infrastructure.Services; using NetRatel.Shared.Operations; using Xunit; @@ -135,6 +140,188 @@ public async Task ProductionArtifactLifecycle_IsCallerBoundRootBoundAndByteWiped .Should().Equal("collect", "artifact_status", "download", "artifact_cleanup"); } + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Artifact_cleanup_replays_after_byte_wipe_without_repeating_cleanup_or_consuming_an_independent_plan(bool simulateTimestampPrecisionLoss) + { + var agentId = Guid.NewGuid(); + await using var db = new OrchestratorDbContext(new DbContextOptionsBuilder() + .UseInMemoryDatabase(Guid.NewGuid().ToString("N")).Options); + using var app = await BuildAppAsync(agentId, includeReadRoot: true, confirmations: new McpOperatorConfirmationService(db)); + var client = AuthorizedClient(app); + var store = app.Services.GetRequiredService(); + var artifact = ArtifactFixture(agentId); + store.Artifacts.Add(artifact); + if (simulateTimestampPrecisionLoss) + { + var nowTicks = DateTimeOffset.UtcNow.UtcTicks; + store.CleanupTimeOverride = new DateTimeOffset(nowTicks - nowTicks % TimeSpan.TicksPerMicrosecond + 7, TimeSpan.Zero); + } + var root = $"/api/v2/mcp/operator/agents/7/{agentId:D}/files/artifacts/{artifact.ArtifactId:D}"; + var previewResponse = await client.PostAsync($"{root}/cleanup/preview", null); + previewResponse.StatusCode.Should().Be(HttpStatusCode.OK); + var preview = (await previewResponse.Content.ReadFromJsonAsync())!; + var independentPreviewResponse = await client.PostAsync($"{root}/cleanup/preview", null); + independentPreviewResponse.StatusCode.Should().Be(HttpStatusCode.OK); + var independentPreview = (await independentPreviewResponse.Content.ReadFromJsonAsync())!; + + var cleaned = await client.PostAsJsonAsync($"{root}/cleanup/confirm", preview); + cleaned.StatusCode.Should().Be(HttpStatusCode.OK); + var original = (await cleaned.Content.ReadFromJsonAsync())!; + if (simulateTimestampPrecisionLoss) + { + var stored = store.Artifacts.Single(); + var timestamp = stored.DeletedAtUtc!.Value; + (timestamp.UtcTicks % TimeSpan.TicksPerMicrosecond).Should().Be(7); + var persistedTimestamp = new DateTimeOffset(timestamp.UtcTicks - timestamp.UtcTicks % TimeSpan.TicksPerMicrosecond, TimeSpan.Zero); + store.Artifacts[0] = stored with { DeletedAtUtc = persistedTimestamp }; + original["deletedAtUtc"]!.GetValue().Should().Be(persistedTimestamp); + } + var replayResponse = await client.PostAsJsonAsync($"{root}/cleanup/confirm", preview); + replayResponse.StatusCode.Should().Be(HttpStatusCode.OK); + var replay = (await replayResponse.Content.ReadFromJsonAsync())!; + replay["replayed"]!.GetValue().Should().BeTrue(); + replay["replayed"] = false; + JsonNode.DeepEquals(original, replay).Should().BeTrue(); + var stale = await client.PostAsJsonAsync($"{root}/cleanup/confirm", independentPreview); + stale.StatusCode.Should().Be(HttpStatusCode.Conflict); + (await stale.Content.ReadAsStringAsync()).Should().Contain("artifact_already_cleaned"); + var unused = await db.McpOperatorConfirmationPlans.SingleAsync(plan => plan.IdempotencyKey == independentPreview.IdempotencyKey); + unused.ConsumedAtUtc.Should().BeNull(); + unused.ConsumedIdempotencyId.Should().BeNull(); + (await db.McpOperatorIdempotencyRecords.CountAsync()).Should().Be(1); + store.Cleanups.Should().Be(1); + store.Artifacts.Should().ContainSingle().Which.Content.Should().BeNull(); + app.Services.GetRequiredService().Accepted.Should().ContainSingle(request => request.Operation == "artifact_cleanup"); + (await client.GetAsync(root)).StatusCode.Should().Be(HttpStatusCode.Gone); + (await client.GetAsync($"{root}/download")).StatusCode.Should().Be(HttpStatusCode.Gone); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task Artifact_cleanup_retains_its_receipt_when_response_delivery_is_cancelled_or_current_authority_is_lost(bool disconnect) + { + var agentId = Guid.NewGuid(); + var writes = new ArtifactReceiptPersistence(); + await using var db = new OrchestratorDbContext(new DbContextOptionsBuilder() + .UseInMemoryDatabase(Guid.NewGuid().ToString("N")).AddInterceptors(writes).Options); + using var app = await BuildAppAsync(agentId, includeReadRoot: true, confirmations: new McpOperatorConfirmationService(db)); + var client = AuthorizedClient(app); + var store = app.Services.GetRequiredService(); + var admission = app.Services.GetRequiredService(); + var artifact = ArtifactFixture(agentId); + store.Artifacts.Add(artifact); + var root = $"/api/v2/mcp/operator/agents/7/{agentId:D}/files/artifacts/{artifact.ArtifactId:D}"; + var previewResponse = await client.PostAsync($"{root}/cleanup/preview", null); + previewResponse.StatusCode.Should().Be(HttpStatusCode.OK); + var preview = (await previewResponse.Content.ReadFromJsonAsync())!; + using var disconnected = new CancellationTokenSource(); + store.AfterCleanup = () => + { + if (disconnect) disconnected.Cancel(); + else admission.FailureCodeOverride = "target_policy_missing"; + }; + + var confirmation = client.PostAsJsonAsync($"{root}/cleanup/confirm", preview, disconnected.Token); + + if (disconnect) + { + var lostResponse = async () => await confirmation; + await lostResponse.Should().ThrowAsync(); + } + else + { + (await confirmation).StatusCode.Should().Be(HttpStatusCode.Forbidden); + (await client.PostAsJsonAsync($"{root}/cleanup/confirm", preview)).StatusCode.Should().Be(HttpStatusCode.Forbidden); + } + await writes.ReceiptSaved.Task.WaitAsync(TimeSpan.FromSeconds(10)); + (await db.McpOperatorIdempotencyRecords.AsNoTracking().SingleAsync()).Outcome.Should().Be(McpOperatorIdempotencyOutcome.Succeeded); + var cleaned = store.Artifacts.Should().ContainSingle().Which; + cleaned.Content.Should().BeNull(); + cleaned.DeletedAtUtc.Should().NotBeNull(); + admission.FailureCodeOverride = null; + var replayResponse = await client.PostAsJsonAsync($"{root}/cleanup/confirm", preview); + replayResponse.StatusCode.Should().Be(HttpStatusCode.OK); + var replay = (await replayResponse.Content.ReadFromJsonAsync())!; + replay["replayed"]!.GetValue().Should().BeTrue(); + replay["artifactId"]!.GetValue().Should().Be(artifact.ArtifactId); + store.Cleanups.Should().Be(1); + admission.Accepted.Should().ContainSingle(request => request.Operation == "artifact_cleanup"); + (await db.McpOperatorIdempotencyRecords.CountAsync()).Should().Be(1); + } + + [Theory] + [InlineData("payload", HttpStatusCode.Conflict)] + [InlineData("subject", HttpStatusCode.NotFound)] + [InlineData("client", HttpStatusCode.NotFound)] + [InlineData("resource", HttpStatusCode.NotFound)] + [InlineData("target", HttpStatusCode.Forbidden)] + [InlineData("policy_revoked", HttpStatusCode.Forbidden)] + [InlineData("policy_version", HttpStatusCode.Conflict)] + [InlineData("expired", HttpStatusCode.Conflict)] + public async Task Artifact_cleanup_replay_retains_current_ownership_policy_and_expiry_checks(string change, HttpStatusCode expectedStatus) + { + var agentId = Guid.NewGuid(); + var changed = false; + await using var db = new OrchestratorDbContext(new DbContextOptionsBuilder() + .UseInMemoryDatabase(Guid.NewGuid().ToString("N")).Options); + using var app = await BuildAppAsync(agentId, includeReadRoot: true, confirmations: new McpOperatorConfirmationService(db), + configureDelegation: delegation => !changed ? delegation : change switch + { + "subject" => delegation with { Identity = delegation.Identity with { Subject = "other-operator" } }, + "client" => delegation with { Identity = delegation.Identity with { ClientId = "other-client" } }, + "resource" => delegation with { Resource = "https://other-mcp.example" }, + "target" => delegation with { AgentId = Guid.NewGuid() }, + _ => delegation + }); + var client = AuthorizedClient(app); + var store = app.Services.GetRequiredService(); + var artifact = ArtifactFixture(agentId); + store.Artifacts.Add(artifact); + var root = $"/api/v2/mcp/operator/agents/7/{agentId:D}/files/artifacts"; + var previewResponse = await client.PostAsync($"{root}/{artifact.ArtifactId:D}/cleanup/preview", null); + previewResponse.StatusCode.Should().Be(HttpStatusCode.OK); + var preview = (await previewResponse.Content.ReadFromJsonAsync())!; + var cleaned = await client.PostAsJsonAsync($"{root}/{artifact.ArtifactId:D}/cleanup/confirm", preview); + cleaned.StatusCode.Should().Be(HttpStatusCode.OK); + var replayArtifactId = artifact.ArtifactId; + if (change == "payload") + { + var otherArtifact = ArtifactFixture(agentId); + store.Artifacts.Add(otherArtifact); + replayArtifactId = otherArtifact.ArtifactId; + } + if (change == "policy_revoked") app.Services.GetRequiredService().FailureCodeOverride = "target_policy_missing"; + if (change == "policy_version") app.Services.GetRequiredService().PolicyVersion++; + if (change == "expired") + { + (await db.McpOperatorConfirmationPlans.SingleAsync()).ExpiresAtUtc = DateTimeOffset.UtcNow.AddSeconds(-1); + await db.SaveChangesAsync(); + } + changed = true; + + var denied = await client.PostAsJsonAsync($"{root}/{replayArtifactId:D}/cleanup/confirm", preview); + + denied.StatusCode.Should().Be(expectedStatus); + if (change == "expired") (await denied.Content.ReadAsStringAsync()).Should().Contain("confirmation_plan_expired"); + store.Cleanups.Should().Be(1); + (await db.McpOperatorIdempotencyRecords.CountAsync()).Should().Be(1); + app.Services.GetRequiredService().Accepted.Should().ContainSingle(request => request.Operation == "artifact_cleanup"); + } + + private static McpOperatorFileArtifact ArtifactFixture(Guid agentId) + { + var content = Encoding.UTF8.GetBytes("synthetic cleanup fixture"); + return new McpOperatorFileArtifact(Guid.NewGuid(), 7, agentId, "operator-1", "operator-client", + "https://mcp.example", "prod", new string('a', 64), "fixture.txt", content.Length, + Convert.ToHexString(SHA256.HashData(content)).ToLowerInvariant(), "text/plain", content, + Guid.NewGuid(), Guid.NewGuid(), DateTimeOffset.UtcNow, DateTimeOffset.UtcNow.AddHours(1), null); + } + + private sealed record ArtifactCleanupPlan(string PlanToken, string IdempotencyKey); + [Fact] public async Task ProductionStat_BindsTheExactDelegationAndReturnsMetadataWithoutFileContent() { @@ -585,7 +772,9 @@ private static HttpClient AuthorizedClient(IHost app) return client; } - private static async Task BuildAppAsync(Guid agentId, bool includeReadRoot, bool includeWriteRoot = true) + private static async Task BuildAppAsync(Guid agentId, bool includeReadRoot, bool includeWriteRoot = true, + IMcpOperatorConfirmationService? confirmations = null, + Func? configureDelegation = null) { var client = new ClientKey(7, agentId); var builder = Host.CreateDefaultBuilder(); @@ -610,7 +799,7 @@ private static async Task BuildAppAsync(Guid agentId, bool includeReadRoo services.AddSingleton(new TestAdmission(includeReadRoot, includeWriteRoot)); services.AddSingleton(provider => provider.GetRequiredService()); services.AddSingleton(); - services.AddSingleton(provider => provider.GetRequiredService()); + services.AddSingleton(provider => confirmations ?? provider.GetRequiredService()); services.AddSingleton(); services.AddSingleton(provider => provider.GetRequiredService()); services.AddSingleton(); @@ -646,7 +835,7 @@ var path when path.EndsWith("/files/copy/confirm", StringComparison.Ordinal) => var path when path.EndsWith("/files/move/preview", StringComparison.Ordinal) => "preview_move", _ => "confirm_move" }; - http.Items[McpOperatorDelegationMiddleware.HttpContextItemKey] = new McpOperatorDelegation( + var delegation = new McpOperatorDelegation( new McpOperatorDelegationIdentity("operator-1", "operator-client", null, [], ["Operator"], ["netratel.mcp.files", "netratel.mcp.write"]), "netratel-mcp-http", "netratel_files", @@ -658,6 +847,7 @@ var path when path.EndsWith("/files/move/preview", StringComparison.Ordinal) => client.TenantId, client.AgentId, "correlation-1"); + http.Items[McpOperatorDelegationMiddleware.HttpContextItemKey] = configureDelegation?.Invoke(delegation) ?? delegation; await next(); }); app.UseAuthorization(); @@ -705,9 +895,12 @@ private sealed class TestAdmission(bool includeReadRoot, bool includeWriteRoot) public ConcurrentQueue Accepted { get; } = []; public IReadOnlyList? WriteRootsOverride { get; set; } public IReadOnlyList? ReadRootsOverride { get; set; } + public string? FailureCodeOverride { get; set; } + public long PolicyVersion { get; set; } = 1; public Task EvaluateAsync(McpOperatorRouteAccessRequest request, CancellationToken cancellationToken) { + cancellationToken.ThrowIfCancellationRequested(); var access = new McpOperatorAccessRequest( request.Environment, request.Principal, @@ -720,6 +913,7 @@ public Task EvaluateAsync(McpOperatorRouteAccessReque request.Operation is "write_text" or "upload" or "move" or "artifact_cleanup" ? McpOperatorConfirmationClass.Destructive : request.Operation is "copy" or "collect" ? McpOperatorConfirmationClass.StandardMutation : McpOperatorConfirmationClass.None, request.CorrelationId, request.RequestId, + TargetSetDigest: new string('a', 64), McpResource: request.McpResource, McpInstance: request.McpInstance, Tool: request.Tool, @@ -733,9 +927,11 @@ public Task EvaluateAsync(McpOperatorRouteAccessReque new McpOperatorConstraints( ReadRoots: ReadRootsOverride ?? (includeReadRoot ? ["/var/log"] : null), WriteRoots: WriteRootsOverride ?? (includeWriteRoot ? ["/var/lib/netratel"] : null)), - "target-digest", + access.TargetSetDigest, access, - 1); + PolicyVersion); + if (FailureCodeOverride is { } failure) + decision = McpOperatorDecision.Denied(access, failure, McpOperatorAuthorizationLayer.Policy); return Task.FromResult(new McpOperatorRouteAdmission(decision, McpOperatorOperationCatalog.Find(request.Tool, request.Operation))); } @@ -886,7 +1082,9 @@ public Task ConfirmAsync(McpOperatorConfirmati return Task.FromResult(McpOperatorConfirmationAdmission.Denied("confirmation_plan_stale")); return Task.FromResult(completed.TryGetValue(request.PayloadHash, out var prior) ? new McpOperatorConfirmationAdmission(false, true, null, idempotencyId, prior.Outcome, prior.ResultReference) - : new McpOperatorConfirmationAdmission(true, false, null, idempotencyId, McpOperatorIdempotencyOutcome.Pending, null)); + : request.ReplayOnly + ? new McpOperatorConfirmationAdmission(false, false, null, null, null, null) + : new McpOperatorConfirmationAdmission(true, false, null, idempotencyId, McpOperatorIdempotencyOutcome.Pending, null)); } public Task CompleteAsync(Guid idempotencyId, McpOperatorIdempotencyOutcome outcome, string? resultReference, CancellationToken cancellationToken) @@ -896,9 +1094,26 @@ public Task CompleteAsync(Guid idempotencyId, McpOperatorIdempotencyOutcome outc } } + private sealed class ArtifactReceiptPersistence : SaveChangesInterceptor + { + public TaskCompletionSource ReceiptSaved { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public override ValueTask SavedChangesAsync(SaveChangesCompletedEventData eventData, int result, + CancellationToken cancellationToken = default) + { + if (eventData.Context is { } db && db.ChangeTracker.Entries() + .Any(entry => entry.Entity.Outcome == McpOperatorIdempotencyOutcome.Succeeded)) + ReceiptSaved.TrySetResult(); + return ValueTask.FromResult(result); + } + } + private sealed class TestArtifactStore : IMcpOperatorFileArtifactStore { public List Artifacts { get; } = []; + public int Cleanups { get; private set; } + public Action? AfterCleanup { get; set; } + public DateTimeOffset? CleanupTimeOverride { get; set; } public Task CreateOrGetAsync(McpOperatorFileArtifactCreateRequest request, CancellationToken cancellationToken) { @@ -947,8 +1162,10 @@ public Task CreateOrGetAsync(McpOperatorFileArtifactCre if (index < 0) return Task.FromResult(null); - var cleaned = Artifacts[index] with { Content = null, DeletedAtUtc = Artifacts[index].DeletedAtUtc ?? now }; + Cleanups++; + var cleaned = Artifacts[index] with { Content = null, DeletedAtUtc = Artifacts[index].DeletedAtUtc ?? CleanupTimeOverride ?? now }; Artifacts[index] = cleaned; + AfterCleanup?.Invoke(); return Task.FromResult(cleaned); } diff --git a/src/NetRatel/NetRatel.Tests/API/McpOperatorJobEndpointTests.cs b/src/NetRatel/NetRatel.Tests/API/McpOperatorJobEndpointTests.cs index 9563698e..6e904f78 100644 --- a/src/NetRatel/NetRatel.Tests/API/McpOperatorJobEndpointTests.cs +++ b/src/NetRatel/NetRatel.Tests/API/McpOperatorJobEndpointTests.cs @@ -18,14 +18,18 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using NetRatel.API.Endpoints.Client; +using NetRatel.API.Endpoints.Auth; using NetRatel.API.Gateway; using NetRatel.API.Middleware; +using NetRatel.API.Security.Integration; using NetRatel.API.Services.Jobs; using NetRatel.Akka.Configuration; using NetRatel.Application.Jobs; using NetRatel.Application.Operations; using NetRatel.Application.Presence; using NetRatel.Infrastructure.Persistence; +using NetRatel.Infrastructure.Identity; +using NetRatel.Infrastructure.Identity.Authorization; using NetRatel.Infrastructure.Services; using NetRatel.Shared.Operations; using Xunit; @@ -46,22 +50,33 @@ public sealed class McpOperatorJobEndpointTests private static string MutationRoot => $"/api/v2/mcp/operator/agents/42/{MutationAgentId:D}/jobs"; [Theory] - [InlineData("create")] - [InlineData("update")] - [InlineData("delete")] - [InlineData("param_add")] - [InlineData("param_update")] - [InlineData("param_delete")] - [InlineData("step_add")] - [InlineData("step_update")] - [InlineData("step_reorder")] - [InlineData("step_delete")] - public async Task Completed_definition_mutations_replay_the_original_result_without_persistence(string action) + [InlineData("create", false)] + [InlineData("update", false)] + [InlineData("delete", false)] + [InlineData("param_add", false)] + [InlineData("param_update", false)] + [InlineData("param_delete", false)] + [InlineData("step_add", false)] + [InlineData("step_update", false)] + [InlineData("step_reorder", false)] + [InlineData("step_delete", false)] + [InlineData("create", true)] + [InlineData("update", true)] + [InlineData("delete", true)] + [InlineData("param_add", true)] + [InlineData("param_update", true)] + [InlineData("param_delete", true)] + [InlineData("step_add", true)] + [InlineData("step_update", true)] + [InlineData("step_reorder", true)] + [InlineData("step_delete", true)] + public async Task Completed_definition_mutations_replay_the_original_result_without_persistence(string action, bool localDelegation) { - await using var app = await BuildMutationAppAsync(); + await using var app = await BuildMutationAppAsync(localDelegation); using var client = app.GetTestClient(); - var request = await PreviewMutationAsync(client, action, MutationRequest(action)); - var confirmed = await ConfirmMutationAsync(client, action, request); + var boundaryApp = localDelegation ? app : null; + var request = await PreviewMutationAsync(client, action, MutationRequest(action), boundaryApp); + var confirmed = await ConfirmMutationAsync(client, action, request, boundaryApp); confirmed.Replayed.Should().BeFalse(); confirmed.Deleted.Should().Be(action == "delete"); confirmed.Version.Should().Be(action == "create" ? "1" : "2"); @@ -69,7 +84,7 @@ public async Task Completed_definition_mutations_replay_the_original_result_with var writes = app.Services.GetRequiredService(); var afterConfirmation = writes.Count; - var replayed = await ConfirmMutationAsync(client, action, request); + var replayed = await ConfirmMutationAsync(client, action, request, boundaryApp); replayed.Should().Be(confirmed with { Replayed = true }); writes.Count.Should().Be(afterConfirmation); @@ -94,12 +109,147 @@ public async Task Completed_definition_mutations_replay_the_original_result_with } var afterLaterChange = writes.Count; - var replayedAfterChange = await ConfirmMutationAsync(client, action, request); + var replayedAfterChange = await ConfirmMutationAsync(client, action, request, boundaryApp); replayedAfterChange.Should().Be(confirmed with { Replayed = true }); writes.Count.Should().Be(afterLaterChange); } + [Fact] + public async Task A_local_deleted_job_receipt_cannot_authorize_a_read_preview_changed_body_or_another_fresh_plan() + { + await using var app = await BuildMutationAppAsync(localDelegation: true); + using var client = app.GetTestClient(); + var original = await PreviewMutationAsync(client, "delete", MutationRequest("delete"), app); + await ConfirmMutationAsync(client, "delete", original, app); + var create = await PreviewMutationAsync(client, "create", MutationRequest("create"), app); + var other = await ConfirmMutationAsync(client, "create", create, app); + var fresh = await PreviewMutationAsync(client, "delete", new McpOperatorJobMutationRequest(JobId: long.Parse(other.JobId), ExpectedVersion: 1), app); + var assertion = await MutationAssertionAsync(app, client, "delete", original); + var writes = app.Services.GetRequiredService(); + var beforeAttempt = writes.Count; + + foreach (var operation in new[] { "get", "delete" }) + { + using var ordinary = await McpJobLocalDelegationTestBoundary.ExchangeAsync(app, client, "netratel_jobs", operation, original.JobId!.ToString(), MutationAgentId); + ordinary.StatusCode.Should().Be(HttpStatusCode.Forbidden); + (await ordinary.Content.ReadAsStringAsync()).Should().Contain("local_delegation_binding_mismatch"); + } + using var preview = await McpJobLocalDelegationTestBoundary.PostAsync(client, MutationRoot + "/preview/delete", original, assertion); + preview.StatusCode.Should().Be(HttpStatusCode.Unauthorized); + using var changed = await McpJobLocalDelegationTestBoundary.PostAsync(client, MutationRoot + "/confirm/delete", original with { ExpectedVersion = 2 }, assertion); + changed.StatusCode.Should().Be(HttpStatusCode.Unauthorized); + using var anotherPlan = await McpJobLocalDelegationTestBoundary.PostAsync(client, MutationRoot + "/confirm/delete", fresh, assertion); + anotherPlan.StatusCode.Should().Be(HttpStatusCode.Unauthorized); + (await anotherPlan.Content.ReadAsStringAsync()).Should().Contain("delegated_identity_invalid"); + McpOperatorJobConfirmationPayload.TryCreateBinding("netratel_jobs", "delete", JsonSerializer.SerializeToElement(original with { ExpectedVersion = 2 }), out var changedBinding).Should().BeTrue(); + using var changedExchange = await McpJobLocalDelegationTestBoundary.ExchangeAsync(app, client, "netratel_jobs", "delete", original.JobId!.ToString(), MutationAgentId, changedBinding); + changedExchange.StatusCode.Should().Be(HttpStatusCode.Forbidden); + writes.Count.Should().Be(beforeAttempt); + await using var verification = app.Services.CreateAsyncScope(); + var db = verification.ServiceProvider.GetRequiredService(); + var freshHash = McpOperatorReplayBinding.Hash(fresh.PlanToken!); + (await db.McpOperatorConfirmationPlans.SingleAsync(row => row.TokenHash == freshHash)).ConsumedAtUtc.Should().BeNull(); + (await db.McpOperatorJobs.SingleAsync(row => row.JobId == long.Parse(other.JobId))).DeletedAtUtc.Should().BeNull(); + } + + [Theory] + [InlineData("permission", "local_credential_permission_denied", HttpStatusCode.Forbidden)] + [InlineData("credential-expired", "local_credential_expired", HttpStatusCode.Unauthorized)] + [InlineData("credential-revoked", "local_credential_revoked", HttpStatusCode.Unauthorized)] + [InlineData("policy-disabled", "local_delegation_binding_mismatch", HttpStatusCode.Forbidden)] + [InlineData("policy-deny", "target_policy_denied", HttpStatusCode.Forbidden)] + [InlineData("plan-expired", "local_delegation_binding_mismatch", HttpStatusCode.Forbidden)] + [InlineData("target-disabled", "target_disabled", HttpStatusCode.Forbidden)] + public async Task An_exchanged_deleted_job_replay_rechecks_current_credential_receipt_and_route_policy(string change, string code, HttpStatusCode status) + { + await using var app = await BuildMutationAppAsync(localDelegation: true); + using var client = app.GetTestClient(); + var request = await PreviewMutationAsync(client, "delete", MutationRequest("delete"), app); + await ConfirmMutationAsync(client, "delete", request, app); + var assertion = await MutationAssertionAsync(app, client, "delete", request); + var boundary = app.Services.GetRequiredService(); + if (change == "permission") boundary.HasPermission = false; + else if (change == "credential-expired") boundary.CurrentFailureCode = "local_credential_expired"; + else if (change == "credential-revoked") boundary.CurrentFailureCode = "local_credential_revoked"; + else + { + await using var scope = app.Services.CreateAsyncScope(); + var db = scope.ServiceProvider.GetRequiredService(); + if (change == "policy-disabled") (await db.McpOperatorPolicies.SingleAsync()).DisabledAtUtc = DateTimeOffset.UtcNow; + else if (change == "plan-expired") (await db.McpOperatorConfirmationPlans.SingleAsync()).ExpiresAtUtc = DateTimeOffset.UtcNow.AddSeconds(-1); + else if (change == "policy-deny") db.McpOperatorPolicies.Add(new McpOperatorPolicyRecord + { + Id = Guid.NewGuid(), Name = "new current denial", Environment = McpOperatorEnvironment.Production, + Effect = McpOperatorPolicyEffect.Deny, Priority = 20, PrincipalSelectorKind = McpOperatorPrincipalSelectorKind.OAuthSubject, + PrincipalSelectorValue = MutationSubject, TargetSelectorKind = McpOperatorTargetSelectorKind.ExactAgent, + TenantId = 42, AgentId = MutationAgentId, OperationFamily = McpOperatorOperationFamily.AutomationWrite, + CreatedAtUtc = DateTimeOffset.UtcNow, CreatedBy = "independent-admin", Version = 1 + }); + else (await db.Agents.SingleAsync()).IsEnabled = false; + await db.SaveChangesAsync(); + } + var writes = app.Services.GetRequiredService(); + var beforeAttempt = writes.Count; + + using var response = await McpJobLocalDelegationTestBoundary.PostAsync(client, MutationRoot + "/confirm/delete", request, assertion); + + response.StatusCode.Should().Be(status, await response.Content.ReadAsStringAsync()); + (await response.Content.ReadAsStringAsync()).Should().Contain(code); + writes.Count.Should().Be(beforeAttempt); + } + + [Fact] + public async Task A_signed_confirmation_proof_requires_the_actual_ingress_credential_as_its_client_even_for_a_live_job() + { + await using var app = await BuildMutationAppAsync(localDelegation: true); + using var client = app.GetTestClient(); + var request = await PreviewMutationAsync(client, "update", MutationRequest("update"), app); + var assertion = await MutationAssertionAsync(app, client, "update", request); + var tokens = app.Services.GetRequiredService(); + tokens.TryValidate(assertion, out var verified).Should().BeTrue(); + var wrongClient = tokens.Create(verified!.Identity with { ClientId = "another-signed-application-client" }, + new McpOperatorDelegationRequest(verified.Tool, verified.Operation, Guid.NewGuid().ToString("N"), verified.Resource, + verified.Instance, verified.TenantId, verified.AgentId, verified.CorrelationId) + { + ObjectReference = verified.ObjectReference, ObjectTargetResolutionEnabled = true, + ReplayBinding = verified.ReplayBinding, IngressCredentialId = verified.IngressCredentialId, + IngressPermission = verified.IngressPermission + }); + var writes = app.Services.GetRequiredService(); + var beforeAttempt = writes.Count; + + using var denied = await McpJobLocalDelegationTestBoundary.PostAsync(client, MutationRoot + "/confirm/update", request, wrongClient); + + denied.StatusCode.Should().Be(HttpStatusCode.Forbidden); + (await denied.Content.ReadAsStringAsync()).Should().Contain("local_delegation_binding_mismatch"); + writes.Count.Should().Be(beforeAttempt); + await using var verification = app.Services.CreateAsyncScope(); + var db = verification.ServiceProvider.GetRequiredService(); + (await db.McpOperatorConfirmationPlans.SingleAsync()).ConsumedAtUtc.Should().BeNull(); + (await db.McpOperatorIdempotencyRecords.CountAsync()).Should().Be(0); + } + + [Theory] + [InlineData("get")] + [InlineData("delete")] + public async Task Ordinary_local_object_requests_keep_ownership_failure_distinct_from_current_permission_denial(string operation) + { + await using var app = await BuildMutationAppAsync(localDelegation: true); + using var client = app.GetTestClient(); + app.Services.GetRequiredService().HasPermission = false; + var writes = app.Services.GetRequiredService(); + var beforeAttempt = writes.Count; + + using var unknown = await McpJobLocalDelegationTestBoundary.ExchangeAsync(app, client, "netratel_jobs", operation, "999999", MutationAgentId); + unknown.StatusCode.Should().Be(HttpStatusCode.Forbidden); + (await unknown.Content.ReadAsStringAsync()).Should().Contain("local_delegation_binding_mismatch"); + using var owned = await McpJobLocalDelegationTestBoundary.ExchangeAsync(app, client, "netratel_jobs", operation, MutationJobId.ToString(), MutationAgentId); + owned.StatusCode.Should().Be(HttpStatusCode.Forbidden); + (await owned.Content.ReadAsStringAsync()).Should().Contain("local_credential_permission_denied"); + writes.Count.Should().Be(beforeAttempt); + } + [Fact] public async Task Changed_confirmation_does_not_consume_the_reviewed_parameter_add() { @@ -292,22 +442,32 @@ public void Output_respects_remaining_line_budget_and_message_bound() _ => throw new ArgumentOutOfRangeException(nameof(action)) }; - private static async Task PreviewMutationAsync(HttpClient client, string action, McpOperatorJobMutationRequest request) + private static async Task PreviewMutationAsync(HttpClient client, string action, McpOperatorJobMutationRequest request, WebApplication? app = null) { - using var response = await client.PostAsJsonAsync(MutationRoot + "/preview/" + action, request); + using var response = app is null ? await client.PostAsJsonAsync(MutationRoot + "/preview/" + action, request) : + await McpJobLocalDelegationTestBoundary.PostAsync(client, MutationRoot + "/preview/" + action, request, + await McpJobLocalDelegationTestBoundary.AssertionAsync(app, client, "netratel_jobs", action, request.JobId?.ToString(), MutationAgentId)); response.StatusCode.Should().Be(HttpStatusCode.OK, await response.Content.ReadAsStringAsync()); var preview = (await response.Content.ReadFromJsonAsync())!; return request with { PlanToken = preview.PlanToken, IdempotencyKey = preview.IdempotencyKey }; } - private static async Task ConfirmMutationAsync(HttpClient client, string action, McpOperatorJobMutationRequest request) + private static async Task ConfirmMutationAsync(HttpClient client, string action, McpOperatorJobMutationRequest request, WebApplication? app = null) { - using var response = await client.PostAsJsonAsync(MutationRoot + "/confirm/" + action, request); + using var response = app is null ? await client.PostAsJsonAsync(MutationRoot + "/confirm/" + action, request) : + await McpJobLocalDelegationTestBoundary.PostAsync(client, MutationRoot + "/confirm/" + action, request, + await MutationAssertionAsync(app, client, action, request)); response.StatusCode.Should().Be(HttpStatusCode.OK, await response.Content.ReadAsStringAsync()); return (await response.Content.ReadFromJsonAsync())!; } - private static async Task BuildMutationAppAsync() + private static Task MutationAssertionAsync(WebApplication app, HttpClient client, string action, McpOperatorJobMutationRequest request) + { + McpOperatorJobConfirmationPayload.TryCreateBinding("netratel_jobs", action, JsonSerializer.SerializeToElement(request), out var binding).Should().BeTrue(); + return McpJobLocalDelegationTestBoundary.AssertionAsync(app, client, "netratel_jobs", action, request.JobId?.ToString(), MutationAgentId, binding); + } + + private static async Task BuildMutationAppAsync(bool localDelegation = false) { var builder = WebApplication.CreateBuilder(new WebApplicationOptions { EnvironmentName = "Production" }); builder.WebHost.UseTestServer(); @@ -327,21 +487,27 @@ private static async Task BuildMutationAppAsync() builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); + if (localDelegation) McpJobLocalDelegationTestBoundary.Configure(builder, MutationSubject, MutationClientId, MutationResource); var app = builder.Build(); app.UseAuthentication(); - // Exercise real route policy, ownership and durable confirmation state - // after the signed delegation middleware's separately tested boundary. - app.Use(async (context, next) => + if (localDelegation) app.UseMiddleware(); + else { - var caller = context.RequestServices.GetRequiredService(); - var action = context.Request.Path.Value!.Split('/')[^1]; - context.Items[McpOperatorDelegationMiddleware.HttpContextItemKey] = new McpOperatorDelegation( - new McpOperatorDelegationIdentity(MutationSubject, caller.ClientId, null, [], ["AutomationOperator"], caller.Scopes), - "mcp-service", "netratel_jobs", action, "definition-request", DateTimeOffset.UtcNow.AddMinutes(1), - caller.Resource, "prod", 42, MutationAgentId, "definition-correlation"); - await next(context); - }); + // Keep the original direct route/store fixture for focused policy + // tests; local cases use the full signed exchange and middleware. + app.Use(async (context, next) => + { + var caller = context.RequestServices.GetRequiredService(); + var action = context.Request.Path.Value!.Split('/')[^1]; + context.Items[McpOperatorDelegationMiddleware.HttpContextItemKey] = new McpOperatorDelegation( + new McpOperatorDelegationIdentity(MutationSubject, caller.ClientId, null, [], ["AutomationOperator"], caller.Scopes), + "mcp-service", "netratel_jobs", action, "definition-request", DateTimeOffset.UtcNow.AddMinutes(1), + caller.Resource, "prod", 42, MutationAgentId, "definition-correlation"); + await next(context); + }); + } app.UseAuthorization(); + if (localDelegation) app.MapMcpLocalDelegationEndpoints(); app.MapMcpOperatorJobEndpoints(); await app.StartAsync(); await using var scope = app.Services.CreateAsyncScope(); @@ -424,7 +590,8 @@ private sealed class MutationAuthentication(IOptionsMonitor(options, logger, encoder) { protected override Task HandleAuthenticateAsync() => Task.FromResult(AuthenticateResult.Success( - new AuthenticationTicket(new ClaimsPrincipal(new ClaimsIdentity([new Claim("sub", "mcp-service")], Scheme.Name)), Scheme.Name))); + new AuthenticationTicket(new ClaimsPrincipal(Context.RequestServices.GetService()?.Identity(Scheme.Name) ?? + new ClaimsIdentity([new Claim("sub", "mcp-service")], Scheme.Name)), Scheme.Name))); } private sealed class MutationPresence : IClientPresenceRouter @@ -488,3 +655,90 @@ public void Production_job_routes_are_versioned_owned_and_m2m_only() .Any(metadata => metadata.Policy == "M2MOnly")); } } + +// The existing definition/run fixtures retain real policy, confirmation and +// owner stores. Only account lookup/grants are controlled here so each test +// can withdraw authority between the real exchange and signed execution. +internal sealed class McpJobLocalDelegationTestBoundary(string subject, string credentialId, string resource) + : IIntegrationCredentialCurrentVerifier, IEffectiveAccessService +{ + public string Subject { get; set; } = subject; + public string CredentialId { get; set; } = credentialId; + public string Resource { get; } = resource; + public bool HasPermission { get; set; } = true; + public string? CurrentFailureCode { get; set; } + + public static void Configure(WebApplicationBuilder builder, string subject, string credentialId, string resource) + { + var boundary = new McpJobLocalDelegationTestBoundary(subject, credentialId, resource); + builder.Services.AddSingleton(boundary); + builder.Services.AddSingleton(boundary); + builder.Services.AddSingleton(boundary); + builder.Services.AddScoped(); + builder.Services.AddSingleton(new McpOperatorDelegationOptions + { + Enabled = true, Issuer = "job-replay-tests", Audience = "job-replay-api", ServicePrincipal = "mcp-service", + KeyId = "test-key", SharedKeyBase64 = Convert.ToBase64String(RandomNumberGenerator.GetBytes(32)) + }); + builder.Services.AddSingleton(); + builder.Services.AddAuthorization(options => options.AddPolicy("McpLocalDelegationExchange", policy => policy.RequireAuthenticatedUser())); + } + + public ClaimsIdentity Identity(string scheme) => new( + [ + new Claim("sub", "mcp-service"), new Claim("netratel_principal_id", Subject), + new Claim(IntegrationCredentialAuthenticationHandler.CredentialIdClaimType, CredentialId), + new Claim("integration_credential_resource", Resource), new Claim("integration_credential_purpose", "http_mcp") + ], scheme); + + public static async Task ExchangeAsync(WebApplication app, HttpClient client, string tool, string operation, + string? objectReference, Guid agentId, McpOperatorReplayBinding? binding = null) + { + var boundary = app.Services.GetRequiredService(); + var tokens = app.Services.GetRequiredService(); + var pairing = tokens.Create(new McpOperatorDelegationIdentity("gateway-is-not-the-owner", "netratel-local-gateway", null, [], [], []), + new McpOperatorDelegationRequest(tool, operation, Guid.NewGuid().ToString("N"), boundary.Resource, "prod", 42, agentId, "local-replay-correlation") + { ObjectReference = objectReference, ObjectTargetResolutionEnabled = true, ReplayBinding = binding }); + using var message = new HttpRequestMessage(HttpMethod.Post, McpLocalDelegationEndpoints.ExchangePath); + message.Headers.Add(McpLocalDelegationEndpoints.PairingHeaderName, pairing); + return await client.SendAsync(message); + } + + public static async Task AssertionAsync(WebApplication app, HttpClient client, string tool, string operation, + string? objectReference, Guid agentId, McpOperatorReplayBinding? binding = null) + { + using var response = await ExchangeAsync(app, client, tool, operation, objectReference, agentId, binding); + response.StatusCode.Should().Be(HttpStatusCode.OK, await response.Content.ReadAsStringAsync()); + var assertion = (await response.Content.ReadFromJsonAsync())!.Assertion; + var tokens = app.Services.GetRequiredService(); + tokens.TryValidate(assertion, out var verified).Should().BeTrue(); + var boundary = app.Services.GetRequiredService(); + verified!.Identity.Subject.Should().Be(boundary.Subject); + verified.Identity.ClientId.Should().Be(boundary.CredentialId); + verified.ReplayBinding.Should().Be(binding); + return assertion; + } + + public static async Task PostAsync(HttpClient client, string path, T body, string assertion) + { + using var message = new HttpRequestMessage(HttpMethod.Post, path) { Content = JsonContent.Create(body) }; + message.Headers.Add(McpOperatorDelegationOptions.HeaderName, assertion); + return await client.SendAsync(message); + } + + public Task CheckCurrentAsync(string credentialId, IntegrationCredentialPurpose purpose, CancellationToken cancellationToken = default) + { + var valid = CurrentFailureCode is null && credentialId == CredentialId && purpose == IntegrationCredentialPurpose.HttpMcp; + return Task.FromResult(new IntegrationCredentialCurrentResult(valid + ? new VerifiedIntegrationCredential(CredentialId, Subject, purpose, []) { Resource = Resource } : null, + valid ? null : CurrentFailureCode ?? "delegated_identity_invalid")); + } + public async Task VerifyCurrentAsync(string credentialId, IntegrationCredentialPurpose purpose, CancellationToken cancellationToken = default) => + (await CheckCurrentAsync(credentialId, purpose, cancellationToken)).Credential; + public Task AuthorizeAsync(ClaimsPrincipal principal, string permission, int? tenantId, CancellationToken cancellationToken = default) => + Task.FromResult(HasPermission && permission == "job.manage" && tenantId == 42 && + principal.HasClaim("netratel_principal_id", Subject) && principal.HasClaim(IntegrationCredentialAuthenticationHandler.CredentialIdClaimType, CredentialId)); + public Task GetSnapshotAsync(ClaimsPrincipal principal, int? tenantId, CancellationToken cancellationToken = default) => + Task.FromResult(new EffectiveAccessSnapshot(Subject, false, false, new HashSet(HasPermission ? ["job.manage"] : []))); + public Task ReconcileBuiltInRolesAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; +} diff --git a/src/NetRatel/NetRatel.Tests/API/McpOperatorJobStartEndpointTests.cs b/src/NetRatel/NetRatel.Tests/API/McpOperatorJobStartEndpointTests.cs index bc146263..f9d4109b 100644 --- a/src/NetRatel/NetRatel.Tests/API/McpOperatorJobStartEndpointTests.cs +++ b/src/NetRatel/NetRatel.Tests/API/McpOperatorJobStartEndpointTests.cs @@ -16,6 +16,7 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using NetRatel.API.Endpoints.Client; +using NetRatel.API.Endpoints.Auth; using NetRatel.API.Gateway; using NetRatel.API.Middleware; using NetRatel.API.Services.Jobs; @@ -44,14 +45,17 @@ public sealed class McpOperatorJobStartEndpointTests private static string ActionRoot(string action) => $"/api/v2/mcp/operator/agents/42/{AgentId:D}/jobs/runs/{RunId}/{action}"; [Theory] - [InlineData("cancel")] - [InlineData("delete")] - public async Task Completed_run_actions_replay_the_original_state_without_repeating_dispatch_or_persistence(string action) + [InlineData("cancel", false)] + [InlineData("delete", false)] + [InlineData("cancel", true)] + [InlineData("delete", true)] + public async Task Completed_run_actions_replay_the_original_state_without_repeating_dispatch_or_persistence(string action, bool localDelegation) { - await using var app = await BuildAsync(action); + await using var app = await BuildAsync(action, localDelegation); using var client = app.GetTestClient(); - var request = await PreviewRunActionAsync(client, action); - var confirmed = await ConfirmRunActionAsync(client, action, request); + var boundaryApp = localDelegation ? app : null; + var request = await PreviewRunActionAsync(client, action, boundaryApp); + var confirmed = await ConfirmRunActionAsync(client, action, request, boundaryApp); confirmed.Replayed.Should().BeFalse(); confirmed.State.Should().Be(action == "cancel" ? nameof(JobRunState.Running) : nameof(JobRunState.Succeeded)); var dispatch = app.Services.GetRequiredService(); @@ -61,6 +65,7 @@ public async Task Completed_run_actions_replay_the_original_state_without_repeat var db = scope.ServiceProvider.GetRequiredService(); if (action == "cancel") (await db.JobRuns.SingleAsync()).Status = (int)JobRunState.Cancelled; (await db.McpOperatorJobs.SingleAsync()).DeletedAtUtc = DateTimeOffset.UtcNow; + if (localDelegation) (await db.McpOperatorJobRuns.SingleAsync()).DeletedAtUtc = DateTimeOffset.UtcNow; await db.SaveChangesAsync(); if (action == "delete") { @@ -71,11 +76,27 @@ public async Task Completed_run_actions_replay_the_original_state_without_repeat var writes = app.Services.GetRequiredService(); var beforeReplay = writes.Count; - var replayed = await ConfirmRunActionAsync(client, action, request); + var replayed = await ConfirmRunActionAsync(client, action, request, boundaryApp); replayed.Should().Be(confirmed with { Replayed = true }); writes.Count.Should().Be(beforeReplay); dispatch.Cancels.Should().Be(action == "cancel" ? 1 : 0); + if (localDelegation) + { + foreach (var operation in new[] { "get", action }) + { + using var ordinary = await McpJobLocalDelegationTestBoundary.ExchangeAsync(app, client, "netratel_job_runs", operation, RunId.ToString(), AgentId); + ordinary.StatusCode.Should().Be(HttpStatusCode.Forbidden); + } + var assertion = await McpJobLocalDelegationTestBoundary.AssertionAsync(app, client, "netratel_job_runs", action, RunId.ToString(), AgentId, + McpOperatorJobConfirmationPayload.RunActionBinding(action, RunId, request.Reason, request.PlanToken!, request.IdempotencyKey!)); + using var preview = await McpJobLocalDelegationTestBoundary.PostAsync(client, ActionRoot(action) + "/preview", request, assertion); + preview.StatusCode.Should().Be(HttpStatusCode.Unauthorized); + using var changed = await McpJobLocalDelegationTestBoundary.PostAsync(client, ActionRoot(action) + "/confirm", request with { Reason = "another reason" }, assertion); + changed.StatusCode.Should().Be(HttpStatusCode.Unauthorized); + writes.Count.Should().Be(beforeReplay); + dispatch.Cancels.Should().Be(action == "cancel" ? 1 : 0); + } await using var verification = app.Services.CreateAsyncScope(); var verificationDb = verification.ServiceProvider.GetRequiredService(); (await verificationDb.McpOperatorAcceptedAudits.CountAsync()).Should().Be(1); @@ -308,23 +329,28 @@ public async Task Start_assertion_cannot_be_reused_to_read_arbitrary_scripts() decision.Decision.FailureCode.Should().Be("oauth_scope_missing"); } - private static async Task PreviewRunActionAsync(HttpClient client, string action) + private static async Task PreviewRunActionAsync(HttpClient client, string action, WebApplication? app = null) { var request = new McpOperatorJobRunActionRequest("reviewed action"); - using var response = await client.PostAsJsonAsync(ActionRoot(action) + "/preview", request); + using var response = app is null ? await client.PostAsJsonAsync(ActionRoot(action) + "/preview", request) : + await McpJobLocalDelegationTestBoundary.PostAsync(client, ActionRoot(action) + "/preview", request, + await McpJobLocalDelegationTestBoundary.AssertionAsync(app, client, "netratel_job_runs", action, RunId.ToString(), AgentId)); response.StatusCode.Should().Be(HttpStatusCode.OK, await response.Content.ReadAsStringAsync()); var preview = (await response.Content.ReadFromJsonAsync())!; return request with { PlanToken = preview.PlanToken, IdempotencyKey = preview.IdempotencyKey }; } - private static async Task ConfirmRunActionAsync(HttpClient client, string action, McpOperatorJobRunActionRequest request) + private static async Task ConfirmRunActionAsync(HttpClient client, string action, McpOperatorJobRunActionRequest request, WebApplication? app = null) { - using var response = await client.PostAsJsonAsync(ActionRoot(action) + "/confirm", request); + using var response = app is null ? await client.PostAsJsonAsync(ActionRoot(action) + "/confirm", request) : + await McpJobLocalDelegationTestBoundary.PostAsync(client, ActionRoot(action) + "/confirm", request, + await McpJobLocalDelegationTestBoundary.AssertionAsync(app, client, "netratel_job_runs", action, RunId.ToString(), AgentId, + McpOperatorJobConfirmationPayload.RunActionBinding(action, RunId, request.Reason, request.PlanToken!, request.IdempotencyKey!))); response.StatusCode.Should().Be(HttpStatusCode.OK, await response.Content.ReadAsStringAsync()); return (await response.Content.ReadFromJsonAsync())!; } - private static async Task BuildAsync(string operation = "start") + private static async Task BuildAsync(string operation = "start", bool localDelegation = false) { var builder = WebApplication.CreateBuilder(new WebApplicationOptions { EnvironmentName = "Production" }); builder.WebHost.UseTestServer(); @@ -344,20 +370,25 @@ private static async Task BuildAsync(string operation = "start") builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); + if (localDelegation) McpJobLocalDelegationTestBoundary.Configure(builder, Subject, ClientId, Resource); var app = builder.Build(); app.UseAuthentication(); - // Current-grant and signed-delegation checks are covered separately in - // McpFileDelegationPermissionTests. Exercise real route/store/plan - // admission here with the exact Execute-only middleware output. - app.Use(async (context, next) => + if (localDelegation) app.UseMiddleware(); + else { - context.Items[McpOperatorDelegationMiddleware.HttpContextItemKey] = new McpOperatorDelegation( - new McpOperatorDelegationIdentity(Subject, ClientId, null, [], ["AutomationOperator"], ["netratel.mcp.execute"]), - "mcp-service", "netratel_job_runs", operation, "start-request", DateTimeOffset.UtcNow.AddMinutes(1), - Resource, "prod", 42, AgentId, "start-correlation"); - await next(context); - }); + // Keep the original direct route/store fixture for its focused + // policy and execution tests; local cases use the full boundary. + app.Use(async (context, next) => + { + context.Items[McpOperatorDelegationMiddleware.HttpContextItemKey] = new McpOperatorDelegation( + new McpOperatorDelegationIdentity(Subject, ClientId, null, [], ["AutomationOperator"], ["netratel.mcp.execute"]), + "mcp-service", "netratel_job_runs", operation, "start-request", DateTimeOffset.UtcNow.AddMinutes(1), + Resource, "prod", 42, AgentId, "start-correlation"); + await next(context); + }); + } app.UseAuthorization(); + if (localDelegation) app.MapMcpLocalDelegationEndpoints(); app.MapMcpOperatorJobEndpoints(); await app.StartAsync(); await using var scope = app.Services.CreateAsyncScope(); @@ -495,6 +526,7 @@ private sealed class Authentication(IOptionsMonitor : AuthenticationHandler(options, logger, encoder) { protected override Task HandleAuthenticateAsync() => Task.FromResult(AuthenticateResult.Success( - new AuthenticationTicket(new ClaimsPrincipal(new ClaimsIdentity([new Claim("sub", "mcp-service")], Scheme.Name)), Scheme.Name))); + new AuthenticationTicket(new ClaimsPrincipal(Context.RequestServices.GetService()?.Identity(Scheme.Name) ?? + new ClaimsIdentity([new Claim("sub", "mcp-service")], Scheme.Name)), Scheme.Name))); } } diff --git a/src/NetRatel/NetRatel.Tests/API/McpOperatorPolicyMutationEndpointTests.cs b/src/NetRatel/NetRatel.Tests/API/McpOperatorPolicyMutationEndpointTests.cs index 05cc3200..f2bd98c1 100644 --- a/src/NetRatel/NetRatel.Tests/API/McpOperatorPolicyMutationEndpointTests.cs +++ b/src/NetRatel/NetRatel.Tests/API/McpOperatorPolicyMutationEndpointTests.cs @@ -12,6 +12,7 @@ using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.TestHost; using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Diagnostics; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; @@ -72,6 +73,247 @@ public async Task Exact_agent_policy_lifecycle_uses_persisted_classification_and (administration.Creates + administration.Replaces + administration.Disables + administration.Revocations).Should().Be(1); } + [Theory] + [InlineData("replace", false)] + [InlineData("disable", false)] + [InlineData("revoke", false)] + [InlineData("disable", true)] + public async Task Policy_mutation_replay_preserves_the_original_result_and_does_not_consume_a_fresh_stale_plan(string action, bool simulateTimestampPrecisionLoss) + { + var draft = ExactAgentDraft() with + { + Name = " reviewed observation policy ", + PrincipalSelector = new McpOperatorPrincipalSelector(McpOperatorPrincipalSelectorKind.OidcGroup, "incident-responders"), + AuditReference = " change-fixture " + }; + await using var db = await CreatePolicyDbAsync(action, draft.TargetSelector); + var administration = new McpOperatorPolicyAdministration(db); + var policy = await CreateStoredPolicyAsync(db, administration, draft); + if (simulateTimestampPrecisionLoss) + { + db.SavingChanges += SetSubMicrosecondTimestamp; + void SetSubMicrosecondTimestamp(object? sender, SavingChangesEventArgs args) + { + var record = db.ChangeTracker.Entries().Single(entry => entry.Entity.Id == policy.PolicyId).Entity; + if (record.DisabledAtUtc is not { } timestamp) return; + record.DisabledAtUtc = new DateTimeOffset(timestamp.UtcTicks - timestamp.UtcTicks % TimeSpan.TicksPerMicrosecond + 7, TimeSpan.Zero); + db.SavingChanges -= SetSubMicrosecondTimestamp; + } + } + using var app = await BuildAppAsync(new McpOperatorAuthorization(db), administration, new McpOperatorConfirmationService(db)); + var client = M2mClient(app); + SetDelegation(client, app, $"preview_{action}", draft.TargetSelector, ["PolicyAdministrator"], ["netratel.mcp.admin"]); + var previewRequest = PolicyPreviewRequest(action, policy, draft); + var previewResponse = await client.PostAsJsonAsync($"/api/v2/mcp/operator/policy/{action}/preview", previewRequest); + previewResponse.StatusCode.Should().Be(HttpStatusCode.OK); + var preview = (await previewResponse.Content.ReadFromJsonAsync())!; + var stalePreviewResponse = await client.PostAsJsonAsync($"/api/v2/mcp/operator/policy/{action}/preview", previewRequest); + stalePreviewResponse.StatusCode.Should().Be(HttpStatusCode.OK); + var stalePreview = (await stalePreviewResponse.Content.ReadFromJsonAsync())!; + SetDelegation(client, app, $"confirm_{action}", draft.TargetSelector, ["PolicyAdministrator"], ["netratel.mcp.admin"]); + var request = PolicyConfirmRequest(action, policy, draft, preview); + + var confirmedResponse = await client.PostAsJsonAsync($"/api/v2/mcp/operator/policy/{action}/confirm", request); + confirmedResponse.StatusCode.Should().Be(HttpStatusCode.OK); + var confirmed = (await confirmedResponse.Content.ReadFromJsonAsync())!; + if (simulateTimestampPrecisionLoss) + { + var originalTimestamp = confirmed.Policy.DisabledAtUtc!.Value; + (originalTimestamp.UtcTicks % TimeSpan.TicksPerMicrosecond).Should().Be(7); + var record = await db.McpOperatorPolicies.SingleAsync(candidate => candidate.Id == policy.PolicyId); + // Simulate the timestamp returned by a PostgreSQL reload, leaving the receipt intact. + record.DisabledAtUtc = new DateTimeOffset(originalTimestamp.UtcTicks - originalTimestamp.UtcTicks % TimeSpan.TicksPerMicrosecond, TimeSpan.Zero); + await db.SaveChangesAsync(); + record.DisabledAtUtc.Should().NotBe(originalTimestamp); + } + var replayResponse = await client.PostAsJsonAsync($"/api/v2/mcp/operator/policy/{action}/confirm", request); + replayResponse.StatusCode.Should().Be(HttpStatusCode.OK); + var replay = (await replayResponse.Content.ReadFromJsonAsync())!; + replay.Replay.Should().BeTrue(); + replay.Policy.Should().BeEquivalentTo(confirmed.Policy); + + var staleResponse = await client.PostAsJsonAsync($"/api/v2/mcp/operator/policy/{action}/confirm", + PolicyConfirmRequest(action, policy, draft, stalePreview)); + staleResponse.StatusCode.Should().Be(HttpStatusCode.Conflict); + (await staleResponse.Content.ReadAsStringAsync()).Should().Contain("operator_policy_version_conflict"); + var unused = await db.McpOperatorConfirmationPlans.SingleAsync(plan => plan.IdempotencyKey == stalePreview.IdempotencyKey); + unused.ConsumedAtUtc.Should().BeNull(); + unused.ConsumedIdempotencyId.Should().BeNull(); + (await db.McpOperatorIdempotencyRecords.CountAsync()).Should().Be(1); + (await db.McpOperatorPolicyChangeAudits.CountAsync(audit => audit.PolicyId == policy.PolicyId)).Should().Be(2); + + if (action == "replace") + { + var later = await administration.ReplaceAsync(policy.PolicyId, confirmed.Policy.Version, + draft with { Name = "later replacement", Constraints = new McpOperatorConstraints(MaxOutputBytes: 256) }, "independent-admin@example.test", CancellationToken.None); + await administration.DisableAsync(policy.PolicyId, later.Version, "independent-admin@example.test", CancellationToken.None); + } + else if (action == "disable") + { + await administration.RevokeAsync(policy.PolicyId, confirmed.Policy.Version, "independent-admin@example.test", CancellationToken.None); + } + var changesBeforeReplay = await db.McpOperatorPolicyChangeAudits.CountAsync(); + var laterReplayResponse = await client.PostAsJsonAsync($"/api/v2/mcp/operator/policy/{action}/confirm", request); + laterReplayResponse.StatusCode.Should().Be(HttpStatusCode.OK); + var laterReplay = (await laterReplayResponse.Content.ReadFromJsonAsync())!; + laterReplay.Replay.Should().BeTrue(); + laterReplay.Policy.Should().BeEquivalentTo(confirmed.Policy); + (await db.McpOperatorPolicyChangeAudits.CountAsync()).Should().Be(changesBeforeReplay); + (await db.McpOperatorAcceptedAudits.CountAsync()).Should().Be(1); + } + + [Theory] + [InlineData("replace")] + [InlineData("disable")] + [InlineData("revoke")] + public async Task Policy_mutation_with_a_lost_response_preserves_its_successful_receipt(string action) + { + using var disconnected = new CancellationTokenSource(); + var writes = new DisconnectAfterPolicyWrite(disconnected.Cancel); + var draft = ExactAgentDraft(); + await using var db = await CreatePolicyDbAsync(action, draft.TargetSelector, writes); + var administration = new McpOperatorPolicyAdministration(db); + var policy = await CreateStoredPolicyAsync(db, administration, draft); + writes.PolicyId = policy.PolicyId; + using var app = await BuildAppAsync(new McpOperatorAuthorization(db), administration, new McpOperatorConfirmationService(db)); + var client = M2mClient(app); + SetDelegation(client, app, $"preview_{action}", draft.TargetSelector, ["PolicyAdministrator"], ["netratel.mcp.admin"]); + var previewResponse = await client.PostAsJsonAsync($"/api/v2/mcp/operator/policy/{action}/preview", PolicyPreviewRequest(action, policy, draft)); + previewResponse.StatusCode.Should().Be(HttpStatusCode.OK); + var preview = (await previewResponse.Content.ReadFromJsonAsync())!; + var request = PolicyConfirmRequest(action, policy, draft, preview); + SetDelegation(client, app, $"confirm_{action}", draft.TargetSelector, ["PolicyAdministrator"], ["netratel.mcp.admin"]); + + var lostResponse = async () => await client.PostAsJsonAsync($"/api/v2/mcp/operator/policy/{action}/confirm", request, disconnected.Token); + + await lostResponse.Should().ThrowAsync(); + await writes.ReceiptSaved.Task.WaitAsync(TimeSpan.FromSeconds(10)); + var persisted = (await administration.GetAsync(policy.PolicyId, CancellationToken.None))!; + persisted.Version.Should().Be(2); + var receipt = await db.McpOperatorIdempotencyRecords.AsNoTracking().SingleAsync(); + receipt.Outcome.Should().Be(McpOperatorIdempotencyOutcome.Succeeded); + var replayResponse = await client.PostAsJsonAsync($"/api/v2/mcp/operator/policy/{action}/confirm", request); + replayResponse.StatusCode.Should().Be(HttpStatusCode.OK); + var replay = (await replayResponse.Content.ReadFromJsonAsync())!; + replay.Replay.Should().BeTrue(); + replay.Policy.Should().BeEquivalentTo(persisted); + (await db.McpOperatorAcceptedAudits.CountAsync()).Should().Be(1); + (await db.McpOperatorPolicyChangeAudits.CountAsync(audit => audit.PolicyId == policy.PolicyId)).Should().Be(2); + (await db.McpOperatorIdempotencyRecords.CountAsync()).Should().Be(1); + } + + [Fact] + public async Task Policy_confirmation_with_a_pending_receipt_returns_retryable_conflict_without_redispatch() + { + await using var db = new OrchestratorDbContext(new DbContextOptionsBuilder() + .UseInMemoryDatabase(Guid.NewGuid().ToString("N")).Options); + var authorization = new RecordingAuthorization(allowed: true); + var administration = new RecordingAdministration(); + var confirmations = new McpOperatorConfirmationService(db); + using var app = await BuildAppAsync(authorization, administration, confirmations); + var client = M2mClient(app); + var draft = Draft(); + var policy = administration.CurrentPolicy; + SetDelegation(client, app, "preview_replace", draft.TargetSelector, ["PolicyAdministrator"], ["netratel.mcp.admin"]); + var previewResponse = await client.PostAsJsonAsync("/api/v2/mcp/operator/policy/replace/preview", PolicyPreviewRequest("replace", policy, draft)); + previewResponse.StatusCode.Should().Be(HttpStatusCode.OK); + var preview = (await previewResponse.Content.ReadFromJsonAsync())!; + var currentDecision = await authorization.EvaluateAsync(authorization.Evaluated.Single(), CancellationToken.None); + var plan = await db.McpOperatorConfirmationPlans.SingleAsync(); + // Reserve the first request without completing its operation, as when + // another confirmation arrives while that admitted dispatch is in flight. + var inFlight = await confirmations.ConfirmAsync(new McpOperatorConfirmationRequest( + preview.PlanToken, preview.IdempotencyKey, plan.PayloadHash, currentDecision), CancellationToken.None); + inFlight.IsNewDispatch.Should().BeTrue(); + SetDelegation(client, app, "confirm_replace", draft.TargetSelector, ["PolicyAdministrator"], ["netratel.mcp.admin"]); + + var duplicate = await client.PostAsJsonAsync("/api/v2/mcp/operator/policy/replace/confirm", PolicyConfirmRequest("replace", policy, draft, preview)); + + duplicate.StatusCode.Should().Be(HttpStatusCode.Conflict); + using var failure = JsonDocument.Parse(await duplicate.Content.ReadAsStringAsync()); + failure.RootElement.GetProperty("failure").GetProperty("code").GetString().Should().Be("idempotency_pending"); + failure.RootElement.GetProperty("failure").GetProperty("retryable").GetBoolean().Should().BeTrue(); + administration.Replaces.Should().Be(0); + authorization.Accepted.Should().BeEmpty(); + var retained = await db.McpOperatorIdempotencyRecords.SingleAsync(); + retained.Id.Should().Be(inFlight.IdempotencyId!.Value); + retained.Outcome.Should().Be(McpOperatorIdempotencyOutcome.Pending); + } + + [Theory] + [InlineData("payload", HttpStatusCode.Conflict)] + [InlineData("subject", HttpStatusCode.Forbidden)] + [InlineData("client", HttpStatusCode.Conflict)] + [InlineData("resource", HttpStatusCode.Conflict)] + [InlineData("instance", HttpStatusCode.Unauthorized)] + [InlineData("target", HttpStatusCode.Unauthorized)] + [InlineData("policy_revoked", HttpStatusCode.Forbidden)] + [InlineData("expired", HttpStatusCode.Conflict)] + public async Task Policy_replay_requires_current_authority_exact_binding_and_an_unexpired_plan(string change, HttpStatusCode expectedStatus) + { + var draft = ExactAgentDraft(); + await using var db = await CreatePolicyDbAsync("replace", draft.TargetSelector); + var administration = new McpOperatorPolicyAdministration(db); + var policy = await CreateStoredPolicyAsync(db, administration, draft); + using var app = await BuildAppAsync(new McpOperatorAuthorization(db), administration, new McpOperatorConfirmationService(db)); + var client = M2mClient(app); + SetDelegation(client, app, "preview_replace", draft.TargetSelector, ["PolicyAdministrator"], ["netratel.mcp.admin"]); + var previewResponse = await client.PostAsJsonAsync("/api/v2/mcp/operator/policy/replace/preview", PolicyPreviewRequest("replace", policy, draft)); + previewResponse.StatusCode.Should().Be(HttpStatusCode.OK); + var preview = (await previewResponse.Content.ReadFromJsonAsync())!; + SetDelegation(client, app, "confirm_replace", draft.TargetSelector, ["PolicyAdministrator"], ["netratel.mcp.admin"]); + var request = (ConfirmMcpOperatorPolicyReplaceRequest)PolicyConfirmRequest("replace", policy, draft, preview); + var confirmed = await client.PostAsJsonAsync("/api/v2/mcp/operator/policy/replace/confirm", request); + confirmed.StatusCode.Should().Be(HttpStatusCode.OK); + if (change == "policy_revoked") + { + var authority = await db.McpOperatorPolicies.SingleAsync(value => value.Id != policy.PolicyId); + authority.DisabledAtUtc = DateTimeOffset.UtcNow; + authority.LifecycleState = McpOperatorPolicyLifecycleState.Revoked; + authority.Version++; + } + if (change == "expired") + (await db.McpOperatorConfirmationPlans.SingleAsync()).ExpiresAtUtc = DateTimeOffset.UtcNow.AddSeconds(-1); + await db.SaveChangesAsync(); + SetDelegation(client, app, "confirm_replace", + change == "target" ? draft.TargetSelector with { AgentId = Guid.NewGuid() } : draft.TargetSelector, + ["PolicyAdministrator"], ["netratel.mcp.admin"], + subject: change == "subject" ? "other.admin@example.test" : "policy.admin@example.test", + clientId: change == "client" ? "other-policy-client" : "policy-admin-client", + resource: change == "resource" ? "https://other-mcp.example/mcp" : "https://mcp.dev.example/mcp", + instance: change == "instance" ? "other" : "dev"); + if (change == "payload") request = request with { Policy = draft with { Name = "changed payload" } }; + + var denied = await client.PostAsJsonAsync("/api/v2/mcp/operator/policy/replace/confirm", request); + + denied.StatusCode.Should().Be(expectedStatus); + if (change == "expired") (await denied.Content.ReadAsStringAsync()).Should().Contain("confirmation_plan_expired"); + (await db.McpOperatorIdempotencyRecords.CountAsync()).Should().Be(1); + (await db.McpOperatorAcceptedAudits.CountAsync()).Should().Be(1); + (await db.McpOperatorPolicyChangeAudits.CountAsync(audit => audit.PolicyId == policy.PolicyId)).Should().Be(2); + (await administration.GetAsync(policy.PolicyId, CancellationToken.None))!.Version.Should().Be(2); + } + + private static async Task CreateStoredPolicyAsync( + OrchestratorDbContext db, McpOperatorPolicyAdministration administration, McpOperatorPolicyDraft draft) + { + var agentId = draft.TargetSelector.AgentId!.Value; + db.Agents.Add(new Agent { Id = agentId, TenantId = draft.TargetSelector.TenantId, IsEnabled = true, + Status = AgentStatus.Active, CreatedAtUtc = DateTimeOffset.UtcNow }); + await db.SaveChangesAsync(); + await administration.UpsertTargetProfileAsync(draft.TargetSelector.TenantId, agentId, + McpOperatorTargetClassification.ManagedStandard, ["campaign-fixture"], null, "independent-admin@example.test", CancellationToken.None); + return await administration.CreateAsync(draft, "independent-admin@example.test", CancellationToken.None); + } + + private static object PolicyPreviewRequest(string action, McpOperatorPolicy policy, McpOperatorPolicyDraft draft) => action == "replace" + ? new PreviewMcpOperatorPolicyReplaceRequest(policy.PolicyId, policy.Version, draft) + : new PreviewMcpOperatorPolicyDisableRequest(policy.PolicyId, policy.Version, draft.TargetSelector); + + private static object PolicyConfirmRequest(string action, McpOperatorPolicy policy, McpOperatorPolicyDraft draft, McpOperatorPolicyMutationPreview preview) => action == "replace" + ? new ConfirmMcpOperatorPolicyReplaceRequest(policy.PolicyId, policy.Version, draft, preview.PlanToken, preview.IdempotencyKey) + : new ConfirmMcpOperatorPolicyDisableRequest(policy.PolicyId, policy.Version, draft.TargetSelector, preview.PlanToken, preview.IdempotencyKey); + [Theory] [InlineData("version", "confirmation_plan_stale")] [InlineData("tags", "confirmation_plan_stale")] @@ -138,10 +380,12 @@ private static McpOperatorPolicyDraft ExactAgentDraft() => Draft() with target.AgentId!.Value, target.TenantId, McpOperatorTargetClassification.ManagedStandard, ["campaign-fixture"], DateTimeOffset.UtcNow, "independent-admin@example.test", 2); - private static async Task CreatePolicyDbAsync(string action, McpOperatorTargetSelector target) + private static async Task CreatePolicyDbAsync(string action, McpOperatorTargetSelector target, + SaveChangesInterceptor? interceptor = null) { - var db = new OrchestratorDbContext(new DbContextOptionsBuilder() - .UseInMemoryDatabase(Guid.NewGuid().ToString("N")).Options); + var options = new DbContextOptionsBuilder().UseInMemoryDatabase(Guid.NewGuid().ToString("N")); + if (interceptor is not null) options.AddInterceptors(interceptor); + var db = new OrchestratorDbContext(options.Options); db.McpOperatorPolicies.Add(new McpOperatorPolicyRecord { Id = Guid.NewGuid(), Name = "exact classified policy administrator", Environment = McpOperatorEnvironment.Development, @@ -490,14 +734,18 @@ private static void SetDelegation( McpOperatorTargetSelector target, IReadOnlyList roles, IReadOnlyList scopes, - IReadOnlyList? groups = null) + IReadOnlyList? groups = null, + string subject = "policy.admin@example.test", + string clientId = "policy-admin-client", + string resource = "https://mcp.dev.example/mcp", + string instance = "dev") { client.DefaultRequestHeaders.Remove(McpOperatorDelegationOptions.HeaderName); var assertion = app.Services.GetRequiredService().Create( new McpOperatorDelegationIdentity( - "policy.admin@example.test", - "policy-admin-client", - "policy-admin-client", + subject, + clientId, + clientId, groups ?? [], roles, scopes), @@ -505,8 +753,8 @@ private static void SetDelegation( "netratel_policy", operation, $"request-{operation}", - "https://mcp.dev.example/mcp", - "dev", + resource, + instance, target.Kind == McpOperatorTargetSelectorKind.ControlPlane ? null : target.TenantId, target.Kind == McpOperatorTargetSelectorKind.ExactAgent ? target.AgentId : null, $"correlation-{operation}")); @@ -515,7 +763,7 @@ private static void SetDelegation( private static async Task BuildAppAsync( IMcpOperatorAuthorization authorization, - RecordingAdministration administration, + IMcpOperatorPolicyAdministration administration, IMcpOperatorConfirmationService confirmations) { var delegationOptions = new McpOperatorDelegationOptions @@ -549,7 +797,7 @@ private static async Task BuildAppAsync( services.AddSingleton(); services.AddSingleton(authorization); services.AddSingleton(administration); - services.AddSingleton(administration); + services.AddSingleton((IMcpOperatorPolicyRevocation)administration); services.AddSingleton(confirmations); }); web.Configure(app => @@ -624,6 +872,31 @@ public Task RecordAcceptedAsync(McpOperatorDecision de } } + private sealed class DisconnectAfterPolicyWrite(Action disconnect) : SaveChangesInterceptor + { + public Guid? PolicyId { get; set; } + public TaskCompletionSource ReceiptSaved { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public override ValueTask SavedChangesAsync(SaveChangesCompletedEventData eventData, int result, + CancellationToken cancellationToken = default) + { + if (eventData.Context is { } db) + { + if (PolicyId is { } policyId && db.ChangeTracker.Entries() + .Any(entry => entry.Entity.Id == policyId && entry.Entity.Version == 2)) + { + // Disconnect only after the real administration service has persisted its mutation. + PolicyId = null; + disconnect(); + } + if (db.ChangeTracker.Entries() + .Any(entry => entry.Entity.Outcome == McpOperatorIdempotencyOutcome.Succeeded)) + ReceiptSaved.TrySetResult(); + } + return ValueTask.FromResult(result); + } + } + private sealed class RecordingAdministration : IMcpOperatorPolicyAdministration, IMcpOperatorPolicyRevocation { public int GuardedMutations { get; private set; } @@ -771,6 +1044,8 @@ public Task CreatePlanAsync(McpOperatorConfirmation public Task ConfirmAsync(McpOperatorConfirmationRequest request, CancellationToken cancellationToken) { + if (request.ReplayOnly) + return Task.FromResult(new McpOperatorConfirmationAdmission(false, false, null, null, null, null)); Confirmed.Add(request); return Task.FromResult(new McpOperatorConfirmationAdmission(true, false, null, AdmissionId, McpOperatorIdempotencyOutcome.Pending, null)); } diff --git a/src/NetRatel/NetRatel.Tests/API/McpOperatorScriptEndpointTests.cs b/src/NetRatel/NetRatel.Tests/API/McpOperatorScriptEndpointTests.cs index ba90c0bd..9303f034 100644 --- a/src/NetRatel/NetRatel.Tests/API/McpOperatorScriptEndpointTests.cs +++ b/src/NetRatel/NetRatel.Tests/API/McpOperatorScriptEndpointTests.cs @@ -1,15 +1,32 @@ +using System.Net; +using System.Net.Http.Json; +using System.Security.Claims; +using System.Security.Cryptography; +using System.Text; +using System.Text.Encodings.Web; +using System.Text.Json; using AwesomeAssertions; +using Microsoft.AspNetCore.Authentication; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Routing; +using Microsoft.AspNetCore.TestHost; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Diagnostics; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using NetRatel.API.Endpoints.Client; using NetRatel.API.Gateway; +using NetRatel.API.Middleware; using NetRatel.Akka.Configuration; using NetRatel.Application.Events; using NetRatel.Application.Jobs; using NetRatel.Application.Operations; using NetRatel.Application.Presence; +using NetRatel.Infrastructure.Persistence; +using NetRatel.Infrastructure.Services; using NetRatel.Shared.Operations; using Xunit; @@ -17,6 +34,374 @@ namespace NetRatel.Tests.API; public sealed class McpOperatorScriptEndpointTests { + private const string Resource = "https://mcp.example.test/mcp"; + private const string Subject = "script-operator"; + private const string ClientId = "script-client"; + private const string ScriptContent = "printf 'reviewed-marker'"; + private const long ScriptId = 801; + private static readonly Guid AgentId = Guid.Parse("865d8a10-b2aa-4f11-93c0-a4d2f3a400e0"); + private static string ScriptRoot => $"/api/v2/mcp/operator/agents/42/{AgentId:D}/scripts"; + + [Fact] + public async Task Deleted_script_replays_its_original_version_without_another_write_or_audit() + { + await using var app = await BuildMutationAppAsync(); + using var client = app.GetTestClient(); + var request = await PreviewDeleteAsync(client); + var confirmed = await ConfirmDeleteAsync(client, request); + confirmed.Should().Be(new McpOperatorScriptMutationResult(ScriptId, 2, true, false, "script-correlation")); + var writes = app.Services.GetRequiredService(); + var afterConfirmation = writes.Count; + + var replayed = await ConfirmDeleteAsync(client, request); + + replayed.Should().Be(confirmed with { Replayed = true }); + writes.Count.Should().Be(afterConfirmation); + await using (var scope = app.Services.CreateAsyncScope()) + { + var db = scope.ServiceProvider.GetRequiredService(); + (await db.McpOperatorIdempotencyRecords.CountAsync()).Should().Be(1); + (await db.McpOperatorAcceptedAudits.CountAsync()).Should().Be(1); + (await db.McpOperatorScriptVersions.SingleAsync()).Action.Should().Be("deleted"); + var scripts = scope.ServiceProvider.GetRequiredService(); + (await scripts.GetOwnedAsync(ScriptId, 42, Principal(), Resource, "prod", default)).Should().BeNull(); + (await scripts.ListOwnedAsync(42, Principal(), Resource, "prod", default)).Should().BeEmpty(); + // A receipt retains the deletion's result even if retained metadata + // later advances; it must not substitute that subsequent version. + (await db.McpOperatorScripts.SingleAsync()).Version++; + await db.SaveChangesAsync(); + } + var afterRetainedChange = writes.Count; + + (await ConfirmDeleteAsync(client, request)).Should().Be(confirmed with { Replayed = true }); + writes.Count.Should().Be(afterRetainedChange); + } + + [Fact] + public async Task Changed_payload_and_fresh_competing_delete_cannot_reuse_a_successful_delete() + { + await using var app = await BuildMutationAppAsync(); + using var client = app.GetTestClient(); + var stale = await PreviewDeleteAsync(client); + var winning = await PreviewDeleteAsync(client); + var confirmed = await ConfirmDeleteAsync(client, winning); + var writes = app.Services.GetRequiredService(); + var beforeAttempts = writes.Count; + + using var changed = await client.PostAsJsonAsync(ScriptRoot + "/confirm/delete", winning with { ExpectedVersion = 2 }); + changed.StatusCode.Should().Be(HttpStatusCode.Conflict); + (await changed.Content.ReadAsStringAsync()).Should().Contain("confirmation_plan_stale"); + using var staleResponse = await client.PostAsJsonAsync(ScriptRoot + "/confirm/delete", stale); + staleResponse.StatusCode.Should().Be(HttpStatusCode.NotFound); + (await staleResponse.Content.ReadAsStringAsync()).Should().Contain("script_not_found"); + + writes.Count.Should().Be(beforeAttempts); + await using var scope = app.Services.CreateAsyncScope(); + var db = scope.ServiceProvider.GetRequiredService(); + (await db.McpOperatorConfirmationPlans.CountAsync(plan => plan.ConsumedAtUtc == null)).Should().Be(1); + (await db.McpOperatorIdempotencyRecords.CountAsync()).Should().Be(1); + (await db.McpOperatorAcceptedAudits.CountAsync()).Should().Be(1); + (await db.McpOperatorScriptVersions.CountAsync()).Should().Be(1); + (await ConfirmDeleteAsync(client, winning)).Should().Be(confirmed with { Replayed = true }); + } + + [Fact] + public async Task Fresh_delete_with_a_stale_live_revision_does_not_consume_its_plan() + { + await using var app = await BuildMutationAppAsync(); + using var client = app.GetTestClient(); + var request = await PreviewDeleteAsync(client); + await using (var scope = app.Services.CreateAsyncScope()) + { + var db = scope.ServiceProvider.GetRequiredService(); + (await db.McpOperatorScripts.SingleAsync()).Version++; + await db.SaveChangesAsync(); + } + var writes = app.Services.GetRequiredService(); + var beforeAttempt = writes.Count; + + using var rejected = await client.PostAsJsonAsync(ScriptRoot + "/confirm/delete", request); + + rejected.StatusCode.Should().Be(HttpStatusCode.Conflict); + (await rejected.Content.ReadAsStringAsync()).Should().Contain("script_etag_mismatch"); + writes.Count.Should().Be(beforeAttempt); + await using var verification = app.Services.CreateAsyncScope(); + var verificationDb = verification.ServiceProvider.GetRequiredService(); + (await verificationDb.McpOperatorConfirmationPlans.SingleAsync()).ConsumedAtUtc.Should().BeNull(); + (await verificationDb.McpOperatorIdempotencyRecords.CountAsync()).Should().Be(0); + (await verificationDb.McpOperatorAcceptedAudits.CountAsync()).Should().Be(0); + (await verificationDb.McpOperatorScripts.SingleAsync()).DeletedAtUtc.Should().BeNull(); + } + + [Theory] + [InlineData("policy-version")] + [InlineData("policy-disabled")] + [InlineData("expired")] + [InlineData("scope")] + [InlineData("subject")] + [InlineData("client")] + [InlineData("resource")] + [InlineData("target")] + [InlineData("target-version")] + [InlineData("tenant")] + [InlineData("owner")] + [InlineData("owner-tenant")] + [InlineData("content")] + public async Task Deleted_script_replay_rechecks_current_authority_and_retained_ownership(string change) + { + await using var app = await BuildMutationAppAsync(); + using var client = app.GetTestClient(); + var request = await PreviewDeleteAsync(client); + await ConfirmDeleteAsync(client, request); + await using (var scope = app.Services.CreateAsyncScope()) + { + var db = scope.ServiceProvider.GetRequiredService(); + var caller = app.Services.GetRequiredService(); + switch (change) + { + case "policy-version": (await db.McpOperatorPolicies.SingleAsync()).Version++; break; + case "policy-disabled": (await db.McpOperatorPolicies.SingleAsync()).DisabledAtUtc = DateTimeOffset.UtcNow; break; + case "expired": (await db.McpOperatorConfirmationPlans.SingleAsync()).ExpiresAtUtc = DateTimeOffset.UtcNow.AddMinutes(-1); break; + case "scope": caller.Scopes = []; break; + case "subject": caller.Subject = "other-operator"; break; + case "client": caller.ClientId = "other-client"; break; + case "resource": caller.Resource = "https://other.example.test/mcp"; break; + case "target": caller.AgentId = Guid.NewGuid(); break; + case "target-version": (await db.McpOperatorTargetProfiles.SingleAsync()).Version++; break; + case "tenant": caller.TenantId = 43; break; + case "owner": (await db.McpOperatorScripts.SingleAsync()).Subject = "other-owner"; break; + case "owner-tenant": (await db.McpOperatorScripts.SingleAsync()).TenantId = 43; break; + case "content": (await db.Scripts.SingleAsync()).Content = "changed outside the retained revision"; break; + } + await db.SaveChangesAsync(); + } + var writes = app.Services.GetRequiredService(); + var beforeReplay = writes.Count; + + using var replayed = await client.PostAsJsonAsync(ScriptRoot + "/confirm/delete", request); + + replayed.IsSuccessStatusCode.Should().BeFalse(); + writes.Count.Should().Be(beforeReplay); + await using var verification = app.Services.CreateAsyncScope(); + var verificationDb = verification.ServiceProvider.GetRequiredService(); + (await verificationDb.McpOperatorScriptVersions.CountAsync()).Should().Be(1); + (await verificationDb.McpOperatorAcceptedAudits.CountAsync()).Should().Be(1); + (await verificationDb.McpOperatorIdempotencyRecords.SingleAsync()).Outcome.Should().Be(McpOperatorIdempotencyOutcome.Succeeded); + } + + [Fact] + public async Task Legacy_script_id_receipt_replays_the_authorized_tombstone_version() + { + await using var app = await BuildMutationAppAsync(); + using var client = app.GetTestClient(); + var request = await PreviewDeleteAsync(client); + var confirmed = await ConfirmDeleteAsync(client, request); + await using (var scope = app.Services.CreateAsyncScope()) + { + var db = scope.ServiceProvider.GetRequiredService(); + (await db.McpOperatorIdempotencyRecords.SingleAsync()).ResultReference = ScriptId.ToString(); + await db.SaveChangesAsync(); + } + var writes = app.Services.GetRequiredService(); + var beforeReplay = writes.Count; + + (await ConfirmDeleteAsync(client, request)).Should().Be(confirmed with { Replayed = true }); + writes.Count.Should().Be(beforeReplay); + } + + [Fact] + public async Task Concurrent_retry_and_lost_response_preserve_one_successful_deletion_receipt() + { + await using var app = await BuildMutationAppAsync(); + using var client = app.GetTestClient(); + var request = await PreviewDeleteAsync(client); + var writes = app.Services.GetRequiredService(); + writes.PauseReceipt = true; + using var disconnected = new CancellationTokenSource(); + var confirmation = client.PostAsJsonAsync(ScriptRoot + "/confirm/delete", request, disconnected.Token); + try + { + await writes.ReceiptStarted.Task.WaitAsync(TimeSpan.FromSeconds(10)); + using var pending = await client.PostAsJsonAsync(ScriptRoot + "/confirm/delete", request); + pending.StatusCode.Should().Be(HttpStatusCode.Conflict); + (await pending.Content.ReadAsStringAsync()).Should().Contain("idempotency_pending"); + disconnected.Cancel(); + } + finally + { + writes.AllowReceipt.TrySetResult(); + } + var lostResponse = async () => await confirmation; + await lostResponse.Should().ThrowAsync(); + await writes.ReceiptSaved.Task.WaitAsync(TimeSpan.FromSeconds(10)); + var afterConfirmation = writes.Count; + + (await ConfirmDeleteAsync(client, request)).Should().Be(new McpOperatorScriptMutationResult(ScriptId, 2, true, true, "script-correlation")); + writes.Count.Should().Be(afterConfirmation); + await using var scope = app.Services.CreateAsyncScope(); + var db = scope.ServiceProvider.GetRequiredService(); + (await db.McpOperatorIdempotencyRecords.SingleAsync()).Outcome.Should().Be(McpOperatorIdempotencyOutcome.Succeeded); + (await db.McpOperatorAcceptedAudits.CountAsync()).Should().Be(1); + (await db.McpOperatorScriptVersions.CountAsync()).Should().Be(1); + (await db.McpOperatorScripts.SingleAsync()).Version.Should().Be(2); + } + + private static async Task PreviewDeleteAsync(HttpClient client) + { + var request = new McpOperatorScriptMutationRequest(ScriptId: ScriptId, ExpectedVersion: 1); + using var response = await client.PostAsJsonAsync(ScriptRoot + "/preview/delete", request); + response.StatusCode.Should().Be(HttpStatusCode.OK, await response.Content.ReadAsStringAsync()); + var preview = (await response.Content.ReadFromJsonAsync())!; + return request with { PlanToken = preview.PlanToken, IdempotencyKey = preview.IdempotencyKey }; + } + + private static async Task ConfirmDeleteAsync(HttpClient client, McpOperatorScriptMutationRequest request) + { + using var response = await client.PostAsJsonAsync(ScriptRoot + "/confirm/delete", request); + response.StatusCode.Should().Be(HttpStatusCode.OK, await response.Content.ReadAsStringAsync()); + var result = (await response.Content.ReadFromJsonAsync())!; + response.Headers.ETag!.Tag.Should().Be($"\"{result.Version}\""); + return result; + } + + private static async Task BuildMutationAppAsync() + { + var builder = WebApplication.CreateBuilder(new WebApplicationOptions { EnvironmentName = "Production" }); + builder.WebHost.UseTestServer(); + builder.Services.AddAuthentication("Test").AddScheme("Test", _ => { }); + builder.Services.AddAuthorization(options => options.AddPolicy("M2MOnly", policy => policy.RequireAuthenticatedUser())); + var databaseName = "script-mutation-" + Guid.NewGuid(); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(); + builder.Services.AddDbContext((provider, options) => options.UseInMemoryDatabase(databaseName) + .AddInterceptors(provider.GetRequiredService())); + builder.Services.AddScoped(); + builder.Services.AddScoped(); + builder.Services.AddScoped(); + builder.Services.AddScoped(); + builder.Services.AddScoped(_ => throw new NotSupportedException()); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(_ => throw new NotSupportedException()); + builder.Services.AddSingleton(); + var app = builder.Build(); + app.UseAuthentication(); + // Reuse the endpoint regression layer with the real policy, ownership + // and confirmation services after trusted delegation ingress. + app.Use(async (context, next) => + { + var caller = context.RequestServices.GetRequiredService(); + context.Items[McpOperatorDelegationMiddleware.HttpContextItemKey] = new McpOperatorDelegation( + new McpOperatorDelegationIdentity(caller.Subject, caller.ClientId, null, [], ["ScriptOperator"], caller.Scopes), + "mcp-service", "netratel_scripts", "delete", "script-request", DateTimeOffset.UtcNow.AddMinutes(1), + caller.Resource, "prod", caller.TenantId, caller.AgentId, "script-correlation"); + await next(context); + }); + app.UseAuthorization(); + app.MapMcpOperatorScriptEndpoints(); + await app.StartAsync(); + await using var scope = app.Services.CreateAsyncScope(); + var db = scope.ServiceProvider.GetRequiredService(); + var now = DateTimeOffset.UtcNow; + var policyId = Guid.NewGuid(); + db.Agents.Add(new Agent { Id = AgentId, TenantId = 42, IsEnabled = true, Status = AgentStatus.Active, CreatedAtUtc = now }); + db.McpOperatorTargetProfiles.Add(new McpOperatorTargetProfileRecord + { + AgentId = AgentId, TenantId = 42, Classification = McpOperatorTargetClassification.ManagedStandard, + TagsJson = "[]", Version = 1, UpdatedAtUtc = now, UpdatedBy = "independent-admin" + }); + db.McpOperatorPolicies.Add(new McpOperatorPolicyRecord + { + Id = policyId, Name = "owned script mutations", Environment = McpOperatorEnvironment.Production, + Effect = McpOperatorPolicyEffect.Allow, Priority = 10, + PrincipalSelectorKind = McpOperatorPrincipalSelectorKind.OAuthSubject, PrincipalSelectorValue = Subject, + TargetSelectorKind = McpOperatorTargetSelectorKind.ExactAgent, TenantId = 42, AgentId = AgentId, + TargetClassification = McpOperatorTargetClassification.ManagedStandard, OperationFamily = McpOperatorOperationFamily.ScriptsWrite, + ConstraintsJson = JsonSerializer.Serialize(new McpOperatorConstraints(AllowedShells: ["sh"], WorkingDirectories: ["/tmp"], + MaxCommandDurationSeconds: 60, MaxScriptBytes: 4096, MaxTaskTargetCount: 1, MaxFanOut: 1), McpOperatorJsonContext.Default.McpOperatorConstraints), + CreatedAtUtc = now, CreatedBy = "independent-admin", Version = 1 + }); + db.Scripts.Add(new ScriptDefinition { Id = ScriptId, Name = "owned-script", Description = "marker", ScriptType = "bash", Content = ScriptContent }); + db.McpOperatorScripts.Add(new McpOperatorScriptRecord + { + Id = Guid.NewGuid(), ScriptId = ScriptId, TenantId = 42, Subject = Subject, ClientId = ClientId, + McpResource = Resource, McpInstance = "prod", PolicyId = policyId, PolicyVersion = 1, + Name = "owned-script", Description = "marker", ShellType = "sh", ContentHash = Hash(ScriptContent), ManifestHash = Hash(string.Empty), + TimeoutSeconds = 15, WorkingDirectory = "/tmp", DeclaredSideEffectsJson = "[1]", Version = 1 + }); + await db.SaveChangesAsync(); + return app; + } + + private static McpOperatorPrincipal Principal() => new(Subject, ClientId, null, new HashSet(), + new HashSet(["ScriptOperator"], StringComparer.Ordinal), new HashSet(["netratel.mcp.write"], StringComparer.Ordinal)); + + private static string Hash(string value) => Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(value))); + + private sealed class MutationCaller + { + public string Subject { get; set; } = McpOperatorScriptEndpointTests.Subject; + public string ClientId { get; set; } = McpOperatorScriptEndpointTests.ClientId; + public string Resource { get; set; } = McpOperatorScriptEndpointTests.Resource; + public string[] Scopes { get; set; } = ["netratel.mcp.write"]; + public int TenantId { get; set; } = 42; + public Guid AgentId { get; set; } = McpOperatorScriptEndpointTests.AgentId; + } + + private sealed class MutationWrites : SaveChangesInterceptor + { + private int _count; + public int Count => _count; + public bool PauseReceipt { get; set; } + public TaskCompletionSource ReceiptStarted { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource AllowReceipt { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource ReceiptSaved { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public override async ValueTask> SavingChangesAsync(DbContextEventData eventData, + InterceptionResult result, CancellationToken cancellationToken = default) + { + Interlocked.Increment(ref _count); + if (PauseReceipt && eventData.Context!.ChangeTracker.Entries() + .Any(entry => entry.State == EntityState.Modified && entry.Entity.Outcome == McpOperatorIdempotencyOutcome.Succeeded)) + { + ReceiptStarted.TrySetResult(); + await AllowReceipt.Task.WaitAsync(cancellationToken); + } + return result; + } + + public override ValueTask SavedChangesAsync(SaveChangesCompletedEventData eventData, int result, + CancellationToken cancellationToken = default) + { + if (PauseReceipt && eventData.Context!.ChangeTracker.Entries() + .Any(entry => entry.Entity.Outcome == McpOperatorIdempotencyOutcome.Succeeded)) + ReceiptSaved.TrySetResult(); + return ValueTask.FromResult(result); + } + } + + private sealed class MutationAuthentication(IOptionsMonitor options, ILoggerFactory logger, UrlEncoder encoder) + : AuthenticationHandler(options, logger, encoder) + { + protected override Task HandleAuthenticateAsync() => Task.FromResult(AuthenticateResult.Success( + new AuthenticationTicket(new ClaimsPrincipal(new ClaimsIdentity([new Claim("sub", "mcp-service")], Scheme.Name)), Scheme.Name))); + } + + private sealed class MutationPresence : IClientPresenceRouter + { + public Task StartSessionAsync(StartGatewayPresenceSession message, CancellationToken cancellationToken) => throw new NotSupportedException(); + public Task RecordHeartbeatAsync(RecordGatewayHeartbeat message, CancellationToken cancellationToken) => throw new NotSupportedException(); + public Task EndSessionAsync(EndGatewayPresenceSession message, CancellationToken cancellationToken) => throw new NotSupportedException(); + public Task ProbeAsync(CancellationToken cancellationToken) => throw new NotSupportedException(); + public Task GetSnapshotAsync(ClientKey client, CancellationToken cancellationToken) => throw new NotSupportedException(); + } + + private sealed class MutationDispatcher : IAgentCommandAuthorityDispatcher + { + public bool IsAvailable(ClientKey client) => throw new NotSupportedException(); + public Task DispatchAsync(ClientKey client, string commandId, string correlationId, string taskType, string payloadJson, int environment, CancellationToken cancellationToken) => throw new NotSupportedException(); + } + [Fact] public void Production_script_routes_are_explicitly_versioned_and_require_the_mcp_m2m_identity() { diff --git a/src/NetRatel/NetRatel.Tests/Infrastructure/McpOperationObjectTargetResolverTests.cs b/src/NetRatel/NetRatel.Tests/Infrastructure/McpOperationObjectTargetResolverTests.cs index 3ff98562..0b15ce5d 100644 --- a/src/NetRatel/NetRatel.Tests/Infrastructure/McpOperationObjectTargetResolverTests.cs +++ b/src/NetRatel/NetRatel.Tests/Infrastructure/McpOperationObjectTargetResolverTests.cs @@ -1,5 +1,6 @@ using AwesomeAssertions; using Microsoft.EntityFrameworkCore; +using NetRatel.Application.Operations; using NetRatel.Infrastructure.Identity; using NetRatel.Infrastructure.Persistence; using NetRatel.Shared.Operations; @@ -9,6 +10,177 @@ namespace NetRatel.Tests.Infrastructure; public sealed class McpOperationObjectTargetResolverTests { + [Theory] + [InlineData("netratel_jobs", "delete", false)] + [InlineData("netratel_jobs", "update", false)] + [InlineData("netratel_jobs", "param_add", false)] + [InlineData("netratel_jobs", "delete", true)] + [InlineData("netratel_job_runs", "delete", false)] + [InlineData("netratel_job_runs", "cancel", false)] + [InlineData("netratel_job_runs", "delete", true)] + public async Task Tombstones_require_an_exact_successful_owned_receipt_and_remain_hidden_to_ordinary_operations(string tool, string operation, bool legacy) + { + await using var db = CreateDb(); + var delegation = await SeedDeletedReceiptAsync(db, tool, operation); + if (legacy) + { + (await db.McpOperatorIdempotencyRecords.SingleAsync()).ResultReference = delegation.ObjectReference; + await db.SaveChangesAsync(); + } + var resolver = new McpOperationObjectTargetResolver(db); + + (await resolver.MatchesDelegationAsync(delegation, default)).Should().BeTrue(); + (await resolver.MatchesDelegationAsync(delegation with { ReplayBinding = null }, default)).Should().BeFalse(); + (await resolver.MatchesDelegationAsync(delegation with { Operation = "get", ReplayBinding = null }, default)).Should().BeFalse(); + (await resolver.MatchesDelegationAsync(delegation with { TenantId = 43 }, default)).Should().BeFalse(); + (await resolver.MatchesDelegationAsync(delegation with { AgentId = Guid.NewGuid() }, default)).Should().BeFalse(); + (await resolver.MatchesDelegationAsync(delegation with { ObjectTargetResolutionEnabled = false }, default)).Should().BeFalse(); + (await resolver.MatchesDelegationAsync(delegation with { IngressCredentialId = null }, default)).Should().BeFalse(); + } + + [Theory] + [InlineData("expired")] + [InlineData("unconsumed")] + [InlineData("unlinked")] + [InlineData("pending")] + [InlineData("failed")] + [InlineData("incomplete")] + [InlineData("result-object")] + [InlineData("result-shape")] + [InlineData("owner")] + [InlineData("client")] + [InlineData("credential")] + [InlineData("resource")] + [InlineData("instance")] + [InlineData("payload")] + [InlineData("key")] + [InlineData("operation")] + [InlineData("family")] + [InlineData("confirmation-class")] + [InlineData("receipt-owner")] + [InlineData("receipt-client")] + [InlineData("receipt-tenant")] + [InlineData("receipt-agent")] + [InlineData("receipt-target")] + [InlineData("receipt-policy")] + [InlineData("receipt-operation")] + [InlineData("receipt-key")] + [InlineData("receipt-payload")] + [InlineData("receipt-environment")] + [InlineData("policy-disabled")] + [InlineData("policy-version")] + [InlineData("policy-expired")] + public async Task A_deleted_object_cannot_be_admitted_by_another_identity_or_a_partial_stale_or_unrelated_receipt(string change) + { + await using var db = CreateDb(); + var delegation = await SeedDeletedReceiptAsync(db, "netratel_jobs", "delete"); + var plan = await db.McpOperatorConfirmationPlans.SingleAsync(); + var receipt = await db.McpOperatorIdempotencyRecords.SingleAsync(); + var policy = await db.McpOperatorPolicies.SingleAsync(); + switch (change) + { + case "expired": plan.ExpiresAtUtc = DateTimeOffset.UtcNow.AddSeconds(-1); break; + case "unconsumed": plan.ConsumedAtUtc = null; break; + case "unlinked": plan.ConsumedIdempotencyId = Guid.NewGuid(); break; + case "pending": receipt.Outcome = McpOperatorIdempotencyOutcome.Pending; break; + case "failed": receipt.Outcome = McpOperatorIdempotencyOutcome.Failed; break; + case "incomplete": receipt.CompletedAtUtc = null; break; + case "result-object": receipt.ResultReference = "job-mutation:2001:2:"; break; + case "result-shape": receipt.ResultReference = "job-mutation:1001:invalid:"; break; + case "owner": delegation = delegation with { Identity = delegation.Identity with { Subject = "another-owner" } }; break; + case "client": delegation = delegation with { Identity = delegation.Identity with { ClientId = "another-client" }, IngressCredentialId = "another-client" }; break; + case "credential": delegation = delegation with { IngressCredentialId = "another-credential" }; break; + case "resource": delegation = delegation with { Resource = "https://other.example.test/mcp" }; break; + case "instance": delegation = delegation with { Instance = "dev" }; break; + case "payload": delegation = delegation with { ReplayBinding = delegation.ReplayBinding! with { PayloadHash = McpOperatorReplayBinding.Hash("another-payload") } }; break; + case "key": delegation = delegation with { ReplayBinding = delegation.ReplayBinding! with { IdempotencyKeyHash = McpOperatorReplayBinding.Hash("another-key") } }; break; + case "operation": plan.Operation = "netratel_jobs/update"; break; + case "family": plan.OperationFamily = McpOperatorOperationFamily.JobExecution; break; + case "confirmation-class": plan.ConfirmationClass = McpOperatorConfirmationClass.None; break; + case "receipt-owner": receipt.Subject = "another-owner"; break; + case "receipt-client": receipt.ClientId = "another-client"; break; + case "receipt-tenant": receipt.TenantId++; break; + case "receipt-agent": receipt.AgentId = Guid.NewGuid(); break; + case "receipt-target": receipt.TargetSetDigest = "another-target"; break; + case "receipt-policy": receipt.PolicyVersion++; break; + case "receipt-operation": receipt.Operation = "netratel_jobs/update"; break; + case "receipt-key": receipt.IdempotencyKey = "another-key"; break; + case "receipt-payload": receipt.PayloadHash = McpOperatorReplayBinding.Hash("another-payload"); break; + case "receipt-environment": receipt.Environment = McpOperatorEnvironment.Development; break; + case "policy-disabled": policy.DisabledAtUtc = DateTimeOffset.UtcNow; break; + case "policy-version": policy.Version++; break; + case "policy-expired": policy.ExpiresAtUtc = DateTimeOffset.UtcNow.AddSeconds(-1); break; + default: throw new ArgumentOutOfRangeException(nameof(change)); + } + await db.SaveChangesAsync(); + + (await new McpOperationObjectTargetResolver(db).MatchesDelegationAsync(delegation, default)).Should().BeFalse(); + } + + [Fact] + public async Task Deleted_run_receipts_cannot_borrow_another_job_records_identity_or_result() + { + await using var db = CreateDb(); + var delegation = await SeedDeletedReceiptAsync(db, "netratel_job_runs", "delete"); + var receipt = await db.McpOperatorIdempotencyRecords.SingleAsync(); + receipt.ResultReference = "job-run-action:1001:Succeeded"; + await db.SaveChangesAsync(); + var resolver = new McpOperationObjectTargetResolver(db); + (await resolver.MatchesDelegationAsync(delegation, default)).Should().BeFalse(); + receipt.ResultReference = "job-run-action:1002:Succeeded"; + (await db.McpOperatorJobRuns.SingleAsync()).JobRecordId = Guid.NewGuid(); + await db.SaveChangesAsync(); + (await resolver.MatchesDelegationAsync(delegation, default)).Should().BeFalse(); + } + + private static async Task SeedDeletedReceiptAsync(OrchestratorDbContext db, string tool, string operation) + { + var now = DateTimeOffset.UtcNow; + var agentId = Guid.NewGuid(); + var policyId = Guid.NewGuid(); + var receiptId = Guid.NewGuid(); + var jobRecordId = Guid.NewGuid(); + var descriptor = McpOperatorOperationCatalog.Find(tool, operation)!; + var binding = new McpOperatorReplayBinding(McpOperatorReplayBinding.Hash("original-plan"), + McpOperatorReplayBinding.Hash("original-idempotency-key"), McpOperatorReplayBinding.Hash("original-canonical-payload")); + var delegation = Delegation(tool, operation, tool == "netratel_jobs" ? "1001" : "1002", 42, agentId) with + { ObjectTargetResolutionEnabled = true, IngressCredentialId = "credential", ReplayBinding = binding }; + db.McpOperatorJobs.Add(new McpOperatorJobRecord + { + Id = jobRecordId, JobId = 1001, TenantId = 42, AgentId = agentId, Subject = "local-owner", ClientId = "credential", + McpResource = delegation.Resource!, McpInstance = "prod", TargetSetDigest = "owned-target", DeletedAtUtc = now, Version = 2 + }); + db.McpOperatorJobRuns.Add(new McpOperatorJobRunRecord + { + Id = Guid.NewGuid(), JobRunId = 1002, JobRecordId = jobRecordId, JobId = 1001, TenantId = 42, AgentId = agentId, + TargetSetDigest = "owned-target", DeletedAtUtc = now, Version = 2 + }); + db.McpOperatorPolicies.Add(new McpOperatorPolicyRecord + { + Id = policyId, Environment = McpOperatorEnvironment.Production, Effect = McpOperatorPolicyEffect.Allow, + OperationFamily = descriptor.OperationFamily, Version = 3 + }); + db.McpOperatorConfirmationPlans.Add(new McpOperatorConfirmationPlanRecord + { + Id = Guid.NewGuid(), TokenHash = binding.PlanTokenHash, Environment = McpOperatorEnvironment.Production, + Subject = "local-owner", ClientId = "credential", McpResource = delegation.Resource, McpInstance = "prod", + TenantId = 42, AgentId = agentId, TargetSetDigest = "owned-target", PolicyId = policyId, PolicyVersion = 3, + OperationFamily = descriptor.OperationFamily, Operation = $"{tool}/{operation}", ConfirmationClass = descriptor.ConfirmationClass, + PayloadHash = binding.PayloadHash, IdempotencyKey = "original-idempotency-key", CreatedAtUtc = now.AddMinutes(-1), + ExpiresAtUtc = now.AddMinutes(2), ConsumedAtUtc = now, ConsumedIdempotencyId = receiptId, Version = 2 + }); + db.McpOperatorIdempotencyRecords.Add(new McpOperatorIdempotencyRecord + { + Id = receiptId, Environment = McpOperatorEnvironment.Production, Subject = "local-owner", ClientId = "credential", + TenantId = 42, AgentId = agentId, TargetSetDigest = "owned-target", PolicyId = policyId, PolicyVersion = 3, + OperationFamily = descriptor.OperationFamily, Operation = $"{tool}/{operation}", PayloadHash = binding.PayloadHash, + IdempotencyKey = "original-idempotency-key", Outcome = McpOperatorIdempotencyOutcome.Succeeded, CreatedAtUtc = now, + CompletedAtUtc = now, ResultReference = tool == "netratel_jobs" ? "job-mutation:1001:2:17" : "job-run-action:1002:Succeeded", Version = 2 + }); + await db.SaveChangesAsync(); + return delegation; + } + [Fact] public async Task Policy_reference_ownership_is_resolved_from_persistence_including_tenant_only_selectors() { diff --git a/src/NetRatel/NetRatel.Tests/Mcp/NetRatelMcpHttpToolTests.cs b/src/NetRatel/NetRatel.Tests/Mcp/NetRatelMcpHttpToolTests.cs index ed4e6cac..b847b68f 100644 --- a/src/NetRatel/NetRatel.Tests/Mcp/NetRatelMcpHttpToolTests.cs +++ b/src/NetRatel/NetRatel.Tests/Mcp/NetRatelMcpHttpToolTests.cs @@ -1,6 +1,18 @@ using AwesomeAssertions; +using Microsoft.AspNetCore.Http; +using Microsoft.IdentityModel.Tokens; +using NetRatel.API.Endpoints.Client; +using NetRatel.Application.Operations; using NetRatel.Mcp.Core; +using NetRatel.Mcp.Http; +using NetRatel.Shared.Operations; +using System.IdentityModel.Tokens.Jwt; using System.Net; +using System.Net.Http.Json; +using System.Security.Claims; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; using System.Text.Json.Nodes; using Xunit; @@ -8,6 +20,223 @@ namespace NetRatel.Tests.Mcp; public sealed class NetRatelMcpHttpToolTests { + private const string ReplayPlanToken = "pppppppppppppppppppppppppppppppp"; + private const string ReplayIdempotencyKey = "iiiiiiiiiiiiiiiiiiiiiiiiiiiiiiii"; + + [Theory] + [InlineData("create")] + [InlineData("update")] + [InlineData("delete")] + [InlineData("param_add")] + [InlineData("param_update")] + [InlineData("param_delete")] + [InlineData("step_add")] + [InlineData("step_update")] + [InlineData("step_reorder")] + [InlineData("step_delete")] + public void Replay_hash_preserves_existing_job_confirmation_serialization_for_typed_and_mcp_inputs(string action) + { + foreach (var optionalFields in new[] { false, true }) + { + var request = ReplayMutationRequest(action, optionalFields); + // This is the pre-shared-helper serialization that existing plans + // already persist. Compare actual API DTOs, including null/default + // nested properties, with the host's decimal-string request form. + var previousJson = JsonSerializer.Serialize(new + { + action, request.JobId, request.ExpectedVersion, request.Job, request.ParameterId, + request.Parameter, request.StepId, request.Step, request.Ordinal + }); + var expected = Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(previousJson))); + McpOperatorJobConfirmationPayload.MutationHash(action, JsonSerializer.SerializeToElement(request)).Should().Be(expected); + + var hostRequest = (JsonObject)JsonSerializer.SerializeToNode(request, new JsonSerializerOptions(JsonSerializerDefaults.Web))!; + foreach (var property in hostRequest.Where(property => property.Value is null).ToArray()) + hostRequest.Remove(property.Key); + hostRequest["tenantId"] = 42; + hostRequest["agentId"] = "e85d8a10-b2aa-4f11-93c0-a4d2f3a400e0"; + foreach (var field in new[] { "jobId", "expectedVersion", "parameterId", "stepId", "ordinal" }) + if (hostRequest[field] is { } value) hostRequest[field] = value.ToJsonString(); + foreach (var field in new[] { "job", "parameter", "step" }) + if (hostRequest[field] is JsonObject nested) + { + foreach (var property in nested.Where(property => property.Value is null).ToArray()) + nested.Remove(property.Key); + foreach (var identifier in new[] { "id", "ordinal", "scriptId", "scriptVersion" }) + if (nested[identifier] is { } value) nested[identifier] = value.ToJsonString(); + } + + McpOperatorJobConfirmationPayload.TryCreateBinding("netratel_jobs", action, Request(hostRequest), out var binding).Should().BeTrue(); + binding!.PayloadHash.Should().Be(expected); + binding.PlanTokenHash.Should().Be(Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(ReplayPlanToken)))); + binding.IdempotencyKeyHash.Should().Be(Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(ReplayIdempotencyKey)))); + } + } + + [Theory] + [InlineData("cancel")] + [InlineData("delete")] + public void Replay_hash_preserves_job_run_reason_normalization_and_decimal_precision(string action) + { + const ulong runId = 9007199254740993; + foreach (var reason in new[] { null, "", " reviewed cleanup ", "has\ncontrol", new string('x', 129) }) + { + var previousReason = string.IsNullOrWhiteSpace(reason) ? "operator_requested" : + reason.Trim().Length > 128 || reason.Any(char.IsControl) ? "operator_requested" : reason.Trim(); + var expected = Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(JsonSerializer.Serialize(new { action, runId, reason = previousReason })))); + McpOperatorJobConfirmationPayload.RunActionHash(action, runId, reason).Should().Be(expected); + var request = JsonSerializer.SerializeToElement(new + { + tenantId = 42, agentId = Guid.NewGuid(), jobRunId = runId.ToString(), reason, + planToken = ReplayPlanToken, idempotencyKey = ReplayIdempotencyKey + }); + McpOperatorJobConfirmationPayload.TryCreateBinding("netratel_job_runs", action, request, out var binding).Should().BeTrue(); + binding!.PayloadHash.Should().Be(expected); + } + } + + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task Replay_proof_propagates_only_hashes_for_confirmed_http_calls(bool localCredentialMode, bool confirmed) + { + var options = ReplayDelegationOptions(); + var tokens = new McpOperatorDelegationTokenService(options); + var host = new NetRatelMcpHostContext(new NetRatelMcpTarget("prod", new Uri("https://api.example.test"), + new Uri("https://mcp.example.test/mcp"), NetRatelMcpCatalog.Revision), NetRatelMcpTransport.StreamableHttp, "test", operatorSurfaceEnabled: true); + var httpOptions = new NetRatelMcpHttpOptions { PublicResourceUri = "https://mcp.example.test/mcp", LocalCredentialMode = localCredentialMode }; + var context = new McpOperatorDelegationContext(); + var http = new DefaultHttpContext(); + http.Request.Headers.Authorization = "Bearer nrt_ic_synthetic-replay-credential"; + var caller = new ClaimsPrincipal(new ClaimsIdentity([new Claim("sub", "replay-owner")], "test")); + McpOperatorDelegation? pairing = null; + using var clients = new ReplayExchangeClients(request => + { + request.RequestUri!.AbsolutePath.Should().Be("/api/v2/mcp/local-delegation/exchange"); + tokens.TryValidate(request.Headers.GetValues("X-NetRatel-Mcp-Pairing").Single(), out pairing).Should().BeTrue(); + pairing!.Identity.ClientId.Should().Be("netratel-local-gateway"); + return new HttpResponseMessage(HttpStatusCode.OK) { Content = JsonContent.Create(new { assertion = "exchanged-assertion" }) }; + }); + var propagation = new McpOperatorDelegationPropagation(options, tokens, context, host, httpOptions, + new HttpContextAccessor { HttpContext = http }, clients, new McpLocalCredentialPairingService(tokens, host, httpOptions)); + var requestBody = JsonSerializer.SerializeToElement(new + { + tenantId = 42, agentId = Guid.NewGuid(), jobId = "9007199254740993", expectedVersion = "3", + planToken = ReplayPlanToken, idempotencyKey = ReplayIdempotencyKey + }); + var arguments = new Dictionary + { + ["operation"] = JsonSerializer.SerializeToElement("delete"), ["confirm"] = JsonSerializer.SerializeToElement(confirmed), + ["request"] = requestBody + }; + + var assertion = await propagation.CreateAssertionAsync(caller, "netratel_jobs", arguments, default); + + McpOperatorDelegation delegation; + if (localCredentialMode) + { + assertion.Should().Be("exchanged-assertion"); + pairing.Should().NotBeNull(); + delegation = pairing!; + } + else + { + tokens.TryValidate(assertion!, out var decoded).Should().BeTrue(); + delegation = decoded!; + var payload = new JwtSecurityTokenHandler().ReadJwtToken(assertion!).Payload.SerializeToJson(); + payload.Should().NotContain(ReplayPlanToken).And.NotContain(ReplayIdempotencyKey).And.NotContain("expectedVersion"); + } + delegation.ObjectReference.Should().Be("9007199254740993"); + if (confirmed) + { + McpOperatorJobConfirmationPayload.TryCreateBinding("netratel_jobs", "delete", requestBody, out var expected).Should().BeTrue(); + delegation.ReplayBinding.Should().Be(expected); + } + else delegation.ReplayBinding.Should().BeNull(); + } + + [Theory] + [InlineData("partial")] + [InlineData("duplicate")] + [InlineData("malformed")] + [InlineData("read-operation")] + public void Replay_proof_rejects_incomplete_duplicate_invalid_or_read_operation_signed_claims(string change) + { + var options = ReplayDelegationOptions(); + var tokens = new McpOperatorDelegationTokenService(options); + var binding = new McpOperatorReplayBinding(new string('A', 64), new string('B', 64), new string('C', 64)); + var valid = tokens.Create(new McpOperatorDelegationIdentity("replay-owner", "replay-client", null, [], [], []), + new McpOperatorDelegationRequest("netratel_jobs", "delete", "replay-request", "https://mcp.example.test/mcp", "prod", 42, Guid.NewGuid()) + { ObjectReference = "901", ObjectTargetResolutionEnabled = true, ReplayBinding = binding }); + tokens.TryValidate(valid, out var decoded).Should().BeTrue(); + decoded!.ReplayBinding.Should().Be(binding); + var jwt = new JwtSecurityTokenHandler().ReadJwtToken(valid); + switch (change) + { + case "partial": jwt.Payload.Remove("mcp_replay_key_hash"); break; + case "duplicate": jwt.Payload["mcp_replay_plan_hash"] = new[] { binding.PlanTokenHash, binding.PlanTokenHash }; break; + case "malformed": jwt.Payload["mcp_replay_payload_hash"] = new string('Z', 64); break; + case "read-operation": jwt.Payload["mcp_operation"] = "get"; break; + } + var key = new SymmetricSecurityKey(Convert.FromBase64String(options.SharedKeyBase64!)) { KeyId = options.KeyId }; + var invalid = new JwtSecurityTokenHandler().WriteToken(new JwtSecurityToken( + new JwtHeader(new SigningCredentials(key, SecurityAlgorithms.HmacSha256)), jwt.Payload)); + + tokens.TryValidate(invalid, out _).Should().BeFalse(); + } + + [Theory] + [InlineData("\"jobId\":\"901\",\"JobId\":\"902\"")] + [InlineData("\"jobId\":\"901\",\"jobId\":\"901\"")] + [InlineData("\"jobId\":\"901\",\"job\":{\"name\":\"one\",\"Name\":\"two\",\"folderPath\":\"/\"}")] + public void Replay_hash_rejects_case_colliding_or_duplicate_request_properties(string properties) + { + using var document = JsonDocument.Parse("{" + properties + ",\"expectedVersion\":1,\"planToken\":\"" + ReplayPlanToken + + "\",\"idempotencyKey\":\"" + ReplayIdempotencyKey + "\"}"); + + McpOperatorJobConfirmationPayload.TryCreateBinding("netratel_jobs", "delete", document.RootElement, out var binding).Should().BeFalse(); + binding.Should().BeNull(); + } + + private static McpOperatorJobMutationRequest ReplayMutationRequest(string action, bool optionalFields) + { + const long jobId = 9007199254740993; + var draft = new McpOperatorJobDraft("reviewed job", "/ops", optionalFields ? "description" : null, optionalFields ? "{\"mode\":\"safe\"}" : null); + var parameter = new McpOperatorJobParameter("region", "string", optionalFields, optionalFields ? "description" : null, + optionalFields ? "north" : null, optionalFields ? ["north", "south"] : null, optionalFields ? "NAMED_REFERENCE" : null, optionalFields ? 301 : null); + var step = new McpOperatorJobStep(optionalFields ? 2 : null, 9007199254740995, 7, new string('A', 64), !optionalFields, optionalFields ? 401 : null); + return new McpOperatorJobMutationRequest( + Job: action is "create" or "update" ? draft : null, JobId: action == "create" ? null : jobId, + ExpectedVersion: action == "create" ? null : 3, ParameterId: action is "param_update" or "param_delete" ? 301 : null, + Parameter: action is "param_add" or "param_update" ? parameter : null, StepId: action is "step_update" or "step_reorder" or "step_delete" ? 401 : null, + Step: action is "step_add" or "step_update" ? step : null, Ordinal: action == "step_reorder" ? 2 : null, + PlanToken: ReplayPlanToken, IdempotencyKey: ReplayIdempotencyKey); + } + + private static McpOperatorDelegationOptions ReplayDelegationOptions() => new() + { + Enabled = true, Issuer = "netratel-mcp-test", Audience = "netratel-api-test", ServicePrincipal = "netratel-mcp-http-test", KeyId = "replay-test", + SharedKeyBase64 = Convert.ToBase64String(Encoding.UTF8.GetBytes("synthetic-replay-delegation-key-at-least-32-bytes")) + }; + + private sealed class ReplayExchangeClients(Func respond) : IHttpClientFactory, IDisposable + { + private readonly HttpClient _client = new(new ReplayExchangeHandler(respond)) { BaseAddress = new Uri("https://api.example.test") }; + public HttpClient CreateClient(string name) => _client; + public void Dispose() => _client.Dispose(); + } + + private sealed class ReplayExchangeHandler(Func respond) : HttpMessageHandler + { + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + return Task.FromResult(respond(request)); + } + } + [Theory] [InlineData("create")] [InlineData("update")]