diff --git a/Directory.Build.props b/Directory.Build.props index 3d151ee9..109a0127 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -1,7 +1,7 @@ 0.1.1 - beta.12 + beta.13 $(VersionPrefix) $(VersionPrefix)-$(VersionSuffix) diff --git a/docs/releases.md b/docs/releases.md index aa9c06e2..4f4d82fc 100644 --- a/docs/releases.md +++ b/docs/releases.md @@ -1,6 +1,17 @@ # Releases -RatelDesk has one repository-owned release line. The root `Directory.Build.props` is the source of truth: maintainers update `VersionPrefix` when preparing the next release. All application projects inherit that value. `VersionSuffix` creates prereleases without changing the release line. The current planned test release is `0.1.1-beta.9`; earlier prereleases remain immutable. +RatelDesk has one repository-owned release line. The root `Directory.Build.props` is the source of truth for `VersionPrefix` and `VersionSuffix`. Applications and release outputs inherit those values. Prereleases remain immutable. + +## 0.1.1-beta.13 — reciprocal link recovery + +Background workers leave live attempts awaiting administrator consent untouched, +preserving the original approval's concurrency revision. Expired unused attempts +still abort and purge temporary pairing material. + +Link recovery reuses short-lived verification and control tokens for the exact +approved peer, tenant, scope and credential revision. It checks current durable +authority on each use and retains the existing rate limits, finite recovery +permissions and rotation deadlines. ## 0.1.1-beta.9 — service credentials and reciprocal NetRatel linking diff --git a/src/Helpdesk.API/Endpoints/ServiceLink/ServiceLinkEndpoints.cs b/src/Helpdesk.API/Endpoints/ServiceLink/ServiceLinkEndpoints.cs index c64d2b9a..6243c5ea 100644 --- a/src/Helpdesk.API/Endpoints/ServiceLink/ServiceLinkEndpoints.cs +++ b/src/Helpdesk.API/Endpoints/ServiceLink/ServiceLinkEndpoints.cs @@ -21,6 +21,7 @@ public static IServiceCollection AddServiceLinkProtocol(this IServiceCollection services.AddOptions().Bind(configuration.GetSection(ServiceLinkOptions.SectionName)).ValidateOnStart(); services.AddSingleton, ServiceLinkOptionsValidator>(); services.AddScoped(); + services.AddScoped(); services.AddHttpClient(client => client.Timeout = TimeSpan.FromSeconds(25)) .ConfigurePrimaryHttpMessageHandler(provider => IntegrationSafeHttpMessageHandler.CreateServiceLink( currentAllowPrivateHttp: () => provider.GetRequiredService>().CurrentValue.AllowPrivateHttp)); diff --git a/src/Helpdesk.Infrastructure/ServiceLink/ServiceLinkCoordinator.cs b/src/Helpdesk.Infrastructure/ServiceLink/ServiceLinkCoordinator.cs index 2720fe76..f49a51ff 100644 --- a/src/Helpdesk.Infrastructure/ServiceLink/ServiceLinkCoordinator.cs +++ b/src/Helpdesk.Infrastructure/ServiceLink/ServiceLinkCoordinator.cs @@ -19,6 +19,7 @@ public sealed partial class ServiceLinkCoordinator( HelpdeskDbContext db, IServicePrincipalRegistry registry, ICurrentUserAccessService accessService, IIntegrationProviderSettingsService providers, ServiceLinkTransport transport, IDataProtectionProvider protection, IOptions options, IOptionsMonitor identityOptions, TimeProvider clock, + ServiceLinkProtocolTokenCache protocolTokens, IOptionsMonitor? currentOptions = null) { private ServiceLinkOptions settings => currentOptions?.CurrentValue ?? options.Value; diff --git a/src/Helpdesk.Infrastructure/ServiceLink/ServiceLinkProtocolTokenCache.cs b/src/Helpdesk.Infrastructure/ServiceLink/ServiceLinkProtocolTokenCache.cs new file mode 100644 index 00000000..830292a5 --- /dev/null +++ b/src/Helpdesk.Infrastructure/ServiceLink/ServiceLinkProtocolTokenCache.cs @@ -0,0 +1,89 @@ +using System.Security.Cryptography; +using System.Text; +using Microsoft.Extensions.Caching.Memory; +using Helpdesk.Shared.ServiceLink; + +namespace Helpdesk.Infrastructure.ServiceLink; + +/// Server-only immutable control-token binding; never serialize its credential into a response or log. +public sealed record ServiceLinkProtocolTokenContext(string AttemptId, string LinkId, long LinkRevision, + string GrantHash, string DescriptorHash, string DirectionId, string Mode, int? OutboundProfileRevision, + ServiceLinkMetadata Local, ServiceLinkMetadata Peer, ServiceDirectionalCredential Credential, + string? RotationId, long? AuthorityExpiresAtUnixSeconds); + +/// A bounded control/verification cache. The caller must resolve current durable authority on every use. +public sealed class ServiceLinkProtocolTokenCache(ServiceLinkTransport transport, IMemoryCache cache, TimeProvider clock) +{ + // Fixed stripes share acquisition across transient worker scopes without retaining a lock per link. + private static readonly SemaphoreSlim[] Stripes = Enumerable.Range(0, 64).Select(_ => new SemaphoreSlim(1, 1)).ToArray(); + private sealed record CachedToken(string Token, DateTimeOffset ReuseUntilUtc); + + public async Task GetAsync(Func> resolveCurrent, + string scope, CancellationToken ct) + { + ct.ThrowIfCancellationRequested(); + if (scope is not (ServiceLinkContract.ControlScope or ServiceLinkContract.VerifyScope)) + throw new ServiceLinkProtocolException(403, "protocol-scope-required", "Only an exact control or verification scope may use this cache."); + var captured = await resolveCurrent(ct); + Validate(captured, ct); + var key = Key(captured, scope); + var stripe = Stripes[(key.GetHashCode(StringComparison.Ordinal) & int.MaxValue) % Stripes.Length]; + await stripe.WaitAsync(ct); + try + { + // Waiting for another scope's token cannot preserve stale authority. + var current = await resolveCurrent(ct); + Validate(current, ct); + RequireSame(key, current, scope); + if (cache.TryGetValue(key, out var cached) && cached is not null) + { + if (cached.ReuseUntilUtc > clock.GetUtcNow()) return cached.Token; + cache.Remove(key); + } + var requestStartedAt = clock.GetUtcNow(); + var issued = await transport.AcquireTokenAsync(current.Credential, scope, ct); + var after = await resolveCurrent(ct); + Validate(after, ct); + RequireSame(key, after, scope); + var now = clock.GetUtcNow(); + if (requestStartedAt.AddSeconds(issued.ExpiresIn) <= now) + throw new ServiceLinkProtocolException(422, "expired-token-response", "The service token lifetime elapsed before its response could be used."); + var reuseUntil = requestStartedAt.AddSeconds(Math.Min(45, issued.ExpiresIn - 5)); + if (after.AuthorityExpiresAtUnixSeconds is { } deadline) + reuseUntil = DateTimeOffset.FromUnixTimeSeconds(deadline) < reuseUntil + ? DateTimeOffset.FromUnixTimeSeconds(deadline) : reuseUntil; + var remaining = reuseUntil - now; + if (remaining > TimeSpan.Zero) cache.Set(key, new CachedToken(issued.AccessToken, reuseUntil), remaining); + return issued.AccessToken; + } + finally { stripe.Release(); } + } + + private void Validate(ServiceLinkProtocolTokenContext context, CancellationToken ct) + { + ct.ThrowIfCancellationRequested(); + if (context.AuthorityExpiresAtUnixSeconds is { } expires && expires <= clock.GetUtcNow().ToUnixTimeSeconds()) + throw new ServiceLinkProtocolException(403, "protocol-authority-expired", "The retained control authority has expired."); + // A cache hit still observes an opt-in removed from current typed options. + transport.ValidateTokenEndpoint(context.Credential); + } + + private static void RequireSame(string capturedKey, ServiceLinkProtocolTokenContext current, string scope) + { + if (Key(current, scope) != capturedKey) + throw new ServiceLinkProtocolException(409, "protocol-profile-conflict", "The current protocol credential or approved link changed during acquisition."); + } + + private static string Key(ServiceLinkProtocolTokenContext context, string scope) => "rateldesk-protocol-token/" + + ServiceLinkCanonicalJson.HashObject(new + { + context.AttemptId, context.LinkId, context.LinkRevision, context.GrantHash, context.DescriptorHash, + context.DirectionId, context.Mode, context.OutboundProfileRevision, context.Local, context.Peer, + context.RotationId, context.AuthorityExpiresAtUnixSeconds, RequestedScope = scope, + Credential = context.Credential with + { + ClientSecret = Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(context.Credential.ClientSecret))), + Scopes = ServiceLinkValidation.Set(context.Credential.Scopes) + } + }); +} diff --git a/src/Helpdesk.Infrastructure/ServiceLink/ServiceLinkProtocolTokens.cs b/src/Helpdesk.Infrastructure/ServiceLink/ServiceLinkProtocolTokens.cs new file mode 100644 index 00000000..698e378e --- /dev/null +++ b/src/Helpdesk.Infrastructure/ServiceLink/ServiceLinkProtocolTokens.cs @@ -0,0 +1,100 @@ +using Microsoft.EntityFrameworkCore; +using Helpdesk.Infrastructure.ServiceIdentity; +using Helpdesk.Shared.ServiceLink; +using static Helpdesk.Infrastructure.ServiceLink.ServiceLinkValidation; + +namespace Helpdesk.Infrastructure.ServiceLink; + +public sealed partial class ServiceLinkCoordinator +{ + private Task ProtocolToken(ServiceLinkAttempt captured, string scope, string route, + ServiceLinkLifecycleRequest? request, ServiceDirectionalCredential credential, CancellationToken ct, bool candidateCredential = false) => + protocolTokens.GetAsync(token => CurrentProtocolTokenContext(captured, scope, route, request, credential, candidateCredential, token), scope, ct); + + private async Task CurrentProtocolTokenContext(ServiceLinkAttempt captured, string scope, + string route, ServiceLinkLifecycleRequest? request, ServiceDirectionalCredential credential, bool candidateCredential, CancellationToken ct) + { + var current = await db.Set().AsNoTracking().SingleOrDefaultAsync(x => x.AttemptId == captured.AttemptId, ct); + Require(current is not null && current.LinkId == captured.LinkId && current.LinkRevision == captured.LinkRevision && + current.GrantHash == captured.GrantHash && current.DescriptorHash == captured.DescriptorHash && + current.Role == captured.Role && current.LocalTenantId == captured.LocalTenantId && + current.PeerInstanceId == captured.PeerInstanceId && current.PeerTenantId == captured.PeerTenantId && + current.OutboundProfileRevision == captured.OutboundProfileRevision && current.ProtectedOutboundCredential is not null, + "protocol-profile-conflict", "The current approved link or outbound profile changed.", 409); + var a = current!; + var summary = Summary(a); + Require(summary.AttemptId == a.AttemptId && summary.LinkId == a.LinkId && summary.ProposedLinkRevision == a.LinkRevision && + summary.DescriptorHash == a.DescriptorHash && ServiceLinkPayloadNormalization.SummaryHashMatches(summary, a.GrantHash!) && + ServiceLinkAuthority.LocalIdentityMatches(summary, a.Role, issuer, settings), + "protocol-authority-unavailable", "The approved local identity is no longer current.", 403); + var grant = OutboundGrant(a); + var peer = Peer(a); + Credential(credential, grant, peer); + var principal = await db.Set().AsNoTracking().SingleOrDefaultAsync(x => x.Id == a.InboundPrincipalId, ct); + var secret = principal is null ? null : await db.Set().AsNoTracking().SingleOrDefaultAsync(x => + x.ServicePrincipalId == principal.Id && x.CredentialRevision == principal.CurrentCredentialRevision, ct); + var customers = principal is null ? Array.Empty() : ServicePrincipalRegistry.ReadArray(principal.CustomerIdsJson); + Require(principal is not null && secret is not null && secret.Status != "revoked" && + secret.ExpiresAtUtc > clock.GetUtcNow() && (secret.RetireAtUtc is null || secret.RetireAtUtc > clock.GetUtcNow()) && + registry.PermittedScopes(principal, secret).Contains(scope, StringComparer.Ordinal) && + await db.Organizations.AsNoTracking().AnyAsync(x => x.Id == principal.OrganizationId && + x.State == Helpdesk.Shared.Models.EntityState.Enabled, ct) && + await db.Customers.AsNoTracking().CountAsync(x => customers.Contains(x.Id) && x.OrganizationId == principal.OrganizationId && + x.State == Helpdesk.Shared.Models.EntityState.Enabled, ct) == customers.Length && + await registry.CanIssueScopesAsync(new(principal, secret), [scope], ct), + "protocol-authority-unavailable", "The exact local registration has no current retained protocol authority.", 403); + + var terminal = a.Decision == "abort" || a.LifecycleState is "revocation_pending" or "revoked" or "expired" or "failed"; + long? authorityDeadline = secret!.ExpiresAtUtc.ToUnixTimeSeconds(); + if (secret.RetireAtUtc is { } retirement) authorityDeadline = Math.Min(authorityDeadline.Value, retirement.ToUnixTimeSeconds()); + if (terminal) + { + Require(scope == ServiceLinkContract.ControlScope && a.TerminalControlExpiresAtUnixSeconds > Now && + (a.LifecycleState is "revocation_pending" or "revoked" + ? route == "revoke" || route == "ack" && request?.AckPhase == "revoked" || route == "status" + : route is "abort" or "status"), + "terminal-control-restricted", "Only the original finite terminal control recovery may continue.", 403); + authorityDeadline = Math.Min(authorityDeadline.Value, a.TerminalControlExpiresAtUnixSeconds!.Value); + } + else Require(a.LifecycleState is not ("awaiting_approval" or "failed" or "expired" or "revoked"), + "protocol-authority-unavailable", "This attempt has no prepared protocol authority.", 403); + + var outbound = Read(Unprotect(a, "outbound-credential", a.ProtectedOutboundCredential!)); + var accepted = ServiceLinkCanonicalJson.HashObject(ServiceLinkPayloadNormalization.Credential(outbound)) == + ServiceLinkCanonicalJson.HashObject(ServiceLinkPayloadNormalization.Credential(credential)); + var candidate = candidateCredential; + if (candidate) + { + Require(request?.RotationId is not null && + (route == "verify" || route == "rotate" && request.RotationPhase is "verified" or "switched" or "abort"), + "candidate-control-restricted", "A candidate may only complete its own pending rotation.", 403); + var rotation = await db.Set().AsNoTracking().SingleOrDefaultAsync(x => + x.LinkId == a.LinkId && x.RotationId == request!.RotationId && !x.IsIssuer, ct); + Require(!terminal && rotation is not null && rotation.DirectionId == grant.DirectionId && + rotation.SuccessorCredentialRevision == credential.CredentialRevision && + rotation.RotationState != "aborted", "candidate-control-restricted", "The candidate belongs to another rotation.", 403); + var durableCandidate = rotation!.ProtectedCandidate is null ? null : Read( + Unprotect(a, "rotation-candidate/" + rotation.RotationId, rotation.ProtectedCandidate)); + // After a durable caller switch, escrow may be gone while an unfinished + // exact switched journal still needs the already installed successor. + var pendingCandidate = rotation.ActiveRotationKey is not null && rotation.RotationState != "completed" && + durableCandidate is not null && ServiceLinkCanonicalJson.HashObject(ServiceLinkPayloadNormalization.Credential(durableCandidate)) == + ServiceLinkCanonicalJson.HashObject(ServiceLinkPayloadNormalization.Credential(credential)); + var retainedSwitch = accepted && rotation.CallerSwitchRevision is not null && rotation.ActivateDecisionId is not null && + route == "rotate" && request!.RotationPhase == "switched" && + await db.Set().AsNoTracking().AnyAsync(x => x.LinkId == a.LinkId && + x.OperationId == request.OperationId && x.Outbound && !x.Completed && x.Kind == "rotate-switched/" + rotation.RotationId, ct); + Require(pendingCandidate || retainedSwitch, + "candidate-control-restricted", "The current candidate or installed successor differs from this rotation.", 403); + if (rotation.ActivateDecisionId is null && request!.RotationPhase != "abort") + { + Require(rotation.OfferExpiresAtUnixSeconds > Now, "candidate-control-expired", "The unactivated candidate offer expired.", 403); + authorityDeadline = Math.Min(authorityDeadline.Value, rotation.OfferExpiresAtUnixSeconds!.Value); + } + } + else Require(accepted, "protocol-profile-conflict", "The current outbound credential differs from the captured credential.", 409); + return new(a.AttemptId, a.LinkId!, a.LinkRevision, a.GrantHash!, a.DescriptorHash, grant.DirectionId, + a.Decision + "/" + a.LifecycleState + (candidate ? "/candidate" : "/current"), a.OutboundProfileRevision, + Local(a), peer, credential, candidate ? request!.RotationId : null, authorityDeadline); + } +} diff --git a/src/Helpdesk.Infrastructure/ServiceLink/ServiceLinkTransport.cs b/src/Helpdesk.Infrastructure/ServiceLink/ServiceLinkTransport.cs index 3463ead8..ddb80cf9 100644 --- a/src/Helpdesk.Infrastructure/ServiceLink/ServiceLinkTransport.cs +++ b/src/Helpdesk.Infrastructure/ServiceLink/ServiceLinkTransport.cs @@ -13,6 +13,8 @@ public sealed class ServiceLinkProtocolException(int statusCode, string code, st public string Code { get; } = code; } +public sealed record ServiceLinkAccessToken(string AccessToken, int ExpiresIn); + /// Every exchange uses the approved immutable destination and the existing connection-time DNS policy. public sealed class ServiceLinkTransport(HttpClient client, IOptions options, IOptionsMonitor? currentOptions = null) { @@ -31,7 +33,15 @@ public async Task PostAsync(string endpoint, object body, CancellationToke return await SendAsync(message, ct); } - public async Task TokenAsync(ServiceDirectionalCredential credential, string scopes, CancellationToken ct) + public async Task TokenAsync(ServiceDirectionalCredential credential, string scopes, CancellationToken ct) => + (await AcquireTokenAsync(credential, scopes, ct)).AccessToken; + + public void ValidateTokenEndpoint(ServiceDirectionalCredential credential) + { + using var request = Message(HttpMethod.Post, credential.TokenEndpoint); + } + + public async Task AcquireTokenAsync(ServiceDirectionalCredential credential, string scopes, CancellationToken ct) { if (credential.TokenEndpointAuthMethod != "client_secret_post") throw new ServiceLinkProtocolException(422, "unsupported-client-authentication", "The approved peer authentication method is unsupported."); using var message = Message(HttpMethod.Post, credential.TokenEndpoint); @@ -40,11 +50,27 @@ public async Task TokenAsync(ServiceDirectionalCredential credential, st ["grant_type"] = "client_credentials", ["client_id"] = credential.ClientId, ["client_secret"] = credential.ClientSecret, ["scope"] = scopes }); - using var result = await SendAsync(message, ct); - var token = result.RootElement.GetProperty("access_token").GetString(); - if (string.IsNullOrEmpty(token) || token.Length > 32768 || result.RootElement.GetProperty("token_type").GetString()?.Equals("Bearer", StringComparison.OrdinalIgnoreCase) != true) + using var result = await ReadTokenResponseAsync(message, ct); + var root = result.RootElement; + if (root.ValueKind != JsonValueKind.Object || + !root.TryGetProperty("access_token", out var tokenValue) || tokenValue.ValueKind != JsonValueKind.String || + !root.TryGetProperty("token_type", out var typeValue) || typeValue.ValueKind != JsonValueKind.String || + !root.TryGetProperty("expires_in", out var expiryValue) || expiryValue.ValueKind != JsonValueKind.Number || + !expiryValue.TryGetInt32(out var expiresIn) || expiresIn is < 1 or > 900) + throw new ServiceLinkProtocolException(422, "invalid-token-response", "The peer returned an invalid service token response."); + var token = tokenValue.GetString(); + if (string.IsNullOrEmpty(token) || token.Length > 32768 || typeValue.GetString()?.Equals("Bearer", StringComparison.OrdinalIgnoreCase) != true) throw new ServiceLinkProtocolException(422, "invalid-token-response", "The peer returned an invalid service token response."); - return token; + return new(token, expiresIn); + } + + private async Task ReadTokenResponseAsync(HttpRequestMessage message, CancellationToken ct) + { + try { return await SendAsync(message, ct); } + catch (Exception error) when (error is JsonException or DecoderFallbackException) + { + throw new ServiceLinkProtocolException(422, "invalid-token-response", "The peer returned an invalid service token response."); + } } private HttpRequestMessage Message(HttpMethod method, string endpoint, string? bearer = null) diff --git a/src/Helpdesk.Infrastructure/ServiceLink/ServiceLinkWorker.cs b/src/Helpdesk.Infrastructure/ServiceLink/ServiceLinkWorker.cs index 5ca42c74..a03b1a5d 100644 --- a/src/Helpdesk.Infrastructure/ServiceLink/ServiceLinkWorker.cs +++ b/src/Helpdesk.Infrastructure/ServiceLink/ServiceLinkWorker.cs @@ -58,7 +58,8 @@ private async Task SendOperation(ServiceLinkAttempt a, string kind, Require(normalizeScopes || operation.RequestFingerprint == ServiceLinkLifecycleProjection.Hash(route, durable, false), "operation-payload-conflict", "The protected durable request differs from its original operation fingerprint.", 409); var credential = candidate ?? Read(Unprotect(a, "outbound-credential", a.ProtectedOutboundCredential!)); - var token = await transport.TokenAsync(credential, route == "verify" ? ServiceLinkContract.VerifyScope : ServiceLinkContract.ControlScope, ct); + var token = await ProtocolToken(a, route == "verify" ? ServiceLinkContract.VerifyScope : ServiceLinkContract.ControlScope, + route, durable, credential, ct, candidate is not null); using var response = await transport.PostAsync(Endpoint(Peer(a).ServiceLinkEndpoint, "/links/" + a.LinkId + "/" + route), ServiceLinkLifecycleProjection.Build(route, durable, normalizeScopes), ct, token); var result = response.RootElement.Clone(); ValidatePeerResult(a, result); operation.ResponseJson = result.GetRawText(); operation.Completed = true; @@ -95,10 +96,21 @@ public async Task WorkAsync(CancellationToken ct) await CleanupBootstrapEscrow(ct); await CleanupRotationEscrow(ct); if (!settings.Enabled) return; - var ids = await db.Set().AsNoTracking().Where(x => x.NextWorkAtUnixSeconds <= Now && x.LifecycleState != "failed" && x.LifecycleState != "expired" && x.LifecycleState != "revoked").OrderBy(x => x.NextWorkAtUnixSeconds).Select(x => x.AttemptId).Take(20).ToListAsync(ct); + // Live attempts without a local principal are waiting for human consent, + // not background work. Scheduling a no-op would still increment their + // optimistic revision and race the original approval/callback scope. + // Expired attempts remain eligible for the existing abort and escrow cleanup. + var ids = await db.Set().AsNoTracking().Where(x => x.NextWorkAtUnixSeconds <= Now && x.LifecycleState != "failed" && x.LifecycleState != "expired" && x.LifecycleState != "revoked" && + !(x.Decision == "undecided" && x.InboundPrincipalId == null && + (x.LifecycleState == "awaiting_approval" || x.LifecycleState == "approved") && x.ExpiresAtUnixSeconds > Now)) + .OrderBy(x => x.NextWorkAtUnixSeconds).Select(x => x.AttemptId).Take(20).ToListAsync(ct); foreach (var id in ids) { db.ChangeTracker.Clear(); var a = await Attempt(id, ct); + // Recheck after loading: a concurrent local action may have changed + // the state since the bounded candidate query. + if (a.Decision == "undecided" && a.InboundPrincipalId is null && + (a.LifecycleState is "awaiting_approval" or "approved") && a.ExpiresAtUnixSeconds > Now) continue; try { a = await ProgressWithRetry(a, ct); a.LastErrorCode = null; } catch (Exception e) when (e is HttpRequestException or ServiceLinkProtocolException or TaskCanceledException) { a.LastErrorCode = e is ServiceLinkProtocolException protocol ? protocol.Code : "peer-unavailable"; } @@ -139,7 +151,7 @@ private async Task Progress(ServiceLinkAttempt a, CancellationToken ct) if (a.Role == "responder" && a.Decision == "undecided" && a.ProtectedOutboundCredential is not null) { var recoveryCredential = Read(Unprotect(a, "outbound-credential", a.ProtectedOutboundCredential)); - var recoveryToken = await transport.TokenAsync(recoveryCredential, ServiceLinkContract.ControlScope, ct); + var recoveryToken = await ProtocolToken(a, ServiceLinkContract.ControlScope, "status", null, recoveryCredential, ct); using var recoveryStatus = await transport.GetAsync(Endpoint(Peer(a).ServiceLinkEndpoint, "/links/" + a.LinkId + "/status"), ct, recoveryToken); var decision = recoveryStatus.RootElement; ValidatePeerResult(a, decision); Require(String(decision, "descriptor_hash") == a.DescriptorHash && String(decision, "attempt_id") == a.AttemptId && String(decision, "coordinator_instance_id") == Descriptor(a).InitiatorInstanceId, "peer-lifecycle-binding-mismatch", "The recovery decision belongs to another attempt."); @@ -191,7 +203,7 @@ private async Task Progress(ServiceLinkAttempt a, CancellationToken ct) if (a.Decision == "undecided") { var outbound = Read(Unprotect(a, "outbound-credential", a.ProtectedOutboundCredential)); - var token = await transport.TokenAsync(outbound, ServiceLinkContract.ControlScope, ct); + var token = await ProtocolToken(a, ServiceLinkContract.ControlScope, "status", null, outbound, ct); using var peerStatus = await transport.GetAsync(Endpoint(Peer(a).ServiceLinkEndpoint, "/links/" + a.LinkId + "/status"), ct, token); var status = peerStatus.RootElement; ValidatePeerResult(a, status); Require(String(status, "descriptor_hash") == a.DescriptorHash && String(status, "attempt_id") == a.AttemptId && String(status, "coordinator_instance_id") == Descriptor(a).InitiatorInstanceId, "peer-lifecycle-binding-mismatch", "The coordinator's stored decision binding differs."); diff --git a/tests/Helpdesk.Tests/Api/ServiceLinkWorkerConsentPostgresTests.cs b/tests/Helpdesk.Tests/Api/ServiceLinkWorkerConsentPostgresTests.cs new file mode 100644 index 00000000..e22364b2 --- /dev/null +++ b/tests/Helpdesk.Tests/Api/ServiceLinkWorkerConsentPostgresTests.cs @@ -0,0 +1,208 @@ +using System.Net; +using System.Net.Http.Json; +using System.Text.Json; +using Helpdesk.Infrastructure.Persistence; +using Helpdesk.Infrastructure.ServiceIdentity; +using Helpdesk.Infrastructure.ServiceLink; +using Helpdesk.Shared.ServiceLink; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Diagnostics; +using Microsoft.Extensions.DependencyInjection; +using Xunit; + +namespace Helpdesk.Tests.Api; + +public sealed partial class ServiceLinkLifecycleTests +{ + [Theory] + [InlineData("responder-consent")] + [InlineData("initiator-callback")] + [InlineData("initiator-consent")] + public async Task Postgres_worker_does_not_write_a_live_human_consent_attempt_between_its_read_and_save(string step) + { + var interleaving = new HumanConsentWorkerInterleaving(step); + await using var peer = await NetRatelServiceLinkContractPeer.CreateAsync(); + await using var local = await LocalAsync(postgres: true, saveInterceptor: interleaving); + string attemptId; + ServiceLinkRequestDescriptor? descriptor = null; + ServiceLinkCallbackRequest? callback = null; + if (step == "responder-consent") + { + descriptor = await PrepareResponderConsentAsync(local, peer); + attemptId = descriptor.AttemptId; + } + else + { + var start = await StartAsync(local, peer); + attemptId = start.AttemptId; + callback = await peer.ApproveAsync(local.BaseUrl, attemptId, BrowserState(start), SessionBinding); + if (step == "initiator-consent") + { + using var reviewed = await local.AdminAsync(HttpMethod.Post, "/api/v1/admin/service-links/callback", callback); + Assert.Equal(HttpStatusCode.OK, reviewed.StatusCode); + } + } + interleaving.Arm(local.Services, attemptId); + using var accepted = step == "responder-consent" + ? await local.AdminAsync(HttpMethod.Post, "/api/v1/admin/service-links/remote-approve", + new ServiceLinkRemoteApproveRequest(attemptId, local.OrganizationId, descriptor!.RequestedGrants, peer.BrowserState)) + : step == "initiator-callback" + ? await local.AdminAsync(HttpMethod.Post, "/api/v1/admin/service-links/callback", callback) + : await local.AdminAsync(HttpMethod.Post, "/api/v1/admin/service-links/attempts/" + attemptId + "/approve", + new ServiceLinkLocalApproveRequest(peer.GrantHash, SessionBinding)); + Assert.Equal(HttpStatusCode.OK, accepted.StatusCode); + Assert.Equal(1, interleaving.Executions); + Assert.NotEqual(interleaving.HumanContextId, interleaving.WorkerContextId); + interleaving.AssertAttemptUnchanged(); + await using var final = local.Services.CreateAsyncScope(); + var db = final.ServiceProvider.GetRequiredService(); + var approved = await db.Set().AsNoTracking().SingleAsync(a => a.AttemptId == attemptId); + Assert.Equal("approved", approved.LifecycleState); + Assert.Equal("undecided", approved.Decision); + Assert.False(approved.LocalInboundActive); + Assert.False(approved.LocalBusinessSenderEnabled); + Assert.NotNull(approved.GrantHash); + var expectedPrincipals = step == "initiator-callback" ? 0 : 1; + Assert.Equal(expectedPrincipals, await db.Set().CountAsync()); + Assert.Equal(expectedPrincipals, await db.Set().CountAsync()); + Assert.Equal(expectedPrincipals, await db.IncidentReceiverPrincipalBindings.CountAsync()); + Assert.All(await db.IncidentReceiverPrincipalBindings.ToListAsync(), binding => Assert.False(binding.IsEnabled)); + Assert.Equal(0, await db.Set().CountAsync()); + Assert.Equal(0, await db.Set().CountAsync()); + Assert.Equal(0, await db.Set().CountAsync()); + } + + [Theory] + [InlineData("initiator-awaiting")] + [InlineData("responder-awaiting")] + [InlineData("initiator-approved")] + public async Task Postgres_worker_expires_human_waits_without_extending_deadlines_or_creating_authority(string state) + { + await using var peer = await NetRatelServiceLinkContractPeer.CreateAsync(); + await using var local = await LocalAsync(postgres: true); + string attemptId; + if (state == "responder-awaiting") attemptId = (await PrepareResponderConsentAsync(local, peer)).AttemptId; + else + { + var start = await StartAsync(local, peer); + attemptId = start.AttemptId; + if (state == "initiator-approved") + { + var callback = await peer.ApproveAsync(local.BaseUrl, attemptId, BrowserState(start), SessionBinding); + using var reviewed = await local.AdminAsync(HttpMethod.Post, "/api/v1/admin/service-links/callback", callback); + Assert.Equal(HttpStatusCode.OK, reviewed.StatusCode); + } + } + var before = await ReadHumanConsentAttemptAsync(local, attemptId); + Assert.Null(before.InboundPrincipalId); + Assert.Equal("undecided", before.Decision); + // This uses the existing deterministic fixture clock for expiry coverage; + // it does not claim real-time rotation or cross-product physical execution. + local.Clock.Advance(TimeSpan.FromSeconds(before.ExpiresAtUnixSeconds - local.Clock.GetUtcNow().ToUnixTimeSeconds() + 1)); + await using (var work = local.Services.CreateAsyncScope()) + await work.ServiceProvider.GetRequiredService().WorkAsync(CancellationToken.None); + var expired = await ReadHumanConsentAttemptAsync(local, attemptId); + Assert.Equal("expired", expired.LifecycleState); + Assert.Equal("abort", expired.Decision); + Assert.NotNull(expired.AbortId); + Assert.Equal(before.ExpiresAtUnixSeconds, expired.ExpiresAtUnixSeconds); + Assert.Equal(before.DescriptorHash, expired.DescriptorHash); + Assert.Equal(before.GrantHash, expired.GrantHash); + Assert.Null(expired.ActiveRelationshipKey); + Assert.Null(expired.InboundPrincipalId); + Assert.False(expired.LocalInboundActive); + Assert.False(expired.LocalBusinessSenderEnabled); + Assert.True(expired.ProtectedBrowserState is null && expired.ProtectedVerifier is null && + expired.ProtectedPairingCode is null && expired.ProtectedInboundEscrow is null && + expired.ProtectedExchangeResponse is null, "Expired unused consent retained bootstrap escrow."); + await using (var check = local.Services.CreateAsyncScope()) + await AssertNoHumanConsentAuthorityAsync(check.ServiceProvider.GetRequiredService(), CancellationToken.None); + await using (var repeatedWork = local.Services.CreateAsyncScope()) + await repeatedWork.ServiceProvider.GetRequiredService().WorkAsync(CancellationToken.None); + Assert.Equal(expired.Revision, (await ReadHumanConsentAttemptAsync(local, attemptId)).Revision); + } + + private static async Task PrepareResponderConsentAsync(ServiceLinkKestrelPeer local, NetRatelServiceLinkContractPeer peer) + { + var metadata = await local.Client.GetFromJsonAsync(ServiceLinkContract.MetadataPath); + Assert.NotNull(metadata); + var descriptor = peer.PrepareInitiator(metadata, local.OrganizationId, local.CustomerId, local.Clock.GetUtcNow()); + using var reviewed = await local.AdminAsync(HttpMethod.Post, "/api/v1/admin/service-links/remote-review", + new ServiceLinkRemoteReviewRequest(peer.BaseUrl, descriptor.AttemptId, peer.BrowserState)); + Assert.Equal(HttpStatusCode.OK, reviewed.StatusCode); + return descriptor; + } + + private static async Task ReadHumanConsentAttemptAsync(ServiceLinkKestrelPeer local, string attemptId) + { + await using var scope = local.Services.CreateAsyncScope(); + return await scope.ServiceProvider.GetRequiredService().Set() + .AsNoTracking().SingleAsync(a => a.AttemptId == attemptId); + } + + private static async Task AssertNoHumanConsentAuthorityAsync(HelpdeskDbContext db, CancellationToken ct) + { + Assert.Equal(0, await db.Set().CountAsync(ct)); + Assert.Equal(0, await db.Set().CountAsync(ct)); + Assert.Equal(0, await db.IncidentReceiverPrincipalBindings.CountAsync(ct)); + Assert.Equal(0, await db.Set().CountAsync(ct)); + Assert.Equal(0, await db.Set().CountAsync(ct)); + Assert.Equal(0, await db.Set().CountAsync(ct)); + } + + private sealed class HumanConsentWorkerInterleaving(string step) : SaveChangesInterceptor + { + private IServiceProvider services = null!; + private string attemptId = ""; + private int armed; + public int Executions { get; private set; } + public Guid HumanContextId { get; private set; } + public Guid WorkerContextId { get; private set; } + private long beforeRevision, afterRevision, beforeNextWork, afterNextWork; + private bool attemptUnchanged; + public void Arm(IServiceProvider provider, string id) { services = provider; attemptId = id; Volatile.Write(ref armed, 1); } + public void AssertAttemptUnchanged() + { + Assert.Equal(beforeRevision, afterRevision); + Assert.Equal(beforeNextWork, afterNextWork); + Assert.True(attemptUnchanged, "Background work changed the durable live human-consent attempt."); + } + + public override async ValueTask> SavingChangesAsync(DbContextEventData eventData, + InterceptionResult result, CancellationToken cancellationToken = default) + { + if (Volatile.Read(ref armed) == 0 || eventData.Context is not { } human) return result; + var attempt = human.ChangeTracker.Entries().SingleOrDefault(e => e.Entity.AttemptId == attemptId); + if (attempt is null || (step == "initiator-callback" + ? attempt.State != EntityState.Modified || attempt.Entity.LifecycleState != "approved" + : !human.ChangeTracker.Entries().Any(e => e.State == EntityState.Added && e.Entity.AttemptId == attemptId))) + return result; + if (Interlocked.Exchange(ref armed, 0) == 0) return result; + HumanContextId = human.ContextId.InstanceId; + await using var scope = services.CreateAsyncScope(); + var db = scope.ServiceProvider.GetRequiredService(); + WorkerContextId = db.ContextId.InstanceId; + Assert.Equal("Npgsql.EntityFrameworkCore.PostgreSQL", db.Database.ProviderName); + var before = await db.Set().AsNoTracking().SingleAsync(a => a.AttemptId == attemptId, cancellationToken); + Assert.Equal(step == "initiator-consent" ? "approved" : "awaiting_approval", before.LifecycleState); + Assert.Equal("undecided", before.Decision); + Assert.Null(before.InboundPrincipalId); + Assert.True(before.ExpiresAtUnixSeconds > services.GetRequiredService().GetUtcNow().ToUnixTimeSeconds()); + await AssertNoHumanConsentAuthorityAsync(db, cancellationToken); + // The actual human scope is suspended before SaveChanges. Execute + // the real worker in a separate PostgreSQL scope, then continue the + // same authenticated command. Never fabricate an exception or authority. + await scope.ServiceProvider.GetRequiredService().WorkAsync(cancellationToken); + var after = await db.Set().AsNoTracking().SingleAsync(a => a.AttemptId == attemptId, cancellationToken); + beforeRevision = before.Revision; afterRevision = after.Revision; + beforeNextWork = before.NextWorkAtUnixSeconds; afterNextWork = after.NextWorkAtUnixSeconds; + attemptUnchanged = string.Equals(JsonSerializer.Serialize(before), JsonSerializer.Serialize(after), StringComparison.Ordinal); + await AssertNoHumanConsentAuthorityAsync(db, cancellationToken); + Executions++; + // Defer snapshot assertions until the actual HTTP command returns; + // the old scheduler must reach the real EF concurrency conflict, + // not a replacement assertion thrown from this interceptor. + return result; + } + } +} diff --git a/tests/Helpdesk.Tests/Infrastructure/ServiceLink/ServiceLinkDynamicNetworkPolicyTests.cs b/tests/Helpdesk.Tests/Infrastructure/ServiceLink/ServiceLinkDynamicNetworkPolicyTests.cs index 097ed223..fb1e7ae0 100644 --- a/tests/Helpdesk.Tests/Infrastructure/ServiceLink/ServiceLinkDynamicNetworkPolicyTests.cs +++ b/tests/Helpdesk.Tests/Infrastructure/ServiceLink/ServiceLinkDynamicNetworkPolicyTests.cs @@ -108,7 +108,7 @@ public void Existing_coordinator_observes_linking_disabled_after_reload() using var provider = Services(configuration); var coordinator = new ServiceLinkCoordinator(null!, null!, null!, null!, null!, null!, provider.GetRequiredService>(), - provider.GetRequiredService>(), TimeProvider.System, + provider.GetRequiredService>(), TimeProvider.System, null!, provider.GetRequiredService>()); Assert.Equal("rd-network-policy", coordinator.Metadata().InstanceId); configuration["ServiceLinks:Enabled"] = "false"; diff --git a/tests/Helpdesk.Tests/Infrastructure/ServiceLink/ServiceLinkProtocolTokenCacheTests.cs b/tests/Helpdesk.Tests/Infrastructure/ServiceLink/ServiceLinkProtocolTokenCacheTests.cs new file mode 100644 index 00000000..e1dc86db --- /dev/null +++ b/tests/Helpdesk.Tests/Infrastructure/ServiceLink/ServiceLinkProtocolTokenCacheTests.cs @@ -0,0 +1,686 @@ +using System.Collections.Concurrent; +using System.Net; +using System.Text; +using System.Text.Json; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Options; +using Helpdesk.Infrastructure.ServiceLink; +using Helpdesk.Shared.ServiceLink; +using Xunit; + +namespace Helpdesk.Tests.Infrastructure.ServiceLink; + +public sealed class ServiceLinkProtocolTokenCacheTests +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Concurrent_same_authority_requests_share_one_issuance_even_across_cache_instances(bool separateInstances) + { + using var harness = new Harness(); + var started = Signal(); + var release = Signal(); + harness.Handler.Reply = async (number, ct) => + { + started.TrySetResult(); + await release.Task.WaitAsync(ct); + return TokenResponse(number, 900); + }; + var other = separateInstances ? harness.NewCache() : harness.Cache; + var first = harness.GetAsync(); + try + { + await started.Task.WaitAsync(TimeSpan.FromSeconds(5)); + var others = Enumerable.Range(0, 12) + .Select(_ => other.GetAsync(harness.ResolveAsync, ServiceLinkContract.ControlScope, CancellationToken.None)) + .ToArray(); + + Assert.Equal(1, harness.Handler.Requests); + release.TrySetResult(); + var tokens = await Task.WhenAll(others.Prepend(first)); + + Assert.All(tokens, token => Assert.Equal("issued-1", token)); + Assert.Equal(1, harness.Handler.Requests); + Assert.True(harness.ResolverCalls >= tokens.Length + 1); + AssertTokenRequest(Assert.Single(harness.Handler.Observed), harness.Context.Credential, ServiceLinkContract.ControlScope); + } + finally { release.TrySetResult(); } + } + + [Fact] + public async Task A_new_cache_instance_reuses_a_shared_entry_from_an_equivalent_context() + { + using var harness = new Harness(); + Assert.Equal("issued-1", await harness.GetAsync()); + harness.Context = harness.Context with + { + Local = harness.Context.Local with { }, + Peer = harness.Context.Peer with { }, + Credential = harness.Context.Credential with { Scopes = [.. harness.Context.Credential.Scopes] } + }; + + var token = await harness.NewCache().GetAsync(harness.ResolveAsync, ServiceLinkContract.ControlScope, CancellationToken.None); + + Assert.Equal("issued-1", token); + Assert.Equal(1, harness.Handler.Requests); + } + + [Fact] + public async Task Reordering_the_approved_credential_scopes_reuses_the_same_entry() + { + using var harness = new Harness(); + Assert.Equal("issued-1", await harness.GetAsync()); + harness.Context = harness.Context with + { + Credential = harness.Context.Credential with { Scopes = harness.Context.Credential.Scopes.Reverse().ToArray() } + }; + + Assert.Equal("issued-1", await harness.GetAsync()); + Assert.Equal(1, harness.Handler.Requests); + } + + [Fact] + public async Task Control_and_verification_scopes_have_separate_tokens_and_unchanged_client_secret_post_requests() + { + using var harness = new Harness(); + + Assert.Equal("issued-1", await harness.GetAsync(ServiceLinkContract.ControlScope)); + Assert.Equal("issued-2", await harness.GetAsync(ServiceLinkContract.VerifyScope)); + Assert.Equal("issued-1", await harness.GetAsync(ServiceLinkContract.ControlScope)); + Assert.Equal("issued-2", await harness.GetAsync(ServiceLinkContract.VerifyScope)); + + Assert.Equal(2, harness.Handler.Requests); + var requests = harness.Handler.Observed.ToArray(); + AssertTokenRequest(requests[0], harness.Context.Credential, ServiceLinkContract.ControlScope); + AssertTokenRequest(requests[1], harness.Context.Credential, ServiceLinkContract.VerifyScope); + } + + [Theory] + [InlineData("attempt")] + [InlineData("link")] + [InlineData("link-revision")] + [InlineData("grant")] + [InlineData("descriptor")] + [InlineData("direction")] + [InlineData("mode")] + [InlineData("profile-revision")] + [InlineData("rotation")] + [InlineData("authority-deadline")] + [InlineData("issuer")] + [InlineData("token-endpoint")] + [InlineData("audience")] + [InlineData("client-id")] + [InlineData("client-secret")] + [InlineData("credential-revision")] + [InlineData("credential-scopes")] + [InlineData("caller-instance")] + [InlineData("caller-tenant")] + [InlineData("target-instance")] + [InlineData("target-tenant")] + [InlineData("local-instance")] + [InlineData("local-source-instance")] + [InlineData("local-issuer")] + [InlineData("local-jwks")] + [InlineData("peer-instance")] + [InlineData("peer-source-instance")] + [InlineData("peer-issuer")] + [InlineData("peer-jwks")] + public async Task Changed_authority_or_credential_descriptor_cannot_reuse_another_contexts_token(string change) + { + using var harness = new Harness(); + var original = harness.Context; + Assert.Equal("issued-1", await harness.GetAsync()); + harness.Context = Changed(original, change); + + Assert.Equal("issued-2", await harness.GetAsync()); + Assert.Equal("issued-2", await harness.GetAsync()); + harness.Context = original; + Assert.Equal("issued-1", await harness.GetAsync()); + + Assert.Equal(2, harness.Handler.Requests); + var requests = harness.Handler.Observed.ToArray(); + AssertTokenRequest(requests[0], original.Credential, ServiceLinkContract.ControlScope); + AssertTokenRequest(requests[1], Changed(original, change).Credential, ServiceLinkContract.ControlScope); + } + + [Theory] + [InlineData(":")] + [InlineData("|")] + public async Task Adjacent_authority_identifiers_cannot_ambiguously_share_tokens(string separator) + { + using var harness = new Harness(); + var first = harness.Context with { AttemptId = $"attempt{separator}part", LinkId = "link" }; + var second = harness.Context with { AttemptId = "attempt", LinkId = $"part{separator}link" }; + harness.Context = first; + Assert.Equal("issued-1", await harness.GetAsync()); + harness.Context = second; + + Assert.Equal("issued-2", await harness.GetAsync()); + harness.Context = first; + Assert.Equal("issued-1", await harness.GetAsync()); + Assert.Equal(2, harness.Handler.Requests); + } + + [Theory] + [InlineData("business.read")] + [InlineData("")] + [InlineData("bostec.service-link.control bostec.service-link.verify")] + public async Task Non_protocol_or_combined_scopes_are_rejected_before_HTTP(string scope) + { + using var harness = new Harness(); + + var error = await Assert.ThrowsAsync(() => harness.GetAsync(scope)); + + Assert.Equal(403, error.StatusCode); + Assert.Equal("protocol-scope-required", error.Code); + Assert.Equal(0, harness.Handler.Requests); + Assert.Empty(harness.Handler.Observed); + } + + [Fact] + public async Task A_current_authority_denial_is_propagated_before_reusing_a_warm_entry() + { + using var harness = new Harness(); + Assert.Equal("issued-1", await harness.GetAsync()); + var before = harness.ResolverCalls; + harness.DenyAuthority = true; + + var error = await Assert.ThrowsAsync(() => harness.GetAsync()); + + Assert.Equal("test-authority-denied", error.Code); + Assert.True(harness.ResolverCalls > before); + Assert.Equal(1, harness.Handler.Requests); + } + + [Fact] + public async Task Removing_the_current_private_endpoint_opt_in_prevents_a_warm_cache_hit() + { + using var harness = new Harness(allowPrivate: true); + harness.Context = harness.Context with + { + Credential = harness.Context.Credential with { TokenEndpoint = "https://10.1.2.3/connect/token" } + }; + Assert.Equal("issued-1", await harness.GetAsync()); + harness.CurrentOptions.Value = new ServiceLinkOptions { AllowPrivateHttp = false }; + + await Assert.ThrowsAsync(() => harness.GetAsync()); + + Assert.Equal(1, harness.Handler.Requests); + } + + [Theory] + [InlineData(12, 7)] + [InlineData(900, 45)] + public async Task Reuse_ends_at_the_expiry_margin_or_45_second_cap(int expiresIn, int reuseSeconds) + { + using var harness = new Harness(); + harness.Handler.Reply = (number, _) => Task.FromResult(TokenResponse(number, expiresIn)); + Assert.Equal("issued-1", await harness.GetAsync()); + harness.Clock.Advance(TimeSpan.FromSeconds(reuseSeconds - 1)); + Assert.Equal("issued-1", await harness.GetAsync()); + harness.Clock.Advance(TimeSpan.FromSeconds(1)); + + Assert.Equal("issued-2", await harness.GetAsync()); + Assert.Equal(2, harness.Handler.Requests); + } + + [Theory] + [InlineData(20, 10, 15)] + [InlineData(900, 30, 45)] + public async Task Slow_HTTP_consumes_the_reuse_lifetime_from_request_start(int expiresIn, int delaySeconds, int reuseSeconds) + { + using var harness = new Harness(); + harness.Handler.Reply = (number, _) => + { + if (number == 1) harness.Clock.Advance(TimeSpan.FromSeconds(delaySeconds)); + return Task.FromResult(TokenResponse(number, expiresIn)); + }; + Assert.Equal("issued-1", await harness.GetAsync()); + harness.Clock.Advance(TimeSpan.FromSeconds(reuseSeconds - delaySeconds - 1)); + Assert.Equal("issued-1", await harness.GetAsync()); + harness.Clock.Advance(TimeSpan.FromSeconds(1)); + + Assert.Equal("issued-2", await harness.GetAsync()); + Assert.Equal(2, harness.Handler.Requests); + } + + [Theory] + [InlineData(1)] + [InlineData(5)] + public async Task Still_valid_tokens_with_no_safe_reuse_lifetime_are_returned_without_caching(int expiresIn) + { + using var harness = new Harness(); + harness.Handler.Reply = (number, _) => Task.FromResult(TokenResponse(number, expiresIn)); + + Assert.Equal("issued-1", await harness.GetAsync()); + Assert.Equal("issued-2", await harness.GetAsync()); + Assert.Equal(2, harness.Handler.Requests); + } + + [Fact] + public async Task A_still_valid_reply_that_consumed_the_safety_margin_is_returned_without_reuse() + { + using var harness = new Harness(); + harness.Handler.Reply = (number, _) => + { + if (number == 1) harness.Clock.Advance(TimeSpan.FromSeconds(6)); + return Task.FromResult(TokenResponse(number, 10)); + }; + + Assert.Equal("issued-1", await harness.GetAsync()); + Assert.Equal("issued-2", await harness.GetAsync()); + Assert.Equal("issued-2", await harness.GetAsync()); + Assert.Equal(2, harness.Handler.Requests); + } + + [Fact] + public async Task An_authority_deadline_prevents_reuse_even_when_the_token_and_45_second_cap_remain_valid() + { + using var harness = new Harness(); + harness.Context = harness.Context with { AuthorityExpiresAtUnixSeconds = harness.Clock.GetUtcNow().ToUnixTimeSeconds() + 10 }; + Assert.Equal("issued-1", await harness.GetAsync()); + harness.Clock.Advance(TimeSpan.FromSeconds(9)); + Assert.Equal("issued-1", await harness.GetAsync()); + harness.Clock.Advance(TimeSpan.FromSeconds(1)); + + var error = await Assert.ThrowsAsync(() => harness.GetAsync()); + + Assert.Equal(403, error.StatusCode); + Assert.Equal(1, harness.Handler.Requests); + } + + [Fact] + public async Task Authority_expiring_during_HTTP_rejects_the_reply_and_does_not_seed_a_later_authority() + { + using var harness = new Harness(); + harness.Context = harness.Context with { AuthorityExpiresAtUnixSeconds = harness.Clock.GetUtcNow().ToUnixTimeSeconds() + 10 }; + harness.Handler.Reply = (number, _) => + { + if (number == 1) harness.Clock.Advance(TimeSpan.FromSeconds(10)); + return Task.FromResult(TokenResponse(number, 900)); + }; + + var error = await Assert.ThrowsAsync(() => harness.GetAsync()); + + Assert.Equal(403, error.StatusCode); + harness.Context = harness.Context with { AuthorityExpiresAtUnixSeconds = harness.Clock.GetUtcNow().ToUnixTimeSeconds() + 600 }; + Assert.Equal("issued-2", await harness.GetAsync()); + Assert.Equal("issued-2", await harness.GetAsync()); + Assert.Equal(2, harness.Handler.Requests); + } + + [Fact] + public async Task A_reply_that_has_already_expired_is_rejected_and_never_reused() + { + using var harness = new Harness(); + harness.Handler.Reply = (number, _) => + { + if (number == 1) + { + harness.Clock.Advance(TimeSpan.FromSeconds(10)); + return Task.FromResult(TokenResponse(number, 10)); + } + return Task.FromResult(TokenResponse(number, 900)); + }; + + await Assert.ThrowsAsync(() => harness.GetAsync()); + Assert.Equal("issued-2", await harness.GetAsync()); + Assert.Equal("issued-2", await harness.GetAsync()); + Assert.Equal(2, harness.Handler.Requests); + } + + [Theory] + [InlineData("0")] + [InlineData("-1")] + [InlineData("901")] + [InlineData("1.5")] + [InlineData("\"60\"")] + [InlineData("null")] + public async Task Invalid_expiry_responses_are_rejected_and_do_not_poison_the_cache(string expiresInJson) + { + using var harness = new Harness(); + harness.Handler.Reply = (number, _) => Task.FromResult(number == 1 + ? JsonResponse($"{{\"access_token\":\"issued-1\",\"token_type\":\"Bearer\",\"expires_in\":{expiresInJson}}}") + : TokenResponse(number, 900)); + + var error = await Assert.ThrowsAsync(() => harness.GetAsync()); + + Assert.Equal("invalid-token-response", error.Code); + Assert.Equal("issued-2", await harness.GetAsync()); + Assert.Equal("issued-2", await harness.GetAsync()); + Assert.Equal(2, harness.Handler.Requests); + } + + [Theory] + [InlineData("{\"access_token\":\"issued-1\",\"token_type\":\"Bearer\"}")] + [InlineData("{\"access_token\":\"issued-1\",\"token_type\":\"Bearer\",\"expires_in\":60,\"expires_in\":60}")] + [InlineData("{\"access_token\":\"issued-1\",\"token_type\":\"Basic\",\"expires_in\":60}")] + [InlineData("{\"access_token\":\"\",\"token_type\":\"Bearer\",\"expires_in\":60}")] + [InlineData("not-json")] + public async Task Invalid_token_response_bodies_are_not_cached(string body) + { + using var harness = new Harness(); + harness.Handler.Reply = (number, _) => Task.FromResult(number == 1 ? JsonResponse(body) : TokenResponse(number, 900)); + + var error = await Assert.ThrowsAsync(() => harness.GetAsync()); + + Assert.Equal("invalid-token-response", error.Code); + Assert.Equal("issued-2", await harness.GetAsync()); + Assert.Equal("issued-2", await harness.GetAsync()); + Assert.Equal(2, harness.Handler.Requests); + } + + [Theory] + [InlineData(500)] + [InlineData(302)] + public async Task HTTP_failure_or_redirect_is_not_cached(int statusCode) + { + using var harness = new Harness(); + harness.Handler.Reply = (number, _) => Task.FromResult(number == 1 + ? new HttpResponseMessage((HttpStatusCode)statusCode) { Content = new StringContent("{}") } + : TokenResponse(number, 900)); + + await Assert.ThrowsAsync(() => harness.GetAsync()); + Assert.Equal("issued-2", await harness.GetAsync()); + Assert.Equal("issued-2", await harness.GetAsync()); + Assert.Equal(2, harness.Handler.Requests); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Authority_changed_or_denied_during_HTTP_rejects_the_issued_token_without_caching_it(bool deny) + { + using var harness = new Harness(); + var original = harness.Context; + harness.Handler.Reply = (number, _) => + { + if (number == 1) + { + if (deny) harness.DenyAuthority = true; + else harness.Context = Changed(original, "credential-revision"); + } + return Task.FromResult(TokenResponse(number, 900)); + }; + + await Assert.ThrowsAsync(() => harness.GetAsync()); + harness.Context = original; + harness.DenyAuthority = false; + Assert.Equal("issued-2", await harness.GetAsync()); + Assert.Equal("issued-2", await harness.GetAsync()); + Assert.Equal(2, harness.Handler.Requests); + } + + [Fact] + public async Task A_cancelled_waiter_does_not_cancel_the_issuer_or_prevent_subsequent_reuse() + { + using var harness = new Harness(); + var started = Signal(); + var release = Signal(); + harness.Handler.Reply = async (number, ct) => + { + started.TrySetResult(); + await release.Task.WaitAsync(ct); + return TokenResponse(number, 900); + }; + var issuer = harness.GetAsync(); + try + { + await started.Task.WaitAsync(TimeSpan.FromSeconds(5)); + using var cancelled = new CancellationTokenSource(); + var waiter = harness.GetAsync(ct: cancelled.Token); + cancelled.Cancel(); + + await Assert.ThrowsAnyAsync(() => waiter); + Assert.Equal(1, harness.Handler.Requests); + release.TrySetResult(); + Assert.Equal("issued-1", await issuer); + Assert.Equal("issued-1", await harness.GetAsync()); + Assert.Equal(1, harness.Handler.Requests); + } + finally { release.TrySetResult(); } + } + + [Fact] + public async Task Authority_changed_while_waiting_is_rechecked_before_reusing_the_issuers_entry() + { + using var harness = new Harness(); + var started = Signal(); + var release = Signal(); + harness.Handler.Reply = async (number, ct) => + { + started.TrySetResult(); + await release.Task.WaitAsync(ct); + return TokenResponse(number, 900); + }; + var issuer = harness.GetAsync(); + try + { + await started.Task.WaitAsync(TimeSpan.FromSeconds(5)); + var waiterContext = harness.Context; + var waiter = harness.NewCache().GetAsync(_ => Task.FromResult(waiterContext), + ServiceLinkContract.ControlScope, CancellationToken.None); + waiterContext = Changed(waiterContext, "credential-revision"); + release.TrySetResult(); + + Assert.Equal("issued-1", await issuer); + var error = await Assert.ThrowsAsync(() => waiter); + + Assert.Equal(409, error.StatusCode); + Assert.Equal("protocol-profile-conflict", error.Code); + Assert.Equal(1, harness.Handler.Requests); + } + finally { release.TrySetResult(); } + } + + [Fact] + public async Task Cancelled_issuance_releases_the_gate_and_does_not_cache_a_result() + { + using var harness = new Harness(); + var started = Signal(); + var release = Signal(); + harness.Handler.Reply = async (number, ct) => + { + if (number == 1) + { + started.TrySetResult(); + await release.Task.WaitAsync(ct); + } + return TokenResponse(number, 900); + }; + using var cancelled = new CancellationTokenSource(); + var pending = harness.GetAsync(ct: cancelled.Token); + try + { + await started.Task.WaitAsync(TimeSpan.FromSeconds(5)); + cancelled.Cancel(); + + await Assert.ThrowsAnyAsync(() => pending); + Assert.Equal("issued-2", await harness.GetAsync()); + Assert.Equal("issued-2", await harness.GetAsync()); + Assert.Equal(2, harness.Handler.Requests); + } + finally + { + cancelled.Cancel(); + release.TrySetResult(); + } + } + + [Fact] + public async Task An_already_cancelled_call_cannot_reuse_a_warm_entry() + { + using var harness = new Harness(); + Assert.Equal("issued-1", await harness.GetAsync()); + using var cancelled = new CancellationTokenSource(); + cancelled.Cancel(); + + await Assert.ThrowsAnyAsync(() => harness.GetAsync(ct: cancelled.Token)); + + Assert.Equal(1, harness.Handler.Requests); + } + + private static ServiceLinkProtocolTokenContext Changed(ServiceLinkProtocolTokenContext context, string change) => change switch + { + "attempt" => context with { AttemptId = "other-attempt" }, + "link" => context with { LinkId = "other-link" }, + "link-revision" => context with { LinkRevision = context.LinkRevision + 1 }, + "grant" => context with { GrantHash = "other-grant" }, + "descriptor" => context with { DescriptorHash = "other-descriptor" }, + "direction" => context with { DirectionId = ServiceLinkContract.ResponderToInitiator }, + "mode" => context with { Mode = "rotation" }, + "profile-revision" => context with { OutboundProfileRevision = context.OutboundProfileRevision + 1 }, + "rotation" => context with { RotationId = "other-rotation" }, + "authority-deadline" => context with { AuthorityExpiresAtUnixSeconds = context.AuthorityExpiresAtUnixSeconds + 1 }, + "issuer" => context with { Credential = context.Credential with { Issuer = "https://other-peer.example.test" } }, + "token-endpoint" => context with { Credential = context.Credential with { TokenEndpoint = "https://peer.example.test/other-token" } }, + "audience" => context with { Credential = context.Credential with { Audience = "other-audience" } }, + "client-id" => context with { Credential = context.Credential with { ClientId = "other-client" } }, + "client-secret" => context with { Credential = context.Credential with { ClientSecret = "other-secret +&=" } }, + "credential-revision" => context with { Credential = context.Credential with { CredentialRevision = context.Credential.CredentialRevision + 1 } }, + "credential-scopes" => context with { Credential = context.Credential with { Scopes = [ServiceLinkContract.ControlScope] } }, + "caller-instance" => context with { Credential = context.Credential with { CallerInstanceId = "other-caller" } }, + "caller-tenant" => context with { Credential = context.Credential with { CallerTenantId = "other-caller-tenant" } }, + "target-instance" => context with { Credential = context.Credential with { TargetInstanceId = "other-target" } }, + "target-tenant" => context with { Credential = context.Credential with { TargetTenantId = "other-target-tenant" } }, + "local-instance" => context with { Local = context.Local with { InstanceId = "other-local" } }, + "local-source-instance" => context with { Local = context.Local with { SourceInstanceId = "other-local-source" } }, + "local-issuer" => context with { Local = context.Local with { OauthIssuer = "https://other-local.example.test" } }, + "local-jwks" => context with { Local = context.Local with { JwksUri = "https://local.example.test/other-jwks" } }, + "peer-instance" => context with { Peer = context.Peer with { InstanceId = "other-peer" } }, + "peer-source-instance" => context with { Peer = context.Peer with { SourceInstanceId = "other-peer-source" } }, + "peer-issuer" => context with { Peer = context.Peer with { OauthIssuer = "https://other-peer.example.test" } }, + "peer-jwks" => context with { Peer = context.Peer with { JwksUri = "https://peer.example.test/other-jwks" } }, + _ => throw new ArgumentOutOfRangeException(nameof(change)) + }; + + private static TaskCompletionSource Signal() => new(TaskCreationOptions.RunContinuationsAsynchronously); + + private static HttpResponseMessage TokenResponse(int number, int expiresIn) => JsonResponse(JsonSerializer.Serialize(new + { + access_token = $"issued-{number}", token_type = "Bearer", expires_in = expiresIn + })); + + private static HttpResponseMessage JsonResponse(string json) => new(HttpStatusCode.Created) + { + Content = new StringContent(json, Encoding.UTF8, "application/json") + }; + + private static void AssertTokenRequest(ObservedRequest request, ServiceDirectionalCredential credential, string scope) + { + Assert.Equal(HttpMethod.Post, request.Method); + Assert.Equal(credential.TokenEndpoint, request.Endpoint); + Assert.Null(request.Authorization); + Assert.Equal("application/x-www-form-urlencoded", request.ContentType); + var form = request.Body.Split('&').Select(pair => pair.Split('=', 2)) + .ToDictionary(pair => Decode(pair[0]), pair => Decode(pair[1]), StringComparer.Ordinal); + Assert.Equal(4, form.Count); + Assert.Equal("client_credentials", form["grant_type"]); + Assert.Equal(credential.ClientId, form["client_id"]); + Assert.Equal(credential.ClientSecret, form["client_secret"]); + Assert.Equal(scope, form["scope"]); + } + + private static string Decode(string value) => Uri.UnescapeDataString(value.Replace('+', ' ')); + + private sealed class Harness : IDisposable + { + private readonly HttpClient client; + private readonly MemoryCache memory = new(new MemoryCacheOptions()); + private int resolverCalls; + + public Harness(bool allowPrivate = false) + { + CurrentOptions = new MutableOptionsMonitor { Value = new ServiceLinkOptions { AllowPrivateHttp = allowPrivate } }; + Handler = new TokenHandler(); + client = new HttpClient(Handler); + Transport = new ServiceLinkTransport(client, Options.Create(CurrentOptions.Value), CurrentOptions); + Cache = NewCache(); + var local = Metadata("local", "rateldesk"); + var peer = Metadata("peer", "netratel"); + Context = new ServiceLinkProtocolTokenContext("attempt", "link", 1, "grant", "descriptor", + ServiceLinkContract.InitiatorToResponder, "active", 1, local, peer, + new ServiceDirectionalCredential + { + ClientId = "protocol-client", ClientSecret = "protocol-secret +&=", + Issuer = peer.OauthIssuer, TokenEndpoint = peer.TokenEndpoint, Audience = peer.Audience, + Scopes = [ServiceLinkContract.ControlScope, ServiceLinkContract.VerifyScope], CredentialRevision = 1, + CallerInstanceId = local.InstanceId, CallerTenantId = "caller-tenant", + TargetInstanceId = peer.InstanceId, TargetTenantId = "target-tenant" + }, "rotation", Clock.GetUtcNow().AddMinutes(10).ToUnixTimeSeconds()); + } + + public ManualTimeProvider Clock { get; } = new(); + public TokenHandler Handler { get; } + public MutableOptionsMonitor CurrentOptions { get; } + public ServiceLinkTransport Transport { get; } + public ServiceLinkProtocolTokenCache Cache { get; } + public ServiceLinkProtocolTokenContext Context { get; set; } + public bool DenyAuthority { get; set; } + public int ResolverCalls => Volatile.Read(ref resolverCalls); + public ServiceLinkProtocolTokenCache NewCache() => new(Transport, memory, Clock); + public Task GetAsync(string scope = ServiceLinkContract.ControlScope, CancellationToken ct = default) => + Cache.GetAsync(ResolveAsync, scope, ct); + + public Task ResolveAsync(CancellationToken ct) + { + Interlocked.Increment(ref resolverCalls); + ct.ThrowIfCancellationRequested(); + if (DenyAuthority) throw new ServiceLinkProtocolException(409, "test-authority-denied", "Current authority was withdrawn."); + return Task.FromResult(Context); + } + + public void Dispose() + { + client.Dispose(); + memory.Dispose(); + } + + private static ServiceLinkMetadata Metadata(string instance, string product) => new() + { + Product = product, ProductVersion = "1.0", InstanceId = instance, SourceInstanceId = $"{instance}-source", + ApiBaseUrl = $"https://{instance}.example.test", WebBaseUrl = $"https://{instance}.example.test/web", + OauthIssuer = $"https://{instance}.example.test", OauthMetadataUrl = $"https://{instance}.example.test/.well-known/oauth-authorization-server", + TokenEndpoint = $"https://{instance}.example.test/connect/token", JwksUri = $"https://{instance}.example.test/jwks", + Audience = $"{instance}-audience", ServiceLinkEndpoint = $"https://{instance}.example.test{ServiceLinkContract.EndpointPath}", + ApprovalEndpoint = $"https://{instance}.example.test/approve", CallbackEndpoint = $"https://{instance}.example.test/callback" + }; + } + + private sealed record ObservedRequest(HttpMethod Method, string Endpoint, string? Authorization, string? ContentType, string Body); + + private sealed class TokenHandler : HttpMessageHandler + { + private int requests; + public int Requests => Volatile.Read(ref requests); + public ConcurrentQueue Observed { get; } = new(); + public Func> Reply { get; set; } = + (number, _) => Task.FromResult(TokenResponse(number, 900)); + + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken ct) + { + var number = Interlocked.Increment(ref requests); + var body = request.Content is null ? "" : await request.Content.ReadAsStringAsync(ct); + Observed.Enqueue(new ObservedRequest(request.Method, request.RequestUri!.AbsoluteUri, + request.Headers.Authorization?.ToString(), request.Content?.Headers.ContentType?.MediaType, body)); + return await Reply(number, ct); + } + } + + private sealed class ManualTimeProvider : TimeProvider + { + // Physical eviction stays separate from the injected logical clock used to enforce reuse. + private DateTimeOffset utcNow = DateTimeOffset.UtcNow.AddHours(1); + public override DateTimeOffset GetUtcNow() => utcNow; + public void Advance(TimeSpan elapsed) => utcNow += elapsed; + } + + private sealed class MutableOptionsMonitor : IOptionsMonitor + { + public ServiceLinkOptions Value { get; set; } = new(); + public ServiceLinkOptions CurrentValue => Value; + public ServiceLinkOptions Get(string? name) => Value; + public IDisposable OnChange(Action listener) => new NoopDisposable(); + private sealed class NoopDisposable : IDisposable { public void Dispose() { } } + } +}