Keep a grading-heartbeat fault from replacing the grade

backend/src/CodeSpace.Core/Services/Agents/AgentRetryContinuity.cs

@@ -11,7 +11,7 @@ namespace CodeSpace.Core.Services.Agents;
 public static class AgentRetryContinuity
 {
     /// <summary>The honest-redo line: fires ONLY when a resumed conversation exists but the workspace was NOT pinned to a prior pushed branch — never on a genuine cold-start retry (no prior attempt at all), which stays byte-identical.</summary>
-    public const string HonestNoContinuityHint = "Note: your prior attempt's conversation is restored, but its git changes were NOT preserved in this workspace (your prior attempt pushed no branch of its own) — you must redo any relevant file changes from scratch.";
+    public const string HonestNoContinuityHint = "Note: your prior attempt's conversation is restored, but its git changes were NOT preserved in this workspace (your prior attempt pushed no branch of its own for this repository) — you must redo any relevant file changes from scratch.";
 
     /// <summary>Append <see cref="HonestNoContinuityHint"/> to a resumed task's goal. One composition, so the two lanes cannot drift on the separator either.</summary>
     public static string WithHonestNoContinuityHint(string goal) => $"{goal}\n\n{HonestNoContinuityHint}";

backend/src/CodeSpace.Core/Services/Agents/Credentials/Broker/LoopbackModelCredentialBroker.cs

@@ -499,6 +499,16 @@ private async Task AcceptAsync(Lease lease)
             // matters more here than it used to — a revoke now closes a listener per FINISHED RUN, where before a
             // listener only ever closed at process teardown — and the only correct response to any of them is to stop
             // serving an address that no longer exists, and to stop CLAIMING it.
+            //
+            // A close can also land between the request above and this re-registration. The managed listener checks
+            // its state and only then queues the wait, and Close() completes-and-clears that queue in between, so the
+            // wait joins a closed listener's queue and this loop never resumes. That strands nothing: every close path
+            // takes the lease out of _byRun (supersede, revoke, sweep and drop before closing, dispose right after),
+            // and what is left is a cycle no root reaches — the closed listener's queue, the wait, this state machine,
+            // the lease, the listener — because Close() has already unhooked the listener from the endpoint manager's
+            // statics. It is collected, decrypted key and all, exactly as a loop that woke and returned would be
+            // (checked on .NET 10 by stranding a loop this way: its lease's weak reference cleared, while a lease a
+            // table still held stayed alive).
             try { context = await lease.Listener.GetContextAsync().ConfigureAwait(false); }
             catch (Exception exception) { DropIfStillServing(lease, exception); return; }
 

backend/src/CodeSpace.Core/Services/Supervisor/SupervisorTurnService.Rehydrate.cs

@@ -11,6 +11,7 @@
 using CodeSpace.Messages.Dtos.Decisions;
 using CodeSpace.Messages.Review;
 using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
 
 namespace CodeSpace.Core.Services.Supervisor;
@@ -1567,20 +1568,34 @@ private async Task<BenchmarkGrade> GradeStopTargetsWithHeartbeatAsync(Guid super
         }
     }
 
-    /// <summary>The heartbeat loop itself: sleeps, logs, repeats — until <paramref name="cancellationToken"/> fires (grading finished). A cancellation mid-sleep is the expected exit, never propagated as a fault. Internal + clock-parameterized so a unit test drives the sleep on a fake <paramref name="timeProvider"/> instead of racing the wall clock; REQUIRED rather than defaulting to the system clock for the reason <see cref="HeartbeatLoop.RunAsync"/> gives — a default is how a call site keeps the wall clock without saying so.</summary>
-    internal async Task RunGradingHeartbeatLoopAsync(Guid supervisorRunId, string nodeId, TimeSpan interval, CancellationToken cancellationToken, TimeProvider timeProvider, string message = "Supervisor stop acceptance grading is still in progress.")
+    /// <summary>
+    /// The heartbeat loop itself: sleeps, pulses, repeats — until <paramref name="cancellationToken"/> fires (grading
+    /// finished). It is <see cref="HeartbeatLoop.RunAsync"/>, so it NEVER completes faulted: a cancellation is the
+    /// expected exit, and a pulse that fails is logged as a warning and retried on the next interval. That is
+    /// load-bearing rather than tidy — every call site awaits this task in the <c>finally</c> around the grade it
+    /// protects, so a fault here would REPLACE the grade with the error of a missed log line.
+    ///
+    /// <para>Internal + clock-parameterized so a unit test drives the sleep on a fake <paramref name="timeProvider"/>
+    /// instead of racing the wall clock; REQUIRED rather than defaulting to the system clock for the reason
+    /// <see cref="HeartbeatLoop.RunAsync"/> gives — a default is how a call site keeps the wall clock without saying so.</para>
+    /// </summary>
+    internal Task RunGradingHeartbeatLoopAsync(Guid supervisorRunId, string nodeId, TimeSpan interval, CancellationToken cancellationToken, TimeProvider timeProvider, string message = "Supervisor stop acceptance grading is still in progress.") =>
+        HeartbeatLoop.RunAsync(ct => PulseGradingHeartbeatAsync(supervisorRunId, nodeId, message, ct), interval, exception => _logger.LogWarning(exception, "Supervisor run {SupervisorRunId} node {NodeId}: a grading heartbeat pulse failed; grading continues and the next pulse retries", supervisorRunId, nodeId), cancellationToken, timeProvider);
+
+    /// <summary>
+    /// One pulse, written through a record logger from a DI scope of its OWN. The pulse runs concurrently with the
+    /// grade, and this service's scope holds the one DbContext the grade is using — its manifest stamps and recorded
+    /// judge calls go through it — so a pulse on this scope's logger that fell due mid-query died on EF's "a second
+    /// operation was started on this context" guard. A scope per pulse rather than one per loop, so an insert that
+    /// failed never waits in a change tracker for the next pulse's save to retry it.
+    /// </summary>
+    private async Task PulseGradingHeartbeatAsync(Guid supervisorRunId, string nodeId, string message, CancellationToken cancellationToken)
     {
-        try
-        {
-            while (true)
-            {
-                await Task.Delay(interval, timeProvider, cancellationToken).ConfigureAwait(false);
+        using var scope = _scopeFactory.CreateScope();
 
-                await _recordLogger.LogAsync(supervisorRunId, nodeId, Workflows.Lifecycle.LogLevel.Info,
-                    message, cancellationToken).ConfigureAwait(false);
-            }
-        }
-        catch (OperationCanceledException) { }
+        var records = scope.ServiceProvider.GetRequiredService<Workflows.Lifecycle.IRunRecordLogger>();
+
+        await records.LogAsync(supervisorRunId, nodeId, Workflows.Lifecycle.LogLevel.Info, message, cancellationToken).ConfigureAwait(false);
     }
 
     /// <summary>

backend/src/CodeSpace.Core/Services/Supervisor/SupervisorTurnService.cs

@@ -10,6 +10,7 @@
 using CodeSpace.Messages.Budget;
 using CodeSpace.Messages.Dtos.Agents;
 using CodeSpace.Messages.Plans;
+using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
 
 namespace CodeSpace.Core.Services.Supervisor;
@@ -41,6 +42,9 @@ public sealed partial class SupervisorTurnService : ISupervisorTurnService, ISco
     private readonly ISupervisorPublishedBranchResolver _publishedBranches;
     private readonly Completion.ICompletionAssessmentComposer _completion;
 
+    /// <summary>Opens the scope each grading-heartbeat pulse writes through — never this service's own, whose DbContext the grade the pulse runs beside is using (see <see cref="PulseGradingHeartbeatAsync"/>).</summary>
+    private readonly IServiceScopeFactory _scopeFactory;
+
     /// <summary>C1 — the rubric judge a BRANCHLESS <c>LlmJudge</c> stop reads its summary with. OPTIONAL so the many hand-built test doubles keep compiling; DI always supplies it, and a null one fails the gate CLOSED rather than passing it silently.</summary>
     private readonly Review.IRubricJudge? _rubricJudge;
 
@@ -49,7 +53,7 @@ public sealed partial class SupervisorTurnService : ISupervisorTurnService, ISco
 
     private readonly ILogger<SupervisorTurnService> _logger;
 
-    public SupervisorTurnService(ISupervisorDecisionLog ledger, ISupervisorDecider decider, ISupervisorActionExecutor executor, CodeSpaceDbContext db, ISupervisorAcceptanceGrader acceptanceGrader, IDecisionQueueService decisionQueue, IDecisionArbiter arbiter, IDecisionAnswerService decisionAnswer, Plans.IWorkPlanService workPlans, Workflows.Lifecycle.IRunRecordLogger recordLogger, Workflows.Artifacts.IArtifactOffloader offloader, IPublishManifestStore manifests, ISupervisorPublishedBranchResolver publishedBranches, Completion.ICompletionAssessmentComposer completion, Workflows.Budget.IBudgetLedger budget, Learning.ILessonReader lessons, ILogger<SupervisorTurnService> logger, Review.IRubricJudge? rubricJudge = null, Completion.IModeProfileRegistry? modes = null)
+    public SupervisorTurnService(ISupervisorDecisionLog ledger, ISupervisorDecider decider, ISupervisorActionExecutor executor, CodeSpaceDbContext db, ISupervisorAcceptanceGrader acceptanceGrader, IDecisionQueueService decisionQueue, IDecisionArbiter arbiter, IDecisionAnswerService decisionAnswer, Plans.IWorkPlanService workPlans, Workflows.Lifecycle.IRunRecordLogger recordLogger, Workflows.Artifacts.IArtifactOffloader offloader, IPublishManifestStore manifests, ISupervisorPublishedBranchResolver publishedBranches, Completion.ICompletionAssessmentComposer completion, Workflows.Budget.IBudgetLedger budget, Learning.ILessonReader lessons, IServiceScopeFactory scopeFactory, ILogger<SupervisorTurnService> logger, Review.IRubricJudge? rubricJudge = null, Completion.IModeProfileRegistry? modes = null)
     {
         _ledger = ledger;
         _decider = decider;
@@ -67,6 +71,7 @@ public SupervisorTurnService(ISupervisorDecisionLog ledger, ISupervisorDecider d
         _manifests = manifests;
         _publishedBranches = publishedBranches;
         _completion = completion;
+        _scopeFactory = scopeFactory;
         _rubricJudge = rubricJudge;
         _modes = modes;
         _logger = logger;

backend/src/CodeSpace.Core/Services/Workflows/Artifacts/Retention/ArtifactRetentionPolicy.cs

@@ -62,8 +62,7 @@ public static class ArtifactRetentionPolicy
             [SessionTranscriptCheckpoint.Class] = SessionTranscriptCheckpoint,
         };
 
-    /// <summary>The rule for <paramref name="value"/>, or null when the running policy does not register it — which the reaper reads as "cannot tell" and keeps.</summary>
-    /// <summary>The rule for a class NAME, or null when this build registers none — including a name a rolled-back build wrote that this one has never heard of. Null settles as keep.</summary>
+    /// <summary>The rule for the class NAME <paramref name="value"/>, or null when this build registers none — including a name a rolled-back build wrote that this one has never heard of. The reaper reads null as "cannot tell" and keeps.</summary>
     public static ArtifactRetentionRule? For(string value) => Enum.TryParse<ArtifactRetentionClass>(value, ignoreCase: false, out var parsed) && Rules.TryGetValue(parsed, out var rule) ? rule : null;
 
     /// <summary>

backend/tests/CodeSpace.IntegrationTests/Agents/ModelPricingUnderCapFlowTests.cs

@@ -764,7 +764,7 @@ private static SupervisorTurnService BuildService(ILifetimeScope scope, ISupervi
             scope.Resolve<Core.Services.Workflows.Artifacts.IArtifactOffloader>(), scope.Resolve<Core.Services.Agents.Publish.IPublishManifestStore>(),
             scope.Resolve<ISupervisorPublishedBranchResolver>(), scope.Resolve<Core.Services.Completion.ICompletionAssessmentComposer>(),
             scope.Resolve<Core.Services.Workflows.Budget.IBudgetLedger>(), scope.Resolve<Core.Services.Learning.ILessonReader>(),
-            scope.Resolve<ILogger<SupervisorTurnService>>());
+            scope.Resolve<Microsoft.Extensions.DependencyInjection.IServiceScopeFactory>(), scope.Resolve<ILogger<SupervisorTurnService>>());
 
     private sealed class AlwaysSpawnDecider : ISupervisorDecider
     {

backend/tests/CodeSpace.IntegrationTests/Learning/LessonArmSupervisorFlowTests.cs

@@ -219,7 +219,7 @@ private async Task<Guid> SeedLessonAsync(Guid teamId)
         scope.Resolve<Core.Services.Completion.ICompletionAssessmentComposer>(),
         scope.Resolve<Core.Services.Workflows.Budget.IBudgetLedger>(),
         lessons ?? scope.Resolve<ILessonReader>(),
-        scope.Resolve<ILogger<SupervisorTurnService>>());
+        scope.Resolve<Microsoft.Extensions.DependencyInjection.IServiceScopeFactory>(), scope.Resolve<ILogger<SupervisorTurnService>>());
 
     private static Lesson Lesson(Guid teamId, string howToApply) => new()
     {

backend/tests/CodeSpace.IntegrationTests/Workflows/Supervisor/RealModelChecksBeforeCriticE2ETests.cs

@@ -105,7 +105,7 @@ private async Task RunPlanTurnAsync(Guid runId, Guid teamId, Guid reviewerRowId,
             scope.Resolve<IDecisionAnswerService>(),
             scope.Resolve<CodeSpace.Core.Services.Plans.IWorkPlanService>(),
             scope.Resolve<CodeSpace.Core.Services.Workflows.Lifecycle.IRunRecordLogger>(), scope.Resolve<CodeSpace.Core.Services.Workflows.Artifacts.IArtifactOffloader>(), scope.Resolve<CodeSpace.Core.Services.Agents.Publish.IPublishManifestStore>(), scope.Resolve<CodeSpace.Core.Services.Supervisor.ISupervisorPublishedBranchResolver>(), scope.Resolve<CodeSpace.Core.Services.Completion.ICompletionAssessmentComposer>(), new AdmitAllBudgetLedger(),
-        scope.Resolve<CodeSpace.Core.Services.Learning.ILessonReader>(), scope.Resolve<ILogger<SupervisorTurnService>>());
+        scope.Resolve<CodeSpace.Core.Services.Learning.ILessonReader>(), scope.Resolve<Microsoft.Extensions.DependencyInjection.IServiceScopeFactory>(), scope.Resolve<ILogger<SupervisorTurnService>>());
 
         var goalConfig = new SupervisorGoalConfig { Goal = Goal, DecisionReviewMode = ReviewMode.Gate, ReviewerModelId = reviewerRowId };