fix(runtime): persist Cancelling before cancellation dispatch

core/src/clients/Elsa.Api.Client/Resources/WorkflowInstances/Enums/WorkflowSubStatus.cs

@@ -40,4 +40,9 @@ public enum WorkflowSubStatus
     /// The instance is resumable and will be picked up by the runtime's shell-activation recovery scan.
     /// </summary>
     Interrupted,
+
+    /// <summary>
+    /// A cancellation request has been accepted and the instance is waiting for cancellation to finish.
+    /// </summary>
+    Cancelling,
 }

core/src/modules/Elsa.Persistence.EFCore/Modules/Management/WorkflowInstanceStore.cs

@@ -216,6 +216,40 @@ public async ValueTask<bool> TryMarkInterruptedAsync(string workflowInstanceId,
         return updated > 0;
     }
 
+    /// <inheritdoc />
+    public async ValueTask<bool> TryTransitionSubStatusAsync(
+        string workflowInstanceId,
+        WorkflowSubStatus expected,
+        DateTimeOffset? expectedUpdatedAt,
+        WorkflowSubStatus next,
+        DateTimeOffset stampedAt,
+        CancellationToken cancellationToken = default)
+    {
+        if (!expected.IsPermittedSubStatusTransition(next))
+            return false;
+
+        await using var dbContext = await _store.CreateDbContextAsync(cancellationToken);
+        // The tenant query filter applies to ExecuteUpdateAsync. Do not ignore it.
+        var query = dbContext.WorkflowInstances.Where(x =>
+            x.Id == workflowInstanceId
+            && x.Status == WorkflowStatus.Running
+            && x.SubStatus == expected);
+
+        if (expectedUpdatedAt is { } expectedStamp)
+            query = query.Where(x => x.UpdatedAt == expectedStamp);
+
+        if (next == WorkflowSubStatus.Cancelling)
+            query = query.Where(x => !x.IsExecuting && x.SubStatus != WorkflowSubStatus.Executing);
+
+        var updated = await query.ExecuteUpdateAsync(
+            setters => setters
+                .SetProperty(x => x.SubStatus, next)
+                .SetProperty(x => x.UpdatedAt, stampedAt),
+            cancellationToken);
+
+        return updated > 0;
+    }
+
     /// <inheritdoc />
     [RequiresUnreferencedCode("Calls Elsa.Workflows.Contracts.IWorkflowStateSerializer.SerializeAsync(WorkflowState, CancellationToken)")]
     public async ValueTask SaveAsync(WorkflowInstance instance, CancellationToken cancellationToken = default)

core/src/modules/Elsa.Workflows.Core/Contexts/WorkflowExecutionContext.cs

@@ -705,6 +705,7 @@ private WorkflowStatus GetMainStatus(WorkflowSubStatus subStatus) =>
             WorkflowSubStatus.Faulted => WorkflowStatus.Finished,
             WorkflowSubStatus.Finished => WorkflowStatus.Finished,
             WorkflowSubStatus.Suspended => WorkflowStatus.Running,
+            WorkflowSubStatus.Cancelling => WorkflowStatus.Running,
             _ => throw new ArgumentOutOfRangeException(nameof(subStatus), subStatus, null)
         };
 

core/src/modules/Elsa.Workflows.Core/Enums/WorkflowSubStatus.cs

@@ -43,4 +43,11 @@ public enum WorkflowSubStatus
     /// sub-status with a stale liveness timestamp.
     /// </summary>
     Interrupted,
+
+    /// <summary>
+    /// A cancellation request has been accepted and the instance is waiting for the cancellation pipeline to finish.
+    /// The main status stays <see cref="WorkflowStatus.Running"/> until the pipeline commits <see cref="Cancelled"/>
+    /// or the mark is restored to the previous sub-status.
+    /// </summary>
+    Cancelling,
 }

core/src/modules/Elsa.Workflows.Core/Extensions/WorkflowSubStatusExtensions.cs

@@ -0,0 +1,40 @@
+namespace Elsa.Workflows;
+
+/// <summary>
+/// Shared checks for the cancellation interim sub-status.
+/// </summary>
+public static class WorkflowSubStatusExtensions
+{
+    /// <summary>
+    /// The sub-status Phase A will leave for <see cref="WorkflowSubStatus.Cancelling"/>.
+    /// Pending stays on the pending run path and Interrupted stays on the interrupted-workflow scan,
+    /// so a lost dispatch cannot take either of them out of recovery.
+    /// </summary>
+    public static bool IsCancellableSubStatus(this WorkflowSubStatus subStatus) =>
+        subStatus == WorkflowSubStatus.Suspended;
+
+    /// <summary>
+    /// A transition into <see cref="WorkflowSubStatus.Cancelling"/> starts from <see cref="IsCancellableSubStatus"/>.
+    /// A transition out of <see cref="WorkflowSubStatus.Cancelling"/> returns to that same sub-status.
+    /// Any other pair is not a cancellation transition.
+    /// </summary>
+    public static bool IsPermittedSubStatusTransition(this WorkflowSubStatus expected, WorkflowSubStatus next)
+    {
+        if (next == WorkflowSubStatus.Cancelling)
+            return expected.IsCancellableSubStatus();
+
+        if (expected == WorkflowSubStatus.Cancelling)
+            return next.IsCancellableSubStatus();
+
+        return true;
+    }
+
+    /// <summary>
+    /// Drops sub-millisecond ticks so stamps survive providers that round timestamps to microseconds.
+    /// </summary>
+    public static DateTimeOffset TruncateToMilliseconds(this DateTimeOffset value)
+    {
+        var ticks = value.Ticks - (value.Ticks % TimeSpan.TicksPerMillisecond);
+        return new DateTimeOffset(ticks, value.Offset);
+    }
+}

core/src/modules/Elsa.Workflows.Management/Contracts/IWorkflowInstanceStore.cs

@@ -199,4 +199,33 @@ public interface IWorkflowInstanceStore
     /// Implementations must not overwrite a concurrent naturally completed commit.
     /// </returns>
     ValueTask<bool> TryMarkInterruptedAsync(string workflowInstanceId, CancellationToken cancellationToken = default, bool allowFinishedCancelled = false);
+
+    /// <summary>
+    /// Sets <see cref="WorkflowInstance.SubStatus"/> to <paramref name="next"/> and
+    /// <see cref="WorkflowInstance.UpdatedAt"/> to <paramref name="stampedAt"/> when the stored instance is still
+    /// <see cref="WorkflowStatus.Running"/> and still <paramref name="expected"/>.
+    /// </summary>
+    /// <remarks>
+    /// When <paramref name="expectedUpdatedAt"/> has a value, <see cref="WorkflowInstance.UpdatedAt"/> must equal it.
+    /// A transition to <see cref="WorkflowSubStatus.Cancelling"/> also requires the row not to be executing and
+    /// <paramref name="expected"/> to be <see cref="WorkflowSubStatus.Suspended"/>.
+    /// A transition out of <see cref="WorkflowSubStatus.Cancelling"/> may only return to <see cref="WorkflowSubStatus.Suspended"/>.
+    /// Only those two columns are written. <see cref="WorkflowInstance.WorkflowState"/>, bookmarks and execution progress stay as they are.
+    /// The default returns <c>false</c> so a provider that cannot do this conditional write fails closed.
+    /// Callers must not fall back to <see cref="SaveAsync"/>, <see cref="SaveManyAsync"/> or <see cref="UpdateAsync"/>.
+    /// </remarks>
+    /// <param name="workflowInstanceId">The workflow instance to update.</param>
+    /// <param name="expected">The sub-status the row must still have.</param>
+    /// <param name="expectedUpdatedAt">The <see cref="WorkflowInstance.UpdatedAt"/> the row must still have, or <c>null</c> to skip that compare.</param>
+    /// <param name="next">The sub-status to write.</param>
+    /// <param name="stampedAt">The stamp written to <see cref="WorkflowInstance.UpdatedAt"/>.</param>
+    /// <param name="cancellationToken">The cancellation token.</param>
+    /// <returns><c>true</c> when the row was updated; otherwise <c>false</c> and the row is unchanged.</returns>
+    ValueTask<bool> TryTransitionSubStatusAsync(
+        string workflowInstanceId,
+        WorkflowSubStatus expected,
+        DateTimeOffset? expectedUpdatedAt,
+        WorkflowSubStatus next,
+        DateTimeOffset stampedAt,
+        CancellationToken cancellationToken = default) => ValueTask.FromResult(false);
 }

core/src/modules/Elsa.Workflows.Management/Stores/MemoryWorkflowInstanceStore.cs

@@ -212,6 +212,35 @@ public ValueTask<bool> TryMarkInterruptedAsync(string workflowInstanceId, Cancel
         }
     }
 
+    /// <inheritdoc />
+    public ValueTask<bool> TryTransitionSubStatusAsync(
+        string workflowInstanceId,
+        WorkflowSubStatus expected,
+        DateTimeOffset? expectedUpdatedAt,
+        WorkflowSubStatus next,
+        DateTimeOffset stampedAt,
+        CancellationToken cancellationToken = default)
+    {
+        if (!expected.IsPermittedSubStatusTransition(next))
+            return ValueTask.FromResult(false);
+
+        // Same lock as Save/Update so a runner's commit cannot land between the predicate and the column writes.
+        lock (_sync)
+        {
+            var instance = _store.Find(x => x.Id == workflowInstanceId && TenantVisibility.IsVisible(x.TenantId, CurrentTenantId));
+            if (instance is null
+                || instance.Status != WorkflowStatus.Running
+                || instance.SubStatus != expected
+                || (expectedUpdatedAt is { } expectedStamp && instance.UpdatedAt != expectedStamp)
+                || (next == WorkflowSubStatus.Cancelling && (instance.IsExecuting || instance.SubStatus == WorkflowSubStatus.Executing)))
+                return ValueTask.FromResult(false);
+
+            instance.SubStatus = next;
+            instance.UpdatedAt = stampedAt;
+            return ValueTask.FromResult(true);
+        }
+    }
+
     private static string GetId(WorkflowInstance workflowInstance) => workflowInstance.Id;
 
     private static WorkflowInstance Clone(WorkflowInstance instance) =>

core/src/modules/Elsa.Workflows.Runtime/Handlers/CancelWorkflowsCommandHandler.cs

@@ -1,23 +1,73 @@
+using Elsa.Common;
 using Elsa.Mediator.Contracts;
 using Elsa.Mediator.Models;
+using Elsa.Workflows.Management;
 using Elsa.Workflows.Runtime.Commands;
 using JetBrains.Annotations;
+using Microsoft.Extensions.Logging;
 
 namespace Elsa.Workflows.Runtime.Handlers;
 
 /// <summary>
 /// Handles the <see cref="CancelWorkflowsCommand"/>.
 /// </summary>
 [UsedImplicitly]
-public class CancelWorkflowsCommandHandler(IWorkflowRuntime workflowRuntime) : ICommandHandler<CancelWorkflowsCommand>
+public class CancelWorkflowsCommandHandler(
+    IWorkflowRuntime workflowRuntime,
+    IWorkflowInstanceStore workflowInstanceStore,
+    ISystemClock systemClock,
+    ILogger<CancelWorkflowsCommandHandler> logger) : ICommandHandler<CancelWorkflowsCommand>
 {
     /// <inheritdoc />
     public async Task<Unit> HandleAsync(CancelWorkflowsCommand command, CancellationToken cancellationToken)
     {
-        var workflowInstanceId = command.Request.WorkflowInstanceId;
-        var workflowClient = await workflowRuntime.CreateClientAsync(workflowInstanceId, cancellationToken);
-        await workflowClient.CancelAsync(cancellationToken);
-
-        return Unit.Instance;
+        try
+        {
+            var workflowInstanceId = command.Request.WorkflowInstanceId;
+            var workflowClient = await workflowRuntime.CreateClientAsync(workflowInstanceId, cancellationToken);
+            await workflowClient.CancelAsync(cancellationToken);
+            return Unit.Instance;
+        }
+        catch (Exception exception)
+        {
+            // BackgroundCommandProcessor logs this exception and does not retry the command.
+            // That makes the failure terminal for the in-process dispatcher. Restore first, then rethrow
+            // so the processor still records it. The compare-and-swap leaves a committed Cancelled row alone.
+            await RestoreIfMarkedAsync(command, exception);
+            throw;
+        }
     }
-}
+
+    private async Task RestoreIfMarkedAsync(CancelWorkflowsCommand command, Exception failure)
+    {
+        var request = command.Request;
+        if (request.PreviousSubStatus is not { } previousSubStatus || request.CancellingAt is not { } cancellingAt)
+            return;
+
+        try
+        {
+            var restored = await workflowInstanceStore.TryTransitionSubStatusAsync(
+                request.WorkflowInstanceId,
+                WorkflowSubStatus.Cancelling,
+                cancellingAt,
+                previousSubStatus,
+                systemClock.UtcNow.TruncateToMilliseconds(),
+                CancellationToken.None);
+
+            if (!restored)
+            {
+                logger.LogDebug(
+                    failure,
+                    "Background cancellation of workflow instance {WorkflowInstanceId} failed, and the Cancelling mark was no longer current.",
+                    request.WorkflowInstanceId);
+            }
+        }
+        catch (Exception restoreFailure)
+        {
+            logger.LogError(
+                restoreFailure,
+                "Failed to restore workflow instance {WorkflowInstanceId} after background cancellation failed.",
+                request.WorkflowInstanceId);
+        }
+    }
+}

core/src/modules/Elsa.Workflows.Runtime/Requests/DispatchCancelWorkflowRequest.cs

@@ -1,3 +1,5 @@
+using Elsa.Workflows;
+
 namespace Elsa.Workflows.Runtime.Requests;
 
 /// <summary>
@@ -9,4 +11,16 @@ public class DispatchCancelWorkflowRequest
     /// The ID of the workflow instance to cancel.
     /// </summary>
     public string WorkflowInstanceId { get; set; } = default!;
+
+    /// <summary>
+    /// The sub-status to restore if this cancellation fails after <see cref="WorkflowSubStatus.Cancelling"/> was persisted.
+    /// Null when no interim mark was written.
+    /// </summary>
+    public WorkflowSubStatus? PreviousSubStatus { get; set; }
+
+    /// <summary>
+    /// The <c>UpdatedAt</c> value written with the Cancelling mark.
+    /// Restore matches this stamp so a newer commit is left alone.
+    /// </summary>
+    public DateTimeOffset? CancellingAt { get; set; }
 }