Fix submission cleanup: recover all non-terminal states, not just Running

src/apps/competitions/tasks.py

@@ -795,11 +795,26 @@ def update_phase_statuses():
 
 @app.task(queue='site-worker')
 def submission_status_cleanup():
-    submissions = Submission.objects.filter(status=Submission.RUNNING, has_children=False).select_related('phase', 'parent')
+    # Recover submissions stuck in any non-terminal state
+    non_terminal_statuses = [
+        Submission.SUBMITTED,
+        Submission.PREPARING,
+        Submission.RUNNING,
+        Submission.SCORING,
+    ]
+    submissions = Submission.objects.filter(
+        status__in=non_terminal_statuses,
+        has_children=False,
+    ).select_related('phase', 'parent')
 
     for sub in submissions:
-        # Check if the submission has been running for 24 hours longer than execution_time_limit
-        if sub.started_when < now() - timedelta(milliseconds=(3600000 * 24) + sub.phase.execution_time_limit):
+        # Use started_when for Running submissions, created_when as fallback for others
+        reference_time = sub.started_when if sub.started_when else sub.created_when
+        deadline = reference_time + timedelta(
+            milliseconds=(3600000 * 24) + sub.phase.execution_time_limit
+        )
+
+        if now() > deadline:
             if sub.parent is not None:
                 sub.parent.cancel(status=Submission.FAILED)
             else:

src/apps/competitions/tests/test_submissions.py

@@ -427,6 +427,51 @@ def test_submissions_are_cancelled_if_running_24_hours_past_execution_time_limit
         assert self.submission_pass.status == Submission.RUNNING
         assert self.submission_fail.status == Submission.FAILED
 
+    def test_cleanup_recovers_stuck_submitted_submissions(self):
+        """Submissions stuck in Submitted should be recovered by cleanup."""
+        sub = self.make_submission()
+        sub.status = Submission.SUBMITTED
+        sub.created_when = timezone.now() - timedelta(hours=48)
+        sub.save(ignore_submission_limit=True)
+
+        submission_status_cleanup()
+        sub.refresh_from_db()
+        assert sub.status == Submission.FAILED
+
+    def test_cleanup_recovers_stuck_preparing_submissions(self):
+        """Submissions stuck in Preparing should be recovered by cleanup."""
+        sub = self.make_submission()
+        sub.status = Submission.PREPARING
+        sub.created_when = timezone.now() - timedelta(hours=48)
+        sub.save(ignore_submission_limit=True)
+
+        submission_status_cleanup()
+        sub.refresh_from_db()
+        assert sub.status == Submission.FAILED
+
+    def test_cleanup_recovers_stuck_scoring_submissions(self):
+        """Submissions stuck in Scoring should be recovered by cleanup."""
+        sub = self.make_submission()
+        sub.status = Submission.SCORING
+        sub.created_when = timezone.now() - timedelta(hours=48)
+        sub.save(ignore_submission_limit=True)
+
+        submission_status_cleanup()
+        sub.refresh_from_db()
+        assert sub.status == Submission.FAILED
+
+    def test_cleanup_does_not_touch_recent_non_terminal_submissions(self):
+        """Recent submissions in non-terminal states should NOT be cleaned up."""
+        for status in [Submission.SUBMITTED, Submission.PREPARING, Submission.SCORING]:
+            sub = self.make_submission()
+            sub.status = status
+            sub.created_when = timezone.now()
+            sub.save(ignore_submission_limit=True)
+
+            submission_status_cleanup()
+            sub.refresh_from_db()
+            assert sub.status == status, f"Recent {status} submission should not be cleaned up"
+
     def test_cancelling_parent_submission_cancels_all_children(self):
         self.parent_submission = self.make_submission()
         self.parent_submission.has_children = True