diff --git a/src/apps/api/serializers/submissions.py b/src/apps/api/serializers/submissions.py index 9c91737ca..769ef0131 100644 --- a/src/apps/api/serializers/submissions.py +++ b/src/apps/api/serializers/submissions.py @@ -1,11 +1,15 @@ import asyncio +import logging from os.path import basename from django.core.cache import cache from django.core.exceptions import ValidationError +from django.db import transaction from rest_framework import serializers from rest_framework.exceptions import PermissionDenied +logger = logging.getLogger(__name__) + from api.mixins import DefaultUserCreateMixin from api.serializers import leaderboards from api.serializers.tasks import TaskSerializer @@ -163,6 +167,7 @@ def validate(self, attrs): return data + @transaction.atomic def update(self, submission, validated_data): # Cannot change submission if secret key is not valid @@ -198,13 +203,29 @@ def update(self, submission, validated_data): self.instance.parent.save() if validated_data.get("status") == Submission.SCORING: - # Start scoring because we're "SCORING" status now from compute worker + # Re-enqueue scoring AFTER the new status is committed: otherwise the + # site-worker may pick the task up before the row reflects SCORING, + # and a broker error here would leave the row stuck in SCORING forever. from competitions.tasks import run_submission - # task = validated_data.get('task_pk') - # if not task: - # raise ValidationError('Cannot update submission. Task pk was not provided') - # task = Task.objects.get(id=task) - run_submission(submission.pk, tasks=[submission.task], is_scoring=True) + submission_pk = submission.pk + scoring_task = submission.task + + def _enqueue_scoring(): + try: + run_submission(submission_pk, tasks=[scoring_task], is_scoring=True) + except Exception: + logger.exception( + "Failed to re-enqueue scoring for submission %s; marking Failed", + submission_pk, + ) + Submission.objects.filter( + pk=submission_pk, status=Submission.SCORING, + ).update( + status=Submission.FAILED, + status_details="Broker unavailable when re-enqueuing scoring task", + ) + + transaction.on_commit(_enqueue_scoring) elif validated_data.get("status") == Submission.FINISHED: # We finished submission, no longer need to store submission stuff in Redis, free it up!