From d1030238d054ce146bc595cd75b305286b3d8c3b Mon Sep 17 00:00:00 2001
From: Teng Yao Long <87478734+tengyaolong2000@users.noreply.github.com>
Date: Tue, 22 Apr 2025 09:40:10 +0800
Subject: [PATCH] fix regex bug in browsecomp

match.group(0) captures "correct: yes/no" when we just want "yes" or "no" (without "correct:") , match.group(1) fixes this
---
 browsecomp_eval.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/browsecomp_eval.py b/browsecomp_eval.py
index e246d52f..1fbaebcf 100644
--- a/browsecomp_eval.py
+++ b/browsecomp_eval.py
@@ -89,7 +89,7 @@ def grade_sample(self, question: str, correct_answer: str, response: str) -> str
         grading_response = self.grader_model(prompt_messages)
 
         match = re.search(r"correct: (yes|no)", grading_response)
-        return match.group(0) if match else "no"  # Default to "no" if no match
+        return match.group(1) if match else "no"  # Default to "no" if no match
 
     def __call__(self, sampler: SamplerBase) -> EvalResult:
             def fn(row: dict):