eval: include judge reasons for passed relevance criteria

The LLM judge already generates a reason for every criterion (pass and
fail), but CheckRelevance was discarding reasons for passed ones.

Changes:
- RelevanceResult now includes a Passed bool field
- CheckRelevance returns []RelevanceResult (all criteria) instead of
  (passed int, failed []RelevanceResult)
- populateEvalResult copies reasons for all criteria into the JSON output
- The JSON output now shows reason on every criterion, not just failures

Assisted-By: docker-agent

Author: hamza-jeddad

pkg/evaluation/eval.go

@@ -357,12 +357,18 @@ func (r *Runner) runSingleEval(ctx context.Context, evalSess *InputSession) (Res
 	if r.judge != nil && len(evals.Relevance) > 0 {
 		// Use transcript for relevance checking to preserve temporal ordering
 		transcript := buildTranscript(events)
-		passed, failed, err := r.judge.CheckRelevance(ctx, transcript, evals.Relevance)
+		results, err := r.judge.CheckRelevance(ctx, transcript, evals.Relevance)
 		if err != nil {
 			return result, fmt.Errorf("relevance check failed: %w", err)
 		}
-		result.RelevancePassed = float64(passed)
-		result.FailedRelevance = failed
+		var passed float64
+		for _, rr := range results {
+			if rr.Passed {
+				passed++
+			}
+		}
+		result.RelevancePassed = passed
+		result.RelevanceResults = results
 	}
 
 	slog.Debug("Evaluation complete", "title", evalSess.Title, "duration", time.Since(startTime))

pkg/evaluation/eval_test.go

@@ -196,7 +196,7 @@ func TestResultCheckResults(t *testing.T) {
 		},
 		{
 			name:         "relevance failures listed",
-			result:       Result{RelevanceExpected: 2, RelevancePassed: 0, FailedRelevance: []RelevanceResult{{Criterion: "check A", Reason: "reason A"}, {Criterion: "check B", Reason: "reason B"}}},
+			result:       Result{RelevanceExpected: 2, RelevancePassed: 0, RelevanceResults: []RelevanceResult{{Criterion: "check A", Passed: false, Reason: "reason A"}, {Criterion: "check B", Passed: false, Reason: "reason B"}}},
 			wantSuccess:  nil,
 			wantFailures: []string{"relevance: check A (reason: reason A)", "relevance: check B (reason: reason B)"},
 		},
@@ -658,7 +658,7 @@ func TestProgressBarPrintResult(t *testing.T) {
 				Size:              "S",
 				RelevanceExpected: 2,
 				RelevancePassed:   1,
-				FailedRelevance:   []RelevanceResult{{Criterion: "check failed", Reason: "did not meet criteria"}},
+				RelevanceResults:   []RelevanceResult{{Criterion: "check failed", Passed: false, Reason: "did not meet criteria"}},
 			},
 			wantContains: []string{
 				"✗ mixed-session", // overall failed

pkg/evaluation/judge.go

@@ -97,17 +97,18 @@ func (j *Judge) Validate(ctx context.Context) error {
 // RelevanceResult contains the result of a single relevance check.
 type RelevanceResult struct {
 	Criterion string `json:"criterion"`
+	Passed    bool   `json:"passed"`
 	Reason    string `json:"reason"`
 }
 
 // CheckRelevance runs all relevance checks concurrently with the configured concurrency.
-// It returns the number of passed checks, a slice of failed results with reasons, and an error
-// if any check encountered an error (e.g. judge model misconfiguration). Errors cause a hard
-// failure so that configuration issues are surfaced immediately rather than silently producing
-// zero-relevance results.
-func (j *Judge) CheckRelevance(ctx context.Context, response string, criteria []string) (passed int, failed []RelevanceResult, err error) {
+// It returns a result for every criterion (both passed and failed, each with a reason from
+// the judge model), and an error if any check encountered an error (e.g. judge model
+// misconfiguration). Errors cause a hard failure so that configuration issues are surfaced
+// immediately rather than silently producing zero-relevance results.
+func (j *Judge) CheckRelevance(ctx context.Context, response string, criteria []string) (results []RelevanceResult, err error) {
 	if len(criteria) == 0 {
-		return 0, nil, nil
+		return nil, nil
 	}
 
 	// Create work channel
@@ -122,23 +123,23 @@ func (j *Judge) CheckRelevance(ctx context.Context, response string, criteria []
 	close(work)
 
 	// Results slice preserves order
-	type result struct {
+	type rawResult struct {
 		passed bool
 		reason string
 		err    error
 	}
-	results := make([]result, len(criteria))
+	rawResults := make([]rawResult, len(criteria))
 
 	var wg sync.WaitGroup
 	for range j.concurrency {
 		wg.Go(func() {
 			for item := range work {
 				if ctx.Err() != nil {
-					results[item.index] = result{err: fmt.Errorf("context cancelled: %w", ctx.Err())}
+					rawResults[item.index] = rawResult{err: fmt.Errorf("context cancelled: %w", ctx.Err())}
 					continue
 				}
 				pass, reason, checkErr := j.checkSingle(ctx, response, item.criterion)
-				results[item.index] = result{passed: pass, reason: reason, err: checkErr}
+				rawResults[item.index] = rawResult{passed: pass, reason: reason, err: checkErr}
 			}
 		})
 	}
@@ -147,26 +148,24 @@ func (j *Judge) CheckRelevance(ctx context.Context, response string, criteria []
 	// Aggregate results. Any error is fatal — return it immediately so the
 	// caller can fail fast on judge misconfiguration.
 	var errs []error
-	for i, r := range results {
+	results = make([]RelevanceResult, len(criteria))
+	for i, r := range rawResults {
 		if r.err != nil {
 			errs = append(errs, fmt.Errorf("checking %q: %w", criteria[i], r.err))
 			continue
 		}
-		if r.passed {
-			passed++
-		} else {
-			failed = append(failed, RelevanceResult{
-				Criterion: criteria[i],
-				Reason:    r.reason,
-			})
+		results[i] = RelevanceResult{
+			Criterion: criteria[i],
+			Passed:    r.passed,
+			Reason:    r.reason,
 		}
 	}
 
 	if len(errs) > 0 {
-		return passed, failed, errors.Join(errs...)
+		return results, errors.Join(errs...)
 	}
 
-	return passed, failed, nil
+	return results, nil
 }
 
 // checkSingle checks a single relevance criterion against the response.

pkg/evaluation/judge_test.go

@@ -47,10 +47,9 @@ func TestJudge_CheckRelevance_EmptyCriteria(t *testing.T) {
 	t.Parallel()
 
 	judge := NewJudge(nil, 1)
-	passed, failed, err := judge.CheckRelevance(t.Context(), "some response", nil)
+	results, err := judge.CheckRelevance(t.Context(), "some response", nil)
 
-	assert.Equal(t, 0, passed)
-	assert.Empty(t, failed)
+	assert.Empty(t, results)
 	assert.NoError(t, err)
 }
 
@@ -63,11 +62,10 @@ func TestJudge_CheckRelevance_ContextCanceled(t *testing.T) {
 	cancel() // Cancel immediately
 
 	criteria := []string{"criterion1", "criterion2", "criterion3"}
-	passed, failed, err := judge.CheckRelevance(ctx, "some response", criteria)
+	results, err := judge.CheckRelevance(ctx, "some response", criteria)
 
 	// All should have errors due to context cancellation
-	assert.Equal(t, 0, passed)
-	assert.Empty(t, failed)
+	assert.Len(t, results, len(criteria))
 	require.Error(t, err)
 	assert.Contains(t, err.Error(), "context cancelled")
 }

pkg/evaluation/save.go

@@ -446,32 +446,13 @@ func populateEvalResult(result *Result) {
 
 	// Populate relevance check if relevance was evaluated
 	if result.RelevanceExpected > 0 {
-		// Build a map of failed criteria for quick lookup
-		failedMap := make(map[string]string, len(result.FailedRelevance))
-		for _, fr := range result.FailedRelevance {
-			failedMap[fr.Criterion] = fr.Reason
-		}
-
-		// Build results for ALL criteria (passed + failed) from the eval input
-		var criteria []string
-		if result.Session.Evals != nil {
-			criteria = result.Session.Evals.Relevance
-		}
-
-		results := make([]session.RelevanceCriterionResult, 0, len(criteria))
-		for _, criterion := range criteria {
-			if reason, failed := failedMap[criterion]; failed {
-				results = append(results, session.RelevanceCriterionResult{
-					Criterion: criterion,
-					Passed:    false,
-					Reason:    reason,
-				})
-			} else {
-				results = append(results, session.RelevanceCriterionResult{
-					Criterion: criterion,
-					Passed:    true,
-				})
-			}
+		results := make([]session.RelevanceCriterionResult, 0, len(result.RelevanceResults))
+		for _, rr := range result.RelevanceResults {
+			results = append(results, session.RelevanceCriterionResult{
+				Criterion: rr.Criterion,
+				Passed:    rr.Passed,
+				Reason:    rr.Reason,
+			})
 		}
 
 		evalResult.Checks.Relevance = &session.RelevanceCheck{

pkg/evaluation/save_test.go

@@ -171,7 +171,11 @@ func TestSaveRunSessionsJSON(t *testing.T) {
 				OutputTokens:      50,
 				RelevancePassed:   2,
 				RelevanceExpected: 2,
-				Session:           sess1,
+				RelevanceResults: []RelevanceResult{
+					{Criterion: "mentions Paris", Passed: true, Reason: "response includes Paris"},
+					{Criterion: "mentions France", Passed: true, Reason: "response includes France"},
+				},
+				Session: sess1,
 			},
 			{
 				Title:             "eval-json-2",
@@ -181,8 +185,9 @@ func TestSaveRunSessionsJSON(t *testing.T) {
 				OutputTokens:      30,
 				RelevancePassed:   1,
 				RelevanceExpected: 2,
-				FailedRelevance: []RelevanceResult{
-					{Criterion: "explains the math", Reason: "no explanation given"},
+				RelevanceResults: []RelevanceResult{
+					{Criterion: "gives the correct answer", Passed: true, Reason: "the response says 4"},
+					{Criterion: "explains the math", Passed: false, Reason: "no explanation given"},
 				},
 				Session: sess2,
 			},
@@ -275,9 +280,9 @@ func TestSaveRunSessionsJSON(t *testing.T) {
 	assert.Equal(t, float64(2), sess2Loaded.EvalResult.Checks.Relevance.Total)
 	require.Len(t, sess2Loaded.EvalResult.Checks.Relevance.Results, 2)
 
-	// First criterion should be passed (not in failed list)
+	// First criterion should be passed with reason
 	assert.True(t, sess2Loaded.EvalResult.Checks.Relevance.Results[0].Passed)
-	assert.Empty(t, sess2Loaded.EvalResult.Checks.Relevance.Results[0].Reason)
+	assert.Equal(t, "the response says 4", sess2Loaded.EvalResult.Checks.Relevance.Results[0].Reason)
 
 	// Second criterion should be failed with reason
 	assert.False(t, sess2Loaded.EvalResult.Checks.Relevance.Results[1].Passed)

pkg/evaluation/types.go

@@ -26,9 +26,9 @@ type Result struct {
 	SizeExpected      string            `json:"size_expected"`
 	ToolCallsScore    float64           `json:"tool_calls_score"`
 	ToolCallsExpected float64           `json:"tool_calls_score_expected"`
-	RelevancePassed   float64           `json:"relevance"`
-	RelevanceExpected float64           `json:"relevance_expected"`
-	FailedRelevance   []RelevanceResult `json:"failed_relevance,omitempty"`
+	RelevancePassed    float64           `json:"relevance"`
+	RelevanceExpected   float64           `json:"relevance_expected"`
+	RelevanceResults    []RelevanceResult `json:"relevance_results,omitempty"`
 	Error             string            `json:"error,omitempty"`
 	RawOutput         []map[string]any  `json:"raw_output,omitempty"`
 	Session           *session.Session  `json:"-"` // Full session for database storage (not in JSON)
@@ -63,11 +63,13 @@ func (r *Result) checkResults() (successes, failures []string) {
 		if r.RelevancePassed >= r.RelevanceExpected {
 			successes = append(successes, fmt.Sprintf("relevance %.0f/%.0f", r.RelevancePassed, r.RelevanceExpected))
 		} else {
-			for _, result := range r.FailedRelevance {
-				if result.Reason != "" {
-					failures = append(failures, fmt.Sprintf("relevance: %s (reason: %s)", result.Criterion, result.Reason))
-				} else {
-					failures = append(failures, "relevance: "+result.Criterion)
+			for _, result := range r.RelevanceResults {
+				if !result.Passed {
+					if result.Reason != "" {
+						failures = append(failures, fmt.Sprintf("relevance: %s (reason: %s)", result.Criterion, result.Reason))
+					} else {
+						failures = append(failures, "relevance: "+result.Criterion)
+					}
 				}
 			}
 		}

pkg/session/session.go

@@ -276,7 +276,7 @@ type RelevanceCheck struct {
 type RelevanceCriterionResult struct {
 	Criterion string `json:"criterion"`
 	Passed    bool   `json:"passed"`
-	Reason    string `json:"reason,omitempty"` // Only set for failed criteria
+	Reason    string `json:"reason,omitempty"`
 }
 
 // EvalCriteria contains the evaluation criteria for a session.