From 213d09d92b451a3deeb0295c4de9d92b7f8f4deb Mon Sep 17 00:00:00 2001 From: Omkar P <45419097+omkar-foss@users.noreply.github.com> Date: Tue, 4 Aug 2026 12:52:28 +0530 Subject: [PATCH 1/9] Add numeric scoring with weights configurable via cli Signed-off-by: Omkar P <45419097+omkar-foss@users.noreply.github.com> --- cmd/cmd.go | 87 +++- cmd/cmd_test.go | 257 ++++++++++ detection/committer/committer.go | 18 +- detection/committer/committer_test.go | 40 +- detection/constants.go | 24 + detection/detection.go | 135 ++++++ detection/detection_test.go | 563 +++++++++++++++++++++- detection/gitnotes/gitnotes.go | 9 +- detection/gitnotes/gitnotes_test.go | 36 +- detection/toolmention/toolmention.go | 8 + detection/toolmention/toolmention_test.go | 73 ++- detection/trailer/trailer.go | 76 ++- detection/trailer/trailer_test.go | 89 +++- output/output.go | 42 +- output/output_test.go | 192 ++++++-- scan/scan.go | 24 +- scan/scan_test.go | 160 +++++- 17 files changed, 1716 insertions(+), 117 deletions(-) diff --git a/cmd/cmd.go b/cmd/cmd.go index 5f4afa7..a949d51 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -4,6 +4,7 @@ import ( "fmt" "io" "log" + "math" "os" "path/filepath" "slices" @@ -83,6 +84,8 @@ func scanCommand(stdout, stderr io.Writer, exitCode *int) *cobra.Command { var rangeFlag string var formatFlag string var minConfFlag string + var weightsFlag string + var confidenceScoresFlag string cmd := &cobra.Command{ Use: "scan [repo-path]", @@ -126,13 +129,78 @@ Examples: repoPath = args[0] } - minConf, err := output.ConfidenceFromString(minConfFlag) + minConf, err := detection.ConfidenceFromString(minConfFlag) if err != nil { fmt.Fprintln(stderr, err) *exitCode = ExitError return err } + // parse confidence-scores override if provided + if strings.TrimSpace(confidenceScoresFlag) != "" { + flagMap := map[string]float64{} + parts := strings.SplitSeq(confidenceScoresFlag, ",") + for p := range parts { + kv := strings.SplitN(p, "=", 2) + if len(kv) != 2 { + err := fmt.Errorf("invalid confidence-scores entry: %q", p) + fmt.Fprintln(stderr, err) + *exitCode = ExitError + return err + } + key := strings.TrimSpace(kv[0]) + var val float64 + if _, err := fmt.Sscan(strings.TrimSpace(kv[1]), &val); err != nil { + fmt.Fprintln(stderr, "invalid number in confidence-scores:", kv[1]) + *exitCode = ExitError + return err + } + if math.IsNaN(val) || math.IsInf(val, 0) || val < 0 || val > 100 { + err := fmt.Errorf("invalid confidence score: %v", val) + fmt.Fprintln(stderr, err) + *exitCode = ExitError + return err + } + flagMap[key] = val + } + if err := detection.SetConfidenceScoresFromStrings(flagMap); err != nil { + fmt.Fprintln(stderr, err) + *exitCode = ExitError + return err + } + } + + // parse weights flag + scan.Weights = nil + if strings.TrimSpace(weightsFlag) != "" { + weightMap := map[string]float64{} + parts := strings.SplitSeq(weightsFlag, ",") + for p := range parts { + kv := strings.SplitN(p, "=", 2) + if len(kv) != 2 { + err := fmt.Errorf("invalid weights entry: %q", p) + fmt.Fprintln(stderr, err) + *exitCode = ExitError + return err + } + name := strings.TrimSpace(kv[0]) + var val float64 + if _, err := fmt.Sscan(strings.TrimSpace(kv[1]), &val); err != nil { + fmt.Fprintln(stderr, "invalid number in weights:", kv[1]) + *exitCode = ExitError + return err + } + if math.IsNaN(val) || math.IsInf(val, 0) { + err := fmt.Errorf("invalid weight value: %v", val) + fmt.Fprintln(stderr, err) + *exitCode = ExitError + return err + } + weightMap[name] = val + } + scan.Weights = weightMap + } + detectors := allDetectors() report, err := scan.ScanCommitRange(repoPath, rangeFlag, detectors) if err != nil { @@ -173,6 +241,8 @@ Examples: cmd.Flags().StringVar(&rangeFlag, "range", "", "commit range in BASE..HEAD format") cmd.Flags().StringVar(&formatFlag, "format", "text", "output format: json or text") cmd.Flags().StringVar(&minConfFlag, "min-confidence", "low", "minimum confidence level: low, medium, high (or 1, 2, 3)") + cmd.Flags().StringVar(&weightsFlag, "weights", "", "comma-separated detector weights, e.g. 'trailer=0.8,toolmention=0.2'") + cmd.Flags().StringVar(&confidenceScoresFlag, "confidence-scores", "", "override confidence->score mapping, e.g. 'low=20,medium=60,high=100'") return cmd } @@ -291,6 +361,9 @@ func filterReport(report scan.Report, minConf detection.Confidence) scan.Report }, } + // collect all findings to compute overall score after filtering + var overallScoreFindings []detection.Finding + for _, cr := range report.Commits { var kept []detection.Finding for _, f := range cr.Findings { @@ -298,7 +371,10 @@ func filterReport(report scan.Report, minConf detection.Confidence) scan.Report kept = append(kept, f) } } - result := scan.CommitResult{Hash: cr.Hash, Findings: kept} + + // Recompute per-commit score from the kept findings and configured weights + commitScore, _ := detection.ConsolidateFindingScore(kept, scan.Weights) + result := scan.CommitResult{Hash: cr.Hash, Findings: kept, Score: commitScore} filtered.Commits = append(filtered.Commits, result) if len(kept) > 0 { @@ -307,9 +383,14 @@ func filterReport(report scan.Report, minConf detection.Confidence) scan.Report for _, f := range kept { filtered.Summary.ToolCounts[f.Tool]++ filtered.Summary.ByConfidence[f.Confidence.String()]++ + overallScoreFindings = append(overallScoreFindings, f) } } + // Compute new overall score for the filtered report using the same weights. + overall, _ := detection.ConsolidateFindingScore(overallScoreFindings, scan.Weights) + filtered.Summary.OverallScore = overall + return filtered } @@ -352,7 +433,7 @@ func generateDocs(exitCode *int) *cobra.Command { docDir = filepath.Clean(filepath.Join(outputDir, formatFlag)) err = os.MkdirAll(docDir, 0o750) } else { - err = fmt.Errorf("unknown format: %s\n", formatFlag) + err = fmt.Errorf("unknown format: %s", formatFlag) } if err != nil { return prepareError(err) diff --git a/cmd/cmd_test.go b/cmd/cmd_test.go index 659d411..59dd29a 100644 --- a/cmd/cmd_test.go +++ b/cmd/cmd_test.go @@ -195,6 +195,42 @@ func TestRunScanNoAI(t *testing.T) { } } +func TestRunTextCommandDetectsAI(t *testing.T) { + tmp := t.TempDir() + file := filepath.Join(tmp, "input.txt") + + os.WriteFile(file, []byte( + "I used Claude to write this", + ), 0644) + + var stdout, stderr bytes.Buffer + + code := Run([]string{ + "text", + "--input=" + file, + }, &stdout, &stderr) + + if code != ExitAI { + t.Errorf("code=%d want AI", code) + } +} + +func TestRunTextCommandNoAI(t *testing.T) { + tmp := t.TempDir() + file := filepath.Join(tmp, "input.txt") + os.WriteFile(file, []byte("plain text"), 0644) + + var stdout, stderr bytes.Buffer + code := Run([]string{ + "text", + "--input=" + file, + }, &stdout, &stderr) + + if code != ExitNoAI { + t.Errorf("code=%d want no AI", code) + } +} + func TestRunScanInvalidRepo(t *testing.T) { var stdout, stderr bytes.Buffer code := Run([]string{"scan", t.TempDir()}, &stdout, &stderr) @@ -203,6 +239,21 @@ func TestRunScanInvalidRepo(t *testing.T) { } } +func TestRunScanInvalidFormat(t *testing.T) { + dir := initTestRepo(t) + + var stdout, stderr bytes.Buffer + code := Run([]string{"scan", "--format=xml", dir}, &stdout, &stderr) + + if code != ExitError { + t.Errorf("exit code = %d, want %d", code, ExitError) + } + + if !strings.Contains(stderr.String(), "unknown format: xml") { + t.Errorf("expected invalid format error, got: %s", stderr.String()) + } +} + func TestFilterReport(t *testing.T) { report := scan.Report{ Commits: []scan.CommitResult{ @@ -234,6 +285,35 @@ func TestFilterReport(t *testing.T) { } } +func TestFilterReportAllFiltered(t *testing.T) { + report := scan.Report{ + Commits: []scan.CommitResult{ + { + Hash: "abc123", + Findings: []detection.Finding{ + { + Detector: "toolmention", + Confidence: detection.ConfidenceLow, + }, + }, + }, + }, + Summary: scan.Summary{ + TotalCommits: 1, + }, + } + + filtered := filterReport(report, detection.ConfidenceHigh) + + if filtered.Summary.AICommits != 0 { + t.Fatalf("AICommits=%d, want 0", filtered.Summary.AICommits) + } + + if len(filtered.Commits[0].Findings) != 0 { + t.Fatalf("expected no findings") + } +} + func TestRunDocsMarkdownDefault(t *testing.T) { // Clean up default directory paths after the test finishes defer func() { @@ -347,3 +427,180 @@ func TestRunDocsWriteError(t *testing.T) { t.Errorf("exit code = %d, want %d", code, ExitError) } } + +func TestRunDocsEmptyFormat(t *testing.T) { + var stdout, stderr bytes.Buffer + + code := Run([]string{ + "docs", + "--format=", + }, &stdout, &stderr) + + if code != ExitError { + t.Errorf("exit code=%d want error", code) + } +} + +func TestRunScanWithWeightsFlag(t *testing.T) { + dir := initTestRepo(t) + + var stdout, stderr bytes.Buffer + // Give trailer weight 0 and toolmention weight 1 so only toolmention contributes + code := Run([]string{"scan", "--format=json", "--weights=trailer=0.55,toolmention=0.45", dir}, &stdout, &stderr) + if code != ExitAI && code != ExitNoAI { + t.Fatalf("unexpected exit code: %d (stderr: %s)", code, stderr.String()) + } + + var report scan.Report + if err := json.Unmarshal(stdout.Bytes(), &report); err != nil { + t.Fatalf("unmarshal: %v (output: %s)", err, stdout.String()) + } + + var all []detection.Finding + for _, cr := range report.Commits { + all = append(all, cr.Findings...) + } + + weights := map[string]float64{"trailer": 0.55, "toolmention": 0.45} + expectedOverall, _ := detection.ConsolidateFindingScore(all, weights) + if report.Summary.OverallScore != expectedOverall { + t.Fatalf("overall score = %v, want %v (weights applied)", report.Summary.OverallScore, expectedOverall) + } +} + +func TestRunScanWithConfidenceScoresFlag(t *testing.T) { + dir := initTestRepo(t) + + var stdout, stderr bytes.Buffer + + // override confidence scores: low=10, medium=50, high=90 + code := Run([]string{"scan", "--format=json", "--confidence-scores=low=10,medium=50,high=90", dir}, &stdout, &stderr) + if code != ExitAI && code != ExitNoAI { + t.Fatalf("unexpected exit code: %d (stderr: %s)", code, stderr.String()) + } + + var report scan.Report + if err := json.Unmarshal(stdout.Bytes(), &report); err != nil { + t.Fatalf("unmarshal: %v (output: %s)", err, stdout.String()) + } + + var all []detection.Finding + for _, cr := range report.Commits { + all = append(all, cr.Findings...) + } + + expectedOverall, _ := detection.ConsolidateFindingScore(all, nil) + if report.Summary.OverallScore != expectedOverall { + t.Fatalf("overall score = %v, want %v (conf scores applied)", report.Summary.OverallScore, expectedOverall) + } +} + +func TestRunScanInvalidMinConfidence(t *testing.T) { + dir := initTestRepo(t) + + var stdout, stderr bytes.Buffer + code := Run([]string{ + "scan", + "--min-confidence=invalid", + dir, + }, &stdout, &stderr) + + if code != ExitError { + t.Errorf("exit code = %d, want %d", code, ExitError) + } + + if !strings.Contains(stderr.String(), "invalid confidence") { + t.Errorf("expected confidence error, got: %s", stderr.String()) + } +} + +func TestRunScanInvalidWeightsFormat(t *testing.T) { + dir := initTestRepo(t) + + var stdout, stderr bytes.Buffer + code := Run([]string{ + "scan", + "--weights=trailer", + dir, + }, &stdout, &stderr) + + if code != ExitError { + t.Errorf("exit code = %d, want %d", code, ExitError) + } +} + +func TestRunScanInvalidConfidenceScoresFormat(t *testing.T) { + dir := initTestRepo(t) + + var stdout, stderr bytes.Buffer + code := Run([]string{ + "scan", + "--confidence-scores=low", + dir, + }, &stdout, &stderr) + + if code != ExitError { + t.Errorf("exit code = %d, want %d", code, ExitError) + } +} + +func TestFilterReportRecalculatesScore(t *testing.T) { + report := scan.Report{ + Commits: []scan.CommitResult{ + { + Hash: "abc123", + Findings: []detection.Finding{ + { + Detector: "toolmention", + Confidence: detection.ConfidenceLow, + Score: 20, + }, + { + Detector: "trailer", + Confidence: detection.ConfidenceHigh, + Score: 100, + }, + }, + }, + }, + } + + filtered := filterReport(report, detection.ConfidenceHigh) + + if filtered.Commits[0].Score != 100 { + t.Errorf( + "score=%v want 100", + filtered.Commits[0].Score, + ) + } +} + +func TestRunScanRejectsNaNWeights(t *testing.T) { + dir := initTestRepo(t) + + var stdout, stderr bytes.Buffer + code := Run([]string{ + "scan", + "--weights=trailer=NaN", + dir, + }, &stdout, &stderr) + + if code != ExitError { + t.Errorf("expected error for NaN weight") + } +} + +func TestRunScanRejectsNaNConfidenceScore(t *testing.T) { + dir := initTestRepo(t) + + var stdout, stderr bytes.Buffer + code := Run([]string{ + "scan", + "--confidence-scores=high=NaN", + dir, + }, &stdout, &stderr) + + if code != ExitError { + t.Errorf("expected error for NaN confidence score") + } +} diff --git a/detection/committer/committer.go b/detection/committer/committer.go index 826f023..030142f 100644 --- a/detection/committer/committer.go +++ b/detection/committer/committer.go @@ -2,6 +2,7 @@ package committer import ( "fmt" + "log" "strings" "github.com/chaoss/disclosure/detection" @@ -25,12 +26,19 @@ type Detector struct{} func (d *Detector) Name() string { return "committer" } func (d *Detector) detectEmail(email, identityField string) []detection.Finding { + // Direct match against known emails if name, ok := detection.KnownAgentCommitters[email]; ok { + score := detection.CommitterMatchBaseScore + detection.CommitterKnownEmailBonusPoints + confidence, err := detection.ScoreToConfidence(score) + if err != nil { + log.Fatal(err) + } return []detection.Finding{{ Detector: d.Name(), Tool: name, - Confidence: detection.ConfidenceHigh, + Score: score, + Confidence: confidence, Detail: fmt.Sprintf("%s email %s matches known AI bot", identityField, email), }} } @@ -38,13 +46,19 @@ func (d *Detector) detectEmail(email, identityField string) []detection.Finding // Numeric prefix match for GitHub noreply emails (#4). // Format: +@users.noreply.github.com if strings.HasSuffix(email, detection.GithubNoReplyEmailSuffix) { + score := detection.CommitterMatchBaseScore + detection.CommitterEmailSuffixBonusPoints + confidence, err := detection.ScoreToConfidence(score) + if err != nil { + log.Fatal(err) + } if idx := strings.Index(email, "+"); idx > 0 { prefix := email[:idx] if name, ok := numericPrefixIndex[prefix]; ok { return []detection.Finding{{ Detector: d.Name(), Tool: name, - Confidence: detection.ConfidenceHigh, + Score: score, + Confidence: confidence, Detail: fmt.Sprintf("%s email %s matches known AI bot", identityField, email), }} } diff --git a/detection/committer/committer_test.go b/detection/committer/committer_test.go index d05ff3d..e11cfd8 100644 --- a/detection/committer/committer_test.go +++ b/detection/committer/committer_test.go @@ -6,6 +6,35 @@ import ( "github.com/chaoss/disclosure/detection" ) +const ( + directMatchScore = detection.CommitterMatchBaseScore + + detection.CommitterKnownEmailBonusPoints + + numericPrefixScore = detection.CommitterMatchBaseScore + + detection.CommitterEmailSuffixBonusPoints +) + +func assertFindingMetadata(t *testing.T, finding detection.Finding, expectedScore float64) { + t.Helper() + + if finding.Score != expectedScore { + t.Errorf("score = %v, want %v", finding.Score, expectedScore) + } + + expectedConfidence, err := detection.ScoreToConfidence(expectedScore) + if err != nil { + t.Fatalf("failed to calculate confidence: %v", err) + } + + if finding.Confidence != expectedConfidence { + t.Errorf("confidence = %d, want %d", finding.Confidence, expectedConfidence) + } + + if finding.Detector != "committer" { + t.Errorf("detector = %q, want %q", finding.Detector, "committer") + } +} + func TestDetectAllKnownEmails(t *testing.T) { d := &Detector{} for email, expectedName := range detection.KnownAgentCommitters { @@ -18,12 +47,7 @@ func TestDetectAllKnownEmails(t *testing.T) { if findings[0].Tool != expectedName { t.Errorf("Detect(%q): tool = %q, want %q", email, findings[0].Tool, expectedName) } - if findings[0].Confidence != detection.ConfidenceHigh { - t.Errorf("Detect(%q): confidence = %d, want %d", email, findings[0].Confidence, detection.ConfidenceHigh) - } - if findings[0].Detector != "committer" { - t.Errorf("Detect(%q): detector = %q, want %q", email, findings[0].Detector, "committer") - } + assertFindingMetadata(t, findings[0], directMatchScore) } } @@ -47,6 +71,7 @@ func TestDetectMixedCase(t *testing.T) { if findings[0].Tool != tc.wantTool { t.Errorf("Detect(%q): tool = %q, want %q", tc.input, findings[0].Tool, tc.wantTool) } + assertFindingMetadata(t, findings[0], directMatchScore) } } @@ -67,6 +92,7 @@ func TestDetectWhitespace(t *testing.T) { if findings[0].Tool != "Claude" { t.Errorf("Detect(%q): tool = %q, want %q", email, findings[0].Tool, "Claude") } + assertFindingMetadata(t, findings[0], directMatchScore) } } @@ -112,6 +138,7 @@ func TestDetectNumericPrefix(t *testing.T) { if findings[0].Tool != tc.wantTool { t.Errorf("Detect(%q): tool = %q, want %q", tc.input, findings[0].Tool, tc.wantTool) } + assertFindingMetadata(t, findings[0], numericPrefixScore) } } @@ -208,6 +235,7 @@ func TestDetectAuthorAndCommitter(t *testing.T) { if finding.Detail != tt.wantDetails[i] { t.Errorf("finding %d detail = %q, want %q", i, finding.Detail, tt.wantDetails[i]) } + assertFindingMetadata(t, finding, directMatchScore) } }) } diff --git a/detection/constants.go b/detection/constants.go index 75da9ed..fee3e0d 100644 --- a/detection/constants.go +++ b/detection/constants.go @@ -186,3 +186,27 @@ var GitNotesAuthorshipPrefix = "authorship/" // TrailerEmailPattern Regex to match email address in commit trailers var TrailerEmailPattern = regexp.MustCompile(`\s*<[^>]+>`) + +// Numeric scoring constants for respective detectors +const ( + // Trailer detector + CoauthoredByTrailerBaseScore float64 = 40.0 + CoauthorKnownEmailBonusPoints float64 = 35.0 + CoauthorModelBonusPoints float64 = 10.0 + AssistedByTrailerBaseScore float64 = 75.0 + TrailerMatchBaseScore float64 = 35.0 + TrailerNotMatchedScore float64 = 0.0 + AdditionalTrailerBonusPoints float64 = 20.0 + SessionIDBonusPoints float64 = 45.0 + + // Tool mention detector + ToolMentionBaseScore float64 = 20.0 + + // Committer detector + CommitterMatchBaseScore float64 = 75 + CommitterKnownEmailBonusPoints float64 = 20 + CommitterEmailSuffixBonusPoints float64 = 10 + + // Gitnotes detector + GitNotesMatchBaseScore float64 = 75 +) diff --git a/detection/detection.go b/detection/detection.go index 0c2e031..f07b096 100644 --- a/detection/detection.go +++ b/detection/detection.go @@ -2,6 +2,9 @@ package detection import ( "fmt" + "math" + "sort" + "strconv" "strings" ) @@ -9,6 +12,7 @@ import ( type Confidence int const ( + ConfidenceNone = 0 // Nil equivalent for confidence ConfidenceLow Confidence = 1 // Tool name mentioned in text ConfidenceMedium Confidence = 2 // Commit message pattern match ConfidenceHigh Confidence = 3 // Bot email, co-author trailer, git AI ref @@ -31,12 +35,80 @@ func (c *Confidence) Increment() { *c = min(*c+1, ConfidenceHigh) } +// Default mapping from Confidence -> numeric score (0..100). +var defaultConfidenceScores = map[Confidence]float64{ + ConfidenceLow: 30.0, + ConfidenceMedium: 70.0, + ConfidenceHigh: 100.0, +} + +// confidenceScores holds the active mapping, can be overridence in tests or via cli. +var confidenceScores = map[Confidence]float64{ + ConfidenceLow: defaultConfidenceScores[ConfidenceLow], + ConfidenceMedium: defaultConfidenceScores[ConfidenceMedium], + ConfidenceHigh: defaultConfidenceScores[ConfidenceHigh], +} + +func ScoreToConfidence(score float64) (Confidence, error) { + if score < 0 || score > 100 { + return ConfidenceNone, fmt.Errorf("invalid score, should be between 0 and 100") + } + levels := []Confidence{ConfidenceLow, ConfidenceMedium, ConfidenceHigh} + for _, level := range levels { + if math.Round(score) <= confidenceScores[level] { + return level, nil + } + } + return ConfidenceNone, fmt.Errorf("confidence intervals unable to categorize score") +} + +// SetConfidenceScoresFromStrings allows to update confidenceScores using a custom map +func SetConfidenceScoresFromStrings(userMapping map[string]float64) error { + tmp := map[Confidence]float64{} + for k, v := range userMapping { + k = strings.ToLower(strings.TrimSpace(k)) + switch k { + case "low": + tmp[ConfidenceLow] = v + case "medium": + tmp[ConfidenceMedium] = v + case "high": + tmp[ConfidenceHigh] = v + default: + return fmt.Errorf("unsupported confidence key: %s", k) + } + } + // set defaults if unspecified in user mapping + for c, def := range defaultConfidenceScores { + if _, ok := tmp[c]; !ok { + tmp[c] = def + } + } + confidenceScores = tmp + return nil +} + +// ConfidenceFromString parses a confidence string or numeric value. +func ConfidenceFromString(s string) (Confidence, error) { + switch strings.ToLower(strings.TrimSpace(s)) { + case "1", "low": + return ConfidenceLow, nil + case "2", "medium": + return ConfidenceMedium, nil + case "3", "high": + return ConfidenceHigh, nil + default: + return 0, fmt.Errorf("invalid confidence %q: use low/1, medium/2, or high/3", s) + } +} + // Finding represents a single detection of AI involvement. type Finding struct { Detector string `json:"detector"` Tool string `json:"tool"` Model string `json:"model,omitempty"` Confidence Confidence `json:"confidence"` + Score float64 `json:"score,omitempty"` Detail string `json:"detail"` } @@ -110,3 +182,66 @@ func (input *Input) GetTextWithCommitMessage() (string, error) { func (input *Input) GetNotes() (GitnoteParseResult, error) { return parseGitnotes(input.Notes) } + +// ConsolidateFindingScore computes per-detector scores and a consolidated overall +// score (0..100) using a weighted average across detectors. +// Things to note: +// - If weights is nil, detectors are equally weighted. We normalize provided weights so they sum to 1. +// - Detectors with missing weight entries are treated as zero weight. +// - Normalization will fallback to equal weights if total weight is zero. +func ConsolidateFindingScore(findings []Finding, weights map[string]float64) (float64, map[string]float64) { + perDetectorScores := map[string]float64{} + for _, f := range findings { + detectorName := strings.TrimSpace(f.Detector) + if cur, ok := perDetectorScores[detectorName]; !ok || f.Score > cur { + perDetectorScores[detectorName] = f.Score + } + } + + // No detectors found + if len(perDetectorScores) == 0 { + return 0.0, perDetectorScores + } + + // Prepare normalized weights + norm := map[string]float64{} + // equal weights for all detectors if weights unspecified + if weights == nil { + weight := 1.0 / float64(len(perDetectorScores)) + for detectorName := range perDetectorScores { + norm[detectorName] = weight + } + } else { + var sum float64 + for detectorName := range perDetectorScores { + currWeight := max(0, weights[detectorName]) + norm[detectorName] = currWeight + sum += currWeight + } + // if user-supplied sum is zero, fallback to equal weights + if sum == 0 { + weight := 1.0 / float64(len(perDetectorScores)) + for detectorName := range perDetectorScores { + norm[detectorName] = weight + } + } else { + for detectorName := range norm { + norm[detectorName] = norm[detectorName] / sum + } + } + } + + // Compute overall weighted average (deterministic order) + detectorNames := make([]string, 0, len(perDetectorScores)) + for detectorName := range perDetectorScores { + detectorNames = append(detectorNames, detectorName) + } + sort.Strings(detectorNames) + var overall float64 + for _, detectorName := range detectorNames { + overall += perDetectorScores[detectorName] * norm[detectorName] + } + overall = max(0, min(overall, 100)) + overall, _ = strconv.ParseFloat(fmt.Sprintf("%.2f", overall), 64) + return overall, perDetectorScores +} diff --git a/detection/detection_test.go b/detection/detection_test.go index 97d5d4c..cc2196e 100644 --- a/detection/detection_test.go +++ b/detection/detection_test.go @@ -1,6 +1,10 @@ package detection -import "testing" +import ( + "math" + "reflect" + "testing" +) func TestFindingDisplayTool(t *testing.T) { tests := []struct { @@ -48,3 +52,560 @@ func TestFindingDisplayTool(t *testing.T) { }) } } + +func TestConsolidateFindings(t *testing.T) { + tests := []struct { + name string + findings []Finding + weights map[string]float64 + wantOverall float64 + wantDetectorScores map[string]float64 + wantNaN bool + }{ + { + name: "equal weights", + findings: []Finding{ + {Detector: "A", Score: 100}, + {Detector: "B", Score: 50}, + {Detector: "C", Score: 75}, + }, + wantOverall: 75, + wantDetectorScores: map[string]float64{ + "A": 100, + "B": 50, + "C": 75, + }, + }, + { + name: "explicit weights", + findings: []Finding{ + {Detector: "A", Score: 100}, + {Detector: "B", Score: 50}, + {Detector: "C", Score: 75}, + }, + weights: map[string]float64{ + "A": 0.6, + "B": 0.2, + "C": 0.2, + }, + wantOverall: 85, + wantDetectorScores: map[string]float64{ + "A": 100, + "B": 50, + "C": 75, + }, + }, + { + name: "weights normalized", + findings: []Finding{ + {Detector: "A", Score: 100}, + {Detector: "B", Score: 50}, + {Detector: "C", Score: 75}, + }, + weights: map[string]float64{ + "A": 0.0, + "B": 0.2, + "C": 0.2, + }, + wantOverall: 62.5, + wantDetectorScores: map[string]float64{ + "A": 100, + "B": 50, + "C": 75, + }, + }, + { + name: "missing weights treated as zero", + findings: []Finding{ + {Detector: "A", Score: 100}, + {Detector: "B", Score: 50}, + {Detector: "C", Score: 75}, + }, + weights: map[string]float64{ + "A": 0.0, + "C": 0.2, + }, + wantOverall: 75, + wantDetectorScores: map[string]float64{ + "A": 100, + "B": 50, + "C": 75, + }, + }, + { + name: "nil findings", + findings: nil, + wantOverall: 0, + wantDetectorScores: map[string]float64{}, + }, + { + name: "empty findings", + findings: []Finding{}, + wantOverall: 0, + wantDetectorScores: map[string]float64{}, + }, + { + name: "duplicate detector keeps max score", + findings: []Finding{ + {Detector: "A", Score: 20}, + {Detector: "A", Score: 80}, + {Detector: "A", Score: 50}, + }, + wantOverall: 80, + wantDetectorScores: map[string]float64{ + "A": 80, + }, + }, + { + name: "detector names trimmed", + findings: []Finding{ + {Detector: " A ", Score: 60}, + {Detector: "A", Score: 90}, + }, + wantOverall: 90, + wantDetectorScores: map[string]float64{ + "A": 90, + }, + }, + { + name: "blank detector names collapse", + findings: []Finding{ + {Detector: "", Score: 10}, + {Detector: " ", Score: 55}, + }, + wantOverall: 55, + wantDetectorScores: map[string]float64{ + "": 55, + }, + }, + { + name: "empty weights fallback to equal", + findings: []Finding{ + {Detector: "A", Score: 100}, + {Detector: "B", Score: 50}, + {Detector: "C", Score: 75}, + }, + weights: map[string]float64{}, + wantOverall: 75, + wantDetectorScores: map[string]float64{ + "A": 100, + "B": 50, + "C": 75, + }, + }, + { + name: "all zero weights fallback to equal", + findings: []Finding{ + {Detector: "A", Score: 100}, + {Detector: "B", Score: 50}, + {Detector: "C", Score: 75}, + }, + weights: map[string]float64{ + "A": 0, + "B": 0, + "C": 0, + }, + wantOverall: 75, + wantDetectorScores: map[string]float64{ + "A": 100, + "B": 50, + "C": 75, + }, + }, + { + name: "negative weights treated as zero", + findings: []Finding{ + {Detector: "A", Score: 100}, + {Detector: "B", Score: 50}, + {Detector: "C", Score: 75}, + }, + weights: map[string]float64{ + "A": -1, + "B": 1, + "C": -2, + }, + wantOverall: 50, + wantDetectorScores: map[string]float64{ + "A": 100, + "B": 50, + "C": 75, + }, + }, + { + name: "extra weights ignored", + findings: []Finding{ + {Detector: "A", Score: 100}, + {Detector: "B", Score: 50}, + {Detector: "C", Score: 75}, + }, + weights: map[string]float64{ + "A": 1, + "B": 1, + "C": 1, + "D": 100, + }, + wantOverall: 75, + wantDetectorScores: map[string]float64{ + "A": 100, + "B": 50, + "C": 75, + }, + }, + { + name: "overall clamped above 100", + findings: []Finding{ + {Detector: "A", Score: 150}, + }, + wantOverall: 100, + wantDetectorScores: map[string]float64{ + "A": 150, + }, + }, + { + name: "overall clamped below zero", + findings: []Finding{ + {Detector: "A", Score: -50}, + }, + wantOverall: 0, + wantDetectorScores: map[string]float64{ + "A": -50, + }, + }, + { + name: "NaN score", + findings: []Finding{ + {Detector: "A", Score: math.NaN()}, + }, + wantNaN: true, + }, + { + name: "NaN weight", + findings: []Finding{ + {Detector: "A", Score: 100}, + {Detector: "B", Score: 50}, + }, + weights: map[string]float64{ + "A": math.NaN(), + "B": 1, + }, + wantNaN: true, + }, + { + name: "duplicate detectors keep max independently", + findings: []Finding{ + {Detector: "A", Score: 20}, + {Detector: "A", Score: 80}, + {Detector: "B", Score: 10}, + {Detector: "B", Score: 30}, + }, + wantOverall: 55, + wantDetectorScores: map[string]float64{ + "A": 80, + "B": 30, + }, + }, + { + name: "mixed positive and negative weights", + findings: []Finding{ + {Detector: "A", Score: 100}, + {Detector: "B", Score: 50}, + {Detector: "C", Score: 75}, + }, + weights: map[string]float64{ + "A": 2, + "B": -5, + "C": 2, + }, + // normalized -> 0.5, 0, 0.5 + wantOverall: 87.5, + wantDetectorScores: map[string]float64{ + "A": 100, + "B": 50, + "C": 75, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + overall, perDetectorScores := ConsolidateFindingScore(tt.findings, tt.weights) + + if tt.wantNaN { + if !math.IsNaN(overall) { + t.Fatalf("overall = %v, want NaN", overall) + } + return + } + + if overall != tt.wantOverall { + t.Fatalf("overall = %v, want %v", overall, tt.wantOverall) + } + + if !reflect.DeepEqual(perDetectorScores, tt.wantDetectorScores) { + t.Fatalf("per = %#v, want %#v", perDetectorScores, tt.wantDetectorScores) + } + }) + } +} + +func TestScoreToConfidence(t *testing.T) { + tests := []struct { + name string + score float64 + want Confidence + wantErr bool + }{ + { + name: "zero score", + score: 0, + want: ConfidenceLow, + wantErr: false, + }, + { + name: "normal low score", + score: 25, + want: ConfidenceLow, + wantErr: false, + }, + { + name: "medium boundary", + score: 50, + want: ConfidenceMedium, + wantErr: false, + }, + { + name: "high boundary", + score: 75, + want: ConfidenceHigh, + wantErr: false, + }, + { + name: "maximum score", + score: 100, + want: ConfidenceHigh, + wantErr: false, + }, + { + name: "negative score", + score: -1, + want: ConfidenceNone, + wantErr: true, + }, + { + name: "above maximum score", + score: 101, + want: ConfidenceNone, + wantErr: true, + }, + { + name: "positive infinity", + score: math.Inf(1), + want: ConfidenceNone, + wantErr: true, + }, + { + name: "negative infinity", + score: math.Inf(-1), + want: ConfidenceNone, + wantErr: true, + }, + { + name: "NaN score", + score: math.NaN(), + want: ConfidenceNone, + wantErr: true, + }, + { + name: "rounding to low boundary", + score: confidenceScores[ConfidenceLow] - 0.4, + want: ConfidenceLow, + wantErr: false, + }, + { + name: "rounding past low boundary", + score: confidenceScores[ConfidenceLow] + 0.5, + want: ConfidenceMedium, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := ScoreToConfidence(tt.score) + + if tt.wantErr { + if err == nil { + t.Fatalf("expected error, got nil") + } + return + } + + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if got != tt.want { + t.Fatalf("confidence=%v, want %v", got, tt.want) + } + }) + } +} + +func TestSetConfidenceScoresFromStrings(t *testing.T) { + tests := []struct { + name string + input map[string]float64 + wantErr bool + check func(t *testing.T) + }{ + { + name: "full custom mapping", + input: map[string]float64{ + "low": 30, + "medium": 60, + "high": 90, + }, + wantErr: false, + check: func(t *testing.T) { + if confidenceScores[ConfidenceLow] != 30 { + t.Fatalf("low score mismatch") + } + if confidenceScores[ConfidenceMedium] != 60 { + t.Fatalf("medium score mismatch") + } + if confidenceScores[ConfidenceHigh] != 90 { + t.Fatalf("high score mismatch") + } + }, + }, + { + name: "case insensitive keys", + input: map[string]float64{ + "LOW": 20, + "Medium": 50, + "HIGH": 80, + }, + wantErr: false, + check: func(t *testing.T) { + if confidenceScores[ConfidenceLow] != 20 { + t.Fatalf("low score mismatch") + } + }, + }, + { + name: "keys with whitespace", + input: map[string]float64{ + " low ": 25, + }, + wantErr: false, + check: func(t *testing.T) { + if confidenceScores[ConfidenceLow] != 25 { + t.Fatalf("low score mismatch") + } + }, + }, + { + name: "empty mapping uses defaults", + input: map[string]float64{}, + wantErr: false, + check: func(t *testing.T) { + if confidenceScores[ConfidenceLow] != defaultConfidenceScores[ConfidenceLow] { + t.Fatalf("default low mismatch") + } + }, + }, + { + name: "partial mapping fills defaults", + input: map[string]float64{ + "low": 10, + }, + wantErr: false, + check: func(t *testing.T) { + if confidenceScores[ConfidenceLow] != 10 { + t.Fatalf("custom low missing") + } + if confidenceScores[ConfidenceHigh] != defaultConfidenceScores[ConfidenceHigh] { + t.Fatalf("high should use default") + } + }, + }, + { + name: "unsupported key", + input: map[string]float64{ + "critical": 100, + }, + wantErr: true, + }, + { + name: "invalid key does not overwrite existing mapping", + input: map[string]float64{ + "low": 10, + "invalid": 20, + }, + wantErr: true, + }, + { + name: "negative threshold accepted", + input: map[string]float64{ + "low": -10, + }, + wantErr: false, + check: func(t *testing.T) { + if confidenceScores[ConfidenceLow] != -10 { + t.Fatalf("expected negative threshold to be accepted") + } + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := SetConfidenceScoresFromStrings(tt.input) + + if tt.wantErr { + if err == nil { + t.Fatalf("expected error, got nil") + } + return + } + + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if tt.check != nil { + tt.check(t) + } + }) + } +} + +func TestConfidenceFromString(t *testing.T) { + tests := []struct { + input string + want Confidence + err bool + }{ + {"low", ConfidenceLow, false}, + {"1", ConfidenceLow, false}, + {"medium", ConfidenceMedium, false}, + {"2", ConfidenceMedium, false}, + {"high", ConfidenceHigh, false}, + {"3", ConfidenceHigh, false}, + {"HIGH", ConfidenceHigh, false}, + {" low ", ConfidenceLow, false}, + {"invalid", 0, true}, + {"4", 0, true}, + {"", 0, true}, + } + + for _, tt := range tests { + got, err := ConfidenceFromString(tt.input) + if (err != nil) != tt.err { + t.Errorf("ConfidenceFromString(%q): err = %v, wantErr = %v", tt.input, err, tt.err) + continue + } + if got != tt.want { + t.Errorf("ConfidenceFromString(%q) = %d, want %d", tt.input, got, tt.want) + } + } +} diff --git a/detection/gitnotes/gitnotes.go b/detection/gitnotes/gitnotes.go index 84a4c76..abc65c7 100644 --- a/detection/gitnotes/gitnotes.go +++ b/detection/gitnotes/gitnotes.go @@ -2,6 +2,7 @@ package gitnotes import ( "fmt" + "log" "sort" "github.com/chaoss/disclosure/detection" @@ -29,6 +30,11 @@ func (d *Detector) Detect(input detection.Input) []detection.Finding { promptIDs = append(promptIDs, promptID) } sort.Strings(promptIDs) + score := detection.GitNotesMatchBaseScore + confidence, err := detection.ScoreToConfidence(score) + if err != nil { + log.Fatal(err) + } for _, promptID := range promptIDs { prompt := parseResult.Metadata.Prompts[promptID] @@ -55,7 +61,8 @@ func (d *Detector) Detect(input detection.Input) []detection.Finding { Detector: d.Name(), Tool: tool, Model: model, - Confidence: detection.ConfidenceHigh, + Score: score, + Confidence: confidence, Detail: detail, }) } diff --git a/detection/gitnotes/gitnotes_test.go b/detection/gitnotes/gitnotes_test.go index 373cc20..743df2c 100644 --- a/detection/gitnotes/gitnotes_test.go +++ b/detection/gitnotes/gitnotes_test.go @@ -6,6 +6,27 @@ import ( "github.com/chaoss/disclosure/detection" ) +func assertFindingMetadata(t *testing.T, finding detection.Finding, wantScore float64) { + t.Helper() + + if finding.Score != wantScore { + t.Errorf("score = %f, want %f", finding.Score, wantScore) + } + + expectedConfidence, err := detection.ScoreToConfidence(wantScore) + if err != nil { + t.Fatalf("failed to calculate confidence: %v", err) + } + + if finding.Confidence != expectedConfidence { + t.Errorf("confidence = %d, want %d", finding.Confidence, expectedConfidence) + } + + if finding.Detector != "gitnotes" { + t.Errorf("detector = %q, want %q", finding.Detector, "gitnotes") + } +} + func TestDetect(t *testing.T) { d := &Detector{} @@ -68,18 +89,21 @@ src/lib.rs notes string wantTools []string wantModels []string + wantScore float64 }{ { name: "valid git-ai note with single tool", notes: validNote, wantTools: []string{"cursor"}, wantModels: []string{"claude-4.5-opus"}, + wantScore: detection.GitNotesMatchBaseScore, }, { name: "multiple tools in note", notes: multiToolNote, wantTools: []string{"cursor", "claude-code"}, wantModels: []string{"claude-4.5-opus", "claude-3-sonnet"}, + wantScore: detection.GitNotesMatchBaseScore, }, { name: "empty notes", @@ -116,12 +140,8 @@ src/lib.rs for i, f := range findings { gotTools[i] = f.Tool gotModels[i] = f.Model - if f.Confidence != detection.ConfidenceHigh { - t.Errorf("confidence = %d, want %d", f.Confidence, detection.ConfidenceHigh) - } - if f.Detector != "gitnotes" { - t.Errorf("detector = %q, want %q", f.Detector, "gitnotes") - } + + assertFindingMetadata(t, f, tt.wantScore) } if len(gotTools) == 0 { @@ -196,6 +216,7 @@ func TestDetectPreservesDistinctToolModelPairs(t *testing.T) { findings := d.Detect(detection.Input{Notes: note}) wantTools := []string{"cursor", "cursor"} wantModels := []string{"claude-4.5-opus", "gpt-4o"} + wantScore := detection.GitNotesMatchBaseScore if len(findings) != len(wantTools) { t.Fatalf("expected %d findings, got %d: %#v", len(wantTools), len(findings), findings) @@ -207,6 +228,7 @@ func TestDetectPreservesDistinctToolModelPairs(t *testing.T) { if finding.Model != wantModels[i] { t.Errorf("model[%d] = %q, want %q", i, finding.Model, wantModels[i]) } + assertFindingMetadata(t, finding, wantScore) } } @@ -237,6 +259,8 @@ func TestDetectDetailIncludesModel(t *testing.T) { t.Fatalf("expected 1 finding, got %d", len(findings)) } + assertFindingMetadata(t, findings[0], detection.GitNotesMatchBaseScore) + if findings[0].Detail == "" { t.Error("expected non-empty detail") } diff --git a/detection/toolmention/toolmention.go b/detection/toolmention/toolmention.go index 0e44f3f..e453182 100644 --- a/detection/toolmention/toolmention.go +++ b/detection/toolmention/toolmention.go @@ -1,6 +1,8 @@ package toolmention import ( + "fmt" + "log" "regexp" "sort" "strings" @@ -92,6 +94,12 @@ func (d *Detector) Detect(input detection.Input) []detection.Finding { lastEnd = match.end } + score := detection.ToolMentionBaseScore + confidence, err := detection.ScoreToConfidence(score) + if err != nil { + log.Fatal(err) + } + findings := make([]detection.Finding, 0, len(toolMatches)) for _, match := range toolMatches { findings = append(findings, detection.Finding{ diff --git a/detection/toolmention/toolmention_test.go b/detection/toolmention/toolmention_test.go index d6d47bd..65695c1 100644 --- a/detection/toolmention/toolmention_test.go +++ b/detection/toolmention/toolmention_test.go @@ -13,96 +13,127 @@ func TestDetect(t *testing.T) { name string input detection.Input wantTools []string + wantScore []float64 }{ { name: "Claude mention in text", input: detection.Input{Text: "I used Claude to write this PR"}, wantTools: []string{"Claude"}, + wantScore: []float64{detection.ToolMentionBaseScore}, }, { name: "Claude Code mention in text", input: detection.Input{Text: "Generated with Claude Code"}, wantTools: []string{"Claude Code"}, + wantScore: []float64{detection.ToolMentionBaseScore}, }, { name: "GitHub Copilot mention", input: detection.Input{Text: "GitHub Copilot helped with this"}, wantTools: []string{"GitHub Copilot"}, + wantScore: []float64{detection.ToolMentionBaseScore}, }, { name: "Copilot mention", input: detection.Input{Text: "Copilot was used to generate docs"}, wantTools: []string{"Copilot"}, + wantScore: []float64{detection.ToolMentionBaseScore}, }, { name: "multiple tools mentioned", input: detection.Input{Text: "I used Cursor and Aider for this PR"}, wantTools: []string{"Cursor", "Aider"}, + wantScore: []float64{ + detection.ToolMentionBaseScore, + detection.ToolMentionBaseScore, + }, }, { name: "case insensitive", input: detection.Input{Text: "I used CLAUDE to write this"}, wantTools: []string{"Claude"}, + wantScore: []float64{detection.ToolMentionBaseScore}, }, { name: "commit message scanned too", input: detection.Input{CommitMessage: "feat: add feature\n\nGenerated with Claude Code"}, wantTools: []string{"Claude Code"}, + wantScore: []float64{detection.ToolMentionBaseScore}, }, { name: "text and commit message combined", input: detection.Input{Text: "Used Cursor", CommitMessage: "aider: fix bug"}, wantTools: []string{"Cursor", "Aider"}, + wantScore: []float64{ + detection.ToolMentionBaseScore, + detection.ToolMentionBaseScore, + }, }, { name: "no mentions", input: detection.Input{Text: "This is a normal PR description"}, wantTools: nil, + wantScore: nil, }, { name: "empty input with spaces", input: detection.Input{Text: " ", CommitMessage: "\n \n"}, wantTools: nil, + wantScore: nil, }, { name: "empty input", input: detection.Input{}, wantTools: nil, + wantScore: nil, }, { name: "word boundary prevents partial match", input: detection.Input{Text: "The cursory review found nothing"}, wantTools: nil, + wantScore: nil, }, { name: "ChatGPT mention", input: detection.Input{Text: "I asked ChatGPT for help"}, wantTools: []string{"ChatGPT"}, + wantScore: []float64{detection.ToolMentionBaseScore}, }, { name: "t3.chat mention", input: detection.Input{Text: "I used t3.chat to compare model outputs"}, wantTools: []string{"t3.chat"}, + wantScore: []float64{detection.ToolMentionBaseScore}, }, { name: "t3.chat mention is case insensitive", input: detection.Input{Text: "Generated with T3.CHAT"}, wantTools: []string{"t3.chat"}, + wantScore: []float64{detection.ToolMentionBaseScore}, }, { name: "t3.chat word boundary prevents partial match", input: detection.Input{Text: "This mentions t3.chatty, not the tool"}, wantTools: nil, + wantScore: nil, }, { name: "Windsurf mention", input: detection.Input{Text: "Written with Windsurf IDE"}, wantTools: []string{"Windsurf"}, + wantScore: []float64{detection.ToolMentionBaseScore}, }, { name: "Devin mention", input: detection.Input{Text: "Devin created this PR"}, wantTools: []string{"Devin"}, + wantScore: []float64{detection.ToolMentionBaseScore}, + }, + { + name: "duplicate tool mentions only produce one finding", + input: detection.Input{Text: "Claude helped here. Claude helped there."}, + wantTools: []string{"Claude"}, + wantScore: []float64{detection.ToolMentionBaseScore}, }, { name: "Qwen coder variant match", @@ -249,29 +280,35 @@ func TestDetect(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { findings := d.Detect(tt.input) - gotTools := make([]string, len(findings)) + + if len(findings) != len(tt.wantTools) { + t.Fatalf("findings count = %d, want %d. findings=%v", len(findings), len(tt.wantTools), findings) + } + + if len(findings) != len(tt.wantScore) { + t.Fatalf("score count = %d, want %d", len(findings), len(tt.wantScore)) + } + for i, f := range findings { - gotTools[i] = f.Tool - if f.Confidence != detection.ConfidenceLow { - t.Errorf("confidence = %d, want %d", f.Confidence, detection.ConfidenceLow) + if f.Tool != tt.wantTools[i] { + t.Errorf("tool[%d] = %q, want %q", i, f.Tool, tt.wantTools[i]) } - if f.Detector != "toolmention" { - t.Errorf("detector = %q, want %q", f.Detector, "toolmention") + + if f.Score != tt.wantScore[i] { + t.Errorf("score[%d] = %v, want %v", i, f.Score, tt.wantScore[i]) } - } - if len(gotTools) == 0 { - gotTools = nil - } + expectedConfidence, err := detection.ScoreToConfidence(tt.wantScore[i]) + if err != nil { + t.Fatalf("failed to calculate confidence: %v", err) + } - if len(gotTools) != len(tt.wantTools) { - t.Errorf("tools = %v, want %v", gotTools, tt.wantTools) - return - } - for i := range gotTools { - if gotTools[i] != tt.wantTools[i] { - t.Errorf("tools = %v, want %v", gotTools, tt.wantTools) - return + if f.Confidence != expectedConfidence { + t.Errorf("confidence[%d] = %d, want %d", i, f.Confidence, expectedConfidence) + } + + if f.Detector != "toolmention" { + t.Errorf("detector[%d] = %q, want %q", i, f.Detector, "toolmention") } } }) diff --git a/detection/trailer/trailer.go b/detection/trailer/trailer.go index cd70ab7..25bafee 100644 --- a/detection/trailer/trailer.go +++ b/detection/trailer/trailer.go @@ -2,6 +2,7 @@ package trailer import ( "fmt" + "log" "slices" "strings" @@ -36,57 +37,66 @@ func extractToolFromText(text string) (string, error) { } var commitMessagePatterns = []struct { - check func(string) (detection.Confidence, bool) + check func(string) (float64, bool) name string }{ { - check: func(msg string) (detection.Confidence, bool) { - return detection.ConfidenceMedium, strings.HasPrefix(strings.ToLower(msg), detection.AiderCommitPrefix) + check: func(msg string) (float64, bool) { + if strings.HasPrefix(strings.ToLower(msg), detection.AiderCommitPrefix) { + return detection.TrailerMatchBaseScore, true + } + return detection.TrailerNotMatchedScore, false }, name: "Aider", }, { - check: func(msg string) (detection.Confidence, bool) { - return detection.ConfidenceMedium, strings.Contains(msg, detection.ClaudeAttributionText) + check: func(msg string) (float64, bool) { + if strings.Contains(msg, detection.ClaudeAttributionText) { + return detection.TrailerMatchBaseScore, true + } + return detection.TrailerNotMatchedScore, false }, name: "Claude Code", }, { - check: func(msg string) (detection.Confidence, bool) { + check: func(msg string) (float64, bool) { + matchedTrailerCount := 0 for _, trailer := range detection.EntireIOTrailers { if strings.Contains(msg, fmt.Sprintf("\n%s:", trailer)) { - return detection.ConfidenceMedium, true + matchedTrailerCount += 1 } } - return detection.ConfidenceMedium, false + if matchedTrailerCount > 0 { + score := detection.TrailerMatchBaseScore + (float64(matchedTrailerCount-1))*detection.AdditionalTrailerBonusPoints + return score, true + } + return detection.TrailerNotMatchedScore, false }, name: "EntireIO", }, { - check: func(msg string) (detection.Confidence, bool) { + check: func(msg string) (float64, bool) { matchResult := detection.ReplitAttributionPattern.FindStringSubmatch(msg) if len(matchResult) == 0 { // replit not detected - return detection.ConfidenceMedium, false + return detection.TrailerNotMatchedScore, false } - var confidence detection.Confidence + var score float64 = 0 switch matchResult[1] { - case "Agent": - confidence = detection.ConfidenceMedium - case "Assistant": - confidence = detection.ConfidenceLow + case "Agent", "Assistant": + score += detection.TrailerMatchBaseScore default: // unknown replit product, we cannot confirm ai use - return detection.ConfidenceLow, false + return detection.TrailerNotMatchedScore, false } - // if commit session id also present, increase confidence + // bonus points if commit session id also present if matchResult[2] != "" { - confidence.Increment() + score += detection.SessionIDBonusPoints } - return confidence, true + return score, true }, name: "Replit", }, @@ -155,18 +165,28 @@ func (d *Detector) detectTrailerCoauthoredBy(commitMessage string) []detection.F namePart := strings.TrimSpace(match[1]) email := strings.ToLower(strings.TrimSpace(match[2])) + score := detection.CoauthoredByTrailerBaseScore if name, ok := detection.KnownCoAuthorEmails[email]; ok { model := extractCoauthorModel(name, namePart) + if model != "" { + score += detection.CoauthorModelBonusPoints + } key := toolModelPair{tool: name, model: model} if seen[key] { continue } + score += detection.CoauthorKnownEmailBonusPoints + confidence, err := detection.ScoreToConfidence(score) + if err != nil { + log.Fatal(err) + } findings = append(findings, detection.Finding{ Detector: d.Name(), Tool: name, Model: model, - Confidence: detection.ConfidenceHigh, + Score: score, + Confidence: confidence, Detail: fmt.Sprintf("Co-Authored-By trailer with email %s", email), }) seen[key] = true @@ -199,10 +219,17 @@ func (d *Detector) detectTrailerAssistedBy(commitMessage string) []detection.Fin continue } + score := detection.AssistedByTrailerBaseScore + confidence, err := detection.ScoreToConfidence(score) + if err != nil { + log.Fatal(err) + } + findings = append(findings, detection.Finding{ Detector: d.Name(), Tool: matchedTool, - Confidence: detection.ConfidenceHigh, + Score: score, + Confidence: confidence, Detail: fmt.Sprintf("Assisted-By trailer with tool %s", matchedTool), }) seen[matchedToolKey] = true @@ -213,10 +240,15 @@ func (d *Detector) detectTrailerAssistedBy(commitMessage string) []detection.Fin func (d *Detector) detectMessagePatterns(commitMessage string) []detection.Finding { var findings []detection.Finding for _, p := range commitMessagePatterns { - if confidence, isDetected := p.check(commitMessage); isDetected { + if score, isDetected := p.check(commitMessage); isDetected { + confidence, err := detection.ScoreToConfidence(score) + if err != nil { + log.Fatal(err) + } findings = append(findings, detection.Finding{ Detector: d.Name(), Tool: p.name, + Score: score, Confidence: confidence, Detail: fmt.Sprintf("commit message matches %s pattern", p.name), }) diff --git a/detection/trailer/trailer_test.go b/detection/trailer/trailer_test.go index 69b496e..2bcff9c 100644 --- a/detection/trailer/trailer_test.go +++ b/detection/trailer/trailer_test.go @@ -13,6 +13,7 @@ func TestDetect(t *testing.T) { message string wantTools []string wantModels []string + wantScore []float64 wantConfidence []detection.Confidence }{ // Co-Authored-By tests start here @@ -21,6 +22,7 @@ func TestDetect(t *testing.T) { message: "fix: update handler\n\nCo-Authored-By: Claude Opus 4 ", wantTools: []string{"Claude Code"}, wantModels: []string{"Opus 4"}, + wantScore: []float64{85}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh}, }, { @@ -28,6 +30,7 @@ func TestDetect(t *testing.T) { message: "fix: update handler\n\nCo-Authored-By: Claude Sonnet 4 ", wantTools: []string{"Claude Code"}, wantModels: []string{"Sonnet 4"}, + wantScore: []float64{85}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh}, }, { @@ -35,6 +38,7 @@ func TestDetect(t *testing.T) { message: "refactor: extract method\n\nCo-Authored-By: Cursor ", wantTools: []string{"Cursor"}, wantModels: []string{""}, + wantScore: []float64{75}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh}, }, { @@ -42,6 +46,7 @@ func TestDetect(t *testing.T) { message: "feat: add endpoint\n\nCo-Authored-By: aider (gpt-4o) ", wantTools: []string{"Aider"}, wantModels: []string{"gpt-4o"}, + wantScore: []float64{85}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh}, }, { @@ -49,6 +54,7 @@ func TestDetect(t *testing.T) { message: "feat: add endpoint\n\nCo-Authored-By: aider (claude-3.5-sonnet) ", wantTools: []string{"Aider"}, wantModels: []string{"claude-3.5-sonnet"}, + wantScore: []float64{85}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh}, }, { @@ -56,6 +62,7 @@ func TestDetect(t *testing.T) { message: "refactor: extract method\n\nCo-Authored-By: Cursor (composer 2.5) ", wantTools: []string{"Cursor"}, wantModels: []string{"composer 2.5"}, + wantScore: []float64{85}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh}, }, { @@ -63,6 +70,7 @@ func TestDetect(t *testing.T) { message: "feat: add endpoint\n\nCo-Authored-By: Copilot (gpt-4.1) ", wantTools: []string{"Copilot"}, wantModels: []string{"gpt-4.1"}, + wantScore: []float64{85}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh}, }, { @@ -70,6 +78,7 @@ func TestDetect(t *testing.T) { message: "fix: update handler\n\nCo-Authored-By: Claude Code (Opus 4.1) ", wantTools: []string{"Claude Code"}, wantModels: []string{"Opus 4.1"}, + wantScore: []float64{85}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh}, }, { @@ -77,6 +86,7 @@ func TestDetect(t *testing.T) { message: "fix: bug\n\nCo-Authored-By: Claude Opus 4 \nCo-Authored-By: Alice ", wantTools: []string{"Claude Code"}, wantModels: []string{"Opus 4"}, + wantScore: []float64{85}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh}, }, { @@ -84,6 +94,7 @@ func TestDetect(t *testing.T) { message: "fix: bug\n\nCo-Authored-By: Claude Opus 4 \nCo-Authored-By: aider (gpt-4o) ", wantTools: []string{"Claude Code", "Aider"}, wantModels: []string{"Opus 4", "gpt-4o"}, + wantScore: []float64{85, 85}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh, detection.ConfidenceHigh}, }, { @@ -91,6 +102,7 @@ func TestDetect(t *testing.T) { message: "fix: bug\n\nCo-Authored-By: Claude Opus 4 \nCo-Authored-By: Claude Sonnet 4 ", wantTools: []string{"Claude Code", "Claude Code"}, wantModels: []string{"Opus 4", "Sonnet 4"}, + wantScore: []float64{85, 85}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh, detection.ConfidenceHigh}, }, { @@ -98,6 +110,7 @@ func TestDetect(t *testing.T) { message: "fix: bug\n\nCo-Authored-By: Claude Opus 4 \nCo-Authored-By: Claude Opus 4 ", wantTools: []string{"Claude Code"}, wantModels: []string{"Opus 4"}, + wantScore: []float64{85}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh}, }, { @@ -105,6 +118,7 @@ func TestDetect(t *testing.T) { message: "fix: thing\n\nco-authored-by: Claude ", wantTools: []string{"Claude Code"}, wantModels: []string{""}, + wantScore: []float64{75}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh}, }, { @@ -112,12 +126,14 @@ func TestDetect(t *testing.T) { message: "fix: thing\n\nCO-AUTHORED-BY: Claude ", wantTools: []string{"Claude Code"}, wantModels: []string{""}, + wantScore: []float64{75}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh}, }, { name: "coauthor: human co-author only", message: "pair programming\n\nCo-Authored-By: Bob ", wantTools: nil, + wantScore: nil, wantConfidence: nil, }, // Co-Authored-By tests end here @@ -127,78 +143,91 @@ func TestDetect(t *testing.T) { name: "assistedby: Claude trailer with Opus model", message: "fix: update handler\n\nAssisted-By: Claude Opus 4 ", wantTools: []string{"Claude Opus 4"}, + wantScore: []float64{75}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh}, }, { name: "assistedby: Claude trailer with Sonnet model", message: "fix: update handler\n\nAssisted-By: Claude Sonnet 4 ", wantTools: []string{"Claude Sonnet 4"}, + wantScore: []float64{75}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh}, }, { name: "assistedby: Cursor trailer", message: "refactor: extract method\n\nAssisted-By: Cursor ", wantTools: []string{"Cursor"}, + wantScore: []float64{75}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh}, }, { name: "assistedby: Aider trailer with model name", message: "feat: add endpoint\n\nAssisted-By: aider (gpt-4o) ", wantTools: []string{"Aider"}, + wantScore: []float64{75}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh}, }, { name: "assistedby: Aider trailer with different model", message: "feat: add endpoint\n\nAssisted-By: aider (claude-4.7-opus) ", wantTools: []string{"Aider"}, + wantScore: []float64{75}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh}, }, { name: "assistedby: multiple trailers with Claude and human", message: "fix: bug\n\nAssisted-By: Claude Opus 4 ", wantTools: []string{"Claude Opus 4"}, + wantScore: []float64{75}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh}, }, { name: "assistedby: multiple AI trailers", message: "fix: bug\n\nAssisted-By: Claude Opus 4 \nAssisted-By: aider (gpt-4o) ", wantTools: []string{"Claude Opus 4", "Aider"}, + wantScore: []float64{75, 75}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh, detection.ConfidenceHigh}, }, { name: "assistedby: case variation", message: "fix: something\n\nassisted-by: Claude ", wantTools: []string{"Claude"}, + wantScore: []float64{75}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh}, }, { name: "assistedby: ASSISTED-BY uppercase", message: "fix: something\n\nASSISTED-BY: Claude ", wantTools: []string{"Claude"}, + wantScore: []float64{75}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh}, }, { name: "assistedby: Assisted-By trailer in commit message", message: "this is a commit message with\nAssisted-By: Claude Code", wantTools: []string{"Claude Code"}, + wantScore: []float64{75}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh}, }, { name: "assistedby: Another Assisted-By trailer in commit message 1", message: "this is a commit message with\nAssisted-By: Gemini", wantTools: []string{"Gemini"}, + wantScore: []float64{75}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh}, }, { name: "assistedby: Another Assisted-By trailer in commit message 2", message: "this is a commit message with\nAssisted-By: Kimi K2.6", wantTools: []string{"Kimi K2.6"}, + wantScore: []float64{75}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh}, }, { name: "assistedby: Multiple Assisted-By trailer in commit message", message: "this is a commit message with\nAssisted-By: Claude Code\nAssisted-By: Gemini", wantTools: []string{"Claude Code", "Gemini"}, + wantScore: []float64{75, 75}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh, detection.ConfidenceHigh}, }, { @@ -217,6 +246,7 @@ Co-Authored-By: Copilot Signed-off-by: some human `, wantTools: []string{"Cursor", "Copilot", "Claude 4.7 Opus", "Claude Sonnet 4", "Kimi K2.6", "ChatGPT", "Gemini"}, + wantScore: []float64{75, 75, 75, 75, 75, 75, 75}, wantConfidence: []detection.Confidence{ detection.ConfidenceHigh, // Cursor detection.ConfidenceHigh, // Copilot @@ -231,48 +261,56 @@ Signed-off-by: some human name: "assistedby: Same tool has 2 Assisted-By trailers in commit message", message: "this is a commit message with\nAssisted-By: Claude Code\nAssisted-By: Claude Code", wantTools: []string{"Claude Code"}, + wantScore: []float64{75}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh}, }, { name: "assistedby: Equivalent tools are deduplicated across multiple trailers", message: "this is a commit message with\nAssisted-By: Claude Code\nAssisted-By: CLAUDE CODE\nAssisted-By: [Claude Code].\nAssisted-By: GitHub Copilot", wantTools: []string{"Claude Code", "GitHub Copilot"}, + wantScore: []float64{75, 75}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh, detection.ConfidenceHigh}, }, { name: "assistedby: Tool with enclosing square brackets", message: "this is a commit message with\nAssisted-By: [Claude Code]", wantTools: []string{"Claude Code"}, + wantScore: []float64{75}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh}, }, { name: "assistedby: Tool with extra whitespace and terminal punctuation", message: "this is a commit message with\nAssisted-By: Claude Code.", wantTools: []string{"Claude Code"}, + wantScore: []float64{75}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh}, }, { name: "assistedby: Internal model punctuation is preserved", message: "this is a commit message with\nAssisted-By: GPT-5.5", wantTools: []string{"GPT-5.5"}, + wantScore: []float64{75}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh}, }, { name: "assistedby: Internal tool punctuation is preserved", message: "this is a commit message with\nAssisted-By: Continue.dev", wantTools: []string{"Continue.dev"}, + wantScore: []float64{75}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh}, }, { name: "assistedby: Assisted-By trailer in commit message in lower case", message: "this is a commit message with\nassisted-by: Claude Code", wantTools: []string{"Claude Code"}, + wantScore: []float64{75}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh}, }, { name: "assistedby: Two different attributions (assistedby and coauthor) both with email address", message: "Fix bug\n\nAssisted-By: Claude Sonnet 4 \nCo-Authored-By: Copilot ", wantTools: []string{"Copilot", "Claude Sonnet 4"}, + wantScore: []float64{75, 75}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh, detection.ConfidenceHigh}, }, { @@ -280,12 +318,14 @@ Signed-off-by: some human message: "Add validation logic\n\nCo-Authored-By: Claude Sonnet 4.6 \nAssisted-by: GitHub Copilot", wantTools: []string{"Claude Code", "GitHub Copilot"}, wantModels: []string{"Sonnet 4.6", ""}, + wantScore: []float64{85, 75}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh, detection.ConfidenceHigh}, }, { name: "assistedby: Claude Opus model attribution trailer", message: "Fix bug\n\nAssisted-by: Claude Opus 4 ", wantTools: []string{"Claude Opus 4"}, + wantScore: []float64{75}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh}, }, //Assisted-by tests end here @@ -295,132 +335,154 @@ Signed-off-by: some human name: "aider prefix", message: "aider: fix the login bug", wantTools: []string{"Aider"}, + wantScore: []float64{35}, wantConfidence: []detection.Confidence{detection.ConfidenceMedium}, }, { name: "aider prefix uppercase", message: "Aider: refactor auth module", wantTools: []string{"Aider"}, + wantScore: []float64{35}, wantConfidence: []detection.Confidence{detection.ConfidenceMedium}, }, { name: "Claude Code footer", message: "Add user validation\n\nGenerated with Claude Code", wantTools: []string{"Claude Code"}, + wantScore: []float64{35}, wantConfidence: []detection.Confidence{detection.ConfidenceMedium}, }, { name: "Claude Code footer with link", message: "Add validation\n\nGenerated with Claude Code\nhttps://claude.ai", wantTools: []string{"Claude Code"}, + wantScore: []float64{35}, wantConfidence: []detection.Confidence{detection.ConfidenceMedium}, }, { name: "EntireIO trailer present in commit", message: "this is some commit message\n\nEntire-Checkpoint: ab123cdefg12", wantTools: []string{"EntireIO"}, + wantScore: []float64{35}, wantConfidence: []detection.Confidence{detection.ConfidenceMedium}, }, { name: "Another EntireIO trailer present in commit", message: "this is some commit message\n\nEntire-Metadata: ab123cdefg12", wantTools: []string{"EntireIO"}, + wantScore: []float64{35}, wantConfidence: []detection.Confidence{detection.ConfidenceMedium}, }, { name: "Another EntireIO trailer present in commit with CRLF line endings", message: "this is some commit message\r\n\r\nEntire-Metadata: ab123cdefg12", wantTools: []string{"EntireIO"}, + wantScore: []float64{35}, wantConfidence: []detection.Confidence{detection.ConfidenceMedium}, }, { name: "EntireIO trailer not used, only mentioned in a commit", message: "this is a commit message with\nEntire-Metadata mentioned", wantTools: nil, + wantScore: nil, wantConfidence: nil, }, { name: "Replit Agent trailer present in a commit", message: "this is a commit message with\nReplit-Commit-Author: Agent", wantTools: []string{"Replit"}, + wantScore: []float64{35}, wantConfidence: []detection.Confidence{detection.ConfidenceMedium}, }, { name: "Replit Agent trailer present in a commit with session id", message: "this is a commit message with\nReplit-Commit-Author: Agent\nReplit-Commit-Session-Id: 1234a1ab-12ab-1234-abcd-0123456a1234", wantTools: []string{"Replit"}, + wantScore: []float64{80}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh}, }, { name: "Replit Assistant trailer present in a commit", message: "this is a commit message with\nReplit-Commit-Author: Assistant", wantTools: []string{"Replit"}, - wantConfidence: []detection.Confidence{detection.ConfidenceLow}, + wantScore: []float64{35}, + wantConfidence: []detection.Confidence{detection.ConfidenceMedium}, }, { name: "Replit Assistant trailer present in a commit with session id", message: "this is a commit message with\nReplit-Commit-Author: Assistant\nReplit-Commit-Session-Id: 1234a1ab-12ab-1234-abcd-0123456a1234", wantTools: []string{"Replit"}, - wantConfidence: []detection.Confidence{detection.ConfidenceMedium}, + wantScore: []float64{80}, + wantConfidence: []detection.Confidence{detection.ConfidenceHigh}, }, { name: "Replit Agent trailer present in commit with CRLF line endings", message: "this is some commit message\r\n\r\nReplit-Commit-Author: Agent", wantTools: []string{"Replit"}, + wantScore: []float64{35}, wantConfidence: []detection.Confidence{detection.ConfidenceMedium}, }, { name: "Replit Assistant trailer present in commit with CRLF line endings", message: "this is some commit message\r\n\r\nReplit-Commit-Author: Assistant", wantTools: []string{"Replit"}, - wantConfidence: []detection.Confidence{detection.ConfidenceLow}, + wantScore: []float64{35}, + wantConfidence: []detection.Confidence{detection.ConfidenceMedium}, }, { name: "Replit Agent trailer present in commit with another trailer with CRLF line endings", message: "this is some commit message\r\n\r\nReplit-Commit-Author: Agent\r\nSomeOther: Trailer", wantTools: []string{"Replit"}, + wantScore: []float64{35}, wantConfidence: []detection.Confidence{detection.ConfidenceMedium}, }, { name: "Replit Assistant trailer present in commit with another trailer with CRLF line endings", message: "this is some commit message\r\n\r\nReplit-Commit-Author: Assistant\r\nSomeOther: Trailer", wantTools: []string{"Replit"}, - wantConfidence: []detection.Confidence{detection.ConfidenceLow}, + wantScore: []float64{35}, + wantConfidence: []detection.Confidence{detection.ConfidenceMedium}, }, { name: "Some other Replit product trailer (not agent or asst) present in a commit", message: "this is a commit message with\nReplit-Commit-Author: SomeOtherReplitProduct", wantTools: nil, + wantScore: nil, wantConfidence: nil, }, { name: "Replit trailer not used, only mentioned in a commit", message: "this is a commit message with\nReplit-Commit-Author: Assistant mentioned", wantTools: nil, + wantScore: nil, wantConfidence: nil, }, { name: "aider in middle of message not prefix", message: "fix the aider: integration test", wantTools: nil, + wantScore: nil, wantConfidence: nil, }, { name: "aider as substring of a word", message: "raider: fix the tests", wantTools: nil, + wantScore: nil, wantConfidence: nil, }, { name: "no trailers", message: "just a normal commit message with no AI signatures", wantTools: nil, + wantScore: nil, wantConfidence: nil, }, { name: "empty message", message: "", wantTools: nil, + wantScore: nil, wantConfidence: nil, }, } @@ -430,10 +492,12 @@ Signed-off-by: some human findings := d.Detect(detection.Input{CommitMessage: tt.message}) gotTools := make([]string, len(findings)) gotModels := make([]string, len(findings)) + gotScore := make([]float64, len(findings)) gotConfidence := make([]detection.Confidence, len(findings)) for i, f := range findings { gotTools[i] = f.Tool gotModels[i] = f.Model + gotScore[i] = f.Score gotConfidence[i] = f.Confidence if f.Detector != "trailer" { @@ -469,6 +533,23 @@ Signed-off-by: some human } } + if tt.wantScore != nil { + if len(gotScore) == 0 { + gotScore = nil + } + if len(gotScore) != len(tt.wantScore) { + t.Errorf("score = %v, want %v", gotScore, tt.wantScore) + return + } + for i := range gotScore { + if gotScore[i] != tt.wantScore[i] { + t.Errorf("score = %v, want %v", gotScore, tt.wantScore) + return + } + } + + } + if len(gotConfidence) == 0 { gotConfidence = nil } diff --git a/output/output.go b/output/output.go index d6025be..45c008e 100644 --- a/output/output.go +++ b/output/output.go @@ -5,7 +5,6 @@ import ( "fmt" "io" "sort" - "strings" "github.com/chaoss/disclosure/detection" "github.com/chaoss/disclosure/scan" @@ -22,6 +21,11 @@ func FormatJSON(w io.Writer, report scan.Report) error { func FormatText(w io.Writer, report scan.Report) error { fmt.Fprintf(w, "Scanned %d commits, %d with AI signals\n\n", report.Summary.TotalCommits, report.Summary.AICommits) + // overall numeric score + if report.Summary.OverallScore > 0 { + fmt.Fprintf(w, "Overall score: %.1f / 100\n\n", report.Summary.OverallScore) + } + if report.Summary.AICommits == 0 { fmt.Fprintln(w, "No AI involvement detected.") return nil @@ -40,7 +44,15 @@ func FormatText(w io.Writer, report scan.Report) error { if len(cr.Findings) == 0 { continue } - fmt.Fprintf(w, "Commit %s\n", cr.Hash[:12]) + hash := cr.Hash + if len(hash) > 12 { + hash = hash[:12] + } + if cr.Score > 0 { + fmt.Fprintf(w, "Commit %s (score: %.1f)\n", hash, cr.Score) + } else { + fmt.Fprintf(w, "Commit %s\n", hash) + } for _, f := range cr.Findings { fmt.Fprintf(w, " [%s] %s (%s): %s\n", f.Confidence, f.DisplayTool(), f.Detector, f.Detail) } @@ -57,6 +69,10 @@ func FormatTextFindings(w io.Writer, findings []detection.Finding) error { } fmt.Fprintf(w, "Found %d AI signal(s):\n", len(findings)) + // compute consolidated score for these findings + overall, _ := detection.ConsolidateFindingScore(findings, nil) + fmt.Fprintf(w, "Overall score: %.1f / 100\n", overall) + for _, f := range findings { fmt.Fprintf(w, " [%s] %s (%s): %s\n", f.Confidence, f.DisplayTool(), f.Detector, f.Detail) } @@ -68,8 +84,12 @@ func FormatJSONFindings(w io.Writer, findings []detection.Finding) error { enc := json.NewEncoder(w) enc.SetIndent("", " ") return enc.Encode(struct { - Findings []detection.Finding `json:"findings"` - }{Findings: findings}) + Findings []detection.Finding `json:"findings"` + OverallScore float64 `json:"overall_score"` + }{ + Findings: findings, + OverallScore: func() float64 { s, _ := detection.ConsolidateFindingScore(findings, nil); return s }(), + }) } func sortedKeys(m map[string]int) []string { @@ -80,17 +100,3 @@ func sortedKeys(m map[string]int) []string { sort.Strings(keys) return keys } - -// ConfidenceFromString parses a confidence string or numeric value. -func ConfidenceFromString(s string) (detection.Confidence, error) { - switch strings.ToLower(strings.TrimSpace(s)) { - case "1", "low": - return detection.ConfidenceLow, nil - case "2", "medium": - return detection.ConfidenceMedium, nil - case "3", "high": - return detection.ConfidenceHigh, nil - default: - return 0, fmt.Errorf("invalid confidence %q: use low/1, medium/2, or high/3", s) - } -} diff --git a/output/output_test.go b/output/output_test.go index 4659dde..29ec88f 100644 --- a/output/output_test.go +++ b/output/output_test.go @@ -3,6 +3,8 @@ package output import ( "bytes" "encoding/json" + "fmt" + "reflect" "strings" "testing" @@ -24,10 +26,12 @@ func sampleReport() scan.Report { Detail: "Co-Authored-By trailer with email noreply@anthropic.com", }, }, + Score: 100.0, }, { Hash: "def789ghi012", Findings: nil, + Score: 0.0, }, }, Summary: scan.Summary{ @@ -35,6 +39,7 @@ func sampleReport() scan.Report { AICommits: 1, ToolCounts: map[string]int{"Claude Code": 1}, ByConfidence: map[string]int{"high": 1}, + OverallScore: 100.0, }, } } @@ -84,6 +89,9 @@ func TestFormatText(t *testing.T) { if !strings.Contains(out, "abc123def456") { t.Errorf("expected commit hash in output, got:\n%s", out) } + if !strings.Contains(out, "Overall score") { + t.Errorf("expected overall score in output, got:\n%s", out) + } } func TestFormatTextNoFindings(t *testing.T) { @@ -155,33 +163,163 @@ func TestFormatTextFindingsEmpty(t *testing.T) { } } -func TestConfidenceFromString(t *testing.T) { - tests := []struct { - input string - want detection.Confidence - err bool - }{ - {"low", detection.ConfidenceLow, false}, - {"1", detection.ConfidenceLow, false}, - {"medium", detection.ConfidenceMedium, false}, - {"2", detection.ConfidenceMedium, false}, - {"high", detection.ConfidenceHigh, false}, - {"3", detection.ConfidenceHigh, false}, - {"HIGH", detection.ConfidenceHigh, false}, - {" low ", detection.ConfidenceLow, false}, - {"invalid", 0, true}, - {"4", 0, true}, - {"", 0, true}, - } - - for _, tt := range tests { - got, err := ConfidenceFromString(tt.input) - if (err != nil) != tt.err { - t.Errorf("ConfidenceFromString(%q): err = %v, wantErr = %v", tt.input, err, tt.err) - continue - } - if got != tt.want { - t.Errorf("ConfidenceFromString(%q) = %d, want %d", tt.input, got, tt.want) +func TestFormatJSONEmptyReport(t *testing.T) { + var buf bytes.Buffer + + report := scan.Report{} + + if err := FormatJSON(&buf, report); err != nil { + t.Fatalf("FormatJSON: %v", err) + } + + var decoded scan.Report + if err := json.Unmarshal(buf.Bytes(), &decoded); err != nil { + t.Fatalf("unmarshal: %v", err) + } + + if decoded.Summary.OverallScore != 0 { + t.Fatalf("overall score = %v, want 0", decoded.Summary.OverallScore) + } +} + +type failingWriter struct{} + +func (failingWriter) Write([]byte) (int, error) { + return 0, fmt.Errorf("write failed") +} + +func TestFormatJSONWriterError(t *testing.T) { + err := FormatJSON(failingWriter{}, sampleReport()) + if err == nil { + t.Fatal("expected error") + } +} + +func TestFormatTextZeroScore(t *testing.T) { + var buf bytes.Buffer + + report := scan.Report{ + Summary: scan.Summary{ + TotalCommits: 1, + AICommits: 1, + OverallScore: 0, + ToolCounts: map[string]int{}, + }, + } + + if err := FormatText(&buf, report); err != nil { + t.Fatal(err) + } + + if strings.Contains(buf.String(), "Overall score") { + t.Error("did not expect overall score for zero") + } +} + +func TestFormatTextShortHash(t *testing.T) { + var buf bytes.Buffer + + report := scan.Report{ + Commits: []scan.CommitResult{ + { + Hash: "abc", + Findings: []detection.Finding{ + {Detector: "test"}, + }, + }, + }, + Summary: scan.Summary{ + AICommits: 1, + }, + } + + defer func() { + if r := recover(); r != nil { + t.Fatalf("panic: %v", r) } + }() + + if err := FormatText(&buf, report); err != nil { + t.Fatal(err) + } +} + +func TestFormatTextFindingsIncludesScore(t *testing.T) { + var buf bytes.Buffer + + findings := []detection.Finding{ + { + Detector: "test", + Score: 100, + }, + } + + if err := FormatTextFindings(&buf, findings); err != nil { + t.Fatal(err) + } + + if !strings.Contains(buf.String(), "Overall score:") { + t.Fatal("missing score") + } +} + +func TestFormatTextFindingsEmptySlice(t *testing.T) { + var buf bytes.Buffer + + if err := FormatTextFindings(&buf, []detection.Finding{}); err != nil { + t.Fatal(err) + } + + if !strings.Contains(buf.String(), "No AI involvement detected") { + t.Fatal("expected no detection message") + } +} + +func TestFormatJSONFindingsStructure(t *testing.T) { + var buf bytes.Buffer + + findings := []detection.Finding{ + { + Detector: "toolmention", + Tool: "Claude", + Score: 100, + }, + } + + if err := FormatJSONFindings(&buf, findings); err != nil { + t.Fatal(err) + } + + var decoded struct { + Findings []detection.Finding `json:"findings"` + Score float64 `json:"overall_score"` + } + + if err := json.Unmarshal(buf.Bytes(), &decoded); err != nil { + t.Fatal(err) + } + + if len(decoded.Findings) != 1 { + t.Fatalf("findings=%d want 1", len(decoded.Findings)) + } + + if decoded.Score != 100 { + t.Fatalf("score=%v want 100", decoded.Score) + } +} + +func TestSortedKeys(t *testing.T) { + got := sortedKeys(map[string]int{ + "c": 1, + "h": 1, + "a": 1, + "o": 1, + "s": 1, + }) + + want := []string{"a", "c", "h", "o", "s"} + + if !reflect.DeepEqual(got, want) { + t.Fatalf("got %v want %v", got, want) } } diff --git a/scan/scan.go b/scan/scan.go index 86084ca..cde7e60 100644 --- a/scan/scan.go +++ b/scan/scan.go @@ -9,14 +9,17 @@ import ( type CommitResult struct { Hash string `json:"hash"` Findings []detection.Finding `json:"findings"` + Score float64 `json:"score"` } // Summary aggregates stats across all commits scanned. type Summary struct { - TotalCommits int `json:"total_commits"` - AICommits int `json:"ai_commits"` - ToolCounts map[string]int `json:"tool_counts"` - ByConfidence map[string]int `json:"by_confidence"` + TotalCommits int `json:"total_commits"` + AICommits int `json:"ai_commits"` + ToolCounts map[string]int `json:"tool_counts"` + ByConfidence map[string]int `json:"by_confidence"` + PerDetectorScores map[string]float64 `json:"per_detector_scores"` + OverallScore float64 `json:"overall_score"` } // Report holds the full scan results. @@ -25,6 +28,10 @@ type Report struct { Summary Summary `json:"summary"` } +// Weights can be set (e.g., from cli) to control detector weighting used when consolidating scores. +// If nil, detectors are equally weighted. +var Weights map[string]float64 + // ScanCommitRange scans all commits in the given range using the provided detectors. func ScanCommitRange(repoPath, commitRange string, detectors []detection.Detector) (Report, error) { commits, err := gitops.ListCommits(repoPath, commitRange) @@ -81,9 +88,12 @@ func scanOneCommit(c gitops.Commit, branchName string, detectors []detection.Det findings = append(findings, d.Detect(input)...) } + score, _ := detection.ConsolidateFindingScore(findings, Weights) + return CommitResult{ Hash: c.Hash, Findings: findings, + Score: score, } } @@ -94,6 +104,7 @@ func buildReport(results []CommitResult) Report { ByConfidence: map[string]int{}, } + var allFindings []detection.Finding for _, r := range results { if len(r.Findings) > 0 { summary.AICommits++ @@ -101,9 +112,14 @@ func buildReport(results []CommitResult) Report { for _, f := range r.Findings { summary.ToolCounts[f.Tool]++ summary.ByConfidence[f.Confidence.String()]++ + allFindings = append(allFindings, f) } } + overall, perDetectorScores := detection.ConsolidateFindingScore(allFindings, Weights) + summary.PerDetectorScores = perDetectorScores + summary.OverallScore = overall + return Report{ Commits: results, Summary: summary, diff --git a/scan/scan_test.go b/scan/scan_test.go index fa50936..7567812 100644 --- a/scan/scan_test.go +++ b/scan/scan_test.go @@ -104,17 +104,17 @@ func TestScanCommitRange(t *testing.T) { dir, hashes := initTestRepo(t) detectors := allDetectors() - report, err := ScanCommitRange(dir, hashes[0]+".."+hashes[3], detectors) + report, err := ScanCommitRange(dir, hashes[0]+".."+hashes[4], detectors) if err != nil { t.Fatalf("ScanCommitRange: %v", err) } - if report.Summary.TotalCommits != 3 { - t.Errorf("total commits = %d, want 3", report.Summary.TotalCommits) + if report.Summary.TotalCommits != 4 { + t.Errorf("total commits = %d, want 4", report.Summary.TotalCommits) } - if report.Summary.AICommits != 3 { - t.Errorf("ai commits = %d, want 3", report.Summary.AICommits) + if report.Summary.AICommits != 4 { + t.Errorf("ai commits = %d, want 4", report.Summary.AICommits) } // Check that Claude Code was detected via co-author @@ -141,6 +141,31 @@ func TestScanCommitRange(t *testing.T) { if count, ok := report.Summary.ToolCounts["Kimi K2.6"]; !ok || count == 0 { t.Error("expected Kimi K2.6 Opus in tool counts") } + + // Check overall score + if report.Summary.OverallScore < 0 || report.Summary.OverallScore > 100 { + t.Error("invalid overall score") + } + perDetectorScores := report.Summary.PerDetectorScores + committerScore := perDetectorScores["committer"] + if committerScore != 95 { + t.Errorf("expected committer score to be 85, found %f", committerScore) + } + gitnotesScore := perDetectorScores["gitnotes"] + if gitnotesScore != 0 { + t.Errorf("expected gitnotes score to be 0, found %f", gitnotesScore) + } + toolmentionScore := perDetectorScores["toolmention"] + if toolmentionScore != 20 { + t.Errorf("expected toolmention score to be 20, found %f", toolmentionScore) + } + trailerScore := perDetectorScores["trailer"] + if trailerScore != 85 { + t.Errorf("expected trailer score to be 85, found %f", trailerScore) + } + if report.Summary.OverallScore != 66.67 { + t.Errorf("expected overall score to be 66.67, found %f", report.Summary.OverallScore) + } } func TestScanCommitRangeAll(t *testing.T) { @@ -205,6 +230,30 @@ func TestScanCommit(t *testing.T) { t.Error("expected findings for assisted-by and co-author trailers") } + for _, f := range result.Findings { + if f.Score <= 0 { + t.Errorf( + "finding %s score=%v, expected positive score", + f.Tool, + f.Score, + ) + } + + expectedConfidence, err := detection.ScoreToConfidence(f.Score) + if err != nil { + t.Fatalf("score conversion failed: %v", err) + } + + if f.Confidence != expectedConfidence { + t.Errorf( + "%s confidence=%d want=%d", + f.Tool, + f.Confidence, + expectedConfidence, + ) + } + } + foundCoauthor := false foundAssistedBy := false for _, f := range result.Findings { @@ -362,3 +411,104 @@ func TestReportSummaryByConfidence(t *testing.T) { t.Error("expected medium confidence findings") } } + +func TestScanReportNoFindingsHasZeroScore(t *testing.T) { + dir, hashes := initTestRepo(t) + + report, err := ScanCommitRange(dir, hashes[0]+".."+hashes[1], nil) + if err != nil { + t.Fatalf("ScanCommitRange: %v", err) + } + + if report.Summary.OverallScore != 0 { + t.Fatalf("overall score = %v, want 0", report.Summary.OverallScore) + } + + for _, cr := range report.Commits { + if cr.Score != 0 { + t.Fatalf("commit %s score = %v, want 0", cr.Hash, cr.Score) + } + } +} + +func TestScanReportInvalidRange(t *testing.T) { + dir, _ := initTestRepo(t) + + _, err := ScanCommitRange(dir, "invalid..range", allDetectors()) + if err == nil { + t.Fatal("expected to return an error") + } +} + +func TestScanCommitNoAI(t *testing.T) { + dir := t.TempDir() + + repo, err := git.PlainInit(dir, false) + if err != nil { + t.Fatalf("init repo: %v", err) + } + + wt, err := repo.Worktree() + if err != nil { + t.Fatalf("worktree: %v", err) + } + + filename := filepath.Join(dir, "main.go") + if err := os.WriteFile(filename, []byte("package main\nfunc main() {}"), 0644); err != nil { + t.Fatalf("write file: %v", err) + } + + if _, err := wt.Add("main.go"); err != nil { + t.Fatalf("add: %v", err) + } + + hash, err := wt.Commit("fix normal bug", &git.CommitOptions{ + Author: &object.Signature{ + Name: "Developer", + Email: "developer@example.com", + When: time.Now(), + }, + Committer: &object.Signature{ + Name: "Developer", + Email: "developer@example.com", + When: time.Now(), + }, + }) + if err != nil { + t.Fatalf("commit: %v", err) + } + + result, err := ScanCommit(dir, hash.String(), allDetectors()) + if err != nil { + t.Fatalf("ScanCommit: %v", err) + } + + if result.Hash != hash.String() { + t.Errorf("hash=%q want %q", result.Hash, hash.String()) + } + + if len(result.Findings) != 0 { + t.Errorf("expected no findings, got %d: %#v", len(result.Findings), result.Findings) + } + + if result.Score != 0 { + t.Errorf("score=%v want 0", result.Score) + } +} + +func TestScanCommitEmptyDetectorList(t *testing.T) { + dir, hashes := initTestRepo(t) + + result, err := ScanCommit(dir, hashes[1], nil) + if err != nil { + t.Fatalf("ScanCommit: %v", err) + } + + if len(result.Findings) != 0 { + t.Errorf("findings=%v want none", result.Findings) + } + + if result.Score != 0 { + t.Errorf("score=%v want 0", result.Score) + } +} From 282cc2078c550e38f46868de51574899953549c9 Mon Sep 17 00:00:00 2001 From: Omkar P <45419097+omkar-foss@users.noreply.github.com> Date: Tue, 4 Aug 2026 14:21:52 +0530 Subject: [PATCH 2/9] Add conf flag validations, replace log.Fatal with conf none Signed-off-by: Omkar P <45419097+omkar-foss@users.noreply.github.com> --- cmd/cmd.go | 23 ++++++++++++++++++----- cmd/cmd_test.go | 12 ++++++++++++ detection/committer/committer.go | 5 ++--- detection/gitnotes/gitnotes.go | 3 +-- detection/toolmention/toolmention.go | 1 - detection/trailer/trailer.go | 7 +++---- 6 files changed, 36 insertions(+), 15 deletions(-) diff --git a/cmd/cmd.go b/cmd/cmd.go index a949d51..7fc97eb 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -8,6 +8,7 @@ import ( "os" "path/filepath" "slices" + "strconv" "strings" "github.com/chaoss/disclosure/detection" @@ -141,6 +142,10 @@ Examples: flagMap := map[string]float64{} parts := strings.SplitSeq(confidenceScoresFlag, ",") for p := range parts { + p = strings.TrimSpace(p) + if p == "" { + continue + } kv := strings.SplitN(p, "=", 2) if len(kv) != 2 { err := fmt.Errorf("invalid confidence-scores entry: %q", p) @@ -149,14 +154,22 @@ Examples: return err } key := strings.TrimSpace(kv[0]) - var val float64 - if _, err := fmt.Sscan(strings.TrimSpace(kv[1]), &val); err != nil { - fmt.Fprintln(stderr, "invalid number in confidence-scores:", kv[1]) + if key == "" { + err := fmt.Errorf("empty key in confidence-scores entry: %q", p) + fmt.Fprintln(stderr, err) + *exitCode = ExitError + return err + } + valStr := strings.TrimSpace(kv[1]) + if valStr == "" { + err := fmt.Errorf("empty value in confidence-scores entry: %q", p) + fmt.Fprintln(stderr, err) *exitCode = ExitError return err } - if math.IsNaN(val) || math.IsInf(val, 0) || val < 0 || val > 100 { - err := fmt.Errorf("invalid confidence score: %v", val) + val, err := strconv.ParseFloat(valStr, 64) + if err != nil || math.IsNaN(val) || math.IsInf(val, 0) || val < 0 || val > 100 { + err = fmt.Errorf("invalid value in confidence-scores entry: %q", p) fmt.Fprintln(stderr, err) *exitCode = ExitError return err diff --git a/cmd/cmd_test.go b/cmd/cmd_test.go index 59dd29a..def830d 100644 --- a/cmd/cmd_test.go +++ b/cmd/cmd_test.go @@ -604,3 +604,15 @@ func TestRunScanRejectsNaNConfidenceScore(t *testing.T) { t.Errorf("expected error for NaN confidence score") } } + +func TestRunScanInvalidWeightsMissingValue(t *testing.T) { + dir := initTestRepo(t) + var stdout, stderr bytes.Buffer + code := Run([]string{"scan", "--weights=trailer=,toolmention=0.5", dir}, &stdout, &stderr) + if code != ExitError { + t.Fatalf("expected ExitError for missing weight value, got %d", code) + } + if !strings.Contains(stderr.String(), "invalid number in weights") { + t.Fatalf("expected numeric parse error, got stderr: %s", stderr.String()) + } +} diff --git a/detection/committer/committer.go b/detection/committer/committer.go index 030142f..afca115 100644 --- a/detection/committer/committer.go +++ b/detection/committer/committer.go @@ -2,7 +2,6 @@ package committer import ( "fmt" - "log" "strings" "github.com/chaoss/disclosure/detection" @@ -32,7 +31,7 @@ func (d *Detector) detectEmail(email, identityField string) []detection.Finding score := detection.CommitterMatchBaseScore + detection.CommitterKnownEmailBonusPoints confidence, err := detection.ScoreToConfidence(score) if err != nil { - log.Fatal(err) + confidence = detection.ConfidenceNone } return []detection.Finding{{ Detector: d.Name(), @@ -49,7 +48,7 @@ func (d *Detector) detectEmail(email, identityField string) []detection.Finding score := detection.CommitterMatchBaseScore + detection.CommitterEmailSuffixBonusPoints confidence, err := detection.ScoreToConfidence(score) if err != nil { - log.Fatal(err) + confidence = detection.ConfidenceNone } if idx := strings.Index(email, "+"); idx > 0 { prefix := email[:idx] diff --git a/detection/gitnotes/gitnotes.go b/detection/gitnotes/gitnotes.go index abc65c7..d6190df 100644 --- a/detection/gitnotes/gitnotes.go +++ b/detection/gitnotes/gitnotes.go @@ -2,7 +2,6 @@ package gitnotes import ( "fmt" - "log" "sort" "github.com/chaoss/disclosure/detection" @@ -33,7 +32,7 @@ func (d *Detector) Detect(input detection.Input) []detection.Finding { score := detection.GitNotesMatchBaseScore confidence, err := detection.ScoreToConfidence(score) if err != nil { - log.Fatal(err) + confidence = detection.ConfidenceNone } for _, promptID := range promptIDs { diff --git a/detection/toolmention/toolmention.go b/detection/toolmention/toolmention.go index e453182..3c25c87 100644 --- a/detection/toolmention/toolmention.go +++ b/detection/toolmention/toolmention.go @@ -2,7 +2,6 @@ package toolmention import ( "fmt" - "log" "regexp" "sort" "strings" diff --git a/detection/trailer/trailer.go b/detection/trailer/trailer.go index 25bafee..2ce2eba 100644 --- a/detection/trailer/trailer.go +++ b/detection/trailer/trailer.go @@ -2,7 +2,6 @@ package trailer import ( "fmt" - "log" "slices" "strings" @@ -179,7 +178,7 @@ func (d *Detector) detectTrailerCoauthoredBy(commitMessage string) []detection.F score += detection.CoauthorKnownEmailBonusPoints confidence, err := detection.ScoreToConfidence(score) if err != nil { - log.Fatal(err) + confidence = detection.ConfidenceNone } findings = append(findings, detection.Finding{ Detector: d.Name(), @@ -222,7 +221,7 @@ func (d *Detector) detectTrailerAssistedBy(commitMessage string) []detection.Fin score := detection.AssistedByTrailerBaseScore confidence, err := detection.ScoreToConfidence(score) if err != nil { - log.Fatal(err) + confidence = detection.ConfidenceNone } findings = append(findings, detection.Finding{ @@ -243,7 +242,7 @@ func (d *Detector) detectMessagePatterns(commitMessage string) []detection.Findi if score, isDetected := p.check(commitMessage); isDetected { confidence, err := detection.ScoreToConfidence(score) if err != nil { - log.Fatal(err) + confidence = detection.ConfidenceNone } findings = append(findings, detection.Finding{ Detector: d.Name(), From 529788f60da681db48229c194c72bdcc0bc195ac Mon Sep 17 00:00:00 2001 From: Omkar P <45419097+omkar-foss@users.noreply.github.com> Date: Tue, 4 Aug 2026 15:57:58 +0530 Subject: [PATCH 3/9] Separate out kv parsing, add more table based tests Signed-off-by: Omkar P <45419097+omkar-foss@users.noreply.github.com> --- cmd/cmd.go | 103 ++++---- cmd/cmd_test.go | 587 ++++++++++++++++++++++++++++-------------- output/output_test.go | 31 +-- 3 files changed, 442 insertions(+), 279 deletions(-) diff --git a/cmd/cmd.go b/cmd/cmd.go index 7fc97eb..5b539f7 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -42,6 +42,39 @@ func allDetectors() []detection.Detector { } } +// parseKeyValueFloatList parses strings like "a=1,b=2.5" into a map[string]float64. +func parseKeyValueFloatList(s string) (map[string]float64, error) { + out := map[string]float64{} + if strings.TrimSpace(s) == "" { + return out, nil + } + parts := strings.SplitSeq(s, ",") + for p := range parts { + p = strings.TrimSpace(p) + if p == "" { + continue + } + kv := strings.SplitN(p, "=", 2) + if len(kv) != 2 { + return nil, fmt.Errorf("invalid entry: %q", p) + } + key := strings.TrimSpace(kv[0]) + if key == "" { + return nil, fmt.Errorf("empty key in entry: %q", p) + } + valStr := strings.TrimSpace(kv[1]) + if valStr == "" { + return nil, fmt.Errorf("empty value in entry: %q", p) + } + v, err := strconv.ParseFloat(valStr, 64) + if err != nil || math.IsNaN(v) || math.IsInf(v, 0) { + return nil, fmt.Errorf("invalid numeric value %q in entry %q", valStr, p) + } + out[key] = v + } + return out, nil +} + // Run is the main entry point for the CLI. Returns an exit code. func Run(args []string, stdout, stderr io.Writer) int { rootCmd := &cobra.Command{ @@ -139,42 +172,11 @@ Examples: // parse confidence-scores override if provided if strings.TrimSpace(confidenceScoresFlag) != "" { - flagMap := map[string]float64{} - parts := strings.SplitSeq(confidenceScoresFlag, ",") - for p := range parts { - p = strings.TrimSpace(p) - if p == "" { - continue - } - kv := strings.SplitN(p, "=", 2) - if len(kv) != 2 { - err := fmt.Errorf("invalid confidence-scores entry: %q", p) - fmt.Fprintln(stderr, err) - *exitCode = ExitError - return err - } - key := strings.TrimSpace(kv[0]) - if key == "" { - err := fmt.Errorf("empty key in confidence-scores entry: %q", p) - fmt.Fprintln(stderr, err) - *exitCode = ExitError - return err - } - valStr := strings.TrimSpace(kv[1]) - if valStr == "" { - err := fmt.Errorf("empty value in confidence-scores entry: %q", p) - fmt.Fprintln(stderr, err) - *exitCode = ExitError - return err - } - val, err := strconv.ParseFloat(valStr, 64) - if err != nil || math.IsNaN(val) || math.IsInf(val, 0) || val < 0 || val > 100 { - err = fmt.Errorf("invalid value in confidence-scores entry: %q", p) - fmt.Fprintln(stderr, err) - *exitCode = ExitError - return err - } - flagMap[key] = val + flagMap, err := parseKeyValueFloatList(confidenceScoresFlag) + if err != nil { + fmt.Fprintln(stderr, err) + *exitCode = ExitError + return err } if err := detection.SetConfidenceScoresFromStrings(flagMap); err != nil { fmt.Fprintln(stderr, err) @@ -186,30 +188,11 @@ Examples: // parse weights flag scan.Weights = nil if strings.TrimSpace(weightsFlag) != "" { - weightMap := map[string]float64{} - parts := strings.SplitSeq(weightsFlag, ",") - for p := range parts { - kv := strings.SplitN(p, "=", 2) - if len(kv) != 2 { - err := fmt.Errorf("invalid weights entry: %q", p) - fmt.Fprintln(stderr, err) - *exitCode = ExitError - return err - } - name := strings.TrimSpace(kv[0]) - var val float64 - if _, err := fmt.Sscan(strings.TrimSpace(kv[1]), &val); err != nil { - fmt.Fprintln(stderr, "invalid number in weights:", kv[1]) - *exitCode = ExitError - return err - } - if math.IsNaN(val) || math.IsInf(val, 0) { - err := fmt.Errorf("invalid weight value: %v", val) - fmt.Fprintln(stderr, err) - *exitCode = ExitError - return err - } - weightMap[name] = val + weightMap, err := parseKeyValueFloatList(weightsFlag) + if err != nil { + fmt.Fprintln(stderr, err) + *exitCode = ExitError + return err } scan.Weights = weightMap } diff --git a/cmd/cmd_test.go b/cmd/cmd_test.go index def830d..f64537b 100644 --- a/cmd/cmd_test.go +++ b/cmd/cmd_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "os" "path/filepath" + "reflect" "strings" "testing" "time" @@ -255,62 +256,159 @@ func TestRunScanInvalidFormat(t *testing.T) { } func TestFilterReport(t *testing.T) { - report := scan.Report{ - Commits: []scan.CommitResult{ - { - Hash: "abc123", - Findings: []detection.Finding{ - {Detector: "toolmention", Tool: "Claude", Confidence: 1, Detail: "text"}, - {Detector: "trailer", Tool: "Claude Code", Confidence: 3, Detail: "trailer"}, + tests := []struct { + name string + report scan.Report + minConf detection.Confidence + wantScore float64 + wantAICommits int + wantFindings int + wantTool string // optional + }{ + { + name: "keep only high confidence findings", + report: scan.Report{ + Commits: []scan.CommitResult{ + { + Hash: "abc123", + Findings: []detection.Finding{ + { + Detector: "toolmention", + Tool: "Claude", + Confidence: detection.ConfidenceLow, + Score: 20, + }, + { + Detector: "trailer", + Tool: "Claude Code", + Confidence: detection.ConfidenceHigh, + Score: 100, + }, + }, + }, + }, + Summary: scan.Summary{ + TotalCommits: 1, + AICommits: 1, + ToolCounts: map[string]int{"Claude": 1, "Claude Code": 1}, + ByConfidence: map[string]int{"low": 1, "high": 1}, }, }, + minConf: detection.ConfidenceHigh, + wantScore: 100, + wantAICommits: 1, + wantFindings: 1, + wantTool: "Claude Code", }, - Summary: scan.Summary{ - TotalCommits: 1, - AICommits: 1, - ToolCounts: map[string]int{"Claude": 1, "Claude Code": 1}, - ByConfidence: map[string]int{"low": 1, "high": 1}, + { + name: "all findings filtered out", + report: scan.Report{ + Commits: []scan.CommitResult{ + { + Hash: "abc123", + Findings: []detection.Finding{ + { + Detector: "toolmention", + Confidence: detection.ConfidenceLow, + Score: 20, + }, + }, + }, + }, + Summary: scan.Summary{ + TotalCommits: 1, + }, + }, + minConf: detection.ConfidenceHigh, + wantScore: 0, + wantAICommits: 0, + wantFindings: 0, }, - } - - filtered := filterReport(report, 3) // high only - if len(filtered.Commits[0].Findings) != 1 { - t.Fatalf("expected 1 finding after filter, got %d", len(filtered.Commits[0].Findings)) - } - if filtered.Commits[0].Findings[0].Tool != "Claude Code" { - t.Errorf("expected Claude Code, got %s", filtered.Commits[0].Findings[0].Tool) - } - if filtered.Summary.AICommits != 1 { - t.Errorf("ai_commits = %d, want 1", filtered.Summary.AICommits) - } -} - -func TestFilterReportAllFiltered(t *testing.T) { - report := scan.Report{ - Commits: []scan.CommitResult{ - { - Hash: "abc123", - Findings: []detection.Finding{ + { + name: "empty findings", + report: scan.Report{ + Commits: []scan.CommitResult{ { - Detector: "toolmention", - Confidence: detection.ConfidenceLow, + Hash: "abc123", + Findings: nil, }, }, }, + minConf: detection.ConfidenceMedium, + wantScore: 0, + wantAICommits: 0, + wantFindings: 0, }, - Summary: scan.Summary{ - TotalCommits: 1, + { + name: "no commits", + report: scan.Report{}, + minConf: detection.ConfidenceHigh, + wantScore: 0, + wantAICommits: 0, + wantFindings: 0, + }, + { + name: "low confidence threshold returns original report", + report: scan.Report{ + Commits: []scan.CommitResult{ + { + Hash: "abc123", + Findings: []detection.Finding{ + { + Detector: "toolmention", + Confidence: detection.ConfidenceLow, + Score: 20, + }, + { + Detector: "trailer", + Confidence: detection.ConfidenceHigh, + Score: 100, + }, + }, + }, + }, + }, + minConf: detection.ConfidenceLow, + wantFindings: 2, }, } - filtered := filterReport(report, detection.ConfidenceHigh) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + filtered := filterReport(tt.report, tt.minConf) - if filtered.Summary.AICommits != 0 { - t.Fatalf("AICommits=%d, want 0", filtered.Summary.AICommits) - } + if tt.minConf == detection.ConfidenceLow { + if got := len(filtered.Commits[0].Findings); got != tt.wantFindings { + t.Fatalf("findings=%d want=%d", got, tt.wantFindings) + } + return + } - if len(filtered.Commits[0].Findings) != 0 { - t.Fatalf("expected no findings") + if filtered.Summary.OverallScore != tt.wantScore { + t.Errorf("overall score=%v want=%v", + filtered.Summary.OverallScore, tt.wantScore) + } + + if filtered.Summary.AICommits != tt.wantAICommits { + t.Errorf("AICommits=%d want=%d", + filtered.Summary.AICommits, tt.wantAICommits) + } + + gotFindings := 0 + for _, c := range filtered.Commits { + gotFindings += len(c.Findings) + } + if gotFindings != tt.wantFindings { + t.Errorf("findings=%d want=%d", gotFindings, tt.wantFindings) + } + + if tt.wantTool != "" { + got := filtered.Commits[0].Findings[0].Tool + if got != tt.wantTool { + t.Errorf("tool=%q want=%q", got, tt.wantTool) + } + } + }) } } @@ -428,191 +526,302 @@ func TestRunDocsWriteError(t *testing.T) { } } -func TestRunDocsEmptyFormat(t *testing.T) { +func TestRunDocsEmptyFormatFlag(t *testing.T) { var stdout, stderr bytes.Buffer - - code := Run([]string{ - "docs", - "--format=", - }, &stdout, &stderr) - + code := Run([]string{"docs", "--format="}, &stdout, &stderr) if code != ExitError { t.Errorf("exit code=%d want error", code) } } -func TestRunScanWithWeightsFlag(t *testing.T) { - dir := initTestRepo(t) - - var stdout, stderr bytes.Buffer - // Give trailer weight 0 and toolmention weight 1 so only toolmention contributes - code := Run([]string{"scan", "--format=json", "--weights=trailer=0.55,toolmention=0.45", dir}, &stdout, &stderr) - if code != ExitAI && code != ExitNoAI { - t.Fatalf("unexpected exit code: %d (stderr: %s)", code, stderr.String()) - } - - var report scan.Report - if err := json.Unmarshal(stdout.Bytes(), &report); err != nil { - t.Fatalf("unmarshal: %v (output: %s)", err, stdout.String()) - } - - var all []detection.Finding - for _, cr := range report.Commits { - all = append(all, cr.Findings...) +func TestParseKeyValueFloatList(t *testing.T) { + tests := []struct { + name string + input string + want map[string]float64 + wantErr bool + }{ + { + name: "Valid input single pair", + input: "low=20", + want: map[string]float64{"low": 20}, + wantErr: false, + }, + { + name: "Valid input multiple pairs with floats", + input: "trailer=0.8,toolmention=0.2", + want: map[string]float64{"trailer": 0.8, "toolmention": 0.2}, + wantErr: false, + }, + { + name: "Valid input with spacing padding", + input: " low = 20 , medium = 60.5 ", + want: map[string]float64{"low": 20, "medium": 60.5}, + wantErr: false, + }, + { + name: "Empty string yields empty map", + input: " ", + want: map[string]float64{}, + wantErr: false, + }, + { + name: "Error on malformed key value syntax", + input: "low:20", + want: nil, + wantErr: true, + }, + { + name: "Error on empty keys", + input: "=20,medium=60", + want: nil, + wantErr: true, + }, + { + name: "Error on empty values", + input: "low=,medium=60", + want: nil, + wantErr: true, + }, + { + name: "Error on invalid float strings", + input: "low=twenty", + want: nil, + wantErr: true, + }, + { + name: "Error on invalid numerical states (NaN)", + input: "low=NaN", + want: nil, + wantErr: true, + }, } - weights := map[string]float64{"trailer": 0.55, "toolmention": 0.45} - expectedOverall, _ := detection.ConsolidateFindingScore(all, weights) - if report.Summary.OverallScore != expectedOverall { - t.Fatalf("overall score = %v, want %v (weights applied)", report.Summary.OverallScore, expectedOverall) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := parseKeyValueFloatList(tt.input) + if (err != nil) != tt.wantErr { + t.Errorf("parseKeyValueFloatList() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !tt.wantErr && !reflect.DeepEqual(got, tt.want) { + t.Errorf("parseKeyValueFloatList() = %v, want %v", got, tt.want) + } + }) } } -func TestRunScanWithConfidenceScoresFlag(t *testing.T) { +func TestRunScanFlags(t *testing.T) { dir := initTestRepo(t) - var stdout, stderr bytes.Buffer - - // override confidence scores: low=10, medium=50, high=90 - code := Run([]string{"scan", "--format=json", "--confidence-scores=low=10,medium=50,high=90", dir}, &stdout, &stderr) - if code != ExitAI && code != ExitNoAI { - t.Fatalf("unexpected exit code: %d (stderr: %s)", code, stderr.String()) - } - - var report scan.Report - if err := json.Unmarshal(stdout.Bytes(), &report); err != nil { - t.Fatalf("unmarshal: %v (output: %s)", err, stdout.String()) + tests := []struct { + name string + args []string + wantCode int + wantErrText string + }{ + { + name: "invalid min confidence", + args: []string{"scan", "--min-confidence=invalid", dir}, + wantCode: ExitError, + wantErrText: "invalid confidence", + }, + { + name: "invalid weights format", + args: []string{"scan", "--weights=trailer", dir}, + wantCode: ExitError, + }, + { + name: "invalid confidence scores format", + args: []string{"scan", "--confidence-scores=low", dir}, + wantCode: ExitError, + }, + { + name: "reject NaN weight", + args: []string{"scan", "--weights=trailer=NaN", dir}, + wantCode: ExitError, + }, + { + name: "reject NaN confidence score", + args: []string{"scan", "--confidence-scores=high=NaN", dir}, + wantCode: ExitError, + }, + { + name: "both valid flags", + args: []string{ + "scan", + "--weights=trailer=0.5,toolmention=0.5", + "--confidence-scores=low=15,medium=55,high=95", + dir, + }, + wantCode: ExitAI, // or ExitNoAI + }, } - var all []detection.Finding - for _, cr := range report.Commits { - all = append(all, cr.Findings...) - } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var stdout, stderr bytes.Buffer - expectedOverall, _ := detection.ConsolidateFindingScore(all, nil) - if report.Summary.OverallScore != expectedOverall { - t.Fatalf("overall score = %v, want %v (conf scores applied)", report.Summary.OverallScore, expectedOverall) - } -} + code := Run(tt.args, &stdout, &stderr) -func TestRunScanInvalidMinConfidence(t *testing.T) { - dir := initTestRepo(t) - - var stdout, stderr bytes.Buffer - code := Run([]string{ - "scan", - "--min-confidence=invalid", - dir, - }, &stdout, &stderr) + if tt.name == "both valid flags" { + if code != ExitAI && code != ExitNoAI { + t.Fatalf("unexpected exit code %d (stderr=%s)", code, stderr.String()) + } + return + } - if code != ExitError { - t.Errorf("exit code = %d, want %d", code, ExitError) - } + if code != tt.wantCode { + t.Fatalf("exit code=%d want=%d", code, tt.wantCode) + } - if !strings.Contains(stderr.String(), "invalid confidence") { - t.Errorf("expected confidence error, got: %s", stderr.String()) + if tt.wantErrText != "" && + !strings.Contains(stderr.String(), tt.wantErrText) { + t.Fatalf("stderr=%q does not contain %q", stderr.String(), tt.wantErrText) + } + }) } } -func TestRunScanInvalidWeightsFormat(t *testing.T) { +func TestRunScanScoreFlags(t *testing.T) { dir := initTestRepo(t) - var stdout, stderr bytes.Buffer - code := Run([]string{ - "scan", - "--weights=trailer", - dir, - }, &stdout, &stderr) - - if code != ExitError { - t.Errorf("exit code = %d, want %d", code, ExitError) + tests := []struct { + name string + args []string + expectedWeights map[string]float64 + }{ + { + name: "weights", + args: []string{ + "scan", + "--format=json", + "--weights=trailer=0.55,toolmention=0.45", + dir, + }, + expectedWeights: map[string]float64{ + "trailer": 0.55, + "toolmention": 0.45, + }, + }, + { + name: "confidence scores", + args: []string{ + "scan", + "--format=json", + "--confidence-scores=low=10,medium=50,high=90", + dir, + }, + expectedWeights: nil, + }, } -} -func TestRunScanInvalidConfidenceScoresFormat(t *testing.T) { - dir := initTestRepo(t) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var stdout, stderr bytes.Buffer - var stdout, stderr bytes.Buffer - code := Run([]string{ - "scan", - "--confidence-scores=low", - dir, - }, &stdout, &stderr) + code := Run(tt.args, &stdout, &stderr) + if code != ExitAI && code != ExitNoAI { + t.Fatalf("unexpected exit code %d: %s", code, stderr.String()) + } - if code != ExitError { - t.Errorf("exit code = %d, want %d", code, ExitError) - } -} + var report scan.Report + if err := json.Unmarshal(stdout.Bytes(), &report); err != nil { + t.Fatalf("unmarshal: %v", err) + } -func TestFilterReportRecalculatesScore(t *testing.T) { - report := scan.Report{ - Commits: []scan.CommitResult{ - { - Hash: "abc123", - Findings: []detection.Finding{ - { - Detector: "toolmention", - Confidence: detection.ConfidenceLow, - Score: 20, - }, - { - Detector: "trailer", - Confidence: detection.ConfidenceHigh, - Score: 100, - }, - }, - }, - }, - } + var findings []detection.Finding + for _, c := range report.Commits { + findings = append(findings, c.Findings...) + } - filtered := filterReport(report, detection.ConfidenceHigh) + expected, _ := detection.ConsolidateFindingScore(findings, tt.expectedWeights) - if filtered.Commits[0].Score != 100 { - t.Errorf( - "score=%v want 100", - filtered.Commits[0].Score, - ) + if report.Summary.OverallScore != expected { + t.Fatalf("overall=%v want=%v", + report.Summary.OverallScore, + expected) + } + }) } } -func TestRunScanRejectsNaNWeights(t *testing.T) { - dir := initTestRepo(t) - - var stdout, stderr bytes.Buffer - code := Run([]string{ - "scan", - "--weights=trailer=NaN", - dir, - }, &stdout, &stderr) - - if code != ExitError { - t.Errorf("expected error for NaN weight") +func TestScanCommandInvalidFlags(t *testing.T) { + tests := []struct { + name string + args []string + }{ + { + name: "weights missing equals", + args: []string{"--weights", "string_without_equals"}, + }, + { + name: "weights empty key", + args: []string{"--weights", "=0.5"}, + }, + { + name: "weights empty value", + args: []string{"--weights", "trailer="}, + }, + { + name: "weights invalid number", + args: []string{"--weights", "trailer=abc"}, + }, + { + name: "weights NaN", + args: []string{"--weights", "trailer=NaN"}, + }, + { + name: "weights Inf", + args: []string{"--weights", "trailer=Inf"}, + }, + { + name: "weights -Inf", + args: []string{"--weights", "trailer=-Inf"}, + }, + { + name: "confidence missing equals", + args: []string{"--confidence-scores", "low"}, + }, + { + name: "confidence empty key", + args: []string{"--confidence-scores", "=10"}, + }, + { + name: "confidence empty value", + args: []string{"--confidence-scores", "low="}, + }, + { + name: "confidence invalid number", + args: []string{"--confidence-scores", "low=abc"}, + }, + { + name: "confidence NaN", + args: []string{"--confidence-scores", "high=NaN"}, + }, + { + name: "confidence Inf", + args: []string{"--confidence-scores", "high=Inf"}, + }, + { + name: "confidence -Inf", + args: []string{"--confidence-scores", "high=-Inf"}, + }, } -} -func TestRunScanRejectsNaNConfidenceScore(t *testing.T) { - dir := initTestRepo(t) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var stdout, stderr bytes.Buffer + exitCode := ExitNoAI - var stdout, stderr bytes.Buffer - code := Run([]string{ - "scan", - "--confidence-scores=high=NaN", - dir, - }, &stdout, &stderr) + cmd := scanCommand(&stdout, &stderr, &exitCode) + cmd.SetArgs(tt.args) - if code != ExitError { - t.Errorf("expected error for NaN confidence score") - } -} + _ = cmd.Execute() -func TestRunScanInvalidWeightsMissingValue(t *testing.T) { - dir := initTestRepo(t) - var stdout, stderr bytes.Buffer - code := Run([]string{"scan", "--weights=trailer=,toolmention=0.5", dir}, &stdout, &stderr) - if code != ExitError { - t.Fatalf("expected ExitError for missing weight value, got %d", code) - } - if !strings.Contains(stderr.String(), "invalid number in weights") { - t.Fatalf("expected numeric parse error, got stderr: %s", stderr.String()) + if exitCode != ExitError { + t.Fatalf("expected ExitError, got %d (stderr=%q)", exitCode, stderr.String()) + } + }) } } diff --git a/output/output_test.go b/output/output_test.go index 29ec88f..87238c2 100644 --- a/output/output_test.go +++ b/output/output_test.go @@ -165,18 +165,14 @@ func TestFormatTextFindingsEmpty(t *testing.T) { func TestFormatJSONEmptyReport(t *testing.T) { var buf bytes.Buffer - report := scan.Report{} - if err := FormatJSON(&buf, report); err != nil { t.Fatalf("FormatJSON: %v", err) } - var decoded scan.Report if err := json.Unmarshal(buf.Bytes(), &decoded); err != nil { t.Fatalf("unmarshal: %v", err) } - if decoded.Summary.OverallScore != 0 { t.Fatalf("overall score = %v, want 0", decoded.Summary.OverallScore) } @@ -197,7 +193,6 @@ func TestFormatJSONWriterError(t *testing.T) { func TestFormatTextZeroScore(t *testing.T) { var buf bytes.Buffer - report := scan.Report{ Summary: scan.Summary{ TotalCommits: 1, @@ -206,11 +201,9 @@ func TestFormatTextZeroScore(t *testing.T) { ToolCounts: map[string]int{}, }, } - if err := FormatText(&buf, report); err != nil { t.Fatal(err) } - if strings.Contains(buf.String(), "Overall score") { t.Error("did not expect overall score for zero") } @@ -218,7 +211,6 @@ func TestFormatTextZeroScore(t *testing.T) { func TestFormatTextShortHash(t *testing.T) { var buf bytes.Buffer - report := scan.Report{ Commits: []scan.CommitResult{ { @@ -232,13 +224,11 @@ func TestFormatTextShortHash(t *testing.T) { AICommits: 1, }, } - defer func() { if r := recover(); r != nil { t.Fatalf("panic: %v", r) } }() - if err := FormatText(&buf, report); err != nil { t.Fatal(err) } @@ -246,18 +236,15 @@ func TestFormatTextShortHash(t *testing.T) { func TestFormatTextFindingsIncludesScore(t *testing.T) { var buf bytes.Buffer - findings := []detection.Finding{ { Detector: "test", Score: 100, }, } - if err := FormatTextFindings(&buf, findings); err != nil { t.Fatal(err) } - if !strings.Contains(buf.String(), "Overall score:") { t.Fatal("missing score") } @@ -265,11 +252,9 @@ func TestFormatTextFindingsIncludesScore(t *testing.T) { func TestFormatTextFindingsEmptySlice(t *testing.T) { var buf bytes.Buffer - if err := FormatTextFindings(&buf, []detection.Finding{}); err != nil { t.Fatal(err) } - if !strings.Contains(buf.String(), "No AI involvement detected") { t.Fatal("expected no detection message") } @@ -277,7 +262,6 @@ func TestFormatTextFindingsEmptySlice(t *testing.T) { func TestFormatJSONFindingsStructure(t *testing.T) { var buf bytes.Buffer - findings := []detection.Finding{ { Detector: "toolmention", @@ -285,40 +269,27 @@ func TestFormatJSONFindingsStructure(t *testing.T) { Score: 100, }, } - if err := FormatJSONFindings(&buf, findings); err != nil { t.Fatal(err) } - var decoded struct { Findings []detection.Finding `json:"findings"` Score float64 `json:"overall_score"` } - if err := json.Unmarshal(buf.Bytes(), &decoded); err != nil { t.Fatal(err) } - if len(decoded.Findings) != 1 { t.Fatalf("findings=%d want 1", len(decoded.Findings)) } - if decoded.Score != 100 { t.Fatalf("score=%v want 100", decoded.Score) } } func TestSortedKeys(t *testing.T) { - got := sortedKeys(map[string]int{ - "c": 1, - "h": 1, - "a": 1, - "o": 1, - "s": 1, - }) - + got := sortedKeys(map[string]int{"c": 1, "h": 1, "a": 1, "o": 1, "s": 1}) want := []string{"a", "c", "h", "o", "s"} - if !reflect.DeepEqual(got, want) { t.Fatalf("got %v want %v", got, want) } From 283fa6474b9c08cd8426d36e1c26a06d2b1945f1 Mon Sep 17 00:00:00 2001 From: Omkar P <45419097+omkar-foss@users.noreply.github.com> Date: Wed, 5 Aug 2026 22:24:21 +0530 Subject: [PATCH 4/9] Use additive scoring, remove weights, update tests Signed-off-by: Omkar P <45419097+omkar-foss@users.noreply.github.com> --- cmd/cmd.go | 43 ++--- cmd/cmd_test.go | 100 ++++------- detection/detection.go | 60 ++----- detection/detection_test.go | 194 ++-------------------- detection/toolmention/toolmention.go | 6 +- detection/toolmention/toolmention_test.go | 41 +---- output/output.go | 4 +- scan/scan.go | 22 ++- scan/scan_test.go | 31 +++- 9 files changed, 118 insertions(+), 383 deletions(-) diff --git a/cmd/cmd.go b/cmd/cmd.go index 5b539f7..edd0abf 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -118,7 +118,6 @@ func scanCommand(stdout, stderr io.Writer, exitCode *int) *cobra.Command { var rangeFlag string var formatFlag string var minConfFlag string - var weightsFlag string var confidenceScoresFlag string cmd := &cobra.Command{ @@ -185,18 +184,6 @@ Examples: } } - // parse weights flag - scan.Weights = nil - if strings.TrimSpace(weightsFlag) != "" { - weightMap, err := parseKeyValueFloatList(weightsFlag) - if err != nil { - fmt.Fprintln(stderr, err) - *exitCode = ExitError - return err - } - scan.Weights = weightMap - } - detectors := allDetectors() report, err := scan.ScanCommitRange(repoPath, rangeFlag, detectors) if err != nil { @@ -237,7 +224,6 @@ Examples: cmd.Flags().StringVar(&rangeFlag, "range", "", "commit range in BASE..HEAD format") cmd.Flags().StringVar(&formatFlag, "format", "text", "output format: json or text") cmd.Flags().StringVar(&minConfFlag, "min-confidence", "low", "minimum confidence level: low, medium, high (or 1, 2, 3)") - cmd.Flags().StringVar(&weightsFlag, "weights", "", "comma-separated detector weights, e.g. 'trailer=0.8,toolmention=0.2'") cmd.Flags().StringVar(&confidenceScoresFlag, "confidence-scores", "", "override confidence->score mapping, e.g. 'low=20,medium=60,high=100'") return cmd @@ -360,31 +346,30 @@ func filterReport(report scan.Report, minConf detection.Confidence) scan.Report // collect all findings to compute overall score after filtering var overallScoreFindings []detection.Finding - for _, cr := range report.Commits { - var kept []detection.Finding - for _, f := range cr.Findings { + for _, commit := range report.Commits { + var commitFindings []detection.Finding + for _, f := range commit.Findings { if f.Confidence >= minConf { - kept = append(kept, f) + commitFindings = append(commitFindings, f) } } - // Recompute per-commit score from the kept findings and configured weights - commitScore, _ := detection.ConsolidateFindingScore(kept, scan.Weights) - result := scan.CommitResult{Hash: cr.Hash, Findings: kept, Score: commitScore} + // Recompute per-commit score from the kept findings + commitScore, _ := detection.ConsolidateScoreByFindings(commitFindings) + result := scan.CommitResult{Hash: commit.Hash, Findings: commitFindings, Score: commitScore} filtered.Commits = append(filtered.Commits, result) - - if len(kept) > 0 { + if len(commitFindings) > 0 { filtered.Summary.AICommits++ } - for _, f := range kept { - filtered.Summary.ToolCounts[f.Tool]++ - filtered.Summary.ByConfidence[f.Confidence.String()]++ - overallScoreFindings = append(overallScoreFindings, f) + for _, commitFinding := range commitFindings { + filtered.Summary.ToolCounts[commitFinding.Tool]++ + filtered.Summary.ByConfidence[commitFinding.Confidence.String()]++ + overallScoreFindings = append(overallScoreFindings, commitFinding) } } - // Compute new overall score for the filtered report using the same weights. - overall, _ := detection.ConsolidateFindingScore(overallScoreFindings, scan.Weights) + // Compute new overall score for the filtered report. + overall, _ := detection.ConsolidateScoreByFindings(overallScoreFindings) filtered.Summary.OverallScore = overall return filtered diff --git a/cmd/cmd_test.go b/cmd/cmd_test.go index f64537b..6059b5e 100644 --- a/cmd/cmd_test.go +++ b/cmd/cmd_test.go @@ -286,18 +286,37 @@ func TestFilterReport(t *testing.T) { }, }, }, + { + Hash: "abc456", + Findings: []detection.Finding{ + { + Detector: "toolmention", + Tool: "Claude", + Confidence: detection.ConfidenceLow, + Score: 20, + }, + { + Detector: "trailer", + Tool: "Claude Code", + Confidence: detection.ConfidenceHigh, + Score: 100, + }, + }, + }, }, Summary: scan.Summary{ - TotalCommits: 1, - AICommits: 1, - ToolCounts: map[string]int{"Claude": 1, "Claude Code": 1}, - ByConfidence: map[string]int{"low": 1, "high": 1}, + TotalCommits: 2, + AICommits: 2, + ToolCounts: map[string]int{"Claude": 2, "Claude Code": 2}, + ByConfidence: map[string]int{"low": 2, "high": 2}, + PerDetectorScores: map[string]float64{"toolmention": 20, "trailer": 100}, + OverallScore: 120, }, }, minConf: detection.ConfidenceHigh, wantScore: 100, - wantAICommits: 1, - wantFindings: 1, + wantAICommits: 2, + wantFindings: 2, // only high confidence ones from both commits wantTool: "Claude Code", }, { @@ -370,6 +389,7 @@ func TestFilterReport(t *testing.T) { }, minConf: detection.ConfidenceLow, wantFindings: 2, + wantScore: 120, }, } @@ -626,34 +646,19 @@ func TestRunScanFlags(t *testing.T) { wantCode: ExitError, wantErrText: "invalid confidence", }, - { - name: "invalid weights format", - args: []string{"scan", "--weights=trailer", dir}, - wantCode: ExitError, - }, { name: "invalid confidence scores format", args: []string{"scan", "--confidence-scores=low", dir}, wantCode: ExitError, }, - { - name: "reject NaN weight", - args: []string{"scan", "--weights=trailer=NaN", dir}, - wantCode: ExitError, - }, { name: "reject NaN confidence score", args: []string{"scan", "--confidence-scores=high=NaN", dir}, wantCode: ExitError, }, { - name: "both valid flags", - args: []string{ - "scan", - "--weights=trailer=0.5,toolmention=0.5", - "--confidence-scores=low=15,medium=55,high=95", - dir, - }, + name: "both valid flags", + args: []string{"scan", "--confidence-scores=low=15,medium=55,high=95", dir}, wantCode: ExitAI, // or ExitNoAI }, } @@ -687,23 +692,9 @@ func TestRunScanScoreFlags(t *testing.T) { dir := initTestRepo(t) tests := []struct { - name string - args []string - expectedWeights map[string]float64 + name string + args []string }{ - { - name: "weights", - args: []string{ - "scan", - "--format=json", - "--weights=trailer=0.55,toolmention=0.45", - dir, - }, - expectedWeights: map[string]float64{ - "trailer": 0.55, - "toolmention": 0.45, - }, - }, { name: "confidence scores", args: []string{ @@ -712,7 +703,6 @@ func TestRunScanScoreFlags(t *testing.T) { "--confidence-scores=low=10,medium=50,high=90", dir, }, - expectedWeights: nil, }, } @@ -735,7 +725,7 @@ func TestRunScanScoreFlags(t *testing.T) { findings = append(findings, c.Findings...) } - expected, _ := detection.ConsolidateFindingScore(findings, tt.expectedWeights) + expected, _ := detection.ConsolidateScoreByFindings(findings) if report.Summary.OverallScore != expected { t.Fatalf("overall=%v want=%v", @@ -751,34 +741,6 @@ func TestScanCommandInvalidFlags(t *testing.T) { name string args []string }{ - { - name: "weights missing equals", - args: []string{"--weights", "string_without_equals"}, - }, - { - name: "weights empty key", - args: []string{"--weights", "=0.5"}, - }, - { - name: "weights empty value", - args: []string{"--weights", "trailer="}, - }, - { - name: "weights invalid number", - args: []string{"--weights", "trailer=abc"}, - }, - { - name: "weights NaN", - args: []string{"--weights", "trailer=NaN"}, - }, - { - name: "weights Inf", - args: []string{"--weights", "trailer=Inf"}, - }, - { - name: "weights -Inf", - args: []string{"--weights", "trailer=-Inf"}, - }, { name: "confidence missing equals", args: []string{"--confidence-scores", "low"}, diff --git a/detection/detection.go b/detection/detection.go index f07b096..887d295 100644 --- a/detection/detection.go +++ b/detection/detection.go @@ -3,8 +3,6 @@ package detection import ( "fmt" "math" - "sort" - "strconv" "strings" ) @@ -183,13 +181,9 @@ func (input *Input) GetNotes() (GitnoteParseResult, error) { return parseGitnotes(input.Notes) } -// ConsolidateFindingScore computes per-detector scores and a consolidated overall -// score (0..100) using a weighted average across detectors. -// Things to note: -// - If weights is nil, detectors are equally weighted. We normalize provided weights so they sum to 1. -// - Detectors with missing weight entries are treated as zero weight. -// - Normalization will fallback to equal weights if total weight is zero. -func ConsolidateFindingScore(findings []Finding, weights map[string]float64) (float64, map[string]float64) { +// ConsolidateScoreByFindings computes per-detector scores and a overall score from findings +func ConsolidateScoreByFindings(findings []Finding) (float64, map[string]float64) { + // For each detector, we take max of all findings for that detector. perDetectorScores := map[string]float64{} for _, f := range findings { detectorName := strings.TrimSpace(f.Detector) @@ -203,45 +197,15 @@ func ConsolidateFindingScore(findings []Finding, weights map[string]float64) (fl return 0.0, perDetectorScores } - // Prepare normalized weights - norm := map[string]float64{} - // equal weights for all detectors if weights unspecified - if weights == nil { - weight := 1.0 / float64(len(perDetectorScores)) - for detectorName := range perDetectorScores { - norm[detectorName] = weight - } - } else { - var sum float64 - for detectorName := range perDetectorScores { - currWeight := max(0, weights[detectorName]) - norm[detectorName] = currWeight - sum += currWeight - } - // if user-supplied sum is zero, fallback to equal weights - if sum == 0 { - weight := 1.0 / float64(len(perDetectorScores)) - for detectorName := range perDetectorScores { - norm[detectorName] = weight - } - } else { - for detectorName := range norm { - norm[detectorName] = norm[detectorName] / sum - } - } - } + // Compute total score for findings + return CalculateTotalScore(perDetectorScores), perDetectorScores +} - // Compute overall weighted average (deterministic order) - detectorNames := make([]string, 0, len(perDetectorScores)) - for detectorName := range perDetectorScores { - detectorNames = append(detectorNames, detectorName) - } - sort.Strings(detectorNames) - var overall float64 - for _, detectorName := range detectorNames { - overall += perDetectorScores[detectorName] * norm[detectorName] +// CalculateTotalScore computes total score from a per-detector scores map +func CalculateTotalScore(perDetectorScores map[string]float64) float64 { + var totalScore float64 + for _, score := range perDetectorScores { + totalScore += score } - overall = max(0, min(overall, 100)) - overall, _ = strconv.ParseFloat(fmt.Sprintf("%.2f", overall), 64) - return overall, perDetectorScores + return totalScore } diff --git a/detection/detection_test.go b/detection/detection_test.go index cc2196e..2135609 100644 --- a/detection/detection_test.go +++ b/detection/detection_test.go @@ -57,19 +57,18 @@ func TestConsolidateFindings(t *testing.T) { tests := []struct { name string findings []Finding - weights map[string]float64 wantOverall float64 wantDetectorScores map[string]float64 wantNaN bool }{ { - name: "equal weights", + name: "sum of individual detector scores", findings: []Finding{ {Detector: "A", Score: 100}, {Detector: "B", Score: 50}, {Detector: "C", Score: 75}, }, - wantOverall: 75, + wantOverall: 225, wantDetectorScores: map[string]float64{ "A": 100, "B": 50, @@ -77,57 +76,31 @@ func TestConsolidateFindings(t *testing.T) { }, }, { - name: "explicit weights", + name: "mix of positive and negative scores", findings: []Finding{ {Detector: "A", Score: 100}, - {Detector: "B", Score: 50}, - {Detector: "C", Score: 75}, - }, - weights: map[string]float64{ - "A": 0.6, - "B": 0.2, - "C": 0.2, - }, - wantOverall: 85, - wantDetectorScores: map[string]float64{ - "A": 100, - "B": 50, - "C": 75, - }, - }, - { - name: "weights normalized", - findings: []Finding{ - {Detector: "A", Score: 100}, - {Detector: "B", Score: 50}, + {Detector: "B", Score: -50}, {Detector: "C", Score: 75}, }, - weights: map[string]float64{ - "A": 0.0, - "B": 0.2, - "C": 0.2, - }, - wantOverall: 62.5, + wantOverall: 125, wantDetectorScores: map[string]float64{ "A": 100, - "B": 50, + "B": -50, "C": 75, }, }, { - name: "missing weights treated as zero", + name: "max used to aggregated detector level scores", findings: []Finding{ {Detector: "A", Score: 100}, {Detector: "B", Score: 50}, + {Detector: "A", Score: 120}, {Detector: "C", Score: 75}, + {Detector: "B", Score: -10}, }, - weights: map[string]float64{ - "A": 0.0, - "C": 0.2, - }, - wantOverall: 75, + wantOverall: 245, wantDetectorScores: map[string]float64{ - "A": 100, + "A": 120, "B": 50, "C": 75, }, @@ -145,7 +118,7 @@ func TestConsolidateFindings(t *testing.T) { wantDetectorScores: map[string]float64{}, }, { - name: "duplicate detector keeps max score", + name: "single detector across all findings", findings: []Finding{ {Detector: "A", Score: 20}, {Detector: "A", Score: 80}, @@ -178,156 +151,19 @@ func TestConsolidateFindings(t *testing.T) { "": 55, }, }, - { - name: "empty weights fallback to equal", - findings: []Finding{ - {Detector: "A", Score: 100}, - {Detector: "B", Score: 50}, - {Detector: "C", Score: 75}, - }, - weights: map[string]float64{}, - wantOverall: 75, - wantDetectorScores: map[string]float64{ - "A": 100, - "B": 50, - "C": 75, - }, - }, - { - name: "all zero weights fallback to equal", - findings: []Finding{ - {Detector: "A", Score: 100}, - {Detector: "B", Score: 50}, - {Detector: "C", Score: 75}, - }, - weights: map[string]float64{ - "A": 0, - "B": 0, - "C": 0, - }, - wantOverall: 75, - wantDetectorScores: map[string]float64{ - "A": 100, - "B": 50, - "C": 75, - }, - }, - { - name: "negative weights treated as zero", - findings: []Finding{ - {Detector: "A", Score: 100}, - {Detector: "B", Score: 50}, - {Detector: "C", Score: 75}, - }, - weights: map[string]float64{ - "A": -1, - "B": 1, - "C": -2, - }, - wantOverall: 50, - wantDetectorScores: map[string]float64{ - "A": 100, - "B": 50, - "C": 75, - }, - }, - { - name: "extra weights ignored", - findings: []Finding{ - {Detector: "A", Score: 100}, - {Detector: "B", Score: 50}, - {Detector: "C", Score: 75}, - }, - weights: map[string]float64{ - "A": 1, - "B": 1, - "C": 1, - "D": 100, - }, - wantOverall: 75, - wantDetectorScores: map[string]float64{ - "A": 100, - "B": 50, - "C": 75, - }, - }, - { - name: "overall clamped above 100", - findings: []Finding{ - {Detector: "A", Score: 150}, - }, - wantOverall: 100, - wantDetectorScores: map[string]float64{ - "A": 150, - }, - }, - { - name: "overall clamped below zero", - findings: []Finding{ - {Detector: "A", Score: -50}, - }, - wantOverall: 0, - wantDetectorScores: map[string]float64{ - "A": -50, - }, - }, { name: "NaN score", findings: []Finding{ {Detector: "A", Score: math.NaN()}, - }, - wantNaN: true, - }, - { - name: "NaN weight", - findings: []Finding{ - {Detector: "A", Score: 100}, {Detector: "B", Score: 50}, }, - weights: map[string]float64{ - "A": math.NaN(), - "B": 1, - }, - wantNaN: true, - }, - { - name: "duplicate detectors keep max independently", - findings: []Finding{ - {Detector: "A", Score: 20}, - {Detector: "A", Score: 80}, - {Detector: "B", Score: 10}, - {Detector: "B", Score: 30}, - }, - wantOverall: 55, - wantDetectorScores: map[string]float64{ - "A": 80, - "B": 30, - }, - }, - { - name: "mixed positive and negative weights", - findings: []Finding{ - {Detector: "A", Score: 100}, - {Detector: "B", Score: 50}, - {Detector: "C", Score: 75}, - }, - weights: map[string]float64{ - "A": 2, - "B": -5, - "C": 2, - }, - // normalized -> 0.5, 0, 0.5 - wantOverall: 87.5, - wantDetectorScores: map[string]float64{ - "A": 100, - "B": 50, - "C": 75, - }, + wantOverall: math.NaN(), + wantNaN: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - overall, perDetectorScores := ConsolidateFindingScore(tt.findings, tt.weights) + overall, perDetectorScores := ConsolidateScoreByFindings(tt.findings) if tt.wantNaN { if !math.IsNaN(overall) { diff --git a/detection/toolmention/toolmention.go b/detection/toolmention/toolmention.go index 3c25c87..aca064b 100644 --- a/detection/toolmention/toolmention.go +++ b/detection/toolmention/toolmention.go @@ -1,7 +1,6 @@ package toolmention import ( - "fmt" "regexp" "sort" "strings" @@ -96,7 +95,7 @@ func (d *Detector) Detect(input detection.Input) []detection.Finding { score := detection.ToolMentionBaseScore confidence, err := detection.ScoreToConfidence(score) if err != nil { - log.Fatal(err) + confidence = detection.ConfidenceNone } findings := make([]detection.Finding, 0, len(toolMatches)) @@ -104,7 +103,8 @@ func (d *Detector) Detect(input detection.Input) []detection.Finding { findings = append(findings, detection.Finding{ Detector: d.Name(), Tool: match.name, - Confidence: detection.ConfidenceLow, + Score: score, + Confidence: confidence, Detail: "text mentions " + match.name, }) } diff --git a/detection/toolmention/toolmention_test.go b/detection/toolmention/toolmention_test.go index 65695c1..5c9769a 100644 --- a/detection/toolmention/toolmention_test.go +++ b/detection/toolmention/toolmention_test.go @@ -13,127 +13,101 @@ func TestDetect(t *testing.T) { name string input detection.Input wantTools []string - wantScore []float64 }{ { name: "Claude mention in text", input: detection.Input{Text: "I used Claude to write this PR"}, wantTools: []string{"Claude"}, - wantScore: []float64{detection.ToolMentionBaseScore}, }, { name: "Claude Code mention in text", input: detection.Input{Text: "Generated with Claude Code"}, wantTools: []string{"Claude Code"}, - wantScore: []float64{detection.ToolMentionBaseScore}, }, { name: "GitHub Copilot mention", input: detection.Input{Text: "GitHub Copilot helped with this"}, wantTools: []string{"GitHub Copilot"}, - wantScore: []float64{detection.ToolMentionBaseScore}, }, { name: "Copilot mention", input: detection.Input{Text: "Copilot was used to generate docs"}, wantTools: []string{"Copilot"}, - wantScore: []float64{detection.ToolMentionBaseScore}, }, { name: "multiple tools mentioned", input: detection.Input{Text: "I used Cursor and Aider for this PR"}, wantTools: []string{"Cursor", "Aider"}, - wantScore: []float64{ - detection.ToolMentionBaseScore, - detection.ToolMentionBaseScore, - }, }, { name: "case insensitive", input: detection.Input{Text: "I used CLAUDE to write this"}, wantTools: []string{"Claude"}, - wantScore: []float64{detection.ToolMentionBaseScore}, }, { name: "commit message scanned too", input: detection.Input{CommitMessage: "feat: add feature\n\nGenerated with Claude Code"}, wantTools: []string{"Claude Code"}, - wantScore: []float64{detection.ToolMentionBaseScore}, }, { name: "text and commit message combined", input: detection.Input{Text: "Used Cursor", CommitMessage: "aider: fix bug"}, wantTools: []string{"Cursor", "Aider"}, - wantScore: []float64{ - detection.ToolMentionBaseScore, - detection.ToolMentionBaseScore, - }, }, { name: "no mentions", input: detection.Input{Text: "This is a normal PR description"}, wantTools: nil, - wantScore: nil, }, { name: "empty input with spaces", input: detection.Input{Text: " ", CommitMessage: "\n \n"}, wantTools: nil, - wantScore: nil, }, { name: "empty input", input: detection.Input{}, wantTools: nil, - wantScore: nil, }, { name: "word boundary prevents partial match", input: detection.Input{Text: "The cursory review found nothing"}, wantTools: nil, - wantScore: nil, }, { name: "ChatGPT mention", input: detection.Input{Text: "I asked ChatGPT for help"}, wantTools: []string{"ChatGPT"}, - wantScore: []float64{detection.ToolMentionBaseScore}, }, { name: "t3.chat mention", input: detection.Input{Text: "I used t3.chat to compare model outputs"}, wantTools: []string{"t3.chat"}, - wantScore: []float64{detection.ToolMentionBaseScore}, }, { name: "t3.chat mention is case insensitive", input: detection.Input{Text: "Generated with T3.CHAT"}, wantTools: []string{"t3.chat"}, - wantScore: []float64{detection.ToolMentionBaseScore}, }, { name: "t3.chat word boundary prevents partial match", input: detection.Input{Text: "This mentions t3.chatty, not the tool"}, wantTools: nil, - wantScore: nil, }, { name: "Windsurf mention", input: detection.Input{Text: "Written with Windsurf IDE"}, wantTools: []string{"Windsurf"}, - wantScore: []float64{detection.ToolMentionBaseScore}, }, { name: "Devin mention", input: detection.Input{Text: "Devin created this PR"}, wantTools: []string{"Devin"}, - wantScore: []float64{detection.ToolMentionBaseScore}, }, { name: "duplicate tool mentions only produce one finding", input: detection.Input{Text: "Claude helped here. Claude helped there."}, wantTools: []string{"Claude"}, - wantScore: []float64{detection.ToolMentionBaseScore}, }, { name: "Qwen coder variant match", @@ -285,8 +259,10 @@ func TestDetect(t *testing.T) { t.Fatalf("findings count = %d, want %d. findings=%v", len(findings), len(tt.wantTools), findings) } - if len(findings) != len(tt.wantScore) { - t.Fatalf("score count = %d, want %d", len(findings), len(tt.wantScore)) + expectedScore := detection.ToolMentionBaseScore + expectedConfidence, err := detection.ScoreToConfidence(expectedScore) + if err != nil { + t.Fatalf("failed to calculate confidence: %v", err) } for i, f := range findings { @@ -294,13 +270,8 @@ func TestDetect(t *testing.T) { t.Errorf("tool[%d] = %q, want %q", i, f.Tool, tt.wantTools[i]) } - if f.Score != tt.wantScore[i] { - t.Errorf("score[%d] = %v, want %v", i, f.Score, tt.wantScore[i]) - } - - expectedConfidence, err := detection.ScoreToConfidence(tt.wantScore[i]) - if err != nil { - t.Fatalf("failed to calculate confidence: %v", err) + if f.Score != expectedScore { + t.Errorf("score[%d] = %v, want %v", i, f.Score, expectedScore) } if f.Confidence != expectedConfidence { diff --git a/output/output.go b/output/output.go index 45c008e..a9a1133 100644 --- a/output/output.go +++ b/output/output.go @@ -70,7 +70,7 @@ func FormatTextFindings(w io.Writer, findings []detection.Finding) error { fmt.Fprintf(w, "Found %d AI signal(s):\n", len(findings)) // compute consolidated score for these findings - overall, _ := detection.ConsolidateFindingScore(findings, nil) + overall, _ := detection.ConsolidateScoreByFindings(findings) fmt.Fprintf(w, "Overall score: %.1f / 100\n", overall) for _, f := range findings { @@ -88,7 +88,7 @@ func FormatJSONFindings(w io.Writer, findings []detection.Finding) error { OverallScore float64 `json:"overall_score"` }{ Findings: findings, - OverallScore: func() float64 { s, _ := detection.ConsolidateFindingScore(findings, nil); return s }(), + OverallScore: func() float64 { s, _ := detection.ConsolidateScoreByFindings(findings); return s }(), }) } diff --git a/scan/scan.go b/scan/scan.go index cde7e60..e7674cf 100644 --- a/scan/scan.go +++ b/scan/scan.go @@ -7,9 +7,10 @@ import ( // CommitResult holds findings for a single commit. type CommitResult struct { - Hash string `json:"hash"` - Findings []detection.Finding `json:"findings"` - Score float64 `json:"score"` + Hash string `json:"hash"` + Findings []detection.Finding `json:"findings"` + PerDetectorScores map[string]float64 `json:"per_detector_scores"` + Score float64 `json:"score"` } // Summary aggregates stats across all commits scanned. @@ -28,10 +29,6 @@ type Report struct { Summary Summary `json:"summary"` } -// Weights can be set (e.g., from cli) to control detector weighting used when consolidating scores. -// If nil, detectors are equally weighted. -var Weights map[string]float64 - // ScanCommitRange scans all commits in the given range using the provided detectors. func ScanCommitRange(repoPath, commitRange string, detectors []detection.Detector) (Report, error) { commits, err := gitops.ListCommits(repoPath, commitRange) @@ -88,12 +85,13 @@ func scanOneCommit(c gitops.Commit, branchName string, detectors []detection.Det findings = append(findings, d.Detect(input)...) } - score, _ := detection.ConsolidateFindingScore(findings, Weights) + score, perDetectorScores := detection.ConsolidateScoreByFindings(findings) return CommitResult{ - Hash: c.Hash, - Findings: findings, - Score: score, + Hash: c.Hash, + Findings: findings, + PerDetectorScores: perDetectorScores, + Score: score, } } @@ -116,7 +114,7 @@ func buildReport(results []CommitResult) Report { } } - overall, perDetectorScores := detection.ConsolidateFindingScore(allFindings, Weights) + overall, perDetectorScores := detection.ConsolidateScoreByFindings(allFindings) summary.PerDetectorScores = perDetectorScores summary.OverallScore = overall diff --git a/scan/scan_test.go b/scan/scan_test.go index 7567812..86114b1 100644 --- a/scan/scan_test.go +++ b/scan/scan_test.go @@ -142,10 +142,7 @@ func TestScanCommitRange(t *testing.T) { t.Error("expected Kimi K2.6 Opus in tool counts") } - // Check overall score - if report.Summary.OverallScore < 0 || report.Summary.OverallScore > 100 { - t.Error("invalid overall score") - } + // Check scoring perDetectorScores := report.Summary.PerDetectorScores committerScore := perDetectorScores["committer"] if committerScore != 95 { @@ -163,8 +160,8 @@ func TestScanCommitRange(t *testing.T) { if trailerScore != 85 { t.Errorf("expected trailer score to be 85, found %f", trailerScore) } - if report.Summary.OverallScore != 66.67 { - t.Errorf("expected overall score to be 66.67, found %f", report.Summary.OverallScore) + if report.Summary.OverallScore != 200 { + t.Errorf("expected overall score to be 200, found %f", report.Summary.OverallScore) } } @@ -269,6 +266,28 @@ func TestScanCommit(t *testing.T) { if !foundAssistedBy { t.Error("expected assistedby finding for Kimi K2.6") } + + // Check scoring + perDetectorScores := result.PerDetectorScores + committerScore := perDetectorScores["committer"] + if committerScore != 0 { + t.Errorf("expected committer score to be 0, found %f", committerScore) + } + gitnotesScore := perDetectorScores["gitnotes"] + if gitnotesScore != 0 { + t.Errorf("expected gitnotes score to be 0, found %f", gitnotesScore) + } + toolmentionScore := perDetectorScores["toolmention"] + if toolmentionScore != 20 { + t.Errorf("expected toolmention score to be 20, found %f", toolmentionScore) + } + trailerScore := perDetectorScores["trailer"] + if trailerScore != 75 { + t.Errorf("expected trailer score to be 85, found %f", trailerScore) + } + if result.Score != 95 { + t.Errorf("expected overall score to be 95, found %f", result.Score) + } } func TestScanText(t *testing.T) { From f2ad72dfcfbf824e6b98267bbb806880bd22ec88 Mon Sep 17 00:00:00 2001 From: Omkar P <45419097+omkar-foss@users.noreply.github.com> Date: Sat, 8 Aug 2026 22:06:28 +0530 Subject: [PATCH 5/9] Remove overall score, rename to conf levels, update tests Signed-off-by: Omkar P <45419097+omkar-foss@users.noreply.github.com> --- cmd/cmd.go | 59 ++-- cmd/cmd_test.go | 164 ++++++----- detection/committer/committer.go | 17 +- detection/committer/committer_test.go | 21 +- detection/detection.go | 95 ++++-- detection/detection_test.go | 214 +++++++------- detection/gitnotes/gitnotes.go | 11 +- detection/gitnotes/gitnotes_test.go | 13 +- detection/toolmention/toolmention.go | 11 +- detection/toolmention/toolmention_test.go | 7 +- detection/trailer/trailer.go | 21 +- detection/trailer/trailer_test.go | 4 +- output/output.go | 41 +-- output/output_test.go | 337 ++++++++++++++++++++-- scan/scan.go | 27 +- scan/scan_test.go | 25 +- 16 files changed, 718 insertions(+), 349 deletions(-) diff --git a/cmd/cmd.go b/cmd/cmd.go index edd0abf..27ef0a7 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -32,13 +32,13 @@ const ( ExitError = 2 ) -func allDetectors() []detection.Detector { +func allDetectors(confidenceLevels map[detection.Confidence]float64) []detection.Detector { return []detection.Detector{ - &committer.Detector{}, - &gitnotes.Detector{}, - &trailer.Detector{}, - &toolmention.Detector{}, - &branchname.Detector{}, + &committer.Detector{ConfidenceLevels: confidenceLevels}, + &gitnotes.Detector{ConfidenceLevels: confidenceLevels}, + &trailer.Detector{ConfidenceLevels: confidenceLevels}, + &toolmention.Detector{ConfidenceLevels: confidenceLevels}, + &branchname.Detector{ConfidenceLevels: confidenceLevels}, } } @@ -118,7 +118,7 @@ func scanCommand(stdout, stderr io.Writer, exitCode *int) *cobra.Command { var rangeFlag string var formatFlag string var minConfFlag string - var confidenceScoresFlag string + var confidenceLevelsFlag string cmd := &cobra.Command{ Use: "scan [repo-path]", @@ -169,22 +169,26 @@ Examples: return err } - // parse confidence-scores override if provided - if strings.TrimSpace(confidenceScoresFlag) != "" { - flagMap, err := parseKeyValueFloatList(confidenceScoresFlag) + // parse confidence-levels override if provided + confidenceLevels := detection.GetDefaultConfidenceLevels() + if strings.TrimSpace(confidenceLevelsFlag) != "" { + flagMap, err := parseKeyValueFloatList(confidenceLevelsFlag) if err != nil { fmt.Fprintln(stderr, err) *exitCode = ExitError return err } - if err := detection.SetConfidenceScoresFromStrings(flagMap); err != nil { + if confidenceLevels, err = detection.SetConfidenceLevelsFromStrings( + confidenceLevels, + flagMap, + ); err != nil { fmt.Fprintln(stderr, err) *exitCode = ExitError return err } } - detectors := allDetectors() + detectors := allDetectors(confidenceLevels) report, err := scan.ScanCommitRange(repoPath, rangeFlag, detectors) if err != nil { fmt.Fprintf(stderr, "error: %v\n", err) @@ -192,7 +196,7 @@ Examples: return err } - report = filterReport(report, minConf) + report = filterReport(report, minConf, confidenceLevels) switch formatFlag { case "json": @@ -224,7 +228,7 @@ Examples: cmd.Flags().StringVar(&rangeFlag, "range", "", "commit range in BASE..HEAD format") cmd.Flags().StringVar(&formatFlag, "format", "text", "output format: json or text") cmd.Flags().StringVar(&minConfFlag, "min-confidence", "low", "minimum confidence level: low, medium, high (or 1, 2, 3)") - cmd.Flags().StringVar(&confidenceScoresFlag, "confidence-scores", "", "override confidence->score mapping, e.g. 'low=20,medium=60,high=100'") + cmd.Flags().StringVar(&confidenceLevelsFlag, "confidence-levels", "", "override confidence->score mapping, e.g. 'low=20,medium=60,high=100'") return cmd } @@ -275,7 +279,7 @@ Examples: return err } - detectors := allDetectors() + detectors := allDetectors(detection.GetDefaultConfidenceLevels()) findings := scan.ScanText(string(textBytes), detectors) switch formatFlag { @@ -329,7 +333,11 @@ Examples: } } -func filterReport(report scan.Report, minConf detection.Confidence) scan.Report { +func filterReport( + report scan.Report, + minConf detection.Confidence, + confidenceLevels map[detection.Confidence]float64, +) scan.Report { if minConf <= detection.ConfidenceLow { return report } @@ -343,9 +351,6 @@ func filterReport(report scan.Report, minConf detection.Confidence) scan.Report }, } - // collect all findings to compute overall score after filtering - var overallScoreFindings []detection.Finding - for _, commit := range report.Commits { var commitFindings []detection.Finding for _, f := range commit.Findings { @@ -355,8 +360,15 @@ func filterReport(report scan.Report, minConf detection.Confidence) scan.Report } // Recompute per-commit score from the kept findings - commitScore, _ := detection.ConsolidateScoreByFindings(commitFindings) - result := scan.CommitResult{Hash: commit.Hash, Findings: commitFindings, Score: commitScore} + commitScore, perDetectorScores := detection.ConsolidateScoreByFindings(commitFindings) + confidence := detection.ScoreToConfidence(confidenceLevels, commitScore) + result := scan.CommitResult{ + Hash: commit.Hash, + Findings: commitFindings, + Score: commitScore, + Confidence: confidence, + PerDetectorScores: perDetectorScores, + } filtered.Commits = append(filtered.Commits, result) if len(commitFindings) > 0 { filtered.Summary.AICommits++ @@ -364,14 +376,9 @@ func filterReport(report scan.Report, minConf detection.Confidence) scan.Report for _, commitFinding := range commitFindings { filtered.Summary.ToolCounts[commitFinding.Tool]++ filtered.Summary.ByConfidence[commitFinding.Confidence.String()]++ - overallScoreFindings = append(overallScoreFindings, commitFinding) } } - // Compute new overall score for the filtered report. - overall, _ := detection.ConsolidateScoreByFindings(overallScoreFindings) - filtered.Summary.OverallScore = overall - return filtered } diff --git a/cmd/cmd_test.go b/cmd/cmd_test.go index 6059b5e..80aba31 100644 --- a/cmd/cmd_test.go +++ b/cmd/cmd_test.go @@ -257,16 +257,18 @@ func TestRunScanInvalidFormat(t *testing.T) { func TestFilterReport(t *testing.T) { tests := []struct { - name string - report scan.Report - minConf detection.Confidence - wantScore float64 - wantAICommits int - wantFindings int - wantTool string // optional + name string + report scan.Report + minConf detection.Confidence + wantAICommits int + wantFindings []int + wantPerDetectorScores []map[string]float64 + wantScores []float64 + wantConfidence detection.Confidence + wantTool string // optional }{ { - name: "keep only high confidence findings", + name: "keep low confidence and above findings", report: scan.Report{ Commits: []scan.CommitResult{ { @@ -275,9 +277,15 @@ func TestFilterReport(t *testing.T) { { Detector: "toolmention", Tool: "Claude", - Confidence: detection.ConfidenceLow, + Confidence: detection.ConfidenceMedium, Score: 20, }, + { + Detector: "trailer", + Tool: "Kimi K3", + Confidence: detection.ConfidenceHigh, + Score: 85, + }, { Detector: "trailer", Tool: "Claude Code", @@ -292,32 +300,28 @@ func TestFilterReport(t *testing.T) { { Detector: "toolmention", Tool: "Claude", - Confidence: detection.ConfidenceLow, + Confidence: detection.ConfidenceMedium, Score: 20, }, { Detector: "trailer", Tool: "Claude Code", Confidence: detection.ConfidenceHigh, - Score: 100, + Score: 105, }, }, }, }, - Summary: scan.Summary{ - TotalCommits: 2, - AICommits: 2, - ToolCounts: map[string]int{"Claude": 2, "Claude Code": 2}, - ByConfidence: map[string]int{"low": 2, "high": 2}, - PerDetectorScores: map[string]float64{"toolmention": 20, "trailer": 100}, - OverallScore: 120, - }, }, - minConf: detection.ConfidenceHigh, - wantScore: 100, + minConf: detection.ConfidenceMedium, wantAICommits: 2, - wantFindings: 2, // only high confidence ones from both commits - wantTool: "Claude Code", + wantFindings: []int{3, 2}, + wantTool: "Claude", + wantPerDetectorScores: []map[string]float64{ + {"toolmention": 20, "trailer": 100}, + {"toolmention": 20, "trailer": 105}, + }, + wantScores: []float64{120, 125}, }, { name: "all findings filtered out", @@ -334,14 +338,11 @@ func TestFilterReport(t *testing.T) { }, }, }, - Summary: scan.Summary{ - TotalCommits: 1, - }, }, minConf: detection.ConfidenceHigh, - wantScore: 0, wantAICommits: 0, - wantFindings: 0, + wantFindings: []int{0}, + wantScores: []float64{0}, }, { name: "empty findings", @@ -354,17 +355,17 @@ func TestFilterReport(t *testing.T) { }, }, minConf: detection.ConfidenceMedium, - wantScore: 0, wantAICommits: 0, - wantFindings: 0, + wantFindings: []int{0}, + wantScores: []float64{0}, }, { name: "no commits", report: scan.Report{}, minConf: detection.ConfidenceHigh, - wantScore: 0, wantAICommits: 0, - wantFindings: 0, + wantFindings: []int{}, + wantScores: []float64{}, }, { name: "low confidence threshold returns original report", @@ -381,45 +382,78 @@ func TestFilterReport(t *testing.T) { { Detector: "trailer", Confidence: detection.ConfidenceHigh, - Score: 100, + Score: 80, }, }, + Score: 100, + Confidence: detection.ConfidenceHigh, }, }, }, minConf: detection.ConfidenceLow, - wantFindings: 2, - wantScore: 120, + wantFindings: []int{2}, + wantScores: []float64{100}, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - filtered := filterReport(tt.report, tt.minConf) + filtered := filterReport(tt.report, tt.minConf, detection.GetDefaultConfidenceLevels()) - if tt.minConf == detection.ConfidenceLow { - if got := len(filtered.Commits[0].Findings); got != tt.wantFindings { - t.Fatalf("findings=%d want=%d", got, tt.wantFindings) + if tt.wantFindings != nil { + lenCommit := len(filtered.Commits) + lenWant := len(tt.wantFindings) + if lenCommit != lenWant { + t.Fatalf("invalid number of items to check, commit len=%d want len=%d", lenCommit, lenWant) + } + for i := range filtered.Commits { + if len(filtered.Commits[i].Findings) != tt.wantFindings[i] { + t.Fatalf("commit findings len=%d want findings=%d", lenCommit, lenWant) + } } - return } - if filtered.Summary.OverallScore != tt.wantScore { - t.Errorf("overall score=%v want=%v", - filtered.Summary.OverallScore, tt.wantScore) + if tt.wantScores != nil { + lenCommit := len(filtered.Commits) + lenWant := len(tt.wantScores) + if lenCommit != lenWant { + t.Fatalf("invalid number of items to check, commit len=%d want len=%d", lenCommit, lenWant) + } + for i := range filtered.Commits { + if filtered.Commits[i].Score != tt.wantScores[i] { + t.Fatalf("commit score=%f want score=%f", filtered.Commits[i].Score, tt.wantScores[i]) + } + } } if filtered.Summary.AICommits != tt.wantAICommits { - t.Errorf("AICommits=%d want=%d", - filtered.Summary.AICommits, tt.wantAICommits) + t.Errorf("AICommits=%d want=%d", filtered.Summary.AICommits, tt.wantAICommits) } gotFindings := 0 - for _, c := range filtered.Commits { - gotFindings += len(c.Findings) - } - if gotFindings != tt.wantFindings { - t.Errorf("findings=%d want=%d", gotFindings, tt.wantFindings) + for i, commit := range filtered.Commits { + gotFindings += len(commit.Findings) + + // compare per-detector scores for each commit + if tt.wantPerDetectorScores == nil { + continue + } + wantPerDetectorScores := tt.wantPerDetectorScores[i] + lenCommitScores := len(commit.PerDetectorScores) + lenWantScores := len(wantPerDetectorScores) + if lenCommitScores != lenWantScores { + t.Errorf("expected %d detectors in map but found=%d", lenWantScores, lenCommitScores) + } + for detector := range commit.PerDetectorScores { + if wantPerDetectorScores[detector] != commit.PerDetectorScores[detector] { + t.Errorf( + "expected %f, found %f for detector %q", + wantPerDetectorScores[detector], + commit.PerDetectorScores[detector], + detector, + ) + } + } } if tt.wantTool != "" { @@ -648,17 +682,17 @@ func TestRunScanFlags(t *testing.T) { }, { name: "invalid confidence scores format", - args: []string{"scan", "--confidence-scores=low", dir}, + args: []string{"scan", "--confidence-levels=low", dir}, wantCode: ExitError, }, { name: "reject NaN confidence score", - args: []string{"scan", "--confidence-scores=high=NaN", dir}, + args: []string{"scan", "--confidence-levels=high=NaN", dir}, wantCode: ExitError, }, { name: "both valid flags", - args: []string{"scan", "--confidence-scores=low=15,medium=55,high=95", dir}, + args: []string{"scan", "--confidence-levels=low=15,medium=55,high=95", dir}, wantCode: ExitAI, // or ExitNoAI }, } @@ -700,7 +734,7 @@ func TestRunScanScoreFlags(t *testing.T) { args: []string{ "scan", "--format=json", - "--confidence-scores=low=10,medium=50,high=90", + "--confidence-levels=low=10,medium=50,high=90", dir, }, }, @@ -724,14 +758,6 @@ func TestRunScanScoreFlags(t *testing.T) { for _, c := range report.Commits { findings = append(findings, c.Findings...) } - - expected, _ := detection.ConsolidateScoreByFindings(findings) - - if report.Summary.OverallScore != expected { - t.Fatalf("overall=%v want=%v", - report.Summary.OverallScore, - expected) - } }) } } @@ -743,31 +769,31 @@ func TestScanCommandInvalidFlags(t *testing.T) { }{ { name: "confidence missing equals", - args: []string{"--confidence-scores", "low"}, + args: []string{"--confidence-levels", "low"}, }, { name: "confidence empty key", - args: []string{"--confidence-scores", "=10"}, + args: []string{"--confidence-levels", "=10"}, }, { name: "confidence empty value", - args: []string{"--confidence-scores", "low="}, + args: []string{"--confidence-levels", "low="}, }, { name: "confidence invalid number", - args: []string{"--confidence-scores", "low=abc"}, + args: []string{"--confidence-levels", "low=abc"}, }, { name: "confidence NaN", - args: []string{"--confidence-scores", "high=NaN"}, + args: []string{"--confidence-levels", "high=NaN"}, }, { name: "confidence Inf", - args: []string{"--confidence-scores", "high=Inf"}, + args: []string{"--confidence-levels", "high=Inf"}, }, { name: "confidence -Inf", - args: []string{"--confidence-scores", "high=-Inf"}, + args: []string{"--confidence-levels", "high=-Inf"}, }, } diff --git a/detection/committer/committer.go b/detection/committer/committer.go index afca115..3f4c5ea 100644 --- a/detection/committer/committer.go +++ b/detection/committer/committer.go @@ -20,19 +20,19 @@ func init() { } } -type Detector struct{} +type Detector struct { + ConfidenceLevels map[detection.Confidence]float64 +} func (d *Detector) Name() string { return "committer" } -func (d *Detector) detectEmail(email, identityField string) []detection.Finding { +func (d *Detector) GetConfidenceLevels() map[detection.Confidence]float64 { return d.ConfidenceLevels } +func (d *Detector) detectEmail(email, identityField string) []detection.Finding { // Direct match against known emails if name, ok := detection.KnownAgentCommitters[email]; ok { score := detection.CommitterMatchBaseScore + detection.CommitterKnownEmailBonusPoints - confidence, err := detection.ScoreToConfidence(score) - if err != nil { - confidence = detection.ConfidenceNone - } + confidence := detection.ScoreToConfidence(d.ConfidenceLevels, score) return []detection.Finding{{ Detector: d.Name(), Tool: name, @@ -46,10 +46,7 @@ func (d *Detector) detectEmail(email, identityField string) []detection.Finding // Format: +@users.noreply.github.com if strings.HasSuffix(email, detection.GithubNoReplyEmailSuffix) { score := detection.CommitterMatchBaseScore + detection.CommitterEmailSuffixBonusPoints - confidence, err := detection.ScoreToConfidence(score) - if err != nil { - confidence = detection.ConfidenceNone - } + confidence := detection.ScoreToConfidence(d.ConfidenceLevels, score) if idx := strings.Index(email, "+"); idx > 0 { prefix := email[:idx] if name, ok := numericPrefixIndex[prefix]; ok { diff --git a/detection/committer/committer_test.go b/detection/committer/committer_test.go index e11cfd8..3ba4e92 100644 --- a/detection/committer/committer_test.go +++ b/detection/committer/committer_test.go @@ -21,10 +21,9 @@ func assertFindingMetadata(t *testing.T, finding detection.Finding, expectedScor t.Errorf("score = %v, want %v", finding.Score, expectedScore) } - expectedConfidence, err := detection.ScoreToConfidence(expectedScore) - if err != nil { - t.Fatalf("failed to calculate confidence: %v", err) - } + expectedConfidence := detection.ScoreToConfidence( + detection.GetDefaultConfidenceLevels(), expectedScore, + ) if finding.Confidence != expectedConfidence { t.Errorf("confidence = %d, want %d", finding.Confidence, expectedConfidence) @@ -36,7 +35,7 @@ func assertFindingMetadata(t *testing.T, finding detection.Finding, expectedScor } func TestDetectAllKnownEmails(t *testing.T) { - d := &Detector{} + d := &Detector{ConfidenceLevels: detection.GetDefaultConfidenceLevels()} for email, expectedName := range detection.KnownAgentCommitters { input := detection.Input{CommitEmail: email} findings := d.Detect(input) @@ -52,7 +51,7 @@ func TestDetectAllKnownEmails(t *testing.T) { } func TestDetectMixedCase(t *testing.T) { - d := &Detector{} + d := &Detector{ConfidenceLevels: detection.GetDefaultConfidenceLevels()} cases := []struct { input string wantTool string @@ -76,7 +75,7 @@ func TestDetectMixedCase(t *testing.T) { } func TestDetectWhitespace(t *testing.T) { - d := &Detector{} + d := &Detector{ConfidenceLevels: detection.GetDefaultConfidenceLevels()} cases := []string{ " 209825114+claude[bot]@users.noreply.github.com", "209825114+claude[bot]@users.noreply.github.com ", @@ -97,7 +96,7 @@ func TestDetectWhitespace(t *testing.T) { } func TestDetectNotFound(t *testing.T) { - d := &Detector{} + d := &Detector{ConfidenceLevels: detection.GetDefaultConfidenceLevels()} cases := []string{ "user@example.com", "", @@ -115,7 +114,7 @@ func TestDetectNotFound(t *testing.T) { } func TestDetectNumericPrefix(t *testing.T) { - d := &Detector{} + d := &Detector{ConfidenceLevels: detection.GetDefaultConfidenceLevels()} // Simulate a renamed bot: same numeric ID, different username cases := []struct { input string @@ -143,7 +142,7 @@ func TestDetectNumericPrefix(t *testing.T) { } func TestDetectNumericPrefixNoFalsePositive(t *testing.T) { - d := &Detector{} + d := &Detector{ConfidenceLevels: detection.GetDefaultConfidenceLevels()} // An email with a numeric prefix that doesn't match any known bot cases := []string{ "999999999+someone@users.noreply.github.com", @@ -165,7 +164,7 @@ func TestDetectAuthorAndCommitter(t *testing.T) { humanEmail = "human@example.com" ) - d := &Detector{} + d := &Detector{ConfidenceLevels: detection.GetDefaultConfidenceLevels()} tests := []struct { name string input detection.Input diff --git a/detection/detection.go b/detection/detection.go index 887d295..7c3c09c 100644 --- a/detection/detection.go +++ b/detection/detection.go @@ -1,6 +1,7 @@ package detection import ( + "encoding/json" "fmt" "math" "strings" @@ -10,7 +11,7 @@ import ( type Confidence int const ( - ConfidenceNone = 0 // Nil equivalent for confidence + ConfidenceNone Confidence = 0 // Nil equivalent for confidence ConfidenceLow Confidence = 1 // Tool name mentioned in text ConfidenceMedium Confidence = 2 // Commit message pattern match ConfidenceHigh Confidence = 3 // Bot email, co-author trailer, git AI ref @@ -29,61 +30,90 @@ func (c Confidence) String() string { } } -func (c *Confidence) Increment() { - *c = min(*c+1, ConfidenceHigh) +func (c Confidence) MarshalJSON() ([]byte, error) { + return json.Marshal(c.String()) } -// Default mapping from Confidence -> numeric score (0..100). -var defaultConfidenceScores = map[Confidence]float64{ - ConfidenceLow: 30.0, - ConfidenceMedium: 70.0, - ConfidenceHigh: 100.0, +func (c *Confidence) UnmarshalJSON(data []byte) error { + var s string + if err := json.Unmarshal(data, &s); err != nil { + return err + } + + switch s { + case "none": + *c = ConfidenceNone + case "low": + *c = ConfidenceLow + case "medium": + *c = ConfidenceMedium + case "high": + *c = ConfidenceHigh + default: + return fmt.Errorf("invalid confidence %q", s) + } + + return nil } -// confidenceScores holds the active mapping, can be overridence in tests or via cli. -var confidenceScores = map[Confidence]float64{ - ConfidenceLow: defaultConfidenceScores[ConfidenceLow], - ConfidenceMedium: defaultConfidenceScores[ConfidenceMedium], - ConfidenceHigh: defaultConfidenceScores[ConfidenceHigh], +func (c *Confidence) Increment() { + *c = min(*c+1, ConfidenceHigh) +} + +func GetDefaultConfidenceLevels() map[Confidence]float64 { + return map[Confidence]float64{ + ConfidenceLow: 30.0, + ConfidenceMedium: 70.0, + ConfidenceHigh: 100.0, + } } -func ScoreToConfidence(score float64) (Confidence, error) { - if score < 0 || score > 100 { - return ConfidenceNone, fmt.Errorf("invalid score, should be between 0 and 100") +func ScoreToConfidence(confidenceLevels map[Confidence]float64, score float64) Confidence { + if score >= confidenceLevels[ConfidenceHigh] { + return ConfidenceHigh + } + if score <= confidenceLevels[ConfidenceLow] { + return ConfidenceLow } levels := []Confidence{ConfidenceLow, ConfidenceMedium, ConfidenceHigh} for _, level := range levels { - if math.Round(score) <= confidenceScores[level] { - return level, nil + if math.Round(score) <= confidenceLevels[level] { + return level } } - return ConfidenceNone, fmt.Errorf("confidence intervals unable to categorize score") + return ConfidenceNone } -// SetConfidenceScoresFromStrings allows to update confidenceScores using a custom map -func SetConfidenceScoresFromStrings(userMapping map[string]float64) error { - tmp := map[Confidence]float64{} +// SetConfidenceLevelsFromStrings allows to update confidenceLevels using a custom map +func SetConfidenceLevelsFromStrings( + confidenceLevels map[Confidence]float64, + userMapping map[string]float64, +) (map[Confidence]float64, error) { for k, v := range userMapping { k = strings.ToLower(strings.TrimSpace(k)) switch k { case "low": - tmp[ConfidenceLow] = v + confidenceLevels[ConfidenceLow] = v case "medium": - tmp[ConfidenceMedium] = v + confidenceLevels[ConfidenceMedium] = v case "high": - tmp[ConfidenceHigh] = v + confidenceLevels[ConfidenceHigh] = v default: - return fmt.Errorf("unsupported confidence key: %s", k) + return nil, fmt.Errorf("unsupported confidence key: %s", k) } } // set defaults if unspecified in user mapping - for c, def := range defaultConfidenceScores { - if _, ok := tmp[c]; !ok { - tmp[c] = def + for c, def := range GetDefaultConfidenceLevels() { + if _, ok := confidenceLevels[c]; !ok { + confidenceLevels[c] = def } } - confidenceScores = tmp - return nil + + if confidenceLevels[ConfidenceLow] >= confidenceLevels[ConfidenceMedium] || + confidenceLevels[ConfidenceMedium] >= confidenceLevels[ConfidenceHigh] { + return nil, fmt.Errorf("low, medium, high must be in ascending order") + } + return confidenceLevels, nil } // ConfidenceFromString parses a confidence string or numeric value. @@ -128,6 +158,7 @@ func (f Finding) DisplayTool() string { type Detector interface { Name() string Detect(input Input) []Finding + GetConfidenceLevels() map[Confidence]float64 } // Input provides data for detectors to examine. Each detector reads the fields @@ -181,7 +212,7 @@ func (input *Input) GetNotes() (GitnoteParseResult, error) { return parseGitnotes(input.Notes) } -// ConsolidateScoreByFindings computes per-detector scores and a overall score from findings +// ConsolidateScoreByFindings computes per-detector scores and a total score from findings func ConsolidateScoreByFindings(findings []Finding) (float64, map[string]float64) { // For each detector, we take max of all findings for that detector. perDetectorScores := map[string]float64{} diff --git a/detection/detection_test.go b/detection/detection_test.go index 2135609..103fe00 100644 --- a/detection/detection_test.go +++ b/detection/detection_test.go @@ -57,7 +57,7 @@ func TestConsolidateFindings(t *testing.T) { tests := []struct { name string findings []Finding - wantOverall float64 + wantTotalScore float64 wantDetectorScores map[string]float64 wantNaN bool }{ @@ -68,7 +68,7 @@ func TestConsolidateFindings(t *testing.T) { {Detector: "B", Score: 50}, {Detector: "C", Score: 75}, }, - wantOverall: 225, + wantTotalScore: 225, wantDetectorScores: map[string]float64{ "A": 100, "B": 50, @@ -82,7 +82,7 @@ func TestConsolidateFindings(t *testing.T) { {Detector: "B", Score: -50}, {Detector: "C", Score: 75}, }, - wantOverall: 125, + wantTotalScore: 125, wantDetectorScores: map[string]float64{ "A": 100, "B": -50, @@ -98,7 +98,7 @@ func TestConsolidateFindings(t *testing.T) { {Detector: "C", Score: 75}, {Detector: "B", Score: -10}, }, - wantOverall: 245, + wantTotalScore: 245, wantDetectorScores: map[string]float64{ "A": 120, "B": 50, @@ -108,13 +108,13 @@ func TestConsolidateFindings(t *testing.T) { { name: "nil findings", findings: nil, - wantOverall: 0, + wantTotalScore: 0, wantDetectorScores: map[string]float64{}, }, { name: "empty findings", findings: []Finding{}, - wantOverall: 0, + wantTotalScore: 0, wantDetectorScores: map[string]float64{}, }, { @@ -124,7 +124,7 @@ func TestConsolidateFindings(t *testing.T) { {Detector: "A", Score: 80}, {Detector: "A", Score: 50}, }, - wantOverall: 80, + wantTotalScore: 80, wantDetectorScores: map[string]float64{ "A": 80, }, @@ -135,7 +135,7 @@ func TestConsolidateFindings(t *testing.T) { {Detector: " A ", Score: 60}, {Detector: "A", Score: 90}, }, - wantOverall: 90, + wantTotalScore: 90, wantDetectorScores: map[string]float64{ "A": 90, }, @@ -146,7 +146,7 @@ func TestConsolidateFindings(t *testing.T) { {Detector: "", Score: 10}, {Detector: " ", Score: 55}, }, - wantOverall: 55, + wantTotalScore: 55, wantDetectorScores: map[string]float64{ "": 55, }, @@ -157,23 +157,23 @@ func TestConsolidateFindings(t *testing.T) { {Detector: "A", Score: math.NaN()}, {Detector: "B", Score: 50}, }, - wantOverall: math.NaN(), - wantNaN: true, + wantTotalScore: math.NaN(), + wantNaN: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - overall, perDetectorScores := ConsolidateScoreByFindings(tt.findings) + totalScore, perDetectorScores := ConsolidateScoreByFindings(tt.findings) if tt.wantNaN { - if !math.IsNaN(overall) { - t.Fatalf("overall = %v, want NaN", overall) + if !math.IsNaN(totalScore) { + t.Fatalf("total score = %v, want NaN", totalScore) } return } - if overall != tt.wantOverall { - t.Fatalf("overall = %v, want %v", overall, tt.wantOverall) + if totalScore != tt.wantTotalScore { + t.Fatalf("total score = %v, want %v", totalScore, tt.wantTotalScore) } if !reflect.DeepEqual(perDetectorScores, tt.wantDetectorScores) { @@ -184,101 +184,77 @@ func TestConsolidateFindings(t *testing.T) { } func TestScoreToConfidence(t *testing.T) { + confidenceLevels := GetDefaultConfidenceLevels() tests := []struct { - name string - score float64 - want Confidence - wantErr bool + name string + score float64 + want Confidence }{ { - name: "zero score", - score: 0, - want: ConfidenceLow, - wantErr: false, + name: "zero score", + score: 0, + want: ConfidenceLow, }, { - name: "normal low score", - score: 25, - want: ConfidenceLow, - wantErr: false, + name: "normal low score", + score: 25, + want: ConfidenceLow, }, { - name: "medium boundary", - score: 50, - want: ConfidenceMedium, - wantErr: false, + name: "medium boundary", + score: 50, + want: ConfidenceMedium, }, { - name: "high boundary", - score: 75, - want: ConfidenceHigh, - wantErr: false, + name: "high boundary", + score: 75, + want: ConfidenceHigh, }, { - name: "maximum score", - score: 100, - want: ConfidenceHigh, - wantErr: false, + name: "maximum score", + score: 100, + want: ConfidenceHigh, }, { - name: "negative score", - score: -1, - want: ConfidenceNone, - wantErr: true, + name: "negative score", + score: -1, + want: ConfidenceLow, }, { - name: "above maximum score", - score: 101, - want: ConfidenceNone, - wantErr: true, + name: "above maximum score", + score: 101, + want: ConfidenceHigh, }, { - name: "positive infinity", - score: math.Inf(1), - want: ConfidenceNone, - wantErr: true, + name: "positive infinity", + score: math.Inf(1), + want: ConfidenceHigh, }, { - name: "negative infinity", - score: math.Inf(-1), - want: ConfidenceNone, - wantErr: true, + name: "negative infinity", + score: math.Inf(-1), + want: ConfidenceLow, }, { - name: "NaN score", - score: math.NaN(), - want: ConfidenceNone, - wantErr: true, + name: "NaN score", + score: math.NaN(), + want: ConfidenceNone, }, { - name: "rounding to low boundary", - score: confidenceScores[ConfidenceLow] - 0.4, - want: ConfidenceLow, - wantErr: false, + name: "rounding to low boundary", + score: confidenceLevels[ConfidenceLow] - 0.4, + want: ConfidenceLow, }, { - name: "rounding past low boundary", - score: confidenceScores[ConfidenceLow] + 0.5, - want: ConfidenceMedium, - wantErr: false, + name: "rounding past low boundary", + score: confidenceLevels[ConfidenceLow] + 0.5, + want: ConfidenceMedium, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := ScoreToConfidence(tt.score) - - if tt.wantErr { - if err == nil { - t.Fatalf("expected error, got nil") - } - return - } - - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - + got := ScoreToConfidence(confidenceLevels, tt.score) if got != tt.want { t.Fatalf("confidence=%v, want %v", got, tt.want) } @@ -286,12 +262,12 @@ func TestScoreToConfidence(t *testing.T) { } } -func TestSetConfidenceScoresFromStrings(t *testing.T) { +func TestSetConfidenceLevelsFromStrings(t *testing.T) { tests := []struct { name string input map[string]float64 wantErr bool - check func(t *testing.T) + check func(t *testing.T, confidenceLevels map[Confidence]float64) }{ { name: "full custom mapping", @@ -301,14 +277,14 @@ func TestSetConfidenceScoresFromStrings(t *testing.T) { "high": 90, }, wantErr: false, - check: func(t *testing.T) { - if confidenceScores[ConfidenceLow] != 30 { + check: func(t *testing.T, confidenceLevels map[Confidence]float64) { + if confidenceLevels[ConfidenceLow] != 30 { t.Fatalf("low score mismatch") } - if confidenceScores[ConfidenceMedium] != 60 { + if confidenceLevels[ConfidenceMedium] != 60 { t.Fatalf("medium score mismatch") } - if confidenceScores[ConfidenceHigh] != 90 { + if confidenceLevels[ConfidenceHigh] != 90 { t.Fatalf("high score mismatch") } }, @@ -321,8 +297,8 @@ func TestSetConfidenceScoresFromStrings(t *testing.T) { "HIGH": 80, }, wantErr: false, - check: func(t *testing.T) { - if confidenceScores[ConfidenceLow] != 20 { + check: func(t *testing.T, confidenceLevels map[Confidence]float64) { + if confidenceLevels[ConfidenceLow] != 20 { t.Fatalf("low score mismatch") } }, @@ -333,8 +309,8 @@ func TestSetConfidenceScoresFromStrings(t *testing.T) { " low ": 25, }, wantErr: false, - check: func(t *testing.T) { - if confidenceScores[ConfidenceLow] != 25 { + check: func(t *testing.T, confidenceLevels map[Confidence]float64) { + if confidenceLevels[ConfidenceLow] != 25 { t.Fatalf("low score mismatch") } }, @@ -343,8 +319,9 @@ func TestSetConfidenceScoresFromStrings(t *testing.T) { name: "empty mapping uses defaults", input: map[string]float64{}, wantErr: false, - check: func(t *testing.T) { - if confidenceScores[ConfidenceLow] != defaultConfidenceScores[ConfidenceLow] { + check: func(t *testing.T, confidenceLevels map[Confidence]float64) { + defaultConfidenceLevels := GetDefaultConfidenceLevels() + if confidenceLevels[ConfidenceLow] != defaultConfidenceLevels[ConfidenceLow] { t.Fatalf("default low mismatch") } }, @@ -355,15 +332,50 @@ func TestSetConfidenceScoresFromStrings(t *testing.T) { "low": 10, }, wantErr: false, - check: func(t *testing.T) { - if confidenceScores[ConfidenceLow] != 10 { + check: func(t *testing.T, confidenceLevels map[Confidence]float64) { + defaultConfidenceLevels := GetDefaultConfidenceLevels() + if confidenceLevels[ConfidenceLow] != 10 { t.Fatalf("custom low missing") } - if confidenceScores[ConfidenceHigh] != defaultConfidenceScores[ConfidenceHigh] { + if confidenceLevels[ConfidenceHigh] != defaultConfidenceLevels[ConfidenceHigh] { t.Fatalf("high should use default") } }, }, + { + name: "all levels must be in ascending order", + input: map[string]float64{ + "low": 40, + "medium": 30, + "high": 90, + }, + wantErr: true, + }, + { + name: "levels must be in ascending order even with defaults", + input: map[string]float64{ + "high": 30, + }, + wantErr: true, + }, + { + name: "levels cannot be equal when defaults are used", + input: map[string]float64{ + "low": 70, + "medium": 70, + "high": 100, + }, + wantErr: true, + }, + { + name: "levels cannot be equal even when defaults are used", + input: map[string]float64{ + "low": 70, + // medium defaults to 70 + "high": 100, + }, + wantErr: true, + }, { name: "unsupported key", input: map[string]float64{ @@ -385,8 +397,8 @@ func TestSetConfidenceScoresFromStrings(t *testing.T) { "low": -10, }, wantErr: false, - check: func(t *testing.T) { - if confidenceScores[ConfidenceLow] != -10 { + check: func(t *testing.T, confidenceLevels map[Confidence]float64) { + if confidenceLevels[ConfidenceLow] != -10 { t.Fatalf("expected negative threshold to be accepted") } }, @@ -395,8 +407,8 @@ func TestSetConfidenceScoresFromStrings(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - err := SetConfidenceScoresFromStrings(tt.input) - + confidenceLevels := GetDefaultConfidenceLevels() + confidenceLevels, err := SetConfidenceLevelsFromStrings(confidenceLevels, tt.input) if tt.wantErr { if err == nil { t.Fatalf("expected error, got nil") @@ -409,7 +421,7 @@ func TestSetConfidenceScoresFromStrings(t *testing.T) { } if tt.check != nil { - tt.check(t) + tt.check(t, confidenceLevels) } }) } diff --git a/detection/gitnotes/gitnotes.go b/detection/gitnotes/gitnotes.go index d6190df..fd91a86 100644 --- a/detection/gitnotes/gitnotes.go +++ b/detection/gitnotes/gitnotes.go @@ -7,10 +7,14 @@ import ( "github.com/chaoss/disclosure/detection" ) -type Detector struct{} +type Detector struct { + ConfidenceLevels map[detection.Confidence]float64 +} func (d *Detector) Name() string { return "gitnotes" } +func (d *Detector) GetConfidenceLevels() map[detection.Confidence]float64 { return d.ConfidenceLevels } + type toolModelPair struct { tool string model string @@ -30,10 +34,7 @@ func (d *Detector) Detect(input detection.Input) []detection.Finding { } sort.Strings(promptIDs) score := detection.GitNotesMatchBaseScore - confidence, err := detection.ScoreToConfidence(score) - if err != nil { - confidence = detection.ConfidenceNone - } + confidence := detection.ScoreToConfidence(d.ConfidenceLevels, score) for _, promptID := range promptIDs { prompt := parseResult.Metadata.Prompts[promptID] diff --git a/detection/gitnotes/gitnotes_test.go b/detection/gitnotes/gitnotes_test.go index 743df2c..d0d5ff9 100644 --- a/detection/gitnotes/gitnotes_test.go +++ b/detection/gitnotes/gitnotes_test.go @@ -13,10 +13,9 @@ func assertFindingMetadata(t *testing.T, finding detection.Finding, wantScore fl t.Errorf("score = %f, want %f", finding.Score, wantScore) } - expectedConfidence, err := detection.ScoreToConfidence(wantScore) - if err != nil { - t.Fatalf("failed to calculate confidence: %v", err) - } + expectedConfidence := detection.ScoreToConfidence( + detection.GetDefaultConfidenceLevels(), wantScore, + ) if finding.Confidence != expectedConfidence { t.Errorf("confidence = %d, want %d", finding.Confidence, expectedConfidence) @@ -28,7 +27,7 @@ func assertFindingMetadata(t *testing.T, finding detection.Finding, wantScore fl } func TestDetect(t *testing.T) { - d := &Detector{} + d := &Detector{ConfidenceLevels: detection.GetDefaultConfidenceLevels()} validNote := `src/main.rs abcd1234abcd1234 1-10,15-20 @@ -182,7 +181,7 @@ src/lib.rs } func TestDetectPreservesDistinctToolModelPairs(t *testing.T) { - d := &Detector{} + d := &Detector{ConfidenceLevels: detection.GetDefaultConfidenceLevels()} note := `src/main.rs first 1-10 second 11-20 @@ -233,7 +232,7 @@ func TestDetectPreservesDistinctToolModelPairs(t *testing.T) { } func TestDetectDetailIncludesModel(t *testing.T) { - d := &Detector{} + d := &Detector{ConfidenceLevels: detection.GetDefaultConfidenceLevels()} note := `src/main.rs abcd1234abcd1234 1-10 --- diff --git a/detection/toolmention/toolmention.go b/detection/toolmention/toolmention.go index aca064b..9c0b2c3 100644 --- a/detection/toolmention/toolmention.go +++ b/detection/toolmention/toolmention.go @@ -41,10 +41,14 @@ func init() { } } -type Detector struct{} +type Detector struct { + ConfidenceLevels map[detection.Confidence]float64 +} func (d *Detector) Name() string { return "toolmention" } +func (d *Detector) GetConfidenceLevels() map[detection.Confidence]float64 { return d.ConfidenceLevels } + type toolMatch struct { start int end int @@ -93,10 +97,7 @@ func (d *Detector) Detect(input detection.Input) []detection.Finding { } score := detection.ToolMentionBaseScore - confidence, err := detection.ScoreToConfidence(score) - if err != nil { - confidence = detection.ConfidenceNone - } + confidence := detection.ScoreToConfidence(d.ConfidenceLevels, score) findings := make([]detection.Finding, 0, len(toolMatches)) for _, match := range toolMatches { diff --git a/detection/toolmention/toolmention_test.go b/detection/toolmention/toolmention_test.go index 5c9769a..3c84b33 100644 --- a/detection/toolmention/toolmention_test.go +++ b/detection/toolmention/toolmention_test.go @@ -7,7 +7,7 @@ import ( ) func TestDetect(t *testing.T) { - d := &Detector{} + d := &Detector{ConfidenceLevels: detection.GetDefaultConfidenceLevels()} tests := []struct { name string @@ -260,10 +260,7 @@ func TestDetect(t *testing.T) { } expectedScore := detection.ToolMentionBaseScore - expectedConfidence, err := detection.ScoreToConfidence(expectedScore) - if err != nil { - t.Fatalf("failed to calculate confidence: %v", err) - } + expectedConfidence := detection.ScoreToConfidence(d.ConfidenceLevels, expectedScore) for i, f := range findings { if f.Tool != tt.wantTools[i] { diff --git a/detection/trailer/trailer.go b/detection/trailer/trailer.go index 2ce2eba..05adba4 100644 --- a/detection/trailer/trailer.go +++ b/detection/trailer/trailer.go @@ -101,10 +101,14 @@ var commitMessagePatterns = []struct { }, } -type Detector struct{} +type Detector struct { + ConfidenceLevels map[detection.Confidence]float64 +} func (d *Detector) Name() string { return "trailer" } +func (d *Detector) GetConfidenceLevels() map[detection.Confidence]float64 { return d.ConfidenceLevels } + type toolModelPair struct { tool string model string @@ -176,10 +180,7 @@ func (d *Detector) detectTrailerCoauthoredBy(commitMessage string) []detection.F continue } score += detection.CoauthorKnownEmailBonusPoints - confidence, err := detection.ScoreToConfidence(score) - if err != nil { - confidence = detection.ConfidenceNone - } + confidence := detection.ScoreToConfidence(d.ConfidenceLevels, score) findings = append(findings, detection.Finding{ Detector: d.Name(), Tool: name, @@ -219,10 +220,7 @@ func (d *Detector) detectTrailerAssistedBy(commitMessage string) []detection.Fin } score := detection.AssistedByTrailerBaseScore - confidence, err := detection.ScoreToConfidence(score) - if err != nil { - confidence = detection.ConfidenceNone - } + confidence := detection.ScoreToConfidence(d.ConfidenceLevels, score) findings = append(findings, detection.Finding{ Detector: d.Name(), @@ -240,10 +238,7 @@ func (d *Detector) detectMessagePatterns(commitMessage string) []detection.Findi var findings []detection.Finding for _, p := range commitMessagePatterns { if score, isDetected := p.check(commitMessage); isDetected { - confidence, err := detection.ScoreToConfidence(score) - if err != nil { - confidence = detection.ConfidenceNone - } + confidence := detection.ScoreToConfidence(d.ConfidenceLevels, score) findings = append(findings, detection.Finding{ Detector: d.Name(), Tool: p.name, diff --git a/detection/trailer/trailer_test.go b/detection/trailer/trailer_test.go index 2bcff9c..c8bbcda 100644 --- a/detection/trailer/trailer_test.go +++ b/detection/trailer/trailer_test.go @@ -7,7 +7,7 @@ import ( ) func TestDetect(t *testing.T) { - d := &Detector{} + d := &Detector{ConfidenceLevels: detection.GetDefaultConfidenceLevels()} tests := []struct { name string message string @@ -568,7 +568,7 @@ Signed-off-by: some human } func BenchmarkDetect(b *testing.B) { - d := &Detector{} + d := &Detector{ConfidenceLevels: detection.GetDefaultConfidenceLevels()} tests := []struct { name string message string diff --git a/output/output.go b/output/output.go index a9a1133..c653b8a 100644 --- a/output/output.go +++ b/output/output.go @@ -21,11 +21,6 @@ func FormatJSON(w io.Writer, report scan.Report) error { func FormatText(w io.Writer, report scan.Report) error { fmt.Fprintf(w, "Scanned %d commits, %d with AI signals\n\n", report.Summary.TotalCommits, report.Summary.AICommits) - // overall numeric score - if report.Summary.OverallScore > 0 { - fmt.Fprintf(w, "Overall score: %.1f / 100\n\n", report.Summary.OverallScore) - } - if report.Summary.AICommits == 0 { fmt.Fprintln(w, "No AI involvement detected.") return nil @@ -48,13 +43,12 @@ func FormatText(w io.Writer, report scan.Report) error { if len(hash) > 12 { hash = hash[:12] } - if cr.Score > 0 { - fmt.Fprintf(w, "Commit %s (score: %.1f)\n", hash, cr.Score) - } else { - fmt.Fprintf(w, "Commit %s\n", hash) - } + fmt.Fprintf(w, "Commit %s (score: %.1f, confidence: %s)\n", hash, cr.Score, cr.Confidence.String()) for _, f := range cr.Findings { - fmt.Fprintf(w, " [%s] %s (%s): %s\n", f.Confidence, f.DisplayTool(), f.Detector, f.Detail) + fmt.Fprintf( + w, " [score: %.1f, confidence: %s] %s (%s): %s\n", + f.Score, f.Confidence, f.DisplayTool(), f.Detector, f.Detail, + ) } } @@ -69,12 +63,17 @@ func FormatTextFindings(w io.Writer, findings []detection.Finding) error { } fmt.Fprintf(w, "Found %d AI signal(s):\n", len(findings)) - // compute consolidated score for these findings - overall, _ := detection.ConsolidateScoreByFindings(findings) - fmt.Fprintf(w, "Overall score: %.1f / 100\n", overall) + + // compute consolidated score and confidence for these findings + score, _ := detection.ConsolidateScoreByFindings(findings) + confidence := detection.ScoreToConfidence(detection.GetDefaultConfidenceLevels(), score) + fmt.Fprintf(w, "Score: %.1f, Confidence: %s\n", score, confidence.String()) for _, f := range findings { - fmt.Fprintf(w, " [%s] %s (%s): %s\n", f.Confidence, f.DisplayTool(), f.Detector, f.Detail) + fmt.Fprintf( + w, " [score: %.1f, confidence: %s] %s (%s): %s\n", + f.Score, f.Confidence.String(), f.DisplayTool(), f.Detector, f.Detail, + ) } return nil } @@ -83,12 +82,16 @@ func FormatTextFindings(w io.Writer, findings []detection.Finding) error { func FormatJSONFindings(w io.Writer, findings []detection.Finding) error { enc := json.NewEncoder(w) enc.SetIndent("", " ") + score, _ := detection.ConsolidateScoreByFindings(findings) + confidence := detection.ScoreToConfidence(detection.GetDefaultConfidenceLevels(), score) return enc.Encode(struct { - Findings []detection.Finding `json:"findings"` - OverallScore float64 `json:"overall_score"` + Findings []detection.Finding `json:"findings"` + Score float64 `json:"score"` + Confidence detection.Confidence `json:"confidence"` }{ - Findings: findings, - OverallScore: func() float64 { s, _ := detection.ConsolidateScoreByFindings(findings); return s }(), + Findings: findings, + Score: score, + Confidence: confidence, }) } diff --git a/output/output_test.go b/output/output_test.go index 87238c2..905e98d 100644 --- a/output/output_test.go +++ b/output/output_test.go @@ -24,14 +24,17 @@ func sampleReport() scan.Report { Model: "Opus 4", Confidence: detection.ConfidenceHigh, Detail: "Co-Authored-By trailer with email noreply@anthropic.com", + Score: 100.0, }, }, - Score: 100.0, + Score: 100.0, + Confidence: detection.ConfidenceHigh, }, { - Hash: "def789ghi012", - Findings: nil, - Score: 0.0, + Hash: "def789ghi012", + Findings: nil, + Score: 0.0, + Confidence: detection.ConfidenceLow, }, }, Summary: scan.Summary{ @@ -39,7 +42,6 @@ func sampleReport() scan.Report { AICommits: 1, ToolCounts: map[string]int{"Claude Code": 1}, ByConfidence: map[string]int{"high": 1}, - OverallScore: 100.0, }, } } @@ -89,9 +91,6 @@ func TestFormatText(t *testing.T) { if !strings.Contains(out, "abc123def456") { t.Errorf("expected commit hash in output, got:\n%s", out) } - if !strings.Contains(out, "Overall score") { - t.Errorf("expected overall score in output, got:\n%s", out) - } } func TestFormatTextNoFindings(t *testing.T) { @@ -173,9 +172,6 @@ func TestFormatJSONEmptyReport(t *testing.T) { if err := json.Unmarshal(buf.Bytes(), &decoded); err != nil { t.Fatalf("unmarshal: %v", err) } - if decoded.Summary.OverallScore != 0 { - t.Fatalf("overall score = %v, want 0", decoded.Summary.OverallScore) - } } type failingWriter struct{} @@ -197,15 +193,14 @@ func TestFormatTextZeroScore(t *testing.T) { Summary: scan.Summary{ TotalCommits: 1, AICommits: 1, - OverallScore: 0, ToolCounts: map[string]int{}, }, } if err := FormatText(&buf, report); err != nil { t.Fatal(err) } - if strings.Contains(buf.String(), "Overall score") { - t.Error("did not expect overall score for zero") + if strings.Contains(buf.String(), "Score") { + t.Error("did not expect score for zero") } } @@ -245,9 +240,13 @@ func TestFormatTextFindingsIncludesScore(t *testing.T) { if err := FormatTextFindings(&buf, findings); err != nil { t.Fatal(err) } - if !strings.Contains(buf.String(), "Overall score:") { + formatTextStr := buf.String() + if !strings.Contains(formatTextStr, "Score:") { t.Fatal("missing score") } + if !strings.Contains(formatTextStr, "Confidence:") { + t.Fatal("missing confidence") + } } func TestFormatTextFindingsEmptySlice(t *testing.T) { @@ -264,17 +263,19 @@ func TestFormatJSONFindingsStructure(t *testing.T) { var buf bytes.Buffer findings := []detection.Finding{ { - Detector: "toolmention", - Tool: "Claude", - Score: 100, + Detector: "toolmention", + Tool: "Claude", + Score: 100, + Confidence: detection.ConfidenceHigh, }, } if err := FormatJSONFindings(&buf, findings); err != nil { t.Fatal(err) } var decoded struct { - Findings []detection.Finding `json:"findings"` - Score float64 `json:"overall_score"` + Findings []detection.Finding `json:"findings"` + Score float64 `json:"score"` + Confidence detection.Confidence `json:"confidence"` } if err := json.Unmarshal(buf.Bytes(), &decoded); err != nil { t.Fatal(err) @@ -285,6 +286,9 @@ func TestFormatJSONFindingsStructure(t *testing.T) { if decoded.Score != 100 { t.Fatalf("score=%v want 100", decoded.Score) } + if decoded.Confidence != detection.ConfidenceHigh { + t.Fatalf("confidence=%v want high", decoded.Confidence) + } } func TestSortedKeys(t *testing.T) { @@ -294,3 +298,296 @@ func TestSortedKeys(t *testing.T) { t.Fatalf("got %v want %v", got, want) } } + +func TestFormatTextExactOutput(t *testing.T) { + var buf bytes.Buffer + + report := sampleReport() + + if err := FormatText(&buf, report); err != nil { + t.Fatalf("FormatText: %v", err) + } + + want := `Scanned 2 commits, 1 with AI signals + +Tools detected: + Claude Code: 1 + +Commit abc123def456 (score: 100.0, confidence: high) + [score: 100.0, confidence: high] Claude Code [Opus 4] (trailer): Co-Authored-By trailer with email noreply@anthropic.com +` + + if got := buf.String(); got != want { + t.Errorf("FormatText output mismatch\n--- got ---\n%q\n--- want ---\n%q", got, want) + } +} + +func TestFormatTextExactOutputMultipleTools(t *testing.T) { + var buf bytes.Buffer + + report := scan.Report{ + Commits: []scan.CommitResult{ + { + Hash: "1234567890abcdef", + Findings: []detection.Finding{ + { + Detector: "toolmention", + Tool: "Copilot", + Confidence: detection.ConfidenceLow, + Score: 20, + Detail: "copilot finding", + }, + { + Detector: "toolmention", + Tool: "Kimi", + Confidence: detection.ConfidenceLow, + Score: 20, + Detail: "kimi finding", + }, + { + Detector: "trailer", + Tool: "Claude", + Confidence: detection.ConfidenceHigh, + Score: 75, + Detail: "claude finding", + }, + }, + Score: 95, + Confidence: detection.ConfidenceHigh, + }, + }, + Summary: scan.Summary{ + TotalCommits: 1, + AICommits: 1, + ToolCounts: map[string]int{ + "Claude": 1, + "Copilot": 1, + "Kimi": 1, + }, + }, + } + + if err := FormatText(&buf, report); err != nil { + t.Fatalf("FormatText: %v", err) + } + + want := `Scanned 1 commits, 1 with AI signals + +Tools detected: + Claude: 1 + Copilot: 1 + Kimi: 1 + +Commit 1234567890ab (score: 95.0, confidence: high) + [score: 20.0, confidence: low] Copilot (toolmention): copilot finding + [score: 20.0, confidence: low] Kimi (toolmention): kimi finding + [score: 75.0, confidence: high] Claude (trailer): claude finding +` + + if got := buf.String(); got != want { + t.Errorf("FormatText output mismatch\n--- got ---\n%q\n--- want ---\n%q", got, want) + } +} + +func TestFormatTextNoAIExactOutput(t *testing.T) { + var buf bytes.Buffer + + report := scan.Report{ + Summary: scan.Summary{ + TotalCommits: 5, + AICommits: 0, + ToolCounts: map[string]int{}, + }, + } + + if err := FormatText(&buf, report); err != nil { + t.Fatalf("FormatText: %v", err) + } + + want := "Scanned 5 commits, 0 with AI signals\n\nNo AI involvement detected.\n" + + if got := buf.String(); got != want { + t.Errorf("FormatText output mismatch\n--- got ---\n%q\n--- want ---\n%q", got, want) + } +} + +func TestFormatTextSkipsCommitsWithoutFindings(t *testing.T) { + var buf bytes.Buffer + + report := scan.Report{ + Commits: []scan.CommitResult{ + { + Hash: "with-findings", + Findings: []detection.Finding{ + { + Detector: "test", + Tool: "Claude", + Confidence: detection.ConfidenceHigh, + Score: 100, + Detail: "detected", + }, + }, + Score: 100, + Confidence: detection.ConfidenceHigh, + }, + { + Hash: "without-findings", + Findings: nil, + Score: 0, + Confidence: detection.ConfidenceNone, + }, + }, + Summary: scan.Summary{ + TotalCommits: 2, + AICommits: 1, + ToolCounts: map[string]int{ + "Claude": 1, + }, + }, + } + + if err := FormatText(&buf, report); err != nil { + t.Fatalf("FormatText: %v", err) + } + + want := `Scanned 2 commits, 1 with AI signals + +Tools detected: + Claude: 1 + +Commit with-finding (score: 100.0, confidence: high) + [score: 100.0, confidence: high] Claude (test): detected +` + + if got := buf.String(); got != want { + t.Errorf("FormatText output mismatch\n--- got ---\n%q\n--- want ---\n%q", got, want) + } +} + +func TestFormatTextFindingsExactOutput(t *testing.T) { + var buf bytes.Buffer + + findings := []detection.Finding{ + { + Detector: "toolmention", + Tool: "Claude", + Confidence: detection.ConfidenceLow, + Score: 25, + Detail: "text mentions Claude", + }, + { + Detector: "gitnotes", + Model: "gpt-4o", + Confidence: detection.ConfidenceHigh, + Score: 100, + Detail: "git notes declares model", + }, + } + + if err := FormatTextFindings(&buf, findings); err != nil { + t.Fatalf("FormatTextFindings: %v", err) + } + + want := `Found 2 AI signal(s): +Score: 125.0, Confidence: high + [score: 25.0, confidence: low] Claude (toolmention): text mentions Claude + [score: 100.0, confidence: high] gpt-4o (gitnotes): git notes declares model +` + + if got := buf.String(); got != want { + t.Errorf("FormatTextFindings output mismatch\n--- got ---\n%q\n--- want ---\n%q", got, want) + } +} + +func TestFormatTextFindingsEmptyExactOutput(t *testing.T) { + var buf bytes.Buffer + + if err := FormatTextFindings(&buf, nil); err != nil { + t.Fatalf("FormatTextFindings: %v", err) + } + + want := "No AI involvement detected.\n" + + if got := buf.String(); got != want { + t.Errorf("FormatTextFindings output mismatch\n--- got ---\n%q\n--- want ---\n%q", got, want) + } +} + +func TestFormatTextFindingsEmptySliceExactOutput(t *testing.T) { + var buf bytes.Buffer + + if err := FormatTextFindings(&buf, []detection.Finding{}); err != nil { + t.Fatalf("FormatTextFindings: %v", err) + } + + want := "No AI involvement detected.\n" + + if got := buf.String(); got != want { + t.Errorf("FormatTextFindings output mismatch\n--- got ---\n%q\n--- want ---\n%q", got, want) + } +} + +func TestFormatJSONExactIndentation(t *testing.T) { + var buf bytes.Buffer + + report := scan.Report{} + + if err := FormatJSON(&buf, report); err != nil { + t.Fatalf("FormatJSON: %v", err) + } + + got := buf.String() + + if !strings.HasSuffix(got, "\n") { + t.Errorf("FormatJSON output does not end with newline: %q", got) + } + + if !strings.Contains(got, "\n ") { + t.Errorf("FormatJSON output is not indented with two spaces:\n%s", got) + } +} + +func TestFormatJSONFindingsExactFormatting(t *testing.T) { + var buf bytes.Buffer + + findings := []detection.Finding{ + { + Detector: "toolmention", + Tool: "Claude", + Confidence: detection.ConfidenceLow, + Score: 100, + Detail: "text mentions Claude", + }, + } + + if err := FormatJSONFindings(&buf, findings); err != nil { + t.Fatalf("FormatJSONFindings: %v", err) + } + + got := buf.String() + + if !strings.HasSuffix(got, "\n") { + t.Errorf("expected trailing newline, got %q", got) + } + + if !strings.Contains(got, "\n ") { + t.Errorf("expected two-space indentation:\n%s", got) + } + + var decoded map[string]any + if err := json.Unmarshal(buf.Bytes(), &decoded); err != nil { + t.Fatalf("unmarshal: %v", err) + } + + if _, ok := decoded["findings"]; !ok { + t.Errorf("missing findings field:\n%s", got) + } + + if _, ok := decoded["score"]; !ok { + t.Errorf("missing score field:\n%s", got) + } + + if _, ok := decoded["confidence"]; !ok { + t.Errorf("missing confidence field:\n%s", got) + } +} diff --git a/scan/scan.go b/scan/scan.go index e7674cf..55c0ca9 100644 --- a/scan/scan.go +++ b/scan/scan.go @@ -7,10 +7,11 @@ import ( // CommitResult holds findings for a single commit. type CommitResult struct { - Hash string `json:"hash"` - Findings []detection.Finding `json:"findings"` - PerDetectorScores map[string]float64 `json:"per_detector_scores"` - Score float64 `json:"score"` + Hash string `json:"hash"` + Findings []detection.Finding `json:"findings"` + PerDetectorScores map[string]float64 `json:"per_detector_scores"` + Score float64 `json:"score"` + Confidence detection.Confidence `json:"confidence"` } // Summary aggregates stats across all commits scanned. @@ -20,7 +21,6 @@ type Summary struct { ToolCounts map[string]int `json:"tool_counts"` ByConfidence map[string]int `json:"by_confidence"` PerDetectorScores map[string]float64 `json:"per_detector_scores"` - OverallScore float64 `json:"overall_score"` } // Report holds the full scan results. @@ -85,13 +85,25 @@ func scanOneCommit(c gitops.Commit, branchName string, detectors []detection.Det findings = append(findings, d.Detect(input)...) } - score, perDetectorScores := detection.ConsolidateScoreByFindings(findings) + if len(detectors) == 0 { + return CommitResult{ + Hash: c.Hash, + Findings: findings, + PerDetectorScores: nil, + Score: 0.0, + Confidence: detection.ConfidenceNone, + } + } + confidenceLevels := detectors[0].GetConfidenceLevels() + score, perDetectorScores := detection.ConsolidateScoreByFindings(findings) + confidence := detection.ScoreToConfidence(confidenceLevels, score) return CommitResult{ Hash: c.Hash, Findings: findings, PerDetectorScores: perDetectorScores, Score: score, + Confidence: confidence, } } @@ -114,9 +126,8 @@ func buildReport(results []CommitResult) Report { } } - overall, perDetectorScores := detection.ConsolidateScoreByFindings(allFindings) + _, perDetectorScores := detection.ConsolidateScoreByFindings(allFindings) summary.PerDetectorScores = perDetectorScores - summary.OverallScore = overall return Report{ Commits: results, diff --git a/scan/scan_test.go b/scan/scan_test.go index 86114b1..8fff12e 100644 --- a/scan/scan_test.go +++ b/scan/scan_test.go @@ -17,11 +17,12 @@ import ( ) func allDetectors() []detection.Detector { + confidenceLevels := detection.GetDefaultConfidenceLevels() return []detection.Detector{ - &committer.Detector{}, - &gitnotes.Detector{}, - &trailer.Detector{}, - &toolmention.Detector{}, + &committer.Detector{ConfidenceLevels: confidenceLevels}, + &gitnotes.Detector{ConfidenceLevels: confidenceLevels}, + &trailer.Detector{ConfidenceLevels: confidenceLevels}, + &toolmention.Detector{ConfidenceLevels: confidenceLevels}, } } @@ -160,9 +161,6 @@ func TestScanCommitRange(t *testing.T) { if trailerScore != 85 { t.Errorf("expected trailer score to be 85, found %f", trailerScore) } - if report.Summary.OverallScore != 200 { - t.Errorf("expected overall score to be 200, found %f", report.Summary.OverallScore) - } } func TestScanCommitRangeAll(t *testing.T) { @@ -236,10 +234,9 @@ func TestScanCommit(t *testing.T) { ) } - expectedConfidence, err := detection.ScoreToConfidence(f.Score) - if err != nil { - t.Fatalf("score conversion failed: %v", err) - } + expectedConfidence := detection.ScoreToConfidence( + detection.GetDefaultConfidenceLevels(), f.Score, + ) if f.Confidence != expectedConfidence { t.Errorf( @@ -286,7 +283,7 @@ func TestScanCommit(t *testing.T) { t.Errorf("expected trailer score to be 85, found %f", trailerScore) } if result.Score != 95 { - t.Errorf("expected overall score to be 95, found %f", result.Score) + t.Errorf("expected score to be 95, found %f", result.Score) } } @@ -439,10 +436,6 @@ func TestScanReportNoFindingsHasZeroScore(t *testing.T) { t.Fatalf("ScanCommitRange: %v", err) } - if report.Summary.OverallScore != 0 { - t.Fatalf("overall score = %v, want 0", report.Summary.OverallScore) - } - for _, cr := range report.Commits { if cr.Score != 0 { t.Fatalf("commit %s score = %v, want 0", cr.Hash, cr.Score) From 1a3b50984198f2432a7932d11c329d753b051f36 Mon Sep 17 00:00:00 2001 From: Omkar P <45419097+omkar-foss@users.noreply.github.com> Date: Mon, 10 Aug 2026 16:47:57 +0530 Subject: [PATCH 6/9] Update String to include ConfidenceNone Signed-off-by: Omkar P <45419097+omkar-foss@users.noreply.github.com> --- detection/detection.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/detection/detection.go b/detection/detection.go index 7c3c09c..e16d8cd 100644 --- a/detection/detection.go +++ b/detection/detection.go @@ -19,6 +19,8 @@ const ( func (c Confidence) String() string { switch c { + case ConfidenceNone: + return "none" case ConfidenceLow: return "low" case ConfidenceMedium: From e015fc776af57a71cb983a1486ea6c171ea0967d Mon Sep 17 00:00:00 2001 From: Omkar P <45419097+omkar-foss@users.noreply.github.com> Date: Tue, 11 Aug 2026 14:25:49 +0530 Subject: [PATCH 7/9] Add scoring for new branch detector Signed-off-by: Omkar P <45419097+omkar-foss@users.noreply.github.com> --- detection/branchname/branchname.go | 15 ++++++++++++-- detection/branchname/branchname_test.go | 19 +++++++++++------ detection/constants.go | 11 ++++++---- scan/scan_test.go | 27 ++++++++++++++++++++++--- 4 files changed, 57 insertions(+), 15 deletions(-) diff --git a/detection/branchname/branchname.go b/detection/branchname/branchname.go index 42f1b81..3a7d305 100644 --- a/detection/branchname/branchname.go +++ b/detection/branchname/branchname.go @@ -9,23 +9,34 @@ import ( "github.com/chaoss/disclosure/detection" ) -type Detector struct{} +type Detector struct { + ConfidenceLevels map[detection.Confidence]float64 +} func (d *Detector) Name() string { return "branchname" } +func (d *Detector) GetConfidenceLevels() map[detection.Confidence]float64 { return d.ConfidenceLevels } + func (d *Detector) Detect(input detection.Input) []detection.Finding { branch, err := input.GetBranchName() if err != nil { return nil } + score := detection.BranchNameBaseScore + confidence := detection.ScoreToConfidence(d.ConfidenceLevels, score) + if err != nil { + confidence = detection.ConfidenceNone + } + lower := strings.ToLower(branch) for prefix, tool := range detection.KnownAgentBranchPrefixes { if strings.HasPrefix(lower, prefix) { return []detection.Finding{{ Detector: d.Name(), Tool: tool, - Confidence: detection.ConfidenceMedium, + Score: score, + Confidence: confidence, Detail: fmt.Sprintf("branch name %q matches %s convention", branch, tool), }} } diff --git a/detection/branchname/branchname_test.go b/detection/branchname/branchname_test.go index dc92fd2..4e0821b 100644 --- a/detection/branchname/branchname_test.go +++ b/detection/branchname/branchname_test.go @@ -15,14 +15,21 @@ func TestDetectKnownPrefixes(t *testing.T) { t.Errorf("Detect(%q): got %d findings, want 1", branch, len(findings)) continue } - if findings[0].Tool != expectedTool { - t.Errorf("Detect(%q): tool = %q, want %q", branch, findings[0].Tool, expectedTool) + finding := findings[0] + if finding.Tool != expectedTool { + t.Errorf("Detect(%q): tool = %q, want %q", branch, finding.Tool, expectedTool) } - if findings[0].Confidence != detection.ConfidenceMedium { - t.Errorf("Detect(%q): confidence = %d, want %d", branch, findings[0].Confidence, detection.ConfidenceMedium) + if finding.Score != detection.BranchNameBaseScore { + t.Errorf("Detect(%q): score = %f, want %f", branch, finding.Score, detection.BranchNameBaseScore) } - if findings[0].Detector != "branchname" { - t.Errorf("Detect(%q): detector = %q, want %q", branch, findings[0].Detector, "branchname") + if finding.Confidence != detection.ConfidenceHigh { + t.Errorf( + "Detect(%q): confidence = %s, want %s", + branch, finding.Confidence.String(), detection.ConfidenceMedium.String(), + ) + } + if finding.Detector != "branchname" { + t.Errorf("Detect(%q): detector = %q, want %q", branch, finding.Detector, "branchname") } } } diff --git a/detection/constants.go b/detection/constants.go index fee3e0d..9c47e61 100644 --- a/detection/constants.go +++ b/detection/constants.go @@ -203,10 +203,13 @@ const ( ToolMentionBaseScore float64 = 20.0 // Committer detector - CommitterMatchBaseScore float64 = 75 - CommitterKnownEmailBonusPoints float64 = 20 - CommitterEmailSuffixBonusPoints float64 = 10 + CommitterMatchBaseScore float64 = 75.0 + CommitterKnownEmailBonusPoints float64 = 20.0 + CommitterEmailSuffixBonusPoints float64 = 10.0 // Gitnotes detector - GitNotesMatchBaseScore float64 = 75 + GitNotesMatchBaseScore float64 = 75.0 + + // Branch name detector + BranchNameBaseScore float64 = 75.0 ) diff --git a/scan/scan_test.go b/scan/scan_test.go index 8fff12e..e26794a 100644 --- a/scan/scan_test.go +++ b/scan/scan_test.go @@ -8,11 +8,13 @@ import ( "time" "github.com/chaoss/disclosure/detection" + "github.com/chaoss/disclosure/detection/branchname" "github.com/chaoss/disclosure/detection/committer" "github.com/chaoss/disclosure/detection/gitnotes" "github.com/chaoss/disclosure/detection/toolmention" "github.com/chaoss/disclosure/detection/trailer" "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/object" ) @@ -23,11 +25,13 @@ func allDetectors() []detection.Detector { &gitnotes.Detector{ConfidenceLevels: confidenceLevels}, &trailer.Detector{ConfidenceLevels: confidenceLevels}, &toolmention.Detector{ConfidenceLevels: confidenceLevels}, + &branchname.Detector{ConfidenceLevels: confidenceLevels}, } } func initTestRepo(t *testing.T) (string, []string) { t.Helper() + const humanEmail = "human@example.com" dir := t.TempDir() @@ -98,6 +102,14 @@ Assisted-by: Gemini (documentation) hashes = append(hashes, hash.String()) } + const branchName = "codex/fix-test" + if err := wt.Checkout(&git.CheckoutOptions{ + Branch: plumbing.NewBranchReferenceName(branchName), + Create: true, + }); err != nil { + t.Fatalf("create branch %s: %v", branchName, err) + } + return dir, hashes } @@ -161,6 +173,10 @@ func TestScanCommitRange(t *testing.T) { if trailerScore != 85 { t.Errorf("expected trailer score to be 85, found %f", trailerScore) } + branchnameScore := perDetectorScores["branchname"] + if branchnameScore != 75 { + t.Errorf("expected branchname score to be 75, found %f", branchnameScore) + } } func TestScanCommitRangeAll(t *testing.T) { @@ -189,6 +205,7 @@ func TestScanCommitDetectsAcrossDetectors(t *testing.T) { (&committer.Detector{}).Name(): "GitHub Copilot (agent)", (&trailer.Detector{}).Name(): "Kimi K2.6", (&toolmention.Detector{}).Name(): "Kimi", + (&branchname.Detector{}).Name(): "OpenAI Codex", } if len(result.Findings) != len(wantToolsByDetector) { t.Fatalf("got %d findings, want %d", len(result.Findings), len(wantToolsByDetector)) @@ -280,10 +297,14 @@ func TestScanCommit(t *testing.T) { } trailerScore := perDetectorScores["trailer"] if trailerScore != 75 { - t.Errorf("expected trailer score to be 85, found %f", trailerScore) + t.Errorf("expected trailer score to be 75, found %f", trailerScore) + } + branchnameScore := perDetectorScores["branchname"] + if branchnameScore != 75 { + t.Errorf("expected branchname score to be 75, found %f", branchnameScore) } - if result.Score != 95 { - t.Errorf("expected score to be 95, found %f", result.Score) + if result.Score != 170 { + t.Errorf("expected overall score to be 170, found %f", result.Score) } } From 427c10ea24ccd007bdbeea68c2fb52476ecdfc5d Mon Sep 17 00:00:00 2001 From: Omkar P <45419097+omkar-foss@users.noreply.github.com> Date: Tue, 11 Aug 2026 16:03:44 +0530 Subject: [PATCH 8/9] Remove extraneous err check Signed-off-by: Omkar P <45419097+omkar-foss@users.noreply.github.com> --- detection/branchname/branchname.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/detection/branchname/branchname.go b/detection/branchname/branchname.go index 3a7d305..433d910 100644 --- a/detection/branchname/branchname.go +++ b/detection/branchname/branchname.go @@ -25,10 +25,6 @@ func (d *Detector) Detect(input detection.Input) []detection.Finding { score := detection.BranchNameBaseScore confidence := detection.ScoreToConfidence(d.ConfidenceLevels, score) - if err != nil { - confidence = detection.ConfidenceNone - } - lower := strings.ToLower(branch) for prefix, tool := range detection.KnownAgentBranchPrefixes { if strings.HasPrefix(lower, prefix) { From 193a535e556920fed6e8178af538106dbb1a9cbe Mon Sep 17 00:00:00 2001 From: Omkar P <45419097+omkar-foss@users.noreply.github.com> Date: Tue, 11 Aug 2026 16:50:42 +0530 Subject: [PATCH 9/9] Return confidence none when score is zero Signed-off-by: Omkar P <45419097+omkar-foss@users.noreply.github.com> --- detection/detection.go | 3 +++ detection/detection_test.go | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/detection/detection.go b/detection/detection.go index e16d8cd..e1cf090 100644 --- a/detection/detection.go +++ b/detection/detection.go @@ -71,6 +71,9 @@ func GetDefaultConfidenceLevels() map[Confidence]float64 { } func ScoreToConfidence(confidenceLevels map[Confidence]float64, score float64) Confidence { + if score == 0 { + return ConfidenceNone + } if score >= confidenceLevels[ConfidenceHigh] { return ConfidenceHigh } diff --git a/detection/detection_test.go b/detection/detection_test.go index 103fe00..7a20968 100644 --- a/detection/detection_test.go +++ b/detection/detection_test.go @@ -193,7 +193,7 @@ func TestScoreToConfidence(t *testing.T) { { name: "zero score", score: 0, - want: ConfidenceLow, + want: ConfidenceNone, }, { name: "normal low score",