diff --git a/cmd/cmd.go b/cmd/cmd.go index 5f4afa7..27ef0a7 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -4,9 +4,11 @@ import ( "fmt" "io" "log" + "math" "os" "path/filepath" "slices" + "strconv" "strings" "github.com/chaoss/disclosure/detection" @@ -30,16 +32,49 @@ const ( ExitError = 2 ) -func allDetectors() []detection.Detector { +func allDetectors(confidenceLevels map[detection.Confidence]float64) []detection.Detector { return []detection.Detector{ - &committer.Detector{}, - &gitnotes.Detector{}, - &trailer.Detector{}, - &toolmention.Detector{}, - &branchname.Detector{}, + &committer.Detector{ConfidenceLevels: confidenceLevels}, + &gitnotes.Detector{ConfidenceLevels: confidenceLevels}, + &trailer.Detector{ConfidenceLevels: confidenceLevels}, + &toolmention.Detector{ConfidenceLevels: confidenceLevels}, + &branchname.Detector{ConfidenceLevels: confidenceLevels}, } } +// parseKeyValueFloatList parses strings like "a=1,b=2.5" into a map[string]float64. +func parseKeyValueFloatList(s string) (map[string]float64, error) { + out := map[string]float64{} + if strings.TrimSpace(s) == "" { + return out, nil + } + parts := strings.SplitSeq(s, ",") + for p := range parts { + p = strings.TrimSpace(p) + if p == "" { + continue + } + kv := strings.SplitN(p, "=", 2) + if len(kv) != 2 { + return nil, fmt.Errorf("invalid entry: %q", p) + } + key := strings.TrimSpace(kv[0]) + if key == "" { + return nil, fmt.Errorf("empty key in entry: %q", p) + } + valStr := strings.TrimSpace(kv[1]) + if valStr == "" { + return nil, fmt.Errorf("empty value in entry: %q", p) + } + v, err := strconv.ParseFloat(valStr, 64) + if err != nil || math.IsNaN(v) || math.IsInf(v, 0) { + return nil, fmt.Errorf("invalid numeric value %q in entry %q", valStr, p) + } + out[key] = v + } + return out, nil +} + // Run is the main entry point for the CLI. Returns an exit code. func Run(args []string, stdout, stderr io.Writer) int { rootCmd := &cobra.Command{ @@ -83,6 +118,7 @@ func scanCommand(stdout, stderr io.Writer, exitCode *int) *cobra.Command { var rangeFlag string var formatFlag string var minConfFlag string + var confidenceLevelsFlag string cmd := &cobra.Command{ Use: "scan [repo-path]", @@ -126,14 +162,33 @@ Examples: repoPath = args[0] } - minConf, err := output.ConfidenceFromString(minConfFlag) + minConf, err := detection.ConfidenceFromString(minConfFlag) if err != nil { fmt.Fprintln(stderr, err) *exitCode = ExitError return err } - detectors := allDetectors() + // parse confidence-levels override if provided + confidenceLevels := detection.GetDefaultConfidenceLevels() + if strings.TrimSpace(confidenceLevelsFlag) != "" { + flagMap, err := parseKeyValueFloatList(confidenceLevelsFlag) + if err != nil { + fmt.Fprintln(stderr, err) + *exitCode = ExitError + return err + } + if confidenceLevels, err = detection.SetConfidenceLevelsFromStrings( + confidenceLevels, + flagMap, + ); err != nil { + fmt.Fprintln(stderr, err) + *exitCode = ExitError + return err + } + } + + detectors := allDetectors(confidenceLevels) report, err := scan.ScanCommitRange(repoPath, rangeFlag, detectors) if err != nil { fmt.Fprintf(stderr, "error: %v\n", err) @@ -141,7 +196,7 @@ Examples: return err } - report = filterReport(report, minConf) + report = filterReport(report, minConf, confidenceLevels) switch formatFlag { case "json": @@ -173,6 +228,7 @@ Examples: cmd.Flags().StringVar(&rangeFlag, "range", "", "commit range in BASE..HEAD format") cmd.Flags().StringVar(&formatFlag, "format", "text", "output format: json or text") cmd.Flags().StringVar(&minConfFlag, "min-confidence", "low", "minimum confidence level: low, medium, high (or 1, 2, 3)") + cmd.Flags().StringVar(&confidenceLevelsFlag, "confidence-levels", "", "override confidence->score mapping, e.g. 'low=20,medium=60,high=100'") return cmd } @@ -223,7 +279,7 @@ Examples: return err } - detectors := allDetectors() + detectors := allDetectors(detection.GetDefaultConfidenceLevels()) findings := scan.ScanText(string(textBytes), detectors) switch formatFlag { @@ -277,7 +333,11 @@ Examples: } } -func filterReport(report scan.Report, minConf detection.Confidence) scan.Report { +func filterReport( + report scan.Report, + minConf detection.Confidence, + confidenceLevels map[detection.Confidence]float64, +) scan.Report { if minConf <= detection.ConfidenceLow { return report } @@ -291,22 +351,31 @@ func filterReport(report scan.Report, minConf detection.Confidence) scan.Report }, } - for _, cr := range report.Commits { - var kept []detection.Finding - for _, f := range cr.Findings { + for _, commit := range report.Commits { + var commitFindings []detection.Finding + for _, f := range commit.Findings { if f.Confidence >= minConf { - kept = append(kept, f) + commitFindings = append(commitFindings, f) } } - result := scan.CommitResult{Hash: cr.Hash, Findings: kept} - filtered.Commits = append(filtered.Commits, result) - if len(kept) > 0 { + // Recompute per-commit score from the kept findings + commitScore, perDetectorScores := detection.ConsolidateScoreByFindings(commitFindings) + confidence := detection.ScoreToConfidence(confidenceLevels, commitScore) + result := scan.CommitResult{ + Hash: commit.Hash, + Findings: commitFindings, + Score: commitScore, + Confidence: confidence, + PerDetectorScores: perDetectorScores, + } + filtered.Commits = append(filtered.Commits, result) + if len(commitFindings) > 0 { filtered.Summary.AICommits++ } - for _, f := range kept { - filtered.Summary.ToolCounts[f.Tool]++ - filtered.Summary.ByConfidence[f.Confidence.String()]++ + for _, commitFinding := range commitFindings { + filtered.Summary.ToolCounts[commitFinding.Tool]++ + filtered.Summary.ByConfidence[commitFinding.Confidence.String()]++ } } @@ -352,7 +421,7 @@ func generateDocs(exitCode *int) *cobra.Command { docDir = filepath.Clean(filepath.Join(outputDir, formatFlag)) err = os.MkdirAll(docDir, 0o750) } else { - err = fmt.Errorf("unknown format: %s\n", formatFlag) + err = fmt.Errorf("unknown format: %s", formatFlag) } if err != nil { return prepareError(err) diff --git a/cmd/cmd_test.go b/cmd/cmd_test.go index 659d411..80aba31 100644 --- a/cmd/cmd_test.go +++ b/cmd/cmd_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "os" "path/filepath" + "reflect" "strings" "testing" "time" @@ -195,6 +196,42 @@ func TestRunScanNoAI(t *testing.T) { } } +func TestRunTextCommandDetectsAI(t *testing.T) { + tmp := t.TempDir() + file := filepath.Join(tmp, "input.txt") + + os.WriteFile(file, []byte( + "I used Claude to write this", + ), 0644) + + var stdout, stderr bytes.Buffer + + code := Run([]string{ + "text", + "--input=" + file, + }, &stdout, &stderr) + + if code != ExitAI { + t.Errorf("code=%d want AI", code) + } +} + +func TestRunTextCommandNoAI(t *testing.T) { + tmp := t.TempDir() + file := filepath.Join(tmp, "input.txt") + os.WriteFile(file, []byte("plain text"), 0644) + + var stdout, stderr bytes.Buffer + code := Run([]string{ + "text", + "--input=" + file, + }, &stdout, &stderr) + + if code != ExitNoAI { + t.Errorf("code=%d want no AI", code) + } +} + func TestRunScanInvalidRepo(t *testing.T) { var stdout, stderr bytes.Buffer code := Run([]string{"scan", t.TempDir()}, &stdout, &stderr) @@ -203,34 +240,229 @@ func TestRunScanInvalidRepo(t *testing.T) { } } +func TestRunScanInvalidFormat(t *testing.T) { + dir := initTestRepo(t) + + var stdout, stderr bytes.Buffer + code := Run([]string{"scan", "--format=xml", dir}, &stdout, &stderr) + + if code != ExitError { + t.Errorf("exit code = %d, want %d", code, ExitError) + } + + if !strings.Contains(stderr.String(), "unknown format: xml") { + t.Errorf("expected invalid format error, got: %s", stderr.String()) + } +} + func TestFilterReport(t *testing.T) { - report := scan.Report{ - Commits: []scan.CommitResult{ - { - Hash: "abc123", - Findings: []detection.Finding{ - {Detector: "toolmention", Tool: "Claude", Confidence: 1, Detail: "text"}, - {Detector: "trailer", Tool: "Claude Code", Confidence: 3, Detail: "trailer"}, + tests := []struct { + name string + report scan.Report + minConf detection.Confidence + wantAICommits int + wantFindings []int + wantPerDetectorScores []map[string]float64 + wantScores []float64 + wantConfidence detection.Confidence + wantTool string // optional + }{ + { + name: "keep low confidence and above findings", + report: scan.Report{ + Commits: []scan.CommitResult{ + { + Hash: "abc123", + Findings: []detection.Finding{ + { + Detector: "toolmention", + Tool: "Claude", + Confidence: detection.ConfidenceMedium, + Score: 20, + }, + { + Detector: "trailer", + Tool: "Kimi K3", + Confidence: detection.ConfidenceHigh, + Score: 85, + }, + { + Detector: "trailer", + Tool: "Claude Code", + Confidence: detection.ConfidenceHigh, + Score: 100, + }, + }, + }, + { + Hash: "abc456", + Findings: []detection.Finding{ + { + Detector: "toolmention", + Tool: "Claude", + Confidence: detection.ConfidenceMedium, + Score: 20, + }, + { + Detector: "trailer", + Tool: "Claude Code", + Confidence: detection.ConfidenceHigh, + Score: 105, + }, + }, + }, }, }, + minConf: detection.ConfidenceMedium, + wantAICommits: 2, + wantFindings: []int{3, 2}, + wantTool: "Claude", + wantPerDetectorScores: []map[string]float64{ + {"toolmention": 20, "trailer": 100}, + {"toolmention": 20, "trailer": 105}, + }, + wantScores: []float64{120, 125}, }, - Summary: scan.Summary{ - TotalCommits: 1, - AICommits: 1, - ToolCounts: map[string]int{"Claude": 1, "Claude Code": 1}, - ByConfidence: map[string]int{"low": 1, "high": 1}, + { + name: "all findings filtered out", + report: scan.Report{ + Commits: []scan.CommitResult{ + { + Hash: "abc123", + Findings: []detection.Finding{ + { + Detector: "toolmention", + Confidence: detection.ConfidenceLow, + Score: 20, + }, + }, + }, + }, + }, + minConf: detection.ConfidenceHigh, + wantAICommits: 0, + wantFindings: []int{0}, + wantScores: []float64{0}, + }, + { + name: "empty findings", + report: scan.Report{ + Commits: []scan.CommitResult{ + { + Hash: "abc123", + Findings: nil, + }, + }, + }, + minConf: detection.ConfidenceMedium, + wantAICommits: 0, + wantFindings: []int{0}, + wantScores: []float64{0}, + }, + { + name: "no commits", + report: scan.Report{}, + minConf: detection.ConfidenceHigh, + wantAICommits: 0, + wantFindings: []int{}, + wantScores: []float64{}, + }, + { + name: "low confidence threshold returns original report", + report: scan.Report{ + Commits: []scan.CommitResult{ + { + Hash: "abc123", + Findings: []detection.Finding{ + { + Detector: "toolmention", + Confidence: detection.ConfidenceLow, + Score: 20, + }, + { + Detector: "trailer", + Confidence: detection.ConfidenceHigh, + Score: 80, + }, + }, + Score: 100, + Confidence: detection.ConfidenceHigh, + }, + }, + }, + minConf: detection.ConfidenceLow, + wantFindings: []int{2}, + wantScores: []float64{100}, }, } - filtered := filterReport(report, 3) // high only - if len(filtered.Commits[0].Findings) != 1 { - t.Fatalf("expected 1 finding after filter, got %d", len(filtered.Commits[0].Findings)) - } - if filtered.Commits[0].Findings[0].Tool != "Claude Code" { - t.Errorf("expected Claude Code, got %s", filtered.Commits[0].Findings[0].Tool) - } - if filtered.Summary.AICommits != 1 { - t.Errorf("ai_commits = %d, want 1", filtered.Summary.AICommits) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + filtered := filterReport(tt.report, tt.minConf, detection.GetDefaultConfidenceLevels()) + + if tt.wantFindings != nil { + lenCommit := len(filtered.Commits) + lenWant := len(tt.wantFindings) + if lenCommit != lenWant { + t.Fatalf("invalid number of items to check, commit len=%d want len=%d", lenCommit, lenWant) + } + for i := range filtered.Commits { + if len(filtered.Commits[i].Findings) != tt.wantFindings[i] { + t.Fatalf("commit findings len=%d want findings=%d", lenCommit, lenWant) + } + } + } + + if tt.wantScores != nil { + lenCommit := len(filtered.Commits) + lenWant := len(tt.wantScores) + if lenCommit != lenWant { + t.Fatalf("invalid number of items to check, commit len=%d want len=%d", lenCommit, lenWant) + } + for i := range filtered.Commits { + if filtered.Commits[i].Score != tt.wantScores[i] { + t.Fatalf("commit score=%f want score=%f", filtered.Commits[i].Score, tt.wantScores[i]) + } + } + } + + if filtered.Summary.AICommits != tt.wantAICommits { + t.Errorf("AICommits=%d want=%d", filtered.Summary.AICommits, tt.wantAICommits) + } + + gotFindings := 0 + for i, commit := range filtered.Commits { + gotFindings += len(commit.Findings) + + // compare per-detector scores for each commit + if tt.wantPerDetectorScores == nil { + continue + } + wantPerDetectorScores := tt.wantPerDetectorScores[i] + lenCommitScores := len(commit.PerDetectorScores) + lenWantScores := len(wantPerDetectorScores) + if lenCommitScores != lenWantScores { + t.Errorf("expected %d detectors in map but found=%d", lenWantScores, lenCommitScores) + } + for detector := range commit.PerDetectorScores { + if wantPerDetectorScores[detector] != commit.PerDetectorScores[detector] { + t.Errorf( + "expected %f, found %f for detector %q", + wantPerDetectorScores[detector], + commit.PerDetectorScores[detector], + detector, + ) + } + } + } + + if tt.wantTool != "" { + got := filtered.Commits[0].Findings[0].Tool + if got != tt.wantTool { + t.Errorf("tool=%q want=%q", got, tt.wantTool) + } + } + }) } } @@ -347,3 +579,237 @@ func TestRunDocsWriteError(t *testing.T) { t.Errorf("exit code = %d, want %d", code, ExitError) } } + +func TestRunDocsEmptyFormatFlag(t *testing.T) { + var stdout, stderr bytes.Buffer + code := Run([]string{"docs", "--format="}, &stdout, &stderr) + if code != ExitError { + t.Errorf("exit code=%d want error", code) + } +} + +func TestParseKeyValueFloatList(t *testing.T) { + tests := []struct { + name string + input string + want map[string]float64 + wantErr bool + }{ + { + name: "Valid input single pair", + input: "low=20", + want: map[string]float64{"low": 20}, + wantErr: false, + }, + { + name: "Valid input multiple pairs with floats", + input: "trailer=0.8,toolmention=0.2", + want: map[string]float64{"trailer": 0.8, "toolmention": 0.2}, + wantErr: false, + }, + { + name: "Valid input with spacing padding", + input: " low = 20 , medium = 60.5 ", + want: map[string]float64{"low": 20, "medium": 60.5}, + wantErr: false, + }, + { + name: "Empty string yields empty map", + input: " ", + want: map[string]float64{}, + wantErr: false, + }, + { + name: "Error on malformed key value syntax", + input: "low:20", + want: nil, + wantErr: true, + }, + { + name: "Error on empty keys", + input: "=20,medium=60", + want: nil, + wantErr: true, + }, + { + name: "Error on empty values", + input: "low=,medium=60", + want: nil, + wantErr: true, + }, + { + name: "Error on invalid float strings", + input: "low=twenty", + want: nil, + wantErr: true, + }, + { + name: "Error on invalid numerical states (NaN)", + input: "low=NaN", + want: nil, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := parseKeyValueFloatList(tt.input) + if (err != nil) != tt.wantErr { + t.Errorf("parseKeyValueFloatList() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !tt.wantErr && !reflect.DeepEqual(got, tt.want) { + t.Errorf("parseKeyValueFloatList() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestRunScanFlags(t *testing.T) { + dir := initTestRepo(t) + + tests := []struct { + name string + args []string + wantCode int + wantErrText string + }{ + { + name: "invalid min confidence", + args: []string{"scan", "--min-confidence=invalid", dir}, + wantCode: ExitError, + wantErrText: "invalid confidence", + }, + { + name: "invalid confidence scores format", + args: []string{"scan", "--confidence-levels=low", dir}, + wantCode: ExitError, + }, + { + name: "reject NaN confidence score", + args: []string{"scan", "--confidence-levels=high=NaN", dir}, + wantCode: ExitError, + }, + { + name: "both valid flags", + args: []string{"scan", "--confidence-levels=low=15,medium=55,high=95", dir}, + wantCode: ExitAI, // or ExitNoAI + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var stdout, stderr bytes.Buffer + + code := Run(tt.args, &stdout, &stderr) + + if tt.name == "both valid flags" { + if code != ExitAI && code != ExitNoAI { + t.Fatalf("unexpected exit code %d (stderr=%s)", code, stderr.String()) + } + return + } + + if code != tt.wantCode { + t.Fatalf("exit code=%d want=%d", code, tt.wantCode) + } + + if tt.wantErrText != "" && + !strings.Contains(stderr.String(), tt.wantErrText) { + t.Fatalf("stderr=%q does not contain %q", stderr.String(), tt.wantErrText) + } + }) + } +} + +func TestRunScanScoreFlags(t *testing.T) { + dir := initTestRepo(t) + + tests := []struct { + name string + args []string + }{ + { + name: "confidence scores", + args: []string{ + "scan", + "--format=json", + "--confidence-levels=low=10,medium=50,high=90", + dir, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var stdout, stderr bytes.Buffer + + code := Run(tt.args, &stdout, &stderr) + if code != ExitAI && code != ExitNoAI { + t.Fatalf("unexpected exit code %d: %s", code, stderr.String()) + } + + var report scan.Report + if err := json.Unmarshal(stdout.Bytes(), &report); err != nil { + t.Fatalf("unmarshal: %v", err) + } + + var findings []detection.Finding + for _, c := range report.Commits { + findings = append(findings, c.Findings...) + } + }) + } +} + +func TestScanCommandInvalidFlags(t *testing.T) { + tests := []struct { + name string + args []string + }{ + { + name: "confidence missing equals", + args: []string{"--confidence-levels", "low"}, + }, + { + name: "confidence empty key", + args: []string{"--confidence-levels", "=10"}, + }, + { + name: "confidence empty value", + args: []string{"--confidence-levels", "low="}, + }, + { + name: "confidence invalid number", + args: []string{"--confidence-levels", "low=abc"}, + }, + { + name: "confidence NaN", + args: []string{"--confidence-levels", "high=NaN"}, + }, + { + name: "confidence Inf", + args: []string{"--confidence-levels", "high=Inf"}, + }, + { + name: "confidence -Inf", + args: []string{"--confidence-levels", "high=-Inf"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var stdout, stderr bytes.Buffer + exitCode := ExitNoAI + + cmd := scanCommand(&stdout, &stderr, &exitCode) + cmd.SetArgs(tt.args) + + _ = cmd.Execute() + + if exitCode != ExitError { + t.Fatalf("expected ExitError, got %d (stderr=%q)", exitCode, stderr.String()) + } + }) + } +} diff --git a/detection/branchname/branchname.go b/detection/branchname/branchname.go index 42f1b81..433d910 100644 --- a/detection/branchname/branchname.go +++ b/detection/branchname/branchname.go @@ -9,23 +9,30 @@ import ( "github.com/chaoss/disclosure/detection" ) -type Detector struct{} +type Detector struct { + ConfidenceLevels map[detection.Confidence]float64 +} func (d *Detector) Name() string { return "branchname" } +func (d *Detector) GetConfidenceLevels() map[detection.Confidence]float64 { return d.ConfidenceLevels } + func (d *Detector) Detect(input detection.Input) []detection.Finding { branch, err := input.GetBranchName() if err != nil { return nil } + score := detection.BranchNameBaseScore + confidence := detection.ScoreToConfidence(d.ConfidenceLevels, score) lower := strings.ToLower(branch) for prefix, tool := range detection.KnownAgentBranchPrefixes { if strings.HasPrefix(lower, prefix) { return []detection.Finding{{ Detector: d.Name(), Tool: tool, - Confidence: detection.ConfidenceMedium, + Score: score, + Confidence: confidence, Detail: fmt.Sprintf("branch name %q matches %s convention", branch, tool), }} } diff --git a/detection/branchname/branchname_test.go b/detection/branchname/branchname_test.go index dc92fd2..4e0821b 100644 --- a/detection/branchname/branchname_test.go +++ b/detection/branchname/branchname_test.go @@ -15,14 +15,21 @@ func TestDetectKnownPrefixes(t *testing.T) { t.Errorf("Detect(%q): got %d findings, want 1", branch, len(findings)) continue } - if findings[0].Tool != expectedTool { - t.Errorf("Detect(%q): tool = %q, want %q", branch, findings[0].Tool, expectedTool) + finding := findings[0] + if finding.Tool != expectedTool { + t.Errorf("Detect(%q): tool = %q, want %q", branch, finding.Tool, expectedTool) } - if findings[0].Confidence != detection.ConfidenceMedium { - t.Errorf("Detect(%q): confidence = %d, want %d", branch, findings[0].Confidence, detection.ConfidenceMedium) + if finding.Score != detection.BranchNameBaseScore { + t.Errorf("Detect(%q): score = %f, want %f", branch, finding.Score, detection.BranchNameBaseScore) } - if findings[0].Detector != "branchname" { - t.Errorf("Detect(%q): detector = %q, want %q", branch, findings[0].Detector, "branchname") + if finding.Confidence != detection.ConfidenceHigh { + t.Errorf( + "Detect(%q): confidence = %s, want %s", + branch, finding.Confidence.String(), detection.ConfidenceMedium.String(), + ) + } + if finding.Detector != "branchname" { + t.Errorf("Detect(%q): detector = %q, want %q", branch, finding.Detector, "branchname") } } } diff --git a/detection/committer/committer.go b/detection/committer/committer.go index 826f023..3f4c5ea 100644 --- a/detection/committer/committer.go +++ b/detection/committer/committer.go @@ -20,17 +20,24 @@ func init() { } } -type Detector struct{} +type Detector struct { + ConfidenceLevels map[detection.Confidence]float64 +} func (d *Detector) Name() string { return "committer" } +func (d *Detector) GetConfidenceLevels() map[detection.Confidence]float64 { return d.ConfidenceLevels } + func (d *Detector) detectEmail(email, identityField string) []detection.Finding { // Direct match against known emails if name, ok := detection.KnownAgentCommitters[email]; ok { + score := detection.CommitterMatchBaseScore + detection.CommitterKnownEmailBonusPoints + confidence := detection.ScoreToConfidence(d.ConfidenceLevels, score) return []detection.Finding{{ Detector: d.Name(), Tool: name, - Confidence: detection.ConfidenceHigh, + Score: score, + Confidence: confidence, Detail: fmt.Sprintf("%s email %s matches known AI bot", identityField, email), }} } @@ -38,13 +45,16 @@ func (d *Detector) detectEmail(email, identityField string) []detection.Finding // Numeric prefix match for GitHub noreply emails (#4). // Format: +@users.noreply.github.com if strings.HasSuffix(email, detection.GithubNoReplyEmailSuffix) { + score := detection.CommitterMatchBaseScore + detection.CommitterEmailSuffixBonusPoints + confidence := detection.ScoreToConfidence(d.ConfidenceLevels, score) if idx := strings.Index(email, "+"); idx > 0 { prefix := email[:idx] if name, ok := numericPrefixIndex[prefix]; ok { return []detection.Finding{{ Detector: d.Name(), Tool: name, - Confidence: detection.ConfidenceHigh, + Score: score, + Confidence: confidence, Detail: fmt.Sprintf("%s email %s matches known AI bot", identityField, email), }} } diff --git a/detection/committer/committer_test.go b/detection/committer/committer_test.go index d05ff3d..3ba4e92 100644 --- a/detection/committer/committer_test.go +++ b/detection/committer/committer_test.go @@ -6,8 +6,36 @@ import ( "github.com/chaoss/disclosure/detection" ) +const ( + directMatchScore = detection.CommitterMatchBaseScore + + detection.CommitterKnownEmailBonusPoints + + numericPrefixScore = detection.CommitterMatchBaseScore + + detection.CommitterEmailSuffixBonusPoints +) + +func assertFindingMetadata(t *testing.T, finding detection.Finding, expectedScore float64) { + t.Helper() + + if finding.Score != expectedScore { + t.Errorf("score = %v, want %v", finding.Score, expectedScore) + } + + expectedConfidence := detection.ScoreToConfidence( + detection.GetDefaultConfidenceLevels(), expectedScore, + ) + + if finding.Confidence != expectedConfidence { + t.Errorf("confidence = %d, want %d", finding.Confidence, expectedConfidence) + } + + if finding.Detector != "committer" { + t.Errorf("detector = %q, want %q", finding.Detector, "committer") + } +} + func TestDetectAllKnownEmails(t *testing.T) { - d := &Detector{} + d := &Detector{ConfidenceLevels: detection.GetDefaultConfidenceLevels()} for email, expectedName := range detection.KnownAgentCommitters { input := detection.Input{CommitEmail: email} findings := d.Detect(input) @@ -18,17 +46,12 @@ func TestDetectAllKnownEmails(t *testing.T) { if findings[0].Tool != expectedName { t.Errorf("Detect(%q): tool = %q, want %q", email, findings[0].Tool, expectedName) } - if findings[0].Confidence != detection.ConfidenceHigh { - t.Errorf("Detect(%q): confidence = %d, want %d", email, findings[0].Confidence, detection.ConfidenceHigh) - } - if findings[0].Detector != "committer" { - t.Errorf("Detect(%q): detector = %q, want %q", email, findings[0].Detector, "committer") - } + assertFindingMetadata(t, findings[0], directMatchScore) } } func TestDetectMixedCase(t *testing.T) { - d := &Detector{} + d := &Detector{ConfidenceLevels: detection.GetDefaultConfidenceLevels()} cases := []struct { input string wantTool string @@ -47,11 +70,12 @@ func TestDetectMixedCase(t *testing.T) { if findings[0].Tool != tc.wantTool { t.Errorf("Detect(%q): tool = %q, want %q", tc.input, findings[0].Tool, tc.wantTool) } + assertFindingMetadata(t, findings[0], directMatchScore) } } func TestDetectWhitespace(t *testing.T) { - d := &Detector{} + d := &Detector{ConfidenceLevels: detection.GetDefaultConfidenceLevels()} cases := []string{ " 209825114+claude[bot]@users.noreply.github.com", "209825114+claude[bot]@users.noreply.github.com ", @@ -67,11 +91,12 @@ func TestDetectWhitespace(t *testing.T) { if findings[0].Tool != "Claude" { t.Errorf("Detect(%q): tool = %q, want %q", email, findings[0].Tool, "Claude") } + assertFindingMetadata(t, findings[0], directMatchScore) } } func TestDetectNotFound(t *testing.T) { - d := &Detector{} + d := &Detector{ConfidenceLevels: detection.GetDefaultConfidenceLevels()} cases := []string{ "user@example.com", "", @@ -89,7 +114,7 @@ func TestDetectNotFound(t *testing.T) { } func TestDetectNumericPrefix(t *testing.T) { - d := &Detector{} + d := &Detector{ConfidenceLevels: detection.GetDefaultConfidenceLevels()} // Simulate a renamed bot: same numeric ID, different username cases := []struct { input string @@ -112,11 +137,12 @@ func TestDetectNumericPrefix(t *testing.T) { if findings[0].Tool != tc.wantTool { t.Errorf("Detect(%q): tool = %q, want %q", tc.input, findings[0].Tool, tc.wantTool) } + assertFindingMetadata(t, findings[0], numericPrefixScore) } } func TestDetectNumericPrefixNoFalsePositive(t *testing.T) { - d := &Detector{} + d := &Detector{ConfidenceLevels: detection.GetDefaultConfidenceLevels()} // An email with a numeric prefix that doesn't match any known bot cases := []string{ "999999999+someone@users.noreply.github.com", @@ -138,7 +164,7 @@ func TestDetectAuthorAndCommitter(t *testing.T) { humanEmail = "human@example.com" ) - d := &Detector{} + d := &Detector{ConfidenceLevels: detection.GetDefaultConfidenceLevels()} tests := []struct { name string input detection.Input @@ -208,6 +234,7 @@ func TestDetectAuthorAndCommitter(t *testing.T) { if finding.Detail != tt.wantDetails[i] { t.Errorf("finding %d detail = %q, want %q", i, finding.Detail, tt.wantDetails[i]) } + assertFindingMetadata(t, finding, directMatchScore) } }) } diff --git a/detection/constants.go b/detection/constants.go index 75da9ed..9c47e61 100644 --- a/detection/constants.go +++ b/detection/constants.go @@ -186,3 +186,30 @@ var GitNotesAuthorshipPrefix = "authorship/" // TrailerEmailPattern Regex to match email address in commit trailers var TrailerEmailPattern = regexp.MustCompile(`\s*<[^>]+>`) + +// Numeric scoring constants for respective detectors +const ( + // Trailer detector + CoauthoredByTrailerBaseScore float64 = 40.0 + CoauthorKnownEmailBonusPoints float64 = 35.0 + CoauthorModelBonusPoints float64 = 10.0 + AssistedByTrailerBaseScore float64 = 75.0 + TrailerMatchBaseScore float64 = 35.0 + TrailerNotMatchedScore float64 = 0.0 + AdditionalTrailerBonusPoints float64 = 20.0 + SessionIDBonusPoints float64 = 45.0 + + // Tool mention detector + ToolMentionBaseScore float64 = 20.0 + + // Committer detector + CommitterMatchBaseScore float64 = 75.0 + CommitterKnownEmailBonusPoints float64 = 20.0 + CommitterEmailSuffixBonusPoints float64 = 10.0 + + // Gitnotes detector + GitNotesMatchBaseScore float64 = 75.0 + + // Branch name detector + BranchNameBaseScore float64 = 75.0 +) diff --git a/detection/detection.go b/detection/detection.go index 0c2e031..e1cf090 100644 --- a/detection/detection.go +++ b/detection/detection.go @@ -1,7 +1,9 @@ package detection import ( + "encoding/json" "fmt" + "math" "strings" ) @@ -9,6 +11,7 @@ import ( type Confidence int const ( + ConfidenceNone Confidence = 0 // Nil equivalent for confidence ConfidenceLow Confidence = 1 // Tool name mentioned in text ConfidenceMedium Confidence = 2 // Commit message pattern match ConfidenceHigh Confidence = 3 // Bot email, co-author trailer, git AI ref @@ -16,6 +19,8 @@ const ( func (c Confidence) String() string { switch c { + case ConfidenceNone: + return "none" case ConfidenceLow: return "low" case ConfidenceMedium: @@ -27,16 +32,116 @@ func (c Confidence) String() string { } } +func (c Confidence) MarshalJSON() ([]byte, error) { + return json.Marshal(c.String()) +} + +func (c *Confidence) UnmarshalJSON(data []byte) error { + var s string + if err := json.Unmarshal(data, &s); err != nil { + return err + } + + switch s { + case "none": + *c = ConfidenceNone + case "low": + *c = ConfidenceLow + case "medium": + *c = ConfidenceMedium + case "high": + *c = ConfidenceHigh + default: + return fmt.Errorf("invalid confidence %q", s) + } + + return nil +} + func (c *Confidence) Increment() { *c = min(*c+1, ConfidenceHigh) } +func GetDefaultConfidenceLevels() map[Confidence]float64 { + return map[Confidence]float64{ + ConfidenceLow: 30.0, + ConfidenceMedium: 70.0, + ConfidenceHigh: 100.0, + } +} + +func ScoreToConfidence(confidenceLevels map[Confidence]float64, score float64) Confidence { + if score == 0 { + return ConfidenceNone + } + if score >= confidenceLevels[ConfidenceHigh] { + return ConfidenceHigh + } + if score <= confidenceLevels[ConfidenceLow] { + return ConfidenceLow + } + levels := []Confidence{ConfidenceLow, ConfidenceMedium, ConfidenceHigh} + for _, level := range levels { + if math.Round(score) <= confidenceLevels[level] { + return level + } + } + return ConfidenceNone +} + +// SetConfidenceLevelsFromStrings allows to update confidenceLevels using a custom map +func SetConfidenceLevelsFromStrings( + confidenceLevels map[Confidence]float64, + userMapping map[string]float64, +) (map[Confidence]float64, error) { + for k, v := range userMapping { + k = strings.ToLower(strings.TrimSpace(k)) + switch k { + case "low": + confidenceLevels[ConfidenceLow] = v + case "medium": + confidenceLevels[ConfidenceMedium] = v + case "high": + confidenceLevels[ConfidenceHigh] = v + default: + return nil, fmt.Errorf("unsupported confidence key: %s", k) + } + } + // set defaults if unspecified in user mapping + for c, def := range GetDefaultConfidenceLevels() { + if _, ok := confidenceLevels[c]; !ok { + confidenceLevels[c] = def + } + } + + if confidenceLevels[ConfidenceLow] >= confidenceLevels[ConfidenceMedium] || + confidenceLevels[ConfidenceMedium] >= confidenceLevels[ConfidenceHigh] { + return nil, fmt.Errorf("low, medium, high must be in ascending order") + } + return confidenceLevels, nil +} + +// ConfidenceFromString parses a confidence string or numeric value. +func ConfidenceFromString(s string) (Confidence, error) { + switch strings.ToLower(strings.TrimSpace(s)) { + case "1", "low": + return ConfidenceLow, nil + case "2", "medium": + return ConfidenceMedium, nil + case "3", "high": + return ConfidenceHigh, nil + default: + return 0, fmt.Errorf("invalid confidence %q: use low/1, medium/2, or high/3", s) + } +} + // Finding represents a single detection of AI involvement. type Finding struct { Detector string `json:"detector"` Tool string `json:"tool"` Model string `json:"model,omitempty"` Confidence Confidence `json:"confidence"` + Score float64 `json:"score,omitempty"` Detail string `json:"detail"` } @@ -58,6 +163,7 @@ func (f Finding) DisplayTool() string { type Detector interface { Name() string Detect(input Input) []Finding + GetConfidenceLevels() map[Confidence]float64 } // Input provides data for detectors to examine. Each detector reads the fields @@ -110,3 +216,32 @@ func (input *Input) GetTextWithCommitMessage() (string, error) { func (input *Input) GetNotes() (GitnoteParseResult, error) { return parseGitnotes(input.Notes) } + +// ConsolidateScoreByFindings computes per-detector scores and a total score from findings +func ConsolidateScoreByFindings(findings []Finding) (float64, map[string]float64) { + // For each detector, we take max of all findings for that detector. + perDetectorScores := map[string]float64{} + for _, f := range findings { + detectorName := strings.TrimSpace(f.Detector) + if cur, ok := perDetectorScores[detectorName]; !ok || f.Score > cur { + perDetectorScores[detectorName] = f.Score + } + } + + // No detectors found + if len(perDetectorScores) == 0 { + return 0.0, perDetectorScores + } + + // Compute total score for findings + return CalculateTotalScore(perDetectorScores), perDetectorScores +} + +// CalculateTotalScore computes total score from a per-detector scores map +func CalculateTotalScore(perDetectorScores map[string]float64) float64 { + var totalScore float64 + for _, score := range perDetectorScores { + totalScore += score + } + return totalScore +} diff --git a/detection/detection_test.go b/detection/detection_test.go index 97d5d4c..7a20968 100644 --- a/detection/detection_test.go +++ b/detection/detection_test.go @@ -1,6 +1,10 @@ package detection -import "testing" +import ( + "math" + "reflect" + "testing" +) func TestFindingDisplayTool(t *testing.T) { tests := []struct { @@ -48,3 +52,408 @@ func TestFindingDisplayTool(t *testing.T) { }) } } + +func TestConsolidateFindings(t *testing.T) { + tests := []struct { + name string + findings []Finding + wantTotalScore float64 + wantDetectorScores map[string]float64 + wantNaN bool + }{ + { + name: "sum of individual detector scores", + findings: []Finding{ + {Detector: "A", Score: 100}, + {Detector: "B", Score: 50}, + {Detector: "C", Score: 75}, + }, + wantTotalScore: 225, + wantDetectorScores: map[string]float64{ + "A": 100, + "B": 50, + "C": 75, + }, + }, + { + name: "mix of positive and negative scores", + findings: []Finding{ + {Detector: "A", Score: 100}, + {Detector: "B", Score: -50}, + {Detector: "C", Score: 75}, + }, + wantTotalScore: 125, + wantDetectorScores: map[string]float64{ + "A": 100, + "B": -50, + "C": 75, + }, + }, + { + name: "max used to aggregated detector level scores", + findings: []Finding{ + {Detector: "A", Score: 100}, + {Detector: "B", Score: 50}, + {Detector: "A", Score: 120}, + {Detector: "C", Score: 75}, + {Detector: "B", Score: -10}, + }, + wantTotalScore: 245, + wantDetectorScores: map[string]float64{ + "A": 120, + "B": 50, + "C": 75, + }, + }, + { + name: "nil findings", + findings: nil, + wantTotalScore: 0, + wantDetectorScores: map[string]float64{}, + }, + { + name: "empty findings", + findings: []Finding{}, + wantTotalScore: 0, + wantDetectorScores: map[string]float64{}, + }, + { + name: "single detector across all findings", + findings: []Finding{ + {Detector: "A", Score: 20}, + {Detector: "A", Score: 80}, + {Detector: "A", Score: 50}, + }, + wantTotalScore: 80, + wantDetectorScores: map[string]float64{ + "A": 80, + }, + }, + { + name: "detector names trimmed", + findings: []Finding{ + {Detector: " A ", Score: 60}, + {Detector: "A", Score: 90}, + }, + wantTotalScore: 90, + wantDetectorScores: map[string]float64{ + "A": 90, + }, + }, + { + name: "blank detector names collapse", + findings: []Finding{ + {Detector: "", Score: 10}, + {Detector: " ", Score: 55}, + }, + wantTotalScore: 55, + wantDetectorScores: map[string]float64{ + "": 55, + }, + }, + { + name: "NaN score", + findings: []Finding{ + {Detector: "A", Score: math.NaN()}, + {Detector: "B", Score: 50}, + }, + wantTotalScore: math.NaN(), + wantNaN: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + totalScore, perDetectorScores := ConsolidateScoreByFindings(tt.findings) + + if tt.wantNaN { + if !math.IsNaN(totalScore) { + t.Fatalf("total score = %v, want NaN", totalScore) + } + return + } + + if totalScore != tt.wantTotalScore { + t.Fatalf("total score = %v, want %v", totalScore, tt.wantTotalScore) + } + + if !reflect.DeepEqual(perDetectorScores, tt.wantDetectorScores) { + t.Fatalf("per = %#v, want %#v", perDetectorScores, tt.wantDetectorScores) + } + }) + } +} + +func TestScoreToConfidence(t *testing.T) { + confidenceLevels := GetDefaultConfidenceLevels() + tests := []struct { + name string + score float64 + want Confidence + }{ + { + name: "zero score", + score: 0, + want: ConfidenceNone, + }, + { + name: "normal low score", + score: 25, + want: ConfidenceLow, + }, + { + name: "medium boundary", + score: 50, + want: ConfidenceMedium, + }, + { + name: "high boundary", + score: 75, + want: ConfidenceHigh, + }, + { + name: "maximum score", + score: 100, + want: ConfidenceHigh, + }, + { + name: "negative score", + score: -1, + want: ConfidenceLow, + }, + { + name: "above maximum score", + score: 101, + want: ConfidenceHigh, + }, + { + name: "positive infinity", + score: math.Inf(1), + want: ConfidenceHigh, + }, + { + name: "negative infinity", + score: math.Inf(-1), + want: ConfidenceLow, + }, + { + name: "NaN score", + score: math.NaN(), + want: ConfidenceNone, + }, + { + name: "rounding to low boundary", + score: confidenceLevels[ConfidenceLow] - 0.4, + want: ConfidenceLow, + }, + { + name: "rounding past low boundary", + score: confidenceLevels[ConfidenceLow] + 0.5, + want: ConfidenceMedium, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := ScoreToConfidence(confidenceLevels, tt.score) + if got != tt.want { + t.Fatalf("confidence=%v, want %v", got, tt.want) + } + }) + } +} + +func TestSetConfidenceLevelsFromStrings(t *testing.T) { + tests := []struct { + name string + input map[string]float64 + wantErr bool + check func(t *testing.T, confidenceLevels map[Confidence]float64) + }{ + { + name: "full custom mapping", + input: map[string]float64{ + "low": 30, + "medium": 60, + "high": 90, + }, + wantErr: false, + check: func(t *testing.T, confidenceLevels map[Confidence]float64) { + if confidenceLevels[ConfidenceLow] != 30 { + t.Fatalf("low score mismatch") + } + if confidenceLevels[ConfidenceMedium] != 60 { + t.Fatalf("medium score mismatch") + } + if confidenceLevels[ConfidenceHigh] != 90 { + t.Fatalf("high score mismatch") + } + }, + }, + { + name: "case insensitive keys", + input: map[string]float64{ + "LOW": 20, + "Medium": 50, + "HIGH": 80, + }, + wantErr: false, + check: func(t *testing.T, confidenceLevels map[Confidence]float64) { + if confidenceLevels[ConfidenceLow] != 20 { + t.Fatalf("low score mismatch") + } + }, + }, + { + name: "keys with whitespace", + input: map[string]float64{ + " low ": 25, + }, + wantErr: false, + check: func(t *testing.T, confidenceLevels map[Confidence]float64) { + if confidenceLevels[ConfidenceLow] != 25 { + t.Fatalf("low score mismatch") + } + }, + }, + { + name: "empty mapping uses defaults", + input: map[string]float64{}, + wantErr: false, + check: func(t *testing.T, confidenceLevels map[Confidence]float64) { + defaultConfidenceLevels := GetDefaultConfidenceLevels() + if confidenceLevels[ConfidenceLow] != defaultConfidenceLevels[ConfidenceLow] { + t.Fatalf("default low mismatch") + } + }, + }, + { + name: "partial mapping fills defaults", + input: map[string]float64{ + "low": 10, + }, + wantErr: false, + check: func(t *testing.T, confidenceLevels map[Confidence]float64) { + defaultConfidenceLevels := GetDefaultConfidenceLevels() + if confidenceLevels[ConfidenceLow] != 10 { + t.Fatalf("custom low missing") + } + if confidenceLevels[ConfidenceHigh] != defaultConfidenceLevels[ConfidenceHigh] { + t.Fatalf("high should use default") + } + }, + }, + { + name: "all levels must be in ascending order", + input: map[string]float64{ + "low": 40, + "medium": 30, + "high": 90, + }, + wantErr: true, + }, + { + name: "levels must be in ascending order even with defaults", + input: map[string]float64{ + "high": 30, + }, + wantErr: true, + }, + { + name: "levels cannot be equal when defaults are used", + input: map[string]float64{ + "low": 70, + "medium": 70, + "high": 100, + }, + wantErr: true, + }, + { + name: "levels cannot be equal even when defaults are used", + input: map[string]float64{ + "low": 70, + // medium defaults to 70 + "high": 100, + }, + wantErr: true, + }, + { + name: "unsupported key", + input: map[string]float64{ + "critical": 100, + }, + wantErr: true, + }, + { + name: "invalid key does not overwrite existing mapping", + input: map[string]float64{ + "low": 10, + "invalid": 20, + }, + wantErr: true, + }, + { + name: "negative threshold accepted", + input: map[string]float64{ + "low": -10, + }, + wantErr: false, + check: func(t *testing.T, confidenceLevels map[Confidence]float64) { + if confidenceLevels[ConfidenceLow] != -10 { + t.Fatalf("expected negative threshold to be accepted") + } + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + confidenceLevels := GetDefaultConfidenceLevels() + confidenceLevels, err := SetConfidenceLevelsFromStrings(confidenceLevels, tt.input) + if tt.wantErr { + if err == nil { + t.Fatalf("expected error, got nil") + } + return + } + + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if tt.check != nil { + tt.check(t, confidenceLevels) + } + }) + } +} + +func TestConfidenceFromString(t *testing.T) { + tests := []struct { + input string + want Confidence + err bool + }{ + {"low", ConfidenceLow, false}, + {"1", ConfidenceLow, false}, + {"medium", ConfidenceMedium, false}, + {"2", ConfidenceMedium, false}, + {"high", ConfidenceHigh, false}, + {"3", ConfidenceHigh, false}, + {"HIGH", ConfidenceHigh, false}, + {" low ", ConfidenceLow, false}, + {"invalid", 0, true}, + {"4", 0, true}, + {"", 0, true}, + } + + for _, tt := range tests { + got, err := ConfidenceFromString(tt.input) + if (err != nil) != tt.err { + t.Errorf("ConfidenceFromString(%q): err = %v, wantErr = %v", tt.input, err, tt.err) + continue + } + if got != tt.want { + t.Errorf("ConfidenceFromString(%q) = %d, want %d", tt.input, got, tt.want) + } + } +} diff --git a/detection/gitnotes/gitnotes.go b/detection/gitnotes/gitnotes.go index 84a4c76..fd91a86 100644 --- a/detection/gitnotes/gitnotes.go +++ b/detection/gitnotes/gitnotes.go @@ -7,10 +7,14 @@ import ( "github.com/chaoss/disclosure/detection" ) -type Detector struct{} +type Detector struct { + ConfidenceLevels map[detection.Confidence]float64 +} func (d *Detector) Name() string { return "gitnotes" } +func (d *Detector) GetConfidenceLevels() map[detection.Confidence]float64 { return d.ConfidenceLevels } + type toolModelPair struct { tool string model string @@ -29,6 +33,8 @@ func (d *Detector) Detect(input detection.Input) []detection.Finding { promptIDs = append(promptIDs, promptID) } sort.Strings(promptIDs) + score := detection.GitNotesMatchBaseScore + confidence := detection.ScoreToConfidence(d.ConfidenceLevels, score) for _, promptID := range promptIDs { prompt := parseResult.Metadata.Prompts[promptID] @@ -55,7 +61,8 @@ func (d *Detector) Detect(input detection.Input) []detection.Finding { Detector: d.Name(), Tool: tool, Model: model, - Confidence: detection.ConfidenceHigh, + Score: score, + Confidence: confidence, Detail: detail, }) } diff --git a/detection/gitnotes/gitnotes_test.go b/detection/gitnotes/gitnotes_test.go index 373cc20..d0d5ff9 100644 --- a/detection/gitnotes/gitnotes_test.go +++ b/detection/gitnotes/gitnotes_test.go @@ -6,8 +6,28 @@ import ( "github.com/chaoss/disclosure/detection" ) +func assertFindingMetadata(t *testing.T, finding detection.Finding, wantScore float64) { + t.Helper() + + if finding.Score != wantScore { + t.Errorf("score = %f, want %f", finding.Score, wantScore) + } + + expectedConfidence := detection.ScoreToConfidence( + detection.GetDefaultConfidenceLevels(), wantScore, + ) + + if finding.Confidence != expectedConfidence { + t.Errorf("confidence = %d, want %d", finding.Confidence, expectedConfidence) + } + + if finding.Detector != "gitnotes" { + t.Errorf("detector = %q, want %q", finding.Detector, "gitnotes") + } +} + func TestDetect(t *testing.T) { - d := &Detector{} + d := &Detector{ConfidenceLevels: detection.GetDefaultConfidenceLevels()} validNote := `src/main.rs abcd1234abcd1234 1-10,15-20 @@ -68,18 +88,21 @@ src/lib.rs notes string wantTools []string wantModels []string + wantScore float64 }{ { name: "valid git-ai note with single tool", notes: validNote, wantTools: []string{"cursor"}, wantModels: []string{"claude-4.5-opus"}, + wantScore: detection.GitNotesMatchBaseScore, }, { name: "multiple tools in note", notes: multiToolNote, wantTools: []string{"cursor", "claude-code"}, wantModels: []string{"claude-4.5-opus", "claude-3-sonnet"}, + wantScore: detection.GitNotesMatchBaseScore, }, { name: "empty notes", @@ -116,12 +139,8 @@ src/lib.rs for i, f := range findings { gotTools[i] = f.Tool gotModels[i] = f.Model - if f.Confidence != detection.ConfidenceHigh { - t.Errorf("confidence = %d, want %d", f.Confidence, detection.ConfidenceHigh) - } - if f.Detector != "gitnotes" { - t.Errorf("detector = %q, want %q", f.Detector, "gitnotes") - } + + assertFindingMetadata(t, f, tt.wantScore) } if len(gotTools) == 0 { @@ -162,7 +181,7 @@ src/lib.rs } func TestDetectPreservesDistinctToolModelPairs(t *testing.T) { - d := &Detector{} + d := &Detector{ConfidenceLevels: detection.GetDefaultConfidenceLevels()} note := `src/main.rs first 1-10 second 11-20 @@ -196,6 +215,7 @@ func TestDetectPreservesDistinctToolModelPairs(t *testing.T) { findings := d.Detect(detection.Input{Notes: note}) wantTools := []string{"cursor", "cursor"} wantModels := []string{"claude-4.5-opus", "gpt-4o"} + wantScore := detection.GitNotesMatchBaseScore if len(findings) != len(wantTools) { t.Fatalf("expected %d findings, got %d: %#v", len(wantTools), len(findings), findings) @@ -207,11 +227,12 @@ func TestDetectPreservesDistinctToolModelPairs(t *testing.T) { if finding.Model != wantModels[i] { t.Errorf("model[%d] = %q, want %q", i, finding.Model, wantModels[i]) } + assertFindingMetadata(t, finding, wantScore) } } func TestDetectDetailIncludesModel(t *testing.T) { - d := &Detector{} + d := &Detector{ConfidenceLevels: detection.GetDefaultConfidenceLevels()} note := `src/main.rs abcd1234abcd1234 1-10 --- @@ -237,6 +258,8 @@ func TestDetectDetailIncludesModel(t *testing.T) { t.Fatalf("expected 1 finding, got %d", len(findings)) } + assertFindingMetadata(t, findings[0], detection.GitNotesMatchBaseScore) + if findings[0].Detail == "" { t.Error("expected non-empty detail") } diff --git a/detection/toolmention/toolmention.go b/detection/toolmention/toolmention.go index 0e44f3f..9c0b2c3 100644 --- a/detection/toolmention/toolmention.go +++ b/detection/toolmention/toolmention.go @@ -41,10 +41,14 @@ func init() { } } -type Detector struct{} +type Detector struct { + ConfidenceLevels map[detection.Confidence]float64 +} func (d *Detector) Name() string { return "toolmention" } +func (d *Detector) GetConfidenceLevels() map[detection.Confidence]float64 { return d.ConfidenceLevels } + type toolMatch struct { start int end int @@ -92,12 +96,16 @@ func (d *Detector) Detect(input detection.Input) []detection.Finding { lastEnd = match.end } + score := detection.ToolMentionBaseScore + confidence := detection.ScoreToConfidence(d.ConfidenceLevels, score) + findings := make([]detection.Finding, 0, len(toolMatches)) for _, match := range toolMatches { findings = append(findings, detection.Finding{ Detector: d.Name(), Tool: match.name, - Confidence: detection.ConfidenceLow, + Score: score, + Confidence: confidence, Detail: "text mentions " + match.name, }) } diff --git a/detection/toolmention/toolmention_test.go b/detection/toolmention/toolmention_test.go index d6d47bd..3c84b33 100644 --- a/detection/toolmention/toolmention_test.go +++ b/detection/toolmention/toolmention_test.go @@ -7,7 +7,7 @@ import ( ) func TestDetect(t *testing.T) { - d := &Detector{} + d := &Detector{ConfidenceLevels: detection.GetDefaultConfidenceLevels()} tests := []struct { name string @@ -104,6 +104,11 @@ func TestDetect(t *testing.T) { input: detection.Input{Text: "Devin created this PR"}, wantTools: []string{"Devin"}, }, + { + name: "duplicate tool mentions only produce one finding", + input: detection.Input{Text: "Claude helped here. Claude helped there."}, + wantTools: []string{"Claude"}, + }, { name: "Qwen coder variant match", input: detection.Input{Text: "Running Qwen as a local autocomplete provider"}, @@ -249,29 +254,29 @@ func TestDetect(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { findings := d.Detect(tt.input) - gotTools := make([]string, len(findings)) + + if len(findings) != len(tt.wantTools) { + t.Fatalf("findings count = %d, want %d. findings=%v", len(findings), len(tt.wantTools), findings) + } + + expectedScore := detection.ToolMentionBaseScore + expectedConfidence := detection.ScoreToConfidence(d.ConfidenceLevels, expectedScore) + for i, f := range findings { - gotTools[i] = f.Tool - if f.Confidence != detection.ConfidenceLow { - t.Errorf("confidence = %d, want %d", f.Confidence, detection.ConfidenceLow) + if f.Tool != tt.wantTools[i] { + t.Errorf("tool[%d] = %q, want %q", i, f.Tool, tt.wantTools[i]) } - if f.Detector != "toolmention" { - t.Errorf("detector = %q, want %q", f.Detector, "toolmention") + + if f.Score != expectedScore { + t.Errorf("score[%d] = %v, want %v", i, f.Score, expectedScore) } - } - if len(gotTools) == 0 { - gotTools = nil - } + if f.Confidence != expectedConfidence { + t.Errorf("confidence[%d] = %d, want %d", i, f.Confidence, expectedConfidence) + } - if len(gotTools) != len(tt.wantTools) { - t.Errorf("tools = %v, want %v", gotTools, tt.wantTools) - return - } - for i := range gotTools { - if gotTools[i] != tt.wantTools[i] { - t.Errorf("tools = %v, want %v", gotTools, tt.wantTools) - return + if f.Detector != "toolmention" { + t.Errorf("detector[%d] = %q, want %q", i, f.Detector, "toolmention") } } }) diff --git a/detection/trailer/trailer.go b/detection/trailer/trailer.go index cd70ab7..05adba4 100644 --- a/detection/trailer/trailer.go +++ b/detection/trailer/trailer.go @@ -36,66 +36,79 @@ func extractToolFromText(text string) (string, error) { } var commitMessagePatterns = []struct { - check func(string) (detection.Confidence, bool) + check func(string) (float64, bool) name string }{ { - check: func(msg string) (detection.Confidence, bool) { - return detection.ConfidenceMedium, strings.HasPrefix(strings.ToLower(msg), detection.AiderCommitPrefix) + check: func(msg string) (float64, bool) { + if strings.HasPrefix(strings.ToLower(msg), detection.AiderCommitPrefix) { + return detection.TrailerMatchBaseScore, true + } + return detection.TrailerNotMatchedScore, false }, name: "Aider", }, { - check: func(msg string) (detection.Confidence, bool) { - return detection.ConfidenceMedium, strings.Contains(msg, detection.ClaudeAttributionText) + check: func(msg string) (float64, bool) { + if strings.Contains(msg, detection.ClaudeAttributionText) { + return detection.TrailerMatchBaseScore, true + } + return detection.TrailerNotMatchedScore, false }, name: "Claude Code", }, { - check: func(msg string) (detection.Confidence, bool) { + check: func(msg string) (float64, bool) { + matchedTrailerCount := 0 for _, trailer := range detection.EntireIOTrailers { if strings.Contains(msg, fmt.Sprintf("\n%s:", trailer)) { - return detection.ConfidenceMedium, true + matchedTrailerCount += 1 } } - return detection.ConfidenceMedium, false + if matchedTrailerCount > 0 { + score := detection.TrailerMatchBaseScore + (float64(matchedTrailerCount-1))*detection.AdditionalTrailerBonusPoints + return score, true + } + return detection.TrailerNotMatchedScore, false }, name: "EntireIO", }, { - check: func(msg string) (detection.Confidence, bool) { + check: func(msg string) (float64, bool) { matchResult := detection.ReplitAttributionPattern.FindStringSubmatch(msg) if len(matchResult) == 0 { // replit not detected - return detection.ConfidenceMedium, false + return detection.TrailerNotMatchedScore, false } - var confidence detection.Confidence + var score float64 = 0 switch matchResult[1] { - case "Agent": - confidence = detection.ConfidenceMedium - case "Assistant": - confidence = detection.ConfidenceLow + case "Agent", "Assistant": + score += detection.TrailerMatchBaseScore default: // unknown replit product, we cannot confirm ai use - return detection.ConfidenceLow, false + return detection.TrailerNotMatchedScore, false } - // if commit session id also present, increase confidence + // bonus points if commit session id also present if matchResult[2] != "" { - confidence.Increment() + score += detection.SessionIDBonusPoints } - return confidence, true + return score, true }, name: "Replit", }, } -type Detector struct{} +type Detector struct { + ConfidenceLevels map[detection.Confidence]float64 +} func (d *Detector) Name() string { return "trailer" } +func (d *Detector) GetConfidenceLevels() map[detection.Confidence]float64 { return d.ConfidenceLevels } + type toolModelPair struct { tool string model string @@ -155,18 +168,25 @@ func (d *Detector) detectTrailerCoauthoredBy(commitMessage string) []detection.F namePart := strings.TrimSpace(match[1]) email := strings.ToLower(strings.TrimSpace(match[2])) + score := detection.CoauthoredByTrailerBaseScore if name, ok := detection.KnownCoAuthorEmails[email]; ok { model := extractCoauthorModel(name, namePart) + if model != "" { + score += detection.CoauthorModelBonusPoints + } key := toolModelPair{tool: name, model: model} if seen[key] { continue } + score += detection.CoauthorKnownEmailBonusPoints + confidence := detection.ScoreToConfidence(d.ConfidenceLevels, score) findings = append(findings, detection.Finding{ Detector: d.Name(), Tool: name, Model: model, - Confidence: detection.ConfidenceHigh, + Score: score, + Confidence: confidence, Detail: fmt.Sprintf("Co-Authored-By trailer with email %s", email), }) seen[key] = true @@ -199,10 +219,14 @@ func (d *Detector) detectTrailerAssistedBy(commitMessage string) []detection.Fin continue } + score := detection.AssistedByTrailerBaseScore + confidence := detection.ScoreToConfidence(d.ConfidenceLevels, score) + findings = append(findings, detection.Finding{ Detector: d.Name(), Tool: matchedTool, - Confidence: detection.ConfidenceHigh, + Score: score, + Confidence: confidence, Detail: fmt.Sprintf("Assisted-By trailer with tool %s", matchedTool), }) seen[matchedToolKey] = true @@ -213,10 +237,12 @@ func (d *Detector) detectTrailerAssistedBy(commitMessage string) []detection.Fin func (d *Detector) detectMessagePatterns(commitMessage string) []detection.Finding { var findings []detection.Finding for _, p := range commitMessagePatterns { - if confidence, isDetected := p.check(commitMessage); isDetected { + if score, isDetected := p.check(commitMessage); isDetected { + confidence := detection.ScoreToConfidence(d.ConfidenceLevels, score) findings = append(findings, detection.Finding{ Detector: d.Name(), Tool: p.name, + Score: score, Confidence: confidence, Detail: fmt.Sprintf("commit message matches %s pattern", p.name), }) diff --git a/detection/trailer/trailer_test.go b/detection/trailer/trailer_test.go index 69b496e..c8bbcda 100644 --- a/detection/trailer/trailer_test.go +++ b/detection/trailer/trailer_test.go @@ -7,12 +7,13 @@ import ( ) func TestDetect(t *testing.T) { - d := &Detector{} + d := &Detector{ConfidenceLevels: detection.GetDefaultConfidenceLevels()} tests := []struct { name string message string wantTools []string wantModels []string + wantScore []float64 wantConfidence []detection.Confidence }{ // Co-Authored-By tests start here @@ -21,6 +22,7 @@ func TestDetect(t *testing.T) { message: "fix: update handler\n\nCo-Authored-By: Claude Opus 4 ", wantTools: []string{"Claude Code"}, wantModels: []string{"Opus 4"}, + wantScore: []float64{85}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh}, }, { @@ -28,6 +30,7 @@ func TestDetect(t *testing.T) { message: "fix: update handler\n\nCo-Authored-By: Claude Sonnet 4 ", wantTools: []string{"Claude Code"}, wantModels: []string{"Sonnet 4"}, + wantScore: []float64{85}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh}, }, { @@ -35,6 +38,7 @@ func TestDetect(t *testing.T) { message: "refactor: extract method\n\nCo-Authored-By: Cursor ", wantTools: []string{"Cursor"}, wantModels: []string{""}, + wantScore: []float64{75}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh}, }, { @@ -42,6 +46,7 @@ func TestDetect(t *testing.T) { message: "feat: add endpoint\n\nCo-Authored-By: aider (gpt-4o) ", wantTools: []string{"Aider"}, wantModels: []string{"gpt-4o"}, + wantScore: []float64{85}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh}, }, { @@ -49,6 +54,7 @@ func TestDetect(t *testing.T) { message: "feat: add endpoint\n\nCo-Authored-By: aider (claude-3.5-sonnet) ", wantTools: []string{"Aider"}, wantModels: []string{"claude-3.5-sonnet"}, + wantScore: []float64{85}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh}, }, { @@ -56,6 +62,7 @@ func TestDetect(t *testing.T) { message: "refactor: extract method\n\nCo-Authored-By: Cursor (composer 2.5) ", wantTools: []string{"Cursor"}, wantModels: []string{"composer 2.5"}, + wantScore: []float64{85}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh}, }, { @@ -63,6 +70,7 @@ func TestDetect(t *testing.T) { message: "feat: add endpoint\n\nCo-Authored-By: Copilot (gpt-4.1) ", wantTools: []string{"Copilot"}, wantModels: []string{"gpt-4.1"}, + wantScore: []float64{85}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh}, }, { @@ -70,6 +78,7 @@ func TestDetect(t *testing.T) { message: "fix: update handler\n\nCo-Authored-By: Claude Code (Opus 4.1) ", wantTools: []string{"Claude Code"}, wantModels: []string{"Opus 4.1"}, + wantScore: []float64{85}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh}, }, { @@ -77,6 +86,7 @@ func TestDetect(t *testing.T) { message: "fix: bug\n\nCo-Authored-By: Claude Opus 4 \nCo-Authored-By: Alice ", wantTools: []string{"Claude Code"}, wantModels: []string{"Opus 4"}, + wantScore: []float64{85}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh}, }, { @@ -84,6 +94,7 @@ func TestDetect(t *testing.T) { message: "fix: bug\n\nCo-Authored-By: Claude Opus 4 \nCo-Authored-By: aider (gpt-4o) ", wantTools: []string{"Claude Code", "Aider"}, wantModels: []string{"Opus 4", "gpt-4o"}, + wantScore: []float64{85, 85}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh, detection.ConfidenceHigh}, }, { @@ -91,6 +102,7 @@ func TestDetect(t *testing.T) { message: "fix: bug\n\nCo-Authored-By: Claude Opus 4 \nCo-Authored-By: Claude Sonnet 4 ", wantTools: []string{"Claude Code", "Claude Code"}, wantModels: []string{"Opus 4", "Sonnet 4"}, + wantScore: []float64{85, 85}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh, detection.ConfidenceHigh}, }, { @@ -98,6 +110,7 @@ func TestDetect(t *testing.T) { message: "fix: bug\n\nCo-Authored-By: Claude Opus 4 \nCo-Authored-By: Claude Opus 4 ", wantTools: []string{"Claude Code"}, wantModels: []string{"Opus 4"}, + wantScore: []float64{85}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh}, }, { @@ -105,6 +118,7 @@ func TestDetect(t *testing.T) { message: "fix: thing\n\nco-authored-by: Claude ", wantTools: []string{"Claude Code"}, wantModels: []string{""}, + wantScore: []float64{75}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh}, }, { @@ -112,12 +126,14 @@ func TestDetect(t *testing.T) { message: "fix: thing\n\nCO-AUTHORED-BY: Claude ", wantTools: []string{"Claude Code"}, wantModels: []string{""}, + wantScore: []float64{75}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh}, }, { name: "coauthor: human co-author only", message: "pair programming\n\nCo-Authored-By: Bob ", wantTools: nil, + wantScore: nil, wantConfidence: nil, }, // Co-Authored-By tests end here @@ -127,78 +143,91 @@ func TestDetect(t *testing.T) { name: "assistedby: Claude trailer with Opus model", message: "fix: update handler\n\nAssisted-By: Claude Opus 4 ", wantTools: []string{"Claude Opus 4"}, + wantScore: []float64{75}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh}, }, { name: "assistedby: Claude trailer with Sonnet model", message: "fix: update handler\n\nAssisted-By: Claude Sonnet 4 ", wantTools: []string{"Claude Sonnet 4"}, + wantScore: []float64{75}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh}, }, { name: "assistedby: Cursor trailer", message: "refactor: extract method\n\nAssisted-By: Cursor ", wantTools: []string{"Cursor"}, + wantScore: []float64{75}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh}, }, { name: "assistedby: Aider trailer with model name", message: "feat: add endpoint\n\nAssisted-By: aider (gpt-4o) ", wantTools: []string{"Aider"}, + wantScore: []float64{75}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh}, }, { name: "assistedby: Aider trailer with different model", message: "feat: add endpoint\n\nAssisted-By: aider (claude-4.7-opus) ", wantTools: []string{"Aider"}, + wantScore: []float64{75}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh}, }, { name: "assistedby: multiple trailers with Claude and human", message: "fix: bug\n\nAssisted-By: Claude Opus 4 ", wantTools: []string{"Claude Opus 4"}, + wantScore: []float64{75}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh}, }, { name: "assistedby: multiple AI trailers", message: "fix: bug\n\nAssisted-By: Claude Opus 4 \nAssisted-By: aider (gpt-4o) ", wantTools: []string{"Claude Opus 4", "Aider"}, + wantScore: []float64{75, 75}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh, detection.ConfidenceHigh}, }, { name: "assistedby: case variation", message: "fix: something\n\nassisted-by: Claude ", wantTools: []string{"Claude"}, + wantScore: []float64{75}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh}, }, { name: "assistedby: ASSISTED-BY uppercase", message: "fix: something\n\nASSISTED-BY: Claude ", wantTools: []string{"Claude"}, + wantScore: []float64{75}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh}, }, { name: "assistedby: Assisted-By trailer in commit message", message: "this is a commit message with\nAssisted-By: Claude Code", wantTools: []string{"Claude Code"}, + wantScore: []float64{75}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh}, }, { name: "assistedby: Another Assisted-By trailer in commit message 1", message: "this is a commit message with\nAssisted-By: Gemini", wantTools: []string{"Gemini"}, + wantScore: []float64{75}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh}, }, { name: "assistedby: Another Assisted-By trailer in commit message 2", message: "this is a commit message with\nAssisted-By: Kimi K2.6", wantTools: []string{"Kimi K2.6"}, + wantScore: []float64{75}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh}, }, { name: "assistedby: Multiple Assisted-By trailer in commit message", message: "this is a commit message with\nAssisted-By: Claude Code\nAssisted-By: Gemini", wantTools: []string{"Claude Code", "Gemini"}, + wantScore: []float64{75, 75}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh, detection.ConfidenceHigh}, }, { @@ -217,6 +246,7 @@ Co-Authored-By: Copilot Signed-off-by: some human `, wantTools: []string{"Cursor", "Copilot", "Claude 4.7 Opus", "Claude Sonnet 4", "Kimi K2.6", "ChatGPT", "Gemini"}, + wantScore: []float64{75, 75, 75, 75, 75, 75, 75}, wantConfidence: []detection.Confidence{ detection.ConfidenceHigh, // Cursor detection.ConfidenceHigh, // Copilot @@ -231,48 +261,56 @@ Signed-off-by: some human name: "assistedby: Same tool has 2 Assisted-By trailers in commit message", message: "this is a commit message with\nAssisted-By: Claude Code\nAssisted-By: Claude Code", wantTools: []string{"Claude Code"}, + wantScore: []float64{75}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh}, }, { name: "assistedby: Equivalent tools are deduplicated across multiple trailers", message: "this is a commit message with\nAssisted-By: Claude Code\nAssisted-By: CLAUDE CODE\nAssisted-By: [Claude Code].\nAssisted-By: GitHub Copilot", wantTools: []string{"Claude Code", "GitHub Copilot"}, + wantScore: []float64{75, 75}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh, detection.ConfidenceHigh}, }, { name: "assistedby: Tool with enclosing square brackets", message: "this is a commit message with\nAssisted-By: [Claude Code]", wantTools: []string{"Claude Code"}, + wantScore: []float64{75}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh}, }, { name: "assistedby: Tool with extra whitespace and terminal punctuation", message: "this is a commit message with\nAssisted-By: Claude Code.", wantTools: []string{"Claude Code"}, + wantScore: []float64{75}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh}, }, { name: "assistedby: Internal model punctuation is preserved", message: "this is a commit message with\nAssisted-By: GPT-5.5", wantTools: []string{"GPT-5.5"}, + wantScore: []float64{75}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh}, }, { name: "assistedby: Internal tool punctuation is preserved", message: "this is a commit message with\nAssisted-By: Continue.dev", wantTools: []string{"Continue.dev"}, + wantScore: []float64{75}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh}, }, { name: "assistedby: Assisted-By trailer in commit message in lower case", message: "this is a commit message with\nassisted-by: Claude Code", wantTools: []string{"Claude Code"}, + wantScore: []float64{75}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh}, }, { name: "assistedby: Two different attributions (assistedby and coauthor) both with email address", message: "Fix bug\n\nAssisted-By: Claude Sonnet 4 \nCo-Authored-By: Copilot ", wantTools: []string{"Copilot", "Claude Sonnet 4"}, + wantScore: []float64{75, 75}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh, detection.ConfidenceHigh}, }, { @@ -280,12 +318,14 @@ Signed-off-by: some human message: "Add validation logic\n\nCo-Authored-By: Claude Sonnet 4.6 \nAssisted-by: GitHub Copilot", wantTools: []string{"Claude Code", "GitHub Copilot"}, wantModels: []string{"Sonnet 4.6", ""}, + wantScore: []float64{85, 75}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh, detection.ConfidenceHigh}, }, { name: "assistedby: Claude Opus model attribution trailer", message: "Fix bug\n\nAssisted-by: Claude Opus 4 ", wantTools: []string{"Claude Opus 4"}, + wantScore: []float64{75}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh}, }, //Assisted-by tests end here @@ -295,132 +335,154 @@ Signed-off-by: some human name: "aider prefix", message: "aider: fix the login bug", wantTools: []string{"Aider"}, + wantScore: []float64{35}, wantConfidence: []detection.Confidence{detection.ConfidenceMedium}, }, { name: "aider prefix uppercase", message: "Aider: refactor auth module", wantTools: []string{"Aider"}, + wantScore: []float64{35}, wantConfidence: []detection.Confidence{detection.ConfidenceMedium}, }, { name: "Claude Code footer", message: "Add user validation\n\nGenerated with Claude Code", wantTools: []string{"Claude Code"}, + wantScore: []float64{35}, wantConfidence: []detection.Confidence{detection.ConfidenceMedium}, }, { name: "Claude Code footer with link", message: "Add validation\n\nGenerated with Claude Code\nhttps://claude.ai", wantTools: []string{"Claude Code"}, + wantScore: []float64{35}, wantConfidence: []detection.Confidence{detection.ConfidenceMedium}, }, { name: "EntireIO trailer present in commit", message: "this is some commit message\n\nEntire-Checkpoint: ab123cdefg12", wantTools: []string{"EntireIO"}, + wantScore: []float64{35}, wantConfidence: []detection.Confidence{detection.ConfidenceMedium}, }, { name: "Another EntireIO trailer present in commit", message: "this is some commit message\n\nEntire-Metadata: ab123cdefg12", wantTools: []string{"EntireIO"}, + wantScore: []float64{35}, wantConfidence: []detection.Confidence{detection.ConfidenceMedium}, }, { name: "Another EntireIO trailer present in commit with CRLF line endings", message: "this is some commit message\r\n\r\nEntire-Metadata: ab123cdefg12", wantTools: []string{"EntireIO"}, + wantScore: []float64{35}, wantConfidence: []detection.Confidence{detection.ConfidenceMedium}, }, { name: "EntireIO trailer not used, only mentioned in a commit", message: "this is a commit message with\nEntire-Metadata mentioned", wantTools: nil, + wantScore: nil, wantConfidence: nil, }, { name: "Replit Agent trailer present in a commit", message: "this is a commit message with\nReplit-Commit-Author: Agent", wantTools: []string{"Replit"}, + wantScore: []float64{35}, wantConfidence: []detection.Confidence{detection.ConfidenceMedium}, }, { name: "Replit Agent trailer present in a commit with session id", message: "this is a commit message with\nReplit-Commit-Author: Agent\nReplit-Commit-Session-Id: 1234a1ab-12ab-1234-abcd-0123456a1234", wantTools: []string{"Replit"}, + wantScore: []float64{80}, wantConfidence: []detection.Confidence{detection.ConfidenceHigh}, }, { name: "Replit Assistant trailer present in a commit", message: "this is a commit message with\nReplit-Commit-Author: Assistant", wantTools: []string{"Replit"}, - wantConfidence: []detection.Confidence{detection.ConfidenceLow}, + wantScore: []float64{35}, + wantConfidence: []detection.Confidence{detection.ConfidenceMedium}, }, { name: "Replit Assistant trailer present in a commit with session id", message: "this is a commit message with\nReplit-Commit-Author: Assistant\nReplit-Commit-Session-Id: 1234a1ab-12ab-1234-abcd-0123456a1234", wantTools: []string{"Replit"}, - wantConfidence: []detection.Confidence{detection.ConfidenceMedium}, + wantScore: []float64{80}, + wantConfidence: []detection.Confidence{detection.ConfidenceHigh}, }, { name: "Replit Agent trailer present in commit with CRLF line endings", message: "this is some commit message\r\n\r\nReplit-Commit-Author: Agent", wantTools: []string{"Replit"}, + wantScore: []float64{35}, wantConfidence: []detection.Confidence{detection.ConfidenceMedium}, }, { name: "Replit Assistant trailer present in commit with CRLF line endings", message: "this is some commit message\r\n\r\nReplit-Commit-Author: Assistant", wantTools: []string{"Replit"}, - wantConfidence: []detection.Confidence{detection.ConfidenceLow}, + wantScore: []float64{35}, + wantConfidence: []detection.Confidence{detection.ConfidenceMedium}, }, { name: "Replit Agent trailer present in commit with another trailer with CRLF line endings", message: "this is some commit message\r\n\r\nReplit-Commit-Author: Agent\r\nSomeOther: Trailer", wantTools: []string{"Replit"}, + wantScore: []float64{35}, wantConfidence: []detection.Confidence{detection.ConfidenceMedium}, }, { name: "Replit Assistant trailer present in commit with another trailer with CRLF line endings", message: "this is some commit message\r\n\r\nReplit-Commit-Author: Assistant\r\nSomeOther: Trailer", wantTools: []string{"Replit"}, - wantConfidence: []detection.Confidence{detection.ConfidenceLow}, + wantScore: []float64{35}, + wantConfidence: []detection.Confidence{detection.ConfidenceMedium}, }, { name: "Some other Replit product trailer (not agent or asst) present in a commit", message: "this is a commit message with\nReplit-Commit-Author: SomeOtherReplitProduct", wantTools: nil, + wantScore: nil, wantConfidence: nil, }, { name: "Replit trailer not used, only mentioned in a commit", message: "this is a commit message with\nReplit-Commit-Author: Assistant mentioned", wantTools: nil, + wantScore: nil, wantConfidence: nil, }, { name: "aider in middle of message not prefix", message: "fix the aider: integration test", wantTools: nil, + wantScore: nil, wantConfidence: nil, }, { name: "aider as substring of a word", message: "raider: fix the tests", wantTools: nil, + wantScore: nil, wantConfidence: nil, }, { name: "no trailers", message: "just a normal commit message with no AI signatures", wantTools: nil, + wantScore: nil, wantConfidence: nil, }, { name: "empty message", message: "", wantTools: nil, + wantScore: nil, wantConfidence: nil, }, } @@ -430,10 +492,12 @@ Signed-off-by: some human findings := d.Detect(detection.Input{CommitMessage: tt.message}) gotTools := make([]string, len(findings)) gotModels := make([]string, len(findings)) + gotScore := make([]float64, len(findings)) gotConfidence := make([]detection.Confidence, len(findings)) for i, f := range findings { gotTools[i] = f.Tool gotModels[i] = f.Model + gotScore[i] = f.Score gotConfidence[i] = f.Confidence if f.Detector != "trailer" { @@ -469,6 +533,23 @@ Signed-off-by: some human } } + if tt.wantScore != nil { + if len(gotScore) == 0 { + gotScore = nil + } + if len(gotScore) != len(tt.wantScore) { + t.Errorf("score = %v, want %v", gotScore, tt.wantScore) + return + } + for i := range gotScore { + if gotScore[i] != tt.wantScore[i] { + t.Errorf("score = %v, want %v", gotScore, tt.wantScore) + return + } + } + + } + if len(gotConfidence) == 0 { gotConfidence = nil } @@ -487,7 +568,7 @@ Signed-off-by: some human } func BenchmarkDetect(b *testing.B) { - d := &Detector{} + d := &Detector{ConfidenceLevels: detection.GetDefaultConfidenceLevels()} tests := []struct { name string message string diff --git a/output/output.go b/output/output.go index d6025be..c653b8a 100644 --- a/output/output.go +++ b/output/output.go @@ -5,7 +5,6 @@ import ( "fmt" "io" "sort" - "strings" "github.com/chaoss/disclosure/detection" "github.com/chaoss/disclosure/scan" @@ -40,9 +39,16 @@ func FormatText(w io.Writer, report scan.Report) error { if len(cr.Findings) == 0 { continue } - fmt.Fprintf(w, "Commit %s\n", cr.Hash[:12]) + hash := cr.Hash + if len(hash) > 12 { + hash = hash[:12] + } + fmt.Fprintf(w, "Commit %s (score: %.1f, confidence: %s)\n", hash, cr.Score, cr.Confidence.String()) for _, f := range cr.Findings { - fmt.Fprintf(w, " [%s] %s (%s): %s\n", f.Confidence, f.DisplayTool(), f.Detector, f.Detail) + fmt.Fprintf( + w, " [score: %.1f, confidence: %s] %s (%s): %s\n", + f.Score, f.Confidence, f.DisplayTool(), f.Detector, f.Detail, + ) } } @@ -57,8 +63,17 @@ func FormatTextFindings(w io.Writer, findings []detection.Finding) error { } fmt.Fprintf(w, "Found %d AI signal(s):\n", len(findings)) + + // compute consolidated score and confidence for these findings + score, _ := detection.ConsolidateScoreByFindings(findings) + confidence := detection.ScoreToConfidence(detection.GetDefaultConfidenceLevels(), score) + fmt.Fprintf(w, "Score: %.1f, Confidence: %s\n", score, confidence.String()) + for _, f := range findings { - fmt.Fprintf(w, " [%s] %s (%s): %s\n", f.Confidence, f.DisplayTool(), f.Detector, f.Detail) + fmt.Fprintf( + w, " [score: %.1f, confidence: %s] %s (%s): %s\n", + f.Score, f.Confidence.String(), f.DisplayTool(), f.Detector, f.Detail, + ) } return nil } @@ -67,9 +82,17 @@ func FormatTextFindings(w io.Writer, findings []detection.Finding) error { func FormatJSONFindings(w io.Writer, findings []detection.Finding) error { enc := json.NewEncoder(w) enc.SetIndent("", " ") + score, _ := detection.ConsolidateScoreByFindings(findings) + confidence := detection.ScoreToConfidence(detection.GetDefaultConfidenceLevels(), score) return enc.Encode(struct { - Findings []detection.Finding `json:"findings"` - }{Findings: findings}) + Findings []detection.Finding `json:"findings"` + Score float64 `json:"score"` + Confidence detection.Confidence `json:"confidence"` + }{ + Findings: findings, + Score: score, + Confidence: confidence, + }) } func sortedKeys(m map[string]int) []string { @@ -80,17 +103,3 @@ func sortedKeys(m map[string]int) []string { sort.Strings(keys) return keys } - -// ConfidenceFromString parses a confidence string or numeric value. -func ConfidenceFromString(s string) (detection.Confidence, error) { - switch strings.ToLower(strings.TrimSpace(s)) { - case "1", "low": - return detection.ConfidenceLow, nil - case "2", "medium": - return detection.ConfidenceMedium, nil - case "3", "high": - return detection.ConfidenceHigh, nil - default: - return 0, fmt.Errorf("invalid confidence %q: use low/1, medium/2, or high/3", s) - } -} diff --git a/output/output_test.go b/output/output_test.go index 4659dde..905e98d 100644 --- a/output/output_test.go +++ b/output/output_test.go @@ -3,6 +3,8 @@ package output import ( "bytes" "encoding/json" + "fmt" + "reflect" "strings" "testing" @@ -22,12 +24,17 @@ func sampleReport() scan.Report { Model: "Opus 4", Confidence: detection.ConfidenceHigh, Detail: "Co-Authored-By trailer with email noreply@anthropic.com", + Score: 100.0, }, }, + Score: 100.0, + Confidence: detection.ConfidenceHigh, }, { - Hash: "def789ghi012", - Findings: nil, + Hash: "def789ghi012", + Findings: nil, + Score: 0.0, + Confidence: detection.ConfidenceLow, }, }, Summary: scan.Summary{ @@ -155,33 +162,432 @@ func TestFormatTextFindingsEmpty(t *testing.T) { } } -func TestConfidenceFromString(t *testing.T) { - tests := []struct { - input string - want detection.Confidence - err bool - }{ - {"low", detection.ConfidenceLow, false}, - {"1", detection.ConfidenceLow, false}, - {"medium", detection.ConfidenceMedium, false}, - {"2", detection.ConfidenceMedium, false}, - {"high", detection.ConfidenceHigh, false}, - {"3", detection.ConfidenceHigh, false}, - {"HIGH", detection.ConfidenceHigh, false}, - {" low ", detection.ConfidenceLow, false}, - {"invalid", 0, true}, - {"4", 0, true}, - {"", 0, true}, - } - - for _, tt := range tests { - got, err := ConfidenceFromString(tt.input) - if (err != nil) != tt.err { - t.Errorf("ConfidenceFromString(%q): err = %v, wantErr = %v", tt.input, err, tt.err) - continue - } - if got != tt.want { - t.Errorf("ConfidenceFromString(%q) = %d, want %d", tt.input, got, tt.want) +func TestFormatJSONEmptyReport(t *testing.T) { + var buf bytes.Buffer + report := scan.Report{} + if err := FormatJSON(&buf, report); err != nil { + t.Fatalf("FormatJSON: %v", err) + } + var decoded scan.Report + if err := json.Unmarshal(buf.Bytes(), &decoded); err != nil { + t.Fatalf("unmarshal: %v", err) + } +} + +type failingWriter struct{} + +func (failingWriter) Write([]byte) (int, error) { + return 0, fmt.Errorf("write failed") +} + +func TestFormatJSONWriterError(t *testing.T) { + err := FormatJSON(failingWriter{}, sampleReport()) + if err == nil { + t.Fatal("expected error") + } +} + +func TestFormatTextZeroScore(t *testing.T) { + var buf bytes.Buffer + report := scan.Report{ + Summary: scan.Summary{ + TotalCommits: 1, + AICommits: 1, + ToolCounts: map[string]int{}, + }, + } + if err := FormatText(&buf, report); err != nil { + t.Fatal(err) + } + if strings.Contains(buf.String(), "Score") { + t.Error("did not expect score for zero") + } +} + +func TestFormatTextShortHash(t *testing.T) { + var buf bytes.Buffer + report := scan.Report{ + Commits: []scan.CommitResult{ + { + Hash: "abc", + Findings: []detection.Finding{ + {Detector: "test"}, + }, + }, + }, + Summary: scan.Summary{ + AICommits: 1, + }, + } + defer func() { + if r := recover(); r != nil { + t.Fatalf("panic: %v", r) } + }() + if err := FormatText(&buf, report); err != nil { + t.Fatal(err) + } +} + +func TestFormatTextFindingsIncludesScore(t *testing.T) { + var buf bytes.Buffer + findings := []detection.Finding{ + { + Detector: "test", + Score: 100, + }, + } + if err := FormatTextFindings(&buf, findings); err != nil { + t.Fatal(err) + } + formatTextStr := buf.String() + if !strings.Contains(formatTextStr, "Score:") { + t.Fatal("missing score") + } + if !strings.Contains(formatTextStr, "Confidence:") { + t.Fatal("missing confidence") + } +} + +func TestFormatTextFindingsEmptySlice(t *testing.T) { + var buf bytes.Buffer + if err := FormatTextFindings(&buf, []detection.Finding{}); err != nil { + t.Fatal(err) + } + if !strings.Contains(buf.String(), "No AI involvement detected") { + t.Fatal("expected no detection message") + } +} + +func TestFormatJSONFindingsStructure(t *testing.T) { + var buf bytes.Buffer + findings := []detection.Finding{ + { + Detector: "toolmention", + Tool: "Claude", + Score: 100, + Confidence: detection.ConfidenceHigh, + }, + } + if err := FormatJSONFindings(&buf, findings); err != nil { + t.Fatal(err) + } + var decoded struct { + Findings []detection.Finding `json:"findings"` + Score float64 `json:"score"` + Confidence detection.Confidence `json:"confidence"` + } + if err := json.Unmarshal(buf.Bytes(), &decoded); err != nil { + t.Fatal(err) + } + if len(decoded.Findings) != 1 { + t.Fatalf("findings=%d want 1", len(decoded.Findings)) + } + if decoded.Score != 100 { + t.Fatalf("score=%v want 100", decoded.Score) + } + if decoded.Confidence != detection.ConfidenceHigh { + t.Fatalf("confidence=%v want high", decoded.Confidence) + } +} + +func TestSortedKeys(t *testing.T) { + got := sortedKeys(map[string]int{"c": 1, "h": 1, "a": 1, "o": 1, "s": 1}) + want := []string{"a", "c", "h", "o", "s"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("got %v want %v", got, want) + } +} + +func TestFormatTextExactOutput(t *testing.T) { + var buf bytes.Buffer + + report := sampleReport() + + if err := FormatText(&buf, report); err != nil { + t.Fatalf("FormatText: %v", err) + } + + want := `Scanned 2 commits, 1 with AI signals + +Tools detected: + Claude Code: 1 + +Commit abc123def456 (score: 100.0, confidence: high) + [score: 100.0, confidence: high] Claude Code [Opus 4] (trailer): Co-Authored-By trailer with email noreply@anthropic.com +` + + if got := buf.String(); got != want { + t.Errorf("FormatText output mismatch\n--- got ---\n%q\n--- want ---\n%q", got, want) + } +} + +func TestFormatTextExactOutputMultipleTools(t *testing.T) { + var buf bytes.Buffer + + report := scan.Report{ + Commits: []scan.CommitResult{ + { + Hash: "1234567890abcdef", + Findings: []detection.Finding{ + { + Detector: "toolmention", + Tool: "Copilot", + Confidence: detection.ConfidenceLow, + Score: 20, + Detail: "copilot finding", + }, + { + Detector: "toolmention", + Tool: "Kimi", + Confidence: detection.ConfidenceLow, + Score: 20, + Detail: "kimi finding", + }, + { + Detector: "trailer", + Tool: "Claude", + Confidence: detection.ConfidenceHigh, + Score: 75, + Detail: "claude finding", + }, + }, + Score: 95, + Confidence: detection.ConfidenceHigh, + }, + }, + Summary: scan.Summary{ + TotalCommits: 1, + AICommits: 1, + ToolCounts: map[string]int{ + "Claude": 1, + "Copilot": 1, + "Kimi": 1, + }, + }, + } + + if err := FormatText(&buf, report); err != nil { + t.Fatalf("FormatText: %v", err) + } + + want := `Scanned 1 commits, 1 with AI signals + +Tools detected: + Claude: 1 + Copilot: 1 + Kimi: 1 + +Commit 1234567890ab (score: 95.0, confidence: high) + [score: 20.0, confidence: low] Copilot (toolmention): copilot finding + [score: 20.0, confidence: low] Kimi (toolmention): kimi finding + [score: 75.0, confidence: high] Claude (trailer): claude finding +` + + if got := buf.String(); got != want { + t.Errorf("FormatText output mismatch\n--- got ---\n%q\n--- want ---\n%q", got, want) + } +} + +func TestFormatTextNoAIExactOutput(t *testing.T) { + var buf bytes.Buffer + + report := scan.Report{ + Summary: scan.Summary{ + TotalCommits: 5, + AICommits: 0, + ToolCounts: map[string]int{}, + }, + } + + if err := FormatText(&buf, report); err != nil { + t.Fatalf("FormatText: %v", err) + } + + want := "Scanned 5 commits, 0 with AI signals\n\nNo AI involvement detected.\n" + + if got := buf.String(); got != want { + t.Errorf("FormatText output mismatch\n--- got ---\n%q\n--- want ---\n%q", got, want) + } +} + +func TestFormatTextSkipsCommitsWithoutFindings(t *testing.T) { + var buf bytes.Buffer + + report := scan.Report{ + Commits: []scan.CommitResult{ + { + Hash: "with-findings", + Findings: []detection.Finding{ + { + Detector: "test", + Tool: "Claude", + Confidence: detection.ConfidenceHigh, + Score: 100, + Detail: "detected", + }, + }, + Score: 100, + Confidence: detection.ConfidenceHigh, + }, + { + Hash: "without-findings", + Findings: nil, + Score: 0, + Confidence: detection.ConfidenceNone, + }, + }, + Summary: scan.Summary{ + TotalCommits: 2, + AICommits: 1, + ToolCounts: map[string]int{ + "Claude": 1, + }, + }, + } + + if err := FormatText(&buf, report); err != nil { + t.Fatalf("FormatText: %v", err) + } + + want := `Scanned 2 commits, 1 with AI signals + +Tools detected: + Claude: 1 + +Commit with-finding (score: 100.0, confidence: high) + [score: 100.0, confidence: high] Claude (test): detected +` + + if got := buf.String(); got != want { + t.Errorf("FormatText output mismatch\n--- got ---\n%q\n--- want ---\n%q", got, want) + } +} + +func TestFormatTextFindingsExactOutput(t *testing.T) { + var buf bytes.Buffer + + findings := []detection.Finding{ + { + Detector: "toolmention", + Tool: "Claude", + Confidence: detection.ConfidenceLow, + Score: 25, + Detail: "text mentions Claude", + }, + { + Detector: "gitnotes", + Model: "gpt-4o", + Confidence: detection.ConfidenceHigh, + Score: 100, + Detail: "git notes declares model", + }, + } + + if err := FormatTextFindings(&buf, findings); err != nil { + t.Fatalf("FormatTextFindings: %v", err) + } + + want := `Found 2 AI signal(s): +Score: 125.0, Confidence: high + [score: 25.0, confidence: low] Claude (toolmention): text mentions Claude + [score: 100.0, confidence: high] gpt-4o (gitnotes): git notes declares model +` + + if got := buf.String(); got != want { + t.Errorf("FormatTextFindings output mismatch\n--- got ---\n%q\n--- want ---\n%q", got, want) + } +} + +func TestFormatTextFindingsEmptyExactOutput(t *testing.T) { + var buf bytes.Buffer + + if err := FormatTextFindings(&buf, nil); err != nil { + t.Fatalf("FormatTextFindings: %v", err) + } + + want := "No AI involvement detected.\n" + + if got := buf.String(); got != want { + t.Errorf("FormatTextFindings output mismatch\n--- got ---\n%q\n--- want ---\n%q", got, want) + } +} + +func TestFormatTextFindingsEmptySliceExactOutput(t *testing.T) { + var buf bytes.Buffer + + if err := FormatTextFindings(&buf, []detection.Finding{}); err != nil { + t.Fatalf("FormatTextFindings: %v", err) + } + + want := "No AI involvement detected.\n" + + if got := buf.String(); got != want { + t.Errorf("FormatTextFindings output mismatch\n--- got ---\n%q\n--- want ---\n%q", got, want) + } +} + +func TestFormatJSONExactIndentation(t *testing.T) { + var buf bytes.Buffer + + report := scan.Report{} + + if err := FormatJSON(&buf, report); err != nil { + t.Fatalf("FormatJSON: %v", err) + } + + got := buf.String() + + if !strings.HasSuffix(got, "\n") { + t.Errorf("FormatJSON output does not end with newline: %q", got) + } + + if !strings.Contains(got, "\n ") { + t.Errorf("FormatJSON output is not indented with two spaces:\n%s", got) + } +} + +func TestFormatJSONFindingsExactFormatting(t *testing.T) { + var buf bytes.Buffer + + findings := []detection.Finding{ + { + Detector: "toolmention", + Tool: "Claude", + Confidence: detection.ConfidenceLow, + Score: 100, + Detail: "text mentions Claude", + }, + } + + if err := FormatJSONFindings(&buf, findings); err != nil { + t.Fatalf("FormatJSONFindings: %v", err) + } + + got := buf.String() + + if !strings.HasSuffix(got, "\n") { + t.Errorf("expected trailing newline, got %q", got) + } + + if !strings.Contains(got, "\n ") { + t.Errorf("expected two-space indentation:\n%s", got) + } + + var decoded map[string]any + if err := json.Unmarshal(buf.Bytes(), &decoded); err != nil { + t.Fatalf("unmarshal: %v", err) + } + + if _, ok := decoded["findings"]; !ok { + t.Errorf("missing findings field:\n%s", got) + } + + if _, ok := decoded["score"]; !ok { + t.Errorf("missing score field:\n%s", got) + } + + if _, ok := decoded["confidence"]; !ok { + t.Errorf("missing confidence field:\n%s", got) } } diff --git a/scan/scan.go b/scan/scan.go index 86084ca..55c0ca9 100644 --- a/scan/scan.go +++ b/scan/scan.go @@ -7,16 +7,20 @@ import ( // CommitResult holds findings for a single commit. type CommitResult struct { - Hash string `json:"hash"` - Findings []detection.Finding `json:"findings"` + Hash string `json:"hash"` + Findings []detection.Finding `json:"findings"` + PerDetectorScores map[string]float64 `json:"per_detector_scores"` + Score float64 `json:"score"` + Confidence detection.Confidence `json:"confidence"` } // Summary aggregates stats across all commits scanned. type Summary struct { - TotalCommits int `json:"total_commits"` - AICommits int `json:"ai_commits"` - ToolCounts map[string]int `json:"tool_counts"` - ByConfidence map[string]int `json:"by_confidence"` + TotalCommits int `json:"total_commits"` + AICommits int `json:"ai_commits"` + ToolCounts map[string]int `json:"tool_counts"` + ByConfidence map[string]int `json:"by_confidence"` + PerDetectorScores map[string]float64 `json:"per_detector_scores"` } // Report holds the full scan results. @@ -81,9 +85,25 @@ func scanOneCommit(c gitops.Commit, branchName string, detectors []detection.Det findings = append(findings, d.Detect(input)...) } + if len(detectors) == 0 { + return CommitResult{ + Hash: c.Hash, + Findings: findings, + PerDetectorScores: nil, + Score: 0.0, + Confidence: detection.ConfidenceNone, + } + } + + confidenceLevels := detectors[0].GetConfidenceLevels() + score, perDetectorScores := detection.ConsolidateScoreByFindings(findings) + confidence := detection.ScoreToConfidence(confidenceLevels, score) return CommitResult{ - Hash: c.Hash, - Findings: findings, + Hash: c.Hash, + Findings: findings, + PerDetectorScores: perDetectorScores, + Score: score, + Confidence: confidence, } } @@ -94,6 +114,7 @@ func buildReport(results []CommitResult) Report { ByConfidence: map[string]int{}, } + var allFindings []detection.Finding for _, r := range results { if len(r.Findings) > 0 { summary.AICommits++ @@ -101,9 +122,13 @@ func buildReport(results []CommitResult) Report { for _, f := range r.Findings { summary.ToolCounts[f.Tool]++ summary.ByConfidence[f.Confidence.String()]++ + allFindings = append(allFindings, f) } } + _, perDetectorScores := detection.ConsolidateScoreByFindings(allFindings) + summary.PerDetectorScores = perDetectorScores + return Report{ Commits: results, Summary: summary, diff --git a/scan/scan_test.go b/scan/scan_test.go index fa50936..e26794a 100644 --- a/scan/scan_test.go +++ b/scan/scan_test.go @@ -8,25 +8,30 @@ import ( "time" "github.com/chaoss/disclosure/detection" + "github.com/chaoss/disclosure/detection/branchname" "github.com/chaoss/disclosure/detection/committer" "github.com/chaoss/disclosure/detection/gitnotes" "github.com/chaoss/disclosure/detection/toolmention" "github.com/chaoss/disclosure/detection/trailer" "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/object" ) func allDetectors() []detection.Detector { + confidenceLevels := detection.GetDefaultConfidenceLevels() return []detection.Detector{ - &committer.Detector{}, - &gitnotes.Detector{}, - &trailer.Detector{}, - &toolmention.Detector{}, + &committer.Detector{ConfidenceLevels: confidenceLevels}, + &gitnotes.Detector{ConfidenceLevels: confidenceLevels}, + &trailer.Detector{ConfidenceLevels: confidenceLevels}, + &toolmention.Detector{ConfidenceLevels: confidenceLevels}, + &branchname.Detector{ConfidenceLevels: confidenceLevels}, } } func initTestRepo(t *testing.T) (string, []string) { t.Helper() + const humanEmail = "human@example.com" dir := t.TempDir() @@ -97,6 +102,14 @@ Assisted-by: Gemini (documentation) hashes = append(hashes, hash.String()) } + const branchName = "codex/fix-test" + if err := wt.Checkout(&git.CheckoutOptions{ + Branch: plumbing.NewBranchReferenceName(branchName), + Create: true, + }); err != nil { + t.Fatalf("create branch %s: %v", branchName, err) + } + return dir, hashes } @@ -104,17 +117,17 @@ func TestScanCommitRange(t *testing.T) { dir, hashes := initTestRepo(t) detectors := allDetectors() - report, err := ScanCommitRange(dir, hashes[0]+".."+hashes[3], detectors) + report, err := ScanCommitRange(dir, hashes[0]+".."+hashes[4], detectors) if err != nil { t.Fatalf("ScanCommitRange: %v", err) } - if report.Summary.TotalCommits != 3 { - t.Errorf("total commits = %d, want 3", report.Summary.TotalCommits) + if report.Summary.TotalCommits != 4 { + t.Errorf("total commits = %d, want 4", report.Summary.TotalCommits) } - if report.Summary.AICommits != 3 { - t.Errorf("ai commits = %d, want 3", report.Summary.AICommits) + if report.Summary.AICommits != 4 { + t.Errorf("ai commits = %d, want 4", report.Summary.AICommits) } // Check that Claude Code was detected via co-author @@ -141,6 +154,29 @@ func TestScanCommitRange(t *testing.T) { if count, ok := report.Summary.ToolCounts["Kimi K2.6"]; !ok || count == 0 { t.Error("expected Kimi K2.6 Opus in tool counts") } + + // Check scoring + perDetectorScores := report.Summary.PerDetectorScores + committerScore := perDetectorScores["committer"] + if committerScore != 95 { + t.Errorf("expected committer score to be 85, found %f", committerScore) + } + gitnotesScore := perDetectorScores["gitnotes"] + if gitnotesScore != 0 { + t.Errorf("expected gitnotes score to be 0, found %f", gitnotesScore) + } + toolmentionScore := perDetectorScores["toolmention"] + if toolmentionScore != 20 { + t.Errorf("expected toolmention score to be 20, found %f", toolmentionScore) + } + trailerScore := perDetectorScores["trailer"] + if trailerScore != 85 { + t.Errorf("expected trailer score to be 85, found %f", trailerScore) + } + branchnameScore := perDetectorScores["branchname"] + if branchnameScore != 75 { + t.Errorf("expected branchname score to be 75, found %f", branchnameScore) + } } func TestScanCommitRangeAll(t *testing.T) { @@ -169,6 +205,7 @@ func TestScanCommitDetectsAcrossDetectors(t *testing.T) { (&committer.Detector{}).Name(): "GitHub Copilot (agent)", (&trailer.Detector{}).Name(): "Kimi K2.6", (&toolmention.Detector{}).Name(): "Kimi", + (&branchname.Detector{}).Name(): "OpenAI Codex", } if len(result.Findings) != len(wantToolsByDetector) { t.Fatalf("got %d findings, want %d", len(result.Findings), len(wantToolsByDetector)) @@ -205,6 +242,29 @@ func TestScanCommit(t *testing.T) { t.Error("expected findings for assisted-by and co-author trailers") } + for _, f := range result.Findings { + if f.Score <= 0 { + t.Errorf( + "finding %s score=%v, expected positive score", + f.Tool, + f.Score, + ) + } + + expectedConfidence := detection.ScoreToConfidence( + detection.GetDefaultConfidenceLevels(), f.Score, + ) + + if f.Confidence != expectedConfidence { + t.Errorf( + "%s confidence=%d want=%d", + f.Tool, + f.Confidence, + expectedConfidence, + ) + } + } + foundCoauthor := false foundAssistedBy := false for _, f := range result.Findings { @@ -220,6 +280,32 @@ func TestScanCommit(t *testing.T) { if !foundAssistedBy { t.Error("expected assistedby finding for Kimi K2.6") } + + // Check scoring + perDetectorScores := result.PerDetectorScores + committerScore := perDetectorScores["committer"] + if committerScore != 0 { + t.Errorf("expected committer score to be 0, found %f", committerScore) + } + gitnotesScore := perDetectorScores["gitnotes"] + if gitnotesScore != 0 { + t.Errorf("expected gitnotes score to be 0, found %f", gitnotesScore) + } + toolmentionScore := perDetectorScores["toolmention"] + if toolmentionScore != 20 { + t.Errorf("expected toolmention score to be 20, found %f", toolmentionScore) + } + trailerScore := perDetectorScores["trailer"] + if trailerScore != 75 { + t.Errorf("expected trailer score to be 75, found %f", trailerScore) + } + branchnameScore := perDetectorScores["branchname"] + if branchnameScore != 75 { + t.Errorf("expected branchname score to be 75, found %f", branchnameScore) + } + if result.Score != 170 { + t.Errorf("expected overall score to be 170, found %f", result.Score) + } } func TestScanText(t *testing.T) { @@ -362,3 +448,100 @@ func TestReportSummaryByConfidence(t *testing.T) { t.Error("expected medium confidence findings") } } + +func TestScanReportNoFindingsHasZeroScore(t *testing.T) { + dir, hashes := initTestRepo(t) + + report, err := ScanCommitRange(dir, hashes[0]+".."+hashes[1], nil) + if err != nil { + t.Fatalf("ScanCommitRange: %v", err) + } + + for _, cr := range report.Commits { + if cr.Score != 0 { + t.Fatalf("commit %s score = %v, want 0", cr.Hash, cr.Score) + } + } +} + +func TestScanReportInvalidRange(t *testing.T) { + dir, _ := initTestRepo(t) + + _, err := ScanCommitRange(dir, "invalid..range", allDetectors()) + if err == nil { + t.Fatal("expected to return an error") + } +} + +func TestScanCommitNoAI(t *testing.T) { + dir := t.TempDir() + + repo, err := git.PlainInit(dir, false) + if err != nil { + t.Fatalf("init repo: %v", err) + } + + wt, err := repo.Worktree() + if err != nil { + t.Fatalf("worktree: %v", err) + } + + filename := filepath.Join(dir, "main.go") + if err := os.WriteFile(filename, []byte("package main\nfunc main() {}"), 0644); err != nil { + t.Fatalf("write file: %v", err) + } + + if _, err := wt.Add("main.go"); err != nil { + t.Fatalf("add: %v", err) + } + + hash, err := wt.Commit("fix normal bug", &git.CommitOptions{ + Author: &object.Signature{ + Name: "Developer", + Email: "developer@example.com", + When: time.Now(), + }, + Committer: &object.Signature{ + Name: "Developer", + Email: "developer@example.com", + When: time.Now(), + }, + }) + if err != nil { + t.Fatalf("commit: %v", err) + } + + result, err := ScanCommit(dir, hash.String(), allDetectors()) + if err != nil { + t.Fatalf("ScanCommit: %v", err) + } + + if result.Hash != hash.String() { + t.Errorf("hash=%q want %q", result.Hash, hash.String()) + } + + if len(result.Findings) != 0 { + t.Errorf("expected no findings, got %d: %#v", len(result.Findings), result.Findings) + } + + if result.Score != 0 { + t.Errorf("score=%v want 0", result.Score) + } +} + +func TestScanCommitEmptyDetectorList(t *testing.T) { + dir, hashes := initTestRepo(t) + + result, err := ScanCommit(dir, hashes[1], nil) + if err != nil { + t.Fatalf("ScanCommit: %v", err) + } + + if len(result.Findings) != 0 { + t.Errorf("findings=%v want none", result.Findings) + } + + if result.Score != 0 { + t.Errorf("score=%v want 0", result.Score) + } +}