From 607f0a439ee7c769bec524d807d864bfe5adfdf3 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Fri, 21 Aug 2026 09:08:00 +0100 Subject: [PATCH 1/2] Add JSON report metadata --- cmd/licenses/benchmark_test.go | 3 + cmd/licenses/licensee_comparison_test.go | 1 + cmd/licenses/main.go | 2 +- cmd/licenses/main_test.go | 66 +++++++++- cmd/licenses/scan.go | 70 ++++++++++- cmd/licenses/scan_test.go | 152 ++++++++++++++++++++++- 6 files changed, 283 insertions(+), 11 deletions(-) diff --git a/cmd/licenses/benchmark_test.go b/cmd/licenses/benchmark_test.go index 9932bea..819444c 100644 --- a/cmd/licenses/benchmark_test.go +++ b/cmd/licenses/benchmark_test.go @@ -55,6 +55,7 @@ func BenchmarkScanRepository(b *testing.B) { matcher, root, options, + testScannerVersion, ) if err != nil { b.Fatal(err) @@ -81,6 +82,7 @@ func BenchmarkScanRepositories(b *testing.B) { matcher, root, options, + testScannerVersion, ) if err != nil { b.Fatal(err) @@ -92,6 +94,7 @@ func BenchmarkScanRepositories(b *testing.B) { matcher, root, options, + testScannerVersion, ) if err != nil { b.Fatal(err) diff --git a/cmd/licenses/licensee_comparison_test.go b/cmd/licenses/licensee_comparison_test.go index d045fb2..2f77781 100644 --- a/cmd/licenses/licensee_comparison_test.go +++ b/cmd/licenses/licensee_comparison_test.go @@ -86,6 +86,7 @@ func TestCompareLicenseeRepositories(t *testing.T) { matcher, root, options, + testScannerVersion, ) if err != nil { t.Fatalf("scan %s: %v", root, err) diff --git a/cmd/licenses/main.go b/cmd/licenses/main.go index 1d0aba9..dc792fd 100644 --- a/cmd/licenses/main.go +++ b/cmd/licenses/main.go @@ -133,7 +133,7 @@ func run( if err != nil { return exitFatal, err } - report, err := scanRepository(ctx, matcher, root, options) + report, err := scanRepository(ctx, matcher, root, options, version) if err != nil { return exitFatal, err } diff --git a/cmd/licenses/main_test.go b/cmd/licenses/main_test.go index a0e8f9e..52119b7 100644 --- a/cmd/licenses/main_test.go +++ b/cmd/licenses/main_test.go @@ -3,9 +3,12 @@ package main import ( "bytes" "context" + "crypto/sha256" + "encoding/hex" "encoding/json" "flag" "path/filepath" + "slices" "strings" "testing" @@ -68,6 +71,9 @@ func TestRunJSON(t *testing.T) { if report.Schema != reportSchemaVersion { t.Errorf("schema = %d, want %d", report.Schema, reportSchemaVersion) } + if report.Scanner != (scannerRecord{Name: scannerName, Version: version}) { + t.Errorf("scanner = %#v, want CLI name and version", report.Scanner) + } if report.Summary.FilesScanned != 1 { t.Errorf("files scanned = %d, want 1", report.Summary.FilesScanned) } @@ -79,6 +85,13 @@ func TestRunJSON(t *testing.T) { if len(report.Files) != 1 || !hasMITExpression(report.Files[0]) { t.Fatalf("files = %#v, want MIT detection", report.Files) } + digest := sha256.Sum256(projectLicense(t)) + if report.Files[0].SHA256 != hex.EncodeToString(digest[:]) { + t.Errorf("sha256 = %q, want original LICENSE hash", report.Files[0].SHA256) + } + if !slices.Equal(report.Files[0].Roles, []string{"license"}) { + t.Errorf("roles = %#v, want license", report.Files[0].Roles) + } if report.Files[0].LicenseTextCoverage <= 0 || report.Files[0].LicenseTextCoverage > 100 { t.Errorf( @@ -97,7 +110,8 @@ func TestRunJSON(t *testing.T) { ) } if len(report.Expressions) != 1 || - report.Expressions[0].Identification != licenses.Identified { + report.Expressions[0].Identification != licenses.Identified || + !report.Expressions[0].Root { t.Errorf("expressions = %#v, want identified", report.Expressions) } if report.Files[0].Detections[0].Matches[0].Matched == "" { @@ -115,12 +129,48 @@ func TestRunJSON(t *testing.T) { } } +func TestRunJSONReportsEmptyRolesAndNonRootExpression(t *testing.T) { + t.Parallel() + + path := filepath.Join(t.TempDir(), "source.txt") + writeTestFile(t, path, projectLicense(t)) + var stdout bytes.Buffer + exitCode, err := run( + context.Background(), + []string{"-json", path}, + &stdout, + &bytes.Buffer{}, + false, + ) + if err != nil { + t.Fatal(err) + } + if exitCode != exitSuccess { + t.Fatalf("exit code = %d, want %d", exitCode, exitSuccess) + } + var report scanReport + if err := json.Unmarshal(stdout.Bytes(), &report); err != nil { + t.Fatal(err) + } + if len(report.Files) != 1 || report.Files[0].Roles == nil || len(report.Files[0].Roles) != 0 { + t.Fatalf("files = %#v, want one file with empty roles", report.Files) + } + if len(report.Expressions) != 1 || report.Expressions[0].Root { + t.Errorf("expressions = %#v, want one non-root expression", report.Expressions) + } + if !strings.Contains(stdout.String(), `"roles": []`) || + !strings.Contains(stdout.String(), `"root": false`) { + t.Errorf("JSON does not preserve empty roles and false root:\n%s", stdout.String()) + } +} + func TestRunJSONIsDeterministicAcrossWorkerCounts(t *testing.T) { t.Parallel() root := t.TempDir() writeTestFile(t, filepath.Join(root, "a", "LICENSE"), projectLicense(t)) writeTestFile(t, filepath.Join(root, "b", "LICENSE"), projectLicense(t)) + writeTestFile(t, filepath.Join(root, "licenses", "NOTICE"), projectLicense(t)) writeTestFile(t, filepath.Join(root, "binary"), []byte("data\x00data")) var first bytes.Buffer @@ -151,6 +201,20 @@ func TestRunJSONIsDeterministicAcrossWorkerCounts(t *testing.T) { if !bytes.Equal(first.Bytes(), second.Bytes()) { t.Errorf("JSON differs by worker count:\n%s\n%s", first.String(), second.String()) } + var report scanReport + if err := json.Unmarshal(first.Bytes(), &report); err != nil { + t.Fatal(err) + } + foundNotice := false + for _, file := range report.Files { + if file.Path == "licenses/NOTICE" && + slices.Equal(file.Roles, []string{"license", "notice"}) { + foundNotice = true + } + } + if !foundNotice { + t.Errorf("files = %#v, want NOTICE with license, notice roles", report.Files) + } } func TestRunScope(t *testing.T) { diff --git a/cmd/licenses/scan.go b/cmd/licenses/scan.go index 42108b9..895ca7f 100644 --- a/cmd/licenses/scan.go +++ b/cmd/licenses/scan.go @@ -4,7 +4,9 @@ import ( "bytes" "cmp" "context" + "crypto/sha256" "encoding/binary" + "encoding/hex" "errors" "fmt" "io" @@ -34,10 +36,13 @@ const ( maxReadPreallocate = 16 << 20 utf8BOMSize = 3 minimumMarkerLength = 2 + legalRoleCount = 2 + percentageScale = 100 encodingUTF8 = "utf-8" encodingUTF16LE = "utf-16le" encodingUTF16BE = "utf-16be" encodingLatin1 = "iso-8859-1" + scannerName = "git-pkgs/licenses" skipReasonBinary = "binary" skipReasonConfiguredDirectory = "configured-directory" @@ -91,6 +96,7 @@ type scanReport struct { Schema int `json:"schema"` Root string `json:"root"` Scope string `json:"scope"` + Scanner scannerRecord `json:"scanner"` Corpus corpusRecord `json:"corpus"` Summary scanSummary `json:"summary"` Declared []declaredRecord `json:"declared"` @@ -100,6 +106,11 @@ type scanReport struct { Errors []scanErrorRecord `json:"errors"` } +type scannerRecord struct { + Name string `json:"name"` + Version string `json:"version"` +} + type corpusRecord struct { Version string `json:"version"` RuleCount int `json:"rule_count"` @@ -126,6 +137,7 @@ type scanSummary struct { type expressionRecord struct { Expression string `json:"expression"` Identification licenses.Identification `json:"identification"` + Root bool `json:"root"` Files int `json:"files"` Matches int `json:"matches"` } @@ -140,7 +152,9 @@ type declaredRecord struct { type fileRecord struct { Path string `json:"path"` Size int64 `json:"size"` + SHA256 string `json:"sha256"` Encoding string `json:"encoding"` + Roles []string `json:"roles"` LicenseTextCoverage float64 `json:"license_text_coverage"` Detections []detectionRecord `json:"detections"` Clues []matchRecord `json:"clues"` @@ -188,6 +202,7 @@ type fileOutcome struct { binary bool tooLarge bool encoding string + sha256 string licenseTextCoverage float64 err error } @@ -219,6 +234,7 @@ func scanRepository( matcher *licenses.Matcher, root string, options scanOptions, + scannerVersion string, ) (scanReport, error) { if matcher == nil { return scanReport{}, errors.New("nil matcher") @@ -241,6 +257,10 @@ func scanRepository( Files: make([]fileRecord, 0), Skipped: make([]skipRecord, 0), Errors: make([]scanErrorRecord, 0), + Scanner: scannerRecord{ + Name: scannerName, + Version: scannerVersion, + }, Corpus: corpusRecord{ Version: corpus.Version, RuleCount: corpus.RuleCount, @@ -289,7 +309,9 @@ func scanRepository( file := makeFileRecord( outcome.task.display, outcome.bytes, + outcome.sha256, outcome.encoding, + legalFileRoles(outcome.task.policyPath), outcome.licenseTextCoverage, outcome.result, ) @@ -693,12 +715,18 @@ func scanFile( applyScanPolicy(task.policyPath, decoded.data, &result) licenseTextCoverage := calculateLicenseTextCoverage(result, len(decoded.data)) remapResultOffsets(&result, decoded) + checksum := "" + if len(result.Detections) != 0 || len(result.Clues) != 0 { + digest := sha256.Sum256(data) + checksum = hex.EncodeToString(digest[:]) + } return fileOutcome{ task: task, result: result, bytes: int64(len(data)), scanned: true, encoding: decoded.encoding, + sha256: checksum, licenseTextCoverage: licenseTextCoverage, } } @@ -865,14 +893,18 @@ func remapResultOffsets(result *licenses.Result, decoded decodedText) { func makeFileRecord( path string, size int64, + checksum string, encoding string, + roles []string, licenseTextCoverage float64, result licenses.Result, ) fileRecord { file := fileRecord{ Path: path, Size: size, + SHA256: checksum, Encoding: encoding, + Roles: roles, LicenseTextCoverage: licenseTextCoverage, } file.Detections = make([]detectionRecord, 0, len(result.Detections)) @@ -945,7 +977,7 @@ func calculateLicenseTextCoverage(result licenses.Result, inputLength int) float current = next } covered += current.end - current.start - return float64(covered) / float64(inputLength) * 100 + return float64(covered) / float64(inputLength) * percentageScale } func makeMatchRecord(match licenses.Match) matchRecord { @@ -1098,16 +1130,23 @@ func isListItem(line []byte) bool { } func isLegalFile(filePath string) bool { + return len(legalFileRoles(filePath)) != 0 +} + +func legalFileRoles(filePath string) []string { cleaned := filepath.ToSlash(filePath) parts := strings.Split(cleaned, "/") + licenseRole := false for _, directory := range parts[:len(parts)-1] { switch strings.ToLower(directory) { case "license", "licenses", "licence", "licences": - return true + licenseRole = true } } name := strings.ToLower(pathpkg.Base(cleaned)) + noticeRole := hasLegalNamePrefix(name, "notices") || + hasLegalNamePrefix(name, "notice") for _, prefix := range []string{ "licenses", "license", @@ -1115,16 +1154,23 @@ func isLegalFile(filePath string) bool { "licence", "copying", "mit-license", - "notices", - "notice", "copyright", "unlicense", } { if hasLegalNamePrefix(name, prefix) { - return true + licenseRole = true + break } } - return false + + roles := make([]string, 0, legalRoleCount) + if licenseRole { + roles = append(roles, "license") + } + if noticeRole { + roles = append(roles, "notice") + } + return roles } func hasLegalNamePrefix(name, prefix string) bool { @@ -1166,6 +1212,7 @@ func compareMatchRecords(first, second matchRecord) int { } func addExpressionRecords(expressions map[string]*expressionRecord, file fileRecord) { + root := isRootExpressionFile(file) for _, detection := range file.Detections { record := expressions[detection.Expression] if record == nil { @@ -1175,11 +1222,22 @@ func addExpressionRecords(expressions map[string]*expressionRecord, file fileRec } expressions[detection.Expression] = record } + record.Root = record.Root || root record.Files++ record.Matches += len(detection.Matches) } } +func isRootExpressionFile(file fileRecord) bool { + return pathDepth(file.Path) == 1 && + (len(file.Roles) != 0 || isReadmeFile(file.Path)) +} + +func isReadmeFile(filePath string) bool { + name := strings.ToLower(pathpkg.Base(filepath.ToSlash(filePath))) + return name == "readme" || strings.HasPrefix(name, "readme.") +} + func addIdentificationSummary( summary *scanSummary, detections []detectionRecord, diff --git a/cmd/licenses/scan_test.go b/cmd/licenses/scan_test.go index d18c64c..aab200a 100644 --- a/cmd/licenses/scan_test.go +++ b/cmd/licenses/scan_test.go @@ -2,7 +2,10 @@ package main import ( "context" + "crypto/sha256" "encoding/binary" + "encoding/hex" + "encoding/json" "os" "path/filepath" "runtime" @@ -15,6 +18,8 @@ import ( "github.com/git-pkgs/magic" ) +const testScannerVersion = "test-version" + func TestScanRepository(t *testing.T) { t.Parallel() @@ -42,6 +47,7 @@ func TestScanRepository(t *testing.T) { Workers: 2, SkipDirs: map[string]bool{"ignored": true}, }, + testScannerVersion, ) if err != nil { t.Fatal(err) @@ -95,6 +101,34 @@ func TestScanRepository(t *testing.T) { if report.Corpus.RuleCount == 0 || report.Corpus.SourceCommit == "" { t.Errorf("corpus = %#v, want populated metadata", report.Corpus) } + if report.Scanner != (scannerRecord{Name: scannerName, Version: testScannerVersion}) { + t.Errorf("scanner = %#v, want name and test version", report.Scanner) + } +} + +func TestScanRepositoryScannerMetadataAllowsEmptyVersion(t *testing.T) { + t.Parallel() + + report, err := scanRepository( + context.Background(), + newTestMatcher(t), + t.TempDir(), + defaultTestScanOptions(), + "", + ) + if err != nil { + t.Fatal(err) + } + if report.Scanner != (scannerRecord{Name: scannerName}) { + t.Errorf("scanner = %#v, want name and empty version", report.Scanner) + } + encoded, err := json.Marshal(report.Scanner) + if err != nil { + t.Fatal(err) + } + if string(encoded) != `{"name":"git-pkgs/licenses","version":""}` { + t.Errorf("scanner JSON = %s", encoded) + } } func TestScanRepositoryDeclaredLicenses(t *testing.T) { @@ -130,6 +164,7 @@ func TestScanRepositoryDeclaredLicenses(t *testing.T) { newTestMatcher(t), root, defaultTestScanOptions(), + testScannerVersion, ) if err != nil { t.Fatal(err) @@ -181,6 +216,7 @@ func TestScanExplicitManifestDeclaredLicense(t *testing.T) { newTestMatcher(t), manifest, defaultTestScanOptions(), + testScannerVersion, ) if err != nil { t.Fatal(err) @@ -202,6 +238,7 @@ func TestScanExplicitMalformedManifestReportsError(t *testing.T) { newTestMatcher(t), manifest, defaultTestScanOptions(), + testScannerVersion, ) if err != nil { t.Fatal(err) @@ -242,6 +279,7 @@ func TestScanRepositorySkipsDetectedBinary(t *testing.T) { newTestMatcher(t), root, defaultTestScanOptions(), + testScannerVersion, ) if err != nil { t.Fatal(err) @@ -306,6 +344,56 @@ func TestIdentificationRecordsAndSummary(t *testing.T) { } } +func TestExpressionRootClassification(t *testing.T) { + t.Parallel() + + expressions := make(map[string]*expressionRecord) + for _, file := range []fileRecord{ + { + Path: "nested/LICENSE", + Roles: []string{"license"}, + Detections: []detectionRecord{{ + Expression: "MIT", + Matches: []matchRecord{{RuleID: "nested.RULE"}}, + }}, + }, + { + Path: "LICENSE", + Roles: []string{"license"}, + Detections: []detectionRecord{{ + Expression: "MIT", + Matches: []matchRecord{{RuleID: "root.RULE"}}, + }}, + }, + { + Path: "README.md", + Detections: []detectionRecord{{ + Expression: "Apache-2.0", + Matches: []matchRecord{{RuleID: "readme.RULE"}}, + }}, + }, + { + Path: "source.go", + Detections: []detectionRecord{{ + Expression: "BSD-2-Clause", + Matches: []matchRecord{{RuleID: "source.RULE"}}, + }}, + }, + } { + addExpressionRecords(expressions, file) + } + + if record := expressions["MIT"]; record == nil || !record.Root || record.Files != 2 { + t.Errorf("MIT expression = %#v, want root with two files", record) + } + if record := expressions["Apache-2.0"]; record == nil || !record.Root { + t.Errorf("Apache expression = %#v, want root README expression", record) + } + if record := expressions["BSD-2-Clause"]; record == nil || record.Root { + t.Errorf("BSD expression = %#v, want ordinary-file expression", record) + } +} + func TestScanRepositoryFollowsExplicitRootSymlinks(t *testing.T) { t.Parallel() @@ -323,7 +411,13 @@ func TestScanRepositoryFollowsExplicitRootSymlinks(t *testing.T) { options := defaultTestScanOptions() for _, path := range []string{directoryLink, fileLink} { - report, err := scanRepository(context.Background(), newTestMatcher(t), path, options) + report, err := scanRepository( + context.Background(), + newTestMatcher(t), + path, + options, + testScannerVersion, + ) if err != nil { t.Fatalf("scan %s: %v", path, err) } @@ -352,6 +446,7 @@ func TestScanRepositorySkipsTreeSymlinks(t *testing.T) { newTestMatcher(t), root, defaultTestScanOptions(), + testScannerVersion, ) if err != nil { t.Fatal(err) @@ -374,7 +469,13 @@ func TestScanRepositoryAllScopeIncludesDefaultSkips(t *testing.T) { options := defaultTestScanOptions() options.NoDefaultSkip = true - report, err := scanRepository(context.Background(), newTestMatcher(t), root, options) + report, err := scanRepository( + context.Background(), + newTestMatcher(t), + root, + options, + testScannerVersion, + ) if err != nil { t.Fatal(err) } @@ -479,6 +580,7 @@ func TestScanRepositoryDecodesLicenseText(t *testing.T) { matcher, path, defaultTestScanOptions(), + testScannerVersion, ) if err != nil { t.Fatal(err) @@ -490,6 +592,11 @@ func TestScanRepositoryDecodesLicenseText(t *testing.T) { if file.Encoding != test.encoding { t.Errorf("encoding = %q, want %q", file.Encoding, test.encoding) } + digest := sha256.Sum256(test.data) + wantSHA256 := hex.EncodeToString(digest[:]) + if file.SHA256 != wantSHA256 { + t.Errorf("sha256 = %q, want %q", file.SHA256, wantSHA256) + } match, ok := findRecordMatch(file, "mit.LICENSE") if !ok { t.Fatalf("detections = %#v, want mit.LICENSE", file.Detections) @@ -537,6 +644,7 @@ func TestScanRepositoryFallsBackToLatin1ForMalformedUTF16(t *testing.T) { newTestMatcher(t), path, defaultTestScanOptions(), + testScannerVersion, ) if err != nil { t.Fatal(err) @@ -609,6 +717,7 @@ func TestScanRepositoryDemotesReferenceAcrossMarkdownBlocks(t *testing.T) { matcher, root, defaultTestScanOptions(), + testScannerVersion, ) if err != nil { t.Fatal(err) @@ -654,6 +763,7 @@ func TestScanRepositoryDemotesReferenceAcrossMarkdownTableRows(t *testing.T) { newTestMatcher(t), root, defaultTestScanOptions(), + testScannerVersion, ) if err != nil { t.Fatal(err) @@ -696,6 +806,7 @@ func TestScanRepositoryKeepsRubyAlternativeWithinMarkdownBlock(t *testing.T) { newTestMatcher(t), root, defaultTestScanOptions(), + testScannerVersion, ) if err != nil { t.Fatal(err) @@ -903,6 +1014,7 @@ func TestScanRepositoryKeepsReferenceInExplicitLicensesFile(t *testing.T) { newTestMatcher(t), filePath, defaultTestScanOptions(), + testScannerVersion, ) if err != nil { t.Fatal(err) @@ -943,6 +1055,7 @@ func TestScanPolicyUsesDecodedText(t *testing.T) { newTestMatcher(t), filePath, defaultTestScanOptions(), + testScannerVersion, ) if err != nil { t.Fatal(err) @@ -977,6 +1090,31 @@ func TestLegalFileNames(t *testing.T) { } } +func TestLegalFileRoles(t *testing.T) { + t.Parallel() + + tests := []struct { + path string + want []string + }{ + {path: "LICENSE", want: []string{"license"}}, + {path: "copying.txt", want: []string{"license"}}, + {path: "NOTICES.md", want: []string{"notice"}}, + {path: "licenses/component.txt", want: []string{"license"}}, + {path: "LICENSES/NOTICE.txt", want: []string{"license", "notice"}}, + {path: "src/source.go", want: []string{}}, + } + for _, test := range tests { + got := legalFileRoles(test.path) + if !slices.Equal(got, test.want) { + t.Errorf("legalFileRoles(%q) = %#v, want %#v", test.path, got, test.want) + } + if got == nil { + t.Errorf("legalFileRoles(%q) returned nil, want an empty or populated array", test.path) + } + } +} + func TestScanFileEnforcesSizeAfterDiscovery(t *testing.T) { t.Parallel() @@ -1025,6 +1163,7 @@ func TestScanRepositoryRecordsUnreadableFile(t *testing.T) { newTestMatcher(t), root, defaultTestScanOptions(), + testScannerVersion, ) if err != nil { t.Fatal(err) @@ -1050,6 +1189,7 @@ func TestScanRepositoryRecordsCandidateLimit(t *testing.T) { newTestMatcher(t), path, options, + testScannerVersion, ) if err != nil { t.Fatal(err) @@ -1156,7 +1296,13 @@ func TestScanRepositoryCancelled(t *testing.T) { MaxFileSize: defaultMaxFileSize, Workers: 1, } - _, err := scanRepository(ctx, newTestMatcher(t), "../../LICENSE", options) + _, err := scanRepository( + ctx, + newTestMatcher(t), + "../../LICENSE", + options, + testScannerVersion, + ) if err != context.Canceled { t.Fatalf("error = %v, want context.Canceled", err) } From e7ac0a9d17097b71bd7a8b2aee8fb1c60a92a322 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Fri, 21 Aug 2026 09:19:18 +0100 Subject: [PATCH 2/2] Skip unreported file record construction --- cmd/licenses/scan.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/licenses/scan.go b/cmd/licenses/scan.go index 895ca7f..c5126f9 100644 --- a/cmd/licenses/scan.go +++ b/cmd/licenses/scan.go @@ -306,6 +306,9 @@ func scanRepository( Reason: skipReasonSize, }) case outcome.scanned: + if len(outcome.result.Detections) == 0 && len(outcome.result.Clues) == 0 { + continue + } file := makeFileRecord( outcome.task.display, outcome.bytes, @@ -323,9 +326,6 @@ func scanRepository( if len(file.Clues) != 0 { report.Summary.FilesWithClues++ } - if len(file.Detections) == 0 && len(file.Clues) == 0 { - continue - } report.Files = append(report.Files, file) addExpressionRecords(expressions, file) }