From e9403fdb220791dbe47dde97abbd2e986ec13238 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Fri, 21 Aug 2026 10:11:04 +0100 Subject: [PATCH 1/3] Expose repository scanner API --- README.md | 21 +- cmd/licenses/licensee_comparison_test.go | 2 +- cmd/licenses/scan.go | 1259 +------------------ cmd/licenses/scan_test_helpers_test.go | 57 + corpus.go | 6 +- scan.go | 1330 +++++++++++++++++++++ scan_api_test.go | 124 ++ cmd/licenses/scan_test.go => scan_test.go | 78 +- 8 files changed, 1593 insertions(+), 1284 deletions(-) create mode 100644 cmd/licenses/scan_test_helpers_test.go create mode 100644 scan.go create mode 100644 scan_api_test.go rename cmd/licenses/scan_test.go => scan_test.go (95%) diff --git a/README.md b/README.md index d1dbef7..98beae7 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ # licenses -Go library for matching license text against ScanCode's license rule corpus. +Go library and command for matching license text and scanning repositories +against ScanCode's license rule corpus. The corpus is embedded in the package. Matching needs no network access, cgo, or Python. @@ -117,6 +118,24 @@ for _, detection := range result.Detections { } ``` +Scan a file or directory with the same matcher: + +```go +options := licenses.DefaultScanOptions() +options.IncludeLegalFiles = true +report, err := licenses.ScanRepository(ctx, matcher, ".", options) +if err != nil { + return err +} + +for _, file := range report.Files { + fmt.Println(file.Path, file.Detections) +} +``` + +`IncludeLegalFiles` retains recognized license and notice files when the corpus +does not produce a match. + Matching uses normalized whole-text hashes, exact token sequences, and `SPDX-License-Identifier` tag lines. It does not use fuzzy or sequence matching, so edits within a license text can prevent a match. diff --git a/cmd/licenses/licensee_comparison_test.go b/cmd/licenses/licensee_comparison_test.go index 2f77781..8840c6b 100644 --- a/cmd/licenses/licensee_comparison_test.go +++ b/cmd/licenses/licensee_comparison_test.go @@ -199,7 +199,7 @@ func isComparisonProjectFile(filePath string) bool { if strings.Contains(cleaned, "/") { return false } - if isLegalFile(filePath) { + if len(licenses.LegalFileRoles(filePath)) != 0 { return true } name := strings.ToLower(pathpkg.Base(cleaned)) diff --git a/cmd/licenses/scan.go b/cmd/licenses/scan.go index c5126f9..045ca6c 100644 --- a/cmd/licenses/scan.go +++ b/cmd/licenses/scan.go @@ -1,232 +1,35 @@ package main import ( - "bytes" - "cmp" "context" - "crypto/sha256" - "encoding/binary" - "encoding/hex" - "errors" "fmt" - "io" - "os" - pathpkg "path" - "path/filepath" - "runtime" - "slices" - "strings" - "sync" - "unicode/utf16" - "unicode/utf8" licenses "github.com/git-pkgs/licenses" - "github.com/git-pkgs/magic" - "github.com/git-pkgs/manifests" - "github.com/git-pkgs/spdx" ) const ( - reportSchemaVersion = 2 - defaultMaxDepth = 32 - defaultMaxFiles = 10_000 - defaultMaxFileSize = 1 << 20 - classificationProbeSize = 8 << 10 - maxWorkers = 16 - maxReadPreallocate = 16 << 20 - utf8BOMSize = 3 - minimumMarkerLength = 2 - legalRoleCount = 2 - percentageScale = 100 - encodingUTF8 = "utf-8" - encodingUTF16LE = "utf-16le" - encodingUTF16BE = "utf-16be" - encodingLatin1 = "iso-8859-1" - scannerName = "git-pkgs/licenses" - - skipReasonBinary = "binary" - skipReasonConfiguredDirectory = "configured-directory" - skipReasonDepth = "depth" - skipReasonHiddenDirectory = "hidden-directory" - skipReasonNonRegular = "non-regular" - skipReasonProjectScope = "project-scope" - skipReasonSize = "size" - skipReasonSymlink = "symlink" - skipReasonVersionControl = "version-control" - scopeAll = "all" - scopeProject = "project" + reportSchemaVersion = licenses.ReportSchemaVersion + defaultMaxDepth = licenses.DefaultMaxDepth + defaultMaxFiles = licenses.DefaultMaxFiles + defaultMaxFileSize = licenses.DefaultMaxFileSize + scannerName = licenses.ScannerName + scopeAll = licenses.ScopeAll + scopeProject = licenses.ScopeProject ) -var errFileLimit = errors.New("file limit reached") - -var defaultSkippedDirectories = map[string]bool{ - "vendor": true, - "node_modules": true, - "__pycache__": true, - ".bundle": true, - ".venv": true, - "venv": true, - "target": true, - "build": true, - "dist": true, - "out": true, - "_build": true, - "deps": true, - "Pods": true, - "third_party": true, - "thirdparty": true, - "external": true, - "testdata": true, - "tmp": true, - "temp": true, - "cache": true, - "coverage": true, -} - -type scanOptions struct { - MaxDepth int - MaxFiles int - MaxFileSize int64 - Workers int - SkipDirs map[string]bool - NoDefaultSkip bool -} - -type scanReport struct { - Schema int `json:"schema"` - Root string `json:"root"` - Scope string `json:"scope"` - Scanner scannerRecord `json:"scanner"` - Corpus corpusRecord `json:"corpus"` - Summary scanSummary `json:"summary"` - Declared []declaredRecord `json:"declared"` - Expressions []expressionRecord `json:"expressions"` - Files []fileRecord `json:"files"` - Skipped []skipRecord `json:"skipped"` - Errors []scanErrorRecord `json:"errors"` -} - -type scannerRecord struct { - Name string `json:"name"` - Version string `json:"version"` -} - -type corpusRecord struct { - Version string `json:"version"` - RuleCount int `json:"rule_count"` - SourceCommit string `json:"source_commit"` -} - -type scanSummary struct { - FilesVisited int `json:"files_visited"` - FilesScanned int `json:"files_scanned"` - FilesWithDetections int `json:"files_with_detections"` - FilesWithIdentifiedDetections int `json:"files_with_identified_detections"` - FilesWithPartialDetections int `json:"files_with_partial_detections"` - FilesWithNoAssertionDetections int `json:"files_with_noassertion_detections"` - FilesWithClues int `json:"files_with_clues"` - BytesScanned int64 `json:"bytes_scanned"` - DirectoriesSkipped int `json:"directories_skipped"` - FilesSkippedBinary int `json:"files_skipped_binary"` - FilesSkippedSize int `json:"files_skipped_size"` - FilesSkippedOther int `json:"files_skipped_other"` - ErrorCount int `json:"error_count"` - Truncated bool `json:"truncated"` -} - -type expressionRecord struct { - Expression string `json:"expression"` - Identification licenses.Identification `json:"identification"` - Root bool `json:"root"` - Files int `json:"files"` - Matches int `json:"matches"` -} - -type declaredRecord struct { - Path string `json:"path"` - Raw []string `json:"raw"` - LicenseFile string `json:"license_file"` - NormalizedExpression string `json:"normalized_expression"` -} - -type fileRecord struct { - Path string `json:"path"` - Size int64 `json:"size"` - SHA256 string `json:"sha256"` - Encoding string `json:"encoding"` - Roles []string `json:"roles"` - LicenseTextCoverage float64 `json:"license_text_coverage"` - Detections []detectionRecord `json:"detections"` - Clues []matchRecord `json:"clues"` -} - -type detectionRecord struct { - Expression string `json:"expression"` - Identification licenses.Identification `json:"identification"` - Matches []matchRecord `json:"matches"` -} - -type matchRecord struct { - RuleID string `json:"rule_id"` - LicenseIDs []string `json:"license_ids,omitempty"` - Kind licenses.Kind `json:"kind"` - Method licenses.Method `json:"method"` - Score float64 `json:"score"` - Coverage float64 `json:"coverage"` - Start int `json:"start"` - End int `json:"end"` - Matched string `json:"matched,omitempty"` -} - -type scanErrorRecord struct { - Path string `json:"path"` - Error string `json:"error"` -} - -type skipRecord struct { - Path string `json:"path"` - Reason string `json:"reason"` -} - -type fileTask struct { - path string - display string - policyPath string -} - -type fileOutcome struct { - task fileTask - result licenses.Result - bytes int64 - scanned bool - binary bool - tooLarge bool - encoding string - sha256 string - licenseTextCoverage float64 - err error -} - -type decodedText struct { - data []byte - offsets []int - offsetBase int - encoding string -} - -type fileDiscovery struct { - ctx context.Context - root string - options scanOptions - summary *scanSummary - tasks []fileTask - declared []declaredRecord - scanErrors []scanErrorRecord - skipped []skipRecord -} +type scanOptions = licenses.ScanOptions +type scanReport = licenses.ScanReport +type scannerRecord = licenses.ScannerRecord +type scanSummary = licenses.ScanSummary +type expressionRecord = licenses.ExpressionRecord +type fileRecord = licenses.FileRecord +type detectionRecord = licenses.DetectionRecord +type matchRecord = licenses.MatchRecord +type scanErrorRecord = licenses.ScanErrorRecord +type skipRecord = licenses.SkipRecord func defaultWorkerCount() int { - return min(runtime.GOMAXPROCS(0), maxWorkers) + return licenses.DefaultScanOptions().Workers } func scanRepository( @@ -236,1032 +39,12 @@ func scanRepository( options scanOptions, scannerVersion string, ) (scanReport, error) { - if matcher == nil { - return scanReport{}, errors.New("nil matcher") - } - if ctx == nil { - return scanReport{}, errors.New("nil context") - } - if err := ctx.Err(); err != nil { - return scanReport{}, err - } - if err := validateScanOptions(options); err != nil { - return scanReport{}, err - } - corpus := matcher.Corpus() - report := scanReport{ - Schema: reportSchemaVersion, - Root: filepath.Clean(root), - Scope: scanScope(options), - Declared: make([]declaredRecord, 0), - Files: make([]fileRecord, 0), - Skipped: make([]skipRecord, 0), - Errors: make([]scanErrorRecord, 0), - Scanner: scannerRecord{ - Name: scannerName, - Version: scannerVersion, - }, - Corpus: corpusRecord{ - Version: corpus.Version, - RuleCount: corpus.RuleCount, - SourceCommit: corpus.SourceCommit, - }, - } - discovery, err := discoverFiles( - ctx, - root, - options, - &report.Summary, - ) - if err != nil { - return scanReport{}, err - } - report.Declared = append(report.Declared, discovery.declared...) - report.Errors = append(report.Errors, discovery.scanErrors...) - report.Skipped = append(report.Skipped, discovery.skipped...) - - outcomes := scanFiles(ctx, matcher, discovery.tasks, options) - expressions := make(map[string]*expressionRecord) - for outcome := range outcomes { - if outcome.scanned { - report.Summary.FilesScanned++ - report.Summary.BytesScanned += outcome.bytes - } - switch { - case outcome.err != nil: - report.Errors = append(report.Errors, scanErrorRecord{ - Path: outcome.task.display, - Error: outcome.err.Error(), - }) - case outcome.binary: - report.Summary.FilesSkippedBinary++ - report.Skipped = append(report.Skipped, skipRecord{ - Path: outcome.task.display, - Reason: skipReasonBinary, - }) - case outcome.tooLarge: - report.Summary.FilesSkippedSize++ - report.Skipped = append(report.Skipped, skipRecord{ - Path: outcome.task.display, - Reason: skipReasonSize, - }) - case outcome.scanned: - if len(outcome.result.Detections) == 0 && len(outcome.result.Clues) == 0 { - continue - } - file := makeFileRecord( - outcome.task.display, - outcome.bytes, - outcome.sha256, - outcome.encoding, - legalFileRoles(outcome.task.policyPath), - outcome.licenseTextCoverage, - outcome.result, - ) - sortFileMatches(&file) - if len(file.Detections) != 0 { - report.Summary.FilesWithDetections++ - addIdentificationSummary(&report.Summary, file.Detections) - } - if len(file.Clues) != 0 { - report.Summary.FilesWithClues++ - } - report.Files = append(report.Files, file) - addExpressionRecords(expressions, file) - } - } - if err := ctx.Err(); err != nil { - return scanReport{}, err - } - - slices.SortFunc(report.Files, func(first, second fileRecord) int { - return strings.Compare(first.Path, second.Path) - }) - slices.SortFunc(report.Declared, func(first, second declaredRecord) int { - return strings.Compare(first.Path, second.Path) - }) - slices.SortFunc(report.Errors, func(first, second scanErrorRecord) int { - if compared := strings.Compare(first.Path, second.Path); compared != 0 { - return compared - } - return strings.Compare(first.Error, second.Error) - }) - slices.SortFunc(report.Skipped, func(first, second skipRecord) int { - if compared := strings.Compare(first.Path, second.Path); compared != 0 { - return compared - } - return strings.Compare(first.Reason, second.Reason) - }) - report.Expressions = make([]expressionRecord, 0, len(expressions)) - for _, expression := range expressions { - report.Expressions = append(report.Expressions, *expression) - } - slices.SortFunc(report.Expressions, func(first, second expressionRecord) int { - return strings.Compare(first.Expression, second.Expression) - }) - report.Summary.ErrorCount = len(report.Errors) - return report, nil -} - -func scanScope(options scanOptions) string { - if options.NoDefaultSkip { - return scopeAll - } - return scopeProject + options.ScannerVersion = scannerVersion + return licenses.ScanRepository(ctx, matcher, root, options) } func validateScanOptions(options scanOptions) error { - switch { - case options.MaxDepth < 0: - return errors.New("max depth must not be negative") - case options.MaxFiles < 0: - return errors.New("max files must not be negative") - case options.MaxFileSize < 0: - return errors.New("max file size must not be negative") - case options.Workers < 1: - return errors.New("workers must be positive") - default: - return nil - } -} - -func discoverFiles( - ctx context.Context, - root string, - options scanOptions, - summary *scanSummary, -) (*fileDiscovery, error) { - if err := ctx.Err(); err != nil { - return nil, err - } - discovery := &fileDiscovery{ - ctx: ctx, - root: root, - options: options, - summary: summary, - } - info, err := os.Stat(root) - if err != nil { - return nil, err - } - if !info.IsDir() { - discovery.discoverExplicitFile(root, info) - return discovery, nil - } - walkRoot, err := filepath.EvalSymlinks(root) - if err != nil { - return nil, err - } - discovery.root = walkRoot - err = filepath.WalkDir(walkRoot, discovery.visit) - if err != nil && !errors.Is(err, errFileLimit) { - return nil, err - } - return discovery, nil -} - -func (discovery *fileDiscovery) discoverExplicitFile( - path string, - info os.FileInfo, -) { - discovery.summary.FilesVisited = 1 - if !info.Mode().IsRegular() { - discovery.summary.FilesSkippedOther = 1 - discovery.skipped = append(discovery.skipped, skipRecord{ - Path: filepath.Base(path), - Reason: skipReasonNonRegular, - }) - return - } - if discovery.options.MaxFileSize > 0 && - info.Size() > discovery.options.MaxFileSize { - discovery.summary.FilesSkippedSize = 1 - discovery.skipped = append(discovery.skipped, skipRecord{ - Path: filepath.Base(path), - Reason: skipReasonSize, - }) - return - } - display := filepath.Base(path) - discovery.tasks = append(discovery.tasks, fileTask{ - path: path, - display: display, - policyPath: explicitPolicyPath(path), - }) - discovery.discoverDeclaredLicense(path, display) -} - -func explicitPolicyPath(filePath string) string { - name := filepath.Base(filePath) - directory := filepath.Base(filepath.Dir(filepath.Clean(filePath))) - if directory == "." || directory == string(filepath.Separator) { - return name - } - return filepath.ToSlash(filepath.Join(directory, name)) -} - -func (discovery *fileDiscovery) visit( - path string, - entry os.DirEntry, - walkErr error, -) error { - if err := discovery.ctx.Err(); err != nil { - return err - } - relative := relativePath(discovery.root, path) - if walkErr != nil { - discovery.scanErrors = append(discovery.scanErrors, scanErrorRecord{ - Path: relative, - Error: walkErr.Error(), - }) - if entry != nil && entry.IsDir() { - return filepath.SkipDir - } - return nil - } - if entry.IsDir() { - return discovery.visitDirectory(relative, entry.Name()) - } - return discovery.visitFile(path, relative, entry) -} - -func (discovery *fileDiscovery) visitDirectory(relative, name string) error { - if relative == "." { - return nil - } - reason := skippedDirectoryReason(name, discovery.options) - if discovery.options.MaxDepth > 0 && - pathDepth(relative) > discovery.options.MaxDepth { - reason = skipReasonDepth - } - if reason != "" { - discovery.summary.DirectoriesSkipped++ - discovery.skipped = append(discovery.skipped, skipRecord{ - Path: relative, - Reason: reason, - }) - return filepath.SkipDir - } - return nil -} - -func (discovery *fileDiscovery) visitFile( - path string, - relative string, - entry os.DirEntry, -) error { - if discovery.options.MaxFiles > 0 && - discovery.summary.FilesVisited >= discovery.options.MaxFiles { - discovery.summary.Truncated = true - return errFileLimit - } - discovery.summary.FilesVisited++ - if entry.Type()&os.ModeSymlink != 0 { - discovery.summary.FilesSkippedOther++ - discovery.skipped = append(discovery.skipped, skipRecord{ - Path: relative, - Reason: skipReasonSymlink, - }) - return nil - } - fileInfo, err := entry.Info() - if err != nil { - discovery.scanErrors = append(discovery.scanErrors, scanErrorRecord{ - Path: relative, - Error: err.Error(), - }) - return nil - } - if !fileInfo.Mode().IsRegular() { - discovery.summary.FilesSkippedOther++ - discovery.skipped = append(discovery.skipped, skipRecord{ - Path: relative, - Reason: skipReasonNonRegular, - }) - return nil - } - if discovery.options.MaxFileSize > 0 && - fileInfo.Size() > discovery.options.MaxFileSize { - discovery.summary.FilesSkippedSize++ - discovery.skipped = append(discovery.skipped, skipRecord{ - Path: relative, - Reason: skipReasonSize, - }) - return nil - } - discovery.tasks = append(discovery.tasks, fileTask{ - path: path, - display: filepath.ToSlash(relative), - policyPath: filepath.ToSlash(relative), - }) - discovery.discoverDeclaredLicense(path, filepath.ToSlash(relative)) - return nil -} - -func (discovery *fileDiscovery) discoverDeclaredLicense(path, display string) { - record, ok, err := declaredLicense(path, display) - if err != nil { - discovery.scanErrors = append(discovery.scanErrors, scanErrorRecord{ - Path: display, - Error: err.Error(), - }) - return - } - if ok { - discovery.declared = append(discovery.declared, record) - } -} - -func declaredLicense(path, display string) (declaredRecord, bool, error) { - if _, kind, ok := manifests.Identify(display); !ok || kind != manifests.Manifest { - return declaredRecord{}, false, nil - } - content, err := os.ReadFile(path) - if err != nil { - return declaredRecord{}, false, err - } - result, err := manifests.Parse(display, content) - if err != nil { - return declaredRecord{}, false, err - } - if result == nil || - (len(result.Licenses) == 0 && result.LicenseFile == "") { - return declaredRecord{}, false, nil - } - return declaredRecord{ - Path: display, - Raw: append([]string{}, result.Licenses...), - LicenseFile: result.LicenseFile, - NormalizedExpression: normalizeDeclaredExpression(result.Licenses), - }, true, nil -} - -func normalizeDeclaredExpression(raw []string) string { - if len(raw) == 0 { - return "" - } - normalized, err := spdx.NormalizeExpressionLax(strings.Join(raw, " OR ")) - if err != nil { - return "" - } - return normalized -} - -func skippedDirectoryReason(name string, options scanOptions) string { - if name == ".git" { - return skipReasonVersionControl - } - if options.SkipDirs[name] { - return skipReasonConfiguredDirectory - } - if options.NoDefaultSkip { - return "" - } - if strings.HasPrefix(name, ".") { - return skipReasonHiddenDirectory - } - if defaultSkippedDirectories[name] { - return skipReasonProjectScope - } - return "" -} - -func relativePath(root, path string) string { - relative, err := filepath.Rel(root, path) - if err != nil { - return filepath.ToSlash(path) - } - return filepath.ToSlash(relative) -} - -func pathDepth(path string) int { - if path == "." || path == "" { - return 0 - } - return strings.Count(filepath.ToSlash(filepath.Clean(path)), "/") + 1 -} - -func scanFiles( - ctx context.Context, - matcher *licenses.Matcher, - tasks []fileTask, - options scanOptions, -) <-chan fileOutcome { - outcomes := make(chan fileOutcome) - if len(tasks) == 0 { - close(outcomes) - return outcomes - } - jobs := make(chan fileTask, len(tasks)) - for _, task := range tasks { - jobs <- task - } - close(jobs) - - var workers sync.WaitGroup - workerCount := effectiveWorkerCount(options.Workers, len(tasks)) - for range workerCount { - workers.Add(1) - go func() { - defer workers.Done() - for task := range jobs { - outcome := scanFile(ctx, matcher, task, options.MaxFileSize) - select { - case outcomes <- outcome: - case <-ctx.Done(): - return - } - } - }() - } - go func() { - workers.Wait() - close(outcomes) - }() - return outcomes -} - -func effectiveWorkerCount(requested, taskCount int) int { - return min(requested, maxWorkers, taskCount) -} - -func scanFile( - ctx context.Context, - matcher *licenses.Matcher, - task fileTask, - maxFileSize int64, -) fileOutcome { - data, detection, tooLarge, err := readScannableFile(task.path, maxFileSize) - if err != nil { - return fileOutcome{task: task, err: err} - } - if tooLarge { - return fileOutcome{task: task, tooLarge: true} - } - if detection.Kind == magic.KindBinary { - return fileOutcome{task: task, binary: true} - } - decoded := decodeText(data, detection) - result, err := matcher.Match(ctx, decoded.data) - if err != nil { - return fileOutcome{ - task: task, - bytes: int64(len(data)), - scanned: true, - encoding: decoded.encoding, - err: err, - } - } - applyScanPolicy(task.policyPath, decoded.data, &result) - licenseTextCoverage := calculateLicenseTextCoverage(result, len(decoded.data)) - remapResultOffsets(&result, decoded) - checksum := "" - if len(result.Detections) != 0 || len(result.Clues) != 0 { - digest := sha256.Sum256(data) - checksum = hex.EncodeToString(digest[:]) - } - return fileOutcome{ - task: task, - result: result, - bytes: int64(len(data)), - scanned: true, - encoding: decoded.encoding, - sha256: checksum, - licenseTextCoverage: licenseTextCoverage, - } -} - -func readScannableFile(path string, maximum int64) ([]byte, magic.Result, bool, error) { - file, err := os.Open(path) - if err != nil { - return nil, magic.Result{}, false, err - } - defer func() { _ = file.Close() }() - - var data bytes.Buffer - probeLimit := int64(classificationProbeSize) - if maximum > 0 { - probeLimit = min(probeLimit, maximum+1) - } - _, err = io.CopyN(&data, file, probeLimit) - if err != nil && !errors.Is(err, io.EOF) { - return nil, magic.Result{}, false, err - } - if detection := magic.DetectPrefix(data.Bytes()); detection.Kind == magic.KindBinary { - return nil, detection, false, nil - } - if maximum > 0 && int64(data.Len()) > maximum { - return nil, magic.Result{}, true, nil - } - growReadBuffer(file, &data, maximum) - reader := io.Reader(file) - if maximum > 0 { - reader = io.LimitReader(file, maximum-int64(data.Len())+1) - } - if _, err := io.Copy(&data, reader); err != nil { - return nil, magic.Result{}, false, err - } - if maximum > 0 && int64(data.Len()) > maximum { - return nil, magic.Result{}, true, nil - } - content := data.Bytes() - return content, magic.Detect(content), false, nil -} - -func growReadBuffer(file *os.File, data *bytes.Buffer, maximum int64) { - info, err := file.Stat() - if err != nil { - return - } - size := info.Size() - if maximum > 0 { - size = min(size, maximum+1) - } - if size <= int64(data.Len()) || size > maxReadPreallocate { - return - } - data.Grow(int(size) - data.Len()) -} - -func decodeText(data []byte, detection magic.Result) decodedText { - switch detection.Encoding { - case encodingUTF8: - if !bytes.HasPrefix(data, []byte{0xef, 0xbb, 0xbf}) { - return decodedText{data: data, encoding: encodingUTF8} - } - return decodedText{ - data: data[utf8BOMSize:], - offsetBase: utf8BOMSize, - encoding: encodingUTF8, - } - case encodingUTF16LE: - return decodeUTF16(data, binary.LittleEndian, encodingUTF16LE) - case encodingUTF16BE: - return decodeUTF16(data, binary.BigEndian, encodingUTF16BE) - } - if detection.Kind == magic.KindUnknown && - detection.Reason == magic.ReasonInvalidText { - return decodeLatin1(data) - } - return decodedText{data: data, encoding: encodingUTF8} -} - -func decodeUTF16(data []byte, order binary.ByteOrder, name string) decodedText { - decoded := decodedText{ - data: make([]byte, 0, len(data)), - offsets: []int{2}, - encoding: name, - } - for position := 2; position < len(data); { - start := position - var character rune - if position+1 >= len(data) { - character = utf8.RuneError - position++ - } else { - first := order.Uint16(data[position : position+2]) - position += 2 - character = rune(first) - if utf16.IsSurrogate(character) { - if position+1 < len(data) { - second := rune(order.Uint16(data[position : position+2])) - if decodedRune := utf16.DecodeRune(character, second); decodedRune != utf8.RuneError { - character = decodedRune - position += 2 - } else { - character = utf8.RuneError - } - } else { - character = utf8.RuneError - } - } - } - decoded.appendRune(character, start, position) - } - return decoded -} - -func decodeLatin1(data []byte) decodedText { - decoded := decodedText{ - data: make([]byte, 0, len(data)), - offsets: []int{0}, - encoding: encodingLatin1, - } - for position, value := range data { - decoded.appendRune(rune(value), position, position+1) - } - return decoded -} - -func (decoded *decodedText) appendRune(character rune, rawStart, rawEnd int) { - start := len(decoded.data) - decoded.data = utf8.AppendRune(decoded.data, character) - for position := start; position < len(decoded.data); position++ { - if position == len(decoded.data)-1 { - decoded.offsets = append(decoded.offsets, rawEnd) - } else { - decoded.offsets = append(decoded.offsets, rawStart) - } - } -} - -func (decoded decodedText) rawOffset(offset int) int { - if decoded.offsets != nil { - return decoded.offsets[offset] - } - return offset + decoded.offsetBase -} - -func remapResultOffsets(result *licenses.Result, decoded decodedText) { - if decoded.offsets == nil && decoded.offsetBase == 0 { - return - } - for detectionIndex := range result.Detections { - for matchIndex := range result.Detections[detectionIndex].Matches { - match := &result.Detections[detectionIndex].Matches[matchIndex] - match.Start = decoded.rawOffset(match.Start) - match.End = decoded.rawOffset(match.End) - } - } - for matchIndex := range result.Clues { - match := &result.Clues[matchIndex] - match.Start = decoded.rawOffset(match.Start) - match.End = decoded.rawOffset(match.End) - } -} - -func makeFileRecord( - path string, - size int64, - checksum string, - encoding string, - roles []string, - licenseTextCoverage float64, - result licenses.Result, -) fileRecord { - file := fileRecord{ - Path: path, - Size: size, - SHA256: checksum, - Encoding: encoding, - Roles: roles, - LicenseTextCoverage: licenseTextCoverage, - } - file.Detections = make([]detectionRecord, 0, len(result.Detections)) - for _, detection := range result.Detections { - record := detectionRecord{ - Expression: detection.Expression, - Identification: detection.Identification, - Matches: make([]matchRecord, 0, len(detection.Matches)), - } - for _, match := range detection.Matches { - record.Matches = append(record.Matches, makeMatchRecord(match)) - } - file.Detections = append(file.Detections, record) - } - file.Clues = make([]matchRecord, 0, len(result.Clues)) - for _, clue := range result.Clues { - file.Clues = append(file.Clues, makeMatchRecord(clue)) - } - return file -} - -type byteRange struct { - start int - end int -} - -func calculateLicenseTextCoverage(result licenses.Result, inputLength int) float64 { - if inputLength == 0 { - return 0 - } - - ranges := make([]byteRange, 0) - addMatch := func(match licenses.Match) { - if match.Kind != licenses.KindText && match.Kind != licenses.KindNotice { - return - } - start := max(0, min(match.Start, inputLength)) - end := max(0, min(match.End, inputLength)) - if start >= end { - return - } - ranges = append(ranges, byteRange{start: start, end: end}) - } - for _, detection := range result.Detections { - for _, match := range detection.Matches { - addMatch(match) - } - } - for _, clue := range result.Clues { - addMatch(clue) - } - if len(ranges) == 0 { - return 0 - } - - slices.SortFunc(ranges, func(first, second byteRange) int { - if compared := cmp.Compare(first.start, second.start); compared != 0 { - return compared - } - return cmp.Compare(first.end, second.end) - }) - covered := 0 - current := ranges[0] - for _, next := range ranges[1:] { - if next.start <= current.end { - current.end = max(current.end, next.end) - continue - } - covered += current.end - current.start - current = next - } - covered += current.end - current.start - return float64(covered) / float64(inputLength) * percentageScale -} - -func makeMatchRecord(match licenses.Match) matchRecord { - return matchRecord{ - RuleID: match.RuleID, - LicenseIDs: match.LicenseIDs, - Kind: match.Kind, - Method: match.Method, - Score: match.Score, - Coverage: match.Coverage, - Start: match.Start, - End: match.End, - Matched: string(match.Matched), - } -} - -func applyScanPolicy(path string, input []byte, result *licenses.Result) { - if isLegalFile(path) { - return - } - - documentMarkers := usesDocumentMarkers(path) - detections := result.Detections[:0] - for _, detection := range result.Detections { - matches := detection.Matches[:0] - for _, match := range detection.Matches { - if match.Kind == licenses.KindReference && - crossesBlockBoundary( - input, - match.Start, - match.End, - documentMarkers, - ) { - result.Clues = append(result.Clues, match) - continue - } - matches = append(matches, match) - } - if len(matches) == 0 { - continue - } - detection.Matches = matches - detections = append(detections, detection) - } - result.Detections = detections -} - -func crossesBlockBoundary( - input []byte, - start int, - end int, - documentMarkers bool, -) bool { - if start < 0 || start >= end || end > len(input) { - return false - } - for searchStart := start; searchStart < end; { - relative := bytes.IndexByte(input[searchStart:end], '\n') - if relative < 0 { - return false - } - newline := searchStart + relative - leftStart := bytes.LastIndexByte(input[:newline], '\n') + 1 - rightEnd := len(input) - if next := bytes.IndexByte(input[newline+1:], '\n'); next >= 0 { - rightEnd = newline + 1 + next - } - left := bytes.TrimSpace(input[leftStart:newline]) - right := bytes.TrimSpace(input[newline+1 : rightEnd]) - paragraphLeft, paragraphRight := stripCommonCommentLeader(left, right) - if len(paragraphLeft) == 0 || len(paragraphRight) == 0 { - return true - } - if documentMarkers && isDocumentBoundary(left, right) { - return true - } - searchStart = newline + 1 - } - return false -} - -func stripCommonCommentLeader(left, right []byte) ([]byte, []byte) { - for _, leader := range [][]byte{ - []byte("//"), - []byte("--"), - []byte("#"), - []byte("*"), - []byte(";"), - []byte("%"), - } { - strippedLeft, leftHasLeader := stripCommentLeader(left, leader) - strippedRight, rightHasLeader := stripCommentLeader(right, leader) - if leftHasLeader && rightHasLeader { - return strippedLeft, strippedRight - } - } - return left, right -} - -func stripCommentLeader(line, leader []byte) ([]byte, bool) { - if !bytes.HasPrefix(line, leader) { - return line, false - } - for bytes.HasPrefix(line, leader) { - line = line[len(leader):] - } - return bytes.TrimSpace(line), true -} - -func usesDocumentMarkers(filePath string) bool { - cleaned := filepath.ToSlash(filePath) - switch strings.ToLower(pathpkg.Ext(cleaned)) { - case ".md", ".markdown", ".mdown", ".mdx", ".mkd": - return true - } - return strings.EqualFold(pathpkg.Base(cleaned), "readme") -} - -func isDocumentBoundary(left, right []byte) bool { - return isHeadingLine(left) || isHeadingLine(right) || - isTableLine(left) || isTableLine(right) || - isListItem(right) -} - -func isHeadingLine(line []byte) bool { - return line[0] == '#' || line[0] == '=' -} - -func isTableLine(line []byte) bool { - return line[0] == '|' || line[len(line)-1] == '|' -} - -func isListItem(line []byte) bool { - if len(line) < minimumMarkerLength { - return false - } - switch line[0] { - case '-', '*', '+', '>': - return line[1] == ' ' || line[1] == '\t' - } - index := 0 - for index < len(line) && line[index] >= '0' && line[index] <= '9' { - index++ - } - if index == 0 || index+1 >= len(line) || - line[index] != '.' && line[index] != ')' { - return false - } - return line[index+1] == ' ' || line[index+1] == '\t' -} - -func isLegalFile(filePath string) bool { - return len(legalFileRoles(filePath)) != 0 -} - -func legalFileRoles(filePath string) []string { - cleaned := filepath.ToSlash(filePath) - parts := strings.Split(cleaned, "/") - licenseRole := false - for _, directory := range parts[:len(parts)-1] { - switch strings.ToLower(directory) { - case "license", "licenses", "licence", "licences": - licenseRole = true - } - } - - name := strings.ToLower(pathpkg.Base(cleaned)) - noticeRole := hasLegalNamePrefix(name, "notices") || - hasLegalNamePrefix(name, "notice") - for _, prefix := range []string{ - "licenses", - "license", - "licences", - "licence", - "copying", - "mit-license", - "copyright", - "unlicense", - } { - if hasLegalNamePrefix(name, prefix) { - licenseRole = true - break - } - } - - roles := make([]string, 0, legalRoleCount) - if licenseRole { - roles = append(roles, "license") - } - if noticeRole { - roles = append(roles, "notice") - } - return roles -} - -func hasLegalNamePrefix(name, prefix string) bool { - if name == prefix { - return true - } - if !strings.HasPrefix(name, prefix) { - return false - } - switch name[len(prefix)] { - case '.', '-', '_': - return true - default: - return false - } -} - -func sortFileMatches(file *fileRecord) { - slices.SortFunc(file.Clues, compareMatchRecords) - slices.SortFunc(file.Detections, func(first, second detectionRecord) int { - if compared := compareMatchRecords(first.Matches[0], second.Matches[0]); compared != 0 { - return compared - } - return strings.Compare(first.Expression, second.Expression) - }) -} - -func compareMatchRecords(first, second matchRecord) int { - if compared := cmp.Compare(first.Start, second.Start); compared != 0 { - return compared - } - if compared := cmp.Compare(first.End, second.End); compared != 0 { - return compared - } - if compared := strings.Compare(first.RuleID, second.RuleID); compared != 0 { - return compared - } - return strings.Compare(string(first.Method), string(second.Method)) -} - -func addExpressionRecords(expressions map[string]*expressionRecord, file fileRecord) { - root := isRootExpressionFile(file) - for _, detection := range file.Detections { - record := expressions[detection.Expression] - if record == nil { - record = &expressionRecord{ - Expression: detection.Expression, - Identification: detection.Identification, - } - expressions[detection.Expression] = record - } - record.Root = record.Root || root - record.Files++ - record.Matches += len(detection.Matches) - } -} - -func isRootExpressionFile(file fileRecord) bool { - return pathDepth(file.Path) == 1 && - (len(file.Roles) != 0 || isReadmeFile(file.Path)) -} - -func isReadmeFile(filePath string) bool { - name := strings.ToLower(pathpkg.Base(filepath.ToSlash(filePath))) - return name == "readme" || strings.HasPrefix(name, "readme.") -} - -func addIdentificationSummary( - summary *scanSummary, - detections []detectionRecord, -) { - var identified, partial, noAssertion bool - for _, detection := range detections { - switch detection.Identification { - case licenses.Identified: - identified = true - case licenses.Partial: - partial = true - case licenses.NoAssertion: - noAssertion = true - } - } - if identified { - summary.FilesWithIdentifiedDetections++ - } - if partial { - summary.FilesWithPartialDetections++ - } - if noAssertion { - summary.FilesWithNoAssertionDetections++ - } + return licenses.ValidateScanOptions(options) } func formatBytes(size int64) string { diff --git a/cmd/licenses/scan_test_helpers_test.go b/cmd/licenses/scan_test_helpers_test.go new file mode 100644 index 0000000..b331e97 --- /dev/null +++ b/cmd/licenses/scan_test_helpers_test.go @@ -0,0 +1,57 @@ +package main + +import ( + "os" + "path/filepath" + "testing" +) + +const testScannerVersion = "test-version" + +func projectLicense(t *testing.T) []byte { + t.Helper() + data, err := os.ReadFile("../../LICENSE") + if err != nil { + t.Fatal(err) + } + return data +} + +func writeTestFile(t *testing.T, path string, data []byte) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatal(err) + } +} + +func hasMITExpression(file fileRecord) bool { + for _, detection := range file.Detections { + if detection.Expression == "MIT" { + return true + } + } + return false +} + +func hasSkip(records []skipRecord, want skipRecord) bool { + for _, record := range records { + if record == want { + return true + } + } + return false +} + +func findRecordMatch(file fileRecord, ruleID string) (matchRecord, bool) { + for _, detection := range file.Detections { + for _, match := range detection.Matches { + if match.RuleID == ruleID { + return match, true + } + } + } + return matchRecord{}, false +} diff --git a/corpus.go b/corpus.go index 4f8853a..d7216c7 100644 --- a/corpus.go +++ b/corpus.go @@ -1,6 +1,6 @@ -// Package licenses matches byte slices against the ScanCode license rule -// corpus. Matching is exact after token normalization, so edits within a -// license can prevent a match. +// Package licenses matches text and scans repositories against the ScanCode +// license rule corpus. Matching is exact after token normalization, so edits +// within a license can prevent a match. package licenses // CorpusInfo identifies the ScanCode corpus used for a result. diff --git a/scan.go b/scan.go new file mode 100644 index 0000000..ccd85ac --- /dev/null +++ b/scan.go @@ -0,0 +1,1330 @@ +package licenses + +import ( + "bytes" + "cmp" + "context" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "errors" + "io" + "os" + pathpkg "path" + "path/filepath" + "runtime" + "slices" + "strings" + "sync" + "unicode/utf16" + "unicode/utf8" + + "github.com/git-pkgs/magic" + "github.com/git-pkgs/manifests" + "github.com/git-pkgs/spdx" +) + +const ( + reportSchemaVersion = 2 + defaultMaxDepth = 32 + defaultMaxFiles = 10_000 + defaultMaxFileSize = 1 << 20 + classificationProbeSize = 8 << 10 + maxWorkers = 16 + maxReadPreallocate = 16 << 20 + utf8BOMSize = 3 + minimumMarkerLength = 2 + legalRoleCount = 2 + percentageScale = 100 + encodingUTF8 = "utf-8" + encodingUTF16LE = "utf-16le" + encodingUTF16BE = "utf-16be" + encodingLatin1 = "iso-8859-1" + scannerName = "git-pkgs/licenses" + + skipReasonBinary = "binary" + skipReasonConfiguredDirectory = "configured-directory" + skipReasonDepth = "depth" + skipReasonHiddenDirectory = "hidden-directory" + skipReasonNonRegular = "non-regular" + skipReasonProjectScope = "project-scope" + skipReasonSize = "size" + skipReasonSymlink = "symlink" + skipReasonVersionControl = "version-control" + scopeAll = "all" + scopeProject = "project" +) + +var errFileLimit = errors.New("file limit reached") + +var defaultSkippedDirectories = map[string]bool{ + "vendor": true, + "node_modules": true, + "__pycache__": true, + ".bundle": true, + ".venv": true, + "venv": true, + "target": true, + "build": true, + "dist": true, + "out": true, + "_build": true, + "deps": true, + "Pods": true, + "third_party": true, + "thirdparty": true, + "external": true, + "testdata": true, + "tmp": true, + "temp": true, + "cache": true, + "coverage": true, +} + +const ( + // ReportSchemaVersion is the additive JSON report schema version. + ReportSchemaVersion = reportSchemaVersion + // DefaultMaxDepth is the default maximum directory depth. + DefaultMaxDepth = defaultMaxDepth + // DefaultMaxFiles is the default maximum number of visited files. + DefaultMaxFiles = defaultMaxFiles + // DefaultMaxFileSize is the default maximum number of bytes per file. + DefaultMaxFileSize = defaultMaxFileSize + // ScannerName identifies this scanner in reports. + ScannerName = scannerName + // ScopeAll includes dependency, build, cache, and test-data directories. + ScopeAll = scopeAll + // ScopeProject skips dependency, build, cache, and test-data directories. + ScopeProject = scopeProject +) + +// DefaultScanOptions returns the default traversal limits and worker count. +func DefaultScanOptions() ScanOptions { + return ScanOptions{ + MaxDepth: defaultMaxDepth, + MaxFiles: defaultMaxFiles, + MaxFileSize: defaultMaxFileSize, + Workers: defaultWorkerCount(), + } +} + +// ValidateScanOptions reports invalid limits or worker counts. +func ValidateScanOptions(options ScanOptions) error { + return validateScanOptions(options) +} + +// ScanRepository scans a file or directory with matcher. +func ScanRepository( + ctx context.Context, + matcher *Matcher, + root string, + options ScanOptions, +) (ScanReport, error) { + return scanRepository(ctx, matcher, root, options, options.ScannerVersion) +} + +// ScanOptions controls repository traversal and file scanning. +type ScanOptions struct { + MaxDepth int // Maximum directory depth; zero disables the limit. + MaxFiles int // Maximum number of visited files; zero disables the limit. + MaxFileSize int64 // Maximum bytes read per file; zero disables the limit. + Workers int // Requested concurrent file scans, capped at 16. + SkipDirs map[string]bool // Directory base names to skip. + NoDefaultSkip bool // Include hidden, dependency, build, cache, and test-data directories. + IncludeLegalFiles bool // Report legal files even when they contain no matches. + ScannerVersion string // Scanner build version recorded in the report. +} + +// ScanReport contains the deterministic result of scanning a file or directory. +type ScanReport struct { + Schema int `json:"schema"` + Root string `json:"root"` + Scope string `json:"scope"` + Scanner ScannerRecord `json:"scanner"` + Corpus CorpusRecord `json:"corpus"` + Summary ScanSummary `json:"summary"` + Declared []DeclaredRecord `json:"declared"` + Expressions []ExpressionRecord `json:"expressions"` + Files []FileRecord `json:"files"` + Skipped []SkipRecord `json:"skipped"` + Errors []ScanErrorRecord `json:"errors"` +} + +// ScannerRecord identifies the scanner and its build version. +type ScannerRecord struct { + Name string `json:"name"` + Version string `json:"version"` +} + +// CorpusRecord identifies the ScanCode corpus used for a scan. +type CorpusRecord struct { + Version string `json:"version"` + RuleCount int `json:"rule_count"` + SourceCommit string `json:"source_commit"` +} + +// ScanSummary contains file, byte, skip, and error counts for a scan. +type ScanSummary struct { + FilesVisited int `json:"files_visited"` + FilesScanned int `json:"files_scanned"` + FilesWithDetections int `json:"files_with_detections"` + FilesWithIdentifiedDetections int `json:"files_with_identified_detections"` + FilesWithPartialDetections int `json:"files_with_partial_detections"` + FilesWithNoAssertionDetections int `json:"files_with_noassertion_detections"` + FilesWithClues int `json:"files_with_clues"` + BytesScanned int64 `json:"bytes_scanned"` + DirectoriesSkipped int `json:"directories_skipped"` + FilesSkippedBinary int `json:"files_skipped_binary"` + FilesSkippedSize int `json:"files_skipped_size"` + FilesSkippedOther int `json:"files_skipped_other"` + ErrorCount int `json:"error_count"` + Truncated bool `json:"truncated"` +} + +// ExpressionRecord summarizes a detected license expression across files. +type ExpressionRecord struct { + Expression string `json:"expression"` + Identification Identification `json:"identification"` + Root bool `json:"root"` + Files int `json:"files"` + Matches int `json:"matches"` +} + +// DeclaredRecord contains license metadata read from a package manifest. +type DeclaredRecord struct { + Path string `json:"path"` + Raw []string `json:"raw"` + LicenseFile string `json:"license_file"` + NormalizedExpression string `json:"normalized_expression"` +} + +// FileRecord contains the license detections and clues reported for one file. +type FileRecord struct { + Path string `json:"path"` + Size int64 `json:"size"` + SHA256 string `json:"sha256"` + Encoding string `json:"encoding"` + Roles []string `json:"roles"` + LicenseTextCoverage float64 `json:"license_text_coverage"` + Detections []DetectionRecord `json:"detections"` + Clues []MatchRecord `json:"clues"` +} + +// DetectionRecord groups matches that report the same license expression. +type DetectionRecord struct { + Expression string `json:"expression"` + Identification Identification `json:"identification"` + Matches []MatchRecord `json:"matches"` +} + +// MatchRecord describes one rule match in a scanned file. +type MatchRecord struct { + RuleID string `json:"rule_id"` + LicenseIDs []string `json:"license_ids,omitempty"` + Kind Kind `json:"kind"` + Method Method `json:"method"` + Score float64 `json:"score"` + Coverage float64 `json:"coverage"` + Start int `json:"start"` + End int `json:"end"` + Matched string `json:"matched,omitempty"` +} + +// ScanErrorRecord describes an error associated with one path. +type ScanErrorRecord struct { + Path string `json:"path"` + Error string `json:"error"` +} + +// SkipRecord describes a path omitted from a scan and the reason it was omitted. +type SkipRecord struct { + Path string `json:"path"` + Reason string `json:"reason"` +} + +type scanOptions = ScanOptions +type scanReport = ScanReport +type scannerRecord = ScannerRecord +type scanSummary = ScanSummary +type expressionRecord = ExpressionRecord +type declaredRecord = DeclaredRecord +type fileRecord = FileRecord +type detectionRecord = DetectionRecord +type matchRecord = MatchRecord +type scanErrorRecord = ScanErrorRecord +type skipRecord = SkipRecord + +type fileTask struct { + path string + display string + policyPath string +} + +type fileOutcome struct { + task fileTask + result Result + roles []string + bytes int64 + scanned bool + binary bool + tooLarge bool + encoding string + sha256 string + licenseTextCoverage float64 + err error +} + +type decodedText struct { + data []byte + offsets []int + offsetBase int + encoding string +} + +type fileDiscovery struct { + ctx context.Context + root string + options scanOptions + summary *scanSummary + tasks []fileTask + declared []declaredRecord + scanErrors []scanErrorRecord + skipped []skipRecord +} + +func defaultWorkerCount() int { + return min(runtime.GOMAXPROCS(0), maxWorkers) +} + +func scanRepository( + ctx context.Context, + matcher *Matcher, + root string, + options scanOptions, + scannerVersion string, +) (scanReport, error) { + if matcher == nil { + return scanReport{}, errors.New("nil matcher") + } + if ctx == nil { + return scanReport{}, errors.New("nil context") + } + if err := ctx.Err(); err != nil { + return scanReport{}, err + } + if err := validateScanOptions(options); err != nil { + return scanReport{}, err + } + corpus := matcher.Corpus() + report := scanReport{ + Schema: reportSchemaVersion, + Root: filepath.Clean(root), + Scope: scanScope(options), + Declared: make([]declaredRecord, 0), + Files: make([]fileRecord, 0), + Skipped: make([]skipRecord, 0), + Errors: make([]scanErrorRecord, 0), + Scanner: scannerRecord{ + Name: scannerName, + Version: scannerVersion, + }, + Corpus: CorpusRecord(corpus), + } + discovery, err := discoverFiles( + ctx, + root, + options, + &report.Summary, + ) + if err != nil { + return scanReport{}, err + } + report.Declared = append(report.Declared, discovery.declared...) + report.Errors = append(report.Errors, discovery.scanErrors...) + report.Skipped = append(report.Skipped, discovery.skipped...) + + outcomes := scanFiles(ctx, matcher, discovery.tasks, options) + expressions := make(map[string]*expressionRecord) + for outcome := range outcomes { + if outcome.scanned { + report.Summary.FilesScanned++ + report.Summary.BytesScanned += outcome.bytes + } + switch { + case outcome.err != nil: + report.Errors = append(report.Errors, scanErrorRecord{ + Path: outcome.task.display, + Error: outcome.err.Error(), + }) + case outcome.binary: + report.Summary.FilesSkippedBinary++ + report.Skipped = append(report.Skipped, skipRecord{ + Path: outcome.task.display, + Reason: skipReasonBinary, + }) + case outcome.tooLarge: + report.Summary.FilesSkippedSize++ + report.Skipped = append(report.Skipped, skipRecord{ + Path: outcome.task.display, + Reason: skipReasonSize, + }) + case outcome.scanned: + if len(outcome.result.Detections) == 0 && + len(outcome.result.Clues) == 0 && + (!options.IncludeLegalFiles || len(outcome.roles) == 0) { + continue + } + file := makeFileRecord( + outcome.task.display, + outcome.bytes, + outcome.sha256, + outcome.encoding, + outcome.roles, + outcome.licenseTextCoverage, + outcome.result, + ) + sortFileMatches(&file) + if len(file.Detections) != 0 { + report.Summary.FilesWithDetections++ + addIdentificationSummary(&report.Summary, file.Detections) + } + if len(file.Clues) != 0 { + report.Summary.FilesWithClues++ + } + report.Files = append(report.Files, file) + addExpressionRecords(expressions, file) + } + } + if err := ctx.Err(); err != nil { + return scanReport{}, err + } + + slices.SortFunc(report.Files, func(first, second fileRecord) int { + return strings.Compare(first.Path, second.Path) + }) + slices.SortFunc(report.Declared, func(first, second declaredRecord) int { + return strings.Compare(first.Path, second.Path) + }) + slices.SortFunc(report.Errors, func(first, second scanErrorRecord) int { + if compared := strings.Compare(first.Path, second.Path); compared != 0 { + return compared + } + return strings.Compare(first.Error, second.Error) + }) + slices.SortFunc(report.Skipped, func(first, second skipRecord) int { + if compared := strings.Compare(first.Path, second.Path); compared != 0 { + return compared + } + return strings.Compare(first.Reason, second.Reason) + }) + report.Expressions = make([]expressionRecord, 0, len(expressions)) + for _, expression := range expressions { + report.Expressions = append(report.Expressions, *expression) + } + slices.SortFunc(report.Expressions, func(first, second expressionRecord) int { + return strings.Compare(first.Expression, second.Expression) + }) + report.Summary.ErrorCount = len(report.Errors) + return report, nil +} + +func scanScope(options scanOptions) string { + if options.NoDefaultSkip { + return scopeAll + } + return scopeProject +} + +func validateScanOptions(options scanOptions) error { + switch { + case options.MaxDepth < 0: + return errors.New("max depth must not be negative") + case options.MaxFiles < 0: + return errors.New("max files must not be negative") + case options.MaxFileSize < 0: + return errors.New("max file size must not be negative") + case options.Workers < 1: + return errors.New("workers must be positive") + default: + return nil + } +} + +func discoverFiles( + ctx context.Context, + root string, + options scanOptions, + summary *scanSummary, +) (*fileDiscovery, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + discovery := &fileDiscovery{ + ctx: ctx, + root: root, + options: options, + summary: summary, + } + info, err := os.Stat(root) + if err != nil { + return nil, err + } + if !info.IsDir() { + discovery.discoverExplicitFile(root, info) + return discovery, nil + } + walkRoot, err := filepath.EvalSymlinks(root) + if err != nil { + return nil, err + } + discovery.root = walkRoot + err = filepath.WalkDir(walkRoot, discovery.visit) + if err != nil && !errors.Is(err, errFileLimit) { + return nil, err + } + return discovery, nil +} + +func (discovery *fileDiscovery) discoverExplicitFile( + path string, + info os.FileInfo, +) { + discovery.summary.FilesVisited = 1 + if !info.Mode().IsRegular() { + discovery.summary.FilesSkippedOther = 1 + discovery.skipped = append(discovery.skipped, skipRecord{ + Path: filepath.Base(path), + Reason: skipReasonNonRegular, + }) + return + } + if discovery.options.MaxFileSize > 0 && + info.Size() > discovery.options.MaxFileSize { + discovery.summary.FilesSkippedSize = 1 + discovery.skipped = append(discovery.skipped, skipRecord{ + Path: filepath.Base(path), + Reason: skipReasonSize, + }) + return + } + display := filepath.Base(path) + discovery.tasks = append(discovery.tasks, fileTask{ + path: path, + display: display, + policyPath: explicitPolicyPath(path), + }) + discovery.discoverDeclaredLicense(path, display) +} + +func explicitPolicyPath(filePath string) string { + name := filepath.Base(filePath) + directory := filepath.Base(filepath.Dir(filepath.Clean(filePath))) + if directory == "." || directory == string(filepath.Separator) { + return name + } + return filepath.ToSlash(filepath.Join(directory, name)) +} + +func (discovery *fileDiscovery) visit( + path string, + entry os.DirEntry, + walkErr error, +) error { + if err := discovery.ctx.Err(); err != nil { + return err + } + relative := relativePath(discovery.root, path) + if walkErr != nil { + discovery.scanErrors = append(discovery.scanErrors, scanErrorRecord{ + Path: relative, + Error: walkErr.Error(), + }) + if entry != nil && entry.IsDir() { + return filepath.SkipDir + } + return nil + } + if entry.IsDir() { + return discovery.visitDirectory(relative, entry.Name()) + } + return discovery.visitFile(path, relative, entry) +} + +func (discovery *fileDiscovery) visitDirectory(relative, name string) error { + if relative == "." { + return nil + } + reason := skippedDirectoryReason(name, discovery.options) + if discovery.options.MaxDepth > 0 && + pathDepth(relative) > discovery.options.MaxDepth { + reason = skipReasonDepth + } + if reason != "" { + discovery.summary.DirectoriesSkipped++ + discovery.skipped = append(discovery.skipped, skipRecord{ + Path: relative, + Reason: reason, + }) + return filepath.SkipDir + } + return nil +} + +func (discovery *fileDiscovery) visitFile( + path string, + relative string, + entry os.DirEntry, +) error { + if discovery.options.MaxFiles > 0 && + discovery.summary.FilesVisited >= discovery.options.MaxFiles { + discovery.summary.Truncated = true + return errFileLimit + } + discovery.summary.FilesVisited++ + if entry.Type()&os.ModeSymlink != 0 { + discovery.summary.FilesSkippedOther++ + discovery.skipped = append(discovery.skipped, skipRecord{ + Path: relative, + Reason: skipReasonSymlink, + }) + return nil + } + fileInfo, err := entry.Info() + if err != nil { + discovery.scanErrors = append(discovery.scanErrors, scanErrorRecord{ + Path: relative, + Error: err.Error(), + }) + return nil + } + if !fileInfo.Mode().IsRegular() { + discovery.summary.FilesSkippedOther++ + discovery.skipped = append(discovery.skipped, skipRecord{ + Path: relative, + Reason: skipReasonNonRegular, + }) + return nil + } + if discovery.options.MaxFileSize > 0 && + fileInfo.Size() > discovery.options.MaxFileSize { + discovery.summary.FilesSkippedSize++ + discovery.skipped = append(discovery.skipped, skipRecord{ + Path: relative, + Reason: skipReasonSize, + }) + return nil + } + discovery.tasks = append(discovery.tasks, fileTask{ + path: path, + display: filepath.ToSlash(relative), + policyPath: filepath.ToSlash(relative), + }) + discovery.discoverDeclaredLicense(path, filepath.ToSlash(relative)) + return nil +} + +func (discovery *fileDiscovery) discoverDeclaredLicense(path, display string) { + record, ok, err := declaredLicense(path, display) + if err != nil { + discovery.scanErrors = append(discovery.scanErrors, scanErrorRecord{ + Path: display, + Error: err.Error(), + }) + return + } + if ok { + discovery.declared = append(discovery.declared, record) + } +} + +func declaredLicense(path, display string) (declaredRecord, bool, error) { + if _, kind, ok := manifests.Identify(display); !ok || kind != manifests.Manifest { + return declaredRecord{}, false, nil + } + content, err := os.ReadFile(path) + if err != nil { + return declaredRecord{}, false, err + } + result, err := manifests.Parse(display, content) + if err != nil { + return declaredRecord{}, false, err + } + if result == nil || + (len(result.Licenses) == 0 && result.LicenseFile == "") { + return declaredRecord{}, false, nil + } + return declaredRecord{ + Path: display, + Raw: append([]string{}, result.Licenses...), + LicenseFile: result.LicenseFile, + NormalizedExpression: normalizeDeclaredExpression(result.Licenses), + }, true, nil +} + +func normalizeDeclaredExpression(raw []string) string { + if len(raw) == 0 { + return "" + } + normalized, err := spdx.NormalizeExpressionLax(strings.Join(raw, " OR ")) + if err != nil { + return "" + } + return normalized +} + +func skippedDirectoryReason(name string, options scanOptions) string { + if name == ".git" { + return skipReasonVersionControl + } + if options.SkipDirs[name] { + return skipReasonConfiguredDirectory + } + if options.NoDefaultSkip { + return "" + } + if strings.HasPrefix(name, ".") { + return skipReasonHiddenDirectory + } + if defaultSkippedDirectories[name] { + return skipReasonProjectScope + } + return "" +} + +func relativePath(root, path string) string { + relative, err := filepath.Rel(root, path) + if err != nil { + return filepath.ToSlash(path) + } + return filepath.ToSlash(relative) +} + +func pathDepth(path string) int { + if path == "." || path == "" { + return 0 + } + return strings.Count(filepath.ToSlash(filepath.Clean(path)), "/") + 1 +} + +func scanFiles( + ctx context.Context, + matcher *Matcher, + tasks []fileTask, + options scanOptions, +) <-chan fileOutcome { + outcomes := make(chan fileOutcome) + if len(tasks) == 0 { + close(outcomes) + return outcomes + } + jobs := make(chan fileTask, len(tasks)) + for _, task := range tasks { + jobs <- task + } + close(jobs) + + var workers sync.WaitGroup + workerCount := effectiveWorkerCount(options.Workers, len(tasks)) + for range workerCount { + workers.Add(1) + go func() { + defer workers.Done() + for task := range jobs { + outcome := scanFile(ctx, matcher, task, options) + select { + case outcomes <- outcome: + case <-ctx.Done(): + return + } + } + }() + } + go func() { + workers.Wait() + close(outcomes) + }() + return outcomes +} + +func effectiveWorkerCount(requested, taskCount int) int { + return min(requested, maxWorkers, taskCount) +} + +func scanFile( + ctx context.Context, + matcher *Matcher, + task fileTask, + options scanOptions, +) fileOutcome { + data, detection, tooLarge, err := readScannableFile(task.path, options.MaxFileSize) + if err != nil { + return fileOutcome{task: task, err: err} + } + if tooLarge { + return fileOutcome{task: task, tooLarge: true} + } + if detection.Kind == magic.KindBinary { + return fileOutcome{task: task, binary: true} + } + decoded := decodeText(data, detection) + result, err := matcher.Match(ctx, decoded.data) + if err != nil { + return fileOutcome{ + task: task, + bytes: int64(len(data)), + scanned: true, + encoding: decoded.encoding, + err: err, + } + } + applyScanPolicy(task.policyPath, decoded.data, &result) + licenseTextCoverage := calculateLicenseTextCoverage(result, len(decoded.data)) + remapResultOffsets(&result, decoded) + roles := LegalFileRoles(task.policyPath) + checksum := "" + if len(result.Detections) != 0 || len(result.Clues) != 0 || + (options.IncludeLegalFiles && len(roles) != 0) { + digest := sha256.Sum256(data) + checksum = hex.EncodeToString(digest[:]) + } + return fileOutcome{ + task: task, + result: result, + roles: roles, + bytes: int64(len(data)), + scanned: true, + encoding: decoded.encoding, + sha256: checksum, + licenseTextCoverage: licenseTextCoverage, + } +} + +func readScannableFile(path string, maximum int64) ([]byte, magic.Result, bool, error) { + file, err := os.Open(path) + if err != nil { + return nil, magic.Result{}, false, err + } + defer func() { _ = file.Close() }() + + var data bytes.Buffer + probeLimit := int64(classificationProbeSize) + if maximum > 0 { + probeLimit = min(probeLimit, maximum+1) + } + _, err = io.CopyN(&data, file, probeLimit) + if err != nil && !errors.Is(err, io.EOF) { + return nil, magic.Result{}, false, err + } + if detection := magic.DetectPrefix(data.Bytes()); detection.Kind == magic.KindBinary { + return nil, detection, false, nil + } + if maximum > 0 && int64(data.Len()) > maximum { + return nil, magic.Result{}, true, nil + } + growReadBuffer(file, &data, maximum) + reader := io.Reader(file) + if maximum > 0 { + reader = io.LimitReader(file, maximum-int64(data.Len())+1) + } + if _, err := io.Copy(&data, reader); err != nil { + return nil, magic.Result{}, false, err + } + if maximum > 0 && int64(data.Len()) > maximum { + return nil, magic.Result{}, true, nil + } + content := data.Bytes() + return content, magic.Detect(content), false, nil +} + +func growReadBuffer(file *os.File, data *bytes.Buffer, maximum int64) { + info, err := file.Stat() + if err != nil { + return + } + size := info.Size() + if maximum > 0 { + size = min(size, maximum+1) + } + if size <= int64(data.Len()) || size > maxReadPreallocate { + return + } + data.Grow(int(size) - data.Len()) +} + +func decodeText(data []byte, detection magic.Result) decodedText { + switch detection.Encoding { + case encodingUTF8: + if !bytes.HasPrefix(data, []byte{0xef, 0xbb, 0xbf}) { + return decodedText{data: data, encoding: encodingUTF8} + } + return decodedText{ + data: data[utf8BOMSize:], + offsetBase: utf8BOMSize, + encoding: encodingUTF8, + } + case encodingUTF16LE: + return decodeUTF16(data, binary.LittleEndian, encodingUTF16LE) + case encodingUTF16BE: + return decodeUTF16(data, binary.BigEndian, encodingUTF16BE) + } + if detection.Kind == magic.KindUnknown && + detection.Reason == magic.ReasonInvalidText { + return decodeLatin1(data) + } + return decodedText{data: data, encoding: encodingUTF8} +} + +func decodeUTF16(data []byte, order binary.ByteOrder, name string) decodedText { + decoded := decodedText{ + data: make([]byte, 0, len(data)), + offsets: []int{2}, + encoding: name, + } + for position := 2; position < len(data); { + start := position + var character rune + if position+1 >= len(data) { + character = utf8.RuneError + position++ + } else { + first := order.Uint16(data[position : position+2]) + position += 2 + character = rune(first) + if utf16.IsSurrogate(character) { + if position+1 < len(data) { + second := rune(order.Uint16(data[position : position+2])) + if decodedRune := utf16.DecodeRune(character, second); decodedRune != utf8.RuneError { + character = decodedRune + position += 2 + } else { + character = utf8.RuneError + } + } else { + character = utf8.RuneError + } + } + } + decoded.appendRune(character, start, position) + } + return decoded +} + +func decodeLatin1(data []byte) decodedText { + decoded := decodedText{ + data: make([]byte, 0, len(data)), + offsets: []int{0}, + encoding: encodingLatin1, + } + for position, value := range data { + decoded.appendRune(rune(value), position, position+1) + } + return decoded +} + +func (decoded *decodedText) appendRune(character rune, rawStart, rawEnd int) { + start := len(decoded.data) + decoded.data = utf8.AppendRune(decoded.data, character) + for position := start; position < len(decoded.data); position++ { + if position == len(decoded.data)-1 { + decoded.offsets = append(decoded.offsets, rawEnd) + } else { + decoded.offsets = append(decoded.offsets, rawStart) + } + } +} + +func (decoded decodedText) rawOffset(offset int) int { + if decoded.offsets != nil { + return decoded.offsets[offset] + } + return offset + decoded.offsetBase +} + +func remapResultOffsets(result *Result, decoded decodedText) { + if decoded.offsets == nil && decoded.offsetBase == 0 { + return + } + for detectionIndex := range result.Detections { + for matchIndex := range result.Detections[detectionIndex].Matches { + match := &result.Detections[detectionIndex].Matches[matchIndex] + match.Start = decoded.rawOffset(match.Start) + match.End = decoded.rawOffset(match.End) + } + } + for matchIndex := range result.Clues { + match := &result.Clues[matchIndex] + match.Start = decoded.rawOffset(match.Start) + match.End = decoded.rawOffset(match.End) + } +} + +func makeFileRecord( + path string, + size int64, + checksum string, + encoding string, + roles []string, + licenseTextCoverage float64, + result Result, +) fileRecord { + file := fileRecord{ + Path: path, + Size: size, + SHA256: checksum, + Encoding: encoding, + Roles: roles, + LicenseTextCoverage: licenseTextCoverage, + } + file.Detections = make([]detectionRecord, 0, len(result.Detections)) + for _, detection := range result.Detections { + record := detectionRecord{ + Expression: detection.Expression, + Identification: detection.Identification, + Matches: make([]matchRecord, 0, len(detection.Matches)), + } + for _, match := range detection.Matches { + record.Matches = append(record.Matches, makeMatchRecord(match)) + } + file.Detections = append(file.Detections, record) + } + file.Clues = make([]matchRecord, 0, len(result.Clues)) + for _, clue := range result.Clues { + file.Clues = append(file.Clues, makeMatchRecord(clue)) + } + return file +} + +type byteRange struct { + start int + end int +} + +func calculateLicenseTextCoverage(result Result, inputLength int) float64 { + if inputLength == 0 { + return 0 + } + + ranges := make([]byteRange, 0) + addMatch := func(match Match) { + if match.Kind != KindText && match.Kind != KindNotice { + return + } + start := max(0, min(match.Start, inputLength)) + end := max(0, min(match.End, inputLength)) + if start >= end { + return + } + ranges = append(ranges, byteRange{start: start, end: end}) + } + for _, detection := range result.Detections { + for _, match := range detection.Matches { + addMatch(match) + } + } + for _, clue := range result.Clues { + addMatch(clue) + } + if len(ranges) == 0 { + return 0 + } + + slices.SortFunc(ranges, func(first, second byteRange) int { + if compared := cmp.Compare(first.start, second.start); compared != 0 { + return compared + } + return cmp.Compare(first.end, second.end) + }) + covered := 0 + current := ranges[0] + for _, next := range ranges[1:] { + if next.start <= current.end { + current.end = max(current.end, next.end) + continue + } + covered += current.end - current.start + current = next + } + covered += current.end - current.start + return float64(covered) / float64(inputLength) * percentageScale +} + +func makeMatchRecord(match Match) matchRecord { + return matchRecord{ + RuleID: match.RuleID, + LicenseIDs: match.LicenseIDs, + Kind: match.Kind, + Method: match.Method, + Score: match.Score, + Coverage: match.Coverage, + Start: match.Start, + End: match.End, + Matched: string(match.Matched), + } +} + +func applyScanPolicy(path string, input []byte, result *Result) { + if len(LegalFileRoles(path)) != 0 { + return + } + + documentMarkers := usesDocumentMarkers(path) + detections := result.Detections[:0] + for _, detection := range result.Detections { + matches := detection.Matches[:0] + for _, match := range detection.Matches { + if match.Kind == KindReference && + crossesBlockBoundary( + input, + match.Start, + match.End, + documentMarkers, + ) { + result.Clues = append(result.Clues, match) + continue + } + matches = append(matches, match) + } + if len(matches) == 0 { + continue + } + detection.Matches = matches + detections = append(detections, detection) + } + result.Detections = detections +} + +func crossesBlockBoundary( + input []byte, + start int, + end int, + documentMarkers bool, +) bool { + if start < 0 || start >= end || end > len(input) { + return false + } + for searchStart := start; searchStart < end; { + relative := bytes.IndexByte(input[searchStart:end], '\n') + if relative < 0 { + return false + } + newline := searchStart + relative + leftStart := bytes.LastIndexByte(input[:newline], '\n') + 1 + rightEnd := len(input) + if next := bytes.IndexByte(input[newline+1:], '\n'); next >= 0 { + rightEnd = newline + 1 + next + } + left := bytes.TrimSpace(input[leftStart:newline]) + right := bytes.TrimSpace(input[newline+1 : rightEnd]) + paragraphLeft, paragraphRight := stripCommonCommentLeader(left, right) + if len(paragraphLeft) == 0 || len(paragraphRight) == 0 { + return true + } + if documentMarkers && isDocumentBoundary(left, right) { + return true + } + searchStart = newline + 1 + } + return false +} + +func stripCommonCommentLeader(left, right []byte) ([]byte, []byte) { + for _, leader := range [][]byte{ + []byte("//"), + []byte("--"), + []byte("#"), + []byte("*"), + []byte(";"), + []byte("%"), + } { + strippedLeft, leftHasLeader := stripCommentLeader(left, leader) + strippedRight, rightHasLeader := stripCommentLeader(right, leader) + if leftHasLeader && rightHasLeader { + return strippedLeft, strippedRight + } + } + return left, right +} + +func stripCommentLeader(line, leader []byte) ([]byte, bool) { + if !bytes.HasPrefix(line, leader) { + return line, false + } + for bytes.HasPrefix(line, leader) { + line = line[len(leader):] + } + return bytes.TrimSpace(line), true +} + +func usesDocumentMarkers(filePath string) bool { + cleaned := filepath.ToSlash(filePath) + switch strings.ToLower(pathpkg.Ext(cleaned)) { + case ".md", ".markdown", ".mdown", ".mdx", ".mkd": + return true + } + return strings.EqualFold(pathpkg.Base(cleaned), "readme") +} + +func isDocumentBoundary(left, right []byte) bool { + return isHeadingLine(left) || isHeadingLine(right) || + isTableLine(left) || isTableLine(right) || + isListItem(right) +} + +func isHeadingLine(line []byte) bool { + return line[0] == '#' || line[0] == '=' +} + +func isTableLine(line []byte) bool { + return line[0] == '|' || line[len(line)-1] == '|' +} + +func isListItem(line []byte) bool { + if len(line) < minimumMarkerLength { + return false + } + switch line[0] { + case '-', '*', '+', '>': + return line[1] == ' ' || line[1] == '\t' + } + index := 0 + for index < len(line) && line[index] >= '0' && line[index] <= '9' { + index++ + } + if index == 0 || index+1 >= len(line) || + line[index] != '.' && line[index] != ')' { + return false + } + return line[index+1] == ' ' || line[index+1] == '\t' +} + +// LegalFileRoles classifies a path as a license, notice, both, or neither. +func LegalFileRoles(filePath string) []string { + cleaned := filepath.ToSlash(filePath) + parts := strings.Split(cleaned, "/") + licenseRole := false + for _, directory := range parts[:len(parts)-1] { + switch strings.ToLower(directory) { + case "license", "licenses", "licence", "licences": + licenseRole = true + } + } + + name := strings.ToLower(pathpkg.Base(cleaned)) + noticeRole := hasLegalNamePrefix(name, "notices") || + hasLegalNamePrefix(name, "notice") + for _, prefix := range []string{ + "licenses", + "license", + "licences", + "licence", + "copying", + "mit-license", + "copyright", + "unlicense", + } { + if hasLegalNamePrefix(name, prefix) { + licenseRole = true + break + } + } + + roles := make([]string, 0, legalRoleCount) + if licenseRole { + roles = append(roles, "license") + } + if noticeRole { + roles = append(roles, "notice") + } + return roles +} + +func hasLegalNamePrefix(name, prefix string) bool { + if name == prefix { + return true + } + if !strings.HasPrefix(name, prefix) { + return false + } + switch name[len(prefix)] { + case '.', '-', '_': + return true + default: + return false + } +} + +func sortFileMatches(file *fileRecord) { + slices.SortFunc(file.Clues, compareMatchRecords) + slices.SortFunc(file.Detections, func(first, second detectionRecord) int { + if compared := compareMatchRecords(first.Matches[0], second.Matches[0]); compared != 0 { + return compared + } + return strings.Compare(first.Expression, second.Expression) + }) +} + +func compareMatchRecords(first, second matchRecord) int { + if compared := cmp.Compare(first.Start, second.Start); compared != 0 { + return compared + } + if compared := cmp.Compare(first.End, second.End); compared != 0 { + return compared + } + if compared := strings.Compare(first.RuleID, second.RuleID); compared != 0 { + return compared + } + return strings.Compare(string(first.Method), string(second.Method)) +} + +func addExpressionRecords(expressions map[string]*expressionRecord, file fileRecord) { + root := isRootExpressionFile(file) + for _, detection := range file.Detections { + record := expressions[detection.Expression] + if record == nil { + record = &expressionRecord{ + Expression: detection.Expression, + Identification: detection.Identification, + } + expressions[detection.Expression] = record + } + record.Root = record.Root || root + record.Files++ + record.Matches += len(detection.Matches) + } +} + +func isRootExpressionFile(file fileRecord) bool { + return pathDepth(file.Path) == 1 && + (len(file.Roles) != 0 || isReadmeFile(file.Path)) +} + +func isReadmeFile(filePath string) bool { + name := strings.ToLower(pathpkg.Base(filepath.ToSlash(filePath))) + return name == "readme" || strings.HasPrefix(name, "readme.") +} + +func addIdentificationSummary( + summary *scanSummary, + detections []detectionRecord, +) { + var identified, partial, noAssertion bool + for _, detection := range detections { + switch detection.Identification { + case Identified: + identified = true + case Partial: + partial = true + case NoAssertion: + noAssertion = true + } + } + if identified { + summary.FilesWithIdentifiedDetections++ + } + if partial { + summary.FilesWithPartialDetections++ + } + if noAssertion { + summary.FilesWithNoAssertionDetections++ + } +} diff --git a/scan_api_test.go b/scan_api_test.go new file mode 100644 index 0000000..590e188 --- /dev/null +++ b/scan_api_test.go @@ -0,0 +1,124 @@ +package licenses_test + +import ( + "context" + "crypto/sha256" + "fmt" + "os" + "path/filepath" + "testing" + + licenses "github.com/git-pkgs/licenses" +) + +func TestScanRepositoryPublicAPI(t *testing.T) { + t.Parallel() + + matcher, err := licenses.New() + if err != nil { + t.Fatal(err) + } + text, err := os.ReadFile("LICENSE") + if err != nil { + t.Fatal(err) + } + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "LICENSE"), text, 0o600); err != nil { + t.Fatal(err) + } + + options := licenses.DefaultScanOptions() + options.ScannerVersion = "consumer-version" + report, err := licenses.ScanRepository( + context.Background(), + matcher, + root, + options, + ) + if err != nil { + t.Fatal(err) + } + if report.Schema != licenses.ReportSchemaVersion { + t.Errorf("schema = %d, want %d", report.Schema, licenses.ReportSchemaVersion) + } + if report.Scanner != (licenses.ScannerRecord{ + Name: licenses.ScannerName, + Version: "consumer-version", + }) { + t.Errorf("scanner = %#v", report.Scanner) + } + if len(report.Files) != 1 { + t.Fatalf("files = %#v, want one", report.Files) + } + file := report.Files[0] + if file.Path != "LICENSE" || file.SHA256 == "" || len(file.Roles) != 1 || file.Roles[0] != "license" { + t.Errorf("file = %#v, want hashed license file", file) + } +} + +func TestScanRepositoryPublicValidation(t *testing.T) { + t.Parallel() + + options := licenses.DefaultScanOptions() + options.Workers = 0 + if err := licenses.ValidateScanOptions(options); err == nil { + t.Fatal("ValidateScanOptions returned nil for zero workers") + } + if roles := licenses.LegalFileRoles("legal/LICENSE.txt"); len(roles) != 1 || roles[0] != "license" { + t.Fatalf("LegalFileRoles = %#v, want license", roles) + } +} + +func TestScanRepositoryIncludesUnmatchedLegalFiles(t *testing.T) { + t.Parallel() + + matcher, err := licenses.New() + if err != nil { + t.Fatal(err) + } + root := t.TempDir() + files := []struct { + path string + content []byte + role string + }{ + {path: "LICENSE.custom", content: []byte("ZQXWV-184729\n"), role: "license"}, + {path: "NOTICE.custom", content: []byte("QAZWSX-593821\n"), role: "notice"}, + } + for _, file := range files { + if err := os.WriteFile(filepath.Join(root, file.path), file.content, 0o600); err != nil { + t.Fatal(err) + } + } + if err := os.WriteFile(filepath.Join(root, "ordinary.txt"), []byte("MNBVCX-274610\n"), 0o600); err != nil { + t.Fatal(err) + } + + options := licenses.DefaultScanOptions() + report, err := licenses.ScanRepository(context.Background(), matcher, root, options) + if err != nil { + t.Fatal(err) + } + if len(report.Files) != 0 { + t.Fatalf("default files = %#v, want none", report.Files) + } + + options.IncludeLegalFiles = true + report, err = licenses.ScanRepository(context.Background(), matcher, root, options) + if err != nil { + t.Fatal(err) + } + if len(report.Files) != len(files) { + t.Fatalf("files = %#v, want %d", report.Files, len(files)) + } + for index, want := range files { + file := report.Files[index] + wantSHA256 := sha256.Sum256(want.content) + if file.Path != want.path || file.Size != int64(len(want.content)) || + file.SHA256 != fmt.Sprintf("%x", wantSHA256) || file.Encoding != "utf-8" || + len(file.Roles) != 1 || file.Roles[0] != want.role || + len(file.Detections) != 0 || len(file.Clues) != 0 { + t.Errorf("file = %#v, want unmatched %s metadata", file, want.role) + } + } +} diff --git a/cmd/licenses/scan_test.go b/scan_test.go similarity index 95% rename from cmd/licenses/scan_test.go rename to scan_test.go index aab200a..7fe1447 100644 --- a/cmd/licenses/scan_test.go +++ b/scan_test.go @@ -1,4 +1,4 @@ -package main +package licenses import ( "context" @@ -14,7 +14,6 @@ import ( "testing" "unicode/utf16" - licenses "github.com/git-pkgs/licenses" "github.com/git-pkgs/magic" ) @@ -307,17 +306,17 @@ func TestIdentificationRecordsAndSummary(t *testing.T) { file := fileRecord{Detections: []detectionRecord{ { Expression: "MIT", - Identification: licenses.Identified, + Identification: Identified, Matches: []matchRecord{{RuleID: "mit.RULE"}}, }, { Expression: "MIT AND LicenseRef-scancode-free-unknown", - Identification: licenses.Partial, + Identification: Partial, Matches: []matchRecord{{RuleID: "partial.RULE"}}, }, { Expression: "LicenseRef-scancode-unknown-license-reference", - Identification: licenses.NoAssertion, + Identification: NoAssertion, Matches: []matchRecord{{RuleID: "unknown.RULE"}}, }, }} @@ -515,7 +514,7 @@ func TestScanRepositoryDecodesLicenseText(t *testing.T) { plain := make([]byte, 0, len(prefix)+len(license)) plain = append(plain, prefix...) plain = append(plain, license...) - matcher, err := licenses.New(licenses.WithMatchedText()) + matcher, err := New(WithMatchedText()) if err != nil { t.Fatal(err) } @@ -663,22 +662,22 @@ func TestScanRepositoryFallsBackToLatin1ForMalformedUTF16(t *testing.T) { func TestCalculateLicenseTextCoverage(t *testing.T) { t.Parallel() - result := licenses.Result{ - Detections: []licenses.Detection{ + result := Result{ + Detections: []Detection{ { - Matches: []licenses.Match{ - {Kind: licenses.KindText, Start: 5, End: 40}, - {Kind: licenses.KindNotice, Start: 30, End: 55}, - {Kind: licenses.KindReference, Start: 55, End: 100}, - {Kind: licenses.KindText, Start: -10, End: 10}, - {Kind: licenses.KindText, Start: 90, End: 120}, - {Kind: licenses.KindText, Start: 80, End: 70}, + Matches: []Match{ + {Kind: KindText, Start: 5, End: 40}, + {Kind: KindNotice, Start: 30, End: 55}, + {Kind: KindReference, Start: 55, End: 100}, + {Kind: KindText, Start: -10, End: 10}, + {Kind: KindText, Start: 90, End: 120}, + {Kind: KindText, Start: 80, End: 70}, }, }, }, - Clues: []licenses.Match{ - {Kind: licenses.KindNotice, Start: 70, End: 80}, - {Kind: licenses.KindClue, Start: 55, End: 70}, + Clues: []Match{ + {Kind: KindNotice, Start: 70, End: 80}, + {Kind: KindClue, Start: 55, End: 70}, }, } @@ -688,7 +687,7 @@ func TestCalculateLicenseTextCoverage(t *testing.T) { if got := calculateLicenseTextCoverage(result, 0); got != 0 { t.Errorf("empty input coverage = %v, want 0", got) } - if got := calculateLicenseTextCoverage(licenses.Result{}, 100); got != 0 { + if got := calculateLicenseTextCoverage(Result{}, 100); got != 0 { t.Errorf("unmatched input coverage = %v, want 0", got) } } @@ -707,7 +706,7 @@ func TestScanRepositoryDemotesReferenceAcrossMarkdownBlocks(t *testing.T) { "[Apache 2](https://opensource.org/licenses/Apache-2.0).\n", ) writeTestFile(t, filepath.Join(root, "README.md"), readme) - matcher, err := licenses.New(licenses.WithMatchedText()) + matcher, err := New(WithMatchedText()) if err != nil { t.Fatal(err) } @@ -735,7 +734,7 @@ func TestScanRepositoryDemotesReferenceAcrossMarkdownBlocks(t *testing.T) { if !ok { t.Fatalf("clues = %#v, want ruby_15.RULE", report.Files[0].Clues) } - if clue.Kind != licenses.KindReference || clue.Score != 80 { + if clue.Kind != KindReference || clue.Score != 80 { t.Errorf("ruby clue = %#v, want relevance-80 reference", clue) } if got := string(readme[clue.Start:clue.End]); got != "ruby>.\n\n## License" { @@ -834,7 +833,7 @@ func TestApplyScanPolicy(t *testing.T) { name string path string text string - kind licenses.Kind + kind Kind score float64 detections int clues int @@ -958,7 +957,7 @@ func TestApplyScanPolicy(t *testing.T) { name: "notice rule", path: "README.md", text: "Ruby\n\nLicense", - kind: licenses.KindNotice, + kind: KindNotice, detections: 1, }, } @@ -966,7 +965,7 @@ func TestApplyScanPolicy(t *testing.T) { t.Run(test.name, func(t *testing.T) { kind := test.kind if kind == "" { - kind = licenses.KindReference + kind = KindReference } score := test.score if score == 0 { @@ -974,9 +973,9 @@ func TestApplyScanPolicy(t *testing.T) { } start := strings.Index(test.text, "Ruby") end := strings.LastIndex(test.text, "License") + len("License") - result := licenses.Result{Detections: []licenses.Detection{{ + result := Result{Detections: []Detection{{ Expression: "Ruby", - Matches: []licenses.Match{{ + Matches: []Match{{ RuleID: "ruby.RULE", Kind: kind, Score: score, @@ -1084,8 +1083,8 @@ func TestLegalFileNames(t *testing.T) { "NOTICES", "NOTICES.txt", } { - if !isLegalFile(filePath) { - t.Errorf("isLegalFile(%q) = false, want true", filePath) + if len(LegalFileRoles(filePath)) == 0 { + t.Errorf("LegalFileRoles(%q) is empty, want a legal-file role", filePath) } } } @@ -1105,12 +1104,12 @@ func TestLegalFileRoles(t *testing.T) { {path: "src/source.go", want: []string{}}, } for _, test := range tests { - got := legalFileRoles(test.path) + got := LegalFileRoles(test.path) if !slices.Equal(got, test.want) { - t.Errorf("legalFileRoles(%q) = %#v, want %#v", test.path, got, test.want) + t.Errorf("LegalFileRoles(%q) = %#v, want %#v", test.path, got, test.want) } if got == nil { - t.Errorf("legalFileRoles(%q) returned nil, want an empty or populated array", test.path) + t.Errorf("LegalFileRoles(%q) returned nil, want an empty or populated array", test.path) } } } @@ -1137,7 +1136,7 @@ func TestScanFileEnforcesSizeAfterDiscovery(t *testing.T) { context.Background(), nil, discovery.tasks[0], - options.MaxFileSize, + options, ) if !outcome.tooLarge { t.Fatalf("outcome = %#v, want tooLarge", outcome) @@ -1197,12 +1196,9 @@ func TestScanRepositoryRecordsCandidateLimit(t *testing.T) { if len(report.Errors) != 1 { t.Fatalf("errors = %#v, want one", report.Errors) } - if !strings.Contains(report.Errors[0].Error, licenses.ErrTooManyMatches.Error()) { + if !strings.Contains(report.Errors[0].Error, ErrTooManyMatches.Error()) { t.Errorf("error = %q, want candidate limit", report.Errors[0].Error) } - if reportExitCode(report) != exitScanErrors { - t.Errorf("exit code = %d, want %d", reportExitCode(report), exitScanErrors) - } } func TestEffectiveWorkerCount(t *testing.T) { @@ -1432,9 +1428,9 @@ func TestValidateScanOptions(t *testing.T) { } } -func newTestMatcher(t *testing.T) *licenses.Matcher { +func newTestMatcher(t *testing.T) *Matcher { t.Helper() - matcher, err := licenses.New() + matcher, err := New() if err != nil { t.Fatal(err) } @@ -1452,7 +1448,7 @@ func defaultTestScanOptions() scanOptions { func projectLicense(t *testing.T) []byte { t.Helper() - data, err := os.ReadFile("../../LICENSE") + data, err := os.ReadFile("LICENSE") if err != nil { t.Fatal(err) } @@ -1505,7 +1501,7 @@ func hasFile(files []fileRecord, path string) bool { return false } -func findRuleMatch(result licenses.Result, ruleID string) (licenses.Match, bool) { +func findRuleMatch(result Result, ruleID string) (Match, bool) { for _, detection := range result.Detections { for _, match := range detection.Matches { if match.RuleID == ruleID { @@ -1513,7 +1509,7 @@ func findRuleMatch(result licenses.Result, ruleID string) (licenses.Match, bool) } } } - return licenses.Match{}, false + return Match{}, false } func findRecordMatch(file fileRecord, ruleID string) (matchRecord, bool) { From a8a3c88e271a3f8240ae8bc3fc92c0f719ff2293 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Fri, 21 Aug 2026 10:25:10 +0100 Subject: [PATCH 2/3] Include decoded legal file text --- README.md | 5 +++-- scan.go | 10 ++++++++++ scan_api_test.go | 4 ++++ scan_test.go | 12 +++++++++++- 4 files changed, 28 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 98beae7..d7324d5 100644 --- a/README.md +++ b/README.md @@ -129,12 +129,13 @@ if err != nil { } for _, file := range report.Files { - fmt.Println(file.Path, file.Detections) + fmt.Println(file.Path, file.Roles, file.Text) } ``` `IncludeLegalFiles` retains recognized license and notice files when the corpus -does not produce a match. +does not produce a match. It also includes their complete text decoded to +UTF-8. Matching uses normalized whole-text hashes, exact token sequences, and `SPDX-License-Identifier` tag lines. It does not use fuzzy or sequence diff --git a/scan.go b/scan.go index ccd85ac..34966ad 100644 --- a/scan.go +++ b/scan.go @@ -204,6 +204,7 @@ type FileRecord struct { Size int64 `json:"size"` SHA256 string `json:"sha256"` Encoding string `json:"encoding"` + Text string `json:"text,omitempty"` Roles []string `json:"roles"` LicenseTextCoverage float64 `json:"license_text_coverage"` Detections []DetectionRecord `json:"detections"` @@ -270,6 +271,7 @@ type fileOutcome struct { tooLarge bool encoding string sha256 string + text string licenseTextCoverage float64 err error } @@ -379,6 +381,7 @@ func scanRepository( outcome.bytes, outcome.sha256, outcome.encoding, + outcome.text, outcome.roles, outcome.licenseTextCoverage, outcome.result, @@ -787,6 +790,10 @@ func scanFile( digest := sha256.Sum256(data) checksum = hex.EncodeToString(digest[:]) } + text := "" + if options.IncludeLegalFiles && len(roles) != 0 { + text = string(decoded.data) + } return fileOutcome{ task: task, result: result, @@ -795,6 +802,7 @@ func scanFile( scanned: true, encoding: decoded.encoding, sha256: checksum, + text: text, licenseTextCoverage: licenseTextCoverage, } } @@ -963,6 +971,7 @@ func makeFileRecord( size int64, checksum string, encoding string, + text string, roles []string, licenseTextCoverage float64, result Result, @@ -972,6 +981,7 @@ func makeFileRecord( Size: size, SHA256: checksum, Encoding: encoding, + Text: text, Roles: roles, LicenseTextCoverage: licenseTextCoverage, } diff --git a/scan_api_test.go b/scan_api_test.go index 590e188..f98952e 100644 --- a/scan_api_test.go +++ b/scan_api_test.go @@ -54,6 +54,9 @@ func TestScanRepositoryPublicAPI(t *testing.T) { if file.Path != "LICENSE" || file.SHA256 == "" || len(file.Roles) != 1 || file.Roles[0] != "license" { t.Errorf("file = %#v, want hashed license file", file) } + if file.Text != "" { + t.Errorf("text = %q, want empty without IncludeLegalFiles", file.Text) + } } func TestScanRepositoryPublicValidation(t *testing.T) { @@ -116,6 +119,7 @@ func TestScanRepositoryIncludesUnmatchedLegalFiles(t *testing.T) { wantSHA256 := sha256.Sum256(want.content) if file.Path != want.path || file.Size != int64(len(want.content)) || file.SHA256 != fmt.Sprintf("%x", wantSHA256) || file.Encoding != "utf-8" || + file.Text != string(want.content) || len(file.Roles) != 1 || file.Roles[0] != want.role || len(file.Detections) != 0 || len(file.Clues) != 0 { t.Errorf("file = %#v, want unmatched %s metadata", file, want.role) diff --git a/scan_test.go b/scan_test.go index 7fe1447..2edff84 100644 --- a/scan_test.go +++ b/scan_test.go @@ -574,11 +574,13 @@ func TestScanRepositoryDecodesLicenseText(t *testing.T) { t.Run(test.name, func(t *testing.T) { path := filepath.Join(t.TempDir(), "LICENSE") writeTestFile(t, path, test.data) + options := defaultTestScanOptions() + options.IncludeLegalFiles = true report, err := scanRepository( context.Background(), matcher, path, - defaultTestScanOptions(), + options, testScannerVersion, ) if err != nil { @@ -591,6 +593,7 @@ func TestScanRepositoryDecodesLicenseText(t *testing.T) { if file.Encoding != test.encoding { t.Errorf("encoding = %q, want %q", file.Encoding, test.encoding) } + assertFileText(t, file.Text, plain) digest := sha256.Sum256(test.data) wantSHA256 := hex.EncodeToString(digest[:]) if file.SHA256 != wantSHA256 { @@ -624,6 +627,13 @@ func TestScanRepositoryDecodesLicenseText(t *testing.T) { } } +func assertFileText(t *testing.T, got string, want []byte) { + t.Helper() + if got != string(want) { + t.Errorf("text differs from decoded UTF-8 reference") + } +} + func TestScanRepositoryFallsBackToLatin1ForMalformedUTF16(t *testing.T) { t.Parallel() From 71938589c5bb85334bd6c4774819c24103e6377a Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Fri, 21 Aug 2026 15:23:44 +0100 Subject: [PATCH 3/3] Guard default JSON output --- cmd/licenses/main_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmd/licenses/main_test.go b/cmd/licenses/main_test.go index 52119b7..5e27e4a 100644 --- a/cmd/licenses/main_test.go +++ b/cmd/licenses/main_test.go @@ -65,6 +65,9 @@ func TestRunJSON(t *testing.T) { if err := json.Unmarshal(stdout.Bytes(), &report); err != nil { t.Fatalf("decode output: %v\n%s", err, stdout.String()) } + if strings.Contains(stdout.String(), `"text":`) { + t.Errorf("default JSON contains legal file text:\n%s", stdout.String()) + } if report.Root != "../../LICENSE" { t.Errorf("root = %q, want ../../LICENSE", report.Root) }