From 57cddc3b3fc8412b877f1e9af75e3762e8f285c2 Mon Sep 17 00:00:00 2001 From: Lorenzo Gabriele Date: Thu, 17 Sep 2026 11:33:36 +0200 Subject: [PATCH 1/2] Batch files passed to opengrep to avoid ARG_MAX errors Large repos hit "argument list too long" since all target files were passed as CLI args in one invocation. Split into batches of 200 files per opengrep call, merging results. --- internal/tool/command.go | 40 ++++++++++++++++++++++++++++++++-------- 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/internal/tool/command.go b/internal/tool/command.go index cff0999..1ce3af1 100644 --- a/internal/tool/command.go +++ b/internal/tool/command.go @@ -47,19 +47,43 @@ type SemgrepErrorLocation struct { Path string `json:"path"` } +// maxFilesPerBatch caps how many files get passed as CLI args per opengrep +// invocation, to avoid hitting the OS argument list size limit (ARG_MAX) +// on repos with a very large number of files. +const maxFilesPerBatch = 200 + func executeCommandForFiles(configurationFile *os.File, toolExecution codacy.ToolExecution, patternDescriptions *[]codacy.PatternDescription, language string, files []string) ([]codacy.Result, error) { - semgrepCmd := createCommand(configurationFile, toolExecution.SourceDir, language, files) + var results []codacy.Result - semgrepOutput, semgrepError, err := runCommand(semgrepCmd) - if err != nil { - return nil, errors.New("Error running semgrep: " + *semgrepError + "\n" + err.Error()) + for _, batch := range chunkFiles(files, maxFilesPerBatch) { + semgrepCmd := createCommand(configurationFile, toolExecution.SourceDir, language, batch) + + semgrepOutput, semgrepError, err := runCommand(semgrepCmd) + if err != nil { + return nil, errors.New("Error running semgrep: " + *semgrepError + "\n" + err.Error()) + } + + output, err := parseCommandOutput(patternDescriptions, *semgrepOutput) + if err != nil { + return nil, err + } + results = append(results, output...) } - output, err := parseCommandOutput(patternDescriptions, *semgrepOutput) - if err != nil { - return nil, err + return results, nil +} + +func chunkFiles(files []string, size int) [][]string { + if len(files) == 0 { + return nil + } + + var chunks [][]string + for start := 0; start < len(files); start += size { + end := min(start+size, len(files)) + chunks = append(chunks, files[start:end]) } - return output, nil + return chunks } func createCommand(configurationFile *os.File, sourceDir, language string, files []string) *exec.Cmd { From 92aae53b8d40776487d128737f6beb3030ba1361 Mon Sep 17 00:00:00 2001 From: Joao Machado <13315199+machadoit@users.noreply.github.com> Date: Mon, 21 Sep 2026 12:36:24 +0200 Subject: [PATCH 2/2] Change batch to 1k files --- internal/tool/command.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/tool/command.go b/internal/tool/command.go index 1ce3af1..20b85b5 100644 --- a/internal/tool/command.go +++ b/internal/tool/command.go @@ -50,7 +50,7 @@ type SemgrepErrorLocation struct { // maxFilesPerBatch caps how many files get passed as CLI args per opengrep // invocation, to avoid hitting the OS argument list size limit (ARG_MAX) // on repos with a very large number of files. -const maxFilesPerBatch = 200 +const maxFilesPerBatch = 1000 func executeCommandForFiles(configurationFile *os.File, toolExecution codacy.ToolExecution, patternDescriptions *[]codacy.PatternDescription, language string, files []string) ([]codacy.Result, error) { var results []codacy.Result