diff --git a/internal/tool/command.go b/internal/tool/command.go index 20b85b5..28addf6 100644 --- a/internal/tool/command.go +++ b/internal/tool/command.go @@ -52,11 +52,11 @@ type SemgrepErrorLocation struct { // on repos with a very large number of files. const maxFilesPerBatch = 1000 -func executeCommandForFiles(configurationFile *os.File, toolExecution codacy.ToolExecution, patternDescriptions *[]codacy.PatternDescription, language string, files []string) ([]codacy.Result, error) { +func executeCommandForFiles(configurationFile *os.File, toolExecution codacy.ToolExecution, patternDescriptions *[]codacy.PatternDescription, files []string) ([]codacy.Result, error) { var results []codacy.Result for _, batch := range chunkFiles(files, maxFilesPerBatch) { - semgrepCmd := createCommand(configurationFile, toolExecution.SourceDir, language, batch) + semgrepCmd := createCommand(configurationFile, toolExecution.SourceDir, batch) semgrepOutput, semgrepError, err := runCommand(semgrepCmd) if err != nil { @@ -86,15 +86,15 @@ func chunkFiles(files []string, size int) [][]string { return chunks } -func createCommand(configurationFile *os.File, sourceDir, language string, files []string) *exec.Cmd { - params := createCommandParameters(language, configurationFile, files) +func createCommand(configurationFile *os.File, sourceDir string, files []string) *exec.Cmd { + params := createCommandParameters(configurationFile, files) cmd := exec.Command("/usr/local/bin/opengrep", params...) cmd.Dir = sourceDir return cmd } -func createCommandParameters(language string, configurationFile *os.File, filesToAnalyse []string) []string { +func createCommandParameters(configurationFile *os.File, filesToAnalyse []string) []string { // Reset file pointer to the beginning for later use if _, err := configurationFile.Seek(0, io.SeekStart); err != nil { @@ -104,7 +104,6 @@ func createCommandParameters(language string, configurationFile *os.File, filesT "scan", "--json", //"-json_nodots", "--config", configurationFile.Name(), - //"-l", language, "--timeout", "10", "--timeout-threshold", "5", "--max-target-bytes", "150000", diff --git a/internal/tool/command_test.go b/internal/tool/command_test.go index ce819bd..914abd6 100644 --- a/internal/tool/command_test.go +++ b/internal/tool/command_test.go @@ -16,11 +16,10 @@ func TestCreateCommand(t *testing.T) { configurationFile, _ := os.CreateTemp("", "config.*.yaml") defer os.Remove(configurationFile.Name()) sourceDir := "/path/to/source" - language := "go" files := []string{"file1.go", "file2.go"} // Act - cmd := createCommand(configurationFile, sourceDir, language, files) + cmd := createCommand(configurationFile, sourceDir, files) // Assert assert.IsType(t, &exec.Cmd{}, cmd) @@ -32,17 +31,15 @@ func TestCreateCommandParameters(t *testing.T) { // Arrange configurationFile, _ := os.CreateTemp("", "semgrep.yaml") defer os.Remove(configurationFile.Name()) - language := "go" filesToAnalyse := []string{"file1.go", "file2.go"} // Act - cmdParams := createCommandParameters(language, configurationFile, filesToAnalyse) + cmdParams := createCommandParameters(configurationFile, filesToAnalyse) // Assert expectedParams := []string{ "scan", "--json", - //"-lang", language, "--config", configurationFile.Name(), "--timeout", "10", "--timeout-threshold", "5", diff --git a/internal/tool/configuration.go b/internal/tool/configuration.go index 8bb2346..55ca196 100644 --- a/internal/tool/configuration.go +++ b/internal/tool/configuration.go @@ -232,135 +232,28 @@ func formatParameterName(name string) string { return result } -var filesByLanguage = make(map[string][]string) - -// Semgrep: supported language tags are: apex, bash, c, c#, c++, cairo, clojure, cpp, csharp, dart, docker, dockerfile, elixir, ex, generic, go, golang, hack, hcl, html, java, javascript, js, json, jsonnet, julia, kotlin, kt, lisp, lua, none, ocaml, php, promql, proto, proto3, protobuf, py, python, python2, python3, r, regex, ruby, rust, scala, scheme, sh, sol, solidity, swift, terraform, tf, ts, typescript, vue, xml, yaml -// Semgrep: https://github.com/semgrep/semgrep/blob/0ec2b95ec8c3afb8e31fc0295d3604e540c982b0/src/parsing/Unit_parsing.ml#L61 -// Codacy: taken from https://github.com/codacy/ragnaros/blob/05d1374b7ca4a0aa3be44972484938b4785c046f/components/language/src/main/scala/codacy/foundation/api/Language.scala#L6 -var extensionToLanguageMap = map[string]string{ - ".js": "javascript", - ".jsx": "javascript", - ".jsm": "javascript", // missing from tests - ".vue": "vue", - ".mjs": "javascript", // missing from tests - ".scala": "scala", - // ".css" - ".php": "php", - ".py": "python", - ".rb": "ruby", - ".gemspec": "ruby", // missing from tests - ".podspec": "ruby", // missing from tests - ".jbuilder": "ruby", // missing from tests - ".rake": "ruby", // missing from tests - ".opal": "ruby", // missing from tests - ".java": "java", - // ".coffee" - ".swift": "swift", - ".cpp": "cpp", - ".hpp": "cpp", // missing from tests - ".cc": "cpp", // missing from tests - ".cxx": "cpp", // missing from tests - ".ino": "cpp", // missing from tests - ".c": "c", - ".h": "c", // missing - ".sh": "sh", // missing from tests - ".bash": "bash", - ".ts": "typescript", - ".tsx": "typescript", - ".dockerfile": "dockerfile", - "Dockerfile": "dockerfile", - ".sql": "generic", - ".pls": "generic", - ".trg": "generic", - ".prc": "generic", - ".fnc": "generic", - ".pld": "generic", - ".plh": "generic", - ".plb": "generic", - ".pck": "generic", - ".pks": "generic", - ".pkh": "generic", - ".pkb": "generic", - ".typ": "generic", - ".tyb": "generic", - ".tps": "generic", - ".tpb": "generic", - // ".tsql" - // ".trg", ".prc", ".fnc", ".pld", ".pls", ".plh", ".plb", ".pck", ".pks", ".pkh", ".pkb", ".typ", ".tyb", ".tps", ".tpb" - ".json": "json", - // ".scss" - // ".less" - ".go": "go", - // ".jsp" - // ".vm" - ".xml": "xml", - ".xsl": "xml", // missing from tests - ".wsdl": "xml", // missing from tests - ".pom": "xml", // missing from tests - ".cls": "apex", // missing from tests - ".trigger": "apex", // missing from testss - // ".component", ".page" - ".cs": "csharp", - ".kt": "kotlin", - ".kts": "kotlin", // missing from tests - ".ex": "elixir", // missing from tests - ".exs": "elixir", - // ".md", ".markdown", ".mdown", ".mkdn", ".mkd", ".mdwn", ".mkdown", ".ron" - // ".ps1", ".psc1", ".psd1", ".psm1", ".ps1xml", ".pssc", ".cdxml", ".clixml" - // ".cr" - // ".cbl", ".cob" - // ".groovy" - // ".abap" - // ".vb" - // ".m" - ".yaml": "yaml", // should these be Terraform? - ".yml": "yaml", - ".dart": "dart", // missing from tests - ".rs": "rust", - ".rlib": "rust", // missing from tests - ".clj": "clojure", - ".cljs": "clojure", // missing from tests - ".cljc": "clojure", // missing from tests - ".edn": "clojure", // missing from tests - // ".hs", ".lhs" - // ".erl" - // ".elm" - ".html": "html", // missing from tests - // ".pl" - // ".fs" - // ".f90", ".f95", ".f03" - ".r": "r", // missing from tests - // ".scratch", ".sb", ".sprite", ".sb2", ".sprite2" - ".lua": "lua", // missing from tests - ".asd": "lisp", // missing from tests - ".el": "lisp", // missing from tests - ".lsp": "lisp", // missing from tests - ".lisp": "lisp", // missing from tests - // ".P", ".swipl" - ".jl": "julia", // missing from tests - // ".ml", ".mli", ".mly", ".mll" - ".sol": "solidity", - ".tf": "terraform", -} +// filesToAnalyse holds every file to scan, regardless of language: opengrep +// picks the rules that apply to each file, so a single run covers all languages. +var filesToAnalyse []string -func populateFilesByLanguage(toolExecutionFiles *[]string, toolExecutionSourceDir string) error { +func populateFilesToAnalyse(toolExecutionFiles *[]string, toolExecutionSourceDir string) error { // If there are files to analyse, analyse only those files if toolExecutionFiles != nil && len(*toolExecutionFiles) > 0 { - return populateFilesByLanguageFromFiles(*toolExecutionFiles) + return populateFilesToAnalyseFromFiles(*toolExecutionFiles) } // If there are no files to analyse, analyse all files from source dir - return populateFilesByLanguageFromSourceDir(toolExecutionSourceDir) + return populateFilesToAnalyseFromSourceDir(toolExecutionSourceDir) } -func populateFilesByLanguageFromFiles(toolExecutionFiles []string) error { +func populateFilesToAnalyseFromFiles(toolExecutionFiles []string) error { for _, file := range toolExecutionFiles { - addFileToFilesByLanguage(file) + addFileToAnalyse(file) } return nil } -func populateFilesByLanguageFromSourceDir(toolExecutionSourceDir string) error { +func populateFilesToAnalyseFromSourceDir(toolExecutionSourceDir string) error { // Semgrep can analyse full directories and its subdirectories // but we will have to analyse every extension from every file // so we will have to do this walk somewhere else if we dont do it here @@ -382,26 +275,12 @@ func processFile(path string, info fs.DirEntry, err error) error { } // if it is a file and it is not a hidden file if !pathInfo.IsDir() && !strings.HasPrefix(pathInfo.Name(), ".") { - addFileToFilesByLanguage(path) + addFileToAnalyse(path) } return nil } -func addFileToFilesByLanguage(fileName string) { - language := detectLanguage(fileName) - filesByLanguage[language] = append(filesByLanguage[language], fileName) -} - -func detectLanguage(fileName string) string { - extension := strings.ToLower(filepath.Ext(fileName)) - extensionOrFilename := extension - if extension == "" { - extensionOrFilename = fileName - } - - if language, ok := extensionToLanguageMap[extensionOrFilename]; ok { - return language - } - return "none" +func addFileToAnalyse(fileName string) { + filesToAnalyse = append(filesToAnalyse, fileName) } diff --git a/internal/tool/configuration_test.go b/internal/tool/configuration_test.go index 4639012..d50314b 100644 --- a/internal/tool/configuration_test.go +++ b/internal/tool/configuration_test.go @@ -398,79 +398,16 @@ func TestWalkDirFuncWithError(t *testing.T) { assert.Error(t, err) } -func TestAddFileToFilesByLanguageWithGoFile(t *testing.T) { +func TestAddFileToAnalyseKeepsFilesOfAllLanguages(t *testing.T) { // Arrange - fileName := "file.go" + filesToAnalyse = nil + fileNames := []string{"file.go", "script.py", "document.docx"} // Act - addFileToFilesByLanguage(fileName) - - // Assert - assert.Contains(t, filesByLanguage["go"], fileName, "Expected file to be added to Go files") -} - -func TestAddFileToFilesByLanguageWithPythonFile(t *testing.T) { - // Arrange - fileName := "script.py" - - // Act - addFileToFilesByLanguage(fileName) - - // Assert - assert.Contains(t, filesByLanguage["python"], fileName, "Expected file to be added to Python files") -} - -func TestAddFileToFilesByLanguageWithUnknownFile(t *testing.T) { - // Arrange - fileName := "document.docx" - - // Act - addFileToFilesByLanguage(fileName) - - // Assert - assert.Contains(t, filesByLanguage, "none", "Expected file to be added to unknown language") -} - -func TestDetectLanguageWithGoExtension(t *testing.T) { - // Arrange - fileName := "file.go" - - // Act - language := detectLanguage(fileName) - - // Assert - assert.Equal(t, "go", language, "Expected language to be Go") -} - -func TestDetectLanguageWithPythonExtension(t *testing.T) { - // Arrange - fileName := "script.py" - - // Act - language := detectLanguage(fileName) - - // Assert - assert.Equal(t, "python", language, "Expected language to be Python") -} - -func TestDetectLanguageWithUnknownExtension(t *testing.T) { - // Arrange - fileName := "document.docx" - - // Act - language := detectLanguage(fileName) - - // Assert - assert.Equal(t, "none", language, "Expected language to be none for .docx file") -} - -func TestDetectLanguageWithoutExtension(t *testing.T) { - // Arrange - fileName := "file" - - // Act - language := detectLanguage(fileName) + for _, fileName := range fileNames { + addFileToAnalyse(fileName) + } // Assert - assert.Equal(t, "none", language, "Expected language to be none for unknown file type") + assert.Equal(t, fileNames, filesToAnalyse, "Expected all files to be analysed together") } diff --git a/internal/tool/tool.go b/internal/tool/tool.go index 5cedd0f..bcc68ae 100644 --- a/internal/tool/tool.go +++ b/internal/tool/tool.go @@ -47,7 +47,7 @@ func prepareToRun(toolExecution codacy.ToolExecution) (*os.File, *[]codacy.Patte return nil, nil, err } - err = populateFilesByLanguage(toolExecution.Files, toolExecution.SourceDir) + err = populateFilesToAnalyse(toolExecution.Files, toolExecution.SourceDir) if err != nil { return nil, nil, errors.New("Error getting files to analyse: " + err.Error()) } @@ -77,14 +77,5 @@ func loadPatternDescriptions() (*[]codacy.PatternDescription, error) { } func run(configurationFile *os.File, toolExecution codacy.ToolExecution, patternDescriptions *[]codacy.PatternDescription) ([]codacy.Result, error) { - var results []codacy.Result - for language, files := range filesByLanguage { - result, err := executeCommandForFiles(configurationFile, toolExecution, patternDescriptions, language, files) - if err != nil { - return nil, err - } - results = append(results, result...) - } - - return results, nil + return executeCommandForFiles(configurationFile, toolExecution, patternDescriptions, filesToAnalyse) }