Skip to content

Scan files of all languages in a single opengrep run #40

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
lolgab wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

Scan files of all languages in a single opengrep run #40

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 5 additions & 6 deletions internal/tool/command.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,11 +52,11 @@ type SemgrepErrorLocation struct {
// on repos with a very large number of files.
const maxFilesPerBatch = 1000

func executeCommandForFiles(configurationFile *os.File, toolExecution codacy.ToolExecution, patternDescriptions *[]codacy.PatternDescription, language string, files []string) ([]codacy.Result, error) {
func executeCommandForFiles(configurationFile *os.File, toolExecution codacy.ToolExecution, patternDescriptions *[]codacy.PatternDescription, files []string) ([]codacy.Result, error) {
var results []codacy.Result

for _, batch := range chunkFiles(files, maxFilesPerBatch) {
semgrepCmd := createCommand(configurationFile, toolExecution.SourceDir, language, batch)
semgrepCmd := createCommand(configurationFile, toolExecution.SourceDir, batch)

semgrepOutput, semgrepError, err := runCommand(semgrepCmd)
if err != nil {
Expand Down Expand Up @@ -86,15 +86,15 @@ func chunkFiles(files []string, size int) [][]string {
return chunks
}

func createCommand(configurationFile *os.File, sourceDir, language string, files []string) *exec.Cmd {
params := createCommandParameters(language, configurationFile, files)
func createCommand(configurationFile *os.File, sourceDir string, files []string) *exec.Cmd {
params := createCommandParameters(configurationFile, files)
cmd := exec.Command("/usr/local/bin/opengrep", params...)
cmd.Dir = sourceDir

return cmd
}

func createCommandParameters(language string, configurationFile *os.File, filesToAnalyse []string) []string {
func createCommandParameters(configurationFile *os.File, filesToAnalyse []string) []string {

// Reset file pointer to the beginning for later use
if _, err := configurationFile.Seek(0, io.SeekStart); err != nil {
Expand All @@ -104,7 +104,6 @@ func createCommandParameters(language string, configurationFile *os.File, filesT
"scan",
"--json", //"-json_nodots",
"--config", configurationFile.Name(),
//"-l", language,
"--timeout", "10",
"--timeout-threshold", "5",
"--max-target-bytes", "150000",
Expand Down
7 changes: 2 additions & 5 deletions internal/tool/command_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,10 @@ func TestCreateCommand(t *testing.T) {
configurationFile, _ := os.CreateTemp("", "config.*.yaml")
defer os.Remove(configurationFile.Name())
sourceDir := "/path/to/source"
language := "go"
files := []string{"file1.go", "file2.go"}

// Act
cmd := createCommand(configurationFile, sourceDir, language, files)
cmd := createCommand(configurationFile, sourceDir, files)

// Assert
assert.IsType(t, &exec.Cmd{}, cmd)
Expand All @@ -32,17 +31,15 @@ func TestCreateCommandParameters(t *testing.T) {
// Arrange
configurationFile, _ := os.CreateTemp("", "semgrep.yaml")
defer os.Remove(configurationFile.Name())
language := "go"
filesToAnalyse := []string{"file1.go", "file2.go"}

// Act
cmdParams := createCommandParameters(language, configurationFile, filesToAnalyse)
cmdParams := createCommandParameters(configurationFile, filesToAnalyse)

// Assert
expectedParams := []string{
"scan",
"--json",
//"-lang", language,
"--config", configurationFile.Name(),
"--timeout", "10",
"--timeout-threshold", "5",
Expand Down
145 changes: 12 additions & 133 deletions internal/tool/configuration.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -232,135 +232,28 @@ func formatParameterName(name string) string {
return result
}

var filesByLanguage = make(map[string][]string)

// Semgrep: supported language tags are: apex, bash, c, c#, c++, cairo, clojure, cpp, csharp, dart, docker, dockerfile, elixir, ex, generic, go, golang, hack, hcl, html, java, javascript, js, json, jsonnet, julia, kotlin, kt, lisp, lua, none, ocaml, php, promql, proto, proto3, protobuf, py, python, python2, python3, r, regex, ruby, rust, scala, scheme, sh, sol, solidity, swift, terraform, tf, ts, typescript, vue, xml, yaml
// Semgrep: https://github.com/semgrep/semgrep/blob/0ec2b95ec8c3afb8e31fc0295d3604e540c982b0/src/parsing/Unit_parsing.ml#L61
// Codacy: taken from https://github.com/codacy/ragnaros/blob/05d1374b7ca4a0aa3be44972484938b4785c046f/components/language/src/main/scala/codacy/foundation/api/Language.scala#L6
var extensionToLanguageMap = map[string]string{
".js": "javascript",
".jsx": "javascript",
".jsm": "javascript", // missing from tests
".vue": "vue",
".mjs": "javascript", // missing from tests
".scala": "scala",
// ".css"
".php": "php",
".py": "python",
".rb": "ruby",
".gemspec": "ruby", // missing from tests
".podspec": "ruby", // missing from tests
".jbuilder": "ruby", // missing from tests
".rake": "ruby", // missing from tests
".opal": "ruby", // missing from tests
".java": "java",
// ".coffee"
".swift": "swift",
".cpp": "cpp",
".hpp": "cpp", // missing from tests
".cc": "cpp", // missing from tests
".cxx": "cpp", // missing from tests
".ino": "cpp", // missing from tests
".c": "c",
".h": "c", // missing
".sh": "sh", // missing from tests
".bash": "bash",
".ts": "typescript",
".tsx": "typescript",
".dockerfile": "dockerfile",
"Dockerfile": "dockerfile",
".sql": "generic",
".pls": "generic",
".trg": "generic",
".prc": "generic",
".fnc": "generic",
".pld": "generic",
".plh": "generic",
".plb": "generic",
".pck": "generic",
".pks": "generic",
".pkh": "generic",
".pkb": "generic",
".typ": "generic",
".tyb": "generic",
".tps": "generic",
".tpb": "generic",
// ".tsql"
// ".trg", ".prc", ".fnc", ".pld", ".pls", ".plh", ".plb", ".pck", ".pks", ".pkh", ".pkb", ".typ", ".tyb", ".tps", ".tpb"
".json": "json",
// ".scss"
// ".less"
".go": "go",
// ".jsp"
// ".vm"
".xml": "xml",
".xsl": "xml", // missing from tests
".wsdl": "xml", // missing from tests
".pom": "xml", // missing from tests
".cls": "apex", // missing from tests
".trigger": "apex", // missing from testss
// ".component", ".page"
".cs": "csharp",
".kt": "kotlin",
".kts": "kotlin", // missing from tests
".ex": "elixir", // missing from tests
".exs": "elixir",
// ".md", ".markdown", ".mdown", ".mkdn", ".mkd", ".mdwn", ".mkdown", ".ron"
// ".ps1", ".psc1", ".psd1", ".psm1", ".ps1xml", ".pssc", ".cdxml", ".clixml"
// ".cr"
// ".cbl", ".cob"
// ".groovy"
// ".abap"
// ".vb"
// ".m"
".yaml": "yaml", // should these be Terraform?
".yml": "yaml",
".dart": "dart", // missing from tests
".rs": "rust",
".rlib": "rust", // missing from tests
".clj": "clojure",
".cljs": "clojure", // missing from tests
".cljc": "clojure", // missing from tests
".edn": "clojure", // missing from tests
// ".hs", ".lhs"
// ".erl"
// ".elm"
".html": "html", // missing from tests
// ".pl"
// ".fs"
// ".f90", ".f95", ".f03"
".r": "r", // missing from tests
// ".scratch", ".sb", ".sprite", ".sb2", ".sprite2"
".lua": "lua", // missing from tests
".asd": "lisp", // missing from tests
".el": "lisp", // missing from tests
".lsp": "lisp", // missing from tests
".lisp": "lisp", // missing from tests
// ".P", ".swipl"
".jl": "julia", // missing from tests
// ".ml", ".mli", ".mly", ".mll"
".sol": "solidity",
".tf": "terraform",
}
// filesToAnalyse holds every file to scan, regardless of language: opengrep
// picks the rules that apply to each file, so a single run covers all languages.
var filesToAnalyse []string

func populateFilesByLanguage(toolExecutionFiles *[]string, toolExecutionSourceDir string) error {
func populateFilesToAnalyse(toolExecutionFiles *[]string, toolExecutionSourceDir string) error {
// If there are files to analyse, analyse only those files
if toolExecutionFiles != nil && len(*toolExecutionFiles) > 0 {
return populateFilesByLanguageFromFiles(*toolExecutionFiles)
return populateFilesToAnalyseFromFiles(*toolExecutionFiles)
}
// If there are no files to analyse, analyse all files from source dir
return populateFilesByLanguageFromSourceDir(toolExecutionSourceDir)
return populateFilesToAnalyseFromSourceDir(toolExecutionSourceDir)
}

func populateFilesByLanguageFromFiles(toolExecutionFiles []string) error {
func populateFilesToAnalyseFromFiles(toolExecutionFiles []string) error {
for _, file := range toolExecutionFiles {
addFileToFilesByLanguage(file)
addFileToAnalyse(file)
}

return nil
}

func populateFilesByLanguageFromSourceDir(toolExecutionSourceDir string) error {
func populateFilesToAnalyseFromSourceDir(toolExecutionSourceDir string) error {
// Semgrep can analyse full directories and its subdirectories
// but we will have to analyse every extension from every file
// so we will have to do this walk somewhere else if we dont do it here
Expand All @@ -382,26 +275,12 @@ func processFile(path string, info fs.DirEntry, err error) error {
}
// if it is a file and it is not a hidden file
if !pathInfo.IsDir() && !strings.HasPrefix(pathInfo.Name(), ".") {
addFileToFilesByLanguage(path)
addFileToAnalyse(path)
}

return nil
}

func addFileToFilesByLanguage(fileName string) {
language := detectLanguage(fileName)
filesByLanguage[language] = append(filesByLanguage[language], fileName)
}

func detectLanguage(fileName string) string {
extension := strings.ToLower(filepath.Ext(fileName))
extensionOrFilename := extension
if extension == "" {
extensionOrFilename = fileName
}

if language, ok := extensionToLanguageMap[extensionOrFilename]; ok {
return language
}
return "none"
func addFileToAnalyse(fileName string) {
filesToAnalyse = append(filesToAnalyse, fileName)
}
77 changes: 7 additions & 70 deletions internal/tool/configuration_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -398,79 +398,16 @@ func TestWalkDirFuncWithError(t *testing.T) {
assert.Error(t, err)
}

func TestAddFileToFilesByLanguageWithGoFile(t *testing.T) {
func TestAddFileToAnalyseKeepsFilesOfAllLanguages(t *testing.T) {
// Arrange
fileName := "file.go"
filesToAnalyse = nil
fileNames := []string{"file.go", "script.py", "document.docx"}

// Act
addFileToFilesByLanguage(fileName)

// Assert
assert.Contains(t, filesByLanguage["go"], fileName, "Expected file to be added to Go files")
}

func TestAddFileToFilesByLanguageWithPythonFile(t *testing.T) {
// Arrange
fileName := "script.py"

// Act
addFileToFilesByLanguage(fileName)

// Assert
assert.Contains(t, filesByLanguage["python"], fileName, "Expected file to be added to Python files")
}

func TestAddFileToFilesByLanguageWithUnknownFile(t *testing.T) {
// Arrange
fileName := "document.docx"

// Act
addFileToFilesByLanguage(fileName)

// Assert
assert.Contains(t, filesByLanguage, "none", "Expected file to be added to unknown language")
}

func TestDetectLanguageWithGoExtension(t *testing.T) {
// Arrange
fileName := "file.go"

// Act
language := detectLanguage(fileName)

// Assert
assert.Equal(t, "go", language, "Expected language to be Go")
}

func TestDetectLanguageWithPythonExtension(t *testing.T) {
// Arrange
fileName := "script.py"

// Act
language := detectLanguage(fileName)

// Assert
assert.Equal(t, "python", language, "Expected language to be Python")
}

func TestDetectLanguageWithUnknownExtension(t *testing.T) {
// Arrange
fileName := "document.docx"

// Act
language := detectLanguage(fileName)

// Assert
assert.Equal(t, "none", language, "Expected language to be none for .docx file")
}

func TestDetectLanguageWithoutExtension(t *testing.T) {
// Arrange
fileName := "file"

// Act
language := detectLanguage(fileName)
for _, fileName := range fileNames {
addFileToAnalyse(fileName)
}

// Assert
assert.Equal(t, "none", language, "Expected language to be none for unknown file type")
assert.Equal(t, fileNames, filesToAnalyse, "Expected all files to be analysed together")
}
13 changes: 2 additions & 11 deletions internal/tool/tool.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ func prepareToRun(toolExecution codacy.ToolExecution) (*os.File, *[]codacy.Patte
return nil, nil, err
}

err = populateFilesByLanguage(toolExecution.Files, toolExecution.SourceDir)
err = populateFilesToAnalyse(toolExecution.Files, toolExecution.SourceDir)
if err != nil {
return nil, nil, errors.New("Error getting files to analyse: " + err.Error())
}
Expand Down Expand Up @@ -77,14 +77,5 @@ func loadPatternDescriptions() (*[]codacy.PatternDescription, error) {
}

func run(configurationFile *os.File, toolExecution codacy.ToolExecution, patternDescriptions *[]codacy.PatternDescription) ([]codacy.Result, error) {
var results []codacy.Result
for language, files := range filesByLanguage {
result, err := executeCommandForFiles(configurationFile, toolExecution, patternDescriptions, language, files)
if err != nil {
return nil, err
}
results = append(results, result...)
}

return results, nil
return executeCommandForFiles(configurationFile, toolExecution, patternDescriptions, filesToAnalyse)
}