Skip to content

Commit 2b66d9d

Browse files
maksim-grebeniuk-sonarsourcesonartech
maksim-grebeniuk-sonarsource
authored and
sonartech
committed
SONARPY-2767 Implement parallel rule execution (#217)
GitOrigin-RevId: dbfbc0c15f05a6643521b01f65d4b3c2687b92b4
1 parent 3ea00e8 commit 2b66d9d

4 files changed

Lines changed: 65 additions & 34 deletions

File tree

‎python-commons/src/main/java/org/sonar/plugins/python/PythonScanner.java‎

Lines changed: 51 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,12 @@
3030
import java.util.Optional;
3131
import java.util.Set;
3232
import java.util.concurrent.ConcurrentHashMap;
33+
import java.util.concurrent.ForkJoinPool;
34+
import java.util.concurrent.atomic.AtomicBoolean;
35+
import java.util.concurrent.atomic.AtomicInteger;
3336
import java.util.function.Supplier;
3437
import java.util.regex.Pattern;
38+
import java.util.stream.Stream;
3539
import javax.annotation.CheckForNull;
3640
import org.slf4j.Logger;
3741
import org.slf4j.LoggerFactory;
@@ -71,6 +75,7 @@
7175
public class PythonScanner extends Scanner {
7276

7377
private static final Logger LOG = LoggerFactory.getLogger(PythonScanner.class);
78+
private static final Pattern DATABRICKS_MAGIC_COMMAND_PATTERN = Pattern.compile("^\\h*#\\h*(MAGIC|COMMAND).*");
7479

7580
private final Supplier<PythonParser> parserSupplier;
7681
private final PythonChecks checks;
@@ -79,9 +84,8 @@ public class PythonScanner extends Scanner {
7984
private final PythonCpdAnalyzer cpdAnalyzer;
8085
private final PythonIndexer indexer;
8186
private final Map<PythonInputFile, Set<Class<? extends PythonCheck>>> checksExecutedWithoutParsingByFiles;
82-
private int recognitionErrorCount = 0;
83-
private static final Pattern DATABRICKS_MAGIC_COMMAND_PATTERN = Pattern.compile("^\\h*#\\h*(MAGIC|COMMAND).*");
84-
private boolean foundDatabricks = false;
87+
private final AtomicInteger recognitionErrorCount;
88+
private final AtomicBoolean foundDatabricks;
8589
private final PythonFileConsumer architectureCallback;
8690

8791
public PythonScanner(
@@ -97,13 +101,32 @@ public PythonScanner(
97101
this.indexer.buildOnce(context);
98102
this.architectureCallback = architectureCallback;
99103
this.checksExecutedWithoutParsingByFiles = new ConcurrentHashMap<>();
104+
this.recognitionErrorCount = new AtomicInteger(0);
105+
this.foundDatabricks = new AtomicBoolean(false);
100106
}
101107

102108
@Override
103109
protected String name() {
104110
return "rules execution";
105111
}
106112

113+
@Override
114+
protected void processFiles(List<PythonInputFile> files, SensorContext context, MultiFileProgressReport progressReport,
115+
AtomicInteger numScannedWithoutParsing) {
116+
ForkJoinPool pool = new ForkJoinPool(1);
117+
try {
118+
pool.submit(() -> super.processFiles(files, context, progressReport, numScannedWithoutParsing))
119+
.join();
120+
} finally {
121+
pool.shutdown();
122+
}
123+
}
124+
125+
@Override
126+
protected Stream<PythonInputFile> getFilesStream(List<PythonInputFile> files) {
127+
return files.stream().parallel();
128+
}
129+
107130
@Override
108131
protected void scanFile(PythonInputFile inputFile) throws IOException {
109132
var pythonFile = SonarQubePythonFile.create(inputFile);
@@ -126,12 +149,13 @@ protected void scanFile(PythonInputFile inputFile) throws IOException {
126149
} catch (RecognitionException e) {
127150
visitorContext = new PythonVisitorContext(pythonFile, e, context.runtime().getProduct());
128151

129-
var line = (inputFile.kind() == PythonInputFile.Kind.IPYTHON) ? ((GeneratedIPythonFile) inputFile).locationMap().get(e.getLine()).line() : e.getLine();
152+
var line = (inputFile.kind() == PythonInputFile.Kind.IPYTHON) ?
153+
((GeneratedIPythonFile) inputFile).locationMap().get(e.getLine()).line() : e.getLine();
130154
var newMessage = e.getMessage().replace("line " + e.getLine(), "line " + line);
131155

132-
LOG.error("Unable to parse file: " + inputFile);
156+
LOG.error("Unable to parse file: {}", inputFile);
133157
LOG.error(newMessage);
134-
recognitionErrorCount++;
158+
recognitionErrorCount.incrementAndGet();
135159
context.newAnalysisError()
136160
.onFile(inputFile.wrappedFile())
137161
.at(inputFile.wrappedFile().newPointer(line, 0))
@@ -140,8 +164,8 @@ protected void scanFile(PythonInputFile inputFile) throws IOException {
140164
}
141165
List<PythonSubscriptionCheck> checksBasedOnTree = new ArrayList<>();
142166
for (PythonCheck check : checks.all()) {
143-
if (!isCheckApplicable(check, fileType)
144-
|| checksExecutedWithoutParsingByFiles.getOrDefault(inputFile, Collections.emptySet()).contains(check.getClass())) {
167+
if (isCheckNotApplicable(check, fileType)
168+
|| checksExecutedWithoutParsingByFiles.getOrDefault(inputFile, Collections.emptySet()).contains(check.getClass())) {
145169
continue;
146170
}
147171
if (check instanceof PythonSubscriptionCheck pythonSubscriptionCheck) {
@@ -163,16 +187,18 @@ protected void scanFile(PythonInputFile inputFile) throws IOException {
163187
}
164188

165189
private void searchForDataBricks(PythonVisitorContext visitorContext) {
166-
foundDatabricks |= visitorContext.pythonFile().content().lines().anyMatch(
190+
var hasDatabricks = visitorContext.pythonFile().content().lines().anyMatch(
167191
line -> DATABRICKS_MAGIC_COMMAND_PATTERN.matcher(line).matches());
192+
foundDatabricks.compareAndSet(false, hasDatabricks);
168193
}
169194

170195
private static PythonTreeMaker getTreeMaker(PythonInputFile inputFile) {
171-
return Python.KEY.equals(inputFile.wrappedFile().language()) ? new PythonTreeMaker() : new IPythonTreeMaker(getOffsetLocations(inputFile));
196+
return Python.KEY.equals(inputFile.wrappedFile().language()) ? new PythonTreeMaker() :
197+
new IPythonTreeMaker(getOffsetLocations(inputFile));
172198
}
173199

174200
private static Map<Integer, IPythonLocation> getOffsetLocations(PythonInputFile inputFile) {
175-
if(inputFile.kind() == PythonInputFile.Kind.IPYTHON){
201+
if (inputFile.kind() == PythonInputFile.Kind.IPYTHON) {
176202
return ((GeneratedIPythonFile) inputFile).locationMap();
177203
}
178204
return Map.of();
@@ -191,7 +217,7 @@ public boolean scanFileWithoutParsing(PythonInputFile inputFile) {
191217
indexer.projectLevelSymbolTable()
192218
);
193219
for (PythonCheck check : checks.all()) {
194-
if (!isCheckApplicable(check, fileType)) {
220+
if (isCheckNotApplicable(check, fileType)) {
195221
continue;
196222
}
197223
if (checkRequiresParsingOfImpactedFile(inputFile, check)) {
@@ -228,12 +254,8 @@ public void endOfAnalysis() {
228254
checks.endOfAnalyses().forEach(c -> c.endOfAnalysis(indexer.cacheContext()));
229255
}
230256

231-
boolean isCheckApplicable(PythonCheck pythonCheck, InputFile.Type fileType) {
232-
PythonCheck.CheckScope checkScope = pythonCheck.scope();
233-
if (checkScope == PythonCheck.CheckScope.ALL) {
234-
return true;
235-
}
236-
return fileType == InputFile.Type.MAIN;
257+
boolean isCheckNotApplicable(PythonCheck pythonCheck, InputFile.Type fileType) {
258+
return fileType != InputFile.Type.MAIN && pythonCheck.scope() != PythonCheck.CheckScope.ALL;
237259
}
238260

239261
// visible for testing
@@ -257,11 +279,12 @@ public boolean canBeScannedWithoutParsing(PythonInputFile inputFile) {
257279

258280
@Override
259281
protected void reportStatistics(int numSkippedFiles, int numTotalFiles) {
260-
LOG.info("The Python analyzer was able to leverage cached data from previous analyses for {} out of {} files. These files were not parsed.",
282+
LOG.info("The Python analyzer was able to leverage cached data from previous analyses for {} out of {} files. These files were not " +
283+
"parsed.",
261284
numSkippedFiles, numTotalFiles);
262285
}
263286

264-
private void saveIssues(PythonInputFile inputFile, List<PreciseIssue> issues) {
287+
private synchronized void saveIssues(PythonInputFile inputFile, List<PreciseIssue> issues) {
265288
for (PreciseIssue preciseIssue : issues) {
266289
RuleKey ruleKey = checks.ruleKey(preciseIssue.check());
267290
NewIssue newIssue = context
@@ -304,7 +327,8 @@ private void saveIssues(PythonInputFile inputFile, List<PreciseIssue> issues) {
304327

305328
@CheckForNull
306329
private InputFile component(String fileId, SensorContext sensorContext) {
307-
InputFile inputFile = Optional.ofNullable(sensorContext.fileSystem().inputFile(sensorContext.fileSystem().predicates().is(new File(fileId))))
330+
var predicate = sensorContext.fileSystem().predicates().is(new File(fileId));
331+
InputFile inputFile = Optional.ofNullable(sensorContext.fileSystem().inputFile(predicate))
308332
.orElseGet(() -> indexer.getFileWithId(fileId));
309333
if (inputFile == null) {
310334
LOG.debug("Failed to find InputFile for {}", fileId);
@@ -320,7 +344,8 @@ private static NewIssueLocation newLocation(PythonInputFile inputFile, NewIssue
320344
if (location.startLineOffset() == IssueLocation.UNDEFINED_OFFSET) {
321345
range = inputFile.wrappedFile().selectLine(location.startLine());
322346
} else {
323-
range = inputFile.wrappedFile().newRange(location.startLine(), location.startLineOffset(), location.endLine(), location.endLineOffset());
347+
range = inputFile.wrappedFile().newRange(location.startLine(), location.startLineOffset(), location.endLine(),
348+
location.endLineOffset());
324349
}
325350
newLocation.at(range);
326351
}
@@ -415,14 +440,15 @@ private static void addQuickFixes(InputFile inputFile, RuleKey ruleKey, Iterable
415440
}
416441

417442
private static TextRange rangeFromTextSpan(InputFile file, PythonTextEdit pythonTextEdit) {
418-
return file.newRange(pythonTextEdit.startLine(), pythonTextEdit.startLineOffset(), pythonTextEdit.endLine(), pythonTextEdit.endLineOffset());
443+
return file.newRange(pythonTextEdit.startLine(), pythonTextEdit.startLineOffset(), pythonTextEdit.endLine(),
444+
pythonTextEdit.endLineOffset());
419445
}
420446

421447
public int getRecognitionErrorCount() {
422-
return recognitionErrorCount;
448+
return recognitionErrorCount.get();
423449
}
424450

425451
public boolean getFoundDatabricks() {
426-
return foundDatabricks;
452+
return foundDatabricks.get();
427453
}
428454
}

‎python-frontend/src/main/java/org/sonar/plugins/python/api/PythonVisitorContext.java‎

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -46,9 +46,7 @@ public class PythonVisitorContext extends PythonInputFileContext {
4646

4747
public PythonVisitorContext(FileInput rootTree, PythonFile pythonFile, @Nullable File workingDirectory, String packageName) {
4848
super(pythonFile, workingDirectory, CacheContextImpl.dummyCache(), ProjectLevelSymbolTable.empty());
49-
50-
SymbolTableBuilder symbolTableBuilder = new SymbolTableBuilder(packageName, pythonFile);
51-
symbolTableBuilder.visitFileInput(rootTree);
49+
buildSymbols(rootTree, pythonFile, packageName);
5250
var symbolTable = new SymbolTableBuilderV2(rootTree).build();
5351
var projectLevelTypeTable = new ProjectLevelTypeTable(ProjectLevelSymbolTable.empty());
5452

@@ -63,8 +61,7 @@ public PythonVisitorContext(FileInput rootTree, PythonFile pythonFile, @Nullable
6361
ProjectLevelSymbolTable projectLevelSymbolTable, CacheContext cacheContext) {
6462
super(pythonFile, workingDirectory, cacheContext, projectLevelSymbolTable);
6563

66-
var symbolTableBuilder = new SymbolTableBuilder(packageName, pythonFile, projectLevelSymbolTable);
67-
symbolTableBuilder.visitFileInput(rootTree);
64+
buildSymbols(rootTree, pythonFile, packageName, projectLevelSymbolTable);
6865
var symbolTable = new SymbolTableBuilderV2(rootTree).build();
6966
var projectLevelTypeTable = new ProjectLevelTypeTable(projectLevelSymbolTable);
7067

@@ -80,8 +77,7 @@ public PythonVisitorContext(FileInput rootTree, PythonFile pythonFile, @Nullable
8077
super(pythonFile, workingDirectory, cacheContext, sonarProduct, projectLevelSymbolTable);
8178
var symbolTableBuilderV2 = new SymbolTableBuilderV2(rootTree);
8279
var symbolTable = symbolTableBuilderV2.build();
83-
var symbolTableBuilder = new SymbolTableBuilder(packageName, pythonFile, projectLevelSymbolTable);
84-
symbolTableBuilder.visitFileInput(rootTree);
80+
buildSymbols(rootTree, pythonFile, packageName, projectLevelSymbolTable);
8581
var projectLevelTypeTable = new ProjectLevelTypeTable(projectLevelSymbolTable);
8682
this.moduleType = new TypeInferenceV2(projectLevelTypeTable, pythonFile, symbolTable, packageName).inferModuleType(rootTree);
8783
this.typeChecker = new TypeChecker(projectLevelTypeTable);
@@ -90,6 +86,15 @@ public PythonVisitorContext(FileInput rootTree, PythonFile pythonFile, @Nullable
9086
this.issues = new ArrayList<>();
9187
}
9288

89+
private static synchronized void buildSymbols(FileInput rootTree, PythonFile pythonFile, String packageName) {
90+
buildSymbols(rootTree, pythonFile, packageName, ProjectLevelSymbolTable.empty());
91+
}
92+
93+
private static synchronized void buildSymbols(FileInput rootTree, PythonFile pythonFile, String packageName, ProjectLevelSymbolTable projectLevelSymbolTable) {
94+
var symbolTableBuilder = new SymbolTableBuilder(packageName, pythonFile, projectLevelSymbolTable);
95+
symbolTableBuilder.visitFileInput(rootTree);
96+
}
97+
9398
public PythonVisitorContext(PythonFile pythonFile, RecognitionException parsingException) {
9499
super(pythonFile, null, CacheContextImpl.dummyCache(), ProjectLevelSymbolTable.empty());
95100
this.rootTree = null;

‎python-frontend/src/main/java/org/sonar/python/semantic/ProjectLevelSymbolTable.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -183,7 +183,7 @@ public Symbol getSymbol(@Nullable String fullyQualifiedName, @Nullable String lo
183183
}
184184

185185
@CheckForNull
186-
public Set<Symbol> getSymbolsFromModule(@Nullable String moduleName) {
186+
public synchronized Set<Symbol> getSymbolsFromModule(@Nullable String moduleName) {
187187
Set<Descriptor> descriptors = getDescriptorsFromModule(moduleName);
188188
if (descriptors == null) {
189189
return null;

‎python-frontend/src/main/java/org/sonar/python/semantic/v2/SymbolsModuleTypeProvider.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ public ModuleType createBuiltinModule() {
6060
return rootModule;
6161
}
6262

63-
public PythonType convertModuleType(List<String> moduleFqn, ModuleType parent) {
63+
public synchronized PythonType convertModuleType(List<String> moduleFqn, ModuleType parent) {
6464
var moduleName = moduleFqn.get(moduleFqn.size() - 1);
6565
var moduleFqnString = getModuleFqnString(moduleFqn);
6666
Optional<ModuleType> result = createModuleTypeFromProjectLevelSymbolTable(moduleName, moduleFqnString, parent)

0 commit comments

Comments
 (0)