Skip to content

Commit b97c7e4

Browse files
maksim-grebeniuk-sonarsourcesonartech
maksim-grebeniuk-sonarsource
authored and
sonartech
committed
SONARPY-2887 Introduce MeasuresRepository to same measures in a thread-safe way (#237)
GitOrigin-RevId: 76af05590ca544815f0ea3082a3e71869b48fbad
1 parent 7937219 commit b97c7e4

4 files changed

Lines changed: 159 additions & 70 deletions

File tree

‎python-commons/src/main/java/org/sonar/plugins/python/MeasuresRepository.java‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
/*
2+
* SonarQube Python Plugin
3+
* Copyright (C) 2011-2025 SonarSource SA
4+
* mailto:info AT sonarsource DOT com
5+
*
6+
* This program is free software; you can redistribute it and/or
7+
* modify it under the terms of the Sonar Source-Available License Version 1, as published by SonarSource SA.
8+
*
9+
* This program is distributed in the hope that it will be useful,
10+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
12+
* See the Sonar Source-Available License for more details.
13+
*
14+
* You should have received a copy of the Sonar Source-Available License
15+
* along with this program; if not, see https://sonarsource.com/license/ssal/
16+
*/
17+
package org.sonar.plugins.python;
18+
19+
import java.util.Set;
20+
import org.sonar.api.batch.sensor.SensorContext;
21+
import org.sonar.api.batch.sensor.measure.NewMeasure;
22+
import org.sonar.api.issue.NoSonarFilter;
23+
import org.sonar.api.measures.CoreMetrics;
24+
import org.sonar.api.measures.FileLinesContext;
25+
import org.sonar.api.measures.FileLinesContextFactory;
26+
import org.sonar.api.measures.Metric;
27+
import org.sonar.plugins.python.api.PythonVisitorContext;
28+
import org.sonar.python.metrics.FileLinesVisitor;
29+
import org.sonar.python.metrics.FileMetrics;
30+
31+
public class MeasuresRepository {
32+
private final SensorContext context;
33+
private final NoSonarFilter noSonarFilter;
34+
private final FileLinesContextFactory fileLinesContextFactory;
35+
private final boolean isInSonarLint;
36+
private final Object monitor;
37+
38+
public MeasuresRepository(SensorContext context, NoSonarFilter noSonarFilter, FileLinesContextFactory fileLinesContextFactory, boolean isInSonarLint, Object monitor) {
39+
this.context = context;
40+
this.noSonarFilter = noSonarFilter;
41+
this.fileLinesContextFactory = fileLinesContextFactory;
42+
this.isInSonarLint = isInSonarLint;
43+
this.monitor = monitor;
44+
}
45+
46+
public synchronized void save(PythonInputFile inputFile, PythonVisitorContext visitorContext) {
47+
FileMetrics fileMetrics = new FileMetrics(visitorContext, isNotebook(inputFile));
48+
FileLinesVisitor fileLinesVisitor = fileMetrics.fileLinesVisitor();
49+
50+
processNoSonarInFile(inputFile, fileLinesVisitor);
51+
52+
if (!isInSonarLint) {
53+
Set<Integer> linesOfCode = fileLinesVisitor.getLinesOfCode();
54+
saveMetricOnFile(inputFile, CoreMetrics.NCLOC, linesOfCode.size());
55+
saveMetricOnFile(inputFile, CoreMetrics.STATEMENTS, fileMetrics.numberOfStatements());
56+
saveMetricOnFile(inputFile, CoreMetrics.FUNCTIONS, fileMetrics.numberOfFunctions());
57+
saveMetricOnFile(inputFile, CoreMetrics.CLASSES, fileMetrics.numberOfClasses());
58+
saveMetricOnFile(inputFile, CoreMetrics.COMPLEXITY, fileMetrics.complexity());
59+
saveMetricOnFile(inputFile, CoreMetrics.COGNITIVE_COMPLEXITY, fileMetrics.cognitiveComplexity());
60+
saveMetricOnFile(inputFile, CoreMetrics.COMMENT_LINES, fileLinesVisitor.getCommentLineCount());
61+
62+
FileLinesContext fileLinesContext = fileLinesContextFactory.createFor(inputFile.wrappedFile());
63+
if (inputFile.kind() == PythonInputFile.Kind.PYTHON) {
64+
for (int line : linesOfCode) {
65+
fileLinesContext.setIntValue(CoreMetrics.NCLOC_DATA_KEY, line, 1);
66+
}
67+
}
68+
for (int line : fileLinesVisitor.getExecutableLines()) {
69+
fileLinesContext.setIntValue(CoreMetrics.EXECUTABLE_LINES_DATA_KEY, line, 1);
70+
}
71+
save(fileLinesContext);
72+
}
73+
}
74+
75+
private void processNoSonarInFile(PythonInputFile inputFile, FileLinesVisitor fileLinesVisitor) {
76+
synchronized (monitor) {
77+
noSonarFilter.noSonarInFile(inputFile.wrappedFile(), fileLinesVisitor.getLinesWithNoSonar());
78+
}
79+
}
80+
81+
static boolean isNotebook(PythonInputFile inputFile) {
82+
return inputFile.kind() == PythonInputFile.Kind.IPYTHON;
83+
}
84+
85+
private void saveMetricOnFile(PythonInputFile inputFile, Metric<Integer> metric, Integer value) {
86+
var measure = context.<Integer>newMeasure()
87+
.withValue(value)
88+
.forMetric(metric)
89+
.on(inputFile.wrappedFile());
90+
save(measure);
91+
}
92+
93+
private void save(FileLinesContext fileLinesContext) {
94+
synchronized (monitor) {
95+
fileLinesContext.save();
96+
}
97+
}
98+
99+
private void save(NewMeasure<Integer> measure) {
100+
synchronized (monitor) {
101+
measure.save();
102+
}
103+
}
104+
}

‎python-commons/src/main/java/org/sonar/plugins/python/PythonScanner.java‎

Lines changed: 17 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -39,10 +39,7 @@
3939
import org.sonar.api.batch.fs.InputFile;
4040
import org.sonar.api.batch.sensor.SensorContext;
4141
import org.sonar.api.issue.NoSonarFilter;
42-
import org.sonar.api.measures.CoreMetrics;
43-
import org.sonar.api.measures.FileLinesContext;
4442
import org.sonar.api.measures.FileLinesContextFactory;
45-
import org.sonar.api.measures.Metric;
4643
import org.sonar.plugins.python.api.PythonCheck;
4744
import org.sonar.plugins.python.api.PythonFile;
4845
import org.sonar.plugins.python.api.PythonFileConsumer;
@@ -54,8 +51,6 @@
5451
import org.sonar.plugins.python.indexer.PythonIndexer;
5552
import org.sonar.python.IPythonLocation;
5653
import org.sonar.python.SubscriptionVisitor;
57-
import org.sonar.python.metrics.FileLinesVisitor;
58-
import org.sonar.python.metrics.FileMetrics;
5954
import org.sonar.python.parser.PythonParser;
6055
import org.sonar.python.tree.IPythonTreeMaker;
6156
import org.sonar.python.tree.PythonTreeMaker;
@@ -69,8 +64,6 @@ public class PythonScanner extends Scanner {
6964

7065
private final Supplier<PythonParser> parserSupplier;
7166
private final PythonChecks checks;
72-
private final FileLinesContextFactory fileLinesContextFactory;
73-
private final NoSonarFilter noSonarFilter;
7467
private final PythonCpdAnalyzer cpdAnalyzer;
7568
private final PythonIndexer indexer;
7669
private final Map<PythonInputFile, Set<Class<? extends PythonCheck>>> checksExecutedWithoutParsingByFiles;
@@ -81,14 +74,13 @@ public class PythonScanner extends Scanner {
8174
private final NewSymbolsCollector newSymbolsCollector;
8275
private final PythonHighlighter pythonHighlighter;
8376
private final IssuesRepository issuesRepository;
77+
private final MeasuresRepository measuresRepository;
8478

8579
public PythonScanner(
8680
SensorContext context, PythonChecks checks, FileLinesContextFactory fileLinesContextFactory, NoSonarFilter noSonarFilter,
8781
Supplier<PythonParser> parserSupplier, PythonIndexer indexer, PythonFileConsumer architectureCallback) {
8882
super(context);
8983
this.checks = checks;
90-
this.fileLinesContextFactory = fileLinesContextFactory;
91-
this.noSonarFilter = noSonarFilter;
9284
this.cpdAnalyzer = new PythonCpdAnalyzer(context);
9385
this.parserSupplier = parserSupplier;
9486
this.indexer = indexer;
@@ -101,6 +93,7 @@ public PythonScanner(
10193
this.newSymbolsCollector = new NewSymbolsCollector(this);
10294
this.pythonHighlighter = new PythonHighlighter(this);
10395
this.issuesRepository = new IssuesRepository(context, checks, indexer, isInSonarLint(context), this);
96+
this.measuresRepository = new MeasuresRepository(context, noSonarFilter, fileLinesContextFactory, isInSonarLint(context), this);
10497
}
10598

10699
@Override
@@ -150,7 +143,8 @@ private PythonVisitorContext createVisitorContext(PythonInputFile inputFile, Pyt
150143
indexer.cacheContext(),
151144
context.runtime().getProduct());
152145
if (fileType == InputFile.Type.MAIN) {
153-
saveMeasures(inputFile, visitorContext);
146+
pushTokens(inputFile, visitorContext);
147+
measuresRepository.save(inputFile, visitorContext);
154148
}
155149
} catch (RecognitionException e) {
156150
visitorContext = new PythonVisitorContext(pythonFile, e, context.runtime().getProduct());
@@ -171,11 +165,18 @@ private PythonVisitorContext createVisitorContext(PythonInputFile inputFile, Pyt
171165
return visitorContext;
172166
}
173167

174-
private void executeChecks(PythonVisitorContext visitorContext, Collection<PythonCheck> checks, InputFile.Type fileType, PythonInputFile inputFile) {
168+
private synchronized void pushTokens(PythonInputFile inputFile, PythonVisitorContext visitorContext) {
169+
if (!isInSonarLint(context) && inputFile.kind() == PythonInputFile.Kind.PYTHON) {
170+
cpdAnalyzer.pushCpdTokens(inputFile.wrappedFile(), visitorContext);
171+
}
172+
}
173+
174+
private void executeChecks(PythonVisitorContext visitorContext, Collection<PythonCheck> checks, InputFile.Type fileType,
175+
PythonInputFile inputFile) {
175176
Collection<PythonSubscriptionCheck> subscriptionChecks = new ArrayList<>();
176177
for (PythonCheck check : checks) {
177178
if (isCheckNotApplicable(check, fileType)
178-
|| checksExecutedWithoutParsingByFiles.getOrDefault(inputFile, Collections.emptySet()).contains(check.getClass())) {
179+
|| checksExecutedWithoutParsingByFiles.getOrDefault(inputFile, Collections.emptySet()).contains(check.getClass())) {
179180
continue;
180181
}
181182
if (check instanceof PythonSubscriptionCheck pythonSubscriptionCheck) {
@@ -251,7 +252,8 @@ public boolean scanFileWithoutParsing(PythonInputFile inputFile) {
251252
return restoreAndPushMeasuresIfApplicable(inputFile);
252253
}
253254

254-
private boolean scanFileWithoutParsingNotSonarPython(PythonInputFile inputFile, PythonCheck check, InputFile.Type fileType, boolean result,
255+
private boolean scanFileWithoutParsingNotSonarPython(PythonInputFile inputFile, PythonCheck check, InputFile.Type fileType,
256+
boolean result,
255257
PythonInputFileContext inputFileContext) {
256258
if (isCheckNotApplicable(check, fileType)) {
257259
return result;
@@ -270,7 +272,8 @@ private boolean scanFileWithoutParsingNotSonarPython(PythonInputFile inputFile,
270272
return result;
271273
}
272274

273-
private boolean scanFileWithoutParsingSonarPython(PythonInputFile inputFile, InputFile.Type fileType, PythonInputFileContext inputFileContext, boolean result) {
275+
private boolean scanFileWithoutParsingSonarPython(PythonInputFile inputFile, InputFile.Type fileType,
276+
PythonInputFileContext inputFileContext, boolean result) {
274277
var ourChecks = checks.sonarPythonChecks();
275278
for (var check : ourChecks) {
276279
if (isCheckNotApplicable(check, fileType)) {
@@ -332,43 +335,6 @@ protected void reportStatistics(int numSkippedFiles, int numTotalFiles) {
332335
numSkippedFiles, numTotalFiles);
333336
}
334337

335-
private synchronized void saveMeasures(PythonInputFile inputFile, PythonVisitorContext visitorContext) {
336-
FileMetrics fileMetrics = new FileMetrics(visitorContext, isNotebook(inputFile));
337-
FileLinesVisitor fileLinesVisitor = fileMetrics.fileLinesVisitor();
338-
339-
noSonarFilter.noSonarInFile(inputFile.wrappedFile(), fileLinesVisitor.getLinesWithNoSonar());
340-
341-
if (!isInSonarLint(context)) {
342-
if (inputFile.kind() == PythonInputFile.Kind.PYTHON) {
343-
cpdAnalyzer.pushCpdTokens(inputFile.wrappedFile(), visitorContext);
344-
}
345-
346-
Set<Integer> linesOfCode = fileLinesVisitor.getLinesOfCode();
347-
saveMetricOnFile(inputFile, CoreMetrics.NCLOC, linesOfCode.size());
348-
saveMetricOnFile(inputFile, CoreMetrics.STATEMENTS, fileMetrics.numberOfStatements());
349-
saveMetricOnFile(inputFile, CoreMetrics.FUNCTIONS, fileMetrics.numberOfFunctions());
350-
saveMetricOnFile(inputFile, CoreMetrics.CLASSES, fileMetrics.numberOfClasses());
351-
saveMetricOnFile(inputFile, CoreMetrics.COMPLEXITY, fileMetrics.complexity());
352-
saveMetricOnFile(inputFile, CoreMetrics.COGNITIVE_COMPLEXITY, fileMetrics.cognitiveComplexity());
353-
saveMetricOnFile(inputFile, CoreMetrics.COMMENT_LINES, fileLinesVisitor.getCommentLineCount());
354-
355-
FileLinesContext fileLinesContext = fileLinesContextFactory.createFor(inputFile.wrappedFile());
356-
if (inputFile.kind() == PythonInputFile.Kind.PYTHON) {
357-
for (int line : linesOfCode) {
358-
fileLinesContext.setIntValue(CoreMetrics.NCLOC_DATA_KEY, line, 1);
359-
}
360-
}
361-
for (int line : fileLinesVisitor.getExecutableLines()) {
362-
fileLinesContext.setIntValue(CoreMetrics.EXECUTABLE_LINES_DATA_KEY, line, 1);
363-
}
364-
fileLinesContext.save();
365-
}
366-
}
367-
368-
static boolean isNotebook(PythonInputFile inputFile) {
369-
return inputFile.kind() == PythonInputFile.Kind.IPYTHON;
370-
}
371-
372338
private synchronized boolean restoreAndPushMeasuresIfApplicable(PythonInputFile inputFile) {
373339
if (inputFile.wrappedFile().type() == InputFile.Type.TEST) {
374340
return true;
@@ -377,14 +343,6 @@ private synchronized boolean restoreAndPushMeasuresIfApplicable(PythonInputFile
377343
return cpdAnalyzer.pushCachedCpdTokens(inputFile.wrappedFile(), indexer.cacheContext());
378344
}
379345

380-
private void saveMetricOnFile(PythonInputFile inputFile, Metric<Integer> metric, Integer value) {
381-
context.<Integer>newMeasure()
382-
.withValue(value)
383-
.forMetric(metric)
384-
.on(inputFile.wrappedFile())
385-
.save();
386-
}
387-
388346
public int getRecognitionErrorCount() {
389347
return recognitionErrorCount.get();
390348
}

‎python-commons/src/test/java/org/sonar/plugins/python/MeasuresRepositoryTest.java‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
/*
2+
* SonarQube Python Plugin
3+
* Copyright (C) 2011-2025 SonarSource SA
4+
* mailto:info AT sonarsource DOT com
5+
*
6+
* This program is free software; you can redistribute it and/or
7+
* modify it under the terms of the Sonar Source-Available License Version 1, as published by SonarSource SA.
8+
*
9+
* This program is distributed in the hope that it will be useful,
10+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
12+
* See the Sonar Source-Available License for more details.
13+
*
14+
* You should have received a copy of the Sonar Source-Available License
15+
* along with this program; if not, see https://sonarsource.com/license/ssal/
16+
*/
17+
package org.sonar.plugins.python;
18+
19+
import org.junit.jupiter.api.Test;
20+
21+
import static org.assertj.core.api.Assertions.assertThat;
22+
import static org.mockito.Mockito.mock;
23+
import static org.mockito.Mockito.when;
24+
25+
class MeasuresRepositoryTest {
26+
27+
28+
@Test
29+
void isNotebookTest() {
30+
var regularPythonFile = mock(PythonInputFile.class);
31+
when(regularPythonFile.kind()).thenReturn(PythonInputFile.Kind.PYTHON);
32+
assertThat(MeasuresRepository.isNotebook(regularPythonFile)).isFalse();
33+
34+
var notebookPythonFile = mock(PythonInputFile.class);
35+
when(notebookPythonFile.kind()).thenReturn(PythonInputFile.Kind.IPYTHON);
36+
assertThat(MeasuresRepository.isNotebook(notebookPythonFile)).isTrue();
37+
}
38+
}

‎python-commons/src/test/java/org/sonar/plugins/python/PythonSensorTest.java‎

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1440,17 +1440,6 @@ void cpd_tokens_failure_does_not_execute_checks_multiple_times() throws IOExcept
14401440
.containsEntry(CPD_TOKENS_STRING_TABLE_KEY_PREFIX + inputFile.wrappedFile().key(), cpdTokens.stringTable);
14411441
}
14421442

1443-
@Test
1444-
void test_scanner_isNotebook() {
1445-
var regularPythonFile = mock(PythonInputFile.class);
1446-
when(regularPythonFile.kind()).thenReturn(PythonInputFile.Kind.PYTHON);
1447-
assertThat(PythonScanner.isNotebook(regularPythonFile)).isFalse();
1448-
1449-
var notebookPythonFile = mock(PythonInputFile.class);
1450-
when(notebookPythonFile.kind()).thenReturn(PythonInputFile.Kind.IPYTHON);
1451-
assertThat(PythonScanner.isNotebook(notebookPythonFile)).isTrue();
1452-
}
1453-
14541443
@Test
14551444
void send_telemetry_with_version() {
14561445
activeRules = new ActiveRulesBuilder()

0 commit comments

Comments
 (0)