SONARPY-2888 Refactor the Scanner implementations to have a single implementation of sequential/parallel analysis (#238)

GitOrigin-RevId: 1d27f34b0457c5af7fed2668b152d1390cf9cb03

Author: ghislainpiot

python-commons/src/main/java/org/sonar/plugins/python/PythonScanner.java

@@ -31,12 +31,10 @@
 import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ForkJoinPool;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Supplier;
 import java.util.regex.Pattern;
-import java.util.stream.Stream;
 import javax.annotation.CheckForNull;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -120,26 +118,8 @@ protected String name() {
   }
 
   @Override
-  protected void processFiles(List<PythonInputFile> files, SensorContext context, MultiFileProgressReport progressReport,
-    AtomicInteger numScannedWithoutParsing) {
-    var numberOfThreads = getNumberOfThreads(context);
-    if (numberOfThreads == 1) {
-      super.processFiles(files, context, progressReport, numScannedWithoutParsing);
-      return;
-    }
-    var pool = new ForkJoinPool(numberOfThreads);
-    try {
-      LOG.debug("Scanning files in {} threads", numberOfThreads);
-      pool.submit(() -> super.processFiles(files, context, progressReport, numScannedWithoutParsing))
-        .join();
-    } finally {
-      pool.shutdown();
-    }
-  }
-
-  @Override
-  protected Stream<PythonInputFile> getFilesStream(List<PythonInputFile> files) {
-    return getNumberOfThreads(context) == 1 ? files.stream() : files.stream().parallel();
+  protected void logStart(int numThreads) {
+    LOG.debug("Scanning files in {} threads", numThreads);
   }
 
   @Override
@@ -165,7 +145,6 @@ protected void scanFile(PythonInputFile inputFile) throws IOException {
     searchForDataBricks(visitorContext);
   }
 
-
   private PythonVisitorContext createVisitorContext(PythonInputFile inputFile, PythonFile pythonFile, InputFile.Type fileType) throws IOException {
     PythonVisitorContext visitorContext;
     try {
@@ -530,7 +509,8 @@ public boolean getFoundDatabricks() {
     return foundDatabricks.get();
   }
 
-  private static Integer getNumberOfThreads(SensorContext context) {
+  @Override
+  protected int getNumberOfThreads(SensorContext context) {
     return context.config().getInt(THREADS_PROPERTY_NAME)
       .orElse(1);
   }

python-commons/src/main/java/org/sonar/plugins/python/Scanner.java

@@ -19,6 +19,7 @@
 import com.sonar.sslr.api.RecognitionException;
 import java.io.IOException;
 import java.util.List;
+import java.util.concurrent.ForkJoinPool;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.Stream;
 import org.slf4j.Logger;
@@ -50,11 +51,27 @@ public void execute(List<PythonInputFile> files, SensorContext context) {
   }
 
   protected void processFiles(List<PythonInputFile> files, SensorContext context, MultiFileProgressReport progressReport, AtomicInteger numScannedWithoutParsing) {
-    getFilesStream(files).forEach(file -> processFile(context, file, progressReport, numScannedWithoutParsing));
+    var numberOfThreads = getNumberOfThreads(context);
+    logStart(numberOfThreads);
+    if (numberOfThreads == 1) {
+      getFilesStream(files).forEach(file -> processFile(context, file, progressReport, numScannedWithoutParsing));
+      return;
+    }
+    var pool = new ForkJoinPool(numberOfThreads);
+    try {
+      pool.submit(() -> getFilesStream(files).forEach(file -> processFile(context, file, progressReport, numScannedWithoutParsing)))
+        .join();
+    } finally {
+      pool.shutdown();
+    }
   }
 
+  protected abstract void logStart(int numThreads);
+
   protected Stream<PythonInputFile> getFilesStream(List<PythonInputFile> files) {
-    return files.stream();
+    return getNumberOfThreads(context) == 1
+      ? files.stream()
+      : files.parallelStream();
   }
 
   private void processFile(SensorContext context, PythonInputFile file, MultiFileProgressReport progressReport, AtomicInteger numScannedWithoutParsing) {
@@ -110,4 +127,7 @@ private static boolean isParseErrorOnTestFile(PythonInputFile file, Exception e)
     // As test files may contain invalid syntax on purpose, we avoid failing the analysis when encountering parse errors on them
     return e instanceof RecognitionException && file.wrappedFile().type() == InputFile.Type.TEST;
   }
+
+  protected abstract int getNumberOfThreads(SensorContext context);
+
 }

python-commons/src/main/java/org/sonar/plugins/python/indexer/PythonIndexer.java

@@ -22,17 +22,13 @@
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ForkJoinPool;
-import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Supplier;
-import java.util.stream.Stream;
 import javax.annotation.CheckForNull;
 import javax.annotation.Nullable;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.sonar.api.batch.fs.InputFile;
 import org.sonar.api.batch.sensor.SensorContext;
-import org.sonar.plugins.python.MultiFileProgressReport;
 import org.sonar.plugins.python.PythonInputFile;
 import org.sonar.plugins.python.Scanner;
 import org.sonar.plugins.python.SonarQubePythonFile;
@@ -148,45 +144,22 @@ protected String name() {
     }
 
     @Override
-    protected void scanFile(PythonInputFile inputFile) throws IOException {
-      // Global Symbol Table is deactivated for Notebooks see: SONARPY-2021
-      if (inputFile.kind() == PythonInputFile.Kind.PYTHON) {
-        addFile(inputFile);
+    protected void logStart(int numThreads) {
+      if (numThreads != 1) {
+        LOG.debug("Scanning global symbols in {} threads", numThreads);
       }
     }
 
     @Override
-    protected Stream<PythonInputFile> getFilesStream(List<PythonInputFile> files) {
-      var numberOfThreads = getNumberOfThreads(context);
-      if (numberOfThreads == 1) {
-        return files.stream();
+    protected void scanFile(PythonInputFile inputFile) throws IOException {
+      // Global Symbol Table is deactivated for Notebooks see: SONARPY-2021
+      if (inputFile.kind() == PythonInputFile.Kind.PYTHON) {
+        addFile(inputFile);
       }
-      return files.stream().parallel();
     }
 
     @Override
-    protected void processFiles(List<PythonInputFile> files, SensorContext context, MultiFileProgressReport progressReport, AtomicInteger numScannedWithoutParsing) {
-      var numberOfThreads = getNumberOfThreads(context);
-      if (numberOfThreads != 1) {
-        processFilesWithThreads(files, context, progressReport, numScannedWithoutParsing, numberOfThreads);
-      } else {
-        super.processFiles(files, context, progressReport, numScannedWithoutParsing);
-      }
-    }
-
-    private void processFilesWithThreads(List<PythonInputFile> files, SensorContext context, MultiFileProgressReport progressReport, AtomicInteger numScannedWithoutParsing,
-      Integer numberOfThreads) {
-      LOG.debug("Scanning global symbols in {} threads", numberOfThreads);
-      ForkJoinPool pool = new ForkJoinPool(numberOfThreads);
-      try {
-        pool.submit(() -> super.processFiles(files, context, progressReport, numScannedWithoutParsing))
-          .join();
-      } finally {
-        pool.shutdown();
-      }
-    }
-
-    private Integer getNumberOfThreads(SensorContext context) {
+    protected int getNumberOfThreads(SensorContext context) {
       return context.config().getInt(THREADS_PROPERTY_NAME)
         .orElse(Math.max(2, Math.min((int) Math.round(Runtime.getRuntime().availableProcessors() * 0.9), 6)));
     }