SONARPY-2852 Rework SymbolVisitor to be a thread safe NewSymbolsCollector (#233)

GitOrigin-RevId: 1eaf362d8dba2c48c39cd01775e2f93f28a3253d

Author: maksim-grebeniuk-sonarsource

python-commons/src/main/java/org/sonar/plugins/python/NewSymbolsCollector.java

@@ -0,0 +1,115 @@
+/*
+ * SonarQube Python Plugin
+ * Copyright (C) 2011-2025 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the Sonar Source-Available License Version 1, as published by SonarSource SA.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ * See the Sonar Source-Available License for more details.
+ *
+ * You should have received a copy of the Sonar Source-Available License
+ * along with this program; if not, see https://sonarsource.com/license/ssal/
+ */
+package org.sonar.plugins.python;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import org.sonar.api.batch.sensor.symbol.NewSymbol;
+import org.sonar.api.batch.sensor.symbol.NewSymbolTable;
+import org.sonar.plugins.python.api.symbols.Symbol;
+import org.sonar.plugins.python.api.symbols.Usage;
+import org.sonar.plugins.python.api.tree.BaseTreeVisitor;
+import org.sonar.plugins.python.api.tree.ClassDef;
+import org.sonar.plugins.python.api.tree.ComprehensionExpression;
+import org.sonar.plugins.python.api.tree.DictCompExpression;
+import org.sonar.plugins.python.api.tree.FileInput;
+import org.sonar.plugins.python.api.tree.FunctionDef;
+import org.sonar.plugins.python.api.tree.LambdaExpression;
+import org.sonar.plugins.python.api.tree.Tree;
+
+public class NewSymbolsCollector {
+  private final Object monitor;
+
+  public NewSymbolsCollector(Object monitor) {
+    this.monitor = monitor;
+  }
+
+  public void collect(NewSymbolTable newSymbolTable, FileInput fileInput) {
+    var visitor = new SymbolsCollectingVisitor(newSymbolTable);
+    visitor.visitFileInput(fileInput);
+    save(newSymbolTable);
+  }
+
+  private void save(NewSymbolTable newSymbolTable) {
+    synchronized (monitor) {
+      newSymbolTable.save();
+    }
+  }
+
+  private static class SymbolsCollectingVisitor extends BaseTreeVisitor {
+    private final NewSymbolTable newSymbolTable;
+
+    public SymbolsCollectingVisitor(NewSymbolTable newSymbolTable) {
+      this.newSymbolTable = newSymbolTable;
+    }
+
+    @Override
+    public void visitClassDef(ClassDef classDef) {
+      classDef.classFields().forEach(this::handleSymbol);
+      classDef.instanceFields().forEach(this::handleSymbol);
+      super.visitClassDef(classDef);
+    }
+
+    @Override
+    public void visitFunctionDef(FunctionDef functionDef) {
+      functionDef.localVariables().forEach(this::handleSymbol);
+      super.visitFunctionDef(functionDef);
+    }
+
+    @Override
+    public void visitLambda(LambdaExpression lambdaExpression) {
+      lambdaExpression.localVariables().forEach(this::handleSymbol);
+      super.visitLambda(lambdaExpression);
+    }
+
+    @Override
+    public void visitPyListOrSetCompExpression(ComprehensionExpression tree) {
+      tree.localVariables().forEach(this::handleSymbol);
+      super.visitPyListOrSetCompExpression(tree);
+    }
+
+    @Override
+    public void visitDictCompExpression(DictCompExpression tree) {
+      tree.localVariables().forEach(this::handleSymbol);
+      super.visitDictCompExpression(tree);
+    }
+
+    @Override
+    public void visitFileInput(FileInput fileInput) {
+      fileInput.globalVariables().forEach(this::handleSymbol);
+      super.visitFileInput(fileInput);
+    }
+
+    private void handleSymbol(Symbol symbol) {
+      if (symbol.usages().isEmpty()) {
+        // global symbols might not have any usages
+        return;
+      }
+      List<Usage> usages = new ArrayList<>(symbol.usages());
+      usages.sort(Comparator.comparingInt(u -> u.tree().firstToken().line()));
+      Tree firstUsageTree = usages.get(0).tree();
+      NewSymbol newSymbol = newSymbolTable.newSymbol(firstUsageTree.firstToken().line(), firstUsageTree.firstToken().column(),
+        firstUsageTree.lastToken().line(), firstUsageTree.lastToken().column() + firstUsageTree.lastToken().value().length());
+      for (int i = 1; i < usages.size(); i++) {
+        Tree usageTree = usages.get(i).tree();
+        newSymbol.newReference(usageTree.firstToken().line(), usageTree.firstToken().column(),
+          usageTree.lastToken().line(), usageTree.lastToken().column() + usageTree.lastToken().value().length());
+      }
+    }
+  }
+}

python-commons/src/main/java/org/sonar/plugins/python/PythonScanner.java

@@ -91,6 +91,7 @@ public class PythonScanner extends Scanner {
   private final AtomicBoolean foundDatabricks;
   private final PythonFileConsumer architectureCallback;
   private final Map<String, Object> repositoryLocks;
+  private final NewSymbolsCollector newSymbolsCollector;
 
   public PythonScanner(
     SensorContext context, PythonChecks checks, FileLinesContextFactory fileLinesContextFactory, NoSonarFilter noSonarFilter,
@@ -108,6 +109,7 @@ public PythonScanner(
     this.recognitionErrorCount = new AtomicInteger(0);
     this.foundDatabricks = new AtomicBoolean(false);
     this.repositoryLocks = new ConcurrentHashMap<>();
+    this.newSymbolsCollector = new NewSymbolsCollector(this);
   }
 
   @Override
@@ -154,8 +156,8 @@ protected void scanFile(PythonInputFile inputFile) throws IOException {
     saveIssues(inputFile, visitorContext.getIssues());
 
     if (visitorContext.rootTree() != null && !isInSonarLint(context)) {
-      new SymbolVisitor(context.newSymbolTable().onFile(inputFile.wrappedFile())).visitFileInput(visitorContext.rootTree());
       new PythonHighlighter(context, inputFile).scanFile(visitorContext);
+      newSymbolsCollector.collect(context.newSymbolTable().onFile(inputFile.wrappedFile()), visitorContext.rootTree());
     }
 
     searchForDataBricks(visitorContext);

python-commons/src/main/java/org/sonar/plugins/python/SymbolVisitor.java

@@ -27,13 +27,11 @@
 import org.sonar.api.batch.fs.internal.DefaultTextRange;
 import org.sonar.api.batch.fs.internal.TestInputFileBuilder;
 import org.sonar.api.batch.sensor.internal.SensorContextTester;
-import org.sonar.plugins.python.api.PythonVisitorContext;
-import org.sonar.plugins.python.api.tree.FileInput;
 import org.sonar.python.TestPythonVisitorRunner;
 
 import static org.assertj.core.api.Assertions.assertThat;
 
-class SymbolVisitorTest {
+class NewSymbolsCollectorTest {
 
   private static SensorContextTester context;
   private static String componentKey;
@@ -49,10 +47,11 @@ static void scanFile() {
     context.fileSystem().add(inputFile);
     componentKey = inputFile.key();
 
-    SymbolVisitor symbolVisitor = new SymbolVisitor(context.newSymbolTable().onFile(inputFile));
-    PythonVisitorContext context = TestPythonVisitorRunner.createContext(file);
-    FileInput fileInput = context.rootTree();
-    fileInput.accept(symbolVisitor);
+    var newSymbolTable = context.newSymbolTable().onFile(inputFile);
+    var visitorContext = TestPythonVisitorRunner.createContext(file);
+    var fileInput = visitorContext.rootTree();
+    var newSymbolsCollector = new NewSymbolsCollector(new Object());
+    newSymbolsCollector.collect(newSymbolTable, fileInput);
   }
 
   @Test