SONARPY-2451 Collect data for the Jupyter notebooks

Author: ghislainpiot

sonar-python-plugin/src/main/java/org/sonar/plugins/python/IPynbSensor.java

@@ -47,6 +47,7 @@ public final class IPynbSensor implements Sensor {
   private final NoSonarFilter noSonarFilter;
   private final PythonIndexer indexer;
   private static final String FAIL_FAST_PROPERTY_NAME = "sonar.internal.analysis.failFast";
+  private final SensorTelemetryStorage sensorTelemetryStorage;
 
   public IPynbSensor(FileLinesContextFactory fileLinesContextFactory, CheckFactory checkFactory, NoSonarFilter noSonarFilter) {
     this(fileLinesContextFactory, checkFactory, noSonarFilter, null);
@@ -58,6 +59,7 @@ public IPynbSensor(FileLinesContextFactory fileLinesContextFactory, CheckFactory
     this.fileLinesContextFactory = fileLinesContextFactory;
     this.noSonarFilter = noSonarFilter;
     this.indexer = indexer;
+    this.sensorTelemetryStorage = new SensorTelemetryStorage();
   }
 
   @Override
@@ -80,6 +82,7 @@ public void execute(SensorContext context) {
     } else {
       processNotebooksFiles(pythonFiles, context);
     }
+    sensorTelemetryStorage.send(context);
   }
 
   private void processNotebooksFiles(List<PythonInputFile> pythonFiles, SensorContext context) {
@@ -89,20 +92,29 @@ private void processNotebooksFiles(List<PythonInputFile> pythonFiles, SensorCont
     PythonIndexer pythonIndexer = new SonarQubePythonIndexer(pythonFiles, cacheContext, context);
     PythonScanner scanner = new PythonScanner(context, checks, fileLinesContextFactory, noSonarFilter, PythonParser.createIPythonParser(), pythonIndexer);
     scanner.execute(pythonFiles, context);
+    sensorTelemetryStorage.updateMetric(SensorTelemetryStorage.MetricKey.NOTEBOOK_RECOGNITION_ERROR_KEY, String.valueOf(scanner.getRecognitionErrorCount()));
   }
 
-  private static List<PythonInputFile> parseNotebooks(List<PythonInputFile> pythonFiles, SensorContext context) {
+  private List<PythonInputFile> parseNotebooks(List<PythonInputFile> pythonFiles, SensorContext context) {
     List<PythonInputFile> generatedIPythonFiles = new ArrayList<>();
+
+    sensorTelemetryStorage.updateMetric(SensorTelemetryStorage.MetricKey.NOTEBOOK_TOTAL_KEY, String.valueOf(pythonFiles.size()));
+    var numberOfExceptions = 0;
+
     for (PythonInputFile inputFile : pythonFiles) {
       try {
+        sensorTelemetryStorage.updateMetric(SensorTelemetryStorage.MetricKey.NOTEBOOK_PRESENT_KEY, "1");
         var result = IpynbNotebookParser.parseNotebook(inputFile);
         result.ifPresent(generatedIPythonFiles::add);
       } catch (Exception e) {
+        numberOfExceptions++;
         if (context.config().getBoolean(FAIL_FAST_PROPERTY_NAME).orElse(false) && !isErrorOnTestFile(inputFile)) {
           throw new IllegalStateException("Exception when parsing " + inputFile, e);
         }
       }
     }
+
+    sensorTelemetryStorage.updateMetric(SensorTelemetryStorage.MetricKey.NOTEBOOK_EXCEPTION_KEY, String.valueOf(numberOfExceptions));
     return generatedIPythonFiles;
   }
 

sonar-python-plugin/src/main/java/org/sonar/plugins/python/PythonScanner.java

@@ -77,6 +77,7 @@ public class PythonScanner extends Scanner {
   private final PythonCpdAnalyzer cpdAnalyzer;
   private final PythonIndexer indexer;
   private final Map<PythonInputFile, Set<PythonCheck>> checksExecutedWithoutParsingByFiles = new HashMap<>();
+  private int recognitionErrorCount = 0;
 
   public PythonScanner(
     SensorContext context, PythonChecks checks,
@@ -123,6 +124,7 @@ protected void scanFile(PythonInputFile inputFile) throws IOException {
 
       LOG.error("Unable to parse file: " + inputFile);
       LOG.error(newMessage);
+      recognitionErrorCount++;
       context.newAnalysisError()
         .onFile(inputFile.wrappedFile())
         .at(inputFile.wrappedFile().newPointer(line, 0))
@@ -401,4 +403,8 @@ private static void addQuickFixes(InputFile inputFile, RuleKey ruleKey, Iterable
   private static TextRange rangeFromTextSpan(InputFile file, PythonTextEdit pythonTextEdit) {
     return file.newRange(pythonTextEdit.startLine(), pythonTextEdit.startLineOffset(), pythonTextEdit.endLine(), pythonTextEdit.endLineOffset());
   }
+
+  public int getRecognitionErrorCount() {
+    return recognitionErrorCount;
+  }
 }

sonar-python-plugin/src/main/java/org/sonar/plugins/python/SensorTelemetryStorage.java

@@ -0,0 +1,57 @@
+/*
+ * SonarQube Python Plugin
+ * Copyright (C) 2011-2024 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the Sonar Source-Available License Version 1, as published by SonarSource SA.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ * See the Sonar Source-Available License for more details.
+ *
+ * You should have received a copy of the Sonar Source-Available License
+ * along with this program; if not, see https://sonarsource.com/license/ssal/
+ */
+package org.sonar.plugins.python;
+
+import java.util.HashMap;
+import java.util.Map;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.sonar.api.batch.sensor.SensorContext;
+
+public class SensorTelemetryStorage {
+  private static final Logger LOG = LoggerFactory.getLogger(SensorTelemetryStorage.class);
+
+  private final Map<String, String> data = new HashMap<>();
+
+  public void send(SensorContext sensorContext) {
+    data.forEach((k, v) -> {
+      LOG.info("Metrics property: {}={}", k, v);
+      sensorContext.addTelemetryProperty(k, v);
+    });
+  }
+
+  public void updateMetric(MetricKey key, String value) {
+    data.put(key.key(), value);
+  }
+
+  public enum MetricKey {
+    NOTEBOOK_PRESENT_KEY("python.notebook.present"),
+    NOTEBOOK_TOTAL_KEY("python.notebook.total"),
+    NOTEBOOK_RECOGNITION_ERROR_KEY("python.notebook.recognition_error"),
+    NOTEBOOK_EXCEPTION_KEY("python.notebook.exceptions");
+
+    private final String key;
+
+    MetricKey(String key) {
+      this.key = key;
+    }
+
+    public String key() {
+      return key;
+    }
+  }
+}

sonar-python-plugin/src/test/java/org/sonar/plugins/python/IPynbSensorTest.java

@@ -57,6 +57,8 @@
 import static org.junit.Assert.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
 class IPynbSensorTest {
@@ -210,10 +212,16 @@ void test_notebook_sensor_cannot_parse_file() {
   }
 
   @Test
-  void test_notebook_sensor_is_excuted_on_json_file() {
+  void test_notebook_sensor_is_executed_on_json_file() {
     inputFile(NOTEBOOK_FILE);
     activeRules = new ActiveRulesBuilder().build();
-    assertDoesNotThrow(() -> notebookSensor().execute(context));
+    var sensor = spy(notebookSensor());
+    var contextSpy = spy(context);
+    assertDoesNotThrow(() -> sensor.execute(contextSpy));
+    verify(contextSpy, Mockito.times(1)).addTelemetryProperty(SensorTelemetryStorage.MetricKey.NOTEBOOK_PRESENT_KEY.key(), "1");
+    verify(contextSpy, Mockito.times(1)).addTelemetryProperty(SensorTelemetryStorage.MetricKey.NOTEBOOK_RECOGNITION_ERROR_KEY.key(), "0");
+    verify(contextSpy, Mockito.times(1)).addTelemetryProperty(SensorTelemetryStorage.MetricKey.NOTEBOOK_TOTAL_KEY.key(), "1");
+    verify(contextSpy, Mockito.times(1)).addTelemetryProperty(SensorTelemetryStorage.MetricKey.NOTEBOOK_EXCEPTION_KEY.key(), "0");
   }
 
   @Test
@@ -227,12 +235,17 @@ void test_notebook_sensor_does_not_execute_cpd_measures() {
   }
 
   @Test
-  void test_notebook_sensor_parse_error_on_valid_line(){
+  void test_notebook_sensor_parse_error_on_valid_line() {
     inputFile("notebook_parse_error.ipynb");
     activeRules = new ActiveRulesBuilder().build();
     var sensor = notebookSensor();
-    sensor.execute(context);
+    var contextSpy = spy(context);
+    sensor.execute(contextSpy);
     var logs = String.join("", logTester.logs());
     assertThat(logs).contains("Unable to parse file: notebook_parse_error.ipynbParse error at line 1");
+    verify(contextSpy, Mockito.times(1)).addTelemetryProperty(SensorTelemetryStorage.MetricKey.NOTEBOOK_PRESENT_KEY.key(), "1");
+    verify(contextSpy, Mockito.times(1)).addTelemetryProperty(SensorTelemetryStorage.MetricKey.NOTEBOOK_RECOGNITION_ERROR_KEY.key(), "1");
+    verify(contextSpy, Mockito.times(1)).addTelemetryProperty(SensorTelemetryStorage.MetricKey.NOTEBOOK_TOTAL_KEY.key(), "1");
+    verify(contextSpy, Mockito.times(1)).addTelemetryProperty(SensorTelemetryStorage.MetricKey.NOTEBOOK_EXCEPTION_KEY.key(), "0");
   }
 }