SONARPY-3749 Fix incorrect rule id parsing of Mypy reports (#832)

GitOrigin-RevId: bbe07e85a8130bc46393c0b41f47935c9337c100

Author: guillaume-dequenne

its/plugin/it-python-plugin-test/projects/mypy_project/mypy_output_long_message.txt

@@ -0,0 +1 @@
+src/type_hints_noncompliant.py:11:11: error: Incompatible types in assignment (expression has type "def [T: BaseClass] process_data(*, items: str | Iterable[str] = ..., values: str | Iterable[str] = ..., categories: str | Iterable[str] = ..., dependencies: Iterable[str] = ..., resources: Iterable[str] = ..., metadata: Iterable[str] = ..., identifier: str | None = ..., version: str | None = ..., owner: str | None = ..., owner_email: str | None = ..., maintainer: str | None = ..., maintainer_email: str | None = ..., source_url: str | None = ..., license_type: str | None = ..., summary: str | None = ..., detailed_info: str | None = ..., download_location: str | None = ..., is_verbose: bool = ..., is_dry_run: bool = ..., needs_help: bool = ..., handlers: MutableMapping[str, type[Handler]] = ..., handler_packages: str | list[str] | None = ..., config_name: str | PathLike[str] | None = ..., config_args: list[str] | None = ..., handler_options: MutableMapping[str, Mapping[str, tuple[str, str]]] = ..., modules: list[str] | None = ..., module_dir: Mapping[str, str] | None = ..., submodules: list[str] | None = ..., extensions: list[tuple[str, BuildConfig]] | None = ..., header_files: list[str] | None = ..., binary_modules: Sequence[BinaryModule] | None = ..., extension_package: str | None = ..., search_dirs: list[str] | None = ..., additional_path: Never = ..., executable_files: list[str] | None = ..., static_files: list[tuple[str, Sequence[str]]] | None = ..., access_key: str = ..., handler_obj: MutableMapping[str, Handler] = ..., execution_state: MutableMapping[str, bool] = ..., config_options: Mapping[str, Mapping[str, str]] | None = ..., license_key: Never = ..., show_help: bool = ..., display_name: str | Literal[False] = ..., contact_info: str | Literal[False] = ..., contact_address: str | Literal[False] = ..., data_files: Mapping[str, list[str]] = ..., distribution_files: list[tuple[str, str, str]] = ..., include_data: bool | None = ..., exclude_data: Mapping[str, list[str]] | None = ..., base_dir: str | None = ..., external_deps: list[str] = ..., build_deps: list[str] = ..., content_format: str | None = ..., web_links: Mapping[Any, Any] = ..., extra_features: MutableMapping[Any, Any] = ..., license_info: str | None = ..., license_path: Never = ..., license_paths: Iterable[str] | None = ..., runtime_deps: str | Iterable[str] = ..., optional_deps: Mapping[Any, Any] = ..., result_class: type[T] = ..., **attributes: Any) -> ProcessedData", variable has type "def [T: BaseClass] process_data(*, items: str | Iterable[str] = ..., values: str | Iterable[str] = ..., categories: str | Iterable[str] = ..., dependencies: Iterable[str] = ..., resources: Iterable[str] = ..., metadata: Iterable[str] = ..., identifier: str | None = ..., version: str | None = ..., owner: str | None = ..., owner_email: str | None = ..., maintainer: str | None = ..., maintainer_email: str | None = ..., source_url: str | None = ..., license_type: str | None = ..., summary: str | None = ..., detailed_info: str | None = ..., download_location: str | None = ..., is_verbose: bool = ..., is_dry_run: bool = ..., needs_help: bool = ..., handlers: MutableMapping[str, type[Handler]] = ..., handler_packages: str | list[str] | None = ..., config_name: str | PathLike[str] | None = ..., config_args: list[str] | None = ..., handler_options: MutableMapping[str, Mapping[str, tuple[str, str]]] = ..., modules: list[str] | None = ..., module_dir: Mapping[str, str] | None = ..., submodules: list[str] | None = ..., extensions: list[tuple[str, BuildConfig]] | None = ..., header_files: list[str] | None = ..., binary_modules: Sequence[BinaryModule] | None = ..., extension_package: str | None = ..., search_dirs: list[str] | None = ..., additional_path: Never = ..., executable_files: list[str] | None = ..., static_files: list[tuple[str, Sequence[str]]] | None = ..., access_key: str = ..., handler_obj: MutableMapping[str, Handler] = ..., execution_state: MutableMapping[str, bool] = ..., config_options: Mapping[str, Mapping[str, str]] | None = ..., license_key: Never = ..., show_help: bool = ..., display_name: str | Literal[False] = ..., contact_info: str | Literal[False] = ..., contact_address: str | Literal[False] = ..., data_files: Mapping[str, list[str]] = ..., distribution_files: list[tuple[str, str, str]] = ..., include_data: bool | None = ..., exclude_data: Mapping[str, list[str]] | None = ..., base_dir: str | None = ..., external_deps: list[str] = ..., build_deps: list[str] = ..., content_format: str | None = ..., web_links: Mapping[Any, Any] = ..., extra_features: MutableMapping[Any, Any] = ..., license_info: str | None = ..., license_path: Never = ..., license_paths: Iterable[str] | None = ..., runtime_deps: str | Iterable[str] = ..., optional_deps: Mapping[Any, Any] = ..., result_class: type[T] = ..., **attributes: Any) -> T")  [assignment]

its/plugin/it-python-plugin-test/src/test/java/com/sonar/python/it/plugin/MypyReportTest.java

@@ -16,6 +16,7 @@
  */
 package com.sonar.python.it.plugin;
 
+import com.sonar.orchestrator.build.BuildResult;
 import com.sonar.python.it.ConcurrentOrchestratorExtension;
 import com.sonar.python.it.TestsUtils;
 import java.io.File;
@@ -52,6 +53,34 @@ void import_report() {
     assertIssue(issues.get(4), "external_mypy:unknown_mypy_rule", "Unused \"type: ignore\" comment");
   }
 
+  @Test
+  void long_message_with_brackets_is_parsed_correctly() {
+    final String projectKey = "mypy_project_long_message";
+    ORCHESTRATOR.getServer().provisionProject(projectKey, projectKey);
+    ORCHESTRATOR.getServer().associateProjectToQualityProfile(projectKey, "py", "no_rule");
+
+    BuildResult result = ORCHESTRATOR.executeBuild(
+      ORCHESTRATOR.createSonarScanner()
+        .setProjectDir(new File("projects/mypy_project"))
+        .setProjectKey(projectKey)
+        .setProjectName(projectKey)
+        .setProperty("sonar.python.mypy.reportPaths", "mypy_output_long_message.txt"));
+
+    assertThat(result.isSuccess()).isTrue();
+
+    List<Issues.Issue> importedIssues = issues(projectKey);
+    assertThat(importedIssues).hasSize(1);
+
+    Issues.Issue issue = importedIssues.get(0);
+    assertThat(issue.getRule()).isEqualTo("external_mypy:assignment");
+
+    // Message should be truncated to 500 characters + "..."
+    assertThat(issue.getMessage()).hasSize(503);
+    assertThat(issue.getMessage()).startsWith("Incompatible types in assignment");
+    assertThat(issue.getMessage()).contains("BaseClass");
+    assertThat(issue.getMessage()).endsWith("...");
+  }
+
   private static void assertIssue(Issues.Issue issue, String rule, String message) {
     assertThat(issue.getComponent()).isEqualTo("mypy_project:src/type_hints_noncompliant.py");
     assertThat(issue.getRule()).isEqualTo(rule);

python-commons/src/main/java/org/sonar/plugins/python/mypy/MypySensor.java

@@ -42,6 +42,8 @@ public class MypySensor extends ExternalIssuesSensor {
   public static final String REPORT_PATH_KEY = "sonar.python.mypy.reportPaths";
 
   private static final String FALLBACK_RULE_KEY = "unknown_mypy_rule";
+  private static final int MAX_RULE_KEY_LENGTH = 200;
+  private static final int MAX_MESSAGE_LENGTH = 500;
 
   // Pattern -> Location ': ' Severity ':' Message '['Code']'
   // Location -> File ':' StartLine (':' StartCol (':' EndLine ':' EndCol))
@@ -50,7 +52,7 @@ public class MypySensor extends ExternalIssuesSensor {
   private static final String END_LOCATION = "(?::(?<endLine>\\d+):(?<endCol>\\d+))?";
 
   private static final Pattern PATTERN = Pattern
-    .compile(String.format("^(?<file>[^:]+):%s%s: (?<severity>\\S+[^:]): (?<message>.*?)(?: \\[(?<code>.*)])?\\s*$", START_LOCATION, END_LOCATION));
+    .compile(String.format("^(?<file>[^:]+):%s%s: (?<severity>\\S+[^:]): (?<message>.*?)(?: \\[(?<code>[^\\]]+)])?\\s*$", START_LOCATION, END_LOCATION));
 
   @Override
   protected void importReport(File reportPath, SensorContext context, Set<String> unresolvedInputFiles) throws IOException {
@@ -98,6 +100,20 @@ private static TextReportReader.Issue extractIssue(Matcher m) {
       errorCode = FALLBACK_RULE_KEY;
     }
 
+    // Skip issues with rule keys that are too long (likely due to parsing errors)
+    if (errorCode.length() > MAX_RULE_KEY_LENGTH) {
+      LOG.warn("Skipping mypy issue with rule key longer than {} characters at {}:{}",
+        MAX_RULE_KEY_LENGTH, filePath, lineNumber);
+      return null;
+    }
+
+    // Mypy messages can be very long, which can cause problems in the UI.
+    if (message.length() > MAX_MESSAGE_LENGTH) {
+      LOG.debug("Truncating mypy message from {} to {} characters at {}:{}",
+        message.length(), MAX_MESSAGE_LENGTH, filePath, lineNumber);
+      message = message.substring(0, MAX_MESSAGE_LENGTH) + "...";
+    }
+
     Integer columnNumber = Optional.ofNullable(m.group("startCol"))
       .map(Integer::parseInt)
       .map(i -> i - 1)

python-commons/src/test/java/org/sonar/plugins/python/mypy/MypySensorTest.java

@@ -193,6 +193,34 @@ void invalid_mypy_file() throws IOException {
     assertThat(onlyOneLogElement(logTester.logs(Level.DEBUG))).isEqualTo("Cannot parse the line: this is not a mypy output");
   }
 
+  @Test
+  void very_long_rule_message() throws IOException {
+    List<ExternalIssue> externalIssues = executeSensorImporting("mypy_output_long_message.txt");
+    assertThat(externalIssues).hasSize(1);
+
+    ExternalIssue issue = externalIssues.get(0);
+
+    assertThat(issue.ruleId()).isEqualTo("assignment");
+    assertThat(issue.ruleId().length()).isLessThan(20);
+
+    // Message should be truncated to 500 characters + "..."
+    String message = issue.primaryLocation().message();
+    assertThat(message)
+      .hasSize(503)
+      .startsWith("Incompatible types in assignment")
+      .contains("BaseClass")
+      .endsWith("...");
+  }
+
+  @Test
+  void rule_id_longer_than_limit_is_skipped() throws IOException {
+    List<ExternalIssue> externalIssues = executeSensorImporting("mypy_output_long_rule_id.txt");
+    assertThat(externalIssues).isEmpty();
+
+    assertThat(onlyOneLogElement(logTester.logs(Level.WARN)))
+      .contains("Skipping mypy issue with rule key longer than 200 characters");
+  }
+
   private static void assertIssue(ExternalIssue issue, String key, String message, int startLine, int startColumn, int endLine, int endColumn) {
     IssueLocation location = issue.primaryLocation();
     assertThat(issue.type()).isEqualTo(RuleType.CODE_SMELL);

python-commons/src/test/resources/org/sonar/plugins/python/mypy/mypy_output_long_message.txt

@@ -0,0 +1 @@
+mypy/type_hints_noncompliant.py:11:11: error: Incompatible types in assignment (expression has type "def [T: BaseClass] process_data(*, items: str | Iterable[str] = ..., values: str | Iterable[str] = ..., categories: str | Iterable[str] = ..., dependencies: Iterable[str] = ..., resources: Iterable[str] = ..., metadata: Iterable[str] = ..., identifier: str | None = ..., version: str | None = ..., owner: str | None = ..., owner_email: str | None = ..., maintainer: str | None = ..., maintainer_email: str | None = ..., source_url: str | None = ..., license_type: str | None = ..., summary: str | None = ..., detailed_info: str | None = ..., download_location: str | None = ..., is_verbose: bool = ..., is_dry_run: bool = ..., needs_help: bool = ..., handlers: MutableMapping[str, type[Handler]] = ..., handler_packages: str | list[str] | None = ..., config_name: str | PathLike[str] | None = ..., config_args: list[str] | None = ..., handler_options: MutableMapping[str, Mapping[str, tuple[str, str]]] = ..., modules: list[str] | None = ..., module_dir: Mapping[str, str] | None = ..., submodules: list[str] | None = ..., extensions: list[tuple[str, BuildConfig]] | None = ..., header_files: list[str] | None = ..., binary_modules: Sequence[BinaryModule] | None = ..., extension_package: str | None = ..., search_dirs: list[str] | None = ..., additional_path: Never = ..., executable_files: list[str] | None = ..., static_files: list[tuple[str, Sequence[str]]] | None = ..., access_key: str = ..., handler_obj: MutableMapping[str, Handler] = ..., execution_state: MutableMapping[str, bool] = ..., config_options: Mapping[str, Mapping[str, str]] | None = ..., license_key: Never = ..., show_help: bool = ..., display_name: str | Literal[False] = ..., contact_info: str | Literal[False] = ..., contact_address: str | Literal[False] = ..., data_files: Mapping[str, list[str]] = ..., distribution_files: list[tuple[str, str, str]] = ..., include_data: bool | None = ..., exclude_data: Mapping[str, list[str]] | None = ..., base_dir: str | None = ..., external_deps: list[str] = ..., build_deps: list[str] = ..., content_format: str | None = ..., web_links: Mapping[Any, Any] = ..., extra_features: MutableMapping[Any, Any] = ..., license_info: str | None = ..., license_path: Never = ..., license_paths: Iterable[str] | None = ..., runtime_deps: str | Iterable[str] = ..., optional_deps: Mapping[Any, Any] = ..., result_class: type[T] = ..., **attributes: Any) -> ProcessedData", variable has type "def [T: BaseClass] process_data(*, items: str | Iterable[str] = ..., values: str | Iterable[str] = ..., categories: str | Iterable[str] = ..., dependencies: Iterable[str] = ..., resources: Iterable[str] = ..., metadata: Iterable[str] = ..., identifier: str | None = ..., version: str | None = ..., owner: str | None = ..., owner_email: str | None = ..., maintainer: str | None = ..., maintainer_email: str | None = ..., source_url: str | None = ..., license_type: str | None = ..., summary: str | None = ..., detailed_info: str | None = ..., download_location: str | None = ..., is_verbose: bool = ..., is_dry_run: bool = ..., needs_help: bool = ..., handlers: MutableMapping[str, type[Handler]] = ..., handler_packages: str | list[str] | None = ..., config_name: str | PathLike[str] | None = ..., config_args: list[str] | None = ..., handler_options: MutableMapping[str, Mapping[str, tuple[str, str]]] = ..., modules: list[str] | None = ..., module_dir: Mapping[str, str] | None = ..., submodules: list[str] | None = ..., extensions: list[tuple[str, BuildConfig]] | None = ..., header_files: list[str] | None = ..., binary_modules: Sequence[BinaryModule] | None = ..., extension_package: str | None = ..., search_dirs: list[str] | None = ..., additional_path: Never = ..., executable_files: list[str] | None = ..., static_files: list[tuple[str, Sequence[str]]] | None = ..., access_key: str = ..., handler_obj: MutableMapping[str, Handler] = ..., execution_state: MutableMapping[str, bool] = ..., config_options: Mapping[str, Mapping[str, str]] | None = ..., license_key: Never = ..., show_help: bool = ..., display_name: str | Literal[False] = ..., contact_info: str | Literal[False] = ..., contact_address: str | Literal[False] = ..., data_files: Mapping[str, list[str]] = ..., distribution_files: list[tuple[str, str, str]] = ..., include_data: bool | None = ..., exclude_data: Mapping[str, list[str]] | None = ..., base_dir: str | None = ..., external_deps: list[str] = ..., build_deps: list[str] = ..., content_format: str | None = ..., web_links: Mapping[Any, Any] = ..., extra_features: MutableMapping[Any, Any] = ..., license_info: str | None = ..., license_path: Never = ..., license_paths: Iterable[str] | None = ..., runtime_deps: str | Iterable[str] = ..., optional_deps: Mapping[Any, Any] = ..., result_class: type[T] = ..., **attributes: Any) -> T")  [assignment]

python-commons/src/test/resources/org/sonar/plugins/python/mypy/mypy_output_long_rule_id.txt

@@ -0,0 +1 @@
+mypy/type_hints_noncompliant.py:11:11: error: Incompatible types in assignment  [this-is-a-genuinely-very-long-mypy-error-code-that-exceeds-the-maximum-allowed-rule-key-length-of-two-hundred-characters-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa]