SONARPY-3976 Rule S8520 "sum()" should not be used with an empty list to concatenate lists (#1012)

Co-authored-by: Marc Jasper <marc.jasper@sonarsource.com>
GitOrigin-RevId: fd5138cf008f8139622f3a830a86663d9fe73325

Author: 2 people

python-checks/src/main/java/org/sonar/python/checks/OpenSourceCheckList.java

@@ -418,6 +418,7 @@ public Stream<Class<?>> getChecks() {
       StringLiteralDuplicationCheck.class,
       StringReplaceCheck.class,
       StrongCryptographicKeysCheck.class,
+      SumListConcatenationCheck.class,
       SklearnCachedPipelineDontAccessTransformersCheck.class,
       MissingHyperParameterCheck.class,
       NetworkCallsWithoutTimeoutsInLambdaCheck.class,

python-checks/src/main/java/org/sonar/python/checks/SumListConcatenationCheck.java

@@ -0,0 +1,63 @@
+/*
+ * SonarQube Python Plugin
+ * Copyright (C) SonarSource Sàrl
+ * mailto:info AT sonarsource DOT com
+ *
+ * You can redistribute and/or modify this program under the terms of
+ * the Sonar Source-Available License Version 1, as published by SonarSource Sàrl.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ * See the Sonar Source-Available License for more details.
+ *
+ * You should have received a copy of the Sonar Source-Available License
+ * along with this program; if not, see https://sonarsource.com/license/ssal/
+ */
+package org.sonar.python.checks;
+
+import javax.annotation.Nullable;
+import org.sonar.check.Rule;
+import org.sonar.plugins.python.api.PythonSubscriptionCheck;
+import org.sonar.plugins.python.api.SubscriptionContext;
+import org.sonar.plugins.python.api.tree.CallExpression;
+import org.sonar.plugins.python.api.tree.ListLiteral;
+import org.sonar.plugins.python.api.tree.RegularArgument;
+import org.sonar.plugins.python.api.tree.Tree;
+import org.sonar.plugins.python.api.types.v2.matchers.TypeMatcher;
+import org.sonar.plugins.python.api.types.v2.matchers.TypeMatchers;
+import org.sonar.python.tree.TreeUtils;
+
+@Rule(key = "S8520")
+public class SumListConcatenationCheck extends PythonSubscriptionCheck {
+
+  private static final String MESSAGE = "Use \"itertools.chain.from_iterable()\" instead of \"sum()\" to flatten or concatenate lists.";
+  private static final TypeMatcher SUM_MATCHER = TypeMatchers.isType("sum");
+
+  @Override
+  public void initialize(Context context) {
+    context.registerSyntaxNodeConsumer(Tree.Kind.CALL_EXPR, SumListConcatenationCheck::checkCall);
+  }
+
+  private static void checkCall(SubscriptionContext ctx) {
+    CallExpression call = (CallExpression) ctx.syntaxNode();
+    if (!SUM_MATCHER.isTrueFor(call.callee(), ctx)) {
+      return;
+    }
+    RegularArgument startArgument = TreeUtils.nthArgumentOrKeyword(1, "start", call.arguments());
+    if (isEmptyListLiteral(startArgument)) {
+      ctx.addIssue(call, MESSAGE);
+    }
+  }
+
+  private static boolean isEmptyListLiteral(@Nullable RegularArgument argument) {
+    if (argument == null) {
+      return false;
+    }
+    if (!argument.expression().is(Tree.Kind.LIST_LITERAL)) {
+      return false;
+    }
+    ListLiteral listLiteral = (ListLiteral) argument.expression();
+    return listLiteral.elements().expressions().isEmpty();
+  }
+}

python-checks/src/main/resources/org/sonar/l10n/py/rules/python/S8520.html

@@ -0,0 +1,38 @@
+<p>This rule raises an issue when <code>sum()</code> is called with an empty list (<code>[]</code>) as the start parameter to concatenate or flatten
+lists.</p>
+<h2>Why is this an issue?</h2>
+<p>Using <code>sum(list_of_lists, [])</code> to concatenate lists is a common anti-pattern in Python. Because each <code>+</code> concatenation
+creates a new list and copies all previous elements into it, the overall operation has quadratic time complexity O(n<sup>2</sup>) instead of the
+linear O(n) that proper alternatives achieve.</p>
+<p>Additionally, <code>sum()</code> is designed for numeric addition, not list operations. Using it for lists is semantically unclear and may confuse
+other developers reading the code.</p>
+<h3>What is the potential impact?</h3>
+<ul>
+  <li><strong>performance degradation</strong>: operations that should take milliseconds can take seconds or even minutes with large datasets</li>
+  <li><strong>memory pressure</strong>: each intermediate concatenation allocates a new list, creating unnecessary memory overhead</li>
+  <li><strong>scalability issues</strong>: code that works fine in development with small datasets may become unusable in production</li>
+</ul>
+<h2>How to fix it</h2>
+<h3>Code examples</h3>
+<h4>Noncompliant code example</h4>
+<pre data-diff-id="1" data-diff-type="noncompliant">
+list_of_lists = [[1, 2], [3, 4], [5, 6]]
+result = sum(list_of_lists, [])  # Noncompliant
+</pre>
+<h4>Compliant solution</h4>
+<pre data-diff-id="1" data-diff-type="compliant">
+import itertools
+
+list_of_lists = [[1, 2], [3, 4], [5, 6]]
+result = list(itertools.chain.from_iterable(list_of_lists))
+</pre>
+<h3>How does this work?</h3>
+<p><code>itertools.chain.from_iterable()</code> creates an iterator that yields elements from each sub-list in sequence without creating intermediate
+list copies. This avoids the repeated copying that causes the quadratic behavior of <code>sum()</code> and results in linear O(n) time complexity.</p>
+<h2>Resources</h2>
+<h3>Documentation</h3>
+<ul>
+  <li>Python Documentation - <a href="https://docs.python.org/3/library/itertools.html#itertools.chain"><code>itertools.chain</code> and
+  <code>chain.from_iterable</code></a></li>
+</ul>
+

python-checks/src/main/resources/org/sonar/l10n/py/rules/python/S8520.json

@@ -0,0 +1,24 @@
+{
+  "title": "\"sum()\" should not be used with an empty list to concatenate lists",
+  "type": "CODE_SMELL",
+  "status": "ready",
+  "remediation": {
+    "func": "Constant\/Issue",
+    "constantCost": "5min"
+  },
+  "tags": [
+    "performance",
+    "pitfall"
+  ],
+  "defaultSeverity": "Major",
+  "ruleSpecification": "RSPEC-8520",
+  "sqKey": "S8520",
+  "scope": "All",
+  "quickfix": "unknown",
+  "code": {
+    "impacts": {
+      "RELIABILITY": "HIGH"
+    },
+    "attribute": "EFFICIENT"
+  }
+}

python-checks/src/main/resources/org/sonar/l10n/py/rules/python/Sonar_way_profile.json

@@ -332,6 +332,7 @@
     "S8495",
     "S8504",
     "S8517",
+    "S8520",
     "S8521"
   ]
 }

python-checks/src/test/java/org/sonar/python/checks/SumListConcatenationCheckTest.java

@@ -0,0 +1,29 @@
+/*
+ * SonarQube Python Plugin
+ * Copyright (C) SonarSource Sàrl
+ * mailto:info AT sonarsource DOT com
+ *
+ * You can redistribute and/or modify this program under the terms of
+ * the Sonar Source-Available License Version 1, as published by SonarSource Sàrl.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ * See the Sonar Source-Available License for more details.
+ *
+ * You should have received a copy of the Sonar Source-Available License
+ * along with this program; if not, see https://sonarsource.com/license/ssal/
+ */
+package org.sonar.python.checks;
+
+import org.junit.jupiter.api.Test;
+import org.sonar.python.checks.utils.PythonCheckVerifier;
+
+class SumListConcatenationCheckTest {
+
+  @Test
+  void test() {
+    PythonCheckVerifier.verify("src/test/resources/checks/sumListConcatenation.py", new SumListConcatenationCheck());
+  }
+
+}

python-checks/src/test/resources/checks/sumListConcatenation.py

@@ -0,0 +1,44 @@
+list_of_lists = [[1, 2], [3, 4]]
+
+
+def noncompliant_empty_list_as_start():
+    sum(list_of_lists, [])  # Noncompliant {{Use "itertools.chain.from_iterable()" instead of "sum()" to flatten or concatenate lists.}}
+#   ^^^^^^^^^^^^^^^^^^^^^^
+
+
+def noncompliant_start_keyword_argument():
+    sum(list_of_lists, start=[])  # Noncompliant {{Use "itertools.chain.from_iterable()" instead of "sum()" to flatten or concatenate lists.}}
+#   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+
+def compliant_callee_is_not_builtin_sum(other_sum):
+    def sum(iterable, start):
+        result = start
+        for item in iterable:
+            result = result + item
+        return result
+
+    sum([[1, 2]], [])
+    other_sum([[1, 2]], [])
+
+
+def compliant_no_start_argument():
+    sum([1, 2, 3])
+    sum()
+
+
+def compliant_start_not_empty_list_literal():
+    sum(list_of_lists, [0])
+
+
+def compliant_start_not_list_literal():
+    sum(list_of_lists, 10)
+
+
+def fn_start_not_syntactic_empty_list():  # FN: only literal []; Name, list(), **kwargs, starred unpack
+    empty = []
+    sum(list_of_lists, empty)
+    sum(list_of_lists, list())
+    kwargs = {"start": []}
+    sum(list_of_lists, **kwargs)
+    sum(list_of_lists, *[[]])