SONARPY-3752 Serialize attributes of FunctionType params and return type (#838)

GitOrigin-RevId: e0334ce302d9cf2606c5044d6cac076290d1c45a

Author: Seppli11

python-checks/src/test/resources/checks/nonCallableCalled.py

@@ -411,3 +411,12 @@ def object_typevar():
 def uncallable_inside_lambda_are_non_compliant():
     lambda_call = (lambda : print("something"))() # OK
     uncallable_inside_lambda = lambda:(print("something"))() # Noncompliant
+
+
+def fp_loop_of_class_list():
+    class MyClass: ... 
+
+    list_of_classes = [MyClass]
+
+    for clazz in list_of_classes:
+        clazz() # Noncompliant

python-frontend/src/main/java/org/sonar/plugins/python/api/types/v2/ObjectType.java

@@ -27,11 +27,11 @@
 @Beta
 public final class ObjectType implements PythonType {
   private final TypeWrapper typeWrapper;
-  private final List<PythonType> attributes;
+  private final List<TypeWrapper> attributes;
   private final List<Member> members;
   private final TypeSource typeSource;
 
-  private ObjectType(TypeWrapper typeWrapper, List<PythonType> attributes, List<Member> members, TypeSource typeSource) {
+  private ObjectType(TypeWrapper typeWrapper, List<TypeWrapper> attributes, List<Member> members, TypeSource typeSource) {
     this.typeWrapper = typeWrapper;
     this.attributes = attributes;
     this.members = members;
@@ -86,16 +86,13 @@ public boolean equals(Object o) {
     ObjectType that = (ObjectType) o;
     List<String> membersNames = members.stream().map(Member::name).toList();
     List<String> otherMembersNames = that.members.stream().map(Member::name).toList();
-    List<String> attributesNames = attributes.stream().map(PythonType::key).toList();
-    List<String> otherAttributesNames = that.attributes.stream().map(PythonType::key).toList();
-    return Objects.equals(typeWrapper, that.typeWrapper) && Objects.equals(membersNames, otherMembersNames) && Objects.equals(attributesNames, otherAttributesNames);
+    return Objects.equals(typeWrapper, that.typeWrapper) && Objects.equals(membersNames, otherMembersNames) && Objects.equals(attributes, that.attributes);
   }
 
   @Override
   public int hashCode() {
     List<String> membersNames = members.stream().map(Member::name).toList();
-    List<String> attributesNames = attributes.stream().map(PythonType::key).toList();
-    return Objects.hash(typeWrapper, attributesNames, membersNames);
+    return Objects.hash(typeWrapper, attributes, membersNames);
   }
 
   public PythonType type() {
@@ -107,7 +104,7 @@ public TypeWrapper typeWrapper() {
   }
 
   public List<PythonType> attributes() {
-    return attributes;
+    return attributes.stream().map(TypeWrapper::type).toList();
   }
 
   public List<Member> members() {
@@ -130,7 +127,7 @@ public String toString() {
 
   public static class Builder implements TypeBuilder<ObjectType> {
     private TypeWrapper typeWrapper;
-    private List<PythonType> attributes = List.of();
+    private List<TypeWrapper> attributes = List.of();
     private List<Member> members = List.of();
     private TypeSource typeSource = TypeSource.EXACT;
 
@@ -145,7 +142,7 @@ public static Builder fromTypeWrapper(TypeWrapper typeWrapper) {
     public static Builder fromType(PythonType type) {
       if (type instanceof ObjectType objectType) {
         return new Builder(objectType.typeWrapper())
-          .withAttributes(objectType.attributes())
+          .withTypeWrapperAttributes(objectType.attributes)
           .withMembers(objectType.members())
           .withTypeSource(objectType.typeSource());
       }
@@ -168,6 +165,11 @@ public Builder withType(PythonType type) {
     }
 
     public Builder withAttributes(List<PythonType> attributes) {
+      this.attributes = attributes.stream().map(TypeWrapper::of).toList();
+      return this;
+    }
+
+    public Builder withTypeWrapperAttributes(List<TypeWrapper> attributes) {
       this.attributes = attributes;
       return this;
     }

python-frontend/src/main/java/org/sonar/python/index/DescriptorsToProtobuf.java

@@ -21,9 +21,9 @@
 import java.util.Collection;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Objects;
 import java.util.Set;
 import javax.annotation.CheckForNull;
-
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.sonar.plugins.python.api.LocationInFile;
@@ -316,8 +316,11 @@ private static TypeAnnotationDescriptor fromProtobuf(SymbolsProtos.Type typeProt
     if (kind == null) {
       return null;
     }
-    List<TypeAnnotationDescriptor> args = new ArrayList<>();
-    typeProto.getArgsList().forEach(proto -> args.add(fromProtobuf(proto)));
+    List<TypeAnnotationDescriptor> args = typeProto.getArgsList().stream()
+      .map(DescriptorsToProtobuf::fromProtobuf)
+      .filter(Objects::nonNull)
+      .toList();
+
     return new TypeAnnotationDescriptor(
       typeProto.getPrettyPrintedName(),
       kind,

python-frontend/src/main/java/org/sonar/python/semantic/v2/converter/PythonTypeToDescriptorConverter.java

@@ -139,10 +139,8 @@ private static Descriptor convert(FunctionType type) {
       .toList();
 
     var returnType = type.returnType();
-    var unwrappedReturnType = returnType.unwrappedType();
-    var isSelfReturnType = containsSelfType(returnType);
-    var annotatedReturnTypeName = FullyQualifiedNameHelper.getFullyQualifiedName(unwrappedReturnType).orElse(null);
-    var returnTypeAnnotationDescriptor = createTypeAnnotationDescriptor(unwrappedReturnType, isSelfReturnType);
+    var annotatedReturnTypeName = FullyQualifiedNameHelper.getFullyQualifiedName(returnType.unwrappedType()).orElse(null);
+    var returnTypeAnnotationDescriptor = createTypeAnnotationDescriptor(returnType);
 
     // Using FunctionType#name and FunctionType#fullyQualifiedName instead of symbol is only accurate if the function has not been reassigned
     // This logic should be revisited when tackling SONARPY-2285
@@ -158,10 +156,6 @@ private static Descriptor convert(FunctionType type) {
       returnTypeAnnotationDescriptor);
   }
 
-  private static boolean containsSelfType(PythonType returnType){
-    return returnType instanceof SelfType || returnType.unwrappedType() instanceof SelfType;
-  }
-
   @VisibleForTesting
   static Descriptor convert(String moduleFqn, String parentFqn, String symbolName, SelfType selfType) {
     var innerType = selfType.innerType();
@@ -238,10 +232,9 @@ private static Descriptor convert(String parentFqn, String symbolName, UnknownTy
 
   private static FunctionDescriptor.Parameter convert(ParameterV2 parameter) {
     var type = parameter.declaredType().type();
-    var isSelf = containsSelfType(type);
     var unwrappedType = type.unwrappedType();
     var annotatedType = FullyQualifiedNameHelper.getFullyQualifiedName(unwrappedType).orElse(null);
-    var typeAnnotationDescriptor = createTypeAnnotationDescriptor(unwrappedType, isSelf);
+    var typeAnnotationDescriptor = createTypeAnnotationDescriptor(type);
 
     return new FunctionDescriptor.Parameter(parameter.name(),
       annotatedType,
@@ -263,17 +256,29 @@ private static boolean usagesContainAssignment(List<UsageV2> symbolUsages) {
   }
 
   @CheckForNull
-  private static TypeAnnotationDescriptor createTypeAnnotationDescriptor(PythonType type, boolean isSelf) {
-    if (type instanceof SelfType selfType) {
-      return createTypeAnnotationDescriptor(selfType.innerType(), isSelf);
+  private static TypeAnnotationDescriptor createTypeAnnotationDescriptor(PythonType type) {
+    return createTypeAnnotationDescriptor(type, false, List.of());
+  }
+
+  @CheckForNull
+  private static TypeAnnotationDescriptor createTypeAnnotationDescriptor(PythonType type, boolean isSelf, List<TypeAnnotationDescriptor> args) {
+    if (type instanceof ObjectType objectType) {
+      List<TypeAnnotationDescriptor> objArgs = objectType.attributes().stream()
+        .map(PythonTypeToDescriptorConverter::createTypeAnnotationDescriptor)
+        .filter(Objects::nonNull)
+        .toList();
+      return createTypeAnnotationDescriptor(objectType.unwrappedType(), isSelf, objArgs);
+    } else if (type instanceof SelfType selfType) {
+      return createTypeAnnotationDescriptor(selfType.innerType(), true, args);
     } else if (type instanceof ClassType classType) {
-      return new TypeAnnotationDescriptor(classType.name(), TypeAnnotationDescriptor.TypeKind.INSTANCE, List.of(), classType.fullyQualifiedName(), isSelf);
+      return new TypeAnnotationDescriptor(classType.name(), TypeAnnotationDescriptor.TypeKind.INSTANCE, args, classType.fullyQualifiedName(), isSelf);
     } else if (type instanceof FunctionType functionType) {
-      return new TypeAnnotationDescriptor(functionType.name(), TypeAnnotationDescriptor.TypeKind.CALLABLE, List.of(), functionType.fullyQualifiedName(), false);
+      return new TypeAnnotationDescriptor(functionType.name(), TypeAnnotationDescriptor.TypeKind.CALLABLE, args, functionType.fullyQualifiedName(), false);
     } else if (type instanceof UnknownType.UnresolvedImportType importType) {
-      return new TypeAnnotationDescriptor(importType.importPath(), TypeAnnotationDescriptor.TypeKind.INSTANCE, List.of(),
-          FullyQualifiedNameHelper.getFullyQualifiedName(importType).orElse(null), false);
+      return new TypeAnnotationDescriptor(importType.importPath(), TypeAnnotationDescriptor.TypeKind.INSTANCE, args,
+        FullyQualifiedNameHelper.getFullyQualifiedName(importType).orElse(null), false);
     }
     return null;
   }
+
 }

python-frontend/src/main/java/org/sonar/python/semantic/v2/converter/TypeAnnotationToPythonTypeConverter.java

@@ -16,10 +16,12 @@
  */
 package org.sonar.python.semantic.v2.converter;
 
+import java.util.List;
 import java.util.Optional;
 import java.util.Set;
 import java.util.function.Predicate;
 import java.util.stream.Collectors;
+import org.sonar.plugins.python.api.types.v2.ObjectType;
 import org.sonar.plugins.python.api.types.v2.PythonType;
 import org.sonar.plugins.python.api.types.v2.SelfType;
 import org.sonar.plugins.python.api.types.v2.TypeWrapper;
@@ -35,9 +37,13 @@ public PythonType convert(ConversionContext context, TypeAnnotationDescriptor ty
         if (fullyQualifiedName == null) {
           return PythonType.UNKNOWN;
         }
-        if (Boolean.TRUE.equals(type.isSelf())) {
+        List<PythonType> attributes = type.args().stream().map(t -> convert(context, t)).toList();
 
-          return SelfType.fromTypeWrapper(TypeWrapper.of(typeFromFQN(fullyQualifiedName, context)));
+        if (Boolean.TRUE.equals(type.isSelf())) {
+          var selfType = SelfType.fromTypeWrapper(TypeWrapper.of(typeFromFQN(fullyQualifiedName, context)));
+          return ObjectType.Builder.fromType(selfType)
+            .withAttributes(attributes)
+            .build();
         }
         // _SpecialForm is the type used for some special types, like Callable, Union, TypeVar, ...
         // It comes from CPython impl: https://github.com/python/cpython/blob/e39ae6bef2c357a88e232dcab2e4b4c0f367544b/Lib/typing.py#L439
@@ -46,7 +52,9 @@ public PythonType convert(ConversionContext context, TypeAnnotationDescriptor ty
         if ("typing._SpecialForm".equals(fullyQualifiedName)) {
           return PythonType.UNKNOWN;
         }
-        return typeFromFQN(fullyQualifiedName, context);
+        return ObjectType.Builder.fromType(typeFromFQN(fullyQualifiedName, context))
+          .withAttributes(attributes)
+          .build();
       case TYPE:
         return context.lazyTypesContext().getOrCreateLazyType("type");
       case TYPE_ALIAS:

python-frontend/src/main/java/org/sonar/python/semantic/v2/types/LoopAssignment.java

@@ -16,13 +16,10 @@
  */
 package org.sonar.python.semantic.v2.types;
 
-import java.util.Collection;
-import java.util.Optional;
-import java.util.stream.Stream;
-import org.sonar.plugins.python.api.TriBool;
 import org.sonar.plugins.python.api.symbols.v2.SymbolV2;
 import org.sonar.plugins.python.api.tree.Expression;
 import org.sonar.plugins.python.api.tree.Name;
+import org.sonar.plugins.python.api.types.v2.ClassType;
 import org.sonar.plugins.python.api.types.v2.ObjectType;
 import org.sonar.plugins.python.api.types.v2.PythonType;
 
@@ -33,13 +30,25 @@ public LoopAssignment(SymbolV2 lhsSymbol, Name lhsName, Expression rhs) {
 
   @Override
   public PythonType rhsType() {
-    return Optional.of(super.rhsType())
-      .filter(ObjectType.class::isInstance)
-      .map(ObjectType.class::cast)
-      .filter(t -> t.hasMember("__iter__") == TriBool.TRUE)
-      .map(ObjectType::attributes)
-      .map(Collection::stream)
-      .flatMap(Stream::findFirst)
-      .orElse(PythonType.UNKNOWN);
+    PythonType rhsType = super.rhsType();
+    if (rhsType instanceof ObjectType objectType && objectType.hasMember("__iter__").isTrue()) {
+      // depending on the origin (e.g. typeshed/user-defined), the attribute type may be wrapped in an ObjectType or not,
+      // independent of if the type is actually an instance or a class. As such, the logic in the map below assumes
+      // the rhsType contains instances of the attribute type, and forces the loop variable type to be an ObjectType.
+      var loopVarType = objectType.attributes().stream()
+        .findFirst()
+        .orElse(PythonType.UNKNOWN);
+
+      if (shouldBeWrappedInObjectType(loopVarType)) {
+        return ObjectType.fromType(loopVarType);
+      }
+
+      return loopVarType;
+    }
+    return PythonType.UNKNOWN;
+  }
+
+  private static boolean shouldBeWrappedInObjectType(PythonType type) {
+    return type instanceof ClassType classType && !"typing.Callable".equals(classType.fullyQualifiedName());
   }
 }

python-frontend/src/main/java/org/sonar/python/semantic/v2/types/typecalculator/CallReturnTypeCalculator.java

@@ -97,8 +97,14 @@ private static PythonType collapseSelfTypeIfNeeded(CallExpression callExpr, Pyth
   }
 
   private static boolean containsSelfType(PythonType type, TypePredicateContext typePredicateContext) {
-    return TypeInferenceMatcher.of(TypeInferenceMatchers.isObjectSatisfying(TypeInferenceMatchers.isSelf()))
-      .evaluate(type, typePredicateContext).isTrue();
+    if (type instanceof SelfType || type.unwrappedType() instanceof SelfType) {
+      return true;
+    }
+    if (type instanceof ObjectType objectType) {
+      return objectType.attributes().stream().anyMatch(t -> containsSelfType(t, typePredicateContext));
+    }
+
+    return false;
   }
 
   private static boolean isInstanceOrClassMethodCall(CallExpression callExpr, TypePredicateContext typePredicateContext) {
@@ -133,10 +139,16 @@ private static PythonType getReceiverType(CallExpression callExpr) {
   }
 
   private static PythonType collapseSelfType(PythonType returnType, PythonType receiverType) {
-    if (returnType instanceof ObjectType objectType && objectType.type() instanceof SelfType) {
-      return ObjectType.Builder.fromType(objectType)
-        .withType(receiverType)
+    if (returnType instanceof ObjectType objectType) {
+      var objectBuilder = ObjectType.Builder.fromType(objectType);
+      if (objectType.type() instanceof SelfType) {
+        objectBuilder.withType(receiverType);
+      }
+      return objectBuilder
+        .withAttributes(objectType.attributes().stream().map(t -> collapseSelfType(t, receiverType)).toList())
         .build();
+    } else if (returnType instanceof SelfType) {
+      return receiverType;
     }
     return PythonType.UNKNOWN;
   }

python-frontend/src/test/java/org/sonar/python/index/FunctionDescriptorTest.java

@@ -69,8 +69,25 @@ void parameterWithDefaultValue() {
     assertDescriptorToProtobuf(functionDescriptor);
   }
 
+  @ParameterizedTest
+  @MethodSource("parameterWithTypeTestCases")
+  void parameterWithType(String code, String expectedAnnotatedType) {
+    FunctionDescriptor functionDescriptor = lastFunctionDescriptor(code);
+    FunctionDescriptor.Parameter parameter = functionDescriptor.parameters().get(0);
+    assertThat(parameter.annotatedType()).isEqualTo(expectedAnnotatedType);
+    assertDescriptorToProtobuf(functionDescriptor);
+  }
+
+  static Stream<Arguments> parameterWithTypeTestCases() {
+    return Stream.of(
+      Arguments.of("def foo(x: str): ...", "str"),
+      Arguments.of("def foo(x: list[something]): ...", "list"),
+      Arguments.of("def foo(x: str): ...", "str")
+    );
+  }
+
   @Test
-  void parameterWithType() {
+  void parameterWithPositional() {
     FunctionDescriptor functionDescriptor = lastFunctionDescriptor("def foo(x: str): ...");
     FunctionDescriptor.Parameter parameter = functionDescriptor.parameters().get(0);
     assertThat(parameter.annotatedType()).isEqualTo("str");
@@ -93,14 +110,6 @@ void parameterWithKeywordOnly() {
     assertDescriptorToProtobuf(functionDescriptor);
   }
 
-  @Test
-  void parameterWithPositional() {
-    FunctionDescriptor functionDescriptor = lastFunctionDescriptor("def foo(x: str): ...");
-    FunctionDescriptor.Parameter parameter = functionDescriptor.parameters().get(0);
-    assertThat(parameter.annotatedType()).isEqualTo("str");
-    assertDescriptorToProtobuf(functionDescriptor);
-  }
-
   @Test
   void variadicParameter() {
     FunctionDescriptor functionDescriptor = lastFunctionDescriptor("def foo(*x, **y): ...");