Make joins with Any more precise

mypy/argmap.py

@@ -233,7 +233,7 @@ def expand_actual_type(
                     item = fallback.args[0]
                 return item
             elif isinstance(actual_type, ParamSpecType):
-                # ParamSpec is valid in *args but it can't be unpacked.
+                # ParamSpec is valid in *args, but it can't be unpacked.
                 return actual_type
             else:
                 return AnyType(TypeOfAny.from_error)
@@ -260,7 +260,7 @@ def expand_actual_type(
                     1
                 ]
             elif isinstance(actual_type, ParamSpecType):
-                # ParamSpec is valid in **kwargs but it can't be unpacked.
+                # ParamSpec is valid in **kwargs, but it can't be unpacked.
                 return actual_type
             else:
                 return AnyType(TypeOfAny.from_error)

mypy/checkexpr.py

@@ -5591,6 +5591,19 @@ def check_typeddict_literal_in_context(
         return typeddict_context.copy_modified()
 
     def visit_dict_expr(self, e: DictExpr) -> Type:
+        old_allow_any_join = type_state.allow_any_join
+        try:
+            ctx = self.type_context[-1]
+            # This is somewhat ad hoc, but we want to cover situations where a dict
+            # appears inside another dict or list literal as well.
+            type_state.allow_any_join = (
+                ctx is None or isinstance(ctx, TypeVarType) and ctx.id.is_meta_var()
+            )
+            return self._visit_dict_expr(e)
+        finally:
+            type_state.allow_any_join = old_allow_any_join
+
+    def _visit_dict_expr(self, e: DictExpr) -> Type:
         """Type check a dict expression.
 
         Translate it into a call to dict(), with provisions for **expr.

mypy/join.py

@@ -62,6 +62,7 @@
     get_proper_types,
     split_with_prefix_and_suffix,
 )
+from mypy.typestate import type_state
 
 
 class InstanceJoiner:
@@ -245,7 +246,9 @@ def join_types(s: Type, t: Type, instance_joiner: InstanceJoiner | None = None)
         s, t = t, s
 
     if isinstance(s, AnyType):
-        return s
+        if isinstance(t, AnyType):
+            return s
+        s, t = t, s
 
     if isinstance(s, ErasedType):
         return t
@@ -284,13 +287,22 @@ def visit_union_type(self, t: UnionType) -> ProperType:
             return mypy.typeops.make_simplified_union([self.s, t])
 
     def visit_any(self, t: AnyType) -> ProperType:
-        return t
+        if type_state.allow_any_join:
+            return t
+        if isinstance(self.s, Instance) and self.s.type.fullname == "builtins.object":
+            return self.s
+        if isinstance(self.s, UnionType) and any(
+            isinstance(get_proper_type(it), AnyType) for it in self.s.items
+        ):
+            # Avoid multiple Anys in a union type.
+            return self.s
+        return UnionType([self.s, t])
 
     def visit_none_type(self, t: NoneType) -> ProperType:
         if state.strict_optional:
             if isinstance(self.s, (NoneType, UninhabitedType)):
                 return t
-            elif isinstance(self.s, (UnboundType, AnyType)):
+            elif isinstance(self.s, UnboundType):
                 return AnyType(TypeOfAny.special_form)
             else:
                 return mypy.typeops.make_simplified_union([self.s, t])
@@ -390,8 +402,6 @@ def visit_instance(self, t: Instance) -> ProperType:
     def visit_callable_type(self, t: CallableType) -> ProperType:
         if isinstance(self.s, CallableType):
             if is_similar_callables(t, self.s):
-                if is_equivalent(t, self.s):
-                    return combine_similar_callables(t, self.s)
                 result = join_similar_callables(t, self.s)
                 if any(
                     isinstance(tp, (NoneType, UninhabitedType))
@@ -462,7 +472,7 @@ def visit_overloaded(self, t: Overloaded) -> ProperType:
                 for s_item in s.items:
                     if is_similar_callables(t_item, s_item):
                         if is_equivalent(t_item, s_item):
-                            result.append(combine_similar_callables(t_item, s_item))
+                            result.append(join_similar_callables(t_item, s_item))
                         elif is_subtype(t_item, s_item):
                             result.append(s_item)
             if result:
@@ -824,7 +834,8 @@ def join_similar_callables(t: CallableType, s: CallableType) -> CallableType:
             arg_types.append(s.arg_types[i])
         else:
             arg_types.append(safe_meet(t.arg_types[i], s.arg_types[i]))
-    # TODO in combine_similar_callables also applies here (names and kinds; user metaclasses)
+    # TODO Handle more kinds and different argument names.
+    # TODO what should happen if one fallback is 'type' and the other is a custom metaclass?
     # The fallback type can be either 'function', 'type', or some user-provided metaclass.
     # The result should always use 'function' as a fallback if either operands are using it.
     if t.fallback.type.fullname == "builtins.function":
@@ -849,7 +860,7 @@ def join_similar_callables(t: CallableType, s: CallableType) -> CallableType:
 
 
 def safe_join(t: Type, s: Type) -> Type:
-    # This is a temporary solution to prevent crashes in combine_similar_callables() etc.,
+    # This is a temporary solution to prevent crashes in join_similar_callables() etc.,
     # until relevant TODOs on handling arg_kinds will be addressed there.
     if not isinstance(t, UnpackType) and not isinstance(s, UnpackType):
         return join_types(t, s)
@@ -880,36 +891,6 @@ def safe_meet(t: Type, s: Type) -> Type:
     return UninhabitedType()
 
 
-def combine_similar_callables(t: CallableType, s: CallableType) -> CallableType:
-    t, s = match_generic_callables(t, s)
-    arg_types: list[Type] = []
-    for i in range(len(t.arg_types)):
-        arg_types.append(safe_join(t.arg_types[i], s.arg_types[i]))
-    # TODO kinds and argument names
-    # TODO what should happen if one fallback is 'type' and the other is a user-provided metaclass?
-    # The fallback type can be either 'function', 'type', or some user-provided metaclass.
-    # The result should always use 'function' as a fallback if either operands are using it.
-    if t.fallback.type.fullname == "builtins.function":
-        fallback = t.fallback
-    elif s.fallback.type.fullname == "builtins.function":
-        fallback = s.fallback
-    else:
-        fallback = InstanceJoiner().join_instances(t.fallback, s.fallback)
-        assert isinstance(fallback, Instance)
-    instance_type = None
-    if t.instance_type is not None and s.instance_type is not None:
-        instance_type = join_types(t.instance_type, s.instance_type)
-    return t.copy_modified(
-        arg_types=arg_types,
-        arg_kinds=combine_arg_kinds(t, s),
-        arg_names=combine_arg_names(t, s),
-        ret_type=join_types(t.ret_type, s.ret_type),
-        instance_type=instance_type,
-        fallback=fallback,
-        name=None,
-    )
-
-
 def combine_arg_names(
     t: CallableType | Parameters, s: CallableType | Parameters
 ) -> list[str | None]:

mypy/meet.py

@@ -961,8 +961,6 @@ def visit_callable_type(self, t: CallableType) -> ProperType:
         if isinstance(self.s, CallableType) and join.is_similar_callables(
             t, self.s, for_meet=True
         ):
-            if is_equivalent(t, self.s):
-                return join.combine_similar_callables(t, self.s)
             result = meet_similar_callables(t, self.s)
             # We set the from_type_type flag to suppress error when a collection of
             # concrete class objects gets inferred as their common abstract superclass.
@@ -1247,7 +1245,7 @@ def meet_similar_callables(t: CallableType, s: CallableType) -> CallableType:
     arg_types: list[Type] = []
     for i in range(len(t.arg_types)):
         arg_types.append(safe_join(t.arg_types[i], s.arg_types[i]))
-    # TODO in combine_similar_callables also applies here (names and kinds)
+    # TODOs in join_similar_callables() also applies here.
     # The fallback type can be either 'function' or 'type'. The result should have 'function' as
     # fallback only if both operands have it as 'function'.
     if t.fallback.type.fullname != "builtins.function":

mypy/test/testsolve.py

@@ -6,7 +6,7 @@
 from mypy.solve import Bounds, Graph, solve_constraints, transitive_closure
 from mypy.test.helpers import Suite, assert_equal
 from mypy.test.typefixture import TypeFixture
-from mypy.types import Type, TypeVarId, TypeVarLikeType, TypeVarType
+from mypy.types import Type, TypeVarId, TypeVarLikeType, TypeVarType, UnionType
 
 
 class SolveSuite(Suite):
@@ -84,7 +84,7 @@ def test_simple_constraints_with_dynamic_type(self) -> None:
         self.assert_solve(
             [self.fx.t],
             [self.supc(self.fx.t, self.fx.anyt), self.supc(self.fx.t, self.fx.a)],
-            [self.fx.anyt],
+            [UnionType([self.fx.a, self.fx.anyt])],
         )
 
         self.assert_solve([self.fx.t], [self.subc(self.fx.t, self.fx.anyt)], [self.fx.anyt])

mypy/test/testtypes.py

@@ -840,21 +840,45 @@ def test_unbound_type(self) -> None:
             self.assert_join(t, UnboundType("X"), self.fx.anyt)
 
     def test_any_type(self) -> None:
-        # Join against 'Any' type always results in 'Any'.
         with state.strict_optional_set(False):
             self.assert_join(NoneType(), self.fx.anyt, self.fx.anyt)
 
         for t in [
-            self.fx.anyt,
             self.fx.a,
-            self.fx.o,
-            NoneType(),
             UnboundType("x"),
             self.fx.t,
             self.tuple(),
             self.callable(self.fx.a, self.fx.b),
         ]:
-            self.assert_join(t, self.fx.anyt, self.fx.anyt)
+            self.assert_join(t, self.fx.anyt, UnionType([t, self.fx.anyt]))
+            self.assert_join(self.fx.anyt, t, UnionType([t, self.fx.anyt]))
+
+        self.assert_join(self.fx.anyt, self.fx.anyt, self.fx.anyt)
+
+        self.assert_join(self.fx.o, self.fx.anyt, self.fx.o)
+        self.assert_join(NoneType(), self.fx.anyt, UnionType([self.fx.anyt, NoneType()]))
+        self.assert_join(UninhabitedType(), self.fx.anyt, self.fx.anyt)
+
+        self.assert_join(self.fx.anyt, self.fx.o, self.fx.o)
+        self.assert_join(self.fx.anyt, NoneType(), UnionType([self.fx.anyt, NoneType()]))
+        self.assert_join(self.fx.anyt, UninhabitedType(), self.fx.anyt)
+
+    def test_any_type_with_union(self) -> None:
+        self.assert_join(
+            UnionType([self.fx.a, self.fx.anyt]),
+            self.fx.anyt,
+            UnionType([self.fx.a, self.fx.anyt]),
+        )
+        self.assert_join(
+            self.fx.anyt,
+            UnionType([self.fx.a, self.fx.anyt]),
+            UnionType([self.fx.a, self.fx.anyt]),
+        )
+        self.assert_join(
+            UnionType([self.fx.a, self.fx.anyt]),
+            UnionType([self.fx.a, self.fx.anyt]),
+            UnionType([self.fx.a, self.fx.anyt]),
+        )
 
     def test_mixed_truth_restricted_type_simple(self) -> None:
         # make_simplified_union against differently restricted truthiness types drops restrictions.
@@ -885,7 +909,7 @@ def test_simple_generics(self) -> None:
         with state.strict_optional_set(True):
             self.assert_join(self.fx.ga, self.fx.nonet, UnionType([self.fx.ga, NoneType()]))
 
-        self.assert_join(self.fx.ga, self.fx.anyt, self.fx.anyt)
+        self.assert_join(self.fx.ga, self.fx.anyt, UnionType([self.fx.ga, self.fx.anyt]))
 
         for t in [
             self.fx.a,
@@ -938,7 +962,9 @@ def test_callables_with_any(self) -> None:
         self.assert_join(
             self.callable(self.fx.a, self.fx.a, self.fx.anyt, self.fx.a),
             self.callable(self.fx.a, self.fx.anyt, self.fx.a, self.fx.anyt),
-            self.callable(self.fx.a, self.fx.anyt, self.fx.anyt, self.fx.anyt),
+            self.callable(
+                self.fx.a, self.fx.anyt, self.fx.anyt, UnionType([self.fx.a, self.fx.anyt])
+            ),
         )
 
     def test_overloaded(self) -> None:
@@ -1016,12 +1042,14 @@ def test_simple_type_objects(self) -> None:
 
     def test_type_type(self) -> None:
         self.assert_join(self.fx.type_a, self.fx.type_b, self.fx.type_a)
-        self.assert_join(self.fx.type_b, self.fx.type_any, self.fx.type_any)
+        self.assert_join(
+            self.fx.type_b, self.fx.type_any, UnionType([self.fx.type_b, self.fx.type_any])
+        )
         self.assert_join(self.fx.type_b, self.fx.type_type, self.fx.type_type)
         self.assert_join(self.fx.type_b, self.fx.type_c, self.fx.type_a)
         self.assert_join(self.fx.type_c, self.fx.type_d, TypeType.make_normalized(self.fx.o))
         self.assert_join(self.fx.type_type, self.fx.type_any, self.fx.type_type)
-        self.assert_join(self.fx.type_b, self.fx.anyt, self.fx.anyt)
+        self.assert_join(self.fx.type_b, self.fx.anyt, UnionType([self.fx.type_b, self.fx.anyt]))
 
     def test_literal_type(self) -> None:
         a = self.fx.a
@@ -1035,7 +1063,7 @@ def test_literal_type(self) -> None:
         self.assert_join(lit1, d, self.fx.o)
         self.assert_join(lit1, lit2, a)
         self.assert_join(lit1, lit3, self.fx.o)
-        self.assert_join(lit1, self.fx.anyt, self.fx.anyt)
+        self.assert_join(lit1, self.fx.anyt, UnionType([lit1, self.fx.anyt]))
         self.assert_join(UnionType([lit1, lit2]), lit2, UnionType([lit1, lit2]))
         self.assert_join(UnionType([lit1, lit2]), a, a)
         self.assert_join(UnionType([lit1, lit3]), a, UnionType([a, lit3]))
@@ -1324,7 +1352,12 @@ def test_callables_with_dynamic(self) -> None:
         self.assert_meet(
             self.callable(self.fx.a, self.fx.a, self.fx.anyt, self.fx.a),
             self.callable(self.fx.a, self.fx.anyt, self.fx.a, self.fx.anyt),
-            self.callable(self.fx.a, self.fx.anyt, self.fx.anyt, self.fx.anyt),
+            self.callable(
+                self.fx.a,
+                UnionType([self.fx.a, self.fx.anyt]),
+                UnionType([self.fx.a, self.fx.anyt]),
+                self.fx.a,
+            ),
         )
 
     def test_meet_interface_types(self) -> None:

mypy/typestate.py

@@ -113,11 +113,15 @@ class TypeState:
     # Whether to use new type inference algorithm that can infer polymorphic types.
     # This is temporary and will be removed soon when new algorithm is more polished.
     infer_polymorphic: bool
-    # No not simplify/normalize unions during type inference in this flag is set.
+    # Do not simplify/normalize unions during type inference in this flag is set.
     # Technically, union simplification gives strictly better type inference, but it
     # may have bad performance (since it is quadratic). So we may want to disable
     # it situations where performance is important.
     keep_unions: bool
+    # Infer join(X, Any) as Any (instead of X | Any) when this flag is set.
+    # This is used to preserve backwards compatibility for situations like
+    # testDictWithUnionAnySpecialCaseInStar2. This is needed until #5382 is fixed.
+    allow_any_join: bool
 
     # N.B: We do all the accesses to these properties through
     # TypeState, instead of making these classmethods and accessing
@@ -137,6 +141,7 @@ def __init__(self) -> None:
         self.infer_unions = False
         self.infer_polymorphic = False
         self.keep_unions = False
+        self.allow_any_join = False
         self.bind_self_cache = {}
 
     def is_assumed_subtype(self, left: Type, right: Type) -> bool:

mypyc/test-data/irbuild-lists.test

@@ -986,7 +986,8 @@ def test():
     r19 :: list
     r20, r21 :: native_int
     r22 :: bit
-    r23, x :: object
+    r23 :: object
+    x :: union[str, tuple[str, str], object]
     r24 :: str
     r25 :: native_int
     a :: list

mypyc/test-data/irbuild-tuple.test

@@ -1042,7 +1042,8 @@ def test():
     r19 :: tuple
     r20, r21 :: native_int
     r22 :: bit
-    r23, x :: object
+    r23 :: object
+    x :: union[str, tuple[str, str], object]
     r24 :: str
     r25 :: native_int
     a :: tuple

test-data/unit/check-classes.test

@@ -9844,7 +9844,9 @@ fs = [f, lambda x: 1, f]
 reveal_type(fs)  # N: Revealed type is "builtins.list[def (x: Any) -> builtins.int]"
 
 [case testLambdaInferredInCollection2]
-def f(x: int, y: int)-> int: ...
+def f(x: int, y: int) -> int: ...
 
 fs = [lambda df, x: df.foo, f]
-reveal_type(fs)  # N: Revealed type is "builtins.list[def (Any, Any) -> Any]"
+reveal_type(fs)  # N: Revealed type is "builtins.list[def (Any, Any) -> builtins.int | Any]"
+fs2 = [f, lambda df, x: df.foo]
+reveal_type(fs2)  # N: Revealed type is "builtins.list[def (Any, Any) -> builtins.int | Any]"

test-data/unit/check-functions.test

@@ -3958,6 +3958,20 @@ Ex = TypeVar("Ex", covariant=True)
 def takes_lambda(func: Callable[[Ex], Any]) -> None:
     pass
 
+[case testDictWithUnionAnySpecialCaseInStar2]
+from typing import Any
+
+a: Any
+d = {"x": 1, "y": a}
+d2 = dict(x=1, y=a)
+ds = [{"x": 1, "y": a} for _ in [1, 2, 3]]
+
+def test(x: int, y: str) -> None: ...
+test(**d)
+test(**d2)
+test(**ds[0])
+[builtins fixtures/dict.pyi]
+
 [case testTypeObjectJoinWithOverloadsCorrectFallback]
 from typing import Iterable, overload
 

test-data/unit/check-generics.test

@@ -3595,7 +3595,7 @@ def draw_none(
         takes_int_str_none(c3)
 [builtins fixtures/tuple.pyi]
 
-[case testDeterminismFromJoinOrderingInSolver2]
+[case testDeterminismFromJoinOrderingInSolver2_no_verbose_reveal]
 # Used to fail non-deterministically
 # https://github.com/python/mypy/issues/21445
 from typing import Generic, Iterable, TypeVar, overload
@@ -3632,7 +3632,7 @@ class dictlike(Generic[K, V]):
     def __init__(self, arg: Iterable[tuple[K, V]]) -> None: pass
 
 x = dictlike(ziplike("012", (f0, f1, g)))
-reveal_type(x)  # N: Revealed type is "__main__.dictlike[builtins.str, Overload(def (Any, Any) -> Any, def (Any, Any) -> Any, def (Any, Any) -> Any, def (Any, Any) -> Any)]"
+reveal_type(x)  # N: Revealed type is "__main__.dictlike[str, Overload(def (Any, Any) -> object, def (Any, Any) -> object, def (Any, Any) -> object, def (Any, Any) -> object)]"
 [builtins fixtures/dict.pyi]
 
 [case testPropertyWithGenericSetter]