From 95cc933d52bcc6ee73d2be8aeb3ddc3a1895781f Mon Sep 17 00:00:00 2001 From: Ryan Heard Date: Mon, 12 Oct 2026 21:30:52 -0400 Subject: [PATCH] [mypyc] Fix *args and **kwargs in super() calls to native methods --- mypyc/codegen/emitclass.py | 13 +- mypyc/codegen/emitmodule.py | 11 +- mypyc/codegen/emitwrapper.py | 11 ++ mypyc/irbuild/expression.py | 17 ++- mypyc/irbuild/ll_builder.py | 77 +++++++++-- mypyc/lib-rt/CPy.h | 1 + mypyc/lib-rt/generic_ops.c | 27 ++++ mypyc/primitives/generic_ops.py | 9 ++ mypyc/test-data/irbuild-classes.test | 73 ++++++++++ mypyc/test-data/run-classes.test | 199 +++++++++++++++++++++++++++ 10 files changed, 408 insertions(+), 30 deletions(-) diff --git a/mypyc/codegen/emitclass.py b/mypyc/codegen/emitclass.py index 69c0b3e7c8c6f..0ef8d9f45e0bc 100644 --- a/mypyc/codegen/emitclass.py +++ b/mypyc/codegen/emitclass.py @@ -25,6 +25,7 @@ generate_len_wrapper, generate_richcompare_wrapper, generate_set_del_item_wrapper, + is_fastcall_supported, ) from mypyc.common import ( BITMAP_BITS, @@ -1116,11 +1117,15 @@ def generate_methods_table( continue emitter.emit_line(f'{{"{fn.name}",') emitter.emit_line(f" (PyCFunction){PREFIX}{fn.cname(emitter.names)},") - flags = ["METH_FASTCALL", "METH_KEYWORDS"] - if fn.decl.kind == FUNC_STATICMETHOD: - flags.append("METH_STATIC") - elif fn.decl.kind == FUNC_CLASSMETHOD: + if is_fastcall_supported(fn, emitter.capi_version): + flags = ["METH_FASTCALL", "METH_KEYWORDS"] + else: + flags = ["METH_VARARGS", "METH_KEYWORDS"] + if fn.decl.kind == FUNC_CLASSMETHOD or fn.name == "__new__": + # The wrapper function of __new__ takes the class as self, like a class method. flags.append("METH_CLASS") + elif fn.decl.kind == FUNC_STATICMETHOD: + flags.append("METH_STATIC") doc = native_function_doc_initializer(fn) emitter.emit_line(" {}, PyDoc_STR({})}},".format(" | ".join(flags), doc)) diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index 6ebc74a96fb89..bb4bf45a33ff1 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -41,6 +41,7 @@ from mypyc.codegen.emitwrapper import ( generate_legacy_wrapper_function, generate_wrapper_function, + is_fastcall_supported, legacy_wrapper_function_header, wrapper_function_header, ) @@ -1726,16 +1727,6 @@ def visit(item: T) -> None: return result -def is_fastcall_supported(fn: FuncIR, capi_version: tuple[int, int]) -> bool: - if fn.class_name is not None: - if fn.name == "__call__": - # We can use vectorcalls (PEP 590) when supported - return True - # TODO: Support fastcall for __init__ and __new__. - return fn.name != "__init__" and fn.name != "__new__" - return True - - def collect_literals(fn: FuncIR, literals: Literals) -> None: """Store all Python literal object refs in fn. diff --git a/mypyc/codegen/emitwrapper.py b/mypyc/codegen/emitwrapper.py index f048778bfb1b8..f6270ecca82de 100644 --- a/mypyc/codegen/emitwrapper.py +++ b/mypyc/codegen/emitwrapper.py @@ -37,6 +37,17 @@ ) from mypyc.namegen import NameGenerator + +def is_fastcall_supported(fn: FuncIR, capi_version: tuple[int, int]) -> bool: + if fn.class_name is not None: + if fn.name == "__call__": + # We can use vectorcalls (PEP 590) when supported + return True + # TODO: Support fastcall for __init__ and __new__. + return fn.name != "__init__" and fn.name != "__new__" + return True + + # Generic vectorcall wrapper functions (Python 3.7+) # # A wrapper function has a signature like this: diff --git a/mypyc/irbuild/expression.py b/mypyc/irbuild/expression.py index 8da4a38aa1ed3..8d24988ca578d 100644 --- a/mypyc/irbuild/expression.py +++ b/mypyc/irbuild/expression.py @@ -507,7 +507,13 @@ def translate_method_call(builder: IRBuilder, expr: CallExpr, callee: MemberExpr and isinstance(callee.expr.node, TypeInfo) and callee.expr.node in builder.mapper.type_to_ir and builder.mapper.type_to_ir[callee.expr.node].has_method(callee.name) - and all(kind in (ARG_POS, ARG_NAMED) for kind in expr.arg_kinds) + and ( + all(kind in (ARG_POS, ARG_NAMED) for kind in expr.arg_kinds) + # A lookup of __init__ would find the slot wrapper, and the __init__ slot of + # a native class usually does nothing (__init__ is called when the object is + # created instead). + or is_native_init(builder.mapper.type_to_ir[callee.expr.node], callee.name) + ) ): # Call a method via the *class* assert isinstance(callee.expr.node, TypeInfo), callee.expr.node @@ -553,6 +559,15 @@ def translate_method_call(builder: IRBuilder, expr: CallExpr, callee: MemberExpr ) +def is_native_init(ir: ClassIR, name: str) -> bool: + """Is this the __init__ method of the class, and does an extension class define it?""" + if name == "__init__": + for base in ir.mro: + if name in base.method_decls: + return base.is_ext_class + return False + + def call_classmethod(builder: IRBuilder, ir: ClassIR, expr: CallExpr, callee: MemberExpr) -> Value: decl = ir.method_decl(callee.name) args = [] diff --git a/mypyc/irbuild/ll_builder.py b/mypyc/irbuild/ll_builder.py index 16c26dec09e92..a61bcf4ff3015 100644 --- a/mypyc/irbuild/ll_builder.py +++ b/mypyc/irbuild/ll_builder.py @@ -149,6 +149,7 @@ from mypyc.primitives.generic_ops import ( generic_len_op, generic_ssize_t_len_op, + method_wrapper_op, py_call_op, py_call_with_kwargs_op, py_call_with_posargs_op, @@ -1365,12 +1366,36 @@ def call( arguments used to track the presence of values for certain arguments. By default, the values of the bitmap arguments are inferred from args. """ + if decl.class_name is not None and not self.can_pass_args_natively( + arg_kinds, arg_names, decl.sig + ): + return self.call_method_wrapper(decl, args, arg_kinds, arg_names, line) # Normalize args to positionals. args = self.native_args_to_positional( args, arg_kinds, arg_names, decl.sig, line, bitmap_args=bitmap_args ) return self.add(Call(decl, args, line)) + def call_method_wrapper( + self, + decl: FuncDecl, + args: Sequence[Value], + arg_kinds: list[ArgKind], + arg_names: Sequence[str | None], + line: int, + ) -> Value: + """Call a native method through its wrapper function, which parses the arguments. + + This is slower than a native call, but like a native call it calls the given + method without looking it up. A lookup could find an override, or the __init__ + slot wrapper of a native class, which usually does nothing (__init__ is called + when the object is created instead). + """ + typ = self.load_native_type_object(f"{decl.module_name}.{decl.class_name}") + wrapper = self.call_c(method_wrapper_op, [typ, self.load_str(decl.name)], line) + result = self.py_call(wrapper, list(args), line, arg_kinds, arg_names) + return self.coerce(result, decl.sig.ret_type, line) + def native_args_to_positional( self, args: Sequence[Value], @@ -1390,22 +1415,9 @@ def native_args_to_positional( and coerce arguments to the appropriate type. """ - sig_args = sig.args + sig_args = sig.real_args() n = sig.num_bitmap_args - if n: - sig_args = sig_args[:-n] - - sig_arg_kinds = [arg.kind for arg in sig_args] - sig_arg_names = [arg.name for arg in sig_args] - - concrete_kinds = [concrete_arg_kind(arg_kind) for arg_kind in arg_kinds] - formal_to_actual = map_actuals_to_formals( - concrete_kinds, - arg_names, - sig_arg_kinds, - sig_arg_names, - lambda n: AnyType(TypeOfAny.special_form), - ) + formal_to_actual = self.map_native_call_args(arg_kinds, arg_names, sig) # First scan for */** and construct those has_star = has_star2 = False @@ -1465,6 +1477,41 @@ def native_args_to_positional( return output_args + def map_native_call_args( + self, arg_kinds: list[ArgKind], arg_names: Sequence[str | None], sig: FuncSignature + ) -> list[list[int]]: + """Map the arguments of a call to the arguments of a native function. + + Return the indexes of the call arguments that each argument of the function + (excluding bitmap arguments) may get its value from. + """ + sig_args = sig.real_args() + return map_actuals_to_formals( + [concrete_arg_kind(arg_kind) for arg_kind in arg_kinds], + arg_names, + [arg.kind for arg in sig_args], + [arg.name for arg in sig_args], + lambda n: AnyType(TypeOfAny.special_form), + ) + + def can_pass_args_natively( + self, arg_kinds: list[ArgKind], arg_names: Sequence[str | None], sig: FuncSignature + ) -> bool: + """Can native_args_to_positional() prepare the arguments of a call? + + It can't unpack a *args or **kwargs argument, since we don't know at compile + time which arguments of the function the items are for. It can only pass one + on as a whole to the *args or **kwargs argument of the function. + """ + if not any(kind.is_star() for kind in arg_kinds): + return True + formal_to_actual = self.map_native_call_args(arg_kinds, arg_names, sig) + for formal, actuals in zip(sig.real_args(), formal_to_actual): + for actual in actuals: + if arg_kinds[actual].is_star() and arg_kinds[actual] != formal.kind: + return False + return True + def gen_method_call( self, base: Value, diff --git a/mypyc/lib-rt/CPy.h b/mypyc/lib-rt/CPy.h index 64cd25f0c3351..af4ae0c6b499d 100644 --- a/mypyc/lib-rt/CPy.h +++ b/mypyc/lib-rt/CPy.h @@ -706,6 +706,7 @@ PyObject *CPyNumber_Power(PyObject *base, PyObject *index); PyObject *CPyNumber_InPlacePower(PyObject *base, PyObject *index); // An omitted slice bound is represented by CPY_INT_TAG. PyObject *CPyObject_GetSlice(PyObject *obj, CPyTagged start, CPyTagged end); +PyObject *CPy_GetMethodWrapper(PyObject *type, PyObject *name); // List operations diff --git a/mypyc/lib-rt/generic_ops.c b/mypyc/lib-rt/generic_ops.c index 2c07d8e7ced80..1a25707617b5c 100644 --- a/mypyc/lib-rt/generic_ops.c +++ b/mypyc/lib-rt/generic_ops.c @@ -84,3 +84,30 @@ PyObject *CPy_SetupObject(PyObject *type) { PyErr_SetString(PyExc_RuntimeError, "Internal mypyc error: Unable to find object setup function"); return NULL; } + +// Return a callable for the wrapper function of a method that a native class defines. +// It takes the same arguments as the method when looked up on the class (including self +// or cls). Unlike an attribute lookup, this doesn't find an override in a subclass or a +// slot wrapper. The __init__ slot of a native class usually does nothing. +PyObject *CPy_GetMethodWrapper(PyObject *type, PyObject *name) { + PyTypeObject *tp = (PyTypeObject *)type; + PyMethodDef *def = tp->tp_methods; + const char *name_str = PyUnicode_AsUTF8(name); + if (name_str == NULL) { + return NULL; + } + for (; def != NULL && def->ml_name != NULL; def++) { + if (strcmp(name_str, def->ml_name) == 0) { + if (def->ml_flags & METH_STATIC) { + return PyCFunction_NewEx(def, NULL, NULL); + } + if (def->ml_flags & METH_CLASS) { + return PyDescr_NewClassMethod(tp, def); + } + return PyDescr_NewMethod(tp, def); + } + } + + PyErr_SetString(PyExc_RuntimeError, "Internal mypyc error: Unable to find method wrapper"); + return NULL; +} diff --git a/mypyc/primitives/generic_ops.py b/mypyc/primitives/generic_ops.py index 738f7bf8808ac..35b99e0f269d9 100644 --- a/mypyc/primitives/generic_ops.py +++ b/mypyc/primitives/generic_ops.py @@ -432,3 +432,12 @@ c_function_name="CPy_SetupObject", error_kind=ERR_MAGIC, ) + +# Get a callable for the wrapper function of a method that a native class defines +# (arguments: type object, method name). +method_wrapper_op = custom_op( + arg_types=[object_rprimitive, object_rprimitive], + return_type=object_rprimitive, + c_function_name="CPy_GetMethodWrapper", + error_kind=ERR_MAGIC, +) diff --git a/mypyc/test-data/irbuild-classes.test b/mypyc/test-data/irbuild-classes.test index 59ab6ced0391c..50e7f0f46b6c4 100644 --- a/mypyc/test-data/irbuild-classes.test +++ b/mypyc/test-data/irbuild-classes.test @@ -682,6 +682,79 @@ def D.g(cls, x): L0: return x +[case testNativeCallWithUnpackedArgs] +from typing import Any + +class A: + def f(self, x: int, y: int = 0) -> int: + return x + y + + def g(self, *args: Any, **kwargs: Any) -> int: + return len(args) + +class B(A): + def call_f(self, *args: int) -> int: + # The items are for ordinary arguments, so the wrapper function has to unpack them + return super().f(*args) + + def call_g(self, *args: Any, **kwargs: Any) -> int: + return super().g(*args, **kwargs) +[out] +def A.f(self, x, y): + self :: __main__.A + x, y, r0 :: int +L0: + if is_error(y) goto L1 else goto L2 +L1: + y = 0 +L2: + r0 = CPyTagged_Add(x, y) + return r0 +def A.g(self, args, kwargs): + self :: __main__.A + args :: tuple + kwargs :: dict + r0 :: native_int + r1 :: short_int +L0: + r0 = var_object_size args + r1 = r0 << 1 + return r1 +def B.call_f(self, args): + self :: __main__.B + args :: tuple + r0 :: object + r1 :: str + r2 :: object + r3 :: list + r4 :: ptr + r5 :: object + r6 :: tuple + r7 :: object + r8 :: int +L0: + r0 = __main__.A :: type + r1 = 'f' + r2 = CPy_GetMethodWrapper(r0, r1) + r3 = PyList_New(1) + r4 = list_items r3 + buf_init_item r4, 0, self + keep_alive r3 + r5 = CPyList_Extend(r3, args) + r6 = PyList_AsTuple(r3) + r7 = PyObject_CallObject(r2, r6) + r8 = unbox(int, r7) + return r8 +def B.call_g(self, args, kwargs): + self :: __main__.B + args :: tuple + kwargs, r0 :: dict + r1 :: int +L0: + r0 = PyDict_Copy(kwargs) + r1 = A.g(self, args, r0) + return r1 + [case testCannotAssignToClsArgument] from typing import Any, cast diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index d8965568af3db..e11dc4c916eab 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -3783,6 +3783,205 @@ import native (31, 12, 23) (61, 42, 53) +[case testNativeCallsWithUnpackedArgs] +from typing import Any, final + +from testutil import assertRaises + +class Base: + def __init__(self, x: int, y: int = 0) -> None: + self.x = x + self.y = y + + def add(self, x: int, y: int = 0) -> int: + return x + y + + def pair(self, a: object, b: object = None) -> tuple[object, object]: + return a, b + + def rest(self, x: int, *args: int, key: str = "k", **kwargs: int) -> str: + return f"{x} {args} {key} {kwargs}" + + def star(self, *args: Any, **kwargs: Any) -> str: + return f"{args} {kwargs}" + + @classmethod + def cm(cls, x: int, y: int = 0) -> str: + return f"{cls.__name__} {x} {y}" + + @staticmethod + def sm(x: int, y: int = 0) -> int: + return x * 10 + y + +class Sub(Base): + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + + def add_args(self, *args: int) -> int: + return super().add(*args) + + def add_kwargs(self, **kwargs: int) -> int: + return super().add(**kwargs) + + def add_both(self, *args: int, **kwargs: int) -> int: + return super().add(*args, **kwargs) + + def add_after_arg(self, *args: int, **kwargs: int) -> list[int]: + return [super().add(1, *args), super().add(1, **kwargs)] + + def add_tuple(self, t: tuple[int, int]) -> int: + return super().add(*t) + + def add_list(self, a: list[int]) -> int: + return super().add(*a) + + def add_dict(self, d: dict[str, int]) -> int: + return super().add(**d) + + def pair_args(self, *args: object) -> tuple[object, object]: + return super().pair(*args) + + def rest_args(self, *args: int, **kwargs: Any) -> str: + return super().rest(*args, **kwargs) + + def star_args(self, *args: Any, **kwargs: Any) -> str: + return super().star(*args, **kwargs) + + def cm_args(self, *args: int) -> str: + return super().cm(*args) + + def sm_args(self, *args: int, **kwargs: int) -> int: + return super().sm(*args, **kwargs) + +def test_super_call_with_unpacked_args() -> None: + s = Sub(1) + assert s.add_args(2, 3) == 5 + assert s.add_args(2) == 2 + assert s.add_kwargs(x=2, y=3) == 5 + assert s.add_kwargs(x=2) == 2 + assert s.add_both(2, y=3) == 5 + assert s.add_after_arg() == [1, 1] + assert s.add_after_arg(2) == [3, 1] + assert s.add_after_arg(y=4) == [1, 5] + assert s.add_tuple((2, 3)) == 5 + assert s.add_list([2, 3]) == 5 + assert s.add_dict({"x": 2, "y": 3}) == 5 + # The unpacked items must not be passed on as a single argument + assert s.pair_args(1, "x") == (1, "x") + assert s.pair_args((1, 2)) == ((1, 2), None) + assert s.rest_args(1) == "1 () k {}" + assert s.rest_args(1, 2, 3, key="a", z=4) == "1 (2, 3) a {'z': 4}" + assert s.star_args(1, 2, a=3) == "(1, 2) {'a': 3}" + assert s.cm_args(1, 2) == "Sub 1 2" + assert s.sm_args(1) == 10 + assert s.sm_args(1, y=2) == 12 + +def test_super_call_with_invalid_unpacked_args() -> None: + s = Sub(1) + with assertRaises(TypeError): + s.add_args() + with assertRaises(TypeError): + s.add_args(1, 2, 3) + with assertRaises(TypeError): + s.add_kwargs(y=1) + with assertRaises(TypeError): + s.add_kwargs(x=1, z=2) + with assertRaises(TypeError): + s.add_after_arg(x=2) + with assertRaises(TypeError): + s.pair_args() + +def test_super_init_with_unpacked_args() -> None: + s = Sub(1, 2) + assert (s.x, s.y) == (1, 2) + s = Sub(3) + assert (s.x, s.y) == (3, 0) + s = Sub(4, y=5) + assert (s.x, s.y) == (4, 5) + s = Sub(y=6, x=7) + assert (s.x, s.y) == (7, 6) + with assertRaises(TypeError): + Sub() + with assertRaises(TypeError): + Sub(1, 2, 3) + +class ViaClass(Base): + def __init__(self, *args: Any, **kwargs: Any) -> None: + Base.__init__(self, *args, **kwargs) + + def add_args(self, *args: int) -> int: + return Base.add(self, *args) + + def add_self_in_args(self, *args: Any) -> int: + return Base.add(*args) + + def cm_args(self, *args: int) -> str: + return Base.cm(*args) + +def test_call_via_class_with_unpacked_args() -> None: + v = ViaClass(1, y=2) + assert (v.x, v.y) == (1, 2) + assert v.add_args(2, 3) == 5 + assert v.add_self_in_args(v, 2, 3) == 5 + assert v.cm_args(1, 2) == "Base 1 2" + with assertRaises(TypeError): + ViaClass() + +class New: + x: int + y: int + + def __new__(cls, x: int, y: int = 0) -> "New": + obj = object.__new__(cls) + obj.x = x + obj.y = y + return obj + +class SubNew(New): + def __new__(cls, *args: Any, **kwargs: Any) -> "SubNew": + obj = super().__new__(cls, *args, **kwargs) + assert isinstance(obj, SubNew) + return obj + +def test_super_new_with_unpacked_args() -> None: + n = SubNew(1, 2) + assert (n.x, n.y) == (1, 2) + n = SubNew(y=3, x=4) + assert (n.x, n.y) == (4, 3) + with assertRaises(TypeError): + SubNew() + +@final +class Final: + @classmethod + def join(cls, a: str, b: str = "?") -> str: + return f"{cls.__name__} {a} {b}" + + @classmethod + def pair(cls, a: object, b: object = None) -> tuple[object, object]: + return a, b + + @classmethod + def join_args(cls, *args: str) -> str: + return cls.join(*args) + + @classmethod + def join_kwargs(cls, **kwargs: str) -> str: + return cls.join(**kwargs) + + @classmethod + def pair_tuple(cls, t: tuple[int, str]) -> tuple[object, object]: + return cls.pair(*t) + +def test_call_via_cls_with_unpacked_args() -> None: + assert Final.join_args("a", "b") == "Final a b" + assert Final.join_args("a") == "Final a ?" + assert Final.join_kwargs(a="a", b="b") == "Final a b" + assert Final.join_kwargs(a="a") == "Final a ?" + assert Final.pair_tuple((1, "x")) == (1, "x") + with assertRaises(TypeError): + Final.join_args() + [case testDataclassInitVar] import dataclasses