[mypyc] Fix *args and **kwargs in super() calls to native methods

mypyc/codegen/emitclass.py

@@ -25,6 +25,7 @@
     generate_len_wrapper,
     generate_richcompare_wrapper,
     generate_set_del_item_wrapper,
+    is_fastcall_supported,
 )
 from mypyc.common import (
     BITMAP_BITS,
@@ -1116,11 +1117,15 @@ def generate_methods_table(
             continue
         emitter.emit_line(f'{{"{fn.name}",')
         emitter.emit_line(f" (PyCFunction){PREFIX}{fn.cname(emitter.names)},")
-        flags = ["METH_FASTCALL", "METH_KEYWORDS"]
-        if fn.decl.kind == FUNC_STATICMETHOD:
-            flags.append("METH_STATIC")
-        elif fn.decl.kind == FUNC_CLASSMETHOD:
+        if is_fastcall_supported(fn, emitter.capi_version):
+            flags = ["METH_FASTCALL", "METH_KEYWORDS"]
+        else:
+            flags = ["METH_VARARGS", "METH_KEYWORDS"]
+        if fn.decl.kind == FUNC_CLASSMETHOD or fn.name == "__new__":
+            # The wrapper function of __new__ takes the class as self, like a class method.
             flags.append("METH_CLASS")
+        elif fn.decl.kind == FUNC_STATICMETHOD:
+            flags.append("METH_STATIC")
 
         doc = native_function_doc_initializer(fn)
         emitter.emit_line(" {}, PyDoc_STR({})}},".format(" | ".join(flags), doc))

mypyc/codegen/emitmodule.py

@@ -41,6 +41,7 @@
 from mypyc.codegen.emitwrapper import (
     generate_legacy_wrapper_function,
     generate_wrapper_function,
+    is_fastcall_supported,
     legacy_wrapper_function_header,
     wrapper_function_header,
 )
@@ -1726,16 +1727,6 @@ def visit(item: T) -> None:
     return result
 
 
-def is_fastcall_supported(fn: FuncIR, capi_version: tuple[int, int]) -> bool:
-    if fn.class_name is not None:
-        if fn.name == "__call__":
-            # We can use vectorcalls (PEP 590) when supported
-            return True
-        # TODO: Support fastcall for __init__ and __new__.
-        return fn.name != "__init__" and fn.name != "__new__"
-    return True
-
-
 def collect_literals(fn: FuncIR, literals: Literals) -> None:
     """Store all Python literal object refs in fn.
 

mypyc/codegen/emitwrapper.py

@@ -37,6 +37,17 @@
 )
 from mypyc.namegen import NameGenerator
 
+
+def is_fastcall_supported(fn: FuncIR, capi_version: tuple[int, int]) -> bool:
+    if fn.class_name is not None:
+        if fn.name == "__call__":
+            # We can use vectorcalls (PEP 590) when supported
+            return True
+        # TODO: Support fastcall for __init__ and __new__.
+        return fn.name != "__init__" and fn.name != "__new__"
+    return True
+
+
 # Generic vectorcall wrapper functions (Python 3.7+)
 #
 # A wrapper function has a signature like this:

mypyc/irbuild/expression.py

@@ -507,7 +507,13 @@ def translate_method_call(builder: IRBuilder, expr: CallExpr, callee: MemberExpr
         and isinstance(callee.expr.node, TypeInfo)
         and callee.expr.node in builder.mapper.type_to_ir
         and builder.mapper.type_to_ir[callee.expr.node].has_method(callee.name)
-        and all(kind in (ARG_POS, ARG_NAMED) for kind in expr.arg_kinds)
+        and (
+            all(kind in (ARG_POS, ARG_NAMED) for kind in expr.arg_kinds)
+            # A lookup of __init__ would find the slot wrapper, and the __init__ slot of
+            # a native class usually does nothing (__init__ is called when the object is
+            # created instead).
+            or is_native_init(builder.mapper.type_to_ir[callee.expr.node], callee.name)
+        )
     ):
         # Call a method via the *class*
         assert isinstance(callee.expr.node, TypeInfo), callee.expr.node
@@ -553,6 +559,15 @@ def translate_method_call(builder: IRBuilder, expr: CallExpr, callee: MemberExpr
         )
 
 
+def is_native_init(ir: ClassIR, name: str) -> bool:
+    """Is this the __init__ method of the class, and does an extension class define it?"""
+    if name == "__init__":
+        for base in ir.mro:
+            if name in base.method_decls:
+                return base.is_ext_class
+    return False
+
+
 def call_classmethod(builder: IRBuilder, ir: ClassIR, expr: CallExpr, callee: MemberExpr) -> Value:
     decl = ir.method_decl(callee.name)
     args = []

mypyc/irbuild/ll_builder.py

@@ -149,6 +149,7 @@
 from mypyc.primitives.generic_ops import (
     generic_len_op,
     generic_ssize_t_len_op,
+    method_wrapper_op,
     py_call_op,
     py_call_with_kwargs_op,
     py_call_with_posargs_op,
@@ -1365,12 +1366,36 @@ def call(
         arguments used to track the presence of values for certain arguments. By
         default, the values of the bitmap arguments are inferred from args.
         """
+        if decl.class_name is not None and not self.can_pass_args_natively(
+            arg_kinds, arg_names, decl.sig
+        ):
+            return self.call_method_wrapper(decl, args, arg_kinds, arg_names, line)
         # Normalize args to positionals.
         args = self.native_args_to_positional(
             args, arg_kinds, arg_names, decl.sig, line, bitmap_args=bitmap_args
         )
         return self.add(Call(decl, args, line))
 
+    def call_method_wrapper(
+        self,
+        decl: FuncDecl,
+        args: Sequence[Value],
+        arg_kinds: list[ArgKind],
+        arg_names: Sequence[str | None],
+        line: int,
+    ) -> Value:
+        """Call a native method through its wrapper function, which parses the arguments.
+
+        This is slower than a native call, but like a native call it calls the given
+        method without looking it up. A lookup could find an override, or the __init__
+        slot wrapper of a native class, which usually does nothing (__init__ is called
+        when the object is created instead).
+        """
+        typ = self.load_native_type_object(f"{decl.module_name}.{decl.class_name}")
+        wrapper = self.call_c(method_wrapper_op, [typ, self.load_str(decl.name)], line)
+        result = self.py_call(wrapper, list(args), line, arg_kinds, arg_names)
+        return self.coerce(result, decl.sig.ret_type, line)
+
     def native_args_to_positional(
         self,
         args: Sequence[Value],
@@ -1390,22 +1415,9 @@ def native_args_to_positional(
         and coerce arguments to the appropriate type.
         """
 
-        sig_args = sig.args
+        sig_args = sig.real_args()
         n = sig.num_bitmap_args
-        if n:
-            sig_args = sig_args[:-n]
-
-        sig_arg_kinds = [arg.kind for arg in sig_args]
-        sig_arg_names = [arg.name for arg in sig_args]
-
-        concrete_kinds = [concrete_arg_kind(arg_kind) for arg_kind in arg_kinds]
-        formal_to_actual = map_actuals_to_formals(
-            concrete_kinds,
-            arg_names,
-            sig_arg_kinds,
-            sig_arg_names,
-            lambda n: AnyType(TypeOfAny.special_form),
-        )
+        formal_to_actual = self.map_native_call_args(arg_kinds, arg_names, sig)
 
         # First scan for */** and construct those
         has_star = has_star2 = False
@@ -1465,6 +1477,41 @@ def native_args_to_positional(
 
         return output_args
 
+    def map_native_call_args(
+        self, arg_kinds: list[ArgKind], arg_names: Sequence[str | None], sig: FuncSignature
+    ) -> list[list[int]]:
+        """Map the arguments of a call to the arguments of a native function.
+
+        Return the indexes of the call arguments that each argument of the function
+        (excluding bitmap arguments) may get its value from.
+        """
+        sig_args = sig.real_args()
+        return map_actuals_to_formals(
+            [concrete_arg_kind(arg_kind) for arg_kind in arg_kinds],
+            arg_names,
+            [arg.kind for arg in sig_args],
+            [arg.name for arg in sig_args],
+            lambda n: AnyType(TypeOfAny.special_form),
+        )
+
+    def can_pass_args_natively(
+        self, arg_kinds: list[ArgKind], arg_names: Sequence[str | None], sig: FuncSignature
+    ) -> bool:
+        """Can native_args_to_positional() prepare the arguments of a call?
+
+        It can't unpack a *args or **kwargs argument, since we don't know at compile
+        time which arguments of the function the items are for. It can only pass one
+        on as a whole to the *args or **kwargs argument of the function.
+        """
+        if not any(kind.is_star() for kind in arg_kinds):
+            return True
+        formal_to_actual = self.map_native_call_args(arg_kinds, arg_names, sig)
+        for formal, actuals in zip(sig.real_args(), formal_to_actual):
+            for actual in actuals:
+                if arg_kinds[actual].is_star() and arg_kinds[actual] != formal.kind:
+                    return False
+        return True
+
     def gen_method_call(
         self,
         base: Value,

mypyc/lib-rt/CPy.h

@@ -706,6 +706,7 @@ PyObject *CPyNumber_Power(PyObject *base, PyObject *index);
 PyObject *CPyNumber_InPlacePower(PyObject *base, PyObject *index);
 // An omitted slice bound is represented by CPY_INT_TAG.
 PyObject *CPyObject_GetSlice(PyObject *obj, CPyTagged start, CPyTagged end);
+PyObject *CPy_GetMethodWrapper(PyObject *type, PyObject *name);
 
 
 // List operations

mypyc/lib-rt/generic_ops.c

@@ -84,3 +84,30 @@ PyObject *CPy_SetupObject(PyObject *type) {
     PyErr_SetString(PyExc_RuntimeError, "Internal mypyc error: Unable to find object setup function");
     return NULL;
 }
+
+// Return a callable for the wrapper function of a method that a native class defines.
+// It takes the same arguments as the method when looked up on the class (including self
+// or cls). Unlike an attribute lookup, this doesn't find an override in a subclass or a
+// slot wrapper. The __init__ slot of a native class usually does nothing.
+PyObject *CPy_GetMethodWrapper(PyObject *type, PyObject *name) {
+    PyTypeObject *tp = (PyTypeObject *)type;
+    PyMethodDef *def = tp->tp_methods;
+    const char *name_str = PyUnicode_AsUTF8(name);
+    if (name_str == NULL) {
+        return NULL;
+    }
+    for (; def != NULL && def->ml_name != NULL; def++) {
+        if (strcmp(name_str, def->ml_name) == 0) {
+            if (def->ml_flags & METH_STATIC) {
+                return PyCFunction_NewEx(def, NULL, NULL);
+            }
+            if (def->ml_flags & METH_CLASS) {
+                return PyDescr_NewClassMethod(tp, def);
+            }
+            return PyDescr_NewMethod(tp, def);
+        }
+    }
+
+    PyErr_SetString(PyExc_RuntimeError, "Internal mypyc error: Unable to find method wrapper");
+    return NULL;
+}

mypyc/primitives/generic_ops.py

@@ -432,3 +432,12 @@
     c_function_name="CPy_SetupObject",
     error_kind=ERR_MAGIC,
 )
+
+# Get a callable for the wrapper function of a method that a native class defines
+# (arguments: type object, method name).
+method_wrapper_op = custom_op(
+    arg_types=[object_rprimitive, object_rprimitive],
+    return_type=object_rprimitive,
+    c_function_name="CPy_GetMethodWrapper",
+    error_kind=ERR_MAGIC,
+)

mypyc/test-data/irbuild-classes.test

@@ -682,6 +682,79 @@ def D.g(cls, x):
 L0:
     return x
 
+[case testNativeCallWithUnpackedArgs]
+from typing import Any
+
+class A:
+    def f(self, x: int, y: int = 0) -> int:
+        return x + y
+
+    def g(self, *args: Any, **kwargs: Any) -> int:
+        return len(args)
+
+class B(A):
+    def call_f(self, *args: int) -> int:
+        # The items are for ordinary arguments, so the wrapper function has to unpack them
+        return super().f(*args)
+
+    def call_g(self, *args: Any, **kwargs: Any) -> int:
+        return super().g(*args, **kwargs)
+[out]
+def A.f(self, x, y):
+    self :: __main__.A
+    x, y, r0 :: int
+L0:
+    if is_error(y) goto L1 else goto L2
+L1:
+    y = 0
+L2:
+    r0 = CPyTagged_Add(x, y)
+    return r0
+def A.g(self, args, kwargs):
+    self :: __main__.A
+    args :: tuple
+    kwargs :: dict
+    r0 :: native_int
+    r1 :: short_int
+L0:
+    r0 = var_object_size args
+    r1 = r0 << 1
+    return r1
+def B.call_f(self, args):
+    self :: __main__.B
+    args :: tuple
+    r0 :: object
+    r1 :: str
+    r2 :: object
+    r3 :: list
+    r4 :: ptr
+    r5 :: object
+    r6 :: tuple
+    r7 :: object
+    r8 :: int
+L0:
+    r0 = __main__.A :: type
+    r1 = 'f'
+    r2 = CPy_GetMethodWrapper(r0, r1)
+    r3 = PyList_New(1)
+    r4 = list_items r3
+    buf_init_item r4, 0, self
+    keep_alive r3
+    r5 = CPyList_Extend(r3, args)
+    r6 = PyList_AsTuple(r3)
+    r7 = PyObject_CallObject(r2, r6)
+    r8 = unbox(int, r7)
+    return r8
+def B.call_g(self, args, kwargs):
+    self :: __main__.B
+    args :: tuple
+    kwargs, r0 :: dict
+    r1 :: int
+L0:
+    r0 = PyDict_Copy(kwargs)
+    r1 = A.g(self, args, r0)
+    return r1
+
 [case testCannotAssignToClsArgument]
 from typing import Any, cast