feat: indie status page MVP -- FastAPI + SQLite

- 8 DB models (services, incidents, monitors, subscribers, etc.)
- Full CRUD API for services, incidents, monitors
- Public status page with live data
- Incident detail page with timeline
- API key authentication
- Uptime monitoring scheduler
- 13 tests passing
- TECHNICAL_DESIGN.md with full spec
This commit is contained in:
IndieStatusBot 2026-04-25 05:00:00 +00:00
commit 902133edd3
4655 changed files with 1342691 additions and 0 deletions

View file

@ -0,0 +1,123 @@
"""IRBuilder AST transform helpers shared between expressions and statements.
Shared code that is tightly coupled to mypy ASTs can be put here instead of
making mypyc.irbuild.builder larger.
"""
from __future__ import annotations
from mypy.nodes import (
LDEF,
BytesExpr,
ComparisonExpr,
Expression,
FloatExpr,
IntExpr,
MemberExpr,
NameExpr,
OpExpr,
StrExpr,
UnaryExpr,
Var,
)
from mypyc.ir.ops import BasicBlock
from mypyc.ir.rtypes import is_fixed_width_rtype, is_tagged
from mypyc.irbuild.builder import IRBuilder
from mypyc.irbuild.constant_fold import constant_fold_expr
def process_conditional(
self: IRBuilder, e: Expression, true: BasicBlock, false: BasicBlock
) -> None:
if isinstance(e, OpExpr) and e.op in ["and", "or"]:
if e.op == "and":
# Short circuit 'and' in a conditional context.
new = BasicBlock()
process_conditional(self, e.left, new, false)
self.activate_block(new)
process_conditional(self, e.right, true, false)
else:
# Short circuit 'or' in a conditional context.
new = BasicBlock()
process_conditional(self, e.left, true, new)
self.activate_block(new)
process_conditional(self, e.right, true, false)
elif isinstance(e, UnaryExpr) and e.op == "not":
process_conditional(self, e.expr, false, true)
else:
res = maybe_process_conditional_comparison(self, e, true, false)
if res:
return
# Catch-all for arbitrary expressions.
reg = self.accept(e)
self.add_bool_branch(reg, true, false)
def maybe_process_conditional_comparison(
self: IRBuilder, e: Expression, true: BasicBlock, false: BasicBlock
) -> bool:
"""Transform simple tagged integer comparisons in a conditional context.
Return True if the operation is supported (and was transformed). Otherwise,
do nothing and return False.
Args:
self: IR form Builder
e: Arbitrary expression
true: Branch target if comparison is true
false: Branch target if comparison is false
"""
if not isinstance(e, ComparisonExpr) or len(e.operands) != 2:
return False
ltype = self.node_type(e.operands[0])
rtype = self.node_type(e.operands[1])
if not (
(is_tagged(ltype) or is_fixed_width_rtype(ltype))
and (is_tagged(rtype) or is_fixed_width_rtype(rtype))
):
return False
op = e.operators[0]
if op not in ("==", "!=", "<", "<=", ">", ">="):
return False
left_expr = e.operands[0]
right_expr = e.operands[1]
borrow_left = is_borrow_friendly_expr(self, right_expr)
left = self.accept(left_expr, can_borrow=borrow_left)
right = self.accept(right_expr, can_borrow=True)
if is_fixed_width_rtype(ltype) or is_fixed_width_rtype(rtype):
if not is_fixed_width_rtype(ltype):
left = self.coerce(left, rtype, e.line)
elif not is_fixed_width_rtype(rtype):
right = self.coerce(right, ltype, e.line)
reg = self.binary_op(left, right, op, e.line)
self.builder.flush_keep_alives(e.line)
self.add_bool_branch(reg, true, false)
else:
# "left op right" for two tagged integers
reg = self.builder.binary_op(left, right, op, e.line)
self.flush_keep_alives(e.line)
self.add_bool_branch(reg, true, false)
return True
def is_borrow_friendly_expr(self: IRBuilder, expr: Expression) -> bool:
"""Can the result of the expression borrowed temporarily?
Borrowing means keeping a reference without incrementing the reference count.
"""
if isinstance(expr, (IntExpr, FloatExpr, StrExpr, BytesExpr)):
# Literals are immortal and can always be borrowed
return True
if (
isinstance(expr, (UnaryExpr, OpExpr, NameExpr, MemberExpr))
and constant_fold_expr(self, expr) is not None
):
# Literal expressions are similar to literals
return True
if isinstance(expr, NameExpr):
if isinstance(expr.node, Var) and expr.kind == LDEF:
# Local variable reference can be borrowed
return True
if isinstance(expr, MemberExpr) and self.is_native_attr_ref(expr):
return True
return False

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,244 @@
"""Generate a class that represents a nested function.
The class defines __call__ for calling the function and allows access to
non-local variables defined in outer scopes.
"""
from __future__ import annotations
from mypyc.common import CPYFUNCTION_NAME, ENV_ATTR_NAME, PROPSET_PREFIX, SELF_NAME
from mypyc.ir.class_ir import ClassIR
from mypyc.ir.func_ir import FuncDecl, FuncIR, FuncSignature, RuntimeArg
from mypyc.ir.ops import BasicBlock, Call, GetAttr, Integer, Register, Return, SetAttr, Value
from mypyc.ir.rtypes import RInstance, c_pointer_rprimitive, int_rprimitive, object_rprimitive
from mypyc.irbuild.builder import IRBuilder
from mypyc.irbuild.context import FuncInfo, ImplicitClass
from mypyc.primitives.misc_ops import (
cpyfunction_get_annotations,
cpyfunction_get_code,
cpyfunction_get_defaults,
cpyfunction_get_kwdefaults,
cpyfunction_get_name,
cpyfunction_set_annotations,
cpyfunction_set_name,
method_new_op,
)
def setup_callable_class(builder: IRBuilder) -> None:
"""Generate an (incomplete) callable class representing a function.
This can be a nested function or a function within a non-extension
class. Also set up the 'self' variable for that class.
This takes the most recently visited function and returns a
ClassIR to represent that function. Each callable class contains
an environment attribute which points to another ClassIR
representing the environment class where some of its variables can
be accessed.
Note that some methods, such as '__call__', are not yet
created here. Use additional functions, such as
add_call_to_callable_class(), to add them.
Return a newly constructed ClassIR representing the callable
class for the nested function.
"""
# Check to see that the name has not already been taken. If so,
# rename the class. We allow multiple uses of the same function
# name because this is valid in if-else blocks. Example:
#
# if True:
# def foo(): ----> foo_obj()
# return True
# else:
# def foo(): ----> foo_obj_0()
# return False
name = base_name = f"{builder.fn_info.namespaced_name()}_obj"
count = 0
while name in builder.callable_class_names:
name = base_name + "_" + str(count)
count += 1
builder.callable_class_names.add(name)
# Define the actual callable class ClassIR, and set its
# environment to point at the previously defined environment
# class.
callable_class_ir = ClassIR(name, builder.module_name, is_generated=True, is_final_class=True)
callable_class_ir.reuse_freed_instance = True
# The functools @wraps decorator attempts to call setattr on
# nested functions, so we create a dict for these nested
# functions.
# https://github.com/python/cpython/blob/3.7/Lib/functools.py#L58
if builder.fn_info.is_nested:
callable_class_ir.has_dict = True
# If the enclosing class doesn't contain nested (which will happen if
# this is a toplevel lambda), don't set up an environment.
if builder.fn_infos[-2].contains_nested:
callable_class_ir.attributes[ENV_ATTR_NAME] = RInstance(builder.fn_infos[-2].env_class)
callable_class_ir.mro = [callable_class_ir]
builder.fn_info.callable_class = ImplicitClass(callable_class_ir)
builder.classes.append(callable_class_ir)
# Add a 'self' variable to the environment of the callable class,
# and store that variable in a register to be accessed later.
self_target = builder.add_self_to_env(callable_class_ir)
builder.fn_info.callable_class.self_reg = builder.read(self_target, builder.fn_info.fitem.line)
if not builder.fn_info.in_non_ext and builder.fn_info.is_coroutine:
add_coroutine_properties(builder, callable_class_ir, builder.fn_info.name)
def add_coroutine_properties(
builder: IRBuilder, callable_class_ir: ClassIR, coroutine_name: str
) -> None:
"""Adds properties to the class to make it look like a regular python function.
Needed to make introspection functions like inspect.iscoroutinefunction work.
"""
callable_class_ir.coroutine_name = coroutine_name
callable_class_ir.attributes[CPYFUNCTION_NAME] = object_rprimitive
properties = {
"__name__": cpyfunction_get_name,
"__code__": cpyfunction_get_code,
"__annotations__": cpyfunction_get_annotations,
"__defaults__": cpyfunction_get_defaults,
"__kwdefaults__": cpyfunction_get_kwdefaults,
}
writable_props = {
"__name__": cpyfunction_set_name,
"__annotations__": cpyfunction_set_annotations,
}
line = builder.fn_info.fitem.line
def get_func_wrapper() -> Value:
return builder.add(GetAttr(builder.self(), CPYFUNCTION_NAME, line))
for name, primitive in properties.items():
with builder.enter_method(callable_class_ir, name, object_rprimitive, internal=True):
func = get_func_wrapper()
val = builder.primitive_op(primitive, [func, Integer(0, c_pointer_rprimitive)], line)
builder.add(Return(val))
for name, primitive in writable_props.items():
with builder.enter_method(
callable_class_ir, f"{PROPSET_PREFIX}{name}", int_rprimitive, internal=True
):
value = builder.add_argument("value", object_rprimitive)
func = get_func_wrapper()
rv = builder.primitive_op(
primitive, [func, value, Integer(0, c_pointer_rprimitive)], line
)
builder.add(Return(rv))
for name in properties:
getter = callable_class_ir.get_method(name)
assert getter
setter = callable_class_ir.get_method(f"{PROPSET_PREFIX}{name}")
callable_class_ir.properties[name] = (getter, setter)
def add_call_to_callable_class(
builder: IRBuilder,
args: list[Register],
blocks: list[BasicBlock],
sig: FuncSignature,
fn_info: FuncInfo,
) -> FuncIR:
"""Generate a '__call__' method for a callable class representing a nested function.
This takes the blocks and signature associated with a function
definition and uses those to build the '__call__' method of a
given callable class, used to represent that function.
"""
# Since we create a method, we also add a 'self' parameter.
nargs = len(sig.args) - sig.num_bitmap_args
sig = FuncSignature(
(RuntimeArg(SELF_NAME, object_rprimitive),) + sig.args[:nargs], sig.ret_type
)
call_fn_decl = FuncDecl("__call__", fn_info.callable_class.ir.name, builder.module_name, sig)
call_fn_ir = FuncIR(
call_fn_decl, args, blocks, fn_info.fitem.line, traceback_name=fn_info.fitem.name
)
fn_info.callable_class.ir.methods["__call__"] = call_fn_ir
fn_info.callable_class.ir.method_decls["__call__"] = call_fn_decl
return call_fn_ir
def add_get_to_callable_class(builder: IRBuilder, fn_info: FuncInfo) -> None:
"""Generate the '__get__' method for a callable class."""
line = fn_info.fitem.line
with builder.enter_method(
fn_info.callable_class.ir,
"__get__",
object_rprimitive,
fn_info,
self_type=object_rprimitive,
):
instance = builder.add_argument("instance", object_rprimitive)
builder.add_argument("owner", object_rprimitive)
# If accessed through the class, just return the callable
# object. If accessed through an object, create a new bound
# instance method object.
instance_block, class_block = BasicBlock(), BasicBlock()
comparison = builder.translate_is_op(
builder.read(instance), builder.none_object(line), "is", line
)
builder.add_bool_branch(comparison, class_block, instance_block)
builder.activate_block(class_block)
builder.add(Return(builder.self()))
builder.activate_block(instance_block)
builder.add(
Return(builder.call_c(method_new_op, [builder.self(), builder.read(instance)], line))
)
def instantiate_callable_class(builder: IRBuilder, fn_info: FuncInfo) -> Value:
"""Create an instance of a callable class for a function.
Calls to the function will actually call this instance.
Note that fn_info refers to the function being assigned, whereas
builder.fn_info refers to the function encapsulating the function
being turned into a callable class.
"""
fitem = fn_info.fitem
func_reg = builder.add(Call(fn_info.callable_class.ir.ctor, [], fitem.line))
# Set the environment attribute of the callable class to point at
# the environment class defined in the callable class' immediate
# outer scope. Note that there are three possible environment
# class registers we may use. This depends on what the encapsulating
# (parent) function is:
#
# - A nested function: the callable class is instantiated
# from the current callable class' '__call__' function, and hence
# the callable class' environment register is used.
# - A generator function: the callable class is instantiated
# from the '__next__' method of the generator class, and hence the
# environment of the generator class is used.
# - Regular function or comprehension scope: we use the environment
# of the original function. Comprehension scopes are inlined (no
# callable class), so they fall into this case despite is_nested.
curr_env_reg = None
if builder.fn_info.is_generator:
curr_env_reg = builder.fn_info.generator_class.curr_env_reg
elif builder.fn_info.is_nested and not builder.fn_info.is_comprehension_scope:
curr_env_reg = builder.fn_info.callable_class.curr_env_reg
elif builder.fn_info.contains_nested:
curr_env_reg = builder.fn_info.curr_env_reg
if curr_env_reg:
builder.add(SetAttr(func_reg, ENV_ATTR_NAME, curr_env_reg, fitem.line))
# Initialize function wrapper for callable classes. As opposed to regular functions,
# each instance of a callable class needs its own wrapper because they might be instantiated
# inside other functions.
if not fn_info.in_non_ext and fn_info.is_coroutine:
builder.add_coroutine_setup_call(fn_info.callable_class.ir.name, func_reg)
return func_reg

View file

@ -0,0 +1,967 @@
"""Transform class definitions from the mypy AST form to IR."""
from __future__ import annotations
from abc import abstractmethod
from collections.abc import Callable
from typing import Final
from mypy.nodes import (
EXCLUDED_ENUM_ATTRIBUTES,
TYPE_VAR_TUPLE_KIND,
AssignmentStmt,
CallExpr,
ClassDef,
Decorator,
EllipsisExpr,
ExpressionStmt,
FuncDef,
Lvalue,
MemberExpr,
NameExpr,
OverloadedFuncDef,
PassStmt,
RefExpr,
StrExpr,
TempNode,
TypeInfo,
TypeParam,
is_class_var,
)
from mypy.types import Instance, UnboundType, get_proper_type
from mypyc.common import PROPSET_PREFIX
from mypyc.ir.class_ir import ClassIR, NonExtClassInfo
from mypyc.ir.func_ir import FuncDecl, FuncSignature
from mypyc.ir.ops import (
NAMESPACE_TYPE,
BasicBlock,
Branch,
Call,
InitStatic,
LoadAddress,
LoadErrorValue,
LoadStatic,
MethodCall,
Register,
Return,
SetAttr,
TupleSet,
Value,
)
from mypyc.ir.rtypes import (
RType,
bool_rprimitive,
dict_rprimitive,
is_none_rprimitive,
is_object_rprimitive,
is_optional_type,
object_rprimitive,
)
from mypyc.irbuild.builder import IRBuilder, create_type_params
from mypyc.irbuild.function import (
gen_property_getter_ir,
gen_property_setter_ir,
handle_ext_method,
handle_non_ext_method,
load_type,
)
from mypyc.irbuild.prepare import GENERATOR_HELPER_NAME
from mypyc.irbuild.util import dataclass_type, get_func_def, is_constant, is_dataclass_decorator
from mypyc.primitives.dict_ops import dict_new_op, exact_dict_set_item_op
from mypyc.primitives.generic_ops import (
iter_op,
next_op,
py_get_item_op,
py_hasattr_op,
py_setattr_op,
)
from mypyc.primitives.misc_ops import (
dataclass_sleight_of_hand,
import_op,
not_implemented_op,
py_calc_meta_op,
py_init_subclass_op,
pytype_from_template_op,
type_object_op,
)
from mypyc.subtype import is_subtype
def transform_class_def(builder: IRBuilder, cdef: ClassDef) -> None:
"""Create IR for a class definition.
This can generate both extension (native) and non-extension
classes. These are generated in very different ways. In the
latter case we construct a Python type object at runtime by doing
the equivalent of "type(name, bases, dict)" in IR. Extension
classes are defined via C structs that are generated later in
mypyc.codegen.emitclass.
This is the main entry point to this module.
"""
if cdef.info not in builder.mapper.type_to_ir:
builder.error("Nested class definitions not supported", cdef.line)
return
ir = builder.mapper.type_to_ir[cdef.info]
# We do this check here because the base field of parent
# classes aren't necessarily populated yet at
# prepare_class_def time.
if any(ir.base_mro[i].base != ir.base_mro[i + 1] for i in range(len(ir.base_mro) - 1)):
builder.error("Multiple inheritance is not supported (except for traits)", cdef.line)
if ir.allow_interpreted_subclasses:
for parent in ir.mro:
if not parent.allow_interpreted_subclasses:
builder.error(
'Base class "{}" does not allow interpreted subclasses'.format(
parent.fullname
),
cdef.line,
)
# Currently, we only create non-extension classes for classes that are
# decorated or inherit from Enum. Classes decorated with @trait do not
# apply here, and are handled in a different way.
if ir.is_ext_class:
cls_type = dataclass_type(cdef)
if cls_type is None:
cls_builder: ClassBuilder = ExtClassBuilder(builder, cdef)
elif cls_type in ["dataclasses", "attr-auto"]:
cls_builder = DataClassBuilder(builder, cdef)
elif cls_type == "attr":
cls_builder = AttrsClassBuilder(builder, cdef)
else:
raise ValueError(cls_type)
else:
cls_builder = NonExtClassBuilder(builder, cdef)
# Set up class body context so that intra-class ClassVar references
# (e.g. C = A | B where A is defined earlier in the same class) can be
# resolved from the class being built instead of module globals.
builder.class_body_classvars = {}
builder.class_body_obj = cls_builder.class_body_obj()
builder.class_body_ir = ir
for stmt in cdef.defs.body:
if (
isinstance(stmt, (FuncDef, Decorator, OverloadedFuncDef))
and stmt.name == GENERATOR_HELPER_NAME
):
builder.error(
f'Method name "{stmt.name}" is reserved for mypyc internal use', stmt.line
)
if isinstance(stmt, OverloadedFuncDef) and stmt.is_property:
if isinstance(cls_builder, NonExtClassBuilder):
# properties with both getters and setters in non_extension
# classes not supported
builder.error("Property setters not supported in non-extension classes", stmt.line)
for item in stmt.items:
with builder.catch_errors(stmt.line):
cls_builder.add_method(get_func_def(item))
elif isinstance(stmt, (FuncDef, Decorator, OverloadedFuncDef)):
# Ignore plugin generated methods (since they have no
# bodies to compile and will need to have the bodies
# provided by some other mechanism.)
if cdef.info.names[stmt.name].plugin_generated:
continue
with builder.catch_errors(stmt.line):
cls_builder.add_method(get_func_def(stmt))
elif isinstance(stmt, PassStmt) or (
isinstance(stmt, ExpressionStmt) and isinstance(stmt.expr, EllipsisExpr)
):
continue
elif isinstance(stmt, AssignmentStmt):
if len(stmt.lvalues) != 1:
builder.error("Multiple assignment in class bodies not supported", stmt.line)
continue
lvalue = stmt.lvalues[0]
if not isinstance(lvalue, NameExpr):
builder.error(
"Only assignment to variables is supported in class bodies", stmt.line
)
continue
# We want to collect class variables in a dictionary for both real
# non-extension classes and fake dataclass ones.
cls_builder.add_attr(lvalue, stmt)
# Track this ClassVar so subsequent class body statements can reference it.
if is_class_var(lvalue) or stmt.is_final_def:
builder.class_body_classvars[lvalue.name] = None
elif isinstance(stmt, ExpressionStmt) and isinstance(stmt.expr, StrExpr):
# Docstring. Ignore
pass
else:
builder.error("Unsupported statement in class body", stmt.line)
# Clear class body context (nested classes are rejected above, so no need to save/restore).
builder.class_body_classvars = {}
builder.class_body_obj = None
builder.class_body_ir = None
# Generate implicit property setters/getters
for name, decl in ir.method_decls.items():
if decl.implicit and decl.is_prop_getter:
getter_ir = gen_property_getter_ir(builder, decl, cdef, ir.is_trait)
builder.functions.append(getter_ir)
ir.methods[getter_ir.decl.name] = getter_ir
setter_ir = None
setter_name = PROPSET_PREFIX + name
if setter_name in ir.method_decls:
setter_ir = gen_property_setter_ir(
builder, ir.method_decls[setter_name], cdef, ir.is_trait
)
builder.functions.append(setter_ir)
ir.methods[setter_name] = setter_ir
ir.properties[name] = (getter_ir, setter_ir)
# TODO: Generate glue method if needed?
# TODO: Do we need interpreted glue methods? Maybe not?
cls_builder.finalize(ir)
class ClassBuilder:
"""Create IR for a class definition.
This is an abstract base class.
"""
def __init__(self, builder: IRBuilder, cdef: ClassDef) -> None:
self.builder = builder
self.cdef = cdef
self.attrs_to_cache: list[tuple[Lvalue, RType]] = []
@abstractmethod
def add_method(self, fdef: FuncDef) -> None:
"""Add a method to the class IR"""
@abstractmethod
def add_attr(self, lvalue: NameExpr, stmt: AssignmentStmt) -> None:
"""Add an attribute to the class IR"""
@abstractmethod
def finalize(self, ir: ClassIR) -> None:
"""Perform any final operations to complete the class IR"""
def class_body_obj(self) -> Value | None:
"""Return the object to use for loading class attributes during class body init.
For extension classes, this is the type object. For non-extension classes,
this is the class dict. Returns None if not applicable.
"""
return None
class NonExtClassBuilder(ClassBuilder):
def __init__(self, builder: IRBuilder, cdef: ClassDef) -> None:
super().__init__(builder, cdef)
self.non_ext = self.create_non_ext_info()
def class_body_obj(self) -> Value | None:
return self.non_ext.dict
def create_non_ext_info(self) -> NonExtClassInfo:
non_ext_bases = populate_non_ext_bases(self.builder, self.cdef)
non_ext_metaclass = find_non_ext_metaclass(self.builder, self.cdef, non_ext_bases)
non_ext_dict = setup_non_ext_dict(
self.builder, self.cdef, non_ext_metaclass, non_ext_bases
)
# We populate __annotations__ for non-extension classes
# because dataclasses uses it to determine which attributes to compute on.
# TODO: Maybe generate more precise types for annotations
non_ext_anns = self.builder.call_c(dict_new_op, [], self.cdef.line)
return NonExtClassInfo(non_ext_dict, non_ext_bases, non_ext_anns, non_ext_metaclass)
def add_method(self, fdef: FuncDef) -> None:
handle_non_ext_method(self.builder, self.non_ext, self.cdef, fdef)
def add_attr(self, lvalue: NameExpr, stmt: AssignmentStmt) -> None:
add_non_ext_class_attr_ann(self.builder, self.non_ext, lvalue, stmt)
add_non_ext_class_attr(
self.builder, self.non_ext, lvalue, stmt, self.cdef, self.attrs_to_cache
)
def finalize(self, ir: ClassIR) -> None:
# Dynamically create the class via the type constructor
non_ext_class = load_non_ext_class(self.builder, ir, self.non_ext, self.cdef.line)
non_ext_class = load_decorated_class(self.builder, self.cdef, non_ext_class)
# Try to avoid contention when using free threading.
self.builder.set_immortal_if_free_threaded(non_ext_class, self.cdef.line)
# Save the decorated class
self.builder.add(
InitStatic(non_ext_class, self.cdef.name, self.builder.module_name, NAMESPACE_TYPE)
)
# Add the non-extension class to the dict
self.builder.call_c(
exact_dict_set_item_op,
[
self.builder.load_globals_dict(),
self.builder.load_str(self.cdef.name),
non_ext_class,
],
self.cdef.line,
)
# Cache any cacheable class attributes
cache_class_attrs(self.builder, self.attrs_to_cache, self.cdef)
class ExtClassBuilder(ClassBuilder):
def __init__(self, builder: IRBuilder, cdef: ClassDef) -> None:
super().__init__(builder, cdef)
# If the class is not decorated, generate an extension class for it.
self.type_obj: Value = allocate_class(builder, cdef)
def class_body_obj(self) -> Value | None:
return self.type_obj
def skip_attr_default(self, name: str, stmt: AssignmentStmt) -> bool:
"""Controls whether to skip generating a default for an attribute."""
return False
def add_method(self, fdef: FuncDef) -> None:
handle_ext_method(self.builder, self.cdef, fdef)
def add_attr(self, lvalue: NameExpr, stmt: AssignmentStmt) -> None:
# Variable declaration with no body
if isinstance(stmt.rvalue, TempNode):
return
# Only treat marked class variables as class variables.
if not (is_class_var(lvalue) or stmt.is_final_def):
return
typ = self.builder.load_native_type_object(self.cdef.fullname)
value = self.builder.accept(stmt.rvalue)
self.builder.primitive_op(
py_setattr_op, [typ, self.builder.load_str(lvalue.name), value], stmt.line
)
if self.builder.non_function_scope() and stmt.is_final_def:
self.builder.init_final_static(lvalue, value, self.cdef.name)
def finalize(self, ir: ClassIR) -> None:
# Call __init_subclass__ after class attributes have been set
self.builder.call_c(py_init_subclass_op, [self.type_obj], self.cdef.line)
attrs_with_defaults, default_assignments = find_attr_initializers(
self.builder, self.cdef, self.skip_attr_default
)
ir.attrs_with_defaults.update(attrs_with_defaults)
generate_attr_defaults_init(self.builder, self.cdef, default_assignments)
create_ne_from_eq(self.builder, self.cdef)
class DataClassBuilder(ExtClassBuilder):
# controls whether an __annotations__ attribute should be added to the class
# __dict__. This is not desirable for attrs classes where auto_attribs is
# disabled, as attrs will reject it.
add_annotations_to_dict = True
def __init__(self, builder: IRBuilder, cdef: ClassDef) -> None:
super().__init__(builder, cdef)
self.non_ext = self.create_non_ext_info()
def create_non_ext_info(self) -> NonExtClassInfo:
"""Set up a NonExtClassInfo to track dataclass attributes.
In addition to setting up a normal extension class for dataclasses,
we also collect its class attributes like a non-extension class so
that we can hand them to the dataclass decorator.
"""
return NonExtClassInfo(
self.builder.call_c(dict_new_op, [], self.cdef.line),
self.builder.add(TupleSet([], self.cdef.line)),
self.builder.call_c(dict_new_op, [], self.cdef.line),
self.builder.add(LoadAddress(type_object_op.type, type_object_op.src, self.cdef.line)),
)
def skip_attr_default(self, name: str, stmt: AssignmentStmt) -> bool:
return stmt.type is not None
def get_type_annotation(self, stmt: AssignmentStmt) -> TypeInfo | None:
# We populate __annotations__ because dataclasses uses it to determine
# which attributes to compute on.
ann_type = get_proper_type(stmt.type)
if isinstance(ann_type, Instance):
return ann_type.type
return None
def add_attr(self, lvalue: NameExpr, stmt: AssignmentStmt) -> None:
add_non_ext_class_attr_ann(
self.builder, self.non_ext, lvalue, stmt, self.get_type_annotation
)
add_non_ext_class_attr(
self.builder, self.non_ext, lvalue, stmt, self.cdef, self.attrs_to_cache
)
super().add_attr(lvalue, stmt)
def finalize(self, ir: ClassIR) -> None:
"""Generate code to finish instantiating a dataclass.
This works by replacing all of the attributes on the class
(which will be descriptors) with whatever they would be in a
non-extension class, calling dataclass, then switching them back.
The resulting class is an extension class and instances of it do not
have a __dict__ (unless something else requires it).
All methods written explicitly in the source are compiled and
may be called through the vtable while the methods generated
by dataclasses are interpreted and may not be.
(If we just called dataclass without doing this, it would think that all
of the descriptors for our attributes are default values and generate an
incorrect constructor. We need to do the switch so that dataclass gets the
appropriate defaults.)
"""
super().finalize(ir)
assert self.type_obj
add_dunders_to_non_ext_dict(
self.builder, self.non_ext, self.cdef.line, self.add_annotations_to_dict
)
dec = self.builder.accept(
next(d for d in self.cdef.decorators if is_dataclass_decorator(d))
)
dataclass_type_val = self.builder.load_str(dataclass_type(self.cdef) or "unknown")
self.builder.call_c(
dataclass_sleight_of_hand,
[dec, self.type_obj, self.non_ext.dict, self.non_ext.anns, dataclass_type_val],
self.cdef.line,
)
class AttrsClassBuilder(DataClassBuilder):
"""Create IR for an attrs class where auto_attribs=False (the default).
When auto_attribs is enabled, attrs classes behave similarly to dataclasses
(i.e. types are stored as annotations on the class) and are thus handled
by DataClassBuilder, but when auto_attribs is disabled the types are
provided via attr.ib(type=...)
"""
add_annotations_to_dict = False
def skip_attr_default(self, name: str, stmt: AssignmentStmt) -> bool:
return True
def get_type_annotation(self, stmt: AssignmentStmt) -> TypeInfo | None:
if isinstance(stmt.rvalue, CallExpr):
# find the type arg in `attr.ib(type=str)`
callee = stmt.rvalue.callee
if (
isinstance(callee, MemberExpr)
and callee.fullname in ["attr.ib", "attr.attr"]
and "type" in stmt.rvalue.arg_names
):
index = stmt.rvalue.arg_names.index("type")
type_name = stmt.rvalue.args[index]
if isinstance(type_name, NameExpr) and isinstance(type_name.node, TypeInfo):
lvalue = stmt.lvalues[0]
assert isinstance(lvalue, NameExpr), lvalue
return type_name.node
return None
def allocate_class(builder: IRBuilder, cdef: ClassDef) -> Value:
# OK AND NOW THE FUN PART
base_exprs = cdef.base_type_exprs + cdef.removed_base_type_exprs
new_style_type_args = cdef.type_args
if new_style_type_args:
bases = [make_generic_base_class(builder, cdef.fullname, new_style_type_args, cdef.line)]
else:
bases = []
if base_exprs or new_style_type_args:
bases.extend([builder.accept(x) for x in base_exprs])
tp_bases = builder.new_tuple(bases, cdef.line)
else:
tp_bases = builder.add(LoadErrorValue(object_rprimitive, is_borrowed=True))
modname = builder.load_str(builder.module_name)
template = builder.add(
LoadStatic(object_rprimitive, cdef.name + "_template", builder.module_name, NAMESPACE_TYPE)
)
# Create the class
tp = builder.call_c(pytype_from_template_op, [template, tp_bases, modname], cdef.line)
# Set type object to be immortal if free threaded, as otherwise reference count contention
# can cause a big performance hit.
builder.set_immortal_if_free_threaded(tp, cdef.line)
# Immediately fix up the trait vtables, before doing anything with the class.
ir = builder.mapper.type_to_ir[cdef.info]
if not ir.is_trait and not ir.builtin_base:
builder.add(
Call(
FuncDecl(
cdef.name + "_trait_vtable_setup",
None,
builder.module_name,
FuncSignature([], bool_rprimitive),
),
[],
cdef.line,
)
)
builder.add_coroutine_setup_call(cdef.name, tp)
# Populate a '__mypyc_attrs__' field containing the list of attrs
builder.primitive_op(
py_setattr_op,
[
tp,
builder.load_str("__mypyc_attrs__"),
create_mypyc_attrs_tuple(builder, builder.mapper.type_to_ir[cdef.info], cdef.line),
],
cdef.line,
)
# Save the class
builder.add(InitStatic(tp, cdef.name, builder.module_name, NAMESPACE_TYPE))
# Add it to the dict
builder.call_c(
exact_dict_set_item_op,
[builder.load_globals_dict(), builder.load_str(cdef.name), tp],
cdef.line,
)
return tp
def make_generic_base_class(
builder: IRBuilder, fullname: str, type_args: list[TypeParam], line: int
) -> Value:
"""Construct Generic[...] base class object for a new-style generic class (Python 3.12)."""
mod = builder.call_c(import_op, [builder.load_str("_typing")], line)
tvs = create_type_params(builder, mod, type_args, line)
args = []
for tv, type_param in zip(tvs, type_args):
if type_param.kind == TYPE_VAR_TUPLE_KIND:
# Evaluate *Ts for a TypeVarTuple
it = builder.primitive_op(iter_op, [tv], line)
tv = builder.call_c(next_op, [it], line)
args.append(tv)
gent = builder.py_get_attr(mod, "Generic", line)
if len(args) == 1:
arg = args[0]
else:
arg = builder.new_tuple(args, line)
base = builder.primitive_op(py_get_item_op, [gent, arg], line)
return base
# Mypy uses these internally as base classes of TypedDict classes. These are
# lies and don't have any runtime equivalent.
MAGIC_TYPED_DICT_CLASSES: Final[tuple[str, ...]] = (
"typing._TypedDict",
"typing_extensions._TypedDict",
)
def populate_non_ext_bases(builder: IRBuilder, cdef: ClassDef) -> Value:
"""Create base class tuple of a non-extension class.
The tuple is passed to the metaclass constructor.
"""
is_named_tuple = cdef.info.is_named_tuple
ir = builder.mapper.type_to_ir[cdef.info]
bases = []
for cls in (b.type for b in cdef.info.bases):
if cls.fullname == "builtins.object":
continue
if is_named_tuple and cls.fullname in (
"typing.Sequence",
"typing.Iterable",
"typing.Collection",
"typing.Reversible",
"typing.Container",
"typing.Sized",
):
# HAX: Synthesized base classes added by mypy don't exist at runtime, so skip them.
# This could break if they were added explicitly, though...
continue
# Add the current class to the base classes list of concrete subclasses
if cls in builder.mapper.type_to_ir:
base_ir = builder.mapper.type_to_ir[cls]
if base_ir.children is not None:
base_ir.children.append(ir)
if cls.fullname in MAGIC_TYPED_DICT_CLASSES:
# HAX: Mypy internally represents TypedDict classes differently from what
# should happen at runtime. Replace with something that works.
module = "typing"
name = "_TypedDict"
base = builder.get_module_attr(module, name, cdef.line)
elif is_named_tuple and cls.fullname == "builtins.tuple":
name = "_NamedTuple"
base = builder.get_module_attr("typing", name, cdef.line)
else:
cls_module = cls.fullname.rsplit(".", 1)[0]
if cls_module == builder.current_module:
base = builder.load_global_str(cls.name, cdef.line)
else:
base = builder.load_module_attr_by_fullname(cls.fullname, cdef.line)
bases.append(base)
if cls.fullname in MAGIC_TYPED_DICT_CLASSES:
# The remaining base classes are synthesized by mypy and should be ignored.
break
return builder.new_tuple(bases, cdef.line)
def find_non_ext_metaclass(builder: IRBuilder, cdef: ClassDef, bases: Value) -> Value:
"""Find the metaclass of a class from its defs and bases."""
if cdef.metaclass:
declared_metaclass = builder.accept(cdef.metaclass)
else:
if cdef.info.typeddict_type is not None:
# In Python 3.9, the metaclass for class-based TypedDict is typing._TypedDictMeta.
# We can't easily calculate it generically, so special case it.
return builder.get_module_attr("typing", "_TypedDictMeta", cdef.line)
elif cdef.info.is_named_tuple:
# In Python 3.9, the metaclass for class-based NamedTuple is typing.NamedTupleMeta.
# We can't easily calculate it generically, so special case it.
return builder.get_module_attr("typing", "NamedTupleMeta", cdef.line)
declared_metaclass = builder.add(
LoadAddress(type_object_op.type, type_object_op.src, cdef.line)
)
return builder.call_c(py_calc_meta_op, [declared_metaclass, bases], cdef.line)
def setup_non_ext_dict(
builder: IRBuilder, cdef: ClassDef, metaclass: Value, bases: Value
) -> Value:
"""Initialize the class dictionary for a non-extension class.
This class dictionary is passed to the metaclass constructor.
"""
# Check if the metaclass defines a __prepare__ method, and if so, call it.
has_prepare = builder.primitive_op(
py_hasattr_op, [metaclass, builder.load_str("__prepare__")], cdef.line
)
non_ext_dict = Register(dict_rprimitive)
true_block, false_block, exit_block = BasicBlock(), BasicBlock(), BasicBlock()
builder.add_bool_branch(has_prepare, true_block, false_block)
builder.activate_block(true_block)
cls_name = builder.load_str(cdef.name)
prepare_meth = builder.py_get_attr(metaclass, "__prepare__", cdef.line)
prepare_dict = builder.py_call(prepare_meth, [cls_name, bases], cdef.line)
builder.assign(non_ext_dict, prepare_dict, cdef.line)
builder.goto(exit_block)
builder.activate_block(false_block)
builder.assign(non_ext_dict, builder.call_c(dict_new_op, [], cdef.line), cdef.line)
builder.goto(exit_block)
builder.activate_block(exit_block)
return non_ext_dict
def add_non_ext_class_attr_ann(
builder: IRBuilder,
non_ext: NonExtClassInfo,
lvalue: NameExpr,
stmt: AssignmentStmt,
get_type_info: Callable[[AssignmentStmt], TypeInfo | None] | None = None,
) -> None:
"""Add a class attribute to __annotations__ of a non-extension class."""
# FIXME: try to better preserve the special forms and type parameters of generics.
typ: Value | None = None
if get_type_info is not None:
type_info = get_type_info(stmt)
if type_info:
# NOTE: Using string type information is similar to using
# `from __future__ import annotations` in standard python.
# NOTE: For string types we need to use the fullname since it
# includes the module. If string type doesn't have the module,
# @dataclass will try to get the current module and fail since the
# current module is not in sys.modules.
if builder.current_module == type_info.module_name and stmt.line < type_info.line:
typ = builder.load_str(type_info.fullname)
else:
typ = load_type(builder, type_info, stmt.unanalyzed_type, stmt.line)
if typ is None:
# FIXME: if get_type_info is not provided, don't fall back to stmt.type?
ann_type = get_proper_type(stmt.type)
if (
isinstance(stmt.unanalyzed_type, UnboundType)
and stmt.unanalyzed_type.original_str_expr is not None
):
# Annotation is a forward reference, so don't attempt to load the actual
# type and load the string instead.
#
# TODO: is it possible to determine whether a non-string annotation is
# actually a forward reference due to the __annotations__ future?
typ = builder.load_str(stmt.unanalyzed_type.original_str_expr)
elif isinstance(ann_type, Instance):
typ = load_type(builder, ann_type.type, stmt.unanalyzed_type, stmt.line)
else:
typ = builder.add(LoadAddress(type_object_op.type, type_object_op.src, stmt.line))
key = builder.load_str(lvalue.name)
builder.call_c(exact_dict_set_item_op, [non_ext.anns, key, typ], stmt.line)
def add_non_ext_class_attr(
builder: IRBuilder,
non_ext: NonExtClassInfo,
lvalue: NameExpr,
stmt: AssignmentStmt,
cdef: ClassDef,
attr_to_cache: list[tuple[Lvalue, RType]],
) -> None:
"""Add a class attribute to __dict__ of a non-extension class."""
# Only add the attribute to the __dict__ if the assignment is of the form:
# x: type = value (don't add attributes of the form 'x: type' to the __dict__).
if not isinstance(stmt.rvalue, TempNode):
rvalue = builder.accept(stmt.rvalue)
builder.add_to_non_ext_dict(non_ext, lvalue.name, rvalue, stmt.line)
# We cache enum attributes to speed up enum attribute lookup since they
# are final.
if (
cdef.info.bases
# Enum class must be the last parent class.
and cdef.info.bases[-1].type.is_enum
# Skip these since Enum will remove it
and lvalue.name not in EXCLUDED_ENUM_ATTRIBUTES
):
# Enum values are always boxed, so use object_rprimitive.
attr_to_cache.append((lvalue, object_rprimitive))
def find_attr_initializers(
builder: IRBuilder, cdef: ClassDef, skip: Callable[[str, AssignmentStmt], bool] | None = None
) -> tuple[set[str], list[tuple[AssignmentStmt, str]]]:
"""Find initializers of attributes in a class body.
If provided, the skip arg should be a callable which will return whether
to skip generating a default for an attribute. It will be passed the name of
the attribute and the corresponding AssignmentStmt.
"""
cls = builder.mapper.type_to_ir[cdef.info]
if cls.builtin_base:
return set(), []
attrs_with_defaults = set()
# Pull out all assignments in classes in the mro so we can initialize them
# TODO: Support nested statements
default_assignments: list[tuple[AssignmentStmt, str]] = []
for info in reversed(cdef.info.mro):
if info not in builder.mapper.type_to_ir:
continue
for stmt in info.defn.defs.body:
if (
isinstance(stmt, AssignmentStmt)
and isinstance(stmt.lvalues[0], NameExpr)
and not is_class_var(stmt.lvalues[0])
and not isinstance(stmt.rvalue, TempNode)
):
name = stmt.lvalues[0].name
if name == "__slots__":
continue
if name == "__deletable__":
check_deletable_declaration(builder, cls, stmt.line)
continue
if skip is not None and skip(name, stmt):
continue
attr_type = cls.attr_type(name)
# If the attribute is initialized to None and type isn't optional,
# doesn't initialize it to anything (special case for "# type:" comments).
if isinstance(stmt.rvalue, RefExpr) and stmt.rvalue.fullname == "builtins.None":
if (
not is_optional_type(attr_type)
and not is_object_rprimitive(attr_type)
and not is_none_rprimitive(attr_type)
):
continue
attrs_with_defaults.add(name)
default_assignments.append((stmt, info.module_name))
return attrs_with_defaults, default_assignments
def generate_attr_defaults_init(
builder: IRBuilder, cdef: ClassDef, default_assignments: list[tuple[AssignmentStmt, str]]
) -> None:
"""Generate an initialization method for default attr values (from class vars)."""
if not default_assignments:
return
cls = builder.mapper.type_to_ir[cdef.info]
if cls.builtin_base:
return
with builder.enter_method(cls, "__mypyc_defaults_setup", bool_rprimitive):
self_var = builder.self()
for stmt, origin_module in default_assignments:
lvalue = stmt.lvalues[0]
assert isinstance(lvalue, NameExpr), lvalue
if not stmt.is_final_def and not is_constant(stmt.rvalue):
builder.warning("Unsupported default attribute value", stmt.rvalue.line)
attr_type = cls.attr_type(lvalue.name)
# When the default comes from a parent in a different module,
# set the globals lookup module so NameExpr references resolve
# against the correct module's globals dict.
builder.globals_lookup_module = (
origin_module if origin_module != builder.module_name else None
)
try:
val = builder.coerce(builder.accept(stmt.rvalue), attr_type, stmt.line)
finally:
builder.globals_lookup_module = None
init = SetAttr(self_var, lvalue.name, val, stmt.rvalue.line)
init.mark_as_initializer()
builder.add(init)
builder.add(Return(builder.true()))
def check_deletable_declaration(builder: IRBuilder, cl: ClassIR, line: int) -> None:
for attr in cl.deletable:
if attr not in cl.attributes:
if not cl.has_attr(attr):
builder.error(f'Attribute "{attr}" not defined', line)
continue
for base in cl.mro:
if attr in base.property_types:
builder.error(f'Cannot make property "{attr}" deletable', line)
break
else:
_, base = cl.attr_details(attr)
builder.error(
('Attribute "{}" not defined in "{}" ' + '(defined in "{}")').format(
attr, cl.name, base.name
),
line,
)
def create_ne_from_eq(builder: IRBuilder, cdef: ClassDef) -> None:
"""Create a "__ne__" method from a "__eq__" method (if only latter exists)."""
cls = builder.mapper.type_to_ir[cdef.info]
if cls.has_method("__eq__") and not cls.has_method("__ne__"):
gen_glue_ne_method(builder, cls, cdef.line)
def gen_glue_ne_method(builder: IRBuilder, cls: ClassIR, line: int) -> None:
"""Generate a "__ne__" method from a "__eq__" method."""
func_ir = cls.get_method("__eq__")
assert func_ir
eq_sig = func_ir.decl.sig
strict_typing = builder.options.strict_dunders_typing
with builder.enter_method(cls, "__ne__", eq_sig.ret_type):
rhs_type = eq_sig.args[0].type if strict_typing else object_rprimitive
rhs_arg = builder.add_argument("rhs", rhs_type)
eqval = builder.add(MethodCall(builder.self(), "__eq__", [rhs_arg], line))
can_return_not_implemented = is_subtype(not_implemented_op.type, eq_sig.ret_type)
return_bool = is_subtype(eq_sig.ret_type, bool_rprimitive)
if not strict_typing or can_return_not_implemented:
# If __eq__ returns NotImplemented, then __ne__ should also
not_implemented_block, regular_block = BasicBlock(), BasicBlock()
not_implemented = builder.add(
LoadAddress(not_implemented_op.type, not_implemented_op.src, line)
)
builder.add(
Branch(
builder.translate_is_op(eqval, not_implemented, "is", line),
not_implemented_block,
regular_block,
Branch.BOOL,
)
)
builder.activate_block(regular_block)
rettype = bool_rprimitive if return_bool and strict_typing else object_rprimitive
retval = builder.coerce(
builder.builder.unary_not(eqval, line, likely_bool=True), rettype, line
)
builder.add(Return(retval))
builder.activate_block(not_implemented_block)
builder.add(Return(not_implemented))
else:
rettype = bool_rprimitive if return_bool and strict_typing else object_rprimitive
retval = builder.coerce(builder.unary_op(eqval, "not", line), rettype, line)
builder.add(Return(retval))
def load_non_ext_class(
builder: IRBuilder, ir: ClassIR, non_ext: NonExtClassInfo, line: int
) -> Value:
cls_name = builder.load_str(ir.name)
add_dunders_to_non_ext_dict(builder, non_ext, line)
class_type_obj = builder.py_call(
non_ext.metaclass, [cls_name, non_ext.bases, non_ext.dict], line
)
return class_type_obj
def load_decorated_class(builder: IRBuilder, cdef: ClassDef, type_obj: Value) -> Value:
"""Apply class decorators to create a decorated (non-extension) class object.
Given a decorated ClassDef and a register containing a
non-extension representation of the ClassDef created via the type
constructor, applies the corresponding decorator functions on that
decorated ClassDef and returns a register containing the decorated
ClassDef.
"""
decorators = cdef.decorators
dec_class = type_obj
for d in reversed(decorators):
decorator = d.accept(builder.visitor)
assert isinstance(decorator, Value), decorator
dec_class = builder.py_call(decorator, [dec_class], dec_class.line)
return dec_class
def cache_class_attrs(
builder: IRBuilder, attrs_to_cache: list[tuple[Lvalue, RType]], cdef: ClassDef
) -> None:
"""Add class attributes to be cached to the global cache."""
typ = builder.load_native_type_object(cdef.info.fullname)
for lval, rtype in attrs_to_cache:
assert isinstance(lval, NameExpr), lval
rval = builder.py_get_attr(typ, lval.name, cdef.line)
builder.init_final_static(lval, rval, cdef.name, type_override=rtype)
def create_mypyc_attrs_tuple(builder: IRBuilder, ir: ClassIR, line: int) -> Value:
attrs = [name for ancestor in ir.mro for name in ancestor.attributes]
if ir.inherits_python:
attrs.append("__dict__")
items = [builder.load_str(attr) for attr in attrs]
return builder.new_tuple(items, line)
def add_dunders_to_non_ext_dict(
builder: IRBuilder, non_ext: NonExtClassInfo, line: int, add_annotations: bool = True
) -> None:
if add_annotations:
# Add __annotations__ to the class dict.
builder.add_to_non_ext_dict(non_ext, "__annotations__", non_ext.anns, line)
# We add a __doc__ attribute so if the non-extension class is decorated with the
# dataclass decorator, dataclass will not try to look for __text_signature__.
# https://github.com/python/cpython/blob/3.7/Lib/dataclasses.py#L957
filler_doc_str = "mypyc filler docstring"
builder.add_to_non_ext_dict(non_ext, "__doc__", builder.load_str(filler_doc_str), line)
builder.add_to_non_ext_dict(non_ext, "__module__", builder.load_str(builder.module_name), line)

View file

@ -0,0 +1,97 @@
"""Constant folding of IR values.
For example, 3 + 5 can be constant folded into 8.
This is mostly like mypy.constant_fold, but we can bind some additional
NameExpr and MemberExpr references here, since we have more knowledge
about which definitions can be trusted -- we constant fold only references
to other compiled modules in the same compilation unit.
"""
from __future__ import annotations
from typing import TYPE_CHECKING, Final
from mypy.constant_fold import constant_fold_binary_op, constant_fold_unary_op
from mypy.nodes import (
BytesExpr,
ComplexExpr,
Expression,
FloatExpr,
IntExpr,
MemberExpr,
NameExpr,
OpExpr,
StrExpr,
UnaryExpr,
Var,
)
from mypyc.irbuild.util import bytes_from_str
if TYPE_CHECKING:
from mypyc.irbuild.builder import IRBuilder
# All possible result types of constant folding
ConstantValue = int | float | complex | str | bytes
CONST_TYPES: Final = (int, float, complex, str, bytes)
def constant_fold_expr(builder: IRBuilder, expr: Expression) -> ConstantValue | None:
"""Return the constant value of an expression for supported operations.
Return None otherwise.
"""
if isinstance(expr, IntExpr):
return expr.value
if isinstance(expr, FloatExpr):
return expr.value
if isinstance(expr, StrExpr):
return expr.value
if isinstance(expr, BytesExpr):
return bytes_from_str(expr.value)
if isinstance(expr, ComplexExpr):
return expr.value
elif isinstance(expr, NameExpr):
node = expr.node
if isinstance(node, Var) and node.is_final:
final_value = node.final_value
if isinstance(final_value, (CONST_TYPES)):
return final_value
elif isinstance(expr, MemberExpr):
final = builder.get_final_ref(expr)
if final is not None:
fn, final_var, native = final
if final_var.is_final:
final_value = final_var.final_value
if isinstance(final_value, (CONST_TYPES)):
return final_value
elif isinstance(expr, OpExpr):
left = constant_fold_expr(builder, expr.left)
right = constant_fold_expr(builder, expr.right)
if left is not None and right is not None:
return constant_fold_binary_op_extended(expr.op, left, right)
elif isinstance(expr, UnaryExpr):
value = constant_fold_expr(builder, expr.expr)
if value is not None and not isinstance(value, bytes):
return constant_fold_unary_op(expr.op, value)
return None
def constant_fold_binary_op_extended(
op: str, left: ConstantValue, right: ConstantValue
) -> ConstantValue | None:
"""Like mypy's constant_fold_binary_op(), but includes bytes support.
mypy cannot use constant folded bytes easily so it's simpler to only support them in mypyc.
"""
if not isinstance(left, bytes) and not isinstance(right, bytes):
return constant_fold_binary_op(op, left, right)
if op == "+" and isinstance(left, bytes) and isinstance(right, bytes):
return left + right
elif op == "*" and isinstance(left, bytes) and isinstance(right, int):
return left * right
elif op == "*" and isinstance(left, int) and isinstance(right, bytes):
return left * right
return None

View file

@ -0,0 +1,206 @@
"""Helpers that store information about functions and the related classes."""
from __future__ import annotations
from mypy.nodes import FuncItem
from mypyc.ir.class_ir import ClassIR
from mypyc.ir.func_ir import INVALID_FUNC_DEF
from mypyc.ir.ops import BasicBlock, Value
from mypyc.irbuild.targets import AssignmentTarget
class FuncInfo:
"""Contains information about functions as they are generated."""
def __init__(
self,
fitem: FuncItem = INVALID_FUNC_DEF,
name: str = "",
class_name: str | None = None,
namespace: str = "",
is_nested: bool = False,
contains_nested: bool = False,
is_decorated: bool = False,
in_non_ext: bool = False,
add_nested_funcs_to_env: bool = False,
is_comprehension_scope: bool = False,
) -> None:
self.fitem = fitem
self.name = name
self.class_name = class_name
self.ns = namespace
# Callable classes implement the '__call__' method, and are used to represent functions
# that are nested inside of other functions.
self._callable_class: ImplicitClass | None = None
# Environment classes are ClassIR instances that contain attributes representing the
# variables in the environment of the function they correspond to. Environment classes are
# generated for functions that contain nested functions.
self._env_class: ClassIR | None = None
# Generator classes implement the '__next__' method, and are used to represent generators
# returned by generator functions.
self._generator_class: GeneratorClass | None = None
# Environment class registers are the local registers associated with instances of an
# environment class, used for getting and setting attributes. curr_env_reg is the register
# associated with the current environment.
self._curr_env_reg: Value | None = None
# These are flags denoting whether a given function is nested, contains a nested function,
# is decorated, or is within a non-extension class.
self.is_nested = is_nested
self.contains_nested = contains_nested
self.is_decorated = is_decorated
self.in_non_ext = in_non_ext
self.add_nested_funcs_to_env = add_nested_funcs_to_env
# Comprehension scopes are lightweight scope boundaries created when
# a comprehension body contains a lambda. The comprehension is still
# inlined (same basic blocks), but we push a new FuncInfo so the
# closure machinery can capture loop variables through env classes.
self.is_comprehension_scope = is_comprehension_scope
# TODO: add field for ret_type: RType = none_rprimitive
def namespaced_name(self) -> str:
return "_".join(x for x in [self.name, self.class_name, self.ns] if x)
@property
def is_generator(self) -> bool:
return self.fitem.is_generator or self.fitem.is_coroutine
@property
def is_coroutine(self) -> bool:
return self.fitem.is_coroutine
@property
def callable_class(self) -> ImplicitClass:
assert self._callable_class is not None
return self._callable_class
@callable_class.setter
def callable_class(self, cls: ImplicitClass) -> None:
self._callable_class = cls
@property
def env_class(self) -> ClassIR:
assert self._env_class is not None
return self._env_class
@env_class.setter
def env_class(self, ir: ClassIR) -> None:
self._env_class = ir
@property
def generator_class(self) -> GeneratorClass:
assert self._generator_class is not None
return self._generator_class
@generator_class.setter
def generator_class(self, cls: GeneratorClass) -> None:
self._generator_class = cls
@property
def curr_env_reg(self) -> Value:
assert self._curr_env_reg is not None
return self._curr_env_reg
def can_merge_generator_and_env_classes(self) -> bool:
# In simple cases we can place the environment into the generator class,
# instead of having two separate classes.
if self._generator_class and not self._generator_class.ir.is_final_class:
result = False
else:
result = self.is_generator and not self.is_nested and not self.contains_nested
return result
class ImplicitClass:
"""Contains information regarding implicitly generated classes.
Implicit classes are generated for nested functions and generator
functions. They are not explicitly defined in the source code.
NOTE: This is both a concrete class and used as a base class.
"""
def __init__(self, ir: ClassIR) -> None:
# The ClassIR instance associated with this class.
self.ir = ir
# The register associated with the 'self' instance for this generator class.
self._self_reg: Value | None = None
# Environment class registers are the local registers associated with instances of an
# environment class, used for getting and setting attributes. curr_env_reg is the register
# associated with the current environment. prev_env_reg is the self.__mypyc_env__ field
# associated with the previous environment.
self._curr_env_reg: Value | None = None
self._prev_env_reg: Value | None = None
@property
def self_reg(self) -> Value:
assert self._self_reg is not None
return self._self_reg
@self_reg.setter
def self_reg(self, reg: Value) -> None:
self._self_reg = reg
@property
def curr_env_reg(self) -> Value:
assert self._curr_env_reg is not None
return self._curr_env_reg
@curr_env_reg.setter
def curr_env_reg(self, reg: Value) -> None:
self._curr_env_reg = reg
@property
def prev_env_reg(self) -> Value:
assert self._prev_env_reg is not None
return self._prev_env_reg
@prev_env_reg.setter
def prev_env_reg(self, reg: Value) -> None:
self._prev_env_reg = reg
class GeneratorClass(ImplicitClass):
"""Contains information about implicit generator function classes."""
def __init__(self, ir: ClassIR) -> None:
super().__init__(ir)
# This register holds the label number that the '__next__' function should go to the next
# time it is called.
self._next_label_reg: Value | None = None
self._next_label_target: AssignmentTarget | None = None
# These registers hold the error values for the generator object for the case that the
# 'throw' function is called.
self.exc_regs: tuple[Value, Value, Value] | None = None
# Holds the arg passed to send
self.send_arg_reg: Value | None = None
# Holds the PyObject ** pointer through which return value can be passed
# instead of raising StopIteration(ret_value) (only if not NULL). This
# is used for faster native-to-native calls.
self.stop_iter_value_reg: Value | None = None
# The switch block is used to decide which instruction to go using the value held in the
# next-label register.
self.switch_block = BasicBlock()
self.continuation_blocks: list[BasicBlock] = []
@property
def next_label_reg(self) -> Value:
assert self._next_label_reg is not None
return self._next_label_reg
@next_label_reg.setter
def next_label_reg(self, reg: Value) -> None:
self._next_label_reg = reg
@property
def next_label_target(self) -> AssignmentTarget:
assert self._next_label_target is not None
return self._next_label_target
@next_label_target.setter
def next_label_target(self, target: AssignmentTarget) -> None:
self._next_label_target = target

View file

@ -0,0 +1,310 @@
"""Generate classes representing function environments (+ related operations).
If we have a nested function that has non-local (free) variables, access to the
non-locals is via an instance of an environment class. Example:
def f() -> int:
x = 0 # Make 'x' an attribute of an environment class instance
def g() -> int:
# We have access to the environment class instance to
# allow accessing 'x'
return x + 2
x = x + 1 # Modify the attribute
return g()
"""
from __future__ import annotations
from mypy.nodes import Argument, FuncDef, SymbolNode, Var
from mypyc.common import (
BITMAP_BITS,
ENV_ATTR_NAME,
GENERATOR_ATTRIBUTE_PREFIX,
SELF_NAME,
bitmap_name,
)
from mypyc.ir.class_ir import ClassIR
from mypyc.ir.ops import Call, GetAttr, SetAttr, Value
from mypyc.ir.rtypes import RInstance, bitmap_rprimitive, object_rprimitive
from mypyc.irbuild.builder import IRBuilder, SymbolTarget
from mypyc.irbuild.context import FuncInfo, GeneratorClass, ImplicitClass
from mypyc.irbuild.targets import AssignmentTargetAttr
def setup_env_class(builder: IRBuilder) -> ClassIR:
"""Generate a class representing a function environment.
Note that the variables in the function environment are not
actually populated here. This is because when the environment
class is generated, the function environment has not yet been
visited. This behavior is allowed so that when the compiler visits
nested functions, it can use the returned ClassIR instance to
figure out free variables it needs to access. The remaining
attributes of the environment class are populated when the
environment registers are loaded.
Return a ClassIR representing an environment for a function
containing a nested function.
"""
env_class = ClassIR(
f"{builder.fn_info.namespaced_name()}_env",
builder.module_name,
is_generated=True,
is_final_class=True,
)
env_class.reuse_freed_instance = True
env_class.attributes[SELF_NAME] = RInstance(env_class)
if builder.fn_info.is_nested and builder.fn_infos[-2]._env_class is not None:
# If the function is nested, its environment class must contain an environment
# attribute pointing to its encapsulating functions' environment class.
env_class.attributes[ENV_ATTR_NAME] = RInstance(builder.fn_infos[-2].env_class)
env_class.mro = [env_class]
builder.fn_info.env_class = env_class
builder.classes.append(env_class)
return env_class
def finalize_env_class(builder: IRBuilder, prefix: str = "") -> None:
"""Generate, instantiate, and set up the environment of an environment class."""
if not builder.fn_info.can_merge_generator_and_env_classes():
instantiate_env_class(builder)
# Iterate through the function arguments and replace local definitions (using registers)
# that were previously added to the environment with references to the function's
# environment class. Comprehension scopes have no arguments to add.
if not builder.fn_info.is_comprehension_scope:
if builder.fn_info.is_nested:
add_args_to_env(
builder, local=False, base=builder.fn_info.callable_class, prefix=prefix
)
else:
add_args_to_env(builder, local=False, base=builder.fn_info, prefix=prefix)
def instantiate_env_class(builder: IRBuilder) -> Value:
"""Assign an environment class to a register named after the given function definition."""
curr_env_reg = builder.add(
Call(builder.fn_info.env_class.ctor, [], builder.fn_info.fitem.line)
)
if builder.fn_info.is_nested and not builder.fn_info.is_comprehension_scope:
builder.fn_info.callable_class._curr_env_reg = curr_env_reg
builder.add(
SetAttr(
curr_env_reg,
ENV_ATTR_NAME,
builder.fn_info.callable_class.prev_env_reg,
builder.fn_info.fitem.line,
)
)
else:
# Top-level functions and comprehension scopes store env reg directly.
builder.fn_info._curr_env_reg = curr_env_reg
# Comprehension scopes link to parent env if it exists.
if (
builder.fn_info.is_nested
and builder.fn_infos[-2]._env_class is not None
and builder.fn_infos[-2]._curr_env_reg is not None
):
builder.add(
SetAttr(
curr_env_reg,
ENV_ATTR_NAME,
builder.fn_infos[-2].curr_env_reg,
builder.fn_info.fitem.line,
)
)
return curr_env_reg
def load_env_registers(builder: IRBuilder, prefix: str = "") -> None:
"""Load the registers for the current FuncItem being visited.
Adds the arguments of the FuncItem to the environment. If the
FuncItem is nested inside of another function, then this also
loads all of the outer environments of the FuncItem into registers
so that they can be used when accessing free variables.
"""
add_args_to_env(builder, local=True, prefix=prefix)
fn_info = builder.fn_info
fitem = fn_info.fitem
if fn_info.is_nested and builder.fn_infos[-2]._env_class is not None:
load_outer_envs(builder, fn_info.callable_class)
# If this is a FuncDef, then make sure to load the FuncDef into its own environment
# class so that the function can be called recursively.
if isinstance(fitem, FuncDef) and fn_info.add_nested_funcs_to_env:
setup_func_for_recursive_call(builder, fitem, fn_info.callable_class, prefix=prefix)
def load_outer_env(
builder: IRBuilder, base: Value, outer_env: dict[SymbolNode, SymbolTarget]
) -> Value:
"""Load the environment class for a given base into a register.
Additionally, iterates through all of the SymbolNode and
AssignmentTarget instances of the environment at the given index's
symtable, and adds those instances to the environment of the
current environment. This is done so that the current environment
can access outer environment variables without having to reload
all of the environment registers.
Returns the register where the environment class was loaded.
"""
env = builder.add(GetAttr(base, ENV_ATTR_NAME, builder.fn_info.fitem.line))
assert isinstance(env.type, RInstance), f"{env} must be of type RInstance"
for symbol, target in outer_env.items():
attr_name = symbol.name
if isinstance(target, AssignmentTargetAttr):
attr_name = target.attr
env.type.class_ir.attributes[attr_name] = target.type
symbol_target = AssignmentTargetAttr(env, attr_name)
builder.add_target(symbol, symbol_target)
return env
def load_outer_envs(builder: IRBuilder, base: ImplicitClass) -> None:
index = len(builder.builders) - 2
# Load the first outer environment. This one is special because it gets saved in the
# FuncInfo instance's prev_env_reg field.
has_outer = index > 1 or (index == 1 and builder.fn_infos[1].contains_nested)
if has_outer and builder.fn_infos[index]._env_class is not None:
# outer_env = builder.fn_infos[index].environment
outer_env = builder.symtables[index]
if isinstance(base, GeneratorClass):
base.prev_env_reg = load_outer_env(builder, base.curr_env_reg, outer_env)
else:
base.prev_env_reg = load_outer_env(builder, base.self_reg, outer_env)
env_reg = base.prev_env_reg
index -= 1
# Load the remaining outer environments into registers.
while index > 1:
if builder.fn_infos[index]._env_class is None:
break
# outer_env = builder.fn_infos[index].environment
outer_env = builder.symtables[index]
env_reg = load_outer_env(builder, env_reg, outer_env)
index -= 1
def num_bitmap_args(builder: IRBuilder, args: list[Argument]) -> int:
n = 0
for arg in args:
t = builder.type_to_rtype(arg.variable.type)
if t.error_overlap and arg.kind.is_optional():
n += 1
return (n + (BITMAP_BITS - 1)) // BITMAP_BITS
def add_args_to_env(
builder: IRBuilder,
local: bool = True,
base: FuncInfo | ImplicitClass | None = None,
reassign: bool = True,
prefix: str = "",
) -> None:
fn_info = builder.fn_info
args = fn_info.fitem.arguments
nb = num_bitmap_args(builder, args)
if local:
for arg in args:
rtype = builder.type_to_rtype(arg.variable.type)
builder.add_local_reg(arg.variable, rtype, is_arg=True)
for i in reversed(range(nb)):
builder.add_local_reg(Var(bitmap_name(i)), bitmap_rprimitive, is_arg=True)
else:
for arg in args:
if (
is_free_variable(builder, arg.variable)
or fn_info.is_generator
or fn_info.is_coroutine
):
rtype = builder.type_to_rtype(arg.variable.type)
assert base is not None, "base cannot be None for adding nonlocal args"
builder.add_var_to_env_class(
arg.variable, rtype, base, reassign=reassign, prefix=prefix
)
def add_vars_to_env(builder: IRBuilder, prefix: str = "") -> None:
"""Add relevant local variables and nested functions to the environment class.
Add all variables and functions that are declared/defined within current
function and are referenced in functions nested within this one to this
function's environment class so the nested functions can reference
them even if they are declared after the nested function's definition.
Note that this is done before visiting the body of the function.
"""
env_for_func: FuncInfo | ImplicitClass = builder.fn_info
if builder.fn_info.is_generator:
env_for_func = builder.fn_info.generator_class
elif (
builder.fn_info.is_nested or builder.fn_info.in_non_ext
) and not builder.fn_info.is_comprehension_scope:
env_for_func = builder.fn_info.callable_class
if builder.fn_info.fitem in builder.free_variables:
# Sort the variables to keep things deterministic
for var in sorted(builder.free_variables[builder.fn_info.fitem], key=lambda x: x.name):
if isinstance(var, Var):
rtype = builder.type_to_rtype(var.type)
builder.add_var_to_env_class(
var, rtype, env_for_func, reassign=False, prefix=prefix
)
if builder.fn_info.fitem in builder.encapsulating_funcs:
for nested_fn in builder.encapsulating_funcs[builder.fn_info.fitem]:
if isinstance(nested_fn, FuncDef):
# The return type is 'object' instead of an RInstance of the
# callable class because differently defined functions with
# the same name and signature across conditional blocks
# will generate different callable classes, so the callable
# class that gets instantiated must be generic.
if nested_fn.is_generator or nested_fn.is_coroutine:
prefix = GENERATOR_ATTRIBUTE_PREFIX
builder.add_var_to_env_class(
nested_fn, object_rprimitive, env_for_func, reassign=False, prefix=prefix
)
def setup_func_for_recursive_call(
builder: IRBuilder, fdef: FuncDef, base: ImplicitClass, prefix: str = ""
) -> None:
"""Enable calling a nested function (with a callable class) recursively.
Adds the instance of the callable class representing the given
FuncDef to a register in the environment so that the function can
be called recursively. Note that this needs to be done only for
nested functions.
"""
# First, set the attribute of the environment class so that GetAttr can be called on it.
prev_env = builder.fn_infos[-2].env_class
attr_name = prefix + fdef.name
prev_env.attributes[attr_name] = builder.type_to_rtype(fdef.type)
line = fdef.line
if isinstance(base, GeneratorClass):
# If we are dealing with a generator class, then we need to first get the register
# holding the current environment class, and load the previous environment class from
# there.
prev_env_reg = builder.add(GetAttr(base.curr_env_reg, ENV_ATTR_NAME, line))
else:
prev_env_reg = base.prev_env_reg
# Obtain the instance of the callable class representing the FuncDef, and add it to the
# current environment.
val = builder.add(GetAttr(prev_env_reg, attr_name, line))
target = builder.add_local_reg(fdef, object_rprimitive)
builder.assign(target, val, line)
def is_free_variable(builder: IRBuilder, symbol: SymbolNode) -> bool:
fitem = builder.fn_info.fitem
return fitem in builder.free_variables and symbol in builder.free_variables[fitem]

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,260 @@
"""Tokenizers for three string formatting methods"""
from __future__ import annotations
from enum import Enum, unique
from typing import Final
from mypy.checkstrformat import (
ConversionSpecifier,
parse_conversion_specifiers,
parse_format_value,
)
from mypy.errors import Errors
from mypy.messages import MessageBuilder
from mypy.nodes import Context, Expression
from mypy.options import Options
from mypyc.ir.ops import Integer, Value
from mypyc.ir.rtypes import (
c_pyssize_t_rprimitive,
is_bytes_rprimitive,
is_int_rprimitive,
is_short_int_rprimitive,
is_str_rprimitive,
)
from mypyc.irbuild.builder import IRBuilder
from mypyc.irbuild.constant_fold import constant_fold_expr
from mypyc.primitives.bytes_ops import bytes_build_op
from mypyc.primitives.int_ops import int_to_ascii_op, int_to_str_op
from mypyc.primitives.str_ops import str_build_op, str_op
@unique
class FormatOp(Enum):
"""FormatOp represents conversion operations of string formatting during
compile time.
Compare to ConversionSpecifier, FormatOp has fewer attributes.
For example, to mark a conversion from any object to string,
ConversionSpecifier may have several representations, like '%s', '{}'
or '{:{}}'. However, there would only exist one corresponding FormatOp.
"""
STR = "s"
INT = "d"
BYTES = "b"
def generate_format_ops(specifiers: list[ConversionSpecifier]) -> list[FormatOp] | None:
"""Convert ConversionSpecifier to FormatOp.
Different ConversionSpecifiers may share a same FormatOp.
"""
format_ops = []
for spec in specifiers:
# TODO: Match specifiers instead of using whole_seq
if spec.whole_seq == "%s" or spec.whole_seq == "{:{}}":
format_op = FormatOp.STR
elif spec.whole_seq == "%d":
format_op = FormatOp.INT
elif spec.whole_seq == "%b":
format_op = FormatOp.BYTES
elif spec.whole_seq:
return None
else:
format_op = FormatOp.STR
format_ops.append(format_op)
return format_ops
def tokenizer_printf_style(format_str: str) -> tuple[list[str], list[FormatOp]] | None:
"""Tokenize a printf-style format string using regex.
Return:
A list of string literals and a list of FormatOps.
"""
literals: list[str] = []
specifiers: list[ConversionSpecifier] = parse_conversion_specifiers(format_str)
format_ops = generate_format_ops(specifiers)
if format_ops is None:
return None
last_end = 0
for spec in specifiers:
cur_start = spec.start_pos
literals.append(format_str[last_end:cur_start])
last_end = cur_start + len(spec.whole_seq)
literals.append(format_str[last_end:])
return literals, format_ops
# The empty Context as an argument for parse_format_value().
# It wouldn't be used since the code has passed the type-checking.
EMPTY_CONTEXT: Final = Context()
def tokenizer_format_call(format_str: str) -> tuple[list[str], list[FormatOp]] | None:
"""Tokenize a str.format() format string.
The core function parse_format_value() is shared with mypy.
With these specifiers, we then parse the literal substrings
of the original format string and convert `ConversionSpecifier`
to `FormatOp`.
Return:
A list of string literals and a list of FormatOps. The literals
are interleaved with FormatOps and the length of returned literals
should be exactly one more than FormatOps.
Return None if it cannot parse the string.
"""
# Creates an empty MessageBuilder here.
# It wouldn't be used since the code has passed the type-checking.
specifiers = parse_format_value(
format_str, EMPTY_CONTEXT, MessageBuilder(Errors(Options()), {})
)
if specifiers is None:
return None
format_ops = generate_format_ops(specifiers)
if format_ops is None:
return None
literals: list[str] = []
last_end = 0
for spec in specifiers:
# Skip { and }
literals.append(format_str[last_end : spec.start_pos - 1])
last_end = spec.start_pos + len(spec.whole_seq) + 1
literals.append(format_str[last_end:])
# Deal with escaped {{
literals = [x.replace("{{", "{").replace("}}", "}") for x in literals]
return literals, format_ops
def convert_format_expr_to_str(
builder: IRBuilder, format_ops: list[FormatOp], exprs: list[Expression], line: int
) -> list[Value] | None:
"""Convert expressions into string literal objects with the guidance
of FormatOps. Return None when fails."""
if len(format_ops) != len(exprs):
return None
converted = []
for x, format_op in zip(exprs, format_ops):
node_type = builder.node_type(x)
if format_op == FormatOp.STR:
if isinstance(folded := constant_fold_expr(builder, x), str):
var_str = builder.load_literal_value(folded)
elif is_str_rprimitive(node_type):
var_str = builder.accept(x)
elif is_int_rprimitive(node_type) or is_short_int_rprimitive(node_type):
var_str = builder.primitive_op(int_to_str_op, [builder.accept(x)], line)
else:
var_str = builder.primitive_op(str_op, [builder.accept(x)], line)
elif format_op == FormatOp.INT:
if isinstance(folded := constant_fold_expr(builder, x), int):
var_str = builder.load_literal_value(str(folded))
elif is_int_rprimitive(node_type) or is_short_int_rprimitive(node_type):
var_str = builder.primitive_op(int_to_str_op, [builder.accept(x)], line)
else:
return None
else:
return None
converted.append(var_str)
return converted
def join_formatted_strings(
builder: IRBuilder, literals: list[str] | None, substitutions: list[Value], line: int
) -> Value:
"""Merge the list of literals and the list of substitutions
alternatively using 'str_build_op'.
`substitutions` is the result value of formatting conversions.
If the `literals` is set to None, we simply join the substitutions;
Otherwise, the `literals` is the literal substrings of the original
format string and its length should be exactly one more than
substitutions.
For example:
(1) 'This is a %s and the value is %d'
-> literals: ['This is a ', ' and the value is', '']
(2) '{} and the value is {}'
-> literals: ['', ' and the value is', '']
"""
# The first parameter for str_build_op is the total size of
# the following PyObject*
result_list: list[Value] = [Integer(0, c_pyssize_t_rprimitive)]
if literals is not None:
for a, b in zip(literals, substitutions):
if a:
result_list.append(builder.load_str(a))
result_list.append(b)
if literals[-1]:
result_list.append(builder.load_str(literals[-1]))
else:
result_list.extend(substitutions)
# Special case for empty string and literal string
if len(result_list) == 1:
return builder.load_str("")
if not substitutions and len(result_list) == 2:
return result_list[1]
result_list[0] = Integer(len(result_list) - 1, c_pyssize_t_rprimitive)
return builder.call_c(str_build_op, result_list, line)
def convert_format_expr_to_bytes(
builder: IRBuilder, format_ops: list[FormatOp], exprs: list[Expression], line: int
) -> list[Value] | None:
"""Convert expressions into bytes literal objects with the guidance
of FormatOps. Return None when fails."""
if len(format_ops) != len(exprs):
return None
converted = []
for x, format_op in zip(exprs, format_ops):
node_type = builder.node_type(x)
# conversion type 's' is an alias of 'b' in bytes formatting
if format_op == FormatOp.BYTES or format_op == FormatOp.STR:
if is_bytes_rprimitive(node_type):
var_bytes = builder.accept(x)
else:
return None
elif format_op == FormatOp.INT:
if isinstance(folded := constant_fold_expr(builder, x), int):
var_bytes = builder.load_literal_value(str(folded).encode("ascii"))
elif is_int_rprimitive(node_type) or is_short_int_rprimitive(node_type):
var_bytes = builder.call_c(int_to_ascii_op, [builder.accept(x)], line)
else:
return None
converted.append(var_bytes)
return converted
def join_formatted_bytes(
builder: IRBuilder, literals: list[str], substitutions: list[Value], line: int
) -> Value:
"""Merge the list of literals and the list of substitutions
alternatively using 'bytes_build_op'."""
result_list: list[Value] = [Integer(0, c_pyssize_t_rprimitive)]
for a, b in zip(literals, substitutions):
if a:
result_list.append(builder.load_bytes_from_str_literal(a))
result_list.append(b)
if literals[-1]:
result_list.append(builder.load_bytes_from_str_literal(literals[-1]))
# Special case for empty bytes and literal
if len(result_list) == 1:
return builder.load_bytes_from_str_literal("")
if not substitutions and len(result_list) == 2:
return result_list[1]
result_list[0] = Integer(len(result_list) - 1, c_pyssize_t_rprimitive)
return builder.call_c(bytes_build_op, result_list, line)

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,438 @@
"""Generate IR for generator functions.
A generator function is represented by a class that implements the
generator protocol and keeps track of the generator state, including
local variables.
The top-level logic for dealing with generator functions is in
mypyc.irbuild.function.
"""
from __future__ import annotations
from collections.abc import Callable
from mypy.nodes import ARG_OPT, FuncDef, Var
from mypyc.common import ENV_ATTR_NAME, GENERATOR_ATTRIBUTE_PREFIX, NEXT_LABEL_ATTR_NAME
from mypyc.ir.class_ir import ClassIR
from mypyc.ir.func_ir import FuncDecl, FuncIR
from mypyc.ir.ops import (
NO_TRACEBACK_LINE_NO,
BasicBlock,
Branch,
Call,
Goto,
Integer,
MethodCall,
RaiseStandardError,
Register,
Return,
SetAttr,
TupleSet,
Unreachable,
Value,
)
from mypyc.ir.rtypes import (
RInstance,
int32_rprimitive,
object_pointer_rprimitive,
object_rprimitive,
)
from mypyc.irbuild.builder import IRBuilder, calculate_arg_defaults, gen_arg_defaults
from mypyc.irbuild.context import FuncInfo
from mypyc.irbuild.env_class import (
add_args_to_env,
add_vars_to_env,
finalize_env_class,
load_env_registers,
load_outer_env,
load_outer_envs,
setup_func_for_recursive_call,
)
from mypyc.irbuild.nonlocalcontrol import ExceptNonlocalControl
from mypyc.irbuild.prepare import GENERATOR_HELPER_NAME
from mypyc.primitives.exc_ops import (
error_catch_op,
exc_matches_op,
raise_exception_with_tb_op,
reraise_exception_op,
restore_exc_info_op,
)
def gen_generator_func(
builder: IRBuilder,
gen_func_ir: Callable[
[list[Register], list[BasicBlock], FuncInfo], tuple[FuncIR, Value | None]
],
) -> tuple[FuncIR, Value | None]:
"""Generate IR for generator function that returns generator object."""
setup_generator_class(builder)
load_env_registers(builder, prefix=GENERATOR_ATTRIBUTE_PREFIX)
gen_arg_defaults(builder)
if builder.fn_info.can_merge_generator_and_env_classes():
gen = instantiate_generator_class(builder)
builder.fn_info._curr_env_reg = gen
finalize_env_class(builder, prefix=GENERATOR_ATTRIBUTE_PREFIX)
else:
finalize_env_class(builder, prefix=GENERATOR_ATTRIBUTE_PREFIX)
gen = instantiate_generator_class(builder)
builder.add(Return(gen))
args, _, blocks, ret_type, fn_info = builder.leave()
func_ir, func_reg = gen_func_ir(args, blocks, fn_info)
return func_ir, func_reg
def gen_generator_func_body(builder: IRBuilder, fn_info: FuncInfo, func_reg: Value | None) -> None:
"""Generate IR based on the body of a generator function.
Add "__next__", "__iter__" and other generator methods to the generator
class that implements the function (each function gets a separate class).
Return the symbol table for the body.
"""
builder.enter(fn_info, ret_type=object_rprimitive)
setup_env_for_generator_class(builder)
load_outer_envs(builder, builder.fn_info.generator_class)
top_level = builder.top_level_fn_info()
fitem = fn_info.fitem
if (
builder.fn_info.is_nested
and isinstance(fitem, FuncDef)
and top_level
and top_level.add_nested_funcs_to_env
):
setup_func_for_recursive_call(
builder, fitem, builder.fn_info.generator_class, prefix=GENERATOR_ATTRIBUTE_PREFIX
)
create_switch_for_generator_class(builder)
add_raise_exception_blocks_to_generator_class(builder, fitem.line)
add_vars_to_env(builder, prefix=GENERATOR_ATTRIBUTE_PREFIX)
builder.accept(fitem.body)
builder.maybe_add_implicit_return()
populate_switch_for_generator_class(builder)
# Hang on to the local symbol table, since the caller will use it
# to calculate argument defaults.
symtable = builder.symtables[-1]
args, _, blocks, ret_type, fn_info = builder.leave()
add_methods_to_generator_class(builder, fn_info, args, blocks, fitem.is_coroutine)
# Evaluate argument defaults in the surrounding scope, since we
# calculate them *once* when the function definition is evaluated.
calculate_arg_defaults(builder, fn_info, func_reg, symtable)
def instantiate_generator_class(builder: IRBuilder) -> Value:
fitem = builder.fn_info.fitem
generator_reg = builder.add(Call(builder.fn_info.generator_class.ir.ctor, [], fitem.line))
if builder.fn_info.can_merge_generator_and_env_classes():
# Set the generator instance to the initial state (zero).
zero = Integer(0)
builder.add(SetAttr(generator_reg, NEXT_LABEL_ATTR_NAME, zero, fitem.line))
else:
# Get the current environment register. If the current function is nested, then the
# generator class gets instantiated from the callable class' '__call__' method, and hence
# we use the callable class' environment register. Otherwise, we use the original
# function's environment register.
if builder.fn_info.is_nested:
curr_env_reg = builder.fn_info.callable_class.curr_env_reg
else:
curr_env_reg = builder.fn_info.curr_env_reg
# Set the generator class' environment attribute to point at the environment class
# defined in the current scope.
builder.add(SetAttr(generator_reg, ENV_ATTR_NAME, curr_env_reg, fitem.line))
# Set the generator instance's environment to the initial state (zero).
zero = Integer(0)
builder.add(SetAttr(curr_env_reg, NEXT_LABEL_ATTR_NAME, zero, fitem.line))
return generator_reg
def setup_generator_class(builder: IRBuilder) -> ClassIR:
mapper = builder.mapper
assert isinstance(builder.fn_info.fitem, FuncDef), builder.fn_info.fitem
generator_class_ir = mapper.fdef_to_generator[builder.fn_info.fitem]
if builder.fn_info.can_merge_generator_and_env_classes():
builder.fn_info.env_class = generator_class_ir
else:
generator_class_ir.attributes[ENV_ATTR_NAME] = RInstance(builder.fn_info.env_class)
builder.classes.append(generator_class_ir)
return generator_class_ir
def create_switch_for_generator_class(builder: IRBuilder) -> None:
builder.add(Goto(builder.fn_info.generator_class.switch_block))
block = BasicBlock()
builder.fn_info.generator_class.continuation_blocks.append(block)
builder.activate_block(block)
def populate_switch_for_generator_class(builder: IRBuilder) -> None:
cls = builder.fn_info.generator_class
line = builder.fn_info.fitem.line
builder.activate_block(cls.switch_block)
for label, true_block in enumerate(cls.continuation_blocks):
false_block = BasicBlock()
comparison = builder.binary_op(cls.next_label_reg, Integer(label), "==", line)
builder.add_bool_branch(comparison, true_block, false_block)
builder.activate_block(false_block)
builder.add(RaiseStandardError(RaiseStandardError.STOP_ITERATION, None, line))
builder.add(Unreachable())
def add_raise_exception_blocks_to_generator_class(builder: IRBuilder, line: int) -> None:
"""Add error handling blocks to a generator class.
Generates blocks to check if error flags are set while calling the
helper method for generator functions, and raises an exception if
those flags are set.
"""
cls = builder.fn_info.generator_class
assert cls.exc_regs is not None
exc_type, exc_val, exc_tb = cls.exc_regs
# Check to see if an exception was raised.
error_block = BasicBlock()
ok_block = BasicBlock()
comparison = builder.translate_is_op(exc_type, builder.none_object(), "is not", line)
builder.add_bool_branch(comparison, error_block, ok_block)
builder.activate_block(error_block)
builder.call_c(raise_exception_with_tb_op, [exc_type, exc_val, exc_tb], line)
builder.add(Unreachable())
builder.goto_and_activate(ok_block)
def add_methods_to_generator_class(
builder: IRBuilder,
fn_info: FuncInfo,
arg_regs: list[Register],
blocks: list[BasicBlock],
is_coroutine: bool,
) -> None:
helper_fn_decl = add_helper_to_generator_class(builder, arg_regs, blocks, fn_info)
add_next_to_generator_class(builder, fn_info, helper_fn_decl)
add_send_to_generator_class(builder, fn_info, helper_fn_decl)
add_iter_to_generator_class(builder, fn_info)
add_throw_to_generator_class(builder, fn_info, helper_fn_decl)
add_close_to_generator_class(builder, fn_info)
if is_coroutine:
add_await_to_generator_class(builder, fn_info)
def add_helper_to_generator_class(
builder: IRBuilder, arg_regs: list[Register], blocks: list[BasicBlock], fn_info: FuncInfo
) -> FuncDecl:
"""Generates a helper method for a generator class, called by '__next__' and 'throw'."""
helper_fn_decl = fn_info.generator_class.ir.method_decls[GENERATOR_HELPER_NAME]
helper_fn_ir = FuncIR(
helper_fn_decl, arg_regs, blocks, fn_info.fitem.line, traceback_name=fn_info.fitem.name
)
fn_info.generator_class.ir.methods[GENERATOR_HELPER_NAME] = helper_fn_ir
builder.functions.append(helper_fn_ir)
fn_info.env_class.env_user_function = helper_fn_ir
return helper_fn_decl
def add_iter_to_generator_class(builder: IRBuilder, fn_info: FuncInfo) -> None:
"""Generates the '__iter__' method for a generator class."""
with builder.enter_method(fn_info.generator_class.ir, "__iter__", object_rprimitive, fn_info):
builder.add(Return(builder.self()))
def add_next_to_generator_class(builder: IRBuilder, fn_info: FuncInfo, fn_decl: FuncDecl) -> None:
"""Generates the '__next__' method for a generator class."""
with builder.enter_method(fn_info.generator_class.ir, "__next__", object_rprimitive, fn_info):
none_reg = builder.none_object()
# Call the helper function with error flags set to Py_None, and return that result.
result = builder.add(
Call(
fn_decl,
[
builder.self(),
none_reg,
none_reg,
none_reg,
none_reg,
Integer(0, object_pointer_rprimitive),
],
fn_info.fitem.line,
)
)
builder.add(Return(result))
def add_send_to_generator_class(builder: IRBuilder, fn_info: FuncInfo, fn_decl: FuncDecl) -> None:
"""Generates the 'send' method for a generator class."""
with builder.enter_method(fn_info.generator_class.ir, "send", object_rprimitive, fn_info):
arg = builder.add_argument("arg", object_rprimitive)
none_reg = builder.none_object()
# Call the helper function with error flags set to Py_None, and return that result.
result = builder.add(
Call(
fn_decl,
[
builder.self(),
none_reg,
none_reg,
none_reg,
builder.read(arg),
Integer(0, object_pointer_rprimitive),
],
fn_info.fitem.line,
)
)
builder.add(Return(result))
def add_throw_to_generator_class(builder: IRBuilder, fn_info: FuncInfo, fn_decl: FuncDecl) -> None:
"""Generates the 'throw' method for a generator class."""
with builder.enter_method(fn_info.generator_class.ir, "throw", object_rprimitive, fn_info):
typ = builder.add_argument("type", object_rprimitive)
val = builder.add_argument("value", object_rprimitive, ARG_OPT)
tb = builder.add_argument("traceback", object_rprimitive, ARG_OPT)
# Because the value and traceback arguments are optional and hence
# can be NULL if not passed in, we have to assign them Py_None if
# they are not passed in.
none_reg = builder.none_object()
builder.assign_if_null(val, lambda: none_reg, fn_info.fitem.line)
builder.assign_if_null(tb, lambda: none_reg, fn_info.fitem.line)
# Call the helper function using the arguments passed in, and return that result.
result = builder.add(
Call(
fn_decl,
[
builder.self(),
builder.read(typ),
builder.read(val),
builder.read(tb),
none_reg,
Integer(0, object_pointer_rprimitive),
],
fn_info.fitem.line,
)
)
builder.add(Return(result))
def add_close_to_generator_class(builder: IRBuilder, fn_info: FuncInfo) -> None:
"""Generates the '__close__' method for a generator class."""
with builder.enter_method(fn_info.generator_class.ir, "close", object_rprimitive, fn_info):
except_block, else_block = BasicBlock(), BasicBlock()
builder.builder.push_error_handler(except_block)
builder.goto_and_activate(BasicBlock())
generator_exit = builder.load_module_attr_by_fullname(
"builtins.GeneratorExit", fn_info.fitem.line
)
builder.add(
MethodCall(
builder.self(),
"throw",
[generator_exit, builder.none_object(), builder.none_object()],
fn_info.fitem.line,
)
)
builder.goto(else_block)
builder.builder.pop_error_handler()
builder.activate_block(except_block)
old_exc = builder.call_c(error_catch_op, [], fn_info.fitem.line)
builder.nonlocal_control.append(
ExceptNonlocalControl(builder.nonlocal_control[-1], old_exc)
)
stop_iteration = builder.load_module_attr_by_fullname(
"builtins.StopIteration", fn_info.fitem.line
)
exceptions = builder.add(TupleSet([generator_exit, stop_iteration], fn_info.fitem.line))
matches = builder.call_c(exc_matches_op, [exceptions], fn_info.fitem.line)
match_block, non_match_block = BasicBlock(), BasicBlock()
builder.add(Branch(matches, match_block, non_match_block, Branch.BOOL))
builder.activate_block(match_block)
builder.call_c(restore_exc_info_op, [builder.read(old_exc)], fn_info.fitem.line)
builder.add(Return(builder.none_object()))
builder.activate_block(non_match_block)
builder.call_c(reraise_exception_op, [], NO_TRACEBACK_LINE_NO)
builder.add(Unreachable())
builder.nonlocal_control.pop()
builder.activate_block(else_block)
builder.add(
RaiseStandardError(
RaiseStandardError.RUNTIME_ERROR,
"generator ignored GeneratorExit",
fn_info.fitem.line,
)
)
builder.add(Unreachable())
def add_await_to_generator_class(builder: IRBuilder, fn_info: FuncInfo) -> None:
"""Generates the '__await__' method for a generator class."""
with builder.enter_method(fn_info.generator_class.ir, "__await__", object_rprimitive, fn_info):
builder.add(Return(builder.self()))
def setup_env_for_generator_class(builder: IRBuilder) -> None:
"""Populates the environment for a generator class."""
fitem = builder.fn_info.fitem
cls = builder.fn_info.generator_class
self_target = builder.add_self_to_env(cls.ir)
# Add the type, value, and traceback variables to the environment.
exc_type = builder.add_local(Var("type"), object_rprimitive, is_arg=True)
exc_val = builder.add_local(Var("value"), object_rprimitive, is_arg=True)
exc_tb = builder.add_local(Var("traceback"), object_rprimitive, is_arg=True)
# TODO: Use the right type here instead of object?
exc_arg = builder.add_local(Var("arg"), object_rprimitive, is_arg=True)
# Parameter that can used to pass a pointer which can used instead of
# raising StopIteration(value). If the value is NULL, this won't be used.
stop_iter_value_arg = builder.add_local(
Var("stop_iter_ptr"), object_pointer_rprimitive, is_arg=True
)
cls.exc_regs = (exc_type, exc_val, exc_tb)
cls.send_arg_reg = exc_arg
cls.stop_iter_value_reg = stop_iter_value_arg
cls.self_reg = builder.read(self_target, fitem.line)
if builder.fn_info.can_merge_generator_and_env_classes():
cls.curr_env_reg = cls.self_reg
else:
cls.curr_env_reg = load_outer_env(builder, cls.self_reg, builder.symtables[-1])
# Define a variable representing the label to go to the next time
# the '__next__' function of the generator is called, and add it
# as an attribute to the environment class.
cls.next_label_target = builder.add_var_to_env_class(
Var(NEXT_LABEL_ATTR_NAME), int32_rprimitive, cls, reassign=False, always_defined=True
)
# Add arguments from the original generator function to the
# environment of the generator class.
add_args_to_env(
builder, local=False, base=cls, reassign=False, prefix=GENERATOR_ATTRIBUTE_PREFIX
)
# Set the next label register for the generator class.
cls.next_label_reg = builder.read(cls.next_label_target, fitem.line)

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,173 @@
"""Transform a mypy AST to the IR form (Intermediate Representation).
For example, consider a function like this:
def f(x: int) -> int:
return x * 2 + 1
It would be translated to something that conceptually looks like this:
r0 = 2
r1 = 1
r2 = x * r0 :: int
r3 = r2 + r1 :: int
return r3
This module deals with the module-level IR transformation logic and
putting it all together. The actual IR is implemented in mypyc.ir.
For the core of the IR transform implementation, look at build_ir()
below, mypyc.irbuild.builder, and mypyc.irbuild.visitor.
"""
from __future__ import annotations
from collections.abc import Callable
from typing import Any, TypeVar, cast
from mypy.build import Graph
from mypy.nodes import ClassDef, Expression, FuncDef, MypyFile
from mypy.state import state
from mypy.types import Type
from mypyc.analysis.attrdefined import analyze_always_defined_attrs
from mypyc.common import TOP_LEVEL_NAME
from mypyc.errors import Errors
from mypyc.ir.func_ir import FuncDecl, FuncIR, FuncSignature
from mypyc.ir.module_ir import ModuleIR, ModuleIRs
from mypyc.ir.rtypes import none_rprimitive
from mypyc.irbuild.builder import IRBuilder
from mypyc.irbuild.mapper import Mapper
from mypyc.irbuild.prebuildvisitor import PreBuildVisitor
from mypyc.irbuild.prepare import (
adjust_generator_classes_of_methods,
build_type_map,
create_generator_class_for_func,
find_singledispatch_register_impls,
)
from mypyc.irbuild.visitor import IRBuilderVisitor
from mypyc.irbuild.vtable import compute_vtable
from mypyc.options import CompilerOptions
# The stubs for callable contextmanagers are busted so cast it to the
# right type...
F = TypeVar("F", bound=Callable[..., Any])
strict_optional_dec = cast(Callable[[F], F], state.strict_optional_set(True))
@strict_optional_dec # Turn on strict optional for any type manipulations we do
def build_ir(
modules: list[MypyFile],
graph: Graph,
types: dict[Expression, Type],
mapper: Mapper,
options: CompilerOptions,
errors: Errors,
) -> ModuleIRs:
"""Build basic IR for a set of modules that have been type-checked by mypy.
The returned IR is not complete and requires additional
transformations, such as the insertion of refcount handling.
"""
build_type_map(mapper, modules, graph, types, options, errors)
adjust_generator_classes_of_methods(mapper)
singledispatch_info = find_singledispatch_register_impls(modules, errors)
result: ModuleIRs = {}
if errors.num_errors > 0:
return result
# Generate IR for all modules.
class_irs = []
for module in modules:
# First pass to determine free symbols.
pbv = PreBuildVisitor(errors, module, singledispatch_info.decorators_to_remove, types)
module.accept(pbv)
# Declare generator classes for nested async functions and generators.
for fdef in pbv.nested_funcs:
if isinstance(fdef, FuncDef):
# Make generator class name sufficiently unique.
suffix = f"___{fdef.line}"
if fdef.is_coroutine or fdef.is_generator:
create_generator_class_for_func(
module.fullname, None, fdef, mapper, name_suffix=suffix
)
# Construct and configure builder objects (cyclic runtime dependency).
visitor = IRBuilderVisitor()
builder = IRBuilder(
module.fullname,
types,
graph,
errors,
mapper,
pbv,
visitor,
options,
singledispatch_info.singledispatch_impls,
)
visitor.builder = builder
# Second pass does the bulk of the work.
transform_mypy_file(builder, module)
module_ir = ModuleIR(
module.fullname,
list(builder.imports),
builder.functions,
builder.classes,
builder.final_names,
builder.type_var_names,
)
result[module.fullname] = module_ir
class_irs.extend(builder.classes)
analyze_always_defined_attrs(class_irs)
# Compute vtables.
for cir in class_irs:
if cir.is_ext_class:
compute_vtable(cir)
return result
def transform_mypy_file(builder: IRBuilder, mypyfile: MypyFile) -> None:
"""Generate IR for a single module."""
if mypyfile.fullname in ("typing", "abc"):
# These module are special; their contents are currently all
# built-in primitives.
return
builder.set_module(mypyfile.fullname, mypyfile.path)
classes = [node for node in mypyfile.defs if isinstance(node, ClassDef)]
# Collect all classes.
for cls in classes:
ir = builder.mapper.type_to_ir[cls.info]
builder.classes.append(ir)
builder.enter("<module>")
# Make sure we have a builtins import
builder.gen_import("builtins", 1)
# Generate ops.
for node in mypyfile.defs:
builder.accept(node)
builder.maybe_add_implicit_return()
# Generate special function representing module top level.
args, _, blocks, ret_type, _ = builder.leave()
sig = FuncSignature([], none_rprimitive)
func_ir = FuncIR(
FuncDecl(TOP_LEVEL_NAME, None, builder.module_name, sig),
args,
blocks,
traceback_name="<module>",
)
builder.functions.append(func_ir)

View file

@ -0,0 +1,244 @@
"""Maintain a mapping from mypy concepts to IR/compiled concepts."""
from __future__ import annotations
from mypy.nodes import ARG_STAR, ARG_STAR2, GDEF, ArgKind, FuncDef, RefExpr, SymbolNode, TypeInfo
from mypy.types import (
AnyType,
CallableType,
Instance,
LiteralType,
NoneTyp,
Overloaded,
PartialType,
TupleType,
Type,
TypedDictType,
TypeType,
TypeVarLikeType,
UnboundType,
UninhabitedType,
UnionType,
find_unpack_in_list,
get_proper_type,
)
from mypyc.ir.class_ir import ClassIR
from mypyc.ir.func_ir import FuncDecl, FuncSignature, RuntimeArg
from mypyc.ir.rtypes import (
KNOWN_NATIVE_TYPES,
RInstance,
RTuple,
RType,
RUnion,
RVec,
bool_rprimitive,
bytearray_rprimitive,
bytes_rprimitive,
dict_rprimitive,
float_rprimitive,
frozenset_rprimitive,
int16_rprimitive,
int32_rprimitive,
int64_rprimitive,
int_rprimitive,
list_rprimitive,
none_rprimitive,
object_rprimitive,
range_rprimitive,
set_rprimitive,
str_rprimitive,
tuple_rprimitive,
uint8_rprimitive,
)
class Mapper:
"""Keep track of mappings from mypy concepts to IR concepts.
For example, we keep track of how the mypy TypeInfos of compiled
classes map to class IR objects.
This state is shared across all modules being compiled in all
compilation groups.
"""
def __init__(self, group_map: dict[str, str | None]) -> None:
self.group_map = group_map
self.type_to_ir: dict[TypeInfo, ClassIR] = {}
self.func_to_decl: dict[SymbolNode, FuncDecl] = {}
self.symbol_fullnames: set[str] = set()
# The corresponding generator class that implements a generator/async function
self.fdef_to_generator: dict[FuncDef, ClassIR] = {}
def type_to_rtype(self, typ: Type | None) -> RType:
if typ is None:
return object_rprimitive
typ = get_proper_type(typ)
if isinstance(typ, Instance):
if typ.type.is_newtype:
# Unwrap NewType to its base type for rprimitive mapping
assert len(typ.type.bases) == 1, typ.type.bases
return self.type_to_rtype(typ.type.bases[0])
if typ.type.fullname == "builtins.int":
return int_rprimitive
elif typ.type.fullname == "builtins.float":
return float_rprimitive
elif typ.type.fullname == "builtins.bool":
return bool_rprimitive
elif typ.type.fullname == "builtins.str":
return str_rprimitive
elif typ.type.fullname == "builtins.bytes":
return bytes_rprimitive
elif typ.type.fullname == "builtins.bytearray":
return bytearray_rprimitive
elif typ.type.fullname == "builtins.list":
return list_rprimitive
# Dict subclasses are at least somewhat common and we
# specifically support them, so make sure that dict operations
# get optimized on them.
elif any(cls.fullname == "builtins.dict" for cls in typ.type.mro):
return dict_rprimitive
elif typ.type.fullname == "builtins.set":
return set_rprimitive
elif typ.type.fullname == "builtins.frozenset":
return frozenset_rprimitive
elif typ.type.fullname == "builtins.tuple":
return tuple_rprimitive # Varying-length tuple
elif typ.type.fullname == "builtins.range":
return range_rprimitive
elif typ.type in self.type_to_ir:
inst = RInstance(self.type_to_ir[typ.type])
# Treat protocols as Union[protocol, object], so that we can do fast
# method calls in the cases where the protocol is explicitly inherited from
# and fall back to generic operations when it isn't.
if typ.type.is_protocol:
return RUnion([inst, object_rprimitive])
else:
return inst
elif typ.type.fullname == "mypy_extensions.i64":
return int64_rprimitive
elif typ.type.fullname == "mypy_extensions.i32":
return int32_rprimitive
elif typ.type.fullname == "mypy_extensions.i16":
return int16_rprimitive
elif typ.type.fullname == "mypy_extensions.u8":
return uint8_rprimitive
elif typ.type.fullname == "librt.vecs.vec":
return RVec(self.type_to_rtype(typ.args[0]))
elif typ.type.fullname in KNOWN_NATIVE_TYPES:
return KNOWN_NATIVE_TYPES[typ.type.fullname]
else:
return object_rprimitive
elif isinstance(typ, TupleType):
# Use our unboxed tuples for raw tuples but fall back to
# being boxed for NamedTuple or for variadic tuples.
if (
typ.partial_fallback.type.fullname == "builtins.tuple"
and find_unpack_in_list(typ.items) is None
):
return RTuple([self.type_to_rtype(t) for t in typ.items])
else:
return tuple_rprimitive
elif isinstance(typ, CallableType):
return object_rprimitive
elif isinstance(typ, NoneTyp):
return none_rprimitive
elif isinstance(typ, UnionType):
return RUnion.make_simplified_union([self.type_to_rtype(item) for item in typ.items])
elif isinstance(typ, AnyType):
return object_rprimitive
elif isinstance(typ, TypeType):
return object_rprimitive
elif isinstance(typ, TypeVarLikeType):
# Erase type variable to upper bound.
# TODO: Erase to union if object has value restriction?
return self.type_to_rtype(typ.upper_bound)
elif isinstance(typ, PartialType):
assert typ.var.type is not None
return self.type_to_rtype(typ.var.type)
elif isinstance(typ, Overloaded):
return object_rprimitive
elif isinstance(typ, TypedDictType):
return dict_rprimitive
elif isinstance(typ, LiteralType):
return self.type_to_rtype(typ.fallback)
elif isinstance(typ, (UninhabitedType, UnboundType)):
# Sure, whatever!
return object_rprimitive
# I think we've covered everything that is supposed to
# actually show up, so anything else is a bug somewhere.
assert False, "unexpected type %s" % type(typ)
def get_arg_rtype(self, typ: Type, kind: ArgKind) -> RType:
if kind == ARG_STAR:
return tuple_rprimitive
elif kind == ARG_STAR2:
return dict_rprimitive
else:
return self.type_to_rtype(typ)
def fdef_to_sig(self, fdef: FuncDef, strict_dunders_typing: bool) -> FuncSignature:
if isinstance(fdef.type, CallableType):
arg_types = [
self.get_arg_rtype(typ, kind)
for typ, kind in zip(fdef.type.arg_types, fdef.type.arg_kinds)
]
arg_pos_onlys = [name is None for name in fdef.type.arg_names]
ret = self.type_to_rtype(fdef.type.ret_type)
else:
# Handle unannotated functions
arg_types = [object_rprimitive for _ in fdef.arguments]
arg_pos_onlys = [arg.pos_only for arg in fdef.arguments]
# We at least know the return type for __init__ methods will be None.
is_init_method = fdef.name == "__init__" and bool(fdef.info)
if is_init_method:
ret = none_rprimitive
else:
ret = object_rprimitive
# mypyc FuncSignatures (unlike mypy types) want to have a name
# present even when the argument is position only, since it is
# the sole way that FuncDecl arguments are tracked. This is
# generally fine except in some cases (like for computing
# init_sig) we need to produce FuncSignatures from a
# deserialized FuncDef that lacks arguments. We won't ever
# need to use those inside of a FuncIR, so we just make up
# some crap.
if hasattr(fdef, "arguments"):
arg_names = [arg.variable.name for arg in fdef.arguments]
else:
arg_names = [name or "" for name in fdef.arg_names]
args = [
RuntimeArg(arg_name, arg_type, arg_kind, arg_pos_only)
for arg_name, arg_kind, arg_type, arg_pos_only in zip(
arg_names, fdef.arg_kinds, arg_types, arg_pos_onlys
)
]
if not strict_dunders_typing:
# We force certain dunder methods to return objects to support letting them
# return NotImplemented. It also avoids some pointless boxing and unboxing,
# since tp_richcompare needs an object anyways.
# However, it also prevents some optimizations.
if fdef.name in ("__eq__", "__ne__", "__lt__", "__gt__", "__le__", "__ge__"):
ret = object_rprimitive
return FuncSignature(args, ret)
def is_native_module(self, module: str) -> bool:
"""Is the given module one compiled by mypyc?"""
return module in self.group_map
def is_native_ref_expr(self, expr: RefExpr) -> bool:
if expr.node is None:
return False
if "." in expr.node.fullname:
name = expr.node.fullname.rpartition(".")[0]
return self.is_native_module(name) or name in self.symbol_fullnames
return True
def is_native_module_ref_expr(self, expr: RefExpr) -> bool:
return self.is_native_ref_expr(expr) and expr.kind == GDEF

View file

@ -0,0 +1,367 @@
from __future__ import annotations
from collections.abc import Generator
from contextlib import contextmanager
from mypy.nodes import MatchStmt, NameExpr, TypeInfo
from mypy.patterns import (
AsPattern,
ClassPattern,
MappingPattern,
OrPattern,
Pattern,
SequencePattern,
SingletonPattern,
StarredPattern,
ValuePattern,
)
from mypy.traverser import TraverserVisitor
from mypy.types import Instance, LiteralType, TupleType, get_proper_type
from mypyc.ir.ops import BasicBlock, Value
from mypyc.ir.rtypes import object_rprimitive
from mypyc.irbuild.builder import IRBuilder
from mypyc.primitives.dict_ops import (
dict_copy,
dict_del_item,
mapping_has_key,
supports_mapping_protocol,
)
from mypyc.primitives.generic_ops import generic_ssize_t_len_op
from mypyc.primitives.list_ops import (
sequence_get_item,
sequence_get_slice,
supports_sequence_protocol,
)
from mypyc.primitives.misc_ops import fast_isinstance_op, slow_isinstance_op
# From: https://peps.python.org/pep-0634/#class-patterns
MATCHABLE_BUILTINS = {
"builtins.bool",
"builtins.bytearray",
"builtins.bytes",
"builtins.dict",
"builtins.float",
"builtins.frozenset",
"builtins.int",
"builtins.list",
"builtins.set",
"builtins.str",
"builtins.tuple",
}
class MatchVisitor(TraverserVisitor):
builder: IRBuilder
code_block: BasicBlock
next_block: BasicBlock
final_block: BasicBlock
subject: Value
match: MatchStmt
as_pattern: AsPattern | None = None
def __init__(self, builder: IRBuilder, match_node: MatchStmt) -> None:
self.builder = builder
self.code_block = BasicBlock()
self.next_block = BasicBlock()
self.final_block = BasicBlock()
self.match = match_node
self.subject = builder.accept(match_node.subject)
def build_match_body(self, index: int) -> None:
self.builder.activate_block(self.code_block)
guard = self.match.guards[index]
if guard:
self.code_block = BasicBlock()
cond = self.builder.accept(guard)
self.builder.add_bool_branch(cond, self.code_block, self.next_block)
self.builder.activate_block(self.code_block)
self.builder.accept(self.match.bodies[index])
self.builder.goto(self.final_block)
def visit_match_stmt(self, m: MatchStmt) -> None:
for i, pattern in enumerate(m.patterns):
self.code_block = BasicBlock()
self.next_block = BasicBlock()
pattern.accept(self)
self.build_match_body(i)
self.builder.activate_block(self.next_block)
self.builder.goto_and_activate(self.final_block)
def visit_value_pattern(self, pattern: ValuePattern) -> None:
value = self.builder.accept(pattern.expr)
cond = self.builder.binary_op(self.subject, value, "==", pattern.expr.line)
self.bind_as_pattern(value)
self.builder.add_bool_branch(cond, self.code_block, self.next_block)
def visit_or_pattern(self, pattern: OrPattern) -> None:
code_block = self.code_block
next_block = self.next_block
for p in pattern.patterns:
self.code_block = BasicBlock()
self.next_block = BasicBlock()
# Hack to ensure the as pattern is bound to each pattern in the
# "or" pattern, but not every subpattern
backup = self.as_pattern
p.accept(self)
self.as_pattern = backup
self.builder.activate_block(self.code_block)
self.builder.goto(code_block)
self.builder.activate_block(self.next_block)
self.code_block = code_block
self.next_block = next_block
self.builder.goto(self.next_block)
def visit_class_pattern(self, pattern: ClassPattern) -> None:
# TODO: use faster instance check for native classes (while still
# making sure to account for inheritance)
isinstance_op = (
fast_isinstance_op
if self.builder.is_builtin_ref_expr(pattern.class_ref)
else slow_isinstance_op
)
cond = self.builder.primitive_op(
isinstance_op, [self.subject, self.builder.accept(pattern.class_ref)], pattern.line
)
self.builder.add_bool_branch(cond, self.code_block, self.next_block)
self.bind_as_pattern(self.subject, new_block=True)
if pattern.positionals:
if pattern.class_ref.fullname in MATCHABLE_BUILTINS:
self.builder.activate_block(self.code_block)
self.code_block = BasicBlock()
pattern.positionals[0].accept(self)
return
node = pattern.class_ref.node
assert isinstance(node, TypeInfo), node
match_args = extract_dunder_match_args_names(node)
for i, expr in enumerate(pattern.positionals):
self.builder.activate_block(self.code_block)
self.code_block = BasicBlock()
# TODO: use faster "get_attr" method instead when calling on native or
# builtin objects
positional = self.builder.py_get_attr(self.subject, match_args[i], expr.line)
with self.enter_subpattern(positional):
expr.accept(self)
for key, value in zip(pattern.keyword_keys, pattern.keyword_values):
self.builder.activate_block(self.code_block)
self.code_block = BasicBlock()
# TODO: same as above "get_attr" comment
attr = self.builder.py_get_attr(self.subject, key, value.line)
with self.enter_subpattern(attr):
value.accept(self)
def visit_as_pattern(self, pattern: AsPattern) -> None:
if pattern.pattern:
old_pattern = self.as_pattern
self.as_pattern = pattern
pattern.pattern.accept(self)
self.as_pattern = old_pattern
elif pattern.name:
target = self.builder.get_assignment_target(pattern.name)
self.builder.assign(target, self.subject, pattern.line)
self.builder.goto(self.code_block)
def visit_singleton_pattern(self, pattern: SingletonPattern) -> None:
if pattern.value is None:
obj = self.builder.none_object()
elif pattern.value is True:
obj = self.builder.true()
else:
obj = self.builder.false()
cond = self.builder.binary_op(self.subject, obj, "is", pattern.line)
self.builder.add_bool_branch(cond, self.code_block, self.next_block)
def visit_mapping_pattern(self, pattern: MappingPattern) -> None:
is_dict = self.builder.call_c(supports_mapping_protocol, [self.subject], pattern.line)
self.builder.add_bool_branch(is_dict, self.code_block, self.next_block)
keys: list[Value] = []
for key, value in zip(pattern.keys, pattern.values):
self.builder.activate_block(self.code_block)
self.code_block = BasicBlock()
key_value = self.builder.accept(key)
keys.append(key_value)
exists = self.builder.call_c(mapping_has_key, [self.subject, key_value], pattern.line)
self.builder.add_bool_branch(exists, self.code_block, self.next_block)
self.builder.activate_block(self.code_block)
self.code_block = BasicBlock()
item = self.builder.gen_method_call(
self.subject, "__getitem__", [key_value], object_rprimitive, pattern.line
)
with self.enter_subpattern(item):
value.accept(self)
if pattern.rest:
self.builder.activate_block(self.code_block)
self.code_block = BasicBlock()
rest = self.builder.primitive_op(dict_copy, [self.subject], pattern.rest.line)
target = self.builder.get_assignment_target(pattern.rest)
self.builder.assign(target, rest, pattern.rest.line)
for i, key_name in enumerate(keys):
self.builder.call_c(dict_del_item, [rest, key_name], pattern.keys[i].line)
self.builder.goto(self.code_block)
def visit_sequence_pattern(self, seq_pattern: SequencePattern) -> None:
star_index, capture, patterns = prep_sequence_pattern(seq_pattern)
is_list = self.builder.call_c(supports_sequence_protocol, [self.subject], seq_pattern.line)
self.builder.add_bool_branch(is_list, self.code_block, self.next_block)
self.builder.activate_block(self.code_block)
self.code_block = BasicBlock()
actual_len = self.builder.call_c(generic_ssize_t_len_op, [self.subject], seq_pattern.line)
min_len = len(patterns)
is_long_enough = self.builder.binary_op(
actual_len,
self.builder.load_int(min_len),
"==" if star_index is None else ">=",
seq_pattern.line,
)
self.builder.add_bool_branch(is_long_enough, self.code_block, self.next_block)
for i, pattern in enumerate(patterns):
self.builder.activate_block(self.code_block)
self.code_block = BasicBlock()
if star_index is not None and i >= star_index:
current = self.builder.binary_op(
actual_len, self.builder.load_int(min_len - i), "-", pattern.line
)
else:
current = self.builder.load_int(i)
item = self.builder.call_c(sequence_get_item, [self.subject, current], pattern.line)
with self.enter_subpattern(item):
pattern.accept(self)
if capture and star_index is not None:
self.builder.activate_block(self.code_block)
self.code_block = BasicBlock()
capture_end = self.builder.binary_op(
actual_len, self.builder.load_int(min_len - star_index), "-", capture.line
)
rest = self.builder.call_c(
sequence_get_slice,
[self.subject, self.builder.load_int(star_index), capture_end],
capture.line,
)
target = self.builder.get_assignment_target(capture)
self.builder.assign(target, rest, capture.line)
self.builder.goto(self.code_block)
def bind_as_pattern(self, value: Value, new_block: bool = False) -> None:
if self.as_pattern and self.as_pattern.pattern and self.as_pattern.name:
if new_block:
self.builder.activate_block(self.code_block)
self.code_block = BasicBlock()
target = self.builder.get_assignment_target(self.as_pattern.name)
self.builder.assign(target, value, self.as_pattern.pattern.line)
self.as_pattern = None
if new_block:
self.builder.goto(self.code_block)
@contextmanager
def enter_subpattern(self, subject: Value) -> Generator[None]:
old_subject = self.subject
self.subject = subject
yield
self.subject = old_subject
def prep_sequence_pattern(
seq_pattern: SequencePattern,
) -> tuple[int | None, NameExpr | None, list[Pattern]]:
star_index: int | None = None
capture: NameExpr | None = None
patterns: list[Pattern] = []
for i, pattern in enumerate(seq_pattern.patterns):
if isinstance(pattern, StarredPattern):
star_index = i
capture = pattern.capture
else:
patterns.append(pattern)
return star_index, capture, patterns
def extract_dunder_match_args_names(info: TypeInfo) -> list[str]:
ty = info.names.get("__match_args__")
assert ty
match_args_type = get_proper_type(ty.type)
assert isinstance(match_args_type, TupleType), match_args_type
match_args: list[str] = []
for item in match_args_type.items:
proper_item = get_proper_type(item)
match_arg = None
if isinstance(proper_item, Instance) and proper_item.last_known_value:
match_arg = proper_item.last_known_value.value
elif isinstance(proper_item, LiteralType):
match_arg = proper_item.value
assert isinstance(match_arg, str), f"Unrecognized __match_args__ item: {item}"
match_args.append(match_arg)
return match_args

View file

@ -0,0 +1,20 @@
from __future__ import annotations
from mypy.nodes import Expression, Node
from mypy.traverser import ExtendedTraverserVisitor
from mypy.types import AnyType, Type, TypeOfAny
class MissingTypesVisitor(ExtendedTraverserVisitor):
"""AST visitor that can be used to add any missing types as a generic AnyType."""
def __init__(self, types: dict[Expression, Type]) -> None:
super().__init__()
self.types: dict[Expression, Type] = types
def visit(self, o: Node) -> bool:
if isinstance(o, Expression) and o not in self.types:
self.types[o] = AnyType(TypeOfAny.special_form)
# If returns True, will continue to nested nodes.
return True

View file

@ -0,0 +1,216 @@
"""Helpers for dealing with nonlocal control such as 'break' and 'return'.
Model how these behave differently in different contexts.
"""
from __future__ import annotations
from abc import abstractmethod
from typing import TYPE_CHECKING
from mypyc.ir.ops import (
NO_TRACEBACK_LINE_NO,
BasicBlock,
Branch,
Goto,
Integer,
Register,
Return,
SetMem,
Unreachable,
Value,
)
from mypyc.ir.rtypes import object_rprimitive
from mypyc.irbuild.targets import AssignmentTarget
from mypyc.primitives.exc_ops import restore_exc_info_op, set_stop_iteration_value
if TYPE_CHECKING:
from mypyc.irbuild.builder import IRBuilder
class NonlocalControl:
"""ABC representing a stack frame of constructs that modify nonlocal control flow.
The nonlocal control flow constructs are break, continue, and
return, and their behavior is modified by a number of other
constructs. The most obvious is loop, which override where break
and continue jump to, but also `except` (which needs to clear
exc_info when left) and (eventually) finally blocks (which need to
ensure that the finally block is always executed when leaving the
try/except blocks).
"""
@abstractmethod
def gen_break(self, builder: IRBuilder, line: int) -> None:
pass
@abstractmethod
def gen_continue(self, builder: IRBuilder, line: int) -> None:
pass
@abstractmethod
def gen_return(self, builder: IRBuilder, value: Value, line: int) -> None:
pass
class BaseNonlocalControl(NonlocalControl):
"""Default nonlocal control outside any statements that affect it."""
def gen_break(self, builder: IRBuilder, line: int) -> None:
assert False, "break outside of loop"
def gen_continue(self, builder: IRBuilder, line: int) -> None:
assert False, "continue outside of loop"
def gen_return(self, builder: IRBuilder, value: Value, line: int) -> None:
builder.add(Return(value, line))
class LoopNonlocalControl(NonlocalControl):
"""Nonlocal control within a loop."""
def __init__(
self, outer: NonlocalControl, continue_block: BasicBlock, break_block: BasicBlock
) -> None:
self.outer = outer
self.continue_block = continue_block
self.break_block = break_block
def gen_break(self, builder: IRBuilder, line: int) -> None:
builder.add(Goto(self.break_block))
def gen_continue(self, builder: IRBuilder, line: int) -> None:
builder.add(Goto(self.continue_block))
def gen_return(self, builder: IRBuilder, value: Value, line: int) -> None:
self.outer.gen_return(builder, value, line)
class GeneratorNonlocalControl(BaseNonlocalControl):
"""Default nonlocal control in a generator function outside statements."""
def gen_return(self, builder: IRBuilder, value: Value, line: int) -> None:
# Assign an invalid next label number so that the next time
# __next__ is called, we jump to the case in which
# StopIteration is raised.
builder.assign(builder.fn_info.generator_class.next_label_target, Integer(-1), line)
# Raise a StopIteration containing a field for the value that
# should be returned. Before doing so, create a new block
# without an error handler set so that the implicitly thrown
# StopIteration isn't caught by except blocks inside of the
# generator function.
builder.builder.push_error_handler(None)
builder.goto_and_activate(BasicBlock())
# Skip creating a traceback frame when we raise here, because
# we don't care about the traceback frame and it is kind of
# expensive since raising StopIteration is an extremely common
# case. Also we call a special internal function to set
# StopIteration instead of using RaiseStandardError because
# the obvious thing doesn't work if the value is a tuple
# (???).
true, false = BasicBlock(), BasicBlock()
stop_iter_reg = builder.fn_info.generator_class.stop_iter_value_reg
assert stop_iter_reg is not None
builder.add(Branch(stop_iter_reg, true, false, Branch.IS_ERROR))
builder.activate_block(true)
# The default/slow path is to raise a StopIteration exception with
# return value.
builder.call_c(set_stop_iteration_value, [value], NO_TRACEBACK_LINE_NO)
builder.add(Unreachable())
builder.builder.pop_error_handler()
builder.activate_block(false)
# The fast path is to store return value via caller-provided pointer
# instead of raising an exception. This can only be used when the
# caller is a native function.
builder.add(SetMem(object_rprimitive, stop_iter_reg, value))
builder.add(Return(Integer(0, object_rprimitive)))
class CleanupNonlocalControl(NonlocalControl):
"""Abstract nonlocal control that runs some cleanup code."""
def __init__(self, outer: NonlocalControl) -> None:
self.outer = outer
@abstractmethod
def gen_cleanup(self, builder: IRBuilder, line: int) -> None: ...
def gen_break(self, builder: IRBuilder, line: int) -> None:
self.gen_cleanup(builder, line)
self.outer.gen_break(builder, line)
def gen_continue(self, builder: IRBuilder, line: int) -> None:
self.gen_cleanup(builder, line)
self.outer.gen_continue(builder, line)
def gen_return(self, builder: IRBuilder, value: Value, line: int) -> None:
self.gen_cleanup(builder, line)
self.outer.gen_return(builder, value, line)
class TryFinallyNonlocalControl(NonlocalControl):
"""Nonlocal control within try/finally."""
def __init__(self, target: BasicBlock) -> None:
self.target = target
self.ret_reg: None | Register | AssignmentTarget = None
def gen_break(self, builder: IRBuilder, line: int) -> None:
builder.error("break inside try/finally block is unimplemented", line)
def gen_continue(self, builder: IRBuilder, line: int) -> None:
builder.error("continue inside try/finally block is unimplemented", line)
def gen_return(self, builder: IRBuilder, value: Value, line: int) -> None:
if self.ret_reg is None:
if builder.fn_info.is_generator:
self.ret_reg = builder.make_spill_target(builder.ret_types[-1])
else:
self.ret_reg = Register(builder.ret_types[-1])
# assert needed because of apparent mypy bug... it loses track of the union
# and infers the type as object
assert isinstance(self.ret_reg, (Register, AssignmentTarget)), self.ret_reg
builder.assign(self.ret_reg, value, line)
builder.add(Goto(self.target))
class ExceptNonlocalControl(CleanupNonlocalControl):
"""Nonlocal control for except blocks.
Just makes sure that sys.exc_info always gets restored when we leave.
This is super annoying.
"""
def __init__(self, outer: NonlocalControl, saved: Value | AssignmentTarget) -> None:
super().__init__(outer)
self.saved = saved
def gen_cleanup(self, builder: IRBuilder, line: int) -> None:
builder.call_c(restore_exc_info_op, [builder.read(self.saved, line)], line)
class FinallyNonlocalControl(CleanupNonlocalControl):
"""Nonlocal control for finally blocks.
Just makes sure that sys.exc_info always gets restored when we
leave and the return register is decrefed if it isn't null.
"""
def __init__(self, outer: NonlocalControl, saved: Value) -> None:
super().__init__(outer)
self.saved = saved
def gen_cleanup(self, builder: IRBuilder, line: int) -> None:
# Restore the old exc_info
target, cleanup = BasicBlock(), BasicBlock()
builder.add(Branch(self.saved, target, cleanup, Branch.IS_ERROR))
builder.activate_block(cleanup)
builder.call_c(restore_exc_info_op, [self.saved], line)
builder.goto_and_activate(target)

View file

@ -0,0 +1,305 @@
from __future__ import annotations
from mypy.nodes import (
AssignmentStmt,
Block,
Decorator,
DictionaryComprehension,
Expression,
FuncDef,
FuncItem,
GeneratorExpr,
Import,
LambdaExpr,
MemberExpr,
MypyFile,
NameExpr,
Node,
SymbolNode,
Var,
)
from mypy.traverser import ExtendedTraverserVisitor, TraverserVisitor
from mypy.types import Type
from mypyc.errors import Errors
from mypyc.irbuild.missingtypevisitor import MissingTypesVisitor
class _LambdaChecker(TraverserVisitor):
"""Check whether an AST subtree contains a lambda expression."""
found = False
def visit_lambda_expr(self, _o: LambdaExpr) -> None:
self.found = True
def _comprehension_has_lambda(node: GeneratorExpr | DictionaryComprehension) -> bool:
"""Return True if a comprehension body contains a lambda.
Only checks body expressions (left_expr/key/value and conditions),
not the sequences, since sequences are evaluated in the enclosing scope.
"""
checker = _LambdaChecker()
if isinstance(node, GeneratorExpr):
node.left_expr.accept(checker)
else:
node.key.accept(checker)
node.value.accept(checker)
for conds in node.condlists:
for cond in conds:
cond.accept(checker)
return checker.found
class PreBuildVisitor(ExtendedTraverserVisitor):
"""Mypy file AST visitor run before building the IR.
This collects various things, including:
* Determine relationships between nested functions and functions that
contain nested functions
* Find non-local variables (free variables)
* Find property setters
* Find decorators of functions
* Find module import groups
The main IR build pass uses this information.
"""
def __init__(
self,
errors: Errors,
current_file: MypyFile,
decorators_to_remove: dict[FuncDef, list[int]],
types: dict[Expression, Type],
) -> None:
super().__init__()
# Dict from a function to symbols defined directly in the
# function that are used as non-local (free) variables within a
# nested function.
self.free_variables: dict[FuncItem, set[SymbolNode]] = {}
# Intermediate data structure used to find the function where
# a SymbolNode is declared. Initially this may point to a
# function nested inside the function with the declaration,
# but we'll eventually update this to refer to the function
# with the declaration.
self.symbols_to_funcs: dict[SymbolNode, FuncItem] = {}
# Stack representing current function nesting.
self.funcs: list[FuncItem] = []
# All property setters encountered so far.
self.prop_setters: set[FuncDef] = set()
# A map from any function that contains nested functions to
# a set of all the functions that are nested within it.
self.encapsulating_funcs: dict[FuncItem, list[FuncItem]] = {}
# Map nested function to its parent/encapsulating function.
self.nested_funcs: dict[FuncItem, FuncItem] = {}
# Map function to its non-special decorators.
self.funcs_to_decorators: dict[FuncDef, list[Expression]] = {}
# Map function to indices of decorators to remove
self.decorators_to_remove: dict[FuncDef, list[int]] = decorators_to_remove
# A mapping of import groups (a series of Import nodes with
# nothing in between) where each group is keyed by its first
# import node.
self.module_import_groups: dict[Import, list[Import]] = {}
self._current_import_group: Import | None = None
self.errors: Errors = errors
self.current_file: MypyFile = current_file
self.missing_types_visitor = MissingTypesVisitor(types)
# Synthetic FuncDef representing the module scope, created on demand
# when a comprehension at module/class level contains a lambda.
self._module_fitem: FuncDef | None = None
# Counter for generating unique synthetic comprehension scope names.
self._comprehension_counter = 0
# Map comprehension AST nodes to synthetic FuncDefs representing
# their scope (only for comprehensions that contain lambdas).
self.comprehension_to_fitem: dict[GeneratorExpr | DictionaryComprehension, FuncDef] = {}
def visit(self, o: Node) -> bool:
if not isinstance(o, Import):
self._current_import_group = None
return True
def visit_assignment_stmt(self, stmt: AssignmentStmt) -> None:
# These are cases where mypy may not have types for certain expressions,
# but mypyc needs some form type to exist.
if stmt.is_alias_def:
stmt.rvalue.accept(self.missing_types_visitor)
return super().visit_assignment_stmt(stmt)
def visit_block(self, block: Block) -> None:
self._current_import_group = None
super().visit_block(block)
self._current_import_group = None
def visit_decorator(self, dec: Decorator) -> None:
if dec.decorators:
# Only add the function being decorated if there exist
# (ordinary) decorators in the decorator list. Certain
# decorators (such as @property, @abstractmethod) are
# special cased and removed from this list by
# mypy. Functions decorated only by special decorators
# (and property setters) are not treated as decorated
# functions by the IR builder.
if isinstance(dec.decorators[0], MemberExpr) and dec.decorators[0].name == "setter":
# Property setters are not treated as decorated methods.
self.prop_setters.add(dec.func)
else:
decorators_to_store = dec.decorators.copy()
if dec.func in self.decorators_to_remove:
to_remove = self.decorators_to_remove[dec.func]
for i in reversed(to_remove):
del decorators_to_store[i]
# if all of the decorators are removed, we shouldn't treat this as a decorated
# function because there aren't any decorators to apply
if not decorators_to_store:
return
self.funcs_to_decorators[dec.func] = decorators_to_store
super().visit_decorator(dec)
def visit_func_def(self, fdef: FuncDef) -> None:
# TODO: What about overloaded functions?
self.visit_func(fdef)
self.visit_symbol_node(fdef)
def visit_lambda_expr(self, expr: LambdaExpr) -> None:
self.visit_func(expr)
def visit_func(self, func: FuncItem) -> None:
# If there were already functions or lambda expressions
# defined in the function stack, then note the previous
# FuncItem as containing a nested function and the current
# FuncItem as being a nested function.
if self.funcs:
# Add the new func to the set of nested funcs within the
# func at top of the func stack.
self.encapsulating_funcs.setdefault(self.funcs[-1], []).append(func)
# Add the func at top of the func stack as the parent of
# new func.
self.nested_funcs[func] = self.funcs[-1]
self.funcs.append(func)
super().visit_func(func)
self.funcs.pop()
def _visit_comprehension_with_scope(self, o: GeneratorExpr | DictionaryComprehension) -> None:
"""Visit a comprehension that contains lambdas.
Creates a synthetic FuncDef to represent the comprehension's scope,
registers it in the function nesting hierarchy, and traverses the
comprehension body with it on the stack.
"""
pushed_module = False
if not self.funcs:
# At module level: push synthetic module FuncDef.
if self._module_fitem is None:
self._module_fitem = FuncDef("__mypyc_module__")
self._module_fitem.line = 1
self.funcs.append(self._module_fitem)
pushed_module = True
# Create synthetic FuncDef for the comprehension scope.
comprehension_fdef = FuncDef(f"__comprehension_{self._comprehension_counter}__")
self._comprehension_counter += 1
comprehension_fdef.line = o.line
self.comprehension_to_fitem[o] = comprehension_fdef
# Register as nested within enclosing function.
self.encapsulating_funcs.setdefault(self.funcs[-1], []).append(comprehension_fdef)
self.nested_funcs[comprehension_fdef] = self.funcs[-1]
# Push and traverse.
self.funcs.append(comprehension_fdef)
if isinstance(o, GeneratorExpr):
super().visit_generator_expr(o)
else:
super().visit_dictionary_comprehension(o)
self.funcs.pop()
if pushed_module:
self.funcs.pop()
def visit_generator_expr(self, o: GeneratorExpr) -> None:
if _comprehension_has_lambda(o):
self._visit_comprehension_with_scope(o)
else:
super().visit_generator_expr(o)
def visit_dictionary_comprehension(self, o: DictionaryComprehension) -> None:
if _comprehension_has_lambda(o):
self._visit_comprehension_with_scope(o)
else:
super().visit_dictionary_comprehension(o)
def visit_import(self, imp: Import) -> None:
if self._current_import_group is not None:
self.module_import_groups[self._current_import_group].append(imp)
else:
self.module_import_groups[imp] = [imp]
self._current_import_group = imp
super().visit_import(imp)
def visit_name_expr(self, expr: NameExpr) -> None:
if isinstance(expr.node, (Var, FuncDef)):
self.visit_symbol_node(expr.node)
def visit_var(self, var: Var) -> None:
self.visit_symbol_node(var)
def visit_symbol_node(self, symbol: SymbolNode) -> None:
if not self.funcs:
# We are not inside a function and hence do not need to do
# anything regarding free variables.
return
if symbol in self.symbols_to_funcs:
orig_func = self.symbols_to_funcs[symbol]
if self.is_parent(self.funcs[-1], orig_func):
# The function in which the symbol was previously seen is
# nested within the function currently being visited. Thus
# the current function is a better candidate to contain the
# declaration.
self.symbols_to_funcs[symbol] = self.funcs[-1]
# TODO: Remove from the orig_func free_variables set?
self.free_variables.setdefault(self.funcs[-1], set()).add(symbol)
elif self.is_parent(orig_func, self.funcs[-1]):
# The SymbolNode instance has already been visited
# before in a parent function, thus it's a non-local
# symbol.
self.add_free_variable(symbol)
else:
# This is the first time the SymbolNode is being
# visited. We map the SymbolNode to the current FuncDef
# being visited to note where it was first visited.
self.symbols_to_funcs[symbol] = self.funcs[-1]
def is_parent(self, fitem: FuncItem, child: FuncItem) -> bool:
# Check if child is nested within fdef (possibly indirectly
# within multiple nested functions).
if child not in self.nested_funcs:
return False
parent = self.nested_funcs[child]
return parent == fitem or self.is_parent(fitem, parent)
def add_free_variable(self, symbol: SymbolNode) -> None:
# Find the function where the symbol was (likely) first declared,
# and mark is as a non-local symbol within that function.
func = self.symbols_to_funcs[symbol]
self.free_variables.setdefault(func, set()).add(symbol)

View file

@ -0,0 +1,900 @@
"""Prepare for IR transform.
This needs to run after type checking and before generating IR.
For example, construct partially initialized FuncIR and ClassIR
objects for all functions and classes. This allows us to bind
references to functions and classes before we've generated full IR for
functions or classes. The actual IR transform will then populate all
the missing bits, such as function bodies (basic blocks).
Also build a mapping from mypy TypeInfos to ClassIR objects.
"""
from __future__ import annotations
from collections import defaultdict
from collections.abc import Iterable
from typing import Final, NamedTuple
from mypy.build import Graph
from mypy.nodes import (
ARG_STAR,
ARG_STAR2,
CallExpr,
ClassDef,
Decorator,
Expression,
FuncDef,
MemberExpr,
MypyFile,
NameExpr,
OverloadedFuncDef,
RefExpr,
SymbolNode,
TypeInfo,
Var,
)
from mypy.semanal import refers_to_fullname
from mypy.traverser import TraverserVisitor
from mypy.types import Instance, Type, get_proper_type
from mypyc.common import FAST_PREFIX, PROPSET_PREFIX, SELF_NAME, get_id_from_name
from mypyc.crash import catch_errors
from mypyc.errors import Errors
from mypyc.ir.class_ir import ClassIR
from mypyc.ir.func_ir import (
FUNC_CLASSMETHOD,
FUNC_NORMAL,
FUNC_STATICMETHOD,
FuncDecl,
FuncSignature,
RuntimeArg,
)
from mypyc.ir.ops import DeserMaps
from mypyc.ir.rtypes import (
RInstance,
RType,
dict_rprimitive,
none_rprimitive,
object_pointer_rprimitive,
object_rprimitive,
tuple_rprimitive,
)
from mypyc.irbuild.mapper import Mapper
from mypyc.irbuild.util import (
get_func_def,
get_mypyc_attrs,
is_dataclass,
is_extension_class,
is_trait,
)
from mypyc.options import CompilerOptions
from mypyc.sametype import is_same_type
GENERATOR_HELPER_NAME: Final = "__mypyc_generator_helper__"
def build_type_map(
mapper: Mapper,
modules: list[MypyFile],
graph: Graph,
types: dict[Expression, Type],
options: CompilerOptions,
errors: Errors,
) -> None:
# Collect all classes defined in everything we are compiling
classes = []
for module in modules:
module_classes = [node for node in module.defs if isinstance(node, ClassDef)]
classes.extend([(module, cdef) for cdef in module_classes])
# Collect all class mappings so that we can bind arbitrary class name
# references even if there are import cycles.
for module, cdef in classes:
class_ir = ClassIR(
cdef.name,
module.fullname,
is_trait(cdef),
is_abstract=cdef.info.is_abstract,
is_final_class=cdef.info.is_final,
)
class_ir.is_ext_class = is_extension_class(module.path, cdef, errors)
if class_ir.is_ext_class:
class_ir.deletable = cdef.info.deletable_attributes.copy()
# If global optimizations are disabled, turn of tracking of class children
if not options.global_opts:
class_ir.children = None
mapper.type_to_ir[cdef.info] = class_ir
mapper.symbol_fullnames.add(class_ir.fullname)
class_ir.is_enum = cdef.info.is_enum and len(cdef.info.enum_members) > 0
# Populate structural information in class IR for extension classes.
for module, cdef in classes:
with catch_errors(module.path, cdef.line):
if mapper.type_to_ir[cdef.info].is_ext_class:
prepare_class_def(module.path, module.fullname, cdef, errors, mapper, options)
else:
prepare_non_ext_class_def(
module.path, module.fullname, cdef, errors, mapper, options
)
# Prepare implicit attribute accessors as needed if an attribute overrides a property.
for module, cdef in classes:
class_ir = mapper.type_to_ir[cdef.info]
if class_ir.is_ext_class:
prepare_implicit_property_accessors(cdef.info, class_ir, module.fullname, mapper)
# Collect all the functions also. We collect from the symbol table
# so that we can easily pick out the right copy of a function that
# is conditionally defined. This doesn't include nested functions!
for module in modules:
for func in get_module_func_defs(module):
prepare_func_def(module.fullname, None, func, mapper, options)
# TODO: what else?
# Check for incompatible attribute definitions that were not
# flagged by mypy but can't be supported when compiling.
for module, cdef in classes:
class_ir = mapper.type_to_ir[cdef.info]
for attr in class_ir.attributes:
for base_ir in class_ir.mro[1:]:
if attr in base_ir.attributes:
if not is_same_type(class_ir.attributes[attr], base_ir.attributes[attr]):
node = cdef.info.names[attr].node
assert node is not None
kind = "trait" if base_ir.is_trait else "class"
errors.error(
f'Type of "{attr}" is incompatible with '
f'definition in {kind} "{base_ir.name}"',
module.path,
node.line,
)
def is_from_module(node: SymbolNode, module: MypyFile) -> bool:
return node.fullname == module.fullname + "." + node.name
def load_type_map(mapper: Mapper, modules: list[MypyFile], deser_ctx: DeserMaps) -> None:
"""Populate a Mapper with deserialized IR from a list of modules."""
for module in modules:
for node in module.names.values():
if (
isinstance(node.node, TypeInfo)
and is_from_module(node.node, module)
and not node.node.is_newtype
and not node.node.is_named_tuple
and node.node.typeddict_type is None
):
ir = deser_ctx.classes[node.node.fullname]
mapper.type_to_ir[node.node] = ir
mapper.symbol_fullnames.add(node.node.fullname)
mapper.func_to_decl[node.node] = ir.ctor
for module in modules:
for func in get_module_func_defs(module):
func_id = get_id_from_name(func.name, func.fullname, func.line)
mapper.func_to_decl[func] = deser_ctx.functions[func_id].decl
def get_module_func_defs(module: MypyFile) -> Iterable[FuncDef]:
"""Collect all of the (non-method) functions declared in a module."""
for node in module.names.values():
# We need to filter out functions that are imported or
# aliases. The best way to do this seems to be by
# checking that the fullname matches.
if isinstance(node.node, (FuncDef, Decorator, OverloadedFuncDef)) and is_from_module(
node.node, module
):
yield get_func_def(node.node)
def prepare_func_def(
module_name: str,
class_name: str | None,
fdef: FuncDef,
mapper: Mapper,
options: CompilerOptions,
) -> FuncDecl:
kind = (
FUNC_CLASSMETHOD
if fdef.is_class
else (FUNC_STATICMETHOD if fdef.is_static else FUNC_NORMAL)
)
sig = mapper.fdef_to_sig(fdef, options.strict_dunders_typing)
decl = FuncDecl(
fdef.name,
class_name,
module_name,
sig,
kind,
is_generator=fdef.is_generator,
is_coroutine=fdef.is_coroutine,
)
mapper.func_to_decl[fdef] = decl
return decl
def create_generator_class_for_func(
module_name: str, class_name: str | None, fdef: FuncDef, mapper: Mapper, name_suffix: str = ""
) -> ClassIR:
"""For a generator/async function, declare a generator class.
Each generator and async function gets a dedicated class that implements the
generator protocol with generated methods.
"""
assert fdef.is_coroutine or fdef.is_generator
name = "_".join(x for x in [fdef.name, class_name] if x) + "_gen" + name_suffix
cir = ClassIR(name, module_name, is_generated=True, is_final_class=class_name is None)
cir.reuse_freed_instance = True
mapper.fdef_to_generator[fdef] = cir
helper_sig = FuncSignature(
(
RuntimeArg(SELF_NAME, object_rprimitive),
RuntimeArg("type", object_rprimitive),
RuntimeArg("value", object_rprimitive),
RuntimeArg("traceback", object_rprimitive),
RuntimeArg("arg", object_rprimitive),
# If non-NULL, used to store return value instead of raising StopIteration(retv)
RuntimeArg("stop_iter_ptr", object_pointer_rprimitive),
),
object_rprimitive,
)
# The implementation of most generator functionality is behind this magic method.
helper_fn_decl = FuncDecl(GENERATOR_HELPER_NAME, name, module_name, helper_sig, internal=True)
cir.method_decls[helper_fn_decl.name] = helper_fn_decl
return cir
def prepare_method_def(
ir: ClassIR,
module_name: str,
cdef: ClassDef,
mapper: Mapper,
node: FuncDef | Decorator,
options: CompilerOptions,
) -> None:
if isinstance(node, FuncDef):
ir.method_decls[node.name] = prepare_func_def(
module_name, cdef.name, node, mapper, options
)
elif isinstance(node, Decorator):
# TODO: do something about abstract methods here. Currently, they are handled just like
# normal methods.
decl = prepare_func_def(module_name, cdef.name, node.func, mapper, options)
if not node.decorators:
ir.method_decls[node.name] = decl
elif isinstance(node.decorators[0], MemberExpr) and node.decorators[0].name == "setter":
# Make property setter name different than getter name so there are no
# name clashes when generating C code, and property lookup at the IR level
# works correctly.
decl.name = PROPSET_PREFIX + decl.name
decl.is_prop_setter = True
# Making the argument implicitly positional-only avoids unnecessary glue methods
decl.sig.args[1].pos_only = True
ir.method_decls[PROPSET_PREFIX + node.name] = decl
if node.func.is_property:
assert node.func.type, f"Expected return type annotation for property '{node.name}'"
decl.is_prop_getter = True
ir.property_types[node.name] = decl.sig.ret_type
def prepare_fast_path(
ir: ClassIR,
module_name: str,
cdef: ClassDef,
mapper: Mapper,
node: SymbolNode | None,
options: CompilerOptions,
) -> None:
"""Add fast (direct) variants of methods in non-extension classes."""
if ir.is_enum:
# We check that non-empty enums are implicitly final in mypy, so we
# can generate direct calls to enum methods.
if isinstance(node, OverloadedFuncDef):
if node.is_property:
return
node = node.impl
if not isinstance(node, FuncDef):
# TODO: support decorated methods (at least @classmethod and @staticmethod).
return
# The simplest case is a regular or overloaded method without decorators. In this
# case we can generate practically identical IR method body, but with a signature
# suitable for direct calls (usual non-extension class methods are converted to
# callable classes, and thus have an extra __mypyc_self__ argument).
name = FAST_PREFIX + node.name
sig = mapper.fdef_to_sig(node, options.strict_dunders_typing)
decl = FuncDecl(name, cdef.name, module_name, sig, FUNC_NORMAL)
ir.method_decls[name] = decl
return
def is_valid_multipart_property_def(prop: OverloadedFuncDef) -> bool:
# Checks to ensure supported property decorator semantics
if len(prop.items) != 2:
return False
getter = prop.items[0]
setter = prop.items[1]
return (
isinstance(getter, Decorator)
and isinstance(setter, Decorator)
and getter.func.is_property
and len(setter.decorators) == 1
and isinstance(setter.decorators[0], MemberExpr)
and setter.decorators[0].name == "setter"
)
def can_subclass_builtin(builtin_base: str) -> bool:
# BaseException and dict are special cased.
return builtin_base in (
(
"builtins.Exception",
"builtins.LookupError",
"builtins.IndexError",
"builtins.Warning",
"builtins.UserWarning",
"builtins.ValueError",
"builtins.object",
)
)
def prepare_class_def(
path: str,
module_name: str,
cdef: ClassDef,
errors: Errors,
mapper: Mapper,
options: CompilerOptions,
) -> None:
"""Populate the interface-level information in a class IR.
This includes attribute and method declarations, and the MRO, among other things, but
method bodies are generated in a later pass.
"""
ir = mapper.type_to_ir[cdef.info]
info = cdef.info
attrs, attrs_lines = get_mypyc_attrs(cdef, path, errors)
if attrs.get("allow_interpreted_subclasses") is True:
ir.allow_interpreted_subclasses = True
if attrs.get("serializable") is True:
# Supports copy.copy and pickle (including subclasses)
ir._serializable = True
if attrs.get("acyclic") is True:
ir.is_acyclic = True
free_list_len = attrs.get("free_list_len")
if free_list_len is not None:
line = attrs_lines["free_list_len"]
if ir.is_trait:
errors.error('"free_list_len" can\'t be used with traits', path, line)
if ir.allow_interpreted_subclasses:
errors.error(
'"free_list_len" can\'t be used in a class that allows interpreted subclasses',
path,
line,
)
if free_list_len == 1:
ir.reuse_freed_instance = True
else:
errors.error(f'Unsupported value for "free_list_len": {free_list_len}', path, line)
# Check for subclassing from builtin types
for cls in info.mro:
# Special case exceptions and dicts
# XXX: How do we handle *other* things??
if cls.fullname == "builtins.BaseException":
ir.builtin_base = "PyBaseExceptionObject"
elif cls.fullname == "builtins.dict":
ir.builtin_base = "PyDictObject"
elif cls.fullname.startswith("builtins."):
if not can_subclass_builtin(cls.fullname):
# Note that if we try to subclass a C extension class that
# isn't in builtins, bad things will happen and we won't
# catch it here! But this should catch a lot of the most
# common pitfalls.
errors.error(
"Inheriting from most builtin types is unimplemented", path, cdef.line
)
errors.note(
"Potential workaround: @mypy_extensions.mypyc_attr(native_class=False)",
path,
cdef.line,
)
errors.note(
"https://mypyc.readthedocs.io/en/stable/native_classes.html#defining-non-native-classes",
path,
cdef.line,
)
# Set up the parent class
bases = [mapper.type_to_ir[base.type] for base in info.bases if base.type in mapper.type_to_ir]
if len(bases) > 1 and any(not c.is_trait for c in bases) and bases[0].is_trait:
# If the first base is a non-trait, don't ever error here. While it is correct
# to error if a trait comes before the next non-trait base (e.g. non-trait, trait,
# non-trait), it's pointless, confusing noise from the bigger issue: multiple
# inheritance is *not* supported.
errors.error("Non-trait base must appear first in parent list", path, cdef.line)
ir.traits = [c for c in bases if c.is_trait]
mro = [] # All mypyc base classes
base_mro = [] # Non-trait mypyc base classes
for cls in info.mro:
if cls not in mapper.type_to_ir:
if cls.fullname != "builtins.object":
ir.inherits_python = True
continue
base_ir = mapper.type_to_ir[cls]
if not base_ir.is_trait:
base_mro.append(base_ir)
mro.append(base_ir)
if cls.defn.removed_base_type_exprs or not base_ir.is_ext_class:
ir.inherits_python = True
base_idx = 1 if not ir.is_trait else 0
if len(base_mro) > base_idx:
ir.base = base_mro[base_idx]
ir.mro = mro
ir.base_mro = base_mro
prepare_methods_and_attributes(cdef, ir, path, module_name, errors, mapper, options)
prepare_init_method(cdef, ir, module_name, mapper)
for base in bases:
if base.children is not None:
base.children.append(ir)
if is_dataclass(cdef):
ir.is_augmented = True
def prepare_methods_and_attributes(
cdef: ClassDef,
ir: ClassIR,
path: str,
module_name: str,
errors: Errors,
mapper: Mapper,
options: CompilerOptions,
) -> None:
"""Populate attribute and method declarations."""
info = cdef.info
for name, node in info.names.items():
# Currently all plugin generated methods are dummies and not included.
if node.plugin_generated:
continue
if isinstance(node.node, Var):
assert node.node.type, "Class member %s missing type" % name
if not node.node.is_classvar and name not in ("__slots__", "__deletable__"):
attr_rtype = mapper.type_to_rtype(node.node.type)
if ir.is_trait and attr_rtype.error_overlap:
# Traits don't have attribute definedness bitmaps, so use
# property accessor methods to access attributes that need them.
# We will generate accessor implementations that use the class bitmap
# for any concrete subclasses.
add_getter_declaration(ir, name, attr_rtype, module_name)
add_setter_declaration(ir, name, attr_rtype, module_name)
ir.attributes[name] = attr_rtype
elif isinstance(node.node, (FuncDef, Decorator)):
prepare_method_def(ir, module_name, cdef, mapper, node.node, options)
elif isinstance(node.node, OverloadedFuncDef):
# Handle case for property with both a getter and a setter
if node.node.is_property:
if is_valid_multipart_property_def(node.node):
for item in node.node.items:
prepare_method_def(ir, module_name, cdef, mapper, item, options)
else:
errors.error("Unsupported property decorator semantics", path, cdef.line)
# Handle case for regular function overload
else:
if not node.node.impl:
errors.error(
"Overloads without implementation are not supported", path, cdef.line
)
else:
prepare_method_def(ir, module_name, cdef, mapper, node.node.impl, options)
if ir.builtin_base:
ir.attributes.clear()
def prepare_implicit_property_accessors(
info: TypeInfo, ir: ClassIR, module_name: str, mapper: Mapper
) -> None:
concrete_attributes = set()
for base in ir.base_mro:
for name, attr_rtype in base.attributes.items():
concrete_attributes.add(name)
add_property_methods_for_attribute_if_needed(
info, ir, name, attr_rtype, module_name, mapper
)
for base in ir.mro[1:]:
if base.is_trait:
for name, attr_rtype in base.attributes.items():
if name not in concrete_attributes:
add_property_methods_for_attribute_if_needed(
info, ir, name, attr_rtype, module_name, mapper
)
def add_property_methods_for_attribute_if_needed(
info: TypeInfo,
ir: ClassIR,
attr_name: str,
attr_rtype: RType,
module_name: str,
mapper: Mapper,
) -> None:
"""Add getter and/or setter for attribute if defined as property in a base class.
Only add declarations. The body IR will be synthesized later during irbuild.
"""
for base in info.mro[1:]:
if base in mapper.type_to_ir:
base_ir = mapper.type_to_ir[base]
n = base.names.get(attr_name)
if n is None:
continue
node = n.node
if isinstance(node, Decorator) and node.name not in ir.method_decls:
# Defined as a read-only property in base class/trait
add_getter_declaration(ir, attr_name, attr_rtype, module_name)
elif isinstance(node, OverloadedFuncDef) and is_valid_multipart_property_def(node):
# Defined as a read-write property in base class/trait
add_getter_declaration(ir, attr_name, attr_rtype, module_name)
add_setter_declaration(ir, attr_name, attr_rtype, module_name)
elif base_ir.is_trait and attr_rtype.error_overlap:
add_getter_declaration(ir, attr_name, attr_rtype, module_name)
add_setter_declaration(ir, attr_name, attr_rtype, module_name)
def add_getter_declaration(
ir: ClassIR, attr_name: str, attr_rtype: RType, module_name: str
) -> None:
self_arg = RuntimeArg("self", RInstance(ir), pos_only=True)
sig = FuncSignature([self_arg], attr_rtype)
decl = FuncDecl(attr_name, ir.name, module_name, sig, FUNC_NORMAL)
decl.is_prop_getter = True
decl.implicit = True # Triggers synthesization
ir.method_decls[attr_name] = decl
ir.property_types[attr_name] = attr_rtype # TODO: Needed??
def add_setter_declaration(
ir: ClassIR, attr_name: str, attr_rtype: RType, module_name: str
) -> None:
self_arg = RuntimeArg("self", RInstance(ir), pos_only=True)
value_arg = RuntimeArg("value", attr_rtype, pos_only=True)
sig = FuncSignature([self_arg, value_arg], none_rprimitive)
setter_name = PROPSET_PREFIX + attr_name
decl = FuncDecl(setter_name, ir.name, module_name, sig, FUNC_NORMAL)
decl.is_prop_setter = True
decl.implicit = True # Triggers synthesization
ir.method_decls[setter_name] = decl
def check_matching_args(init_sig: FuncSignature, new_sig: FuncSignature) -> bool:
num_init_args = len(init_sig.args) - init_sig.num_bitmap_args
num_new_args = len(new_sig.args) - new_sig.num_bitmap_args
if num_init_args != num_new_args:
return False
for idx in range(1, num_init_args):
init_arg = init_sig.args[idx]
new_arg = new_sig.args[idx]
if init_arg.type != new_arg.type:
return False
if init_arg.kind != new_arg.kind:
return False
return True
def prepare_init_method(cdef: ClassDef, ir: ClassIR, module_name: str, mapper: Mapper) -> None:
# Set up a constructor decl
init_node = cdef.info["__init__"].node
new_node: SymbolNode | None = None
new_symbol = cdef.info.get("__new__")
# We are only interested in __new__ method defined in a user-defined class,
# so we ignore it if it comes from a builtin type. It's usually builtins.object
# but could also be builtins.type for metaclasses so we detect the prefix which
# matches both.
if new_symbol and new_symbol.fullname and not new_symbol.fullname.startswith("builtins."):
new_node = new_symbol.node
if isinstance(new_node, (Decorator, OverloadedFuncDef)):
new_node = get_func_def(new_node)
if not ir.is_trait and not ir.builtin_base and isinstance(init_node, FuncDef):
init_sig = mapper.fdef_to_sig(init_node, True)
args_match = True
if isinstance(new_node, FuncDef):
new_sig = mapper.fdef_to_sig(new_node, True)
args_match = check_matching_args(init_sig, new_sig)
defining_ir = mapper.type_to_ir.get(init_node.info)
# If there is a nontrivial __init__ that wasn't defined in an
# extension class, we need to make the constructor take *args,
# **kwargs so it can call tp_init.
if (
(
defining_ir is None
or not defining_ir.is_ext_class
or cdef.info["__init__"].plugin_generated
)
and init_node.info.fullname != "builtins.object"
) or not args_match:
init_sig = FuncSignature(
[
init_sig.args[0],
RuntimeArg("args", tuple_rprimitive, ARG_STAR),
RuntimeArg("kwargs", dict_rprimitive, ARG_STAR2),
],
init_sig.ret_type,
)
last_arg = len(init_sig.args) - init_sig.num_bitmap_args
ctor_sig = FuncSignature(init_sig.args[1:last_arg], RInstance(ir))
ir.ctor = FuncDecl(cdef.name, None, module_name, ctor_sig)
mapper.func_to_decl[cdef.info] = ir.ctor
def prepare_non_ext_class_def(
path: str,
module_name: str,
cdef: ClassDef,
errors: Errors,
mapper: Mapper,
options: CompilerOptions,
) -> None:
ir = mapper.type_to_ir[cdef.info]
info = cdef.info
for node in info.names.values():
if isinstance(node.node, (FuncDef, Decorator)):
prepare_method_def(ir, module_name, cdef, mapper, node.node, options)
elif isinstance(node.node, OverloadedFuncDef):
# Handle case for property with both a getter and a setter
if node.node.is_property:
if not is_valid_multipart_property_def(node.node):
errors.error("Unsupported property decorator semantics", path, cdef.line)
for item in node.node.items:
prepare_method_def(ir, module_name, cdef, mapper, item, options)
# Handle case for regular function overload
else:
prepare_method_def(ir, module_name, cdef, mapper, get_func_def(node.node), options)
prepare_fast_path(ir, module_name, cdef, mapper, node.node, options)
if any(cls in mapper.type_to_ir and mapper.type_to_ir[cls].is_ext_class for cls in info.mro):
errors.error(
"Non-extension classes may not inherit from extension classes", path, cdef.line
)
RegisterImplInfo = tuple[TypeInfo, FuncDef]
class SingledispatchInfo(NamedTuple):
singledispatch_impls: dict[FuncDef, list[RegisterImplInfo]]
decorators_to_remove: dict[FuncDef, list[int]]
def find_singledispatch_register_impls(
modules: list[MypyFile], errors: Errors
) -> SingledispatchInfo:
visitor = SingledispatchVisitor(errors)
for module in modules:
visitor.current_path = module.path
module.accept(visitor)
return SingledispatchInfo(visitor.singledispatch_impls, visitor.decorators_to_remove)
class SingledispatchVisitor(TraverserVisitor):
current_path: str
def __init__(self, errors: Errors) -> None:
super().__init__()
# Map of main singledispatch function to list of registered implementations
self.singledispatch_impls: defaultdict[FuncDef, list[RegisterImplInfo]] = defaultdict(list)
# Map of decorated function to the indices of any decorators to remove
self.decorators_to_remove: dict[FuncDef, list[int]] = {}
self.errors: Errors = errors
self.func_stack_depth = 0
def visit_func_def(self, o: FuncDef) -> None:
self.func_stack_depth += 1
super().visit_func_def(o)
self.func_stack_depth -= 1
def visit_decorator(self, dec: Decorator) -> None:
if dec.decorators:
decorators_to_store = dec.decorators.copy()
decorators_to_remove: list[int] = []
# the index of the last non-register decorator before finding a register decorator
# when going through decorators from top to bottom
last_non_register: int | None = None
for i, d in enumerate(decorators_to_store):
impl = get_singledispatch_register_call_info(d, dec.func)
if impl is not None:
if self.func_stack_depth > 0:
self.errors.error(
"Registering nested functions not supported", self.current_path, d.line
)
self.singledispatch_impls[impl.singledispatch_func].append(
(impl.dispatch_type, dec.func)
)
decorators_to_remove.append(i)
if last_non_register is not None:
# found a register decorator after a non-register decorator, which we
# don't support because we'd have to make a copy of the function before
# calling the decorator so that we can call it later, which complicates
# the implementation for something that is probably not commonly used
self.errors.error(
"Calling decorator after registering function not supported",
self.current_path,
decorators_to_store[last_non_register].line,
)
else:
if refers_to_fullname(d, "functools.singledispatch"):
if self.func_stack_depth > 0:
self.errors.error(
"Nested singledispatch functions not supported",
self.current_path,
d.line,
)
decorators_to_remove.append(i)
# make sure that we still treat the function as a singledispatch function
# even if we don't find any registered implementations (which might happen
# if all registered implementations are registered dynamically)
self.singledispatch_impls.setdefault(dec.func, [])
last_non_register = i
if decorators_to_remove:
# calling register on a function that tries to dispatch based on type annotations
# raises a TypeError because compiled functions don't have an __annotations__
# attribute
self.decorators_to_remove[dec.func] = decorators_to_remove
super().visit_decorator(dec)
class RegisteredImpl(NamedTuple):
singledispatch_func: FuncDef
dispatch_type: TypeInfo
def get_singledispatch_register_call_info(
decorator: Expression, func: FuncDef
) -> RegisteredImpl | None:
# @fun.register(complex)
# def g(arg): ...
if (
isinstance(decorator, CallExpr)
and len(decorator.args) == 1
and isinstance(decorator.args[0], RefExpr)
):
callee = decorator.callee
dispatch_type = decorator.args[0].node
if not isinstance(dispatch_type, TypeInfo):
return None
if isinstance(callee, MemberExpr):
return registered_impl_from_possible_register_call(callee, dispatch_type)
# @fun.register
# def g(arg: int): ...
elif isinstance(decorator, MemberExpr):
# we don't know if this is a register call yet, so we can't be sure that the function
# actually has arguments
if not func.arguments:
return None
arg_type = get_proper_type(func.arguments[0].variable.type)
if not isinstance(arg_type, Instance):
return None
info = arg_type.type
return registered_impl_from_possible_register_call(decorator, info)
return None
def registered_impl_from_possible_register_call(
expr: MemberExpr, dispatch_type: TypeInfo
) -> RegisteredImpl | None:
if expr.name == "register" and isinstance(expr.expr, NameExpr):
node = expr.expr.node
if isinstance(node, Decorator):
return RegisteredImpl(node.func, dispatch_type)
return None
def adjust_generator_classes_of_methods(mapper: Mapper) -> None:
"""Make optimizations and adjustments to generated generator classes of methods.
This is a separate pass after type map has been built, since we need all classes
to be processed to analyze class hierarchies.
"""
generator_methods = []
for fdef, fn_ir in mapper.func_to_decl.items():
if isinstance(fdef, FuncDef) and (fdef.is_coroutine or fdef.is_generator):
gen_ir = create_generator_class_for_func(
fn_ir.module_name, fn_ir.class_name, fdef, mapper
)
# TODO: We could probably support decorators sometimes (static and class method?)
if not fdef.is_decorated:
name = fn_ir.name
precise_ret_type = True
if fn_ir.class_name is not None:
class_ir = mapper.type_to_ir[fdef.info]
subcls = class_ir.subclasses()
if subcls is None:
# Override could be of a different type, so we can't make assumptions.
precise_ret_type = False
elif class_ir.is_trait:
# Give up on traits. We could possibly have an abstract base class
# for generator return types to make this use precise types.
precise_ret_type = False
else:
for s in subcls:
if name in s.method_decls:
m = s.method_decls[name]
if (
m.is_generator != fn_ir.is_generator
or m.is_coroutine != fn_ir.is_coroutine
):
# Override is of a different kind, and the optimization
# to use a precise generator return type doesn't work.
precise_ret_type = False
else:
class_ir = None
if precise_ret_type:
# Give a more precise type for generators, so that we can optimize
# code that uses them. They return a generator object, which has a
# specific class. Without this, the type would have to be 'object'.
fn_ir.sig.ret_type = RInstance(gen_ir)
if fn_ir.bound_sig:
fn_ir.bound_sig.ret_type = RInstance(gen_ir)
if class_ir is not None:
if class_ir.is_method_final(name):
gen_ir.is_final_class = True
generator_methods.append((name, class_ir, gen_ir))
new_bases = {}
for name, class_ir, gen in generator_methods:
# For generator methods, we need to have subclass generator classes inherit from
# baseclass generator classes when there are overrides to maintain LSP.
base = class_ir.real_base()
if base is not None:
if base.has_method(name):
base_sig = base.method_sig(name)
if isinstance(base_sig.ret_type, RInstance):
base_gen = base_sig.ret_type.class_ir
new_bases[gen] = base_gen
# Add generator inheritance relationships by adjusting MROs.
for deriv, base in new_bases.items():
if base.children is not None:
base.children.append(deriv)
while True:
deriv.mro.append(base)
deriv.base_mro.append(base)
if base not in new_bases:
break
base = new_bases[base]

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,70 @@
from __future__ import annotations
from mypyc.ir.ops import Register, Value
from mypyc.ir.rtypes import RInstance, RType, object_rprimitive
class AssignmentTarget:
"""Abstract base class for assignment targets during IR building."""
type: RType = object_rprimitive
class AssignmentTargetRegister(AssignmentTarget):
"""Register as an assignment target.
This is used for local variables and some temporaries.
"""
def __init__(self, register: Register) -> None:
self.register = register
self.type = register.type
def __repr__(self) -> str:
return f"AssignmentTargetRegister({self.register.name})"
class AssignmentTargetIndex(AssignmentTarget):
"""base[index] as assignment target"""
def __init__(self, base: Value, index: Value) -> None:
self.base = base
self.index = index
# TODO: object_rprimitive won't be right for user-defined classes. Store the
# lvalue type in mypy and use a better type to avoid unneeded boxing.
self.type = object_rprimitive
def __repr__(self) -> str:
return f"AssignmentTargetIndex({self.base!r}, {self.index!r})"
class AssignmentTargetAttr(AssignmentTarget):
"""obj.attr as assignment target"""
def __init__(self, obj: Value, attr: str, can_borrow: bool = False) -> None:
self.obj = obj
self.attr = attr
self.can_borrow = can_borrow
if isinstance(obj.type, RInstance) and obj.type.class_ir.has_attr(attr):
# Native attribute reference
self.obj_type: RType = obj.type
self.type = obj.type.attr_type(attr)
else:
# Python attribute reference
self.obj_type = object_rprimitive
self.type = object_rprimitive
def __repr__(self) -> str:
can_borrow_str = ", can_borrow=True" if self.can_borrow else ""
return f"AssignmentTargetAttr({self.obj!r}.{self.attr}{can_borrow_str})"
class AssignmentTargetTuple(AssignmentTarget):
"""x, ..., y as assignment target"""
def __init__(self, items: list[AssignmentTarget], star_idx: int | None = None) -> None:
self.items = items
self.star_idx = star_idx
def __repr__(self) -> str:
return f"AssignmentTargetTuple({self.items}, {self.star_idx})"

View file

@ -0,0 +1,305 @@
"""Various utilities that don't depend on other modules in mypyc.irbuild."""
from __future__ import annotations
from typing import Any, Final, Literal, TypedDict
from typing_extensions import NotRequired
from mypy.nodes import (
ARG_NAMED,
ARG_NAMED_OPT,
ARG_OPT,
ARG_POS,
GDEF,
ArgKind,
BytesExpr,
CallExpr,
ClassDef,
Decorator,
Expression,
FloatExpr,
FuncDef,
IntExpr,
NameExpr,
OverloadedFuncDef,
RefExpr,
StrExpr,
TupleExpr,
UnaryExpr,
Var,
)
from mypy.semanal import refers_to_fullname
from mypy.types import FINAL_DECORATOR_NAMES
from mypyc.errors import Errors
MYPYC_ATTRS: Final[frozenset[MypycAttr]] = frozenset(
["native_class", "allow_interpreted_subclasses", "serializable", "free_list_len", "acyclic"]
)
DATACLASS_DECORATORS: Final = frozenset(["dataclasses.dataclass", "attr.s", "attr.attrs"])
MypycAttr = Literal[
"native_class", "allow_interpreted_subclasses", "serializable", "free_list_len", "acyclic"
]
class MypycAttrs(TypedDict):
native_class: NotRequired[bool]
allow_interpreted_subclasses: NotRequired[bool]
serializable: NotRequired[bool]
free_list_len: NotRequired[int]
acyclic: NotRequired[bool]
def is_final_decorator(d: Expression) -> bool:
return refers_to_fullname(d, FINAL_DECORATOR_NAMES)
def is_trait_decorator(d: Expression) -> bool:
return isinstance(d, RefExpr) and d.fullname == "mypy_extensions.trait"
def is_trait(cdef: ClassDef) -> bool:
return any(is_trait_decorator(d) for d in cdef.decorators) or cdef.info.is_protocol
def dataclass_decorator_type(d: Expression) -> str | None:
if isinstance(d, RefExpr) and d.fullname in DATACLASS_DECORATORS:
return d.fullname.split(".")[0]
elif (
isinstance(d, CallExpr)
and isinstance(d.callee, RefExpr)
and d.callee.fullname in DATACLASS_DECORATORS
):
name = d.callee.fullname.split(".")[0]
if name == "attr" and "auto_attribs" in d.arg_names:
# Note: the mypy attrs plugin checks that the value of auto_attribs is
# not computed at runtime, so we don't need to perform that check here
auto = d.args[d.arg_names.index("auto_attribs")]
if isinstance(auto, NameExpr) and auto.name == "True":
return "attr-auto"
return name
else:
return None
def is_dataclass_decorator(d: Expression) -> bool:
return dataclass_decorator_type(d) is not None
def is_dataclass(cdef: ClassDef) -> bool:
return any(is_dataclass_decorator(d) for d in cdef.decorators)
# The string values returned by this function are inspected in
# mypyc/lib-rt/misc_ops.c:CPyDataclass_SleightOfHand(...).
def dataclass_type(cdef: ClassDef) -> str | None:
for d in cdef.decorators:
typ = dataclass_decorator_type(d)
if typ is not None:
return typ
return None
def get_mypyc_attr_literal(e: Expression) -> Any:
"""Convert an expression from a mypyc_attr decorator to a value.
Supports a pretty limited range."""
if isinstance(e, (StrExpr, IntExpr, FloatExpr)):
return e.value
elif isinstance(e, RefExpr) and e.fullname == "builtins.True":
return True
elif isinstance(e, RefExpr) and e.fullname == "builtins.False":
return False
elif isinstance(e, RefExpr) and e.fullname == "builtins.None":
return None
elif isinstance(e, IntExpr):
return e.value
return NotImplemented
def get_mypyc_attr_call(d: Expression) -> CallExpr | None:
"""Check if an expression is a call to mypyc_attr and return it if so."""
if (
isinstance(d, CallExpr)
and isinstance(d.callee, RefExpr)
and d.callee.fullname == "mypy_extensions.mypyc_attr"
):
return d
return None
def get_mypyc_attrs(
stmt: ClassDef | Decorator, path: str, errors: Errors
) -> tuple[MypycAttrs, dict[MypycAttr, int]]:
"""Collect all the mypyc_attr attributes on a class definition or a function."""
attrs: MypycAttrs = {}
lines: dict[MypycAttr, int] = {}
def set_mypyc_attr(key: str, value: Any, line: int) -> None:
if key in MYPYC_ATTRS:
attrs[key] = value
lines[key] = line
else:
errors.error(f'"{key}" is not a supported "mypyc_attr"', path, line)
supported_keys = '", "'.join(sorted(MYPYC_ATTRS))
errors.note(f'supported keys: "{supported_keys}"', path, line)
for dec in stmt.decorators:
if d := get_mypyc_attr_call(dec):
line = d.line
for name, arg in zip(d.arg_names, d.args):
if name is None:
if isinstance(arg, StrExpr):
set_mypyc_attr(arg.value, True, line)
else:
errors.error(
'All "mypyc_attr" positional arguments must be string literals.',
path,
line,
)
else:
arg_value = get_mypyc_attr_literal(arg)
set_mypyc_attr(name, arg_value, line)
return attrs, lines
def is_extension_class(path: str, cdef: ClassDef, errors: Errors) -> bool:
# Check for @mypyc_attr(native_class=True/False) decorator.
explicit_native_class = get_explicit_native_class(path, cdef, errors)
# Classes with native_class=False are explicitly marked as non extension.
if explicit_native_class is False:
return False
implicit_extension_class, reason = is_implicit_extension_class(cdef)
# Classes with native_class=True should be extension classes, but they might
# not be able to be due to other reasons. Print an error in that case.
if explicit_native_class is True and not implicit_extension_class:
errors.error(
f"Class is marked as native_class=True but it can't be a native class. {reason}",
path,
cdef.line,
)
return implicit_extension_class
def get_explicit_native_class(path: str, cdef: ClassDef, errors: Errors) -> bool | None:
"""Return value of @mypyc_attr(native_class=True/False) decorator.
Look for a @mypyc_attr decorator with native_class=True/False and return
the value assigned or None if it doesn't exist. Other values are an error.
"""
for d in cdef.decorators:
mypyc_attr_call = get_mypyc_attr_call(d)
if not mypyc_attr_call:
continue
for i, name in enumerate(mypyc_attr_call.arg_names):
if name != "native_class":
continue
arg = mypyc_attr_call.args[i]
if not isinstance(arg, NameExpr):
errors.error("native_class must be used with True or False only", path, cdef.line)
return None
if arg.name == "False":
return False
elif arg.name == "True":
return True
else:
errors.error("native_class must be used with True or False only", path, cdef.line)
return None
return None
def is_implicit_extension_class(cdef: ClassDef) -> tuple[bool, str]:
"""Check if class can be extension class and return a user-friendly reason it can't be one."""
for d in cdef.decorators:
if (
not is_trait_decorator(d)
and not is_dataclass_decorator(d)
and not get_mypyc_attr_call(d)
and not is_final_decorator(d)
):
return (
False,
"Classes that have decorators other than supported decorators"
" can't be native classes.",
)
if cdef.info.typeddict_type:
return False, "TypedDict classes can't be native classes."
if cdef.info.is_named_tuple:
return False, "NamedTuple classes can't be native classes."
if cdef.info.metaclass_type and cdef.info.metaclass_type.type.fullname not in (
"abc.ABCMeta",
"typing.TypingMeta",
"typing.GenericMeta",
):
return (
False,
"Classes with a metaclass other than ABCMeta, TypingMeta or"
" GenericMeta can't be native classes.",
)
return True, ""
def get_func_def(op: FuncDef | Decorator | OverloadedFuncDef) -> FuncDef:
if isinstance(op, OverloadedFuncDef):
assert op.impl
op = op.impl
if isinstance(op, Decorator):
op = op.func
return op
def concrete_arg_kind(kind: ArgKind) -> ArgKind:
"""Find the concrete version of an arg kind that is being passed."""
if kind == ARG_OPT:
return ARG_POS
elif kind == ARG_NAMED_OPT:
return ARG_NAMED
else:
return kind
def is_constant(e: Expression) -> bool:
"""Check whether we allow an expression to appear as a default value.
We don't currently properly support storing the evaluated
values for default arguments and default attribute values, so
we restrict what expressions we allow. We allow literals of
primitives types, None, and references to Final global
variables.
"""
return (
isinstance(e, (StrExpr, BytesExpr, IntExpr, FloatExpr))
or (isinstance(e, UnaryExpr) and e.op == "-" and isinstance(e.expr, (IntExpr, FloatExpr)))
or (isinstance(e, TupleExpr) and all(is_constant(e) for e in e.items))
or (
isinstance(e, RefExpr)
and e.kind == GDEF
and (
e.fullname in ("builtins.True", "builtins.False", "builtins.None")
or (isinstance(e.node, Var) and e.node.is_final)
)
)
)
def bytes_from_str(value: str) -> bytes:
"""Convert a string representing bytes into actual bytes.
This is needed because the literal characters of BytesExpr (the
characters inside b'') are stored in BytesExpr.value, whose type is
'str' not 'bytes'.
"""
return bytes(value, "utf8").decode("unicode-escape").encode("raw-unicode-escape")

View file

@ -0,0 +1,542 @@
"""Generate IR for librt.vecs.vec operations"""
from __future__ import annotations
from typing import TYPE_CHECKING, Final, cast
from mypyc.common import IS_32_BIT_PLATFORM, PLATFORM_SIZE
from mypyc.ir.ops import (
ERR_MAGIC,
ERR_NEVER,
Assign,
BasicBlock,
Branch,
CallC,
ComparisonOp,
DecRef,
GetElement,
GetElementPtr,
Integer,
IntOp,
RaiseStandardError,
Register,
SetElement,
TupleGet,
TupleSet,
Unborrow,
Undef,
Unreachable,
Value,
)
from mypyc.ir.rtypes import (
RInstance,
RPrimitive,
RTuple,
RType,
RUnion,
RVec,
VecNestedBufItem,
bool_rprimitive,
c_pyssize_t_rprimitive,
c_size_t_rprimitive,
int32_rprimitive,
int64_rprimitive,
is_c_py_ssize_t_rprimitive,
is_int32_rprimitive,
is_int64_rprimitive,
is_int_rprimitive,
is_short_int_rprimitive,
object_rprimitive,
optional_value_type,
pointer_rprimitive,
vec_api_by_item_type,
vec_item_type_tags,
)
from mypyc.primitives.registry import builtin_names
if TYPE_CHECKING:
from mypyc.irbuild.ll_builder import LowLevelIRBuilder
def as_platform_int(builder: LowLevelIRBuilder, v: Value, line: int) -> Value:
rtype = v.type
if is_c_py_ssize_t_rprimitive(rtype):
return v
if isinstance(v, Integer):
if is_short_int_rprimitive(rtype) or is_int_rprimitive(rtype):
return Integer(v.value // 2, c_pyssize_t_rprimitive)
return Integer(v.value, c_pyssize_t_rprimitive)
if isinstance(rtype, RPrimitive):
if PLATFORM_SIZE == 8 and is_int64_rprimitive(rtype):
return v
if PLATFORM_SIZE == 4 and is_int32_rprimitive(rtype):
return v
return builder.coerce(v, c_pyssize_t_rprimitive, line)
def vec_create(builder: LowLevelIRBuilder, vtype: RVec, length: int | Value, line: int) -> Value:
if isinstance(length, int):
length = Integer(length, c_pyssize_t_rprimitive)
length = as_platform_int(builder, length, line)
item_type = vtype.item_type
api_name = vec_api_by_item_type.get(item_type)
if api_name is not None:
call = CallC(
f"{api_name}.alloc",
[length, length],
vtype,
False,
False,
error_kind=ERR_MAGIC,
line=line,
)
return builder.add(call)
typeobj, optional, depth = vec_item_type_info(builder, item_type, line)
if typeobj is not None:
typeval: Value
if isinstance(typeobj, Integer):
typeval = typeobj
else:
# Create an integer which will hold the type object * as an integral value.
# Assign implicitly coerces between pointer/integer types.
typeval = Register(pointer_rprimitive)
builder.add(Assign(typeval, typeobj))
if optional:
typeval = builder.add(
IntOp(pointer_rprimitive, typeval, Integer(1, pointer_rprimitive), IntOp.OR)
)
if depth == 0:
call = CallC(
"VecTApi.alloc",
[length, length, typeval],
vtype,
False,
False,
error_kind=ERR_MAGIC,
line=line,
)
return builder.add(call)
else:
call = CallC(
"VecNestedApi.alloc",
[length, length, typeval, Integer(depth, int32_rprimitive)],
vtype,
False,
False,
error_kind=ERR_MAGIC,
line=line,
)
return builder.add(call)
assert False, "unsupported: %s" % vtype
def vec_create_initialized(
builder: LowLevelIRBuilder, vtype: RVec, length: int | Value, init: Value, line: int
) -> Value:
"""Create vec with items initialized to the given value."""
if isinstance(length, int):
length = Integer(length, c_pyssize_t_rprimitive)
length = as_platform_int(builder, length, line)
item_type = vtype.item_type
init = builder.coerce(init, item_type, line)
vec = vec_create(builder, vtype, length, line)
items_start = vec_items(builder, vec)
step = step_size(item_type)
items_end = builder.int_add(items_start, builder.int_mul(length, step))
for_loop = builder.begin_for(
items_start, items_end, Integer(step, c_pyssize_t_rprimitive), signed=False
)
builder.set_mem(for_loop.index, item_type, init)
for_loop.finish()
builder.keep_alive([vec], line)
return vec
def vec_create_from_values(
builder: LowLevelIRBuilder, vtype: RVec, values: list[Value], line: int
) -> Value:
vec = vec_create(builder, vtype, len(values), line)
ptr = vec_items(builder, vec)
item_type = vtype.item_type
step = step_size(item_type)
for value in values:
builder.set_mem(ptr, item_type, value)
ptr = builder.int_add(ptr, step)
builder.keep_alive([vec], line)
return vec
def step_size(item_type: RType) -> int:
if isinstance(item_type, RPrimitive):
return item_type.size
elif isinstance(item_type, RVec):
return PLATFORM_SIZE * 2
else:
return PLATFORM_SIZE
VEC_TYPE_INFO_I64: Final = 2
def vec_item_type_info(
builder: LowLevelIRBuilder, typ: RType, line: int
) -> tuple[Value | None, bool, int]:
if isinstance(typ, RPrimitive) and typ.is_refcounted:
typ, src = builtin_names[typ.name]
return builder.load_address(src, typ), False, 0
elif isinstance(typ, RInstance):
return builder.load_native_type_object(typ.name), False, 0
elif typ in vec_item_type_tags:
return Integer(vec_item_type_tags[typ], c_size_t_rprimitive), False, 0
elif isinstance(typ, RUnion):
non_opt = optional_value_type(typ)
assert non_opt is not None
typeval, _, _ = vec_item_type_info(builder, non_opt, line)
if typeval is not None:
return typeval, True, 0
elif isinstance(typ, RVec):
typeval, optional, depth = vec_item_type_info(builder, typ.item_type, line)
if typeval is not None:
return typeval, optional, depth + 1
return None, False, 0
def vec_len(builder: LowLevelIRBuilder, val: Value) -> Value:
"""Return len(<vec>) as i64."""
len_val = vec_len_native(builder, val)
if IS_32_BIT_PLATFORM:
return builder.coerce(len_val, int64_rprimitive, -1)
return len_val
def vec_len_native(builder: LowLevelIRBuilder, val: Value) -> Value:
"""Return len(<vec>) as platform integer type (32-bit/64-bit)."""
return builder.get_element(val, "len")
def vec_items(builder: LowLevelIRBuilder, vecobj: Value) -> Value:
"""Return pointer to first item in vec's buf.
Safe to call even when buf is NULL (empty vec), since GetElementPtr
uses offsetof-based arithmetic instead of &((T*)p)->field.
"""
vtype = cast(RVec, vecobj.type)
buf = builder.get_element(vecobj, "buf")
return builder.add(GetElementPtr(buf, vtype.buf_type, "items"))
def vec_item_ptr(builder: LowLevelIRBuilder, vecobj: Value, index: Value) -> Value:
items_addr = vec_items(builder, vecobj)
assert isinstance(vecobj.type, RVec)
# TODO: Do we need to care about alignment?
item_type = vecobj.type.item_type
if isinstance(item_type, RPrimitive):
item_size = item_type.size
elif isinstance(item_type, RVec):
item_size = 2 * PLATFORM_SIZE
else:
item_size = object_rprimitive.size
delta = builder.int_mul(index, item_size)
return builder.int_add(items_addr, delta)
def vec_check_and_adjust_index(
builder: LowLevelIRBuilder, lenv: Value, index: Value, line: int
) -> Value:
r = Register(int64_rprimitive)
index = builder.coerce(index, int64_rprimitive, line)
lenv = builder.coerce(lenv, int64_rprimitive, line)
ok, ok2, ok3 = BasicBlock(), BasicBlock(), BasicBlock()
fail, fail2 = BasicBlock(), BasicBlock()
is_less = builder.comparison_op(index, lenv, ComparisonOp.ULT, line)
builder.add_bool_branch(is_less, ok2, fail)
builder.activate_block(fail)
x = builder.int_add(index, lenv)
is_less2 = builder.comparison_op(x, lenv, ComparisonOp.ULT, line)
builder.add_bool_branch(is_less2, ok, fail2)
builder.activate_block(fail2)
# TODO: Include index in exception
builder.add(RaiseStandardError(RaiseStandardError.INDEX_ERROR, None, line))
builder.add(Unreachable())
builder.activate_block(ok)
builder.assign(r, x)
builder.goto(ok3)
builder.activate_block(ok2)
builder.assign(r, index)
builder.goto(ok3)
builder.activate_block(ok3)
return r
def vec_get_item(
builder: LowLevelIRBuilder, base: Value, index: Value, line: int, *, can_borrow: bool = False
) -> Value:
"""Generate inlined vec __getitem__ call.
We inline this, since it's simple but performance-critical.
"""
# TODO: Support more item types
# TODO: Support more index types
len_val = vec_len(builder, base)
index = vec_check_and_adjust_index(builder, len_val, index, line)
return vec_get_item_unsafe(builder, base, index, line, can_borrow=can_borrow)
def vec_get_item_unsafe(
builder: LowLevelIRBuilder, base: Value, index: Value, line: int, *, can_borrow: bool = False
) -> Value:
"""Get vec item, assuming index is non-negative and within bounds."""
assert isinstance(base.type, RVec)
index = as_platform_int(builder, index, line)
vtype = base.type
item_addr = vec_item_ptr(builder, base, index)
result = builder.load_mem(item_addr, vtype.item_type, borrow=can_borrow)
builder.keep_alives.append(base)
return result
def vec_set_item(
builder: LowLevelIRBuilder, base: Value, index: Value, item: Value, line: int
) -> None:
assert isinstance(base.type, RVec)
vtype = base.type
len_val = vec_len(builder, base)
index = vec_check_and_adjust_index(builder, len_val, index, line)
index = builder.coerce(index, c_pyssize_t_rprimitive, line)
item_addr = vec_item_ptr(builder, base, index)
item_type = vtype.item_type
item = builder.coerce(item, item_type, line)
if item_type.is_refcounted:
# Read an unborrowed reference to cause a decref to be
# generated for the old item.
old_item = builder.load_mem(item_addr, item_type, borrow=True)
builder.add(DecRef(old_item))
builder.set_mem(item_addr, item_type, item)
builder.keep_alive([base], line)
def vec_init_item_unsafe(
builder: LowLevelIRBuilder, base: Value, index: Value, item: Value, line: int
) -> None:
assert isinstance(base.type, RVec)
index = as_platform_int(builder, index, line)
vtype = base.type
item_addr = vec_item_ptr(builder, base, index)
item_type = vtype.item_type
item = builder.coerce(item, item_type, line)
builder.set_mem(item_addr, item_type, item)
builder.keep_alive([base], line)
def convert_to_t_ext_item(builder: LowLevelIRBuilder, item: Value) -> Value:
vec_len = builder.add(GetElement(item, "len"))
vec_buf = builder.add(GetElement(item, "buf"))
temp = builder.add(SetElement(Undef(VecNestedBufItem), "len", vec_len))
return builder.add(SetElement(temp, "buf", vec_buf))
def convert_from_t_ext_item(builder: LowLevelIRBuilder, item: Value, vec_type: RVec) -> Value:
"""Convert a value of type VecNestedBufItem to the corresponding RVec value."""
api_name = vec_api_by_item_type.get(vec_type.item_type)
if api_name is not None:
name = f"{api_name}.convert_from_nested"
elif isinstance(vec_type.item_type, RVec):
name = "VecNestedApi.convert_from_nested"
else:
name = "VecTApi.convert_from_nested"
return builder.add(
CallC(
name, [item], vec_type, steals=[True], is_borrowed=False, error_kind=ERR_NEVER, line=-1
)
)
def vec_item_type(builder: LowLevelIRBuilder, item_type: RType, line: int) -> Value:
typeobj, optional, depth = vec_item_type_info(builder, item_type, line)
assert typeobj is not None
if isinstance(typeobj, Integer):
return typeobj
else:
# Create an integer which will hold the type object * as an integral value.
# Assign implicitly coerces between pointer/integer types.
typeval: Value
typeval = Register(pointer_rprimitive)
builder.add(Assign(typeval, typeobj))
if optional:
typeval = builder.add(
IntOp(pointer_rprimitive, typeval, Integer(1, pointer_rprimitive), IntOp.OR)
)
return typeval
def vec_append(builder: LowLevelIRBuilder, vec: Value, item: Value, line: int) -> Value:
vec_type = vec.type
assert isinstance(vec_type, RVec)
item_type = vec_type.item_type
coerced_item = builder.coerce(item, item_type, line)
item_type_arg = []
api_name = vec_api_by_item_type.get(item_type)
if api_name is not None:
name = f"{api_name}.append"
elif vec_type.depth() == 0:
name = "VecTApi.append"
item_type_arg = [vec_item_type(builder, item_type, line)]
else:
coerced_item = convert_to_t_ext_item(builder, coerced_item)
name = "VecNestedApi.append"
call = builder.add(
CallC(
name,
[vec, coerced_item] + item_type_arg,
vec_type,
steals=[True, False] + ([False] if item_type_arg else []),
is_borrowed=False,
error_kind=ERR_MAGIC,
line=line,
)
)
if vec_type.depth() > 0:
builder.keep_alive([item], line)
return call
def vec_pop(builder: LowLevelIRBuilder, base: Value, index: Value, line: int) -> Value:
assert isinstance(base.type, RVec)
vec_type = base.type
item_type = vec_type.item_type
index = as_platform_int(builder, index, line)
api_name = vec_api_by_item_type.get(item_type)
if api_name is not None:
name = f"{api_name}.pop"
elif vec_type.depth() == 0:
name = "VecTApi.pop"
else:
name = "VecNestedApi.pop"
# Nested vecs return a generic vec struct.
item_type = VecNestedBufItem
result = builder.add(
CallC(
name,
[base, index],
RTuple([vec_type, item_type]),
steals=[True, False],
is_borrowed=False,
error_kind=ERR_MAGIC,
line=line,
)
)
if vec_type.depth() > 0:
orig = result
x = builder.add(TupleGet(result, 0, borrow=True))
x = builder.add(Unborrow(x))
y = builder.add(TupleGet(result, 1, borrow=True))
y = builder.add(Unborrow(y))
assert isinstance(vec_type.item_type, RVec)
z = convert_from_t_ext_item(builder, y, vec_type.item_type)
result = builder.add(TupleSet([x, z], line))
builder.keep_alive([orig], line, steal=True)
return result
def vec_remove(builder: LowLevelIRBuilder, vec: Value, item: Value, line: int) -> Value:
assert isinstance(vec.type, RVec)
vec_type = vec.type
item_type = vec_type.item_type
coerced_item = builder.coerce(item, item_type, line)
if item_type in vec_api_by_item_type:
name = f"{vec_api_by_item_type[item_type]}.remove"
elif vec_type.depth() == 0:
name = "VecTApi.remove"
else:
coerced_item = convert_to_t_ext_item(builder, coerced_item)
name = "VecNestedApi.remove"
call = builder.add(
CallC(
name,
[vec, coerced_item],
vec_type,
steals=[True, False],
is_borrowed=False,
error_kind=ERR_MAGIC,
line=line,
)
)
if vec_type.depth() > 0:
builder.keep_alive([item], line)
return call
def vec_contains(builder: LowLevelIRBuilder, vec: Value, target: Value, line: int) -> Value:
assert isinstance(vec.type, RVec)
vec_type = vec.type
item_type = vec_type.item_type
target = builder.coerce(target, item_type, line)
step = step_size(item_type)
len_val = vec_len_native(builder, vec)
items_start = vec_items(builder, vec)
items_end = builder.int_add(items_start, builder.int_mul(len_val, step))
true, end = BasicBlock(), BasicBlock()
for_loop = builder.begin_for(
items_start, items_end, Integer(step, c_pyssize_t_rprimitive), signed=False
)
item = builder.load_mem(for_loop.index, item_type, borrow=True)
comp = builder.binary_op(item, target, "==", line)
false = BasicBlock()
builder.add(Branch(comp, true, false, Branch.BOOL))
builder.activate_block(false)
for_loop.finish()
builder.keep_alive([vec], line)
res = Register(bool_rprimitive)
builder.assign(res, Integer(0, bool_rprimitive))
builder.goto(end)
builder.activate_block(true)
builder.assign(res, Integer(1, bool_rprimitive))
builder.goto_and_activate(end)
return res
def vec_slice(
builder: LowLevelIRBuilder, vec: Value, begin: Value, end: Value, line: int
) -> Value:
assert isinstance(vec.type, RVec)
vec_type = vec.type
item_type = vec_type.item_type
begin = builder.coerce(begin, int64_rprimitive, line)
end = builder.coerce(end, int64_rprimitive, line)
api_name = vec_api_by_item_type.get(item_type)
if api_name is not None:
name = f"{api_name}.slice"
elif vec_type.depth() == 0:
name = "VecTApi.slice"
else:
name = "VecNestedApi.slice"
call = CallC(
name,
[vec, begin, end],
vec_type,
steals=[False, False, False],
is_borrowed=False,
error_kind=ERR_MAGIC,
line=line,
)
return builder.add(call)

View file

@ -0,0 +1,414 @@
"""Dispatcher used when transforming a mypy AST to the IR form.
mypyc.irbuild.builder and mypyc.irbuild.main are closely related.
"""
from __future__ import annotations
from typing import NoReturn
from mypy.nodes import (
AssertStmt,
AssertTypeExpr,
AssignmentExpr,
AssignmentStmt,
AwaitExpr,
Block,
BreakStmt,
BytesExpr,
CallExpr,
CastExpr,
ClassDef,
ComparisonExpr,
ComplexExpr,
ConditionalExpr,
ContinueStmt,
Decorator,
DelStmt,
DictExpr,
DictionaryComprehension,
EllipsisExpr,
EnumCallExpr,
ExpressionStmt,
FloatExpr,
ForStmt,
FuncDef,
GeneratorExpr,
GlobalDecl,
IfStmt,
Import,
ImportAll,
ImportFrom,
IndexExpr,
IntExpr,
LambdaExpr,
ListComprehension,
ListExpr,
MatchStmt,
MemberExpr,
MypyFile,
NamedTupleExpr,
NameExpr,
NewTypeExpr,
NonlocalDecl,
OperatorAssignmentStmt,
OpExpr,
OverloadedFuncDef,
ParamSpecExpr,
PassStmt,
PromoteExpr,
RaiseStmt,
ReturnStmt,
RevealExpr,
SetComprehension,
SetExpr,
SliceExpr,
StarExpr,
StrExpr,
SuperExpr,
TemplateStrExpr,
TempNode,
TryStmt,
TupleExpr,
TypeAliasExpr,
TypeAliasStmt,
TypeApplication,
TypedDictExpr,
TypeFormExpr,
TypeVarExpr,
TypeVarTupleExpr,
UnaryExpr,
Var,
WhileStmt,
WithStmt,
YieldExpr,
YieldFromExpr,
)
from mypyc.ir.ops import Value
from mypyc.irbuild.builder import IRBuilder, IRVisitor, UnsupportedException
from mypyc.irbuild.classdef import transform_class_def
from mypyc.irbuild.expression import (
transform_assignment_expr,
transform_bytes_expr,
transform_call_expr,
transform_comparison_expr,
transform_complex_expr,
transform_conditional_expr,
transform_dict_expr,
transform_dictionary_comprehension,
transform_ellipsis,
transform_float_expr,
transform_generator_expr,
transform_index_expr,
transform_int_expr,
transform_list_comprehension,
transform_list_expr,
transform_member_expr,
transform_name_expr,
transform_op_expr,
transform_set_comprehension,
transform_set_expr,
transform_slice_expr,
transform_str_expr,
transform_super_expr,
transform_tuple_expr,
transform_unary_expr,
)
from mypyc.irbuild.function import (
transform_decorator,
transform_func_def,
transform_lambda_expr,
transform_overloaded_func_def,
)
from mypyc.irbuild.statement import (
transform_assert_stmt,
transform_assignment_stmt,
transform_await_expr,
transform_block,
transform_break_stmt,
transform_continue_stmt,
transform_del_stmt,
transform_expression_stmt,
transform_for_stmt,
transform_if_stmt,
transform_import,
transform_import_all,
transform_import_from,
transform_match_stmt,
transform_operator_assignment_stmt,
transform_raise_stmt,
transform_return_stmt,
transform_try_stmt,
transform_type_alias_stmt,
transform_while_stmt,
transform_with_stmt,
transform_yield_expr,
transform_yield_from_expr,
)
class IRBuilderVisitor(IRVisitor):
"""Mypy node visitor that dispatches to node transform implementations.
This class should have no non-trivial logic.
This visitor is separated from the rest of code to improve modularity and
to avoid import cycles.
This is based on the visitor pattern
(https://en.wikipedia.org/wiki/Visitor_pattern).
"""
# This gets passed to all the implementations and contains all the
# state and many helpers. The attribute is initialized outside
# this class since this class and IRBuilder form a reference loop.
builder: IRBuilder
def visit_mypy_file(self, mypyfile: MypyFile) -> None:
assert False, "use transform_mypy_file instead"
def visit_class_def(self, cdef: ClassDef) -> None:
transform_class_def(self.builder, cdef)
def visit_import(self, node: Import) -> None:
transform_import(self.builder, node)
def visit_import_from(self, node: ImportFrom) -> None:
transform_import_from(self.builder, node)
def visit_import_all(self, node: ImportAll) -> None:
transform_import_all(self.builder, node)
def visit_func_def(self, fdef: FuncDef) -> None:
transform_func_def(self.builder, fdef)
def visit_overloaded_func_def(self, o: OverloadedFuncDef) -> None:
transform_overloaded_func_def(self.builder, o)
def visit_decorator(self, dec: Decorator) -> None:
transform_decorator(self.builder, dec)
def visit_block(self, block: Block) -> None:
transform_block(self.builder, block)
# Statements
def visit_expression_stmt(self, stmt: ExpressionStmt) -> None:
transform_expression_stmt(self.builder, stmt)
def visit_return_stmt(self, stmt: ReturnStmt) -> None:
transform_return_stmt(self.builder, stmt)
self.builder.mark_block_unreachable()
def visit_assignment_stmt(self, stmt: AssignmentStmt) -> None:
transform_assignment_stmt(self.builder, stmt)
def visit_operator_assignment_stmt(self, stmt: OperatorAssignmentStmt) -> None:
transform_operator_assignment_stmt(self.builder, stmt)
def visit_if_stmt(self, stmt: IfStmt) -> None:
transform_if_stmt(self.builder, stmt)
def visit_while_stmt(self, stmt: WhileStmt) -> None:
transform_while_stmt(self.builder, stmt)
def visit_for_stmt(self, stmt: ForStmt) -> None:
transform_for_stmt(self.builder, stmt)
def visit_break_stmt(self, stmt: BreakStmt) -> None:
transform_break_stmt(self.builder, stmt)
self.builder.mark_block_unreachable()
def visit_continue_stmt(self, stmt: ContinueStmt) -> None:
transform_continue_stmt(self.builder, stmt)
self.builder.mark_block_unreachable()
def visit_raise_stmt(self, stmt: RaiseStmt) -> None:
transform_raise_stmt(self.builder, stmt)
self.builder.mark_block_unreachable()
def visit_try_stmt(self, stmt: TryStmt) -> None:
transform_try_stmt(self.builder, stmt)
def visit_with_stmt(self, stmt: WithStmt) -> None:
transform_with_stmt(self.builder, stmt)
def visit_pass_stmt(self, stmt: PassStmt) -> None:
pass
def visit_assert_stmt(self, stmt: AssertStmt) -> None:
transform_assert_stmt(self.builder, stmt)
def visit_del_stmt(self, stmt: DelStmt) -> None:
transform_del_stmt(self.builder, stmt)
def visit_global_decl(self, stmt: GlobalDecl) -> None:
# Pure declaration -- no runtime effect
pass
def visit_nonlocal_decl(self, stmt: NonlocalDecl) -> None:
# Pure declaration -- no runtime effect
pass
def visit_match_stmt(self, stmt: MatchStmt) -> None:
transform_match_stmt(self.builder, stmt)
def visit_type_alias_stmt(self, stmt: TypeAliasStmt) -> None:
transform_type_alias_stmt(self.builder, stmt)
# Expressions
def visit_name_expr(self, expr: NameExpr) -> Value:
return transform_name_expr(self.builder, expr)
def visit_member_expr(self, expr: MemberExpr) -> Value:
return transform_member_expr(self.builder, expr)
def visit_super_expr(self, expr: SuperExpr) -> Value:
return transform_super_expr(self.builder, expr)
def visit_call_expr(self, expr: CallExpr) -> Value:
return transform_call_expr(self.builder, expr)
def visit_unary_expr(self, expr: UnaryExpr) -> Value:
return transform_unary_expr(self.builder, expr)
def visit_op_expr(self, expr: OpExpr) -> Value:
return transform_op_expr(self.builder, expr)
def visit_index_expr(self, expr: IndexExpr) -> Value:
return transform_index_expr(self.builder, expr)
def visit_conditional_expr(self, expr: ConditionalExpr) -> Value:
return transform_conditional_expr(self.builder, expr)
def visit_comparison_expr(self, expr: ComparisonExpr) -> Value:
return transform_comparison_expr(self.builder, expr)
def visit_int_expr(self, expr: IntExpr) -> Value:
return transform_int_expr(self.builder, expr)
def visit_float_expr(self, expr: FloatExpr) -> Value:
return transform_float_expr(self.builder, expr)
def visit_complex_expr(self, expr: ComplexExpr) -> Value:
return transform_complex_expr(self.builder, expr)
def visit_str_expr(self, expr: StrExpr) -> Value:
return transform_str_expr(self.builder, expr)
def visit_bytes_expr(self, expr: BytesExpr) -> Value:
return transform_bytes_expr(self.builder, expr)
def visit_ellipsis(self, expr: EllipsisExpr) -> Value:
return transform_ellipsis(self.builder, expr)
def visit_list_expr(self, expr: ListExpr) -> Value:
return transform_list_expr(self.builder, expr)
def visit_tuple_expr(self, expr: TupleExpr) -> Value:
return transform_tuple_expr(self.builder, expr)
def visit_dict_expr(self, expr: DictExpr) -> Value:
return transform_dict_expr(self.builder, expr)
def visit_template_str_expr(self, expr: TemplateStrExpr) -> Value:
self.bail("Template strings are not supported by mypyc", expr.line)
def visit_set_expr(self, expr: SetExpr) -> Value:
return transform_set_expr(self.builder, expr)
def visit_list_comprehension(self, expr: ListComprehension) -> Value:
return transform_list_comprehension(self.builder, expr)
def visit_set_comprehension(self, expr: SetComprehension) -> Value:
return transform_set_comprehension(self.builder, expr)
def visit_dictionary_comprehension(self, expr: DictionaryComprehension) -> Value:
return transform_dictionary_comprehension(self.builder, expr)
def visit_slice_expr(self, expr: SliceExpr) -> Value:
return transform_slice_expr(self.builder, expr)
def visit_generator_expr(self, expr: GeneratorExpr) -> Value:
return transform_generator_expr(self.builder, expr)
def visit_lambda_expr(self, expr: LambdaExpr) -> Value:
return transform_lambda_expr(self.builder, expr)
def visit_yield_expr(self, expr: YieldExpr) -> Value:
return transform_yield_expr(self.builder, expr)
def visit_yield_from_expr(self, o: YieldFromExpr) -> Value:
return transform_yield_from_expr(self.builder, o)
def visit_await_expr(self, o: AwaitExpr) -> Value:
return transform_await_expr(self.builder, o)
def visit_assignment_expr(self, o: AssignmentExpr) -> Value:
return transform_assignment_expr(self.builder, o)
# Constructs that shouldn't ever show up
def visit_enum_call_expr(self, o: EnumCallExpr) -> Value:
assert False, "can't compile analysis-only expressions"
def visit__promote_expr(self, o: PromoteExpr) -> Value:
assert False, "can't compile analysis-only expressions"
def visit_namedtuple_expr(self, o: NamedTupleExpr) -> Value:
assert False, "can't compile analysis-only expressions"
def visit_newtype_expr(self, o: NewTypeExpr) -> Value:
assert False, "can't compile analysis-only expressions"
def visit_temp_node(self, o: TempNode) -> Value:
assert False, "can't compile analysis-only expressions"
def visit_type_alias_expr(self, o: TypeAliasExpr) -> Value:
assert False, "can't compile analysis-only expressions"
def visit_type_application(self, o: TypeApplication) -> Value:
assert False, "can't compile analysis-only expressions"
def visit_type_var_expr(self, o: TypeVarExpr) -> Value:
assert False, "can't compile analysis-only expressions"
def visit_paramspec_expr(self, o: ParamSpecExpr) -> Value:
assert False, "can't compile analysis-only expressions"
def visit_type_var_tuple_expr(self, o: TypeVarTupleExpr) -> Value:
assert False, "can't compile analysis-only expressions"
def visit_typeddict_expr(self, o: TypedDictExpr) -> Value:
assert False, "can't compile analysis-only expressions"
def visit_reveal_expr(self, o: RevealExpr) -> Value:
assert False, "can't compile analysis-only expressions"
def visit_var(self, o: Var) -> None:
assert False, "can't compile Var; should have been handled already?"
def visit_cast_expr(self, o: CastExpr) -> Value:
assert False, "CastExpr should have been handled in CallExpr"
def visit_type_form_expr(self, o: TypeFormExpr) -> Value:
assert False, "TypeFormExpr should have been handled in CallExpr"
def visit_assert_type_expr(self, o: AssertTypeExpr) -> Value:
assert False, "AssertTypeExpr should have been handled in CallExpr"
def visit_star_expr(self, o: StarExpr) -> Value:
assert False, "should have been handled in Tuple/List/Set/DictExpr or CallExpr"
# Helpers
def bail(self, msg: str, line: int) -> NoReturn:
"""Reports an error and aborts compilation up until the last accept() call
(accept() catches the UnsupportedException and keeps on
processing. This allows errors to be non-blocking without always
needing to write handling for them.
"""
self.builder.error(msg, line)
raise UnsupportedException()

View file

@ -0,0 +1,82 @@
"""Compute vtables of native (extension) classes."""
from __future__ import annotations
import itertools
from mypyc.ir.class_ir import ClassIR, VTableEntries, VTableMethod
from mypyc.sametype import is_same_method_signature
def compute_vtable(cls: ClassIR) -> None:
"""Compute the vtable structure for a class."""
if cls.vtable is not None:
return
if not cls.is_generated:
cls.has_dict = any(x.inherits_python for x in cls.mro)
for t in cls.mro[1:]:
# Make sure all ancestors are processed first
compute_vtable(t)
# Merge attributes from traits into the class
if not t.is_trait:
continue
for name, typ in t.attributes.items():
if not cls.is_trait and not any(name in b.attributes for b in cls.base_mro):
cls.attributes[name] = typ
cls.vtable = {}
if cls.base:
assert cls.base.vtable is not None
cls.vtable.update(cls.base.vtable)
cls.vtable_entries = specialize_parent_vtable(cls, cls.base)
# Include the vtable from the parent classes, but handle method overrides.
entries = cls.vtable_entries
all_traits = [t for t in cls.mro if t.is_trait]
for t in [cls] + [t for t in all_traits if t is not cls]:
for fn in itertools.chain(t.methods.values()):
# TODO: don't generate a new entry when we overload without changing the type
if fn == cls.get_method(fn.name, prefer_method=True):
cls.vtable[fn.name] = len(entries)
# If the class contains a glue method referring to itself, that is a
# shadow glue method to support interpreted subclasses.
shadow = cls.glue_methods.get((cls, fn.name))
entries.append(VTableMethod(t, fn.name, fn, shadow))
# Compute vtables for all of the traits that the class implements
if not cls.is_trait:
for trait in all_traits:
compute_vtable(trait)
cls.trait_vtables[trait] = specialize_parent_vtable(cls, trait)
def specialize_parent_vtable(cls: ClassIR, parent: ClassIR) -> VTableEntries:
"""Generate the part of a vtable corresponding to a parent class or trait"""
updated = []
for entry in parent.vtable_entries:
# Find the original method corresponding to this vtable entry.
# (This may not be the method in the entry, if it was overridden.)
orig_parent_method = entry.cls.get_method(entry.name, prefer_method=True)
assert orig_parent_method
method_cls = cls.get_method_and_class(entry.name, prefer_method=True)
if method_cls:
child_method, defining_cls = method_cls
# TODO: emit a wrapper for __init__ that raises or something
if (
is_same_method_signature(orig_parent_method.sig, child_method.sig)
or orig_parent_method.name == "__init__"
):
entry = VTableMethod(entry.cls, entry.name, child_method, entry.shadow_method)
else:
entry = VTableMethod(
entry.cls,
entry.name,
defining_cls.glue_methods[(entry.cls, entry.name)],
entry.shadow_method,
)
updated.append(entry)
return updated