From 862f455406663c9e89ddb0c6a02a97d2aaa38720 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Thu, 10 Sep 2026 11:49:15 +1000 Subject: [PATCH 01/16] native: answer equality, tagged ints, exceptions, with blocks and fields as python does compiled `==` asks the type rather than answering true for identical objects, tagged ints handle the short minimum and literals too wide to tag, the handled exception survives a re-raise and an except clause's class evaluation, a with block looks its methods up on the type, a name's value is released when the name is rebound, an instance made without __init__ has no fields, comparisons used as values keep the object their method returned, a second import hands back the first module object, and a float subclass goes to the interpreted definition. also releases temporaries at their last read, stores closure cells natively and speeds up big tagged ints --- crates/by_build/tests/differential.rs | 3281 ++++++++++++++++- crates/by_build/tests/end_to_end.rs | 4 +- crates/by_codegen_c/src/lib.rs | 1106 +++++- crates/by_ir/src/builder.rs | 1 + crates/by_ir/src/function.rs | 58 +- crates/by_ir/src/ops.rs | 179 +- crates/by_ir/src/print.rs | 93 +- crates/by_ir/src/rtype.rs | 38 + crates/by_ir/src/verify.rs | 260 +- crates/by_irbuild/src/closures.rs | 42 +- crates/by_irbuild/src/comprehension_scopes.rs | 214 ++ crates/by_irbuild/src/generators.rs | 12 +- crates/by_irbuild/src/lib.rs | 1456 +++++++- crates/by_irbuild/src/tests.rs | 329 +- crates/by_opt/src/borrow.rs | 418 ++- crates/by_opt/src/copy_propagation.rs | 8 + crates/by_opt/src/dead_allocations.rs | 193 +- crates/by_opt/src/dict_find.rs | 16 +- crates/by_opt/src/field_proofs.rs | 403 ++ crates/by_opt/src/fold.rs | 8 + crates/by_opt/src/infallible.rs | 16 +- crates/by_opt/src/lib.rs | 25 + crates/by_opt/src/liveness.rs | 122 + crates/by_opt/src/release_temporaries.rs | 774 ++++ crates/by_opt/src/store_moves.rs | 488 +++ crates/by_opt/src/str_append.rs | 93 +- crates/by_opt/src/str_concat_int.rs | 31 +- crates/by_rt/include/by.h | 1535 ++++++-- .../basedpython/development/compilation/ir.md | 2 +- .../development/compilation/runtime.md | 61 + 30 files changed, 10334 insertions(+), 932 deletions(-) create mode 100644 crates/by_irbuild/src/comprehension_scopes.rs create mode 100644 crates/by_opt/src/field_proofs.rs create mode 100644 crates/by_opt/src/liveness.rs create mode 100644 crates/by_opt/src/release_temporaries.rs create mode 100644 crates/by_opt/src/store_moves.rs diff --git a/crates/by_build/tests/differential.rs b/crates/by_build/tests/differential.rs index cac4d15f88..edfc6ecdd1 100644 --- a/crates/by_build/tests/differential.rs +++ b/crates/by_build/tests/differential.rs @@ -1061,6 +1061,32 @@ def mix(a: int, b: int) -> int: ); } +#[test] +fn a_remainder_of_two_shorts_agrees_for_every_sign() { + // `%` floors, so its answer takes the divisor's sign, and it works on the tagged words + // directly. every sign pairing, both ends of the short range, a zero divisor in the + // running interpreter's own words, and operands past the short range on the way out + agree( + "tagmod", + "\ +def fmod(a: int, b: int) -> int: + return a % b + +def by_seven(a: int) -> int: + return a % 7 +", + &[ + "[m.fmod(a, b) for a in (7, -7, 0, 1, -1) for b in (2, -2, 3, -3, 7, -7, 1, -1)]", + "[m.fmod(a, b) for a in (-(2**62), 2**62 - 1) for b in (-(2**62), 2**62 - 1, -1, 1, 3, -3)]", + "[m.by_seven(a) for a in range(-15, 16)]", + "str(_capture(m.fmod, 5, 0))", + "str(_capture(m.fmod, 0, 0))", + "[m.fmod(a, b) for a in (2**70, -(2**70)) for b in (7, -7, 2**65)]", + "[m.fmod(a, b) for a in (7, -7) for b in (2**70, -(2**70))]", + ], + ); +} + #[test] fn arithmetic_that_leaves_the_short_range_agrees() { // a tagged integer holds one bit fewer than a machine word, so an operation whose @@ -1068,7 +1094,8 @@ fn arithmetic_that_leaves_the_short_range_agrees() { // then has to come back through cpython's own arithmetic. the runtime keeps that // half of each operator out of line, so this is the boundary between two pieces of // code rather than two branches of one — including at the very edge, where the - // result is exactly one past what a short can hold + // result is exactly one past what a short can hold. the short range is not + // symmetric, so negating its minimum, or dividing it by -1, is the other way out agree( "shortedge", "\ @@ -1080,8 +1107,20 @@ def sub(a: int, b: int) -> int: def mul(a: int, b: int) -> int: return a * b + +def neg(a: int) -> int: + return -a + +def fdiv(a: int, b: int) -> int: + return a // b + +def fmod(a: int, b: int) -> int: + return a % b ", &[ + "[m.neg(a) for a in (-(2**62), -(2**62) + 1, 2**62 - 1)]", + "[m.fdiv(a, b) for a in (-(2**62), 2**62 - 1) for b in (-1, 1, -2, 2)]", + "[m.fmod(a, b) for a in (-(2**62), 2**62 - 1) for b in (-1, 1, -2, 2)]", "m.add(2**62 - 1, 1)", "m.add(-(2**62), -1)", "m.sub(-(2**62), 1)", @@ -1094,6 +1133,82 @@ def mul(a: int, b: int) -> int: ); } +#[test] +fn a_big_int_against_a_short_one_compares_by_value() { + // a big `int` is held behind a pointer and a small one in the word, so a comparison + // between the two is always between two different representations. every operator, + // both ways round, and both signs: a value too big to be a short is never equal to + // one and is larger or smaller by its sign alone. an `int` subclass is compared by + // its own methods, which is what the last two calls pin — the comparisons there + // answer the opposite of what the numbers would + agree( + "bigcompare", + "\ +def compare(a: int, b: int) -> list[bool]: + return [a < b, a <= b, a > b, a >= b, a == b, a != b] +", + &[ + "[m.compare(a, b) for a in (10**30, -(10**30), 2**62, -(2**62) - 1) for b in (0, 5, -5)]", + "[m.compare(b, a) for a in (10**30, -(10**30), 2**62, -(2**62) - 1) for b in (0, 5, -5)]", + "m.compare(type('Big', (int,), {'__gt__': lambda s, o: False, '__lt__': lambda s, o: True})(10**30), 0)", + "m.compare(0, type('Big', (int,), {'__gt__': lambda s, o: True, '__lt__': lambda s, o: False})(10**30))", + ], + ); +} + +#[test] +fn an_integer_literal_outside_the_short_range_agrees() { + // a short holds one bit fewer than a machine word, so a literal can fit an `i64` + // and still be too wide to tag. such a literal is a real `int` the module builds + // once, wherever it is written: returned, compared against, stepped by, taken as + // a default, or made by folding two short ones. returning it many times over + // shows the frame hands out a reference of its own rather than the module's + agree( + "longliteral", + "\ +def edge() -> int: + return 4611686018427387904 + +def edges() -> list[int]: + return [4611686018427387903, 4611686018427387904, -4611686018427387904, -4611686018427387905, 9223372036854775807] + +def below(a: int) -> bool: + return a < 4611686018427387904 + +def plus(a: int) -> int: + return a + 4611686018427387904 + +def folded() -> int: + return 4611686018427387903 + 1 + +def defaulted(a: int = 4611686018427387904) -> int: + return a + +def boxed() -> object: + return 4611686018427387904 + +def stepped(n: int) -> list[int]: + out: list[int] = [] + for i in range(0, n, 4611686018427387904): + out.append(i) + if len(out) == 3: + break + return out +", + &[ + "m.edge()", + "sum(m.edge() for _ in range(10000))", + "m.edges()", + "[m.below(a) for a in (2**62 - 1, 2**62, 2**62 + 1)]", + "[m.plus(a) for a in (0, -1, -(2**62), 2**62)]", + "m.folded()", + "(m.defaulted(), m.defaulted(1))", + "m.boxed()", + "m.stepped(2**63)", + ], + ); +} + #[test] fn an_operation_that_keeps_leaving_the_short_range_does_not_leak() { // each trip round this loop boxes both operands, calls cpython, and tags the @@ -1139,6 +1254,49 @@ def climb(a: int, n: int) -> int: assert_eq!(out, "True\nstable"); } +#[test] +fn a_result_that_is_not_an_int_is_refused_rather_than_tagged() { + // an operator on a value too big to be a short goes through the object protocol, + // and an `int` subclass may answer it with anything at all. a `-> int` body cannot + // hold a `str`, so that answer has to raise the `TypeError` an unbox of one raises + // — the conversion back used to swallow its own error and tag the `str` as an int. + // this asserts on the compiled leg alone: python hands the `str` back + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = diff_root().join("by_diff_notanint"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +def halve(value: int) -> int: + return value // 2 +"; + if build_source( + source, + "by_diff_notanint", + &toolchain, + &dir, + &Options::default(), + ) + .is_err() + { + eprintln!("skipping: no working C toolchain"); + return; + } + let out = run( + &python, + &dir, + "import by_diff_notanint as m\n\ + assert m.__file__.endswith(('.so', '.pyd')), m.__file__\n\ + Loud = type('Loud', (int,), {'__floordiv__': lambda self, other: 'floordiv override'})\n\ + print(m.halve(10**30) == 5 * 10**29)\n\ + try:\n\ + \x20 print(repr(m.halve(Loud(10**30))))\n\ + except TypeError as error:\n\ + \x20 print(f'TypeError: {error}')\n", + ); + assert_eq!(out, "True\nTypeError: expected int, got str"); +} + #[test] fn floor_division_and_modulo_agree_on_every_sign() { agree( @@ -1658,6 +1816,269 @@ def reassigns(n: int) -> int: ); } +#[test] +fn a_value_nothing_names_is_dropped_as_soon_as_it_is_used() { + // python drops a value no name holds the moment the expression that made it is done + // with it: a file written through `open(p, 'w').write(...)` is closed, and so + // flushed, before the next line reads it back, and a finalizer on a discarded + // result runs before the statement after it. a named local is held until the name + // is rebound, which `a_value_a_name_held_is_dropped_when_the_name_is_rebound` asks + // about, so only the unnamed values are asked about here + agree_python( + "temporaries", + "\ +import os +import tempfile +from collections.abc import Callable + + +def write_then_read() -> str: + fd, path = tempfile.mkstemp() + os.close(fd) + try: + open(path, 'w').write('written') + with open(path) as handle: + return handle.read() + finally: + os.unlink(path) + + +class Fresh: + make: Callable[[], object] + log: list[str] + + def __init__(self, make: Callable[[], object], log: list[str]) -> None: + self.make = make + self.log = log + + def __enter__(self) -> object: + return self.make() + + def __exit__(self, kind: object, value: object, trace: object) -> None: + self.log.append('exit') + + +class Swallow: + def __enter__(self) -> 'Swallow': + return self + + def __exit__(self, kind: object, value: object, trace: object) -> bool: + return True + + +def fresh_enter(make: Callable[[], object], log: list[str]) -> list[str]: + with Fresh(make, log): + log.append('body') + log.append('after') + return list(log) + + +def swallowed(make: Callable[[], BaseException], log: list[str]) -> list[str]: + with Swallow(): + raise make() + log.append('after') + return list(log) + + +def discarded(make: Callable[[], object], log: list[str]) -> list[str]: + make() + log.append('after') + return list(log) +", + &[ + "m.write_then_read()", + "(lambda log: m.fresh_enter(type('T', (), {'__del__': lambda s: log.append('dropped')}), log))([])", + "(lambda log: m.swallowed(type('E', (Exception,), {'__del__': lambda s: log.append('dropped')}), log))([])", + "(lambda log: m.discarded(type('T', (), {'__del__': lambda s: log.append('dropped')}), log))([])", + ], + ); +} + +#[test] +fn a_tuple_nothing_names_is_dropped_as_soon_as_it_is_used() { + // a fixed-length tuple is held as a struct of its elements, and python drops the + // tuple the moment the expression that made it is done with it: a discarded one + // before the next statement, and the rest of one subscripted for a single element + // before that element is used. a tuple drops its elements last to first. the + // element read off `nested(i)[0]` is itself a tuple, held on loan from the outer one, + // so letting go of both at once would free the inner one twice + agree_python( + "tupletemps", + "\ +log: list[str] = [] + + +class Loud: + name: str + + def __init__(self, name: str) -> None: + self.name = name + + def __del__(self) -> None: + log.append('del ' + self.name) + + +def make(i: int) -> tuple[Loud, int]: + return Loud(str(i)), i + + +def pair(i: int) -> tuple[Loud, Loud]: + return Loud('a' + str(i)), Loud('b' + str(i)) + + +def nested(i: int) -> tuple[tuple[Loud, int], Loud]: + return (Loud('in' + str(i)), i), Loud('out' + str(i)) + + +def show(held: Loud) -> None: + log.append('show ' + held.name) + + +def discarded(n: int) -> list[str]: + log.clear() + i = 0 + while i < n: + make(i) + log.append('after ' + str(i)) + i = i + 1 + return list(log) + + +def indexed(n: int) -> list[str]: + log.clear() + i = 0 + while i < n: + part = make(i)[1] + log.append('part ' + str(part)) + i = i + 1 + return list(log) + + +def both_discarded(n: int) -> list[str]: + log.clear() + i = 0 + while i < n: + pair(i) + log.append('after ' + str(i)) + i = i + 1 + return list(log) + + +def one_passed_on(n: int) -> list[str]: + log.clear() + i = 0 + while i < n: + show(pair(i)[0]) + log.append('after ' + str(i)) + i = i + 1 + return list(log) + + +def through_the_inner_tuple(n: int) -> list[str]: + log.clear() + i = 0 + while i < n: + part = nested(i)[0][1] + log.append('part ' + str(part)) + i = i + 1 + return list(log) +", + &[ + "m.discarded(3)", + "m.indexed(3)", + "m.both_discarded(2)", + "m.one_passed_on(2)", + "m.through_the_inner_tuple(2)", + ], + ); +} + +#[test] +fn a_value_stored_into_a_field_lives_as_long_as_the_field() { + // a store that is its value's last read hands the register's reference to the field + // rather than taking one of its own. the field is then the only thing holding the + // value, as it is in python once the expression that made it is done — so rewriting + // the field is what drops it, and a counter kept in one agrees with the interpreter + const SOURCE: &str = "\ +from collections.abc import Callable + + +class Holder: + held: object + depth: int + + def __init__(self) -> None: + self.held = None + self.depth = 0 + + def keep(self, make: Callable[[], object]) -> None: + self.held = make() + + def step(self, k: int) -> int: + self.depth = self.depth + k + return self.depth + + +def swap(make: Callable[[], object], log: list[str]) -> list[str]: + h = Holder() + h.keep(make) + log.append('kept') + h.keep(lambda: None) + log.append('replaced') + return list(log) + + +def count(n: int) -> int: + h = Holder() + i = 0 + while i < n: + h.step(i) + i = i + 1 + return h.depth +"; + agree_python( + "fieldmove", + SOURCE, + &[ + "(lambda log: m.swap(type('T', (), {'__del__': lambda s: log.append('dropped')}), log))([])", + "m.count(1000)", + ], + ); + + // and the stores are the compiled methods' own: an interpreted `Holder` would answer + // the same, so which build answered is asked directly + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = diff_root().join("by_diff_fieldmove_pin"); + let _ = std::fs::remove_dir_all(&dir); + let built = match build_source( + SOURCE, + "by_diff_fieldmove_pin", + &toolchain, + &dir, + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + assert!(built.declined.is_empty(), "declined: {:?}", built.declined); + let out = run( + &python, + &dir, + "import by_diff_fieldmove_pin as m\n\ + print(type(m.Holder.keep).__name__, type(m.Holder.step).__name__, m.count(1000))\n", + ); + assert_eq!(out, "method_descriptor method_descriptor 499500"); +} + #[test] fn a_temporary_object_argument_does_not_crash() { let Some((python, toolchain)) = environment() else { @@ -3783,6 +4204,52 @@ def unmatched(a: int, b: int) -> int: ); } +#[test] +fn an_exception_leaving_a_handler_keeps_the_context_it_was_raised_with() { + // an exception raised inside an `except` block chains onto the one being handled + // at the moment it is raised. leaving the block puts the previous handled exception + // back and lets the new one continue as it is — python's re-raise never chains a + // second time, so the context it picked up where it was raised is the one it keeps + agree_python( + "handledchain", + "\ +def nested() -> None: + try: + raise ValueError('a') + except ValueError: + try: + raise KeyError('b') + except KeyError: + raise IndexError('c') + + +def unmatched() -> None: + try: + raise ValueError('a') + except ValueError: + try: + raise KeyError('b') + except IndexError: + pass + + +def through_finally() -> None: + try: + raise ValueError('a') + except ValueError: + try: + raise KeyError('b') + finally: + pass +", + &[ + "_chain(_capture(m.nested))", + "_chain(_capture(m.unmatched))", + "_chain(_capture(m.through_finally))", + ], + ); +} + /// the operation that would have rebound a name raised, so the name is still bound /// to what it was — and the handler in the same function goes on to read it /// @@ -6122,29 +6589,145 @@ def grow(seed: str, n: int) -> str: assert_eq!(out, "stable"); } -#[test] -fn native_classes_agree() { - agree( - "classes", +/// a `str` subclass defining `__add__`, `__radd__` and `__iadd__`, built in the calling +/// process so that neither leg compiles it +const ADDS: &str = "type('Adds', (str,), {'__add__': lambda s, o: 'ADD', \ + '__radd__': lambda s, o: 'RADD', '__iadd__': lambda s, o: 'IADD'})"; + +#[test] +fn a_str_concatenation_asks_a_str_subclass_for_its_operators() { + // `a + b` over two `str`s is the interpreter's own concatenation only while both + // are exact: a subclass may define `__add__` or `__radd__`, and `a += b` asks the + // left operand's `__iadd__` first. python asks them whichever side the subclass is + // on, in an accumulation as much as in one expression, and in the fused `s + + // str(n)`. an f-string joins its pieces without asking anything, which is the case + // that has to stay as it is + let adds = ADDS; + let radd = "type('RAdd', (str,), {'__radd__': lambda s, o: 'RADD'})"; + let adds_int = "type('AddsInt', (str,), {'__add__': lambda s, o: 42})"; + // each call runs in a process of its own, so nothing has to put `str` back + let with_str = |value: &str| { + format!("(m.__dict__.__setitem__('str', {value}), repr(_capture(m.key, 7)), m.key(7))[1:]") + }; + let calls = [ + format!("m.cat({adds}('x'), 'y')"), + format!("m.cat('x', {adds}('y'))"), + format!("m.lit_plus({adds}('y'))"), + format!("m.acc_right({adds}('y'))"), + format!("m.acc_left({adds}('s'))"), + format!("m.aug({adds}('s'))"), + format!("m.aug_loop({adds}('s'), 3)"), + format!("m.aug_loop({adds}('s'), 0)"), + format!("m.join_all(['a', {adds}('q'), 'b'])"), + format!("(m.chain({adds}('a'), 'b'), m.chain('a', {adds}('b')))"), + format!("type(_capture(m.chain, {adds_int}('a'), 'b')).__name__"), + format!("(m.plus_digits({adds}('k'), 7), m.plus_digits({adds}('k'), 10**30))"), + format!("(m.plus_digits('k', 7), type(m.plus_digits({radd}('k'), 7)).__name__)"), + // the module's own `str` bound to a subclass that answers `__radd__`, and to a + // function answering something that is no `str` at all + with_str(radd), + "(m.__dict__.__setitem__('str', lambda n: 5), repr(_capture(m.key, 7)))[1]".to_string(), + format!("(m.shown({adds}('v')), type(m.shown({adds}('v'))).__name__)"), + "(m.build(4), m.aug_loop('', 0))".to_string(), + ]; + let calls: Vec<&str> = calls.iter().map(String::as_str).collect(); + agree_python( + "strsubadd", "\ -data class Point: - x: int - y: int +def cat(a: str, b: str) -> str: + return a + b - def total(self) -> int: - return self.x + self.y - def scaled(self, k: int) -> int: - return self.total() * k +def lit_plus(b: str) -> str: + return \"w\" + b -data class Named: - label: str - count: int - def shout(self) -> str: - return self.label + \"!\" +def acc_right(b: str) -> str: + out = \"a\" + out = out + b + return out -def make(a: int, b: int) -> object: + +def acc_left(start: str) -> str: + out = start + out = out + \"w\" + return out + + +def aug(start: str) -> str: + out = start + out += \"w\" + return out + + +def aug_loop(start: str, n: int) -> str: + out = start + i = 0 + while i < n: + out += \"w\" + i = i + 1 + return out + + +def join_all(parts: list[str]) -> str: + out = \"\" + for p in parts: + out = out + p + return out + + +def chain(a: str, b: str) -> str: + out = a + \"-\" + b + \".\" + return out + + +def plus_digits(left: str, n: int) -> str: + return left + str(n) + + +def key(n: int) -> str: + return \"k\" + str(n) + + +def build(n: int) -> str: + out = \"\" + i = 0 + while i < n: + out = out + \"word\" + str(i % 10) + \" \" + i = i + 1 + return out + + +def shown(v: str) -> str: + return f\"<{v}>{v}\" +", + &calls, + ); +} + +#[test] +fn native_classes_agree() { + agree( + "classes", + "\ +data class Point: + x: int + y: int + + def total(self) -> int: + return self.x + self.y + + def scaled(self, k: int) -> int: + return self.total() * k + +data class Named: + label: str + count: int + + def shout(self) -> str: + return self.label + \"!\" + +def make(a: int, b: int) -> object: return Point(a, b) ", &[ @@ -6196,6 +6779,91 @@ fn a_native_constructor_checks_its_argument_types() { assert_eq!(out, "TypeError: expected int, got str"); } +#[test] +fn importing_a_compiled_module_again_hands_back_the_module_already_imported() { + // a compiled module keeps its namespace, and every memo of a global in it, in state the + // whole process shares. a second module object would have that state pointed at its own + // namespace, and the first object's compiled code would then read and write the second's + // — so a second import in the same interpreter hands back the module the first one made, + // as a single-phase C extension's does, and an import in another interpreter is refused + let Some((python, toolchain)) = environment() else { + return; + }; + if !supports(&toolchain, (3, 13)) { + return; + } + let dir = diff_root().join("by_diff_reimport"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +_step = 0 +_limit = 0 + + +def total() -> int: + acc = 0 + i = 0 + while i < _limit: + acc = acc + _step + i = i + 1 + return acc + + +def setup() -> None: + global _step, _limit + _step = 15 + _limit = 10 + + +def drop() -> None: + global _step + del _step +"; + let options = Options { + language: by_irbuild::Language::Python, + ..Options::default() + }; + if build_source(source, "by_diff_reimport", &toolchain, &dir, &options).is_err() { + eprintln!("skipping: no working C toolchain"); + return; + } + let out = run( + &python, + &dir, + "import importlib, sys\n\ + import by_diff_reimport as first\n\ + assert first.__file__.endswith('.so'), first.__file__\n\ + first.setup()\n\ + print(first.total())\n\ + del sys.modules['by_diff_reimport']\n\ + import by_diff_reimport as second\n\ + print(second is first, first.total(), second.total())\n\ + importlib.reload(first)\n\ + print(sys.modules['by_diff_reimport'] is first, first.total())\n\ + first._limit = 1\n\ + first.drop()\n\ + print('_step' in vars(first), second.total() if '_step' in vars(second) else 'dropped')\n", + ); + assert_eq!(out, "150\nTrue 150 150\nTrue 150\nFalse dropped"); + // a legacy subinterpreter shares the process, and with it the module's state + let out = run( + &python, + &dir, + "import _interpreters\n\ + import by_diff_reimport as m\n\ + m.setup()\n\ + sub = _interpreters.create('legacy')\n\ + import os\n\ + home = os.path.dirname(m.__file__)\n\ + err = _interpreters.exec(sub, f'import sys; sys.path.insert(0, {home!r}); import by_diff_reimport')\n\ + print(err.type.__name__, 'another interpreter' in err.msg)\n\ + m._step = 2\n\ + print(m.total())\n\ + _interpreters.destroy(sub)\n\ + print(m.total())\n", + ); + assert_eq!(out, "ImportError True\n20\n20"); +} + #[test] fn a_native_class_has_a_fixed_layout() { let Some((python, toolchain)) = environment() else { @@ -6306,6 +6974,141 @@ fn a_class_takes_an_attribute_its_layout_never_had() { ); } +/// classes whose instances' `__dict__` is written to directly +const DICT_WRITES: &str = "\ +class Cell: + def __init__(self) -> None: + self._v = 0 + + @property + def v(self) -> int: + return self._v + + @v.setter + def v(self, given: int) -> None: + self._v = given + + +class Ro: + def __init__(self) -> None: + self._v = 1 + + @property + def v(self) -> int: + return self._v + + +class Ext: + def __init__(self) -> None: + self._v = 0 + + def width(self) -> int: + return self._v + + +class Dropped: + def __init__(self) -> None: + self._v = 0 + + @property + def v(self) -> int: + return self._v + + def drop(self) -> None: + del self._v + + +class ExtSub(Ext): + pass + + +def bump(cell: Cell, n: int) -> int: + i = 0 + while i < n: + cell.v = cell.v + 1 + i = i + 1 + return cell.v +"; + +#[test] +fn writing_into_an_instance_dict_runs_nothing_but_the_write() { + // python's instance `__dict__` is a plain dict: a write through it stores the entry + // and runs no descriptor and no `__setattr__`. a compiled instance keeps the names its + // layout declares in fields, so a key naming one writes the field; any other key is an + // entry in the dict and nothing more — even one a property on the class answers for + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = diff_root().join("by_diff_dictwrites_check"); + let _ = std::fs::remove_dir_all(&dir); + let options = Options { + language: by_irbuild::Language::Python, + ..Options::default() + }; + if build_source( + DICT_WRITES, + "by_diff_dictwrites", + &toolchain, + &dir, + &options, + ) + .is_err() + { + eprintln!("skipping: no working C toolchain"); + return; + } + let out = run( + &python, + &dir, + "import by_diff_dictwrites as m\n\ + print(type(m.Ext.width).__name__, type(m.Cell().__dict__).__name__)\n", + ); + assert_eq!(out, "method_descriptor instance_dict"); + let subclass = "type('S', (m.Ext,), {'__setattr__': lambda s, k, x: s.log.append(k)})"; + let calls = [ + "(lambda c: (c.__dict__.__setitem__('v', 500), m.bump(c, 3), c.__dict__.get('v'), c.v))\ + (m.Cell())" + .to_string(), + "(lambda c: (c.__dict__.update({'v': 41}), c.v, dict(c.__dict__)))(m.Cell())".to_string(), + "(lambda c: (c.__dict__.update(v=40, _v=1), c.v, vars(c)))(m.Cell())".to_string(), + "(lambda c: (c.__dict__.update([('v', 2), ('_v', 3)], w=4), c.v, vars(c)))(m.Cell())" + .to_string(), + "(lambda c: (repr(_capture(c.__dict__.update, [('v', 2), ('_v',)])), vars(c)))(m.Cell())" + .to_string(), + "(lambda c: (c.__dict__.setdefault('v', 9), c.__dict__.setdefault('_v', 9), c.v, \ + vars(c)))(m.Cell())" + .to_string(), + "(lambda c: (c.__dict__.__setitem__('_v', 7), m.bump(c, 3), vars(c)))(m.Cell())" + .to_string(), + "(lambda c: (c.__dict__.__setitem__('v', 5), c.__dict__.__delitem__('v'), c.v, \ + repr(_capture(c.__dict__.__delitem__, 'v'))))(m.Cell())" + .to_string(), + "(lambda c: (c.__dict__.__setitem__('w', 3), c.__dict__.pop('w'), \ + c.__dict__.pop('w', 'gone'), repr(_capture(c.__dict__.pop, 'w')), c.__dict__.pop('v', 0), \ + c.v))(m.Cell())" + .to_string(), + // a field python can take away again, which is one some method of the class deletes + "(lambda c: (c.__dict__.__setitem__('extra', 1), c.__dict__.clear(), hasattr(c, 'extra'), \ + repr(_capture(getattr, c, '_v')), vars(c)))(m.Dropped())" + .to_string(), + "(lambda c: (c.__dict__.pop('_v'), repr(_capture(getattr, c, 'v')), \ + repr(_capture(c.__dict__.__delitem__, '_v')), c.__dict__.__setitem__('_v', 8), c.v))\ + (m.Dropped())" + .to_string(), + "(lambda r: (r.__dict__.__setitem__('v', 5), r.v, r.__dict__.get('v'), \ + r.__dict__.__delitem__('v'), vars(r)))(m.Ro())" + .to_string(), + "(lambda c: (c.__dict__.__ior__({'v': 6, '_v': 2}), c.v, vars(c)))(m.Cell())".to_string(), + format!( + "(lambda s: (s.__dict__.__setitem__('log', []), s.__dict__.__setitem__('zz', 1), \ + s.__dict__.__setitem__('_v', 4), s.zz, s.width(), s.log))\ + ((lambda cls: cls.__new__(cls))({subclass}))" + ), + ]; + let calls: Vec<&str> = calls.iter().map(String::as_str).collect(); + agree_python("dictwrites", DICT_WRITES, &calls); +} + #[test] fn a_class_declaring_slots_keeps_the_bare_layout() { // `__slots__` is python's own way of saying an instance's attributes are exactly the @@ -7633,6 +8436,51 @@ def counted(table: dict[str, int], keys: list[str], passes: int) -> int: return running "; +#[test] +fn a_field_read_in_a_loop_python_can_see_the_bound_of_agrees() { + // a loop whose bound is an `int` is copied once for a bound that fits a machine word, + // and a field read in its body is written once in each copy. the read holds the + // field's value on loan for the add that uses it, which a value past the short range + // exercises — and a body that writes the field between the read and the use must go + // on holding what it read + agree_python( + "unswitchfield", + "\ +class Holder: + count: int + + def __init__(self, count: int) -> None: + self.count = count + + +def total(h: Holder, n: int) -> int: + t = 0 + i = 0 + while i < n: + t = t + h.count + i = i + 1 + return t + + +def rebinding(h: Holder, n: int) -> int: + t = 0 + i = 0 + while i < n: + c = h.count + h.count = c + 1 + t = t + c + i = i + 1 + return t +", + &[ + "m.total(m.Holder(3), 5)", + "m.total(m.Holder(2**70), 3)", + "m.rebinding(m.Holder(2**70), 4)", + "(lambda h: (m.total(h, 1000), __import__('sys').getrefcount(h.count)))(m.Holder(2**70))", + ], + ); +} + #[test] fn a_borrowed_narrowing_does_not_over_release_what_it_narrowed() { // the narrowing no longer retains, so the register is reading through something it @@ -8768,50 +9616,361 @@ def make(n: int) -> object: assert_eq!(out, "[]\n3"); } -#[test] -fn a_raise_out_of_a_try_body_does_not_leak_what_it_wrote() { - // the exception edge is a CFG edge, and the refcount pass used not to follow it - // — so everything the `try` body had written leaked on the exceptional path - let Some((python, toolchain)) = environment() else { - return; - }; - let dir = diff_root().join("by_diff_handlerleak"); - let _ = std::fs::remove_dir_all(&dir); - let source = "\ -def guarded(words: list[str], index: int) -> str: - held = \"held\" + words[0] - try: - return held + words[index] - except IndexError: - return held +/// nested functions a decorator hands back, for the tests below on what one is +const DECORATING_CLOSURES: &str = "\ +import functools +from collections.abc import Callable + + +def tag(f: Callable[[object], str]) -> Callable[..., str]: + def wrapper(self: object = None) -> str: + return f(self) + + return wrapper + + +def strict(f: Callable[[object, int], int]) -> Callable[..., int]: + def wrapper(self: object, k: int) -> int: + return f(self, k) + + return wrapper + + +def double(x: int) -> int: + \"twice as much\" + return x * 2 + + +def logged(f: Callable[[int], int]) -> Callable[[int], int]: + @functools.wraps(f) + def inner(x: int) -> int: + return f(x) + 1 + + return inner "; - if build_source( - source, - "by_diff_handlerleak", - &toolchain, - &dir, - &Options::default(), - ) - .is_err() - { - eprintln!("skipping: no working C toolchain"); - return; - } - let out = run( - &python, - &dir, - "import sys, by_diff_handlerleak as m\n\ - words = ['x' * 40]\n\ - label = words[0]\n\ - before = sys.getrefcount(label)\n\ - for _ in range(20000):\n\ - \x20 m.guarded(words, 9)\n\ - after = sys.getrefcount(label)\n\ - print('stable' if after == before else f'leaked {before}->{after}')\n\ - print(m.guarded(words, 0)[:8])\n", - ); - assert_eq!(out, "stable\nheldxxxx"); -} + +#[test] +fn a_nested_function_installed_on_a_class_binds_the_receiver() { + // python's function is a descriptor, so one stored on a class is handed the instance + // it is reached through. a decorator that returns its own nested function is the + // ordinary way to put one there, and a receiver with a default is where missing the + // binding is silent rather than an arity error + agree_python( + "nestedbind", + DECORATING_CLOSURES, + &[ + "type('Thing', (), {'label': m.tag(lambda s: 'bound' if s is not None else 'UNBOUND')})().label()", + "type('Thing', (), {'add': m.strict(lambda s, k: k + (1 if type(s).__name__ == 'Thing' else 100))})().add(2)", + "type('Thing', (), {'add': m.strict(lambda s, k: k)}).add(None, 5)", + "m.tag(repr)()", + ], + ); +} + +#[test] +fn a_nested_function_can_be_dressed_by_functools_wraps() { + // `functools.wraps` writes the wrapped function's name, qualname and docstring over + // the wrapper's, fills in its `__dict__` and points `__wrapped__` back + agree_python( + "nestedwraps", + DECORATING_CLOSURES, + &[ + "(lambda w: (w(3), w.__name__, w.__qualname__, w.__doc__, w.__wrapped__ is m.double))(m.logged(m.double))", + "(lambda w: (setattr(w, 'label', 7), w.label, sorted(w.__dict__)))(m.tag(repr))", + ], + ); +} + +#[test] +fn a_nested_function_answers_for_where_it_was_written() { + // a nested function is named by the frames it is written in, `counter..step`, + // in its `__qualname__`, its `repr` and the arity errors a call raises. it belongs to + // the module whose body wrote it, and copying one hands back the same function, as + // copying any function does. pickling one refuses because it cannot be found by that + // name, and the refusal names it + agree_python( + "nestedname", + "\ +from collections.abc import Callable + + +def counter(start: int) -> Callable[[int], int]: + total = start + + def step(by: int) -> int: + \"add to the total\" + nonlocal total + total = total + by + return total + + return step + + +def lam(k: int) -> Callable[[int], int]: + return lambda x: x + k + + +def two_deep(n: int) -> Callable[[], Callable[[], int]]: + def middle() -> Callable[[], int]: + def inner() -> int: + return n + + return inner + + return middle + + +class Box: + def method(self, n: int) -> Callable[[], int]: + def inner() -> int: + return n + + return inner +", + &[ + "(lambda f: (f.__name__, f.__qualname__, f.__module__, f.__doc__))(m.counter(0))", + "(lambda f: (f.__name__, f.__qualname__, f(2)))(m.lam(1))", + "(lambda f: (f.__name__, f.__qualname__, f()))(m.two_deep(1)())", + "(lambda f: (f.__name__, f.__qualname__, f()))(m.Box().method(4))", + "repr(m.counter(0)).split(' at ')[0]", + "str(_capture(m.counter(0), 1, 2))", + "str(_capture(m.counter(0)))", + "str(_capture(lambda: m.counter(0)(by=1, other=2)))", + "(lambda copy, f: (copy.copy(f) is f, copy.deepcopy(f) is f, copy.deepcopy([f])[0] is f))(__import__('copy'), m.counter(0))", + "(lambda e: (type(e).__name__, __import__('re').sub(' at 0x[0-9a-f]+', '', str(e))))(_capture(__import__('pickle').dumps, m.counter(0)))", + ], + ); +} + +#[test] +fn a_recursive_closure_and_its_environment_are_collected() { + // a nested function that calls itself reads its own name out of its environment, + // so the environment holds the function and the function holds the environment. + // reference counting never frees that; python's collector does, and so the captured + // value's finalizer runs once nothing else holds either + agree_python( + "nestedcycle", + "\ +from collections.abc import Callable + + +def recursive(held: object) -> Callable[[int], int]: + def again(n: int) -> int: + if n <= 0: + return 0 if held is None else 1 + return again(n - 1) + + return again +", + &[ + "(lambda log: (m.recursive(type('L', (), {'__del__': lambda s: log.append('del')})())(3), gc.collect() >= 0, log))([])", + ], + ); +} + +#[test] +fn a_shared_cell_holding_an_int_agrees() { + // a cell both frames write holds whatever is stored in it, and one only ever given + // an `int` is held as one. it starts unset all the same: a read before the first + // write raises, in either frame, and a zero is a value like any other + agree_python( + "intcell", + "\ +from collections.abc import Callable + + +def counter(start: int) -> Callable[[int], int]: + total = start + + def step(by: int) -> int: + nonlocal total + total = total + by + return total + + return step + + +def early(flag: bool) -> int: + def peek() -> int: + return total + + if flag: + return peek() + total = 0 + return peek() + + +def late(flag: bool) -> Callable[[], int]: + def peek() -> int: + return count + + if flag: + count = 5 + return peek + + +def local_first(flag: bool) -> int: + def bump() -> None: + nonlocal count + count = count + 1 + + if flag: + return count + count = 0 + bump() + return count + + +def written_inside(n: int) -> int: + seen = 0 + + def fill() -> None: + nonlocal seen + seen = n * 2 + + fill() + return seen + + +def parameter(seed: int) -> Callable[[], int]: + def grow() -> int: + nonlocal seed + seed = seed * 10 + return seed + + return grow + + +def two_deep(start: int) -> Callable[[], Callable[[], int]]: + total = start + + def middle() -> Callable[[], int]: + def inner() -> int: + nonlocal total + total = total + 1 + return total + + return inner + + return middle +", + &[ + "(lambda s: (s(3), s(4), s(-7)))(m.counter(0))", + "(lambda s: (s(2**70), s(1)))(m.counter(2**62 - 1))", + "m.early(False)", + "str(_capture(m.early, True))", + "m.late(True)()", + "str(_capture(m.late(False)))", + "str(_capture(m.local_first, True))", + "m.local_first(False)", + "m.written_inside(21)", + "(lambda g: (g(), g()))(m.parameter(7))", + "(lambda g: (g(), g()))(m.parameter(2**61))", + "(lambda f: (f(), f()))(m.two_deep(0)())", + "(lambda f: (f(), f()))(m.two_deep(2**63)())", + ], + ); +} + +#[test] +fn an_unbound_free_variable_is_reported_in_python_s_words() { + agree_python( + "unboundfree", + "\ +def viaif(x: object) -> object: + if x: + a = 1 + + def inner() -> object: + return a + + return inner() +", + &["str(_capture(m.viaif, 0))", "m.viaif(1)"], + ); +} + +#[test] +fn a_nested_function_is_the_compiled_one() { + // `agree` cannot say which build answered. a compiled nested function is a type of + // the runtime's own rather than python's `function`, which is how to tell + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = diff_root().join("by_diff_nestedwhich"); + let _ = std::fs::remove_dir_all(&dir); + let built = match build_source( + DECORATING_CLOSURES, + "by_diff_nestedwhich", + &toolchain, + &dir, + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + assert!(built.declined.is_empty(), "declined: {:?}", built.declined); + let out = run( + &python, + &dir, + "import types, by_diff_nestedwhich as m\n\ + f = m.tag(repr)\n\ + print(type(f).__module__, type(f).__name__, isinstance(f, types.FunctionType))\n", + ); + assert_eq!(out, "by function False"); +} + +#[test] +fn a_raise_out_of_a_try_body_does_not_leak_what_it_wrote() { + // the exception edge is a CFG edge, and the refcount pass used not to follow it + // — so everything the `try` body had written leaked on the exceptional path + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = diff_root().join("by_diff_handlerleak"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +def guarded(words: list[str], index: int) -> str: + held = \"held\" + words[0] + try: + return held + words[index] + except IndexError: + return held +"; + if build_source( + source, + "by_diff_handlerleak", + &toolchain, + &dir, + &Options::default(), + ) + .is_err() + { + eprintln!("skipping: no working C toolchain"); + return; + } + let out = run( + &python, + &dir, + "import sys, by_diff_handlerleak as m\n\ + words = ['x' * 40]\n\ + label = words[0]\n\ + before = sys.getrefcount(label)\n\ + for _ in range(20000):\n\ + \x20 m.guarded(words, 9)\n\ + after = sys.getrefcount(label)\n\ + print('stable' if after == before else f'leaked {before}->{after}')\n\ + print(m.guarded(words, 0)[:8])\n", + ); + assert_eq!(out, "stable\nheldxxxx"); +} #[test] fn a_mutable_capture_agrees() { @@ -9524,6 +10683,132 @@ async def forwards(n: int) -> int: ); } +#[test] +fn a_coroutine_reads_its_variadic_parameters() { + // `*args` and `**kwargs` are parameters like any other, and a coroutine's body reads + // them whether it suspends before doing so or never suspends at all — reached by + // `asyncio.run`, or awaited from another coroutine + agree_python( + "asyncvariadic", + "\ +import asyncio + + +async def counted(*args: object) -> int: + return len(args) + + +async def named(first: int, *rest: int, **extra: int) -> tuple[int, int, list[str]]: + await asyncio.sleep(0) + return (first + sum(rest), len(rest), sorted(extra)) + + +async def awaiting(n: int) -> tuple[int, tuple[int, int, list[str]]]: + a = await counted(n, n, n) + b = await named(n, 1, 2, k=3) + return (a, b) + + +def each(*args: int, **extra: int): + for arg in args: + yield arg + len(extra) +", + &[ + "__import__('asyncio').run(m.counted(1, 2))", + "__import__('asyncio').run(m.counted())", + "__import__('asyncio').run(m.named(1, 2, 3, z=4, a=5))", + "__import__('asyncio').run(m.awaiting(4))", + // a generator keeps its parameters in the same kind of state object + "list(m.each(1, 2, k=0, j=0))", + ], + ); +} + +#[test] +fn a_generator_based_coroutine_can_be_awaited() { + // `types.coroutine` marks a generator as a coroutine python's `await` accepts, with + // no `__await__` of its own. an `async with` awaits what `__aenter__` and `__aexit__` + // answer the same way a written `await` does. the generator function is python's own, + // installed by each call, so that nothing compiled stands in for what is awaited + let install = "m.__dict__.__setitem__('ready', \ + __import__('types').coroutine(lambda value: ((yield), value)[1]))"; + let calls = [ + format!("({install}, __import__('asyncio').run(m.awaited()))[1]"), + format!("({install}, __import__('asyncio').run(m.managed(m.Manager())))[1]"), + ]; + let calls: Vec<&str> = calls.iter().map(String::as_str).collect(); + agree_python( + "genawait", + "\ +from collections.abc import Callable + +ready: Callable[[object], object] + + +class Manager: + def __init__(self) -> None: + self.log: list[str] = [] + + def __aenter__(self): + self.log.append(\"enter\") + return ready(\"entered\") + + def __aexit__(self, *exc: object): + self.log.append(\"exit\") + return ready(False) + + +async def awaited() -> object: + return await ready(7) + + +async def managed(m: Manager) -> tuple[object, list[str]]: + async with m as got: + m.log.append(\"body\") + return (got, m.log) +", + &calls, + ); +} + +#[test] +fn a_context_manager_missing_half_its_protocol_is_refused_before_it_is_entered() { + // python looks up both halves of the protocol before it calls either, so a manager + // with no `__exit__` is refused before its `__enter__` runs, and one with no + // `__enter__` is refused whatever it has. which half is looked up first, and so which + // one the message names, changed in 3.14 + let log = "__import__('builtins').__dict__.setdefault('_entered', [])"; + let only_enter = + format!("type('OnlyEnter', (), {{'__enter__': lambda s: {log}.append('enter')}})"); + let only_exit = "type('OnlyExit', (), {'__exit__': lambda s, *a: None})"; + let only_aenter = + format!("type('OnlyAenter', (), {{'__aenter__': lambda s: {log}.append('aenter')}})"); + let only_aexit = "type('OnlyAexit', (), {'__aexit__': lambda s, *a: None})"; + let calls = [ + format!("(repr(_capture(m.entered, {only_enter}())), {log})"), + format!("(repr(_capture(m.entered, {only_exit}())), {log})"), + "(repr(_capture(m.entered, object())), 0)".to_string(), + format!("(_capture_async(m.aentered, {only_aenter}()), {log})"), + format!("(_capture_async(m.aentered, {only_aexit}()), {log})"), + "_capture_async(m.aentered, object())".to_string(), + ]; + let calls: Vec<&str> = calls.iter().map(String::as_str).collect(); + agree_python( + "withhalves", + "\ +def entered(manager: object) -> str: + with manager: # type: ignore + return \"body\" + + +async def aentered(manager: object) -> str: + async with manager: # type: ignore + return \"body\" +", + &calls, + ); +} + /// `await f(...)` of a coroutine of ours that never suspends is compiled to the call /// /// the object such an await would build is made by the expression, awaited once and @@ -10492,9 +11777,62 @@ def recovering(n: int) -> object: /// /// `close()` asks the same question and answers `None`: there is nothing to unwind #[test] -fn a_throw_into_a_frame_with_no_suspension_raises_at_the_call_site() { +fn a_generator_suspended_in_an_except_block_keeps_its_exception_to_itself() { + // python gives a suspended frame its own handled exception: whoever resumes it sees + // their own again the moment it yields, and it sees its own again the moment it is + // resumed, whatever its caller was handling by then. an exception thrown in at the + // suspension chains onto the frame's, not onto the caller's agree_python( - "throwfinished", + "handledsuspend", + "\ +import sys +from collections.abc import Iterator + + +def gen() -> Iterator[str]: + try: + raise ValueError('inner') + except ValueError: + yield repr(sys.exception()) + yield repr(sys.exception()) + yield repr(sys.exception()) + + +def outside() -> list[str]: + out: list[str] = [] + g = gen() + out.append(next(g)) + out.append(repr(sys.exception())) + try: + raise KeyError('caller') + except KeyError: + out.append(next(g)) + out.append(repr(sys.exception())) + out.append(next(g)) + out.append(repr(sys.exception())) + return out + + +def thrown() -> str: + g = gen() + next(g) + try: + raise KeyError('caller') + except KeyError: + try: + g.throw(IndexError('thrown')) + except IndexError as error: + return repr(error.__context__) + return 'none' +", + &["m.outside()", "m.thrown()"], + ); +} + +#[test] +fn a_throw_into_a_frame_with_no_suspension_raises_at_the_call_site() { + agree_python( + "throwfinished", "\ from typing import Any @@ -10795,6 +12133,123 @@ def swallowed(mgr: object) -> str: ); } +#[test] +fn an_exit_that_is_deciding_sees_the_exception_as_the_one_being_handled() { + // `with` runs its exit inside what python treats as an `except` block for the + // exception unwinding the body: `sys.exception()` answers it there, and an exception + // the exit raises chains onto it, which is what puts "during handling of the above + // exception" into a traceback. an `async with` does the same across an exit that + // suspends, and while it is suspended the exception is its own: another task running + // in the meantime sees nothing being handled + const SOURCE: &str = "\ +import asyncio +import sys + + +class Seeing: + seen: list[str] + + def __init__(self) -> None: + self.seen = [] + + def __enter__(self) -> 'Seeing': + return self + + def __exit__(self, kind: object, value: object, trace: object) -> bool: + self.seen.append(repr(sys.exception())) + return True + + +class Replacing: + def __enter__(self) -> 'Replacing': + return self + + def __exit__(self, kind: object, value: object, trace: object) -> None: + raise KeyError('from exit') + + +class Awaiting: + seen: list[str] + + def __init__(self) -> None: + self.seen = [] + + async def __aenter__(self) -> 'Awaiting': + return self + + async def __aexit__(self, kind: object, value: object, trace: object) -> bool: + self.seen.append(repr(sys.exception())) + await asyncio.sleep(0) + await asyncio.sleep(0) + self.seen.append(repr(sys.exception())) + return True + + +class AsyncReplacing: + async def __aenter__(self) -> 'AsyncReplacing': + return self + + async def __aexit__(self, kind: object, value: object, trace: object) -> None: + raise KeyError('from exit') + + +def seen() -> list[str]: + s = Seeing() + with s: + raise ValueError('body') + s.seen.append(repr(sys.exception())) + return s.seen + + +def context() -> str: + try: + with Replacing(): + raise ValueError('body') + except KeyError as error: + return repr(error.__context__) + return 'none' + + +async def async_seen() -> list[str]: + s = Awaiting() + async with s: + raise ValueError('body') + s.seen.append(repr(sys.exception())) + return s.seen + + +async def async_context() -> str: + try: + async with AsyncReplacing(): + raise ValueError('body') + except KeyError as error: + return repr(error.__context__) + return 'none' + + +async def watcher(seen: list[str]) -> None: + await asyncio.sleep(0) + seen.append(repr(sys.exception())) + + +async def watched() -> list[str]: + seen: list[str] = [] + await asyncio.gather(async_seen(), watcher(seen)) + return seen +"; + agree_python( + "withhandled", + SOURCE, + &[ + "m.seen()", + "m.context()", + "_run(m.async_seen())", + "_run(m.async_context())", + "_run(m.watched())", + ], + ); +} + #[test] fn a_with_block_over_a_class_the_module_declares_reaches_both_halves_of_it() { // a manager the compiler knows the class of reaches `__enter__` and `__exit__` @@ -12098,6 +13553,91 @@ def raised(fn: object) -> str: /// only a getter has no setter at all — the base's is not inherited into it. the emitted /// type has to shadow the base's entry the same way, or a write the interpreted class /// refuses would quietly reach the base's setter +/// a class something extends, whose field an interpreted subclass or the class itself can +/// take over by name +const EXTENDED_FIELD: &str = "\ +class Field: + def __init__(self) -> None: + self.n = 3 + + def doubled(self) -> int: + self.n = self.n * 2 + return self.n + + +class FieldSub(Field): + pass + + +def read_n(f: Field) -> int: + return f.n + + +def write_n(f: Field, x: int) -> None: + f.n = x + + +def bump_n(f: Field) -> int: + f.n += 1 + return f.n +"; + +#[test] +fn a_field_of_a_class_something_extends_is_read_by_name_where_the_name_is_taken_over() { + // a field is a descriptor on the class, and a class that can be extended can have that + // name answered by something else: a property on an interpreted subclass, an override + // of `__getattribute__` or `__setattr__`, or a property put on the class itself after + // import. a read or a write at the field's offset is only what python does while none + // of those has happened, and every other receiver goes through the attribute + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = diff_root().join("by_diff_extfield_check"); + let _ = std::fs::remove_dir_all(&dir); + let options = Options { + language: by_irbuild::Language::Python, + ..Options::default() + }; + if build_source( + EXTENDED_FIELD, + "by_diff_extfield", + &toolchain, + &dir, + &options, + ) + .is_err() + { + eprintln!("skipping: no working C toolchain"); + return; + } + let out = run( + &python, + &dir, + "import by_diff_extfield as m\n\ + print(type(m.Field.doubled).__name__)\n", + ); + assert_eq!(out, "method_descriptor"); + let property = "type('S', (m.Field,), {'n': property(lambda s: 77, lambda s, x: None)})"; + let getattribute = "type('S', (m.Field,), {'__getattribute__': \ + lambda s, k: 55 if k == 'n' else object.__getattribute__(s, k)})"; + let setattr = "type('S', (m.Field,), {'__setattr__': \ + lambda s, k, x: object.__setattr__(s, k, x + 100)})"; + let calls = [ + "(m.read_n(m.Field()), m.bump_n(m.Field()), m.Field().doubled())".to_string(), + "(lambda f: (m.write_n(f, 9), m.read_n(f), f.n))(m.FieldSub())".to_string(), + format!("(lambda s: (m.write_n(s, 9), m.read_n(s), s.n, m.bump_n(s)))({property}())"), + format!("(lambda s: (m.read_n(s), s.doubled()))({property}())"), + format!("(lambda s: (m.read_n(s), s.n))({getattribute}())"), + format!("(lambda s: (m.write_n(s, 1), s.n, m.bump_n(s), s.doubled()))({setattr}())"), + // the class itself, changed after import + "(lambda f: (setattr(m.Field, 'n', property(lambda s: 42, lambda s, x: None)), \ + m.read_n(f), m.bump_n(f), f.doubled()))(m.Field())" + .to_string(), + ]; + let calls: Vec<&str> = calls.iter().map(String::as_str).collect(); + agree_python("extfield", EXTENDED_FIELD, &calls); +} + #[test] fn a_subclass_overriding_one_half_of_a_property_agrees() { agree_python( @@ -14068,6 +15608,124 @@ fn a_subclass_that_appends_nothing_is_the_compiled_type() { ); } +/// a class that adds no field of its own, reaching one its base declares, beside a class +/// that has to fall back +/// +/// `Restated` writes and reads `notes`, which `Held` declares. it adds nothing of its +/// own, so it is laid out with no fields at all — its instances are `Held`'s, at `Held`'s +/// offsets — and the attribute has to be named against `Held` to be reached at all. +/// +/// naming it against `Restated` instead cost far more than the load. the write went out +/// as `PyObject_SetAttr` over a boxed receiver, which is the shape +/// `answers_for_its_classes` hunts for, and a module holding one keeps whatever +/// whole-module answer it already gave: nothing in it may fall back on its own. so +/// `Refused` — which cannot be built from a spec, its storage sitting past a heap base — +/// took `Held` and `Restated` down with it at import, and the module answered every one +/// of its own definitions from the interpreted source. `logging.handlers` is the stdlib's +/// version of this, and gave up all fourteen of its classes to one write of +/// `self.closeOnError` in `DatagramHandler.__init__` +const WRITES_A_BASE_S_FIELD: &str = "\ +from subprocess import SubprocessError + + +class Refused(SubprocessError): + def __init__(self, tag): + super().__init__(tag) + self.tag = tag + + def read(self): + return self.tag + + +class Held(list): + def __init__(self, *args): + super().__init__(*args) + self.notes = [] + + def note_count(self): + return len(self.notes) + + +class Restated(Held): + def __init__(self, *args): + super().__init__(*args) + self.notes = ['restated'] + + def first_note(self): + return self.notes[0] + + def note(self, value): + self.notes = self.notes + [value] + return self.note_count() +"; + +#[test] +fn a_subclass_write_to_a_base_s_field_agrees() { + agree_python( + "basefieldwrite", + WRITES_A_BASE_S_FIELD, + &[ + "(list(m.Restated([1, 2])), m.Restated([1, 2]).notes, m.Restated([1]).first_note())", + "(m.Restated([1]).note('x'), m.Restated([1]).note_count(), m.Held([1]).notes)", + "(m.Held([3]).note_count(), isinstance(m.Restated([1]), m.Held))", + // written from outside the family, where the receiver is not `self` + "[(r.notes, r.note_count()) for r in [m.Restated([1])] if setattr(r, 'notes', ['a', 'b']) is None]", + "(m.Refused('t').read(), m.Refused('t').args)", + ], + ); +} + +#[test] +fn a_subclass_write_to_a_base_s_field_keeps_the_rest_of_the_module() { + // the behaviour above is the same either way — the base publishes a descriptor over + // the field, so the dynamic write reached it too. what moved is which build answers, + // and only `type(...).__name__` says: `Refused` falls back on its own, and `Held` and + // `Restated` stand where the whole module used to give itself up with it + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = diff_root().join("by_diff_basefieldwrite_t"); + let _ = std::fs::remove_dir_all(&dir); + let built = match build_source( + WRITES_A_BASE_S_FIELD, + "by_diff_basefieldwrite_t", + &toolchain, + &dir, + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + // nothing declines at compile time: `Refused` is turned down by the import, where the + // spec meets the heap base it would have to append to + assert!(built.declined.is_empty(), "declined: {:?}", built.declined); + let out = run( + &python, + &dir, + "import by_diff_basefieldwrite_t as m\n\ + print(type(m.Held.__dict__['note_count']).__name__,\n\ + \x20 type(m.Restated.__dict__['first_note']).__name__,\n\ + \x20 type(m.Restated.__dict__['note']).__name__)\n\ + print(type(m.Refused.__dict__['read']).__name__)\n\ + print(m.Held.__basicsize__ == m.Restated.__basicsize__)\n\ + print(m.Restated([1]).notes, m.Restated([1]).first_note())\n", + ); + assert_eq!( + out, + "method_descriptor method_descriptor method_descriptor\n\ + function\n\ + True\n\ + ['restated'] restated" + ); +} + /// a class whose storage would sit past a **heap** base, beside classes whose base is one /// python allocates itself /// @@ -20888,6 +22546,64 @@ def other_array(xs: list[float], ys: list[float]) -> float: ); } +#[test] +fn a_float_subclass_handed_to_a_float_parameter_reaches_the_interpreted_definition() { + // a `float` parameter compiles to a `double`, and a subclass of `float` is more than + // its value: it can define its own operators, and it is its own type. the boundary + // unboxing one would run the body on the bare value, so a call handed a subclass goes + // to the interpreted definition — which is what already happens where python's + // `float` admits an `int`, and has to happen as well in `.by`, where it does not + let loud = "type('Loud', (float,), {'__mul__': lambda s, o: 'Loud.__mul__', \ + '__rmul__': lambda s, o: 'Loud.__rmul__', '__neg__': lambda s: 'Loud.__neg__', \ + '__gt__': lambda s, o: 'Loud.__gt__'})"; + let plain = "type('Plain', (float,), {})"; + let calls = [ + format!("m.mul({loud}(2.0), 3.0)"), + format!("m.mul(3.0, {loud}(2.0))"), + format!("m.neg({loud}(1.0))"), + format!("m.gt4({loud}(1.0))"), + format!("type(m.ident({plain}(1.5))).__name__"), + format!("repr(_capture(m.escape, {loud}(0.3), 0.0, 40))"), + format!("m.escape({plain}(0.3), 0.0, 40)"), + "(m.mul(2.0, 3.5), m.neg(1.0), m.gt4(5.0), m.escape(0.3, 0.0, 40))".to_string(), + ]; + let calls: Vec<&str> = calls.iter().map(String::as_str).collect(); + agree( + "floatsubclass", + "\ +def mul(a: float, b: float) -> float: + return a * b + + +def neg(a: float) -> float: + return -a + + +def gt4(a: float) -> bool: + return a > 4.0 + + +def ident(a: float) -> float: + return a + + +def escape(cr: float, ci: float, limit: int) -> int: + zr = 0.0 + zi = 0.0 + k = 0 + while k < limit: + if zr * zr + zi * zi > 4.0: + return k + t = zr * zr - zi * zi + cr + zi = 2.0 * zr * zi + ci + zr = t + k = k + 1 + return limit +", + &calls, + ); +} + #[test] fn a_buffer_reaches_a_callee_without_being_boxed() { // one body cannot have two representations, so a function whose `list[T]` @@ -23678,6 +25394,134 @@ def kept(d: dict[str, int], k: str) -> object: ); } +#[test] +fn a_membership_test_and_its_read_run_a_stored_keys_eq_as_python_does() { + // an exact dict and an exact `str` key are not enough for the fused lookup to ask + // the table once: a key *already stored* beside the probe, hashing like it, has its + // own `__eq__` asked on every probe that walks past it, and that `__eq__` can count + // the probes or change the dict between the test and the read. only a table holding + // nothing but exact `str` keys has no such key in it. + // + // how many times one lookup walks past the stored key depends on where the string's + // hash sends the probe, and each leg runs under a hash seed of its own. so a count is + // reported in lookups, divided by what one plain membership test asks in the same + // process, and an `__eq__` that acts does so on the first call the read makes + agree_python( + "dictfindeq", + "\ +class Collide: + def __init__(self, like: str) -> None: + self.like = like + self.calls = 0 + self.fire = 0 + self.rewrite = False + self.table: dict[object, int] = {} + + def __hash__(self) -> int: + return hash(self.like) + + def __eq__(self, other: object) -> bool: + self.calls = self.calls + 1 + if self.calls == self.fire: + if self.rewrite: + self.table[self.like] = 999 + else: + del self.table[self.like] + return False + + +class Tagged(str): + def __eq__(self, other: object) -> bool: + return str.__eq__(self, other) + + def __hash__(self) -> int: + return str.__hash__(self) + + +def beside(c: Collide) -> Collide: + c.table[c] = 0 + c.table[c.like] = 5 + c.calls = 0 + return c + + +def per_lookup(c: Collide) -> int: + c.calls = 0 + found = c.like in c.table + asked = c.calls + c.calls = 0 + return asked + + +def read_hit(d: dict[object, int], k: str) -> int: + if k in d: + return d[k] + return -1 + + +def counted(words: list[str], seen: dict[object, int]) -> int: + for word in words: + if word in seen: + seen[word] = seen[word] + 1 + else: + seen[word] = 1 + return len(seen) + + +def turned_general(words: list[str], late: Collide) -> tuple[int, int, int]: + seen: dict[object, int] = {} + late.calls = 0 + i = 0 + for word in words: + if i == 2: + seen[late] = 0 + if word in seen: + seen[word] = seen[word] + 1 + else: + seen[word] = 1 + i = i + 1 + return (len(seen), seen[late.like], late.calls) + + +def turned_general_by_get(words: list[str], late: Collide) -> tuple[int, int, int]: + seen: dict[object, int] = {} + late.calls = 0 + i = 0 + for word in words: + if i == 2: + seen[late] = 0 + if word in seen: + seen[word] = seen.get(word, 0) + 1 + else: + seen[word] = 1 + i = i + 1 + return (len(seen), seen[late.like], late.calls) +", + &[ + "(lambda c: (lambda n: (m.read_hit(c.table, 'a'), c.calls // n))(m.per_lookup(c)))\ + (m.beside(m.Collide('a')))", + "(lambda c: (lambda n: (m.counted(['a', 'a', 'a'], c.table), c.calls // n))\ + (m.per_lookup(c)))(m.beside(m.Collide('a')))", + // the read's first `__eq__` deletes the key the test found, or rewrites the + // value the test saw + "(lambda c: (lambda n: repr(_capture(m.read_hit, c.table, 'a')) \ + if c.__setattr__('fire', n + 1) is None else None)(m.per_lookup(c)))\ + (m.beside(m.Collide('a')))", + "(lambda c: (lambda n: (m.read_hit(c.table, 'a'), c.table['a']) \ + if (c.__setattr__('fire', n + 1), c.__setattr__('rewrite', True)) else None)\ + (m.per_lookup(c)))(m.beside(m.Collide('a')))", + // a table that holds only exact `str` keys when the loop starts, and has a key + // with an `__eq__` of its own inserted part way through. the same loop reading + // through `get`, which is never fused, says what python asks + "(lambda w: m.turned_general(w, m.Collide('b')) == \ + m.turned_general_by_get(w, m.Collide('b')))(['x', 'x', 'b', 'b', 'b'])", + "m.turned_general(['x', 'x', 'b', 'b', 'b'], m.Collide('b'))[:2]", + "m.counted([m.Tagged('a'), 'a', m.Tagged('b')], {'a': 1})", + "m.read_hit({m.Tagged('a'): 3}, 'a')", + ], + ); +} + #[test] fn a_dict_subscript_answers_only_for_an_exact_dict() { // `d[k]` and `d[k] = v` are read and written straight into the table, which is @@ -24040,6 +25884,119 @@ class Hashed: ); } +#[test] +fn an_equality_on_the_object_protocol_asks_the_type_rather_than_the_pointer() { + // python's `==` never takes identity as equality. `a == a` is whatever the type's + // `__eq__` answers, and for a NaN, or for a class that refuses itself, that is + // `False`. the C API's `PyObject_RichCompareBool` does short-circuit on identity + // before it asks the type at all, so a comparison lowered onto it answered `True` + // for both — a compiled program quietly disagreeing with ordinary python. + // + // the operands are typed `object`, which is what keeps the comparison on the + // abstract protocol: a pair of one emitted class is called directly and never + // reaches this. so a mixed pair, an `object`-typed pair and a builtin whose type + // the compiler cannot name are the shapes left + agree_python( + "richcmpidentity", + "\ +class Odd: + def __init__(self, n: int) -> None: + self.n = n + + def __eq__(self, other: object) -> bool: + return False + + def __hash__(self) -> int: + return 0 + + +class Always: + def __eq__(self, other: object) -> bool: + return True + + def __ne__(self, other: object) -> bool: + return True + + def __hash__(self) -> int: + return 0 + + +def equal(a: object, b: object) -> bool: + return a == b + + +def unequal(a: object, b: object) -> bool: + return a != b +", + &[ + // two distinct instances, which never needed the shortcut to answer + "m.equal(m.Odd(1), m.Odd(1))", + // and the same instance twice, which did + "(lambda o: m.equal(o, o))(m.Odd(1))", + "(lambda o: m.unequal(o, o))(m.Odd(1))", + // a written `__ne__` that says `True` whatever it is handed + "(lambda o: m.unequal(o, o))(m.Always())", + "(lambda o: m.equal(o, o))(m.Always())", + // no class of ours at all: a NaN is not equal to itself + "(lambda x: m.equal(x, x))(float('nan'))", + "(lambda x: m.unequal(x, x))(float('nan'))", + // the shapes that were already right, so a fix cannot cost them + "(lambda x: m.equal(x, x))(1.5)", + "m.equal('ab', 'a' + 'b')", + "m.unequal(1, 1.0)", + "(lambda x: m.equal(x, x))([1, 2])", + ], + ); +} + +#[test] +fn membership_and_index_keep_the_identity_shortcut() { + // `x in xs` and `xs.index(x)` are not comparisons. python answers the first with + // `PySequence_Contains` and the second with `list.index`, and *both* of those + // compare identity before they ask `__eq__` — so a class that refuses even itself + // is still found in a list holding it, and interpreted python says so too. + // + // that shortcut is the interpreter's own semantics rather than a shortcut the + // compiler took, and removing it along with the one `==` had would make these + // disagree in the other direction + agree_python( + "containsidentity", + "\ +class Odd: + def __init__(self, n: int) -> None: + self.n = n + + def __eq__(self, other: object) -> bool: + return False + + def __hash__(self) -> int: + return 0 + + +def holds(xs: list[object], x: object) -> bool: + return x in xs + + +def missing(xs: list[object], x: object) -> bool: + return x not in xs + + +def at(xs: list[object], x: object) -> int: + return xs.index(x) +", + &[ + "(lambda o: m.holds([o], o))(m.Odd(1))", + "(lambda o: m.missing([o], o))(m.Odd(1))", + "(lambda o: m.at([o], o))(m.Odd(1))", + "(lambda x: m.holds([x], x))(float('nan'))", + "(lambda x: m.at([x], x))(float('nan'))", + // a *different* instance is the one `__eq__` gets asked about, and it says no + "m.holds([m.Odd(1)], m.Odd(1))", + "type(_capture(m.at, [m.Odd(1)], m.Odd(1))).__name__", + ], + ); +} + #[test] fn a_complex_conversion_is_found_by_name() { // `PyNumberMethods` has `nb_int`, `nb_float` and `nb_index` and no complex field @@ -26470,6 +28427,90 @@ def through(f: object, x: str) -> object: ); } +#[test] +fn a_keyword_given_twice_through_a_merge_is_refused_as_python_refuses_it() { + // a call's `**` merges a mapping into the keywords, and unlike a dict display it + // refuses a keyword that is already there — from a name written before it, from an + // earlier `**`, or from a name written after it. both refusals, and the one for an + // operand that is no mapping, name the callee the way python does. python rewords + // any single-argument `KeyError` the merge raises the same way, one a mapping's own + // `__getitem__` raised included + agree_python( + "kwmerge", + "\ +class Keys: + def __init__(self, keys: list[str]) -> None: + self.names = keys + + def keys(self) -> list[str]: + return self.names + + def __getitem__(self, key: str) -> int: + if key == \"missing\": + raise KeyError(key) + return len(key) + + +class Counter: + def m(self, a: int, **extra: int) -> int: + return a + len(extra) + + +def f(a: int, step: int = 1, **extra: int) -> int: + return a + step + len(extra) + + +def explicit_then_merged(d: dict[str, int]) -> int: + return f(1, step=5, **d) + + +def merged_then_explicit(d: dict[str, int]) -> int: + return f(1, **d, step=5) + + +def merged_twice(d: dict[str, int], e: dict[str, int]) -> int: + return f(1, **d, **e) + + +def merged_mapping(k: Keys) -> int: + return f(1, step=2, **k) + + +def not_a_mapping(x: object) -> int: + return f(1, **x) # type: ignore + + +def method(c: Counter, d: dict[str, int]) -> int: + return c.m(1, b=2, **d) + + +def display(d: dict[str, int], e: dict[str, int]) -> dict[str, int]: + return {**d, \"x\": 0, **e} +", + &[ + "m.explicit_then_merged({'other': 7})", + "repr(_capture(m.explicit_then_merged, {'step': 7}))", + "repr(_capture(m.merged_then_explicit, {'step': 7}))", + "m.merged_twice({'x': 1}, {'y': 2})", + "repr(_capture(m.merged_twice, {'x': 1}, {'x': 2}))", + "m.merged_mapping(m.Keys(['x', 'yy']))", + "repr(_capture(m.merged_mapping, m.Keys(['x', 'step'])))", + "repr(_capture(m.merged_mapping, m.Keys(['missing'])))", + "repr(_capture(m.not_a_mapping, [1]))", + "repr(_capture(m.not_a_mapping, 3))", + // a dict subclass is walked as a dict, whatever its `keys` says + "m.merged_twice(type('D', (dict,), {'keys': lambda s: ['step']})({'z': 1}), {})", + "repr(_capture(m.merged_twice, {1: 1}, {1: 2}))", + // a compiled method carries no `__module__`, which is its own difference, so + // only what follows the callee's name is compared + "str(_capture(m.method, m.Counter(), {'b': 1})).split('()', 1)[1]", + "m.method(m.Counter(), {'x': 1, 'y': 2})", + // a display keeps the later value + "m.display({'x': 1}, {'x': 2})", + ], + ); +} + #[test] fn the_remaining_statement_forms_agree() { agree( @@ -26703,6 +28744,123 @@ def unpacked(n: int) -> str: ); } +#[test] +fn a_comprehensions_targets_are_its_own() { + // a comprehension is a scope of its own, so the names its `for` clauses bind are not + // the enclosing function's: a local, a parameter or a global of the same name keeps + // what it held, and a read of the name before the comprehension still reads the + // global. the first iterable is read in the enclosing scope and every later clause + // inside the comprehension's. a walrus is the one binding that reaches out, and a + // closure made inside captures the comprehension's variable, whose last value every + // one of them sees + agree_python( + "compscope", + "\ +G = 5 + + +def local_float(xs: list[float]) -> float: + x = 5.0 + out = [x + 0.5 for x in xs] + return x + + +def parameter(x: int, xs: list[int]) -> int: + out = [x * 2 for x in xs] + return x + + +def text(xs: list[str]) -> str: + s = \"kept\" + out = [s.upper() for s in xs] + return s + + +def each_form(xs: list[int]) -> tuple[int, int, int, int]: + x = 5 + a = {x for x in xs} + b = x + c = {x: 1 for x in xs} + d = x + e = sum(x for x in xs) + return (b, d, x, len(a) + len(c) + e) + + +def clauses(xss: list[list[int]]) -> tuple[int, int, list[int]]: + x = 5 + ys = [7, 7, 7] + out = [x + 1 for ys in xss for x in ys] + return (x, len(ys), out) + + +def iterable_named_like_target(x: list[int]) -> tuple[list[int], list[int]]: + return ([x * 2 for x in x], x) + + +def global_after(xs: list[int]) -> int: + out = [G + 1 for G in xs] + return G + + +def global_before(xs: list[int]) -> tuple[int, list[int]]: + before = G + out = [G + 1 for G in xs] + return (before, out) + + +def walrus(xs: list[int]) -> tuple[int, list[int]]: + y = 0 + out = [(y := x * 3) for x in xs] + return (y, out) + + +def reused(rows: list[list[int]]) -> tuple[list[list[int]], int]: + x = -1 + out = [[x * 10 for x in x] + [len(x)] for x in rows] + return (out, x) + + +def closures(xs: list[int]) -> tuple[list[int], int]: + x = 100 + made = [lambda: x for x in xs] + return ([f() for f in made], x) + + +def closures_twice(xs: list[int], ys: list[int]) -> list[int]: + first = [lambda: x for x in xs] + second = [lambda: x for x in ys] + return [f() for f in first] + [f() for f in second] +", + &[ + "m.local_float([1.0, 2.0])", + "m.parameter(9, [1, 2])", + "m.text(['a', 'b'])", + "m.each_form([1, 2])", + "m.clauses([[1, 2], [3]])", + "m.iterable_named_like_target([1, 2])", + "(m.global_after([1, 2]), m.G)", + "m.global_before([1, 2])", + "m.walrus([1, 2])", + "m.walrus([])", + "m.reused([[1, 2], [3]])", + "m.closures([1, 2, 3])", + "m.closures_twice([1, 2], [7, 8])", + ], + ); + // basedpython gives each trip of a loop its own binding, a comprehension's included, + // so each closure holds the value it was made with — and still not the frame's `x` + agree( + "compscopeby", + "\ +def closures(xs: list[int]) -> tuple[list[int], int]: + x = 100 + made = [lambda: x for x in xs] + return ([f() for f in made], x) +", + &["m.closures([1, 2, 3])"], + ); +} + #[test] fn the_debug_f_string_form_agrees() { agree( @@ -26927,6 +29085,133 @@ def through(n: int) -> str: ); } +#[test] +fn an_except_clauses_class_is_evaluated_while_the_exception_is_handled() { + // python takes the exception as the one being handled before it evaluates any + // `except` clause's class. so an error raised by that evaluation chains onto it, + // anything the evaluation calls sees it in `sys.exception()`, and when no clause + // matches the exception handled before the `try` is back in place + agree_python( + "handlerclass", + "\ +import sys + + +class Refused(Exception): + pass + + +class Other(Exception): + pass + + +def parse(text: str) -> int: + if len(text) == 0: + raise Refused(\"empty\") + return len(text) + + +def current() -> object: + return type(sys.exception()) + + +def missing_name() -> int: + try: + parse(\"\") + except Other: + return 1 + return 0 + + +def missing_second() -> int: + try: + parse(\"\") + except KeyError: + return 1 + except Other: + return 2 + return 0 + + +def asked_while_handling() -> str: + try: + parse(\"\") + except current() as e: # type: ignore + return \"matched \" + type(e).__name__ + return \"none\" + + +def restored_after(log: list[str]) -> str: + try: + raise KeyError(\"outer\") + except KeyError: + try: + try: + parse(\"\") + except Other: + return \"other\" + except Refused: + log.append(repr(sys.exception())) + return repr(sys.exception()) +", + &[ + "(m.__dict__.pop('Other'), (lambda e: (type(e).__name__, \ + repr(e.__context__), repr(sys.exception())))(_capture(m.missing_name)))[1]", + "(m.__dict__.pop('Other'), (lambda e: (type(e).__name__, \ + repr(e.__context__)))(_capture(m.missing_second)))[1]", + "m.asked_while_handling()", + "(lambda log: (m.restored_after(log), log))([])", + ], + ); +} + +#[test] +fn a_raise_raises_what_the_constructor_answered() { + // `raise Refused(...)` raises whatever calling the name answered. python asks + // nothing of that object beyond its being an exception, so a module whose `Refused` + // has been rebound raises the replacement — and a handler naming `Refused` reads the + // same rebound name, so it still catches it + agree_python( + "raiserebound", + "\ +class Refused(Exception): + pass + + +def parse(text: str) -> int: + if len(text) == 0: + raise Refused(\"empty\") + return len(text) + + +def caught() -> str: + try: + parse(\"\") + except Refused: + return \"caught by name\" + return \"none\" + + +def chained(text: str) -> int: + try: + return int(text) + except ValueError as e: + raise Refused(text) from e +", + &[ + "(m.__dict__.__setitem__('Refused', type('Replacement', (Exception,), {})), \ + m.caught(), repr(_capture(m.parse, '')))[1:]", + "(m.__dict__.__setitem__('Refused', type('Replacement', (Exception,), {})), \ + (lambda e: (repr(e), repr(e.__cause__)))(_capture(m.chained, 'x')))[1]", + "(m.__dict__.__setitem__('Refused', lambda text: KeyError(text)), \ + repr(_capture(m.parse, '')))[1]", + "(m.__dict__.__setitem__('Refused', lambda text: 5), \ + repr(_capture(m.parse, '')))[1]", + "(repr(_capture(m.parse, '')), m.caught())", + ], + ); +} + #[test] fn the_exception_surface_agrees() { agree_with_declines( @@ -30395,7 +32680,7 @@ def joined(n: int) -> str: // be quadratic — which is worth more than the one small allocation fusing it // would save let body = emitted_function(&emitted, "by_by_diff_strconcatint_shape_joined"); - assert!(body.contains("By_StrAppend("), "{body}"); + assert!(body.contains("By_StrAddAppend("), "{body}"); assert!(!body.contains("By_StrConcatInt("), "{body}"); } @@ -30449,15 +32734,14 @@ def once(i: int) -> str: } #[test] -fn fusing_a_prefix_onto_the_digits_does_not_change_what_a_bad_str_raises() { - // a rebound `str` may hand back anything, and what the unfused shape did with - // that is raise from its own unbox — before the concatenation, and in the - // compiler's own words rather than the interpreter's. the fused operation makes - // the same check in the same place, so the two shapes have to say the same thing. +fn fusing_a_prefix_onto_the_digits_raises_what_python_raises_for_a_bad_str() { + // a rebound `str` may hand back anything. the fused operation hands what it answered + // straight to `+`, as python does, so a value that is no `str` is refused by the + // operator in cpython's own words — and a `str` subclass is asked for its `__radd__`. // - // that they say something other than what cpython says for `'k' + 1` is older - // than this fusion and is what any annotated `-> str` already answers; the point - // here is only that fusing did not move it + // the unfused shape names the answer first, and narrowing that name to the `str` the + // checker says it holds is what refuses it there, in the compiler's words: the + // annotation-trust refusal any `-> str` answer already gets let Some((python, toolchain)) = environment() else { return; }; @@ -30512,7 +32796,11 @@ def unfused(n: int) -> str: print(said[0])\n\ print('same' if said[0] == said[1] else f'differs: {said[1]}')\n", ); - assert_eq!(out, "TypeError: expected str, got int\nsame"); + assert_eq!( + out, + "TypeError: can only concatenate str (not \"int\") to str\n\ + differs: TypeError: expected str, got int" + ); } #[test] @@ -30813,6 +33101,853 @@ class Counting: ); } +#[test] +fn a_value_a_name_held_is_dropped_when_the_name_is_rebound() { + // python holds what a name is bound to until the name is rebound or deleted, not + // until the function returns: rebinding drops the old value straight after the new + // one is built, so its finalizer runs before the next statement. that holds whether + // or not anything reads the name again, whatever the name is annotated as, and + // however the value reached the name — a constructor, a call, or an element of a + // tuple a call answered + agree_python( + "namerebind", + "\ +log: list[str] = [] + + +class Cell: + x: int + + def __init__(self, x: int) -> None: + log.append('init ' + str(x)) + self.x = x + + def __del__(self) -> None: + log.append('del ' + str(self.x)) + + +class Linked: + x: int + other: 'Linked | None' + + def __init__(self, x: int, other: 'Linked | None') -> None: + self.x = x + self.other = other + + def __del__(self) -> None: + if self.other is not None: + self.other.x = 99 + + +class Slotted: + __slots__ = ('x',) + + def __init__(self, x: int) -> None: + log.append('init ' + str(x)) + self.x = x + + def __del__(self) -> None: + log.append('del ' + str(self.x)) + + +class Bare: + def __del__(self) -> None: + log.append('del bare') + + +def make(x: int) -> Cell: + return Cell(x) + + +def split(cell: Linked, value: int) -> tuple[Linked, int]: + return cell, value % 7 + + +def ctor_rebound() -> list[str]: + log.clear() + held = Cell(1) + held = Cell(2) + log.append('after rebind') + return list(log) + + +def call_rebound() -> list[str]: + log.clear() + held = make(1) + held = make(2) + log.append('after rebind') + return list(log) + + +def ctor_deleted() -> list[str]: + log.clear() + held = Cell(1) + del held + log.append('after del') + return list(log) + + +def read_then_deleted() -> list[str]: + log.clear() + held = Cell(1) + log.append('read ' + str(held.x)) + del held + log.append('after del') + return list(log) + + +def bare_rebound() -> list[str]: + log.clear() + held = Bare() + held = Bare() + log.append('after rebind') + return list(log) + + +def object_rebound() -> list[str]: + log.clear() + held: object = Cell(1) + held = Cell(2) + log.append('after rebind') + return list(log) + + +def loop_rebound() -> list[str]: + log.clear() + i = 0 + while i < 3: + held = Cell(i) + log.append('made ' + str(i)) + i = i + 1 + return list(log) + + +def for_rebound() -> list[str]: + log.clear() + for i in range(3): + held = Cell(i) + return list(log) + + +def slotted_rebound() -> list[str]: + log.clear() + i = 0 + while i < 3: + held = Slotted(i) + i = i + 1 + return list(log) + + +def slotted_for_rebound() -> list[str]: + log.clear() + for i in range(3): + held = Slotted(i) + return list(log) + + +def bound_before_the_loop() -> list[str]: + log.clear() + held = Cell(-1) + for i in range(3): + held = Cell(i) + del held + log.append('after del') + return list(log) + + +def slotted_bound_before_the_loop() -> list[str]: + log.clear() + held = Slotted(-1) + for i in range(3): + held = Slotted(i) + del held + log.append('after del') + return list(log) + + +def through_a_tuple(second: Linked) -> int: + held, part = split(Linked(1, second), 0) + held, part = split(second, 1) + return held.x + part +", + &[ + "m.ctor_rebound()", + "m.call_rebound()", + "m.ctor_deleted()", + "m.read_then_deleted()", + "m.bare_rebound()", + "m.object_rebound()", + "m.loop_rebound()", + // in a loop the old value goes once the new one is built, and not before its + // `__init__` has run + "m.for_rebound()", + "m.slotted_rebound()", + "m.slotted_for_rebound()", + "m.bound_before_the_loop()", + "m.slotted_bound_before_the_loop()", + "m.through_a_tuple(m.Linked(5, None))", + ], + ); + // a class that fell back to its interpreted definition would agree as well + if let Some((python, _)) = environment() { + let out = run( + &python, + &diff_root().join("by_diff_namerebind_c"), + "import by_diff_namerebind as m\n\ + print(m.__file__.endswith(('.so', '.pyd')), type(m.Cell.__dict__['__del__']).__name__, \ + type(m.Slotted.__dict__['__del__']).__name__)\n", + ); + assert_eq!(out, "True wrapper_descriptor wrapper_descriptor"); + } +} + +#[test] +fn a_value_an_expression_statement_discards_is_not_checked() { + // python throws an expression statement's value away, so nothing about that value + // can make the statement raise. a call declared to answer an `int`, or reached + // through a method a subclass overrides to answer something else, answers what it + // answers — and the statement goes on to the next line + agree_python( + "discarded", + "\ +from collections.abc import Callable + + +class Loud(list): + def append(self, value): + super().append(value) + return 'appended' + + +class Shout(str): + def upper(self): + return 42 + + +class Counter: + def __init__(self) -> None: + self.count = 0 + + def bump(self) -> int: + self.count = self.count + 1 + return self.count + + +class Shape: + def area(self) -> int: + return 1 + + +class Square(Shape): + def area(self) -> int: + return 4 + + +def helper() -> int: + return 1 + + +def push(items: list[str], value: str) -> int: + items.append(value) + return len(items) + + +def touch(s: str) -> str: + s.upper() + return 'touched' + + +def through_callable(cb: Callable[[], int]) -> str: + cb() + return 'ok' + + +def parenthesised(cb: Callable[[], int]) -> str: + (cb()) + return 'ok' + + +def unpacked(cb: Callable[..., int], args: tuple[int, ...]) -> str: + cb(*args) + return 'ok' + + +def through_module_function() -> str: + helper() + return 'ok' + + +def bumped(counter: Counter) -> int: + counter.bump() + return counter.count + + +def measured(shapes: list[Shape]) -> str: + for shape in shapes: + shape.area() + return 'ok' + + +def still_checked(cb: Callable[[], int]) -> int: + cb() + return cb() + 1 +", + &[ + "[m.push([], 'a'), m.push(m.Loud(), 'a')]", + "[m.touch('a'), m.touch(m.Shout('a'))]", + "[m.through_callable(lambda: 1), m.through_callable(lambda: 'x')]", + "m.parenthesised(lambda: 'x')", + "m.unpacked(lambda *a: 'x', (1, 2))", + "[m.helper.__module__, m.through_module_function()]", + // an instance whose own dict holds a callable of the method's name, which + // the call reaches instead of the compiled body + "[_c := m.Counter(), m.bumped(_c), setattr(_c, 'bump', lambda: 'shadowed'), m.bumped(_c)][1::2]", + "m.measured([m.Shape(), m.Square()])", + // a value that is used is still what the call was declared to answer + "(lambda: m.still_checked(lambda: 2))()", + ], + ); +} + +#[test] +fn a_builtin_the_module_rebinds_is_the_one_called() { + // a compiled `len(xs)`, `for i in range(n)`, `globals()` and zero-argument `super()` + // take a native path only while the name still resolves to the interpreter's own + // builtin. a module attribute written from outside — which is what + // `mock.patch('mod.len')` does — is what python calls instead + agree_python( + "rebuiltin", + "\ +import sys +from collections.abc import Callable + + +class Base: + def who(self) -> str: + return 'base' + + +class Child(Base): + def who(self) -> str: + return 'child ' + super().who() + + +def use_len(xs: list[int]) -> int: + return len(xs) + + +def count_chars(s: str) -> int: + seen = 0 + i = 0 + while i < len(s): + seen = seen + 1 + i = i + 1 + return seen + + +def built(n: int) -> float: + xs = [i * 0.5 for i in range(n)] + total = 0.0 + j = 0 + while j < len(xs): + total = total + xs[j] + j = j + 1 + return total + + +def use_range(n: int) -> int: + total = 0 + for i in range(n): + total = total + i + return total + + +def gradual(n: object) -> int: + total = 0 + for i in range(n): + total = total + i + return total + + +def comprehension(n: int) -> list[int]: + return [i * 2 for i in range(n)] + + +def use_globals() -> object: + return globals() + + +def patched_builtin(name: str, value: object, call: Callable[[], object]) -> object: + import builtins + + original = getattr(builtins, name) + setattr(builtins, name, value) + try: + return call() + except Exception as error: + return repr(error) + finally: + setattr(builtins, name, original) + + +def patched(name: str, value: object, call: Callable[[], object]) -> object: + namespace = sys.modules[__name__].__dict__ + namespace[name] = value + try: + return call() + except Exception as error: + return repr(error) + finally: + del namespace[name] +", + &[ + "[m.use_len([1, 2]), m.patched('len', lambda xs: 99, lambda: m.use_len([1, 2]))]", + // the interpreter's own builtins are the other namespace a name resolves through + "[m.use_len([1, 2]), m.patched_builtin('len', lambda xs: 3, lambda: m.use_len([1, 2]))]", + "[m.use_range(3), m.patched_builtin('range', lambda n: iter([4]), lambda: m.use_range(3))]", + "[m.count_chars('abc'), m.patched('len', lambda s: 1, lambda: m.count_chars('abc'))]", + "m.built(3)", + "[m.use_range(3), m.patched('range', lambda n: iter([100, 200]), lambda: m.use_range(3))]", + // a bound the counting loop cannot take calls `range` with what was written + "[m.gradual(3), m.patched('range', lambda n: iter([7]), lambda: m.gradual(3))]", + "[m.comprehension(3), m.patched('range', lambda n: iter([5]), lambda: m.comprehension(3))]", + "[m.use_globals() is vars(m), m.patched('globals', lambda: 'fake', m.use_globals)]", + // python calls a rebound `super` with no arguments at all + "[m.Child().who(), m.patched('super', lambda *a: type('S', (), {'who': lambda self: 'fake' + str(len(a))})(), lambda: m.Child().who())]", + ], + ); + if let Some((python, _)) = environment() { + let out = run( + &python, + &diff_root().join("by_diff_rebuiltin_c"), + "import by_diff_rebuiltin as m\n\ + print(m.__file__.endswith(('.so', '.pyd')), type(m.Child.__dict__['who']).__name__)\n\ + # a list held as a buffer has no list object to hand a rebound `len`, and a\n\ + # copy would be a different list, so that call refuses rather than answering\n\ + print(m.patched('len', lambda xs: 1, lambda: m.built(3)).startswith('RuntimeError'))\n", + ); + assert_eq!(out, "True method_descriptor\nTrue"); + } +} + +#[test] +fn a_special_method_a_descriptor_builds_is_looked_up_as_python_looks_it_up() { + // a `with` statement and a method call each remember what the name resolved to on + // the receiver's type. what is on the type is not always what reading it answers: a + // `functools.partialmethod`, or any `__get__` of its own, builds a fresh object on + // every read, and python binds it to the receiver rather than calling it with one + // prepended. a `staticmethod` binds to nothing, a `classmethod` to the class, and a + // method a metaclass defines is not the instance's at all + agree_python( + "specialsite", + "\ +def use(manager: object, n: int) -> int: + total = 0 + i = 0 + while i < n: + with manager as got: + total = total + 1 + i = i + 1 + return total + + +def entered(manager: object) -> object: + with manager as got: + return got + + +async def aentered(manager: object) -> object: + async with manager as got: + return got + + +async def tagged(manager: object, tag: str) -> str: + return tag + + +async def counted(first: object = None) -> object: + return ('fresh', first is None) + + +async def declined(manager: object, *args: object) -> bool: + return False + + +def popped(receiver: object) -> object: + return receiver.pop() +", + &[ + // freed as soon as the lookup is done, and then called through what was kept + "(lambda P: sum(m.use(P(), 3) + len([object() for _ in range(20)]) for _ in range(2000)))(type('P', (), {'__enter__': __import__('functools').partialmethod(lambda self, tag: tag, 'tag'), '__exit__': lambda self, *a: False}))", + "(lambda P: [m.entered(P()) for _ in range(50)][-1])(type('P', (), {'__enter__': __import__('functools').partialmethod(lambda self, tag: tag, 'tag'), '__exit__': lambda self, *a: False}))", + "(lambda Get: [m.entered(type('F', (), {'__enter__': Get(), '__exit__': lambda self, *a: False})()) for _ in range(50)][-1])(type('Get', (), {'__get__': lambda self, obj, cls: (lambda *a: ('fresh', obj is not None, len(a)))}))", + "[m.entered(type('S', (), {'__enter__': staticmethod(lambda *a: ('static', len(a))), '__exit__': lambda self, *a: False})()) for _ in range(50)][-1]", + "[m.entered(type('K', (), {'__enter__': classmethod(lambda cls, *a: (cls.__name__, len(a))), '__exit__': lambda self, *a: False})()) for _ in range(50)][-1]", + "(lambda Meta: str(_capture(m.entered, Meta('ByMeta', (), {})())).startswith(\"'ByMeta' object does not support the context manager protocol\"))(type('Meta', (type,), {'__enter__': lambda cls: 'meta', '__exit__': lambda cls, *a: False}))", + // `async with` asks through the same kind of site + "(lambda P: [(__import__('asyncio').run(m.aentered(P())), len([object() for _ in range(20)]))[0] for _ in range(300)][-1])(type('P', (), {'__aenter__': __import__('functools').partialmethod(m.tagged, 'tag'), '__aexit__': m.declined}))", + "(lambda Get: [__import__('asyncio').run(m.aentered(type('A', (), {'__aenter__': Get(), '__aexit__': m.declined})())) for _ in range(50)][-1])(type('Get', (), {'__get__': lambda self, obj, cls: m.counted}))", + // a method borrowed from a builtin type answers for that type's instances only + "[repr(_capture(m.popped, type('B', (), {'__slots__': (), 'pop': list.pop})())) for _ in range(50)][-1]", + "[m.popped(type('G', (), {'__slots__': (), 'pop': type('Get', (), {'__get__': lambda self, obj, cls: (lambda *a: ('fresh', obj is not None, len(a)))})()})()) for _ in range(50)][-1]", + "(lambda Q: [(m.popped(Q()), len([object() for _ in range(20)]))[0] for _ in range(3000)][-1])(type('Q', (), {'__slots__': (), 'pop': __import__('functools').partialmethod(lambda self, tag: ('partial', tag), 'tag')}))", + ], + ); +} + +#[test] +fn a_tuple_holds_its_elements_until_python_would_have_dropped_it() { + // python builds a tuple display from every element before anything lets go of one, + // and a tuple dropped lets go of its elements last to first. so an element of a + // display that is thrown away, or subscripted for another element, lives until the + // last element has been made — and a tuple a name holds goes when the name does + agree_python( + "tuplelife", + "\ +from collections.abc import Callable + + +class Loud: + def __init__(self, i: int, log: list[str]) -> None: + self.i = i + self.log = log + log.append('make ' + str(i)) + + def __del__(self) -> None: + self.log.append('del ' + str(self.i)) + + +def display_discarded(f: Callable[[int], object], log: list[str]) -> None: + (f(1), f(2)) + log.append('after') + + +def display_indexed(f: Callable[[int], object], log: list[str]) -> None: + x = (f(1), f(2))[1] + log.append('after') + del x + log.append('end') + + +def display_named(f: Callable[[int], object], log: list[str]) -> None: + t = (f(1), f(2)) + log.append('after') + del t + log.append('end') + + +def display_returned(f: Callable[[int], object], log: list[str]) -> int: + t = (f(1), f(2)) + log.append('after') + return len(t) + + +def mk2(f: Callable[[int], object]) -> tuple[object, object]: + return f(1), f(2) + + +def call_discarded(f: Callable[[int], object], log: list[str]) -> None: + mk2(f) + log.append('after') + + +def call_named(f: Callable[[int], object], log: list[str]) -> None: + t = mk2(f) + log.append('after') + + +def call_unpacked(f: Callable[[int], object], log: list[str]) -> None: + a, b = mk2(f) + log.append('after') + + +def call_rebound(f: Callable[[int], object], log: list[str]) -> None: + t = mk2(f) + log.append('after') + t = mk2(f) + log.append('rebound') +", + &[ + "(lambda log: (m.display_discarded(lambda i: m.Loud(i, log), log), log)[1])([])", + "(lambda log: (m.display_indexed(lambda i: m.Loud(i, log), log), log)[1])([])", + "(lambda log: (m.display_named(lambda i: m.Loud(i, log), log), log)[1])([])", + "(lambda log: (m.display_returned(lambda i: m.Loud(i, log), log), log)[1])([])", + "(lambda log: (m.call_discarded(lambda i: m.Loud(i, log), log), log)[1])([])", + "(lambda log: (m.call_named(lambda i: m.Loud(i, log), log), log)[1])([])", + "(lambda log: (m.call_unpacked(lambda i: m.Loud(i, log), log), log)[1])([])", + "(lambda log: (m.call_rebound(lambda i: m.Loud(i, log), log), log)[1])([])", + ], + ); +} + +#[test] +fn a_length_past_the_short_range_agrees() { + // a length is a `Py_ssize_t`, and a tagged `int` holds one bit fewer, so a length an + // object answers from `__len__` may need an object of its own + agree_python( + "biglen", + "\ +def length(xs: object) -> int: + return len(xs) + + +def nonempty(xs: object) -> bool: + return len(xs) > 0 +", + &[ + "[m.length(range(2**62)), m.length(range(2**63 - 1)), m.length(range(2**62 - 1))]", + "[m.nonempty(range(2**62)), m.length([1, 2, 3]), m.length('abc')]", + ], + ); +} + +#[test] +fn an_instance_made_without_init_has_no_fields_until_they_are_assigned() { + // `C.__new__(C)` makes an instance `__init__` never ran on, and each attribute + // `__init__` would have given it is absent until something assigns it: a read raises + // `AttributeError`, wherever it is read from, and the instance's `__dict__`, + // `vars()` and `__getstate__` list only what it has. `copy` and `pickle` build their + // copies exactly that way and then fill them in + agree_python( + "unsetfields", + "\ +class Fields: + i: int + f: float + b: bool + s: str + o: object + + def __init__(self, i: int, f: float, b: bool, s: str, o: object) -> None: + self.i = i + self.f = f + self.b = b + self.s = s + self.o = o + + +def read_i(x: Fields) -> int: + return x.i + + +def read_f(x: Fields) -> float: + return x.f + + +def read_b(x: Fields) -> bool: + return x.b + + +def read_s(x: Fields) -> str: + return x.s + + +def read_o(x: Fields) -> object: + return x.o + + +def assigned_then_read(x: Fields) -> int: + x.i = 4 + return x.i + x.i + + +class P: + def __init__(self, s: str, xs: list[int]) -> None: + self.s = s + self.xs = xs + + def size(self) -> int: + return len(self.s) + len(self.xs) + + +def get_s(p: P) -> str: + return p.s + + +class Slotted: + __slots__ = ('n', 'label') + + def __init__(self, n: int, label: str) -> None: + self.n = n + self.label = label + + +class Money: + amount: int + + def __init__(self, amount: int) -> None: + self.amount = amount + + def __lt__(self, other: 'Money') -> bool: + return self.amount < other.amount + + def __eq__(self, other: object) -> bool: + return isinstance(other, Money) and self.amount == other.amount + + +def less(a: Money, b: Money) -> bool: + return a < b + + +class Point: + x: int + y: int + + __match_args__ = ('x', 'y') + + def __init__(self, x: int, y: int) -> None: + self.x = x + self.y = y + + +def classify(value: object) -> int: + match value: + case Point(a, b) if a > b: + return a - b + case Point(a, b): + return a + b + case _: + return 3 + + +def by_keyword(value: object) -> int: + match value: + case Point(y=b): + return b + case Point(): + return 5 + case _: + return 6 +", + &[ + "[repr(_capture(getattr, m.Fields.__new__(m.Fields), name)) for name in 'ifbso']", + "[repr(_capture(getattr(m, 'read_' + name), m.Fields.__new__(m.Fields))) for name in 'ifbso']", + "[m.read_i(m.Fields(1, 2.0, True, 's', None)), m.assigned_then_read(m.Fields.__new__(m.Fields))]", + "(lambda x: [x.i, repr(_capture(getattr, x, 'f'))])((lambda x: (setattr(x, 'i', 7), x)[1])(m.Fields.__new__(m.Fields)))", + // `copy` and `pickle` go through `__new__`, `__dict__` and `__getstate__` + "(lambda c: (c.s, c.xs, c.size()))(__import__('copy').copy(m.P('ab', [1])))", + "(lambda c: (c.s, c.xs, c.size()))(__import__('copy').deepcopy(m.P('ab', [1])))", + "(lambda c: (c.s, c.xs, c.size()))(__import__('pickle').loads(__import__('pickle').dumps(m.P('ab', [1]))))", + "[repr(_capture(getattr, m.P.__new__(m.P), 's')), repr(_capture(m.get_s, m.P.__new__(m.P))), repr(_capture(m.P.__new__(m.P).size))]", + "[m.P.__new__(m.P).__dict__, vars(m.P.__new__(m.P)), m.P.__new__(m.P).__getstate__()]", + "(lambda p: [p.__dict__, vars(p), p.__getstate__(), repr(_capture(p.size))])((lambda p: (setattr(p, 's', 'x'), p)[1])(m.P.__new__(m.P)))", + "[repr(_capture(getattr, m.Slotted.__new__(m.Slotted), name)) for name in ('n', 'label')]", + "[repr(_capture(m.less, m.Money.__new__(m.Money), m.Money(1))), repr(_capture(lambda: m.Money(0) == m.Money.__new__(m.Money)))]", + // a class pattern capturing an attribute the instance lacks does not match, + // and matching goes on with the next case — a guard reading the capture never + // runs + "[m.classify(m.Point(3, 1)), m.classify(m.Point(1, 3)), m.classify(m.Point.__new__(m.Point))]", + "(lambda p: [m.classify(p), m.by_keyword(p)])((lambda p: (setattr(p, 'x', 9), p)[1])(m.Point.__new__(m.Point)))", + "[m.by_keyword(m.Point(1, 2)), m.by_keyword(m.Point.__new__(m.Point)), m.by_keyword(0)]", + ], + ); + if let Some((python, _)) = environment() { + let out = run( + &python, + &diff_root().join("by_diff_unsetfields_c"), + "import by_diff_unsetfields as m\n\ + print(m.__file__.endswith(('.so', '.pyd')), type(m.P.__dict__['size']).__name__, \ + type(m.Money.__dict__['__lt__']).__name__)\n", + ); + assert_eq!(out, "True method_descriptor wrapper_descriptor"); + } +} + +#[test] +fn a_comparison_answers_with_what_the_comparison_method_answered() { + // `a > b` is whatever `__gt__` returned, which need not be a `bool`: numpy answers an + // array, and an `__eq__` may answer anything at all. only a condition asks for the + // answer's truth, the way `if` and `while` ask of any value + agree_python( + "richcompare", + "\ +class Loose: + def __init__(self, tag: str) -> None: + self.tag = tag + + def __gt__(self, other: object) -> object: + return 'gt ' + self.tag + + def __lt__(self, other: object) -> object: + return '' + + def __eq__(self, other: object) -> object: + return [self.tag] + + def __ne__(self, other: object) -> object: + return None + + def __le__(self, other: object) -> object: + return 0 + + def __ge__(self, other: object) -> object: + return (1,) + + +def gt(a: object, b: object) -> object: + return a > b + + +def every(a: object, b: object) -> list[object]: + return [a > b, a < b, a == b, a != b, a <= b, a >= b] + + +def stored(a: object, b: object) -> object: + answer = a > b + return answer + + +def chained(a: object, b: object, c: object) -> object: + return a < b < c + + +def chained_through(a: object, b: object, c: object) -> object: + return a > b > c + + +def against_a_float(a: object, x: float) -> object: + return x < a + + +def branch(a: object, b: object) -> str: + if a > b: + return 'taken' + return 'not taken' + + +def falsy_branch(a: object, b: object) -> str: + while a < b: + return 'looped' + return 'skipped' + + +def either(a: object, b: object) -> object: + return a < b or a > b + + +def negated(a: object, b: object) -> object: + return not a == b + + +def plain(a: int, b: int) -> object: + return a < b +", + &[ + "m.gt(m.Loose('x'), 1)", + "m.every(m.Loose('y'), 2)", + "m.stored(m.Loose('z'), 3)", + "[m.chained(1, m.Loose('p'), 3), m.chained(m.Loose('q'), 2, 3)]", + "[m.chained_through(m.Loose('r'), 1, 2), m.chained_through(3, 2, 1), m.chained_through(1, 2, 3)]", + "[m.against_a_float(m.Loose('f'), 1.5), m.against_a_float(2, 1.5)]", + "[m.branch(m.Loose('b'), 0), m.falsy_branch(m.Loose('c'), 0)]", + "[m.either(m.Loose('e'), 0), m.negated(m.Loose('n'), 0)]", + "[m.plain(1, 2), m.gt(2, 1), m.every(1.5, 2)]", + // the answer is the very object the method returned + "(lambda marker: m.gt(type('M', (), {'__gt__': lambda self, other: marker})(), 0) is marker)(object())", + ], + ); +} + #[test] fn a_returned_value_outlives_the_releases_that_follow_it() { // a `return` now hands the caller the reference the frame already held rather diff --git a/crates/by_build/tests/end_to_end.rs b/crates/by_build/tests/end_to_end.rs index 93991a5449..d629a15061 100644 --- a/crates/by_build/tests/end_to_end.rs +++ b/crates/by_build/tests/end_to_end.rs @@ -668,8 +668,8 @@ def kept(a: str, b: str) -> object: ) .expect("the module emits"); let emitted = std::fs::read_to_string(&built.artifact.source).expect("the C is readable"); - let appends = emitted.matches("By_StrAppend(").count(); - let copies = emitted.matches("By_StrConcat(").count(); + let appends = emitted.matches("By_StrAddAppend(").count(); + let copies = emitted.matches("By_StrAdd(").count(); // one per copy of the loop body — `unswitch` duplicates it assert!( appends >= 1, diff --git a/crates/by_codegen_c/src/lib.rs b/crates/by_codegen_c/src/lib.rs index 1094067fbc..47c4ed5a55 100644 --- a/crates/by_codegen_c/src/lib.rs +++ b/crates/by_codegen_c/src/lib.rs @@ -31,7 +31,8 @@ use by_ir::function::{ SlotAlias, Surface, cleaned_doc, }; use by_ir::ops::{ - BinOp, BlockId, CmpOp, LicenceKind, Mutation, Op, RegisterId, Terminator, UnaryOp, Value, + BinOp, BlockId, CmpOp, Concatenation, LicenceKind, Mutation, Op, RegisterId, Terminator, + UnaryOp, Value, }; use by_ir::rtype::{Primitive, RType, tuple_mangle}; @@ -60,7 +61,7 @@ fn default_expr(ty: &RType, default: &Value) -> String { format!("By_NewRef(Py_{})", if *value { "True" } else { "False" }) } Value::Int(_) => format!("By_BoxInt({expr})"), - Value::Fixed(value) => format!("By_BoxInt(By_ShortFrom({value}))"), + Value::Fixed(value) => format!("PyLong_FromLongLong(INT64_C({value}))"), Value::Float(_) => format!("By_BoxFloat({expr})"), Value::Str(_) | Value::Bytes(_) | Value::Register(_) => format!("By_NewRef({expr})"), } @@ -115,6 +116,7 @@ fn frame_kind(surface: Surface) -> &'static str { fn mangle_member(name: &str) -> String { by_ir::function::FieldDecl { + cell: false, name: name.to_string(), ty: RType::OBJECT, default: None, @@ -228,6 +230,17 @@ pub fn emit_module(module: &ModuleIr) -> String { out.push('\n'); } + // an integer literal too wide to tag is a real `int`, built once like a bytes + // literal and read as a borrowed tagged reference from then on + let long_literals = collect_long_literals(module); + LONG_LITERALS.with_borrow_mut(|slot| slot.clone_from(&long_literals)); + for (index, literal) in long_literals.iter().enumerate() { + let _ = writeln!(out, "static PyObject *by_long{index} = NULL; // {literal}"); + } + if !long_literals.is_empty() { + out.push('\n'); + } + for class in &module.classes { out.push_str(&emit_class_struct(module, class)); } @@ -275,6 +288,13 @@ pub fn emit_module(module: &ModuleIr) -> String { accessor_licence(module, &class, &name) ); } + for (class, field) in field_licences(module) { + let _ = writeln!( + out, + "static ByAccessorLicence {} = BY_ACCESSOR_LICENCE_INIT;", + field_licence(module, &class, &field) + ); + } if !module.classes.is_empty() { out.push('\n'); } @@ -283,17 +303,33 @@ pub fn emit_module(module: &ModuleIr) -> String { let _ = writeln!(out, "{};", signature(module, function)); } // the wrappers are declared before the method tables that name them, and the - // tables before the bodies — because a `MakeClosure` in a body names a table. - // interleaving per class made that a forward reference + // tables before the bodies — because a `MakeClosure` in a body names a nested + // function's spec, and the spec names its wrapper. interleaving per class made that + // a forward reference for class in &module.classes { for method in &class.methods { let _ = writeln!( out, - "static PyObject *{}(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames);", - method.wrapper_symbol(module.name.dotted()) + "static PyObject *{}({});", + method.wrapper_symbol(module.name.dotted()), + wrapper_parameters(method) ); } } + for class in &module.classes { + for method in &class.methods { + if let Some(nested) = &method.nested { + let wrapper = method.wrapper_symbol(module.name.dotted()); + let _ = writeln!( + out, + "static const ByFunctionSpec {wrapper}_spec = {{ {wrapper}, {}, {}, {}, &by_module_dict }};", + c_string(&nested.name), + c_string(&nested.qualname), + method_doc(method) + ); + } + } + } out.push('\n'); for class in &module.classes { @@ -391,6 +427,25 @@ fn collect_string_literals(module: &ModuleIr) -> Vec { seen.into_iter().collect() } +/// the range a short holds on a 64-bit word, `BY_SHORT_MIN..=BY_SHORT_MAX` in `by.h` +/// +/// an integer literal is sorted into short or not when the C is written, so this has to +/// be the bound the C compiler will see. `by.h` asserts the word it was written for +const SHORT_RANGE: std::ops::RangeInclusive = -(1 << 62)..=(1 << 62) - 1; + +/// every distinct integer literal in the module too wide to tag, in a stable order +fn collect_long_literals(module: &ModuleIr) -> Vec { + let mut seen = BTreeSet::new(); + for value in module_values(module) { + if let Value::Int(value) = value + && !SHORT_RANGE.contains(value) + { + seen.insert(*value); + } + } + seen.into_iter().collect() +} + /// every distinct bytes literal in the module, in a stable order fn collect_bytes_literals(module: &ModuleIr) -> Vec> { let mut seen = BTreeSet::new(); @@ -517,6 +572,41 @@ fn emit_collected_instance(module: &ModuleIr, class: &ClassIr) -> String { ) } +/// `tp_traverse` and `tp_clear` for a closure environment +/// +/// a nested function that reads its own name holds the environment it reads it out of, +/// and the environment holds the function, so counting alone never frees either — the +/// collector has to be able to walk the cycle, and clearing the environment's fields is +/// what breaks it. the type is static, so there is no reference to it to report +fn emit_collected_environment(module: &ModuleIr, class: &ClassIr) -> String { + let struct_name = class.struct_name(module.name.dotted()); + let type_name = class.type_name(module.name.dotted()); + let mut visits = String::new(); + let mut clears = String::new(); + for field in class.fields.iter().filter(|field| collectable(field)) { + let _ = writeln!( + out_slot(&mut visits), + " Py_VISIT(self->{});", + field.member() + ); + let _ = writeln!( + out_slot(&mut clears), + " Py_CLEAR(self->{});", + field.member() + ); + } + format!( + "static int {type_name}_traverse({struct_name} *self, visitproc visit, void *arg) {{\n\ + \x20 (void)self; (void)visit; (void)arg;\n\ + {visits}\ + \x20 return 0;\n}}\n\n\ + static int {type_name}_clear({struct_name} *self) {{\n\ + \x20 (void)self;\n\ + {clears}\ + \x20 return 0;\n}}\n\n" + ) +} + /// `tp_dealloc`, `tp_traverse` and `tp_clear` for a class whose fields sit past a /// base's instance, and for a base such a class stands on /// @@ -679,7 +769,7 @@ fn emit_class_struct(module: &ModuleIr, class: &ClassIr) -> String { let _ = writeln!(out, " {} {};", ctype(module, &field.ty), field.member()); // `tp_alloc` zeroes the instance, so "never written" is the state an object // starts in and the constructor has nothing to do - if field.optional { + if field.has_presence_byte() { let _ = writeln!(out, " char {};", field.presence()); } } @@ -687,6 +777,63 @@ fn emit_class_struct(module: &ModuleIr, class: &ClassIr) -> String { out } +/// the fields of `class` whose absent state is a value an allocation does not leave +/// +/// a tagged `int` is absent when it holds its error value, and the zero an allocation +/// leaves is the `int` zero. every other field reads a zeroed block as absent already +fn absent_by_value(class: &ClassIr) -> impl Iterator { + published_fields(class).iter().filter(|field| { + field.tracks_absence() && !field.has_presence_byte() && field.ty == RType::INT + }) +} + +/// whether this class's own allocation writes the absent value into fields of its layout +/// +/// a class that keeps nothing of its own inherits its base's allocation, which already +/// writes every field the two share +fn writes_absent_values(module: &ModuleIr, class: &ClassIr) -> bool { + class.resume.is_none() + && absent_by_value(class).next().is_some() + && (external_storage(module, class) || !inherits_layout(module, class)) +} + +/// the allocation of a class whose fields start absent by value, and what it writes +/// +/// every instance comes from `tp_alloc`, however it was asked for — a construction, a +/// bare `__new__`, `object.__new__` inside a written one, `copy` and `pickle` — so this is +/// where an instance `__init__` never runs on gets fields that read as absent. a class +/// that keeps its dead instances' memory writes them from its own allocator instead +fn emit_absent_values(module: &ModuleIr, class: &ClassIr) -> String { + if !writes_absent_values(module, class) { + return String::new(); + } + let type_name = class.type_name(module.name.dotted()); + let mut out = format!( + "static void {type_name}_unset(PyObject *selfobj) {{\n\x20 {}\n", + bind_storage_chain(module, class) + ); + for field in absent_by_value(class) { + let _ = writeln!( + out, + "\x20 {}->{} = BY_INT_ERROR;", + storage_name(module, class, &field.name), + field.member() + ); + } + out.push_str("}\n\n"); + if !recycles_instances(module, class) { + let _ = write!( + out, + "static PyObject *{type_name}_alloc(PyTypeObject *by_type, Py_ssize_t by_items) {{\n\ + \x20 PyObject *by_obj =\n\ + \x20 ((PyTypeObject *){type_name}_OBJ)->tp_base->tp_alloc(by_type, by_items);\n\ + \x20 if (by_obj != NULL) {type_name}_unset(by_obj);\n\ + \x20 return by_obj;\n}}\n\n" + ); + } + out +} + /// how many instances' worth of memory a class keeps back for the next instance /// /// the depth a loop actually needs is one: an object built and dropped every time @@ -768,7 +915,7 @@ fn emit_instance_recycling(module: &ModuleIr, class: &ClassIr) -> String { // address. `PyType_GenericAlloc` reads the flag to decide, and these two have // to agree with it or an instance is allocated under one discipline and freed // under another - let (release, track) = if instance_dict(module, class) { + let (release, track) = if instance_dict(module, class) || class.environment { ("PyObject_GC_Del", " PyObject_GC_Track(by_block);\n") } else { ("PyObject_Free", "") @@ -779,6 +926,14 @@ fn emit_instance_recycling(module: &ModuleIr, class: &ClassIr) -> String { // list through the object header itself — the memset that resets an instance // would cut a live list in two let recycling = "#if !defined(Py_GIL_DISABLED) && !defined(Py_TRACE_REFS)"; + let (unset_recycled, unset_fresh) = if writes_absent_values(module, class) { + ( + format!("\x20 {type_name}_unset(by_block);\n"), + format!("\x20 if (by_fresh != NULL) {type_name}_unset(by_fresh);\n"), + ) + } else { + (String::new(), String::new()) + }; format!( "{recycling}\n\ static PyObject *{type_name}_recycled[{RECYCLED_INSTANCES}];\n\ @@ -798,10 +953,13 @@ fn emit_instance_recycling(module: &ModuleIr, class: &ClassIr) -> String { \x20 Py_SET_REFCNT(by_block, 1);\n\ \x20 if (by_type->tp_flags & Py_TPFLAGS_HEAPTYPE) Py_INCREF(by_type);\n\ {track}\ + {unset_recycled}\ \x20 return by_block;\n\ \x20 }}\n\ #endif\n\ - \x20 return PyType_GenericAlloc(by_type, by_items);\n\ + \x20 PyObject *by_fresh = PyType_GenericAlloc(by_type, by_items);\n\ + {unset_fresh}\ + \x20 return by_fresh;\n\ }}\n\n\ static void {type_name}_free(void *by_block) {{\n\ {recycling}\n\ @@ -840,6 +998,7 @@ fn emit_class_type(module: &ModuleIr, class: &ClassIr) -> String { let mut out = String::new(); let keeps_a_dict = instance_dict(module, class); + out.push_str(&emit_absent_values(module, class)); if frees_its_instances(module, class) { out.push_str(&emit_appended_storage(module, class)); } else { @@ -849,6 +1008,8 @@ fn emit_class_type(module: &ModuleIr, class: &ClassIr) -> String { // turns into an error if keeps_a_dict { out.push_str(&emit_collected_instance(module, class)); + } else if class.environment { + out.push_str(&emit_collected_environment(module, class)); } // dealloc releases each refcounted field, then the object let _ = writeln!( @@ -859,7 +1020,7 @@ fn emit_class_type(module: &ModuleIr, class: &ClassIr) -> String { // a list holding a half-freed object is what the next collection walks. a class // without a dict is not a collected type, and asking about an object with no // collector header in front of it reads memory that is not there - if keeps_a_dict { + if keeps_a_dict || class.environment { out.push_str(" PyObject_GC_UnTrack(self);\n"); } // a finalizer does not run itself: `subtype_dealloc` calls it, and a type that @@ -1029,9 +1190,9 @@ fn emit_class_type(module: &ModuleIr, class: &ClassIr) -> String { let _ = writeln!(out, " {release}"); } let _ = writeln!(out, " {storage}->{} = by_v;", field.member()); - // the byte beside an optional field is what a later read and the deallocation - // both ask, so filling one here has to answer them - if field.optional { + // the byte beside a field is what a later read and the deallocation both ask, + // so filling one here has to answer them + if field.has_presence_byte() { let _ = writeln!(out, " {storage}->{} = 1;", field.presence()); } // a fresh instance has published nothing, but `o.__init__(...)` runs this a @@ -1300,9 +1461,41 @@ fn release_old(field: &by_ir::function::FieldDecl, storage: &str) -> String { } } -/// the assignment that records an optional field as written +/// raising python's `AttributeError` for an instance `object` lacking `field` +/// +/// the wording is the lookup's that would have failed: a slot's descriptor names the +/// type in full, and the lookup through an instance's dict names it as the type does +fn missing_attribute(module: &ModuleIr, class: &ClassIr, object: &str, field: &str) -> String { + format!( + "By_FieldMissing({object}, {}, {});", + c_string(field), + i32::from(class.declares_slots && !instance_dict(module, class)) + ) +} + +/// the C condition under which an instance lacks `field`, read through `storage` +/// +/// `None` for a field no instance can lack. an optional field and one whose +/// representation has nothing spare keep a byte; a tagged `int` holds its error value, +/// which the allocation writes; an object holds `NULL`, which the allocation leaves +fn field_absent(field: &by_ir::function::FieldDecl, storage: &str) -> Option { + if field.has_presence_byte() { + return Some(format!("!{storage}->{}", field.presence())); + } + if !field.tracks_absence() { + return None; + } + let member = format!("{storage}->{}", field.member()); + Some(if field.ty == RType::INT { + format!("{member} == BY_INT_ERROR") + } else { + format!("{member} == NULL") + }) +} + +/// the assignment that records a field with a presence byte as written fn mark_present(field: &by_ir::function::FieldDecl, storage: &str) -> String { - if field.optional { + if field.has_presence_byte() { format!("\x20 {storage}->{} = 1;\n", field.presence()) } else { String::new() @@ -1491,18 +1684,15 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { } // getters and setters, so python sees ordinary attributes - for field in &class.fields { + for field in published_fields(class) { // python's own answer for an attribute the instance never got, and the // getter is where a caller from python meets it - let absent = if field.optional { - format!( - "\x20 if (!self->{}) {{ PyErr_Format(PyExc_AttributeError, \ - \"'%s' object has no attribute '%s'\", By_TypeName(selfobj), {}); return NULL; }}\n", - field.presence(), - c_string(&field.name) - ) - } else { - String::new() + let absent = match field_absent(field, "self") { + Some(absent) => format!( + "\x20 if ({absent}) {{ {} return NULL; }}\n", + missing_attribute(module, class, "selfobj", &field.name) + ), + None => String::new(), }; let _ = writeln!( out, @@ -1533,7 +1723,7 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { // a mutable field needs a setter, and the setter has to check its value for // the same reason the constructor does: an unboxed field cannot hold the // wrong representation - for field in &class.fields { + for field in published_fields(class) { if !class.writable() { continue; } @@ -1615,11 +1805,18 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { } else { "NULL".to_string() }; - let _ = writeln!(out, " {{{}, {present}}},", c_string(&field.name)); + // the same setter the field's descriptor publishes, and the one condition it + // is emitted under + let set = if class.writable() { + format!("{type_name}_set_{}", field.name) + } else { + "NULL".to_string() + }; + let _ = writeln!(out, " {{{}, {present}, {set}}},", c_string(&field.name)); } let _ = write!( out, - " {{NULL, NULL}}\n}};\n\ + " {{NULL, NULL, NULL}}\n}};\n\ static PyObject *{type_name}_get___dict__(PyObject *selfobj, void *closure) {{\n\ \x20 (void)closure;\n\ \x20 return By_InstanceDict(selfobj, {type_name}_dictfields);\n}}\n\ @@ -1640,7 +1837,7 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { } } let _ = writeln!(out, "static PyGetSetDef {type_name}_getset[] = {{"); - for field in &class.fields { + for field in published_fields(class) { // a defaulted field's entry is a descriptor of ours, written into the type's dict // at init over whatever stands there. an entry here as well would be replaced by // it and change nothing — but only while the two agree about which fields are @@ -1836,8 +2033,7 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { method_doc(method) ); } - // after the class's own, so that nothing counting entries in the table moves — - // `MakeClosure` takes the address of one + // after the class's own, so that nothing counting entries in the table moves if publishes_a_state_method(module, class) { let _ = writeln!( out, @@ -2080,9 +2276,16 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { // a static struct is what a class no name reaches gets: a generator's state or a // closure's environment, neither of which anything can ask about if !heap_type(module, class) { + let flags = if class.environment { + format!( + " .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,\n .tp_traverse = (traverseproc){type_name}_traverse,\n .tp_clear = (inquiry){type_name}_clear,\n" + ) + } else { + " .tp_flags = Py_TPFLAGS_DEFAULT,\n".to_string() + }; let _ = write!( out, - "static PyTypeObject {type_name} = {{\n PyVarObject_HEAD_INIT(NULL, 0)\n .tp_name = \"{dotted}.{}\",\n .tp_basicsize = sizeof({struct_name}),\n .tp_itemsize = 0,\n .tp_dealloc = (destructor){type_name}_dealloc,\n .tp_flags = Py_TPFLAGS_DEFAULT,\n{iterator}{dunders} .tp_methods = {type_name}_methods,\n .tp_getset = {type_name}_getset,\n .tp_init = {type_name}_init,\n .tp_new = PyType_GenericNew,\n }};\n\ + "static PyTypeObject {type_name} = {{\n PyVarObject_HEAD_INIT(NULL, 0)\n .tp_name = \"{dotted}.{}\",\n .tp_basicsize = sizeof({struct_name}),\n .tp_itemsize = 0,\n .tp_dealloc = (destructor){type_name}_dealloc,\n{flags}{iterator}{dunders} .tp_methods = {type_name}_methods,\n .tp_getset = {type_name}_getset,\n .tp_init = {type_name}_init,\n .tp_new = PyType_GenericNew,\n }};\n\ ", class.name ); @@ -2135,6 +2338,13 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { // none of the three slots that would take that over. it has to be **transitive** — // a subclass of a subclass that declared its own size would declare one smaller // than its base, which python rejects outright + // an allocation of this class's own where its fields start absent by value, which + // the recycling pair below supplies itself where there is one + let absent = if writes_absent_values(module, class) && !recycles_instances(module, class) { + format!("\x20 {{Py_tp_alloc, (void *){type_name}_alloc}},\n") + } else { + String::new() + }; let (basicsize, own_slots) = if external_storage(module, class) { // PEP 697: a *negative* size asks for that much room past whatever the base // allocated. `tp_new` stays the base's — it is the one that knows how big the @@ -2146,7 +2356,7 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { "\x20 {{Py_tp_dealloc, (void *){type_name}_dealloc}},\n\ \x20 {{Py_tp_traverse, (void *){type_name}_traverse}},\n\ \x20 {{Py_tp_clear, (void *){type_name}_clear}},\n\ - {init}" + {absent}{init}" ), ) } else if inherits_layout(module, class) { @@ -2214,7 +2424,7 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { format!("sizeof({struct_name})"), format!( "\x20 {{Py_tp_dealloc, (void *){type_name}_dealloc}},\n\ - {recycled}{walked}{construction}" + {recycled}{absent}{walked}{construction}" ), ) }; @@ -2946,7 +3156,7 @@ fn answers_for_its_classes(module: &ModuleIr) -> bool { if let Some(class) = instance_held(&boxed, receiver) && let Some(owner) = class_named(module, class) && !instance_dict(module, owner) - && !owner.fields.iter().any(|field| field.name == *name) + && publishing_class(module, owner, name).is_none() { return true; } @@ -2959,6 +3169,30 @@ fn answers_for_its_classes(module: &ModuleIr) -> bool { }) } +/// the nearest class from `class` up its in-module bases whose type publishes `field` +/// +/// a class that adds no field of its own is laid out with an empty field list and publishes +/// no descriptor for what its base declares: a lookup on it finds the base's +fn publishing_class<'a>( + module: &'a ModuleIr, + class: &'a ClassIr, + field: &str, +) -> Option<&'a ClassIr> { + let mut current = class; + // bounded by the class count, for the reason [`inherits_layout`] gives + for _ in 0..=module.classes.len() { + if current.fields.iter().any(|decl| decl.name == field) { + return Some(current); + } + current = current + .base + .as_ref() + .and_then(ClassBase::in_module) + .and_then(|base| class_named(module, base))?; + } + None +} + /// the fields this class keeps in storage of its own /// /// for a class appended over an emitted base that is the run past its base's fields: @@ -3176,6 +3410,32 @@ fn accessor_licence(module: &ModuleIr, class: &str, name: &str) -> String { ) } +/// the static holding the licence to read and write `class`'s `field` at its offset — see +/// `By_ArmField` +fn field_licence(module: &ModuleIr, class: &str, field: &str) -> String { + format!( + "by_field_stands_{}_{}_{}", + mangle(module.name.dotted()), + mangle(class), + mangle(field) + ) +} + +/// every `(class, field)` pair some read or write in this module tests +fn field_licences(module: &ModuleIr) -> BTreeSet<(String, String)> { + let mut wanted = BTreeSet::new(); + for function in module.all_functions() { + for block in &function.blocks { + for op in &block.ops { + if let Op::FieldStands { class, field, .. } = op { + wanted.insert((class.clone(), field.clone())); + } + } + } + } + wanted +} + /// every `(class, property)` pair some read or write in this module tests fn accessor_licences(module: &ModuleIr) -> BTreeSet<(String, String)> { let mut wanted = BTreeSet::new(); @@ -4462,6 +4722,16 @@ fn check_expr(module: &ModuleIr, ty: &RType, expr: &str) -> Option { } } +/// the interpreter's own object for an immediate that is one of its singletons +fn singleton(value: &Value) -> Option<&'static str> { + match value { + Value::None => Some("Py_None"), + Value::Bool(true) | Value::Bit(true) => Some("Py_True"), + Value::Bool(false) | Value::Bit(false) => Some("Py_False"), + _ => None, + } +} + /// unbox `expr` into the representation `ty` fn unbox_expr(ty: &RType, expr: &str) -> String { match ty { @@ -4763,7 +5033,7 @@ fn emit_cleanup( moved: Option, ) -> String { let owned = owned_registers(function); - let mut out = String::new(); + let mut releases = Vec::new(); for (index, decl) in function.registers.iter().enumerate() { if owned.get(index).copied() != Some(true) { continue; // a borrowed parameter belongs to the caller @@ -4777,9 +5047,21 @@ fn emit_cleanup( continue; // provably dead here } if let Some(release) = dec_ref(&decl.ty, &local(RegisterId(index))) { - let _ = writeln!(out, "{indent}{release}"); + releases.push(format!("{indent}{release}\n")); } } + // a release can run a finalizer, and the order a frame lets go of its locals in is + // one a program can see: 3.14 clears them last to first, where every version before + // it cleared them first to last. registers are numbered in the order the names were + // first bound, which is the order python lays the locals out in + if releases.len() < 2 { + return releases.concat(); + } + let mut out = String::from("#if PY_VERSION_HEX >= 0x030E0000\n"); + out.extend(releases.iter().rev().cloned()); + out.push_str("#else\n"); + out.extend(releases.iter().cloned()); + out.push_str("#endif\n"); out } @@ -4850,9 +5132,12 @@ fn dec_ref(ty: &RType, expr: &str) -> Option { } Some(match ty { RType::Primitive(Primitive::Int) => format!("By_DecRefTagged({expr});"), + // last to first, the order a `tuple` drops its elements in, so the finalizers + // they run come in python's order RType::Tuple(items) => items .iter() .enumerate() + .rev() .filter_map(|(index, item)| dec_ref(item, &format!("{expr}.f{index}"))) .collect::>() .join(" "), @@ -4899,7 +5184,14 @@ fn rich_compare_op(op: CmpOp) -> &'static str { fn value_expr(value: &Value) -> String { match value { Value::Register(id) => local(*id), - Value::Int(v) => format!("By_ShortFrom({v})"), + Value::Int(v) if SHORT_RANGE.contains(v) => format!("By_ShortFrom({v})"), + Value::Int(v) => LONG_LITERALS.with_borrow(|literals| { + match literals.iter().position(|literal| literal == v) { + Some(index) => format!("By_TaggedLiteral(by_long{index})"), + // only reachable if a literal appeared after collection, as for a string + None => format!("By_IntFromI64(INT64_C({v}))"), + } + }), Value::Fixed(v) => format!("INT64_C({v})"), Value::Float(v) => format!("{v:?}"), Value::Bool(v) | Value::Bit(v) => i32::from(*v).to_string(), @@ -4934,6 +5226,9 @@ thread_local! { /// the module's bytes literals, likewise static BYTE_LITERALS: std::cell::RefCell>> = const { std::cell::RefCell::new(Vec::new()) }; + /// the module's integer literals too wide to tag, likewise + static LONG_LITERALS: std::cell::RefCell> = + const { std::cell::RefCell::new(Vec::new()) }; } /// whether `value` is the register this function's own `*args` holds @@ -4968,15 +5263,22 @@ fn object_expr(function: &Function, value: &Value) -> String { } } +/// the fields a class publishes as attributes +/// +/// a closure environment publishes none. python has no name for it, and a cell's unset +/// value is not one a getter could hand out +fn published_fields(class: &ClassIr) -> &[by_ir::function::FieldDecl] { + if class.environment { + &[] + } else { + &class.fields + } +} + /// the entries the emitted method table carries *before* the class's own methods /// -/// the table is these followed by [`ClassIr::table_methods`], which drops `__new__` and the -/// halves of a property. anything that wants to *point* at an entry has to count the same -/// way — `MakeClosure` takes the address of one — so both the emission and the index read -/// this one list rather than each deriving the shape from the class. they did not, and the -/// index was taken from the unfiltered `methods` with no prefix at all: right only for a -/// class that is neither generic nor resumable and whose methods are all in the table, which -/// is every closure environment built so far and is not a rule anything states +/// the table is these followed by [`ClassIr::table_methods`], which drops `__new__`, the +/// halves of a property and a closure environment's nested functions fn synthetic_table_entries(class: &by_ir::function::ClassIr, type_name: &str) -> Vec { let mut out = Vec::new(); if class.generic { @@ -5369,6 +5671,24 @@ fn emit_op( assign_owned(module, function, *dest, &expr) }; } + // an operation whose fast path cannot fail is written out with its two paths + // apart, so the error test sits on the slow one alone — see `By_IntAddShort` + if function.value_type(&Value::Register(*dest)) == Some(RType::INT) + && let Some((fast, slow)) = split_int_binary(*op, lhs, rhs) + { + let target = local(*dest); + return format!( + " {{ ByTagged by_t;\n\ + \x20 if (BY_UNLIKELY(!{fast}({}, {}, &by_t))) {{\n\ + \x20 by_t = {slow};\n\ + \x20 if (BY_UNLIKELY(by_t == BY_INT_ERROR)) goto {};\n\ + \x20 }}\n\ + \x20 By_DecRefTagged({target}); {target} = by_t; }}\n", + value_expr(lhs), + value_expr(rhs), + error_label(error_target) + ); + } let call = match op { BinOp::Add => "By_IntAdd", BinOp::Sub => "By_IntSub", @@ -5456,6 +5776,18 @@ fn emit_op( }; assign_checked(module, function, *dest, &expr, error_target) } + Op::BuiltinStands { dest, name } => { + let site = format!("by_bs_{}", mangle(name)); + format!( + " {{ static ByBuiltinSite {site} = BY_BUILTIN_SITE_INIT;\n \ + char by_t = By_BuiltinStands(&{site}, by_module_dict, {});\n \ + if (BY_UNLIKELY(by_t == 2)) goto {};\n \ + {} = by_t; }}\n", + c_string(name), + error_label(error_target), + local(*dest), + ) + } Op::MethodStands { dest, src, @@ -5490,6 +5822,23 @@ fn emit_op( accessor_licence(module, class, name), ) } + Op::FieldStands { + dest, + src, + class, + field, + } => { + let Some(owner) = class_named(module, class) else { + return format!(" {} = 0;\n", local(*dest)); + }; + format!( + " {} = By_AccessorStands((PyObject *){}, {}_OBJ, &{});\n", + local(*dest), + value_expr(src), + owner.type_name(module.name.dotted()), + field_licence(module, class, field), + ) + } Op::LicenceHolds { src, class, @@ -5746,6 +6095,15 @@ fn emit_op( ); assign_checked(module, function, *dest, &expr, error_target) } + Op::ObjectRichCompare { dest, op, lhs, rhs } => { + let expr = format!( + "PyObject_RichCompare({}, {}, {})", + value_expr(lhs), + value_expr(rhs), + rich_compare_op(*op) + ); + assign_checked(module, function, *dest, &expr, error_target) + } Op::StrCompare { dest, op, lhs, rhs } => { let expr = format!( "By_StrCompare({}, {}, {})", @@ -5902,6 +6260,13 @@ fn emit_op( assign_checked(module, function, *dest, &expr, error_target) } Op::Box { dest, src } => { + // the borrow pass lends only a singleton this way, and a singleton needs no + // reference of the frame's own to stay alive + if function.register(*dest).is_some_and(|decl| decl.borrowed) + && let Some(singleton) = singleton(src) + { + return format!(" {} = {singleton};\n", local(*dest)); + } let Some(src_ty) = function.value_type(src) else { return String::new(); }; @@ -5992,6 +6357,20 @@ fn emit_op( ); assign_checked(module, function, *dest, &expr, error_target) } + Op::MergeKeywords { + dest, + container, + source, + callee, + } => { + let expr = format!( + "By_MergeKeywords({}, {}, {})", + value_expr(container), + value_expr(source), + value_expr(callee) + ); + assign_checked(module, function, *dest, &expr, error_target) + } Op::CallUnpacked { dest, callee, @@ -6262,9 +6641,6 @@ fn emit_op( fields_of(module, owner, "by_obj") ); for (field, value) in owner.fields.iter().zip(fields) { - // `tp_alloc` zeroes the block, so `None` leaves the field NULL — - // which is what an unset cell is - let Some(value) = value else { continue }; // a field a base declared sits in that base's region rather than this // class's, so it is reached through the base's type — `by_new` is only // the storage of what this class adds @@ -6274,6 +6650,17 @@ fn emit_op( } else { fields_of(module, declared, "by_obj") }; + // `None` leaves a field unset. `tp_alloc` zeroes the block, which is an + // unset pointer, but a zeroed tagged `int` is the value zero: its unset + // value is the error value, and it has to be written. a block the + // allocator hands back from its recycled ones is zeroed the same way, so + // this is what resets a cell there too + let Some(value) = value else { + if field.ty == RType::INT { + let _ = writeln!(out, " {target}->{} = BY_INT_ERROR;", field.member()); + } + continue; + }; if let Some(retain) = inc_ref(&field.ty, &value_expr(value)) { let _ = writeln!(out, " {retain}"); } @@ -6285,7 +6672,7 @@ fn emit_op( ); // the zero `tp_alloc` left says "never written", so a field filled here // has to say otherwise - if field.optional { + if field.has_presence_byte() { let _ = writeln!(out, " {target}->{} = 1;", field.presence()); } } @@ -6402,13 +6789,26 @@ fn emit_op( free, .. } => { - let call = format!( - "By_ReadCell({}->{}, {}, {})", + let ty = function + .register(*dest) + .map_or(RType::OBJECT, |decl| decl.ty.clone()); + let reader = if ty == RType::INT { + "By_ReadCellTagged" + } else { + "By_ReadCell" + }; + let read = format!( + "{reader}({}->{}, {}, {})", value_expr(receiver), mangle_member(field), c_string(field), i32::from(*free) ); + let call = if ty == RType::INT || ty == RType::OBJECT { + read + } else { + format!("({}){read}", ctype(module, &ty)) + }; assign_checked(module, function, *dest, &call, error_target) } Op::MakeClosure { @@ -6417,30 +6817,23 @@ fn emit_op( method, env, } => { - let Some(owner) = module + let Some(nested) = module .classes .iter() .find(|candidate| candidate.name == *class) - else { - return String::new(); - }; - // the index has to be into the *emitted* table, which is the synthetic - // entries followed by `table_methods` — not into `methods`, which still - // holds the `__new__` and property halves the table leaves out - let Some(index) = owner - .table_methods() - .position(|candidate| candidate.name == *method) - .map(|position| { - position - + synthetic_table_entries(owner, &owner.type_name(module.name.dotted())) - .len() + .and_then(|owner| { + owner + .methods + .iter() + .find(|candidate| candidate.name == *method) }) + .filter(|nested| nested.nested.is_some()) else { return String::new(); }; - let table = format!("{}_methods", owner.type_name(module.name.dotted())); let call = format!( - "By_MakeClosure(&{table}[{index}], (PyObject *)({}))", + "By_MakeFunction(&{}_spec, (PyObject *)({}))", + nested.wrapper_symbol(module.name.dotted()), value_expr(env) ); assign_checked(module, function, *dest, &call, error_target) @@ -6832,17 +7225,58 @@ fn emit_op( out.push_str(&assign_owned(module, function, *dest, &expr)); out } + Op::RequireField { + receiver, + class, + field, + } => { + let fields = receiver_fields(module, class, field, receiver); + // an optional field's own read asks, and answers with a class-level value + // where there is one, which raising here would pre-empt + let absent = field_decl(module, class, field) + .filter(|decl| !decl.optional) + .and_then(|decl| field_absent(decl, &fields)); + match absent { + Some(absent) => format!( + " if (BY_UNLIKELY({absent})) {{ {} goto {}; }}\n", + class_named(module, class).map_or_else(String::new, |owner| missing_attribute( + module, + owner, + &format!("(PyObject *){}", value_expr(receiver)), + field + )), + error_label(error_target) + ), + None => String::new(), + } + } + Op::FieldIsSet { + dest, + receiver, + class, + field, + } => { + let fields = receiver_fields(module, class, field, receiver); + match field_decl(module, class, field).and_then(|decl| field_absent(decl, &fields)) { + Some(absent) => format!(" {} = (char)!({absent});\n", local(*dest)), + None => format!(" {} = 1;\n", local(*dest)), + } + } Op::SetField { receiver, class, field, value, + moves, + present, } => { let fields = receiver_fields(module, class, field, receiver); let target = format!("{fields}->{}", mangle_member(field)); let mut out = String::new(); let ty = function.value_type(value).unwrap_or(RType::OBJECT); - if let Some(retain) = inc_ref(&ty, &value_expr(value)) { + // a moving store hands over the register's own reference, so there is none to + // take for the field + if !*moves && let Some(retain) = inc_ref(&ty, &value_expr(value)) { let _ = writeln!(out, " {retain}"); } // the release reads the old value, so an optional field that was never @@ -6863,8 +7297,14 @@ fn emit_op( } } let _ = writeln!(out, " {target} = {};", value_expr(value)); + // and the register is left empty, so the release every later exit makes of it + // finds nothing + if *moves && let Value::Register(id) = value { + let _ = writeln!(out, " {} = {};", local(*id), ty.undefined()); + } if let Some(decl) = field_decl(module, class, field) - && decl.optional + && decl.has_presence_byte() + && !*present { let _ = writeln!(out, " {}->{} = 1;", fields, decl.presence()); } @@ -7074,6 +7514,55 @@ fn emit_op( assign_owned(module, function, *dest, &expr) } Op::PopHandled { value } => format!(" By_PopHandled({});\n", value_expr(value)), + // emptied before the release, so a finalizer the release runs finds nothing left + // in the register + Op::Move { dest, src, path } => { + let Value::Register(id) = src else { + return String::new(); + }; + let Some(ty) = function + .register(*id) + .and_then(|decl| decl.ty.element(path)) + else { + return String::new(); + }; + let mut place = local(*id); + for index in path { + let _ = write!(place, ".f{index}"); + } + // the place is emptied before the destination lets go of what it held, + // which can run a finalizer + let mut out = format!( + " {{ {} by_m = {place}; {place} = {};\n", + ctype(module, ty), + ty.undefined() + ); + out.push_str(&assign_owned(module, function, *dest, "by_m")); + out.push_str(" }\n"); + out + } + Op::Release { value, path } => { + let Value::Register(id) = value else { + return String::new(); + }; + let Some(decl) = function.register(*id).filter(|decl| !decl.borrowed) else { + return String::new(); + }; + let Some(ty) = decl.ty.element(path) else { + return String::new(); + }; + let mut target = local(*id); + for index in path { + let _ = write!(target, ".f{index}"); + } + dec_ref(ty, "by_old").map_or_else(String::new, |release| { + format!( + " {{ {} by_old = {target}; {target} = {}; {release} }}\n", + ctype(module, ty), + ty.undefined() + ) + }) + } Op::RaiseObject { exception, cause } => format!( " By_RaiseObject({}, {});\n goto {};\n", value_expr(exception), @@ -7179,13 +7668,19 @@ fn emit_op( out.push_str(&commit_checked(function, *dest, error_target)); out } - Op::StrConcatInt { dest, lhs, value } => { + Op::StrConcatInt { + dest, + lhs, + value, + mutation, + } => { let mut out = resolve_str(error_target); let _ = writeln!( out, - " PyObject *by_t = By_StrConcatInt({}, by_fn, {});", + " PyObject *by_t = By_StrConcatInt({}, by_fn, {}, {});", value_expr(lhs), - value_expr(value) + value_expr(value), + u8::from(*mutation == Mutation::InPlace) ); let _ = writeln!(out, " Py_DECREF(by_fn);"); out.push_str(&commit_checked(function, *dest, error_target)); @@ -7196,12 +7691,26 @@ fn emit_op( lhs, rhs, consumes_lhs, + concatenation, } => { + // a join asks neither piece anything, and an operator asks both operands' + // methods whenever either is not an exact `str` + let (concat, append, in_place) = match concatenation { + Concatenation::Join => ("By_StrConcat", "By_StrAppend", ""), + Concatenation::Operator(Mutation::Fresh) => ("By_StrAdd", "By_StrAddAppend", ", 0"), + Concatenation::Operator(Mutation::InPlace) => { + ("By_StrAdd", "By_StrAddAppend", ", 1") + } + }; let Some(Value::Register(source)) = consumes_lhs.then_some(lhs) else { - let expr = format!("By_StrConcat({}, {})", value_expr(lhs), value_expr(rhs)); + let expr = format!( + "{concat}({}, {}{in_place})", + value_expr(lhs), + value_expr(rhs) + ); return assign_checked(module, function, *dest, &expr, error_target); }; - // the register is emptied *before* the call, because `By_StrAppend` takes + // the register is emptied *before* the call, because the append takes // the reference over whether it succeeds or not — a register still naming // it would be released a second time on the way out. // @@ -7212,7 +7721,7 @@ fn emit_op( format!( " {{ PyObject * by_rhs = {};\n \ PyObject * by_lhs = {held}; {held} = NULL;\n \ - PyObject * by_t = By_StrAppend(by_lhs, by_rhs);\n{}", + PyObject * by_t = {append}(by_lhs, by_rhs{in_place});\n{}", value_expr(rhs), commit_checked(function, *dest, error_target), ) @@ -7363,28 +7872,87 @@ fn defer_tests(function: &Function, receiver: bool) -> Vec { .collect() } +/// the fast path and the slow call of a tagged operation whose fast path cannot fail +/// +/// a short is an even word, and a sum, difference, product or bitwise combination of two +/// is one too — so the fast path never produces the error value, and only the slow call, +/// which is the one that reaches cpython, needs testing for it. `//` and `%` are not here: +/// a zero divisor fails on the fast path +fn split_int_binary(op: BinOp, lhs: &Value, rhs: &Value) -> Option<(&'static str, String)> { + let (l, r) = (value_expr(lhs), value_expr(rhs)); + Some(match op { + BinOp::Add => ( + "By_IntAddShort", + format!("By_IntSlowBinary({l}, {r}, \"+\")"), + ), + BinOp::Sub => ( + "By_IntSubShort", + format!("By_IntSlowBinary({l}, {r}, \"-\")"), + ), + BinOp::Mul => ( + "By_IntMulShort", + format!("By_IntSlowBinary({l}, {r}, \"*\")"), + ), + BinOp::BitAnd => ( + "By_IntAndShort", + format!("By_IntSlowBitwise({l}, {r}, '&')"), + ), + BinOp::BitOr => ("By_IntOrShort", format!("By_IntSlowBitwise({l}, {r}, '|')")), + BinOp::BitXor => ( + "By_IntXorShort", + format!("By_IntSlowBitwise({l}, {r}, '^')"), + ), + BinOp::FloorDiv | BinOp::Mod | BinOp::TrueDiv | BinOp::Pow | BinOp::Shl | BinOp::Shr => { + return None; + } + }) +} + +/// the parameters a wrapper is declared with: the fastcall convention a method table +/// calls through, or — for a nested function — the vectorcall one its function object +/// is called through, which hands over the object rather than a receiver +fn wrapper_parameters(function: &Function) -> &'static str { + match function.nested { + Some(_) => "PyObject *by_callable, PyObject *const *args, size_t nargsf, PyObject *kwnames", + None => "PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames", + } +} + /// the python-facing wrapper: unbox each argument, call the native entry, box the /// result. the argument checks here are the `parameters` soundness position fn emit_wrapper(module: &ModuleIr, function: &Function, is_method: bool) -> String { let mut out = format!( - "static PyObject *{}(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) {{\n", - function.wrapper_symbol(module.name.dotted()) + "static PyObject *{}({}) {{\n", + function.wrapper_symbol(module.name.dotted()), + wrapper_parameters(function) ); + // a nested function is called as the function object itself, and its receiver is + // the environment that object holds + if function.nested.is_some() { + out.push_str( + " PyObject *self = By_FunctionEnvironment(by_callable);\n\ + \x20 Py_ssize_t nargs = PyVectorcall_NARGS(nargsf);\n", + ); + } // a method's receiver arrives in `self`, so it does not count as an argument if !is_method { out.push_str(" (void)self;\n"); } + // python counts a method's `self` in an arity error, and a nested function has no + // receiver it can see + let counted = i32::from(is_method && function.nested.is_none()); let bound = Bound::of(function, is_method); let explicit = bound.params.len(); // one binding pass over positionals and keywords together, so a caller may use // either. `by_bound[i]` is NULL where the caller did not supply that parameter, // and the per-parameter code below fills those from the defaults out.push_str(&bound.declare()); - // python names a method by its class in an arity error, and a bare function by - // itself - let reported = c_string(&match &function.owner { - Some(owner) => format!("{owner}.{}", function.name), - None => function.name.clone(), + // python names a method by its class in an arity error, a nested function by the + // frames it is written in, and a bare function by itself + let reported = c_string(&match (&function.nested, &function.owner) { + (Some(nested), _) => nested.qualname.clone(), + (None, Some(owner)) => format!("{owner}.{}", function.name), + (None, None) => function.name.clone(), }); // a boundary with nothing optional and nothing out of positional reach can say // so, and take the call that fills each parameter once without walking the @@ -7393,18 +7961,16 @@ fn emit_wrapper(module: &ModuleIr, function: &Function, is_method: bool) -> Stri let _ = if bound.is_plain() { writeln!( out, - " if (By_BindArgsQuick(args, nargs, kwnames, by_names, {explicit}, by_required, by_bound, {reported}, {}) < 0) return NULL;", - i32::from(is_method) + " if (By_BindArgsQuick(args, nargs, kwnames, by_names, {explicit}, by_required, by_bound, {reported}, {counted}) < 0) return NULL;" ) } else { writeln!( out, - " if (By_BindArgs(args, nargs, kwnames, by_names, {explicit}, by_required, {}, {}, by_bound, {}, {}, {reported}, {}) < 0) return NULL;", + " if (By_BindArgs(args, nargs, kwnames, by_names, {explicit}, by_required, {}, {}, by_bound, {}, {}, {reported}, {counted}) < 0) return NULL;", bound.posonly, bound.kwonly, i32::from(bound.vararg), - i32::from(bound.kwarg), - i32::from(is_method) + i32::from(bound.kwarg) ) }; @@ -8395,6 +8961,28 @@ fn emit_module_init(module: &ModuleIr) -> String { ); } + // the type every nested function is, ready before anything below can make one + let function_type = if module + .all_functions() + .any(|function| function.nested.is_some()) + { + " if (PyType_Ready(&By_FunctionType) < 0) return -1;\n" + } else { + "" + }; + // `By_DictFind` reads a private field only once dicts of known form have been seen to + // read as that form, and they are built here, before any code that could look one up + let dict_kinds = if module.all_functions().any(|function| { + function + .blocks + .iter() + .flat_map(|block| &block.ops) + .any(|op| matches!(op, Op::DictFind { .. })) + }) { + " By_CheckDictKinds();\n" + } else { + "" + }; let mut class_init = String::new(); // a class decorator is arbitrary python handed the class, and what it reads is the // class *body* — its annotations, its class-level defaults, what its `__dict__` @@ -8738,6 +9326,12 @@ fn emit_module_init(module: &ModuleIr) -> String { literal.len() ); } + for (index, literal) in collect_long_literals(module).iter().enumerate() { + let _ = writeln!( + literal_init, + " by_long{index} = PyLong_FromLongLong(INT64_C({literal}));\n if (by_long{index} == NULL) return -1;" + ); + } // taken while the interpreted definition is still the one under this name let mut interpreted_init = String::new(); @@ -8833,6 +9427,21 @@ fn emit_module_init(module: &ModuleIr) -> String { )) }) .collect(); + // the field's getter is what the descriptor standing under the name has to carry, + // whichever of the two descriptors a field is published through + let arm_fields: String = field_licences(module) + .iter() + .filter_map(|(class, field)| { + let owner = class_named(module, class)?; + let type_name = owner.type_name(module.name.dotted()); + let publisher = publishing_class(module, owner, field)?.type_name(module.name.dotted()); + Some(format!( + " By_ArmField(&{}, {type_name}_OBJ, {}, {publisher}_get_{field});\n", + field_licence(module, class, field), + c_string(field), + )) + }) + .collect(); // a class holding instance memory back holds it for as long as it might be asked // for another instance, which is as long as the module stands. handing it back when // the module goes is what keeps the whole scheme from being a leak @@ -8907,7 +9516,16 @@ fn emit_module_init(module: &ModuleIr) -> String { out.push_str(&emit_install_verification(module)); let _ = write!( out, - "static int by_exec(PyObject *module) {{\n\ + "static struct PyModuleDef by_module;\n\n\ + static PyObject *by_create(PyObject *spec, PyModuleDef *def) {{\n\ + \x20 return By_CreateModule(spec, def);\n\ + }}\n\n\ + static int by_exec(PyObject *module) {{\n\ + \x20 /* the module this interpreter already imported, handed back by `by_create`\n\ + \x20 * and then to this slot again: its state is set up, and running the body a\n\ + \x20 * second time would rebind every name in it */\n\ + \x20 int by_executed = By_ModuleExecuted(module, &by_module);\n\ + \x20 if (by_executed != 0) return by_executed < 0 ? -1 : 0;\n\ \x20 PyObject *dict = PyModule_GetDict(module);\n\ \x20 if (dict == NULL) return -1;\n\ \x20 by_module_dict = dict;\n\ @@ -8924,6 +9542,8 @@ fn emit_module_init(module: &ModuleIr) -> String { \x20 * and a second execution of this module invalidates every memo the\n\ \x20 * first one left behind */\n\ \x20 By_WatchModule(dict);\n\ + {dict_kinds}\ + {function_type}\ {literal_init}\ {run_body}\ {layout_guard}\ @@ -8941,9 +9561,12 @@ fn emit_module_init(module: &ModuleIr) -> String { {decorators}\ {arm_dispatch}\ {arm_accessors}\ + {arm_fields}\ + \x20 if (By_RecordModule(module, &by_module) < 0) return -1;\n\ \x20 return 0;\n\ }}\n\n\ static PyModuleDef_Slot by_slots[] = {{\n\ + \x20 {{Py_mod_create, (void *)by_create}},\n\ \x20 {{Py_mod_exec, (void *)by_exec}},\n\ #if PY_VERSION_HEX >= 0x030D0000\n\ \x20 /* compiled functions hold no shared mutable state: every register\n\ @@ -9235,6 +9858,7 @@ mod tests { module.classes.push(ClassIr { name: "Wrapped".to_string(), immutable: false, + environment: false, exported: true, base: Some(ClassBase::External(vec!["Exception".to_string()])), inherited_init: false, @@ -9246,6 +9870,7 @@ mod tests { properties: Vec::new(), slot_aliases: Vec::new(), fields: vec![by_ir::function::FieldDecl { + cell: false, name: "tag".to_string(), ty: RType::OBJECT, default: None, @@ -9531,6 +10156,7 @@ mod tests { wakeup.base = None; wakeup.declares_slots = true; wakeup.fields.push(by_ir::function::FieldDecl { + cell: false, name: "spare".to_string(), ty: RType::OBJECT, default: None, @@ -10108,6 +10734,7 @@ mod tests { ClassIr { name: "Wrapped".to_string(), immutable: false, + environment: false, exported: true, base: Some(ClassBase::External(vec!["Exception".to_string()])), inherited_init: false, @@ -10119,6 +10746,7 @@ mod tests { properties: Vec::new(), slot_aliases: Vec::new(), fields: vec![by_ir::function::FieldDecl { + cell: false, name: "tag".to_string(), ty: RType::OBJECT, default: None, @@ -10212,6 +10840,7 @@ mod tests { inner.name = "Deeper".to_string(); inner.base = Some(ClassBase::InModule("Wrapped".to_string())); inner.fields.push(by_ir::function::FieldDecl { + cell: false, name: "depth".to_string(), ty: RType::OBJECT, default: None, @@ -10397,14 +11026,13 @@ mod tests { #[test] fn a_write_computes_before_it_releases() { let c = emit_module(&module_with(add())); - assert!( - c.contains( - "{ ByTagged by_t = By_IntAdd(r0, r1);\n\ - \x20 if (BY_UNLIKELY(by_t == BY_INT_ERROR)) goto by_error;\n\ - \x20 By_DecRefTagged(r2); r2 = by_t; }" - ), - "{c}" - ); + let compute = c + .find("By_IntAddShort(r0, r1, &by_t)") + .expect("the addition is emitted"); + let release = c[compute..] + .find("By_DecRefTagged(r2); r2 = by_t;") + .expect("the old value is released"); + assert!(release > 0, "the release must follow the computation: {c}"); } #[test] @@ -10423,7 +11051,7 @@ mod tests { builder.terminate(Terminator::Return(Value::Register(acc))); let c = emit_module(&module_with(builder.finish())); let compute = c - .find("By_IntAdd(r1, r0)") + .find("By_IntAddShort(r1, r0, &by_t)") .expect("the addition is emitted"); let release = c[compute..] .find("By_DecRefTagged(r1);") @@ -10434,6 +11062,68 @@ mod tests { ); } + #[test] + fn a_tagged_add_tests_for_an_error_on_its_slow_path_only() { + // the fast path of `+` adds two tagged shorts, and a sum of two even words is even, + // so it can never be the error value `1`. only the slow path, which calls into + // cpython, can fail — and a test after the two paths merge is one the C compiler + // cannot see past, so every sum paid for it + for (op, fast, slow) in [ + ( + BinOp::Add, + "By_IntAddShort", + "By_IntSlowBinary(r0, r1, \"+\")", + ), + ( + BinOp::Sub, + "By_IntSubShort", + "By_IntSlowBinary(r0, r1, \"-\")", + ), + ( + BinOp::Mul, + "By_IntMulShort", + "By_IntSlowBinary(r0, r1, \"*\")", + ), + ( + BinOp::BitAnd, + "By_IntAndShort", + "By_IntSlowBitwise(r0, r1, '&')", + ), + ( + BinOp::BitOr, + "By_IntOrShort", + "By_IntSlowBitwise(r0, r1, '|')", + ), + ( + BinOp::BitXor, + "By_IntXorShort", + "By_IntSlowBitwise(r0, r1, '^')", + ), + ] { + let mut builder = FunctionBuilder::new("sum", RType::INT); + let a = builder.param("a", RType::INT); + let b = builder.param("b", RType::INT); + let out = builder.temp(RType::INT); + builder.push(Op::IntBinary { + dest: out, + op, + lhs: Value::Register(a), + rhs: Value::Register(b), + }); + builder.terminate(Terminator::Return(Value::Register(out))); + let c = emit_module(&module_with(builder.finish())); + let expected = format!( + " {{ ByTagged by_t;\n\ + \x20 if (BY_UNLIKELY(!{fast}(r0, r1, &by_t))) {{\n\ + \x20 by_t = {slow};\n\ + \x20 if (BY_UNLIKELY(by_t == BY_INT_ERROR)) goto by_error;\n\ + \x20 }}\n\ + \x20 By_DecRefTagged(r2); r2 = by_t; }}\n" + ); + assert!(c.contains(&expected), "{op:?}: {c}"); + } + } + #[test] fn copying_a_register_retains_before_it_releases() { // `a = a` must not release the value it is about to store @@ -10604,6 +11294,7 @@ mod tests { module.classes.push(ClassIr { name: "Point".to_string(), immutable: false, + environment: false, exported: true, base: None, inherited_init: false, @@ -10615,6 +11306,7 @@ mod tests { properties: Vec::new(), slot_aliases: Vec::new(), fields: vec![by_ir::function::FieldDecl { + cell: false, name: "x".to_string(), ty: RType::INT, default: None, @@ -10895,6 +11587,7 @@ mod tests { lhs: Value::Register(a), rhs: Value::Register(b), consumes_lhs: false, + concatenation: Concatenation::Operator(Mutation::Fresh), }); builder.terminate(Terminator::Return(Value::Register(joined))); let c = emit_module(&module_with(builder.finish())); @@ -10916,6 +11609,7 @@ mod tests { lhs: Value::Register(out), rhs: Value::Register(seed), consumes_lhs: true, + concatenation: Concatenation::Operator(Mutation::Fresh), }); builder.terminate(Terminator::Return(Value::Register(grown))); let c = emit_module(&module_with(builder.finish())); @@ -10926,8 +11620,30 @@ mod tests { )), "{c}" ); - assert!(c.contains("By_StrAppend(by_lhs, by_rhs)"), "{c}"); - assert!(!c.contains("By_StrConcat("), "{c}"); + assert!(c.contains("By_StrAddAppend(by_lhs, by_rhs, 0)"), "{c}"); + assert!(!c.contains("By_StrAdd("), "{c}"); + } + + #[test] + fn a_borrowed_singleton_is_stored_without_a_reference() { + let mut builder = FunctionBuilder::new("pick", RType::OBJECT); + let none = builder.temp(RType::OBJECT); + let out = builder.temp(RType::OBJECT); + builder.push(Op::Box { + dest: none, + src: Value::None, + }); + builder.push(Op::CallPython { + dest: out, + callee: "repr".to_string(), + args: vec![Value::Register(none)], + }); + builder.terminate(Terminator::Return(Value::Register(out))); + let mut function = builder.finish(); + function.registers[none.index()].borrowed = true; + let c = emit_module(&module_with(function)); + assert!(c.contains("r0 = Py_None;"), "{c}"); + assert!(!c.contains("By_BoxNone()"), "{c}"); } #[test] @@ -10946,6 +11662,24 @@ mod tests { ); } + #[test] + fn an_integer_literal_too_wide_to_tag_is_built_once() { + // a short holds one bit fewer than the word, so `By_ShortFrom` of this literal + // would shift its top bit away and hand back a negative number + let mut builder = FunctionBuilder::new("edge", RType::INT); + let out = builder.temp(RType::INT); + builder.assign(out, Value::Int(1 << 62)); + builder.terminate(Terminator::Return(Value::Register(out))); + let c = emit_module(&module_with(builder.finish())); + assert!(c.contains("static PyObject *by_long0 = NULL;"), "{c}"); + assert!( + c.contains("by_long0 = PyLong_FromLongLong(INT64_C(4611686018427387904));"), + "{c}" + ); + assert!(c.contains("By_TaggedLiteral(by_long0)"), "{c}"); + assert!(!c.contains("By_ShortFrom(4611686018427387904)"), "{c}"); + } + #[test] fn a_guard_clause_still_releases_what_was_written_before_it() { // the refcount pass narrows the release set; getting the direction wrong @@ -11210,6 +11944,41 @@ mod tests { assert!(text.contains("Py_XDECREF(r1.f0)"), "{text}"); } + #[test] + fn a_moved_element_empties_its_place_before_the_destination_lets_go() { + // the destination's old value can have a finalizer, which must find the place + // the reference came from already empty rather than still holding it + let mut builder = FunctionBuilder::new("f", RType::INT); + let text_in = builder.param("s", RType::STR); + let pair = builder.temp(RType::Tuple(Box::new([RType::STR, RType::STR]))); + let head = builder.local("head", RType::STR); + let length = builder.temp(RType::INT); + builder.push(Op::CallNative { + owner: None, + dest: Some(pair), + callee: "split".to_string(), + args: vec![Value::Register(text_in)], + }); + builder.push(Op::Move { + dest: head, + src: Value::Register(pair), + path: Box::new([0]), + }); + builder.push(Op::Len { + dest: length, + src: Value::Register(head), + }); + builder.terminate(Terminator::Return(Value::Register(length))); + let function = builder.finish(); + + let text = emit_function(&ModuleIr::new("app"), &function); + assert!( + text.contains(" { PyObject * by_m = r1.f0; r1.f0 = NULL;\n { PyObject * by_t = by_m; Py_XDECREF(r2); r2 = by_t; }\n }\n"), + "{text}" + ); + assert!(!text.contains("By_XIncRef(r1.f0)"), "{text}"); + } + #[test] fn a_line_directive_points_at_the_by_source() { let mut builder = FunctionBuilder::new("f", RType::INT); @@ -11312,6 +12081,8 @@ mod tests { class: "Holder".to_string(), field: "value".to_string(), value: Value::Register(value), + moves: false, + present: false, }); builder.terminate(Terminator::Return(Value::None)); let mut init = builder.finish(); @@ -11320,6 +12091,7 @@ mod tests { module.classes.push(ClassIr { name: "Holder".to_string(), immutable: false, + environment: false, exported: true, base: None, inherited_init: false, @@ -11331,6 +12103,7 @@ mod tests { properties: Vec::new(), slot_aliases: Vec::new(), fields: vec![by_ir::function::FieldDecl { + cell: false, name: "value".to_string(), ty: RType::OBJECT, default: None, @@ -11410,6 +12183,7 @@ mod tests { module.classes.push(ClassIr { name: "outer$env".to_string(), immutable: false, + environment: false, exported: false, base: None, inherited_init: false, @@ -11421,6 +12195,7 @@ mod tests { properties: Vec::new(), slot_aliases: Vec::new(), fields: vec![by_ir::function::FieldDecl { + cell: false, name: "$default$cb$name".to_string(), ty: RType::OBJECT, default: None, @@ -11474,6 +12249,8 @@ mod tests { class: "Var".to_string(), field: name.to_string(), value: Value::Register(value), + moves: false, + present: false, }); } builder.terminate(Terminator::Return(Value::None)); @@ -11483,6 +12260,7 @@ mod tests { module.classes.push(ClassIr { name: "Var".to_string(), immutable: false, + environment: false, exported: true, base: None, inherited_init: false, @@ -11496,6 +12274,7 @@ mod tests { fields: ["a", "rest", "extra"] .into_iter() .map(|name| by_ir::function::FieldDecl { + cell: false, name: name.to_string(), ty: RType::OBJECT, default: None, @@ -11599,71 +12378,52 @@ mod tests { ); } - /// a closure takes the *address* of a table entry, so the index has to be into the - /// table as emitted — past the synthetic entries a generic or resumable class carries, - /// and not counting the `__new__` the table leaves out. it used to be read off the - /// unfiltered method list with no prefix, which is right only for a class that has - /// neither, and every closure environment built so far happens to be one. - /// - /// the two effects are asserted apart because together they cancel: a `__new__` - /// dropped and a `__class_getitem__` added land on the number the old reading gave + /// a nested function is a function object of its own over its environment, not a + /// method of that environment. so the environment's table does not list it, and a + /// closure is made from the nested function's spec, which names the boundary a call + /// reaches and the names python calls it by #[test] - fn a_closure_points_at_the_entry_the_table_actually_holds() { - let index_of_call = |generic: bool, with_new: bool| { - let entry = |name: &str| { - let mut builder = FunctionBuilder::new(name, RType::OBJECT); - builder.param("self", RType::OBJECT); - builder.terminate(Terminator::Return(Value::None)); - let mut method = builder.finish(); - method.owner = Some("Env".to_string()); - method - }; - let mut class = appending_class(); - class.name = "Env".to_string(); - class.base = None; - class.generic = generic; - class.methods = if with_new { - vec![entry("__new__"), entry("call")] - } else { - vec![entry("call")] - }; - - let mut builder = FunctionBuilder::new("make", RType::OBJECT); - let env = builder.param("env", RType::OBJECT); - let closure = builder.temp(RType::OBJECT); - builder.push(Op::MakeClosure { - dest: closure, - class: "Env".to_string(), - method: "call".to_string(), - env: Value::Register(env), - }); - builder.terminate(Terminator::Return(Value::Register(closure))); + fn a_closure_is_made_from_its_spec_and_is_no_method_of_its_environment() { + let mut builder = FunctionBuilder::new("call", RType::OBJECT); + builder.param("self", RType::OBJECT); + builder.terminate(Terminator::Return(Value::None)); + let mut nested = builder.finish(); + nested.owner = Some("Env".to_string()); + nested.nested = Some(by_ir::function::NestedName { + name: "call".to_string(), + qualname: "make..call".to_string(), + }); + let mut class = appending_class(); + class.name = "Env".to_string(); + class.base = None; + class.environment = true; + class.methods = vec![nested]; - let mut module = module_with(builder.finish()); - module.classes = vec![class]; - emit_module(&module) - }; + let mut builder = FunctionBuilder::new("make", RType::OBJECT); + let env = builder.param("env", RType::OBJECT); + let closure = builder.temp(RType::OBJECT); + builder.push(Op::MakeClosure { + dest: closure, + class: "Env".to_string(), + method: "call".to_string(), + env: Value::Register(env), + }); + builder.terminate(Terminator::Return(Value::Register(closure))); + let mut module = module_with(builder.finish()); + module.classes = vec![class]; + let c = emit_module(&module); - // a `__new__` is not in the table, so the method after it is still entry zero. - // the unfiltered reading said one, which is the entry past the end here - let dropped = index_of_call(false, true); assert!( - dropped.contains("By_MakeClosure(&By_app_Env_Type_methods[0],"), - "{dropped}" + c.contains("By_MakeFunction(&byw_app_Env_call_spec, (PyObject *)(r0))"), + "{c}" ); - - // and a generic class opens with `__class_getitem__`, so its own first method is - // entry one. the unfiltered reading said zero, which is `__class_getitem__` - let shifted = index_of_call(true, false); assert!( - shifted.contains("By_MakeClosure(&By_app_Env_Type_methods[1],"), - "{shifted}" + c.contains( + "static const ByFunctionSpec byw_app_Env_call_spec = { byw_app_Env_call, \"call\", \"make..call\", NULL, &by_module_dict };" + ), + "{c}" ); - let table = &shifted[shifted - .find("static PyMethodDef By_app_Env_Type_methods[]") - .expect("a table")..]; - let table = &table[..table.find("};").expect("a table end")]; - assert!(table.contains("__class_getitem__"), "{table}"); + assert!(!c.contains("{\"call\", (PyCFunction)"), "{c}"); } /// a sealed leaf class — no base, nothing derived from it, no decorator — keeps a /// dead instance's memory back for the next one, and hands it over when the module @@ -11696,6 +12456,36 @@ mod tests { ); } + /// an `int` field reads a zeroed block as the value zero, so an instance whose + /// `__init__` never runs has to be given the error value there by its allocation — + /// and an object field reads the zero as `NULL`, which needs nothing + #[test] + fn an_int_field_starts_absent_from_the_allocation() { + let mut module = module_with(add()); + let mut counted = sealed_class(); + counted.fields[0].ty = RType::INT; + module.classes = vec![counted]; + let c = emit_module(&module); + assert!( + c.contains("static void By_app_Point_Type_unset(PyObject *selfobj) {") + && c.contains("self->by_f_x = BY_INT_ERROR;"), + "{c}" + ); + assert!( + c.contains("By_app_Point_Type_unset(by_block);") + && c.contains("if (by_fresh != NULL) By_app_Point_Type_unset(by_fresh);"), + "a recycled block and a fresh one are both reset: {c}" + ); + + let module = { + let mut module = module_with(add()); + module.classes = vec![sealed_class()]; + module + }; + let c = emit_module(&module); + assert!(!c.contains("_unset"), "{c}"); + } + /// a class python can subclass must not: `tp_alloc` and `tp_free` are inherited, so a /// subclass would ask this allocator for an instance of a size it has never seen and /// be handed a block laid out for the base. neither may the subclass, whose layout is @@ -11826,6 +12616,7 @@ mod tests { ClassIr { name: "Point".to_string(), immutable: false, + environment: false, exported: true, base: None, inherited_init: false, @@ -11837,6 +12628,7 @@ mod tests { properties: Vec::new(), slot_aliases: Vec::new(), fields: vec![by_ir::function::FieldDecl { + cell: false, name: "x".to_string(), ty: RType::OBJECT, default: None, diff --git a/crates/by_ir/src/builder.rs b/crates/by_ir/src/builder.rs index d87f6befef..2e6e88c84f 100644 --- a/crates/by_ir/src/builder.rs +++ b/crates/by_ir/src/builder.rs @@ -348,6 +348,7 @@ impl FunctionBuilder { coroutine_body: None, doc: self.doc, takes_a_weak_reference: self.takes_a_weak_reference, + nested: None, } } } diff --git a/crates/by_ir/src/function.rs b/crates/by_ir/src/function.rs index 1a8f1b182c..d92dfd2fb4 100644 --- a/crates/by_ir/src/function.rs +++ b/crates/by_ir/src/function.rs @@ -297,7 +297,9 @@ pub struct Function { /// so the wrapper hands the whole call to the interpreted definition, which is /// exactly the code the annotation describes. /// - /// `.by` opts out of the promotion, so this is empty for every `.by` function + /// `.by` and `strict-float` opt out of the promotion, and the annotation still + /// admits a subclass of `float`, whose operators and type a `double` does not + /// carry — so every `double` parameter of a module-level function is one of these pub deferring: Vec, /// the indices of parameters whose default is not an immediate. /// @@ -364,6 +366,23 @@ pub struct Function { /// instead: a frame that hands one of these an instance of a class this module lays /// out is the frame that declines, and declining it takes its class with it pub takes_a_weak_reference: bool, + /// what python calls a *nested* function, which only a nested function has + /// + /// its receiver is the environment its captures live in, but python sees neither the + /// environment nor a method of one: it sees a function, named by the frames it is + /// written in. so such a function is published as a function object of its own over + /// the environment, and its boundary takes the environment off that object and + /// reports arity in its own name, counting no receiver + pub nested: Option, +} + +/// the names python gives a nested function — see [`Function::nested`] +#[derive(Debug, Clone, PartialEq)] +pub struct NestedName { + /// `__name__`, which is `` for a lambda + pub name: String, + /// `__qualname__`: `counter..step` + pub qualname: String, } impl Function { @@ -489,6 +508,10 @@ impl Function { pub struct FieldDecl { pub name: String, pub ty: crate::rtype::RType, + /// whether this is a closure environment's *shared* cell: a name more than one frame + /// writes, which every one of them reads and writes here, and which starts unset. a + /// capture that is only copied in is always set, so it is read without the test + pub cell: bool, /// the constructor's default for it, `None` where it has none /// /// only an immediate, for the same reason a parameter default is: it is @@ -523,6 +546,28 @@ impl FieldDecl { self.name.contains('$') } + /// whether an instance can be without this field + /// + /// every field a class declares can be: `C.__new__(C)` makes an instance `__init__` + /// never ran on. what cannot is a field the compiler made up, which no instance of + /// the class python can see ever lacks, and a closure's shared cell, whose own test + /// for being unset is [`Self::cell`] + pub fn tracks_absence(&self) -> bool { + !self.cell && !self.generated() + } + + /// whether this field keeps a byte beside it saying whether it has been written + /// + /// an [optional](Self::optional) field does, and so does any field whose + /// representation has no value to spare for "absent" and is not a tagged `int`, + /// whose error value is spare. an object's is `NULL`, which the allocation leaves + pub fn has_presence_byte(&self) -> bool { + self.optional + || (self.tracks_absence() + && !self.ty.is_object_reference() + && self.ty != crate::rtype::RType::INT) + } + /// the C struct member this field is stored in /// /// prefixed, and unconditionally: a python attribute may be called `int`, `const` @@ -542,7 +587,7 @@ impl FieldDecl { /// the C struct member holding whether this field has been written /// - /// only an [optional](Self::optional) field has one. zero means absent, and + /// only a field [with one](Self::has_presence_byte) has one. zero means absent, and /// `tp_alloc` zeroes the instance, so "never written" needs no constructor work pub fn presence(&self) -> String { format!("{}{}", self.member_prefix("p"), mangle(&self.name)) @@ -700,6 +745,10 @@ pub struct ClassIr { /// `@dataclass(frozen=True)` has none, and — the part a type system is needed for /// — two reads of one field are a *single* read even across an arbitrary call pub immutable: bool, + /// whether this is a closure environment: the object a frame's nested functions read + /// their captures from, which a nested function reading its own name holds a cycle + /// through, so the collector has to be able to walk it + pub environment: bool, /// whether the module's python namespace exposes this class. /// /// a closure environment is a real type with a real layout, but it is an @@ -841,9 +890,13 @@ impl ClassIr { /// the finished type, because that is the only thing that reaches the slot fixup a /// class statement runs and a type spec does not. a table entry would put a second /// descriptor under the same name, filled before the assignment and dropped by it + /// + /// a nested function is left out too: its environment is its receiver, but python + /// never sees it as a method of one — see [`Function::nested`] pub fn table_methods(&self) -> impl Iterator { self.methods.iter().filter(|method| { method.name != "__new__" + && method.nested.is_none() && !self .properties .iter() @@ -1367,6 +1420,7 @@ mod tests { coroutine_body: None, doc: None, takes_a_weak_reference: false, + nested: None, } } diff --git a/crates/by_ir/src/ops.rs b/crates/by_ir/src/ops.rs index 9eee27f53e..5b3ce49723 100644 --- a/crates/by_ir/src/ops.rs +++ b/crates/by_ir/src/ops.rs @@ -85,6 +85,20 @@ pub enum Mutation { InPlace, } +/// what a concatenation of two `str` registers stands for +/// +/// the two look alike and are not the same operation. python joins the pieces of an +/// f-string without asking either piece anything, while `a + b` asks the operands' +/// `__add__` and `__radd__` — which are the interpreter's own concatenation only when +/// both operands are exact `str`s, and a subclass's own methods otherwise +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum Concatenation { + /// the pieces of an f-string + Join, + /// `a + b`, or `a += b` where the mutation is in place + Operator(Mutation), +} + /// arithmetic operators, over both tagged integers and doubles #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum BinOp { @@ -288,13 +302,22 @@ pub enum Op { rhs: Value, mutation: Mutation, }, - /// comparison through the abstract object protocol + /// comparison through the abstract object protocol, as a condition: the truth of + /// whatever the comparison answered ObjectCompare { dest: RegisterId, op: CmpOp, lhs: Value, rhs: Value, }, + /// comparison through the abstract object protocol, as a value: whatever object the + /// comparison method answered, which need not be a `bool` + ObjectRichCompare { + dest: RegisterId, + op: CmpOp, + lhs: Value, + rhs: Value, + }, /// comparison of two `str`s, which the type of the operands settles /// /// the abstract protocol exists to find *whose* comparison to run, and a pair @@ -400,6 +423,17 @@ pub enum Op { class: String, method: String, }, + /// whether `name` resolves, through the module namespace and then builtins, to the + /// interpreter's own builtin of that name + /// + /// a few builtins have a native lowering, and python calls whatever the name + /// resolves to when the call runs — a module attribute written from outside, which + /// is what `mock.patch("mod.len")` does, or a patched `builtins`. so the native + /// path is taken only where this answers yes. it resolves the name itself and keeps + /// no reference, because the answer is asked in loops and the object is only + /// wanted where it is not the builtin. what it costs at runtime is in + /// `By_BuiltinStands` + BuiltinStands { dest: RegisterId, name: String }, /// whether a read or a write of `name` on `src` may go straight to the property /// half this module emitted for `class`, rather than round the descriptor protocol /// @@ -414,6 +448,19 @@ pub enum Op { class: String, name: String, }, + /// whether `field` of `src` may be read and written at its offset in `class`'s layout + /// + /// the questions [`Op::AccessorStands`] asks, about the descriptor a field publishes: + /// `src` has to be *exactly* a `class`, since an interpreted subclass may answer the + /// name with a property or take over `__getattribute__` or `__setattr__`, and `class` + /// has to still answer the name with the field's own descriptor. only a class + /// something can extend or change needs asking + FieldStands { + dest: RegisterId, + src: Value, + class: String, + field: String, + }, /// re-ask, against the receiver in hand, the lookup one licensed direct call /// skipped — and abort naming `class` and `member` when the answer is not the body /// that call is about to run @@ -581,6 +628,16 @@ pub enum Op { source: Value, mapping: bool, }, + /// merge the mapping a `**` names into the keywords of a call to `callee` + /// + /// not the `**` of a display: a call refuses a keyword given twice, where a display + /// keeps the later value, and both refusals name the callee + MergeKeywords { + dest: RegisterId, + container: Value, + source: Value, + callee: Value, + }, /// build an unboxed array from its elements — a `list` display whose elements /// are stored in a buffer of their own rather than as a `PyObject *` each ArrayNew { dest: RegisterId, items: Vec }, @@ -744,10 +801,10 @@ pub enum Op { /// which is how a shared closure cell starts fields: Vec>, }, - /// bind a method of an emitted class to a receiver, giving a callable + /// make the function object a nested function's name is bound to /// - /// this is what a nested function's name is bound to: `PyCFunction_NewEx` with - /// the environment as `self` + /// the body is a method of the environment class, and the function object holds + /// the environment it hands that method as its receiver MakeClosure { dest: RegisterId, class: String, @@ -883,12 +940,43 @@ pub enum Op { class: String, field: String, }, + /// raise `AttributeError` unless `receiver` has `field` + /// + /// an instance made by `__new__` without `__init__` has none of the fields + /// `__init__` would have given it, and a field read at a compile-time offset would + /// read what the allocation left there. so a read that nothing proves assigned is + /// preceded by this, and [`Self::GetField`] itself stays a plain load. once a field + /// is written nothing can take it away again — only a field `__init__` might leave + /// out can be deleted, and that one's read tests for itself — which is what lets a + /// later pass drop the ones an earlier write or test already answered + RequireField { + receiver: Value, + class: String, + field: String, + }, + /// whether `receiver` has `field`, as [`Self::RequireField`] asks it, for a + /// question that has an answer other than raising: a class pattern naming an + /// attribute the instance lacks does not match + FieldIsSet { + dest: RegisterId, + receiver: Value, + class: String, + field: String, + }, /// `receiver.field = value` as a direct struct write SetField { receiver: Value, class: String, field: String, value: Value, + /// whether the store takes over the reference `value`'s register holds and + /// leaves the register empty, rather than retaining one of its own. set where + /// the store is the register's last read + moves: bool, + /// whether the instance is already known to have the field, so a field with a + /// byte saying whether it has been written need not be told again. set where an + /// earlier write or test of the same field of the same object proves it + present: bool, }, /// `receiver.name` GetAttr { @@ -1008,6 +1096,28 @@ pub enum Op { PushHandled { dest: RegisterId, value: Value }, /// leave one, putting back what [`Self::PushHandled`] handed over PopHandled { value: Value }, + /// let go of what a register holds, and leave it empty + /// + /// the value is always a register the frame owns. emptying it is what lets every + /// later release of the same register — the one each exit makes, and the one a + /// write makes of the value it replaces — find nothing rather than release twice + /// + /// a non-empty `path` lets go of one reference inside a fixed-length tuple instead: + /// the element it names, following elements outermost first. only that one is + /// emptied, and the rest of the tuple goes on holding what it holds + Release { value: Value, path: Box<[usize]> }, + /// `dest = src`, or one element of it, taking over the reference `src` holds + /// there rather than retaining one of its own, and leaving that place empty + /// + /// a copy or an element read that is the last thing to read its source place + /// becomes one of these, and the release that would have followed it goes: the + /// value is handed along, so nothing is let go of and nothing is retained. the + /// empty place is what every later release of it finds + Move { + dest: RegisterId, + src: Value, + path: Box<[usize]>, + }, /// `raise `, optionally `from ` /// /// the general form. a class is instantiated, an instance raised as it is @@ -1067,6 +1177,7 @@ pub enum Op { /// nothing reads again — the error edge included, so a failure that empties /// the register cannot be observed either consumes_lhs: bool, + concatenation: Concatenation, }, /// `str(n)` where `n` is a tagged integer /// @@ -1107,6 +1218,8 @@ pub enum Op { dest: RegisterId, lhs: Value, value: Value, + /// whether this is `lhs += str(n)`, which asks a subclass's `__iadd__` first + mutation: Mutation, }, /// raise a standard error with a fixed message RaiseStandard { @@ -1135,6 +1248,8 @@ impl Op { | Self::MakeClosure { class, .. } | Self::LoadClass { class, .. } | Self::GetField { class, .. } + | Self::RequireField { class, .. } + | Self::FieldIsSet { class, .. } | Self::SetField { class, .. } => vec![class.as_str()], // a method reached directly rather than through the type, which only a // class the emitter laid out has @@ -1148,6 +1263,7 @@ impl Op { | Self::IntCompare { .. } | Self::ObjectBinary { .. } | Self::ObjectCompare { .. } + | Self::ObjectRichCompare { .. } | Self::StrCompare { .. } | Self::Truthy { .. } | Self::FloatCompare { .. } @@ -1157,7 +1273,9 @@ impl Op { | Self::IsInstance { .. } | Self::MatchAttr { .. } | Self::MethodStands { .. } + | Self::BuiltinStands { .. } | Self::AccessorStands { .. } + | Self::FieldStands { .. } | Self::DictShadows { .. } | Self::IsMissing { .. } | Self::MatchSlice { .. } @@ -1175,6 +1293,7 @@ impl Op { | Self::Unpack { .. } | Self::CallUnpacked { .. } | Self::Extend { .. } + | Self::MergeKeywords { .. } | Self::ArrayNew { .. } | Self::ArrayGet { .. } | Self::ArraySet { .. } @@ -1220,6 +1339,8 @@ impl Op { | Self::ExceptionMatches { .. } | Self::PushHandled { .. } | Self::PopHandled { .. } + | Self::Release { .. } + | Self::Move { .. } | Self::RaiseObject { .. } | Self::Reraise { .. } | Self::GetIter { .. } @@ -1273,6 +1394,7 @@ impl Op { pub fn dest(&self) -> Option { match self { Self::Assign { dest, .. } + | Self::Move { dest, .. } | Self::Contains { dest, .. } | Self::AsyncContext { dest, .. } | Self::AsyncIter { dest, .. } @@ -1282,7 +1404,9 @@ impl Op { | Self::IsMapping { dest, .. } | Self::MatchAttr { dest, .. } | Self::MethodStands { dest, .. } + | Self::BuiltinStands { dest, .. } | Self::AccessorStands { dest, .. } + | Self::FieldStands { dest, .. } | Self::DictShadows { dest, .. } | Self::IsMissing { dest, .. } | Self::MatchSlice { dest, .. } @@ -1296,6 +1420,7 @@ impl Op { | Self::IntCompare { dest, .. } | Self::ObjectBinary { dest, .. } | Self::ObjectCompare { dest, .. } + | Self::ObjectRichCompare { dest, .. } | Self::StrCompare { dest, .. } | Self::Truthy { dest, .. } | Self::FloatCompare { dest, .. } @@ -1328,6 +1453,7 @@ impl Op { | Self::MakeClosure { dest, .. } | Self::CallMethod { dest, .. } | Self::GetField { dest, .. } + | Self::FieldIsSet { dest, .. } | Self::GetAttr { dest, .. } | Self::SetAttr { dest, .. } | Self::BuildList { dest, .. } @@ -1358,6 +1484,7 @@ impl Op { | Self::DeleteAttr { dest, .. } | Self::ArrayPush { dest, .. } | Self::Extend { dest, .. } + | Self::MergeKeywords { dest, .. } | Self::CallUnpacked { dest, .. } | Self::PushHandled { dest, .. } => Some(*dest), Self::CallNative { dest, .. } => *dest, @@ -1366,8 +1493,10 @@ impl Op { | Self::FinishFrame { .. } | Self::RaiseObject { .. } | Self::PopHandled { .. } + | Self::Release { .. } | Self::Reraise { .. } | Self::LicenceHolds { .. } + | Self::RequireField { .. } | Self::SetField { .. } => None, } } @@ -1380,6 +1509,7 @@ impl Op { pub fn dest_mut(&mut self) -> Option<&mut RegisterId> { match self { Self::Assign { dest, .. } + | Self::Move { dest, .. } | Self::Contains { dest, .. } | Self::AsyncContext { dest, .. } | Self::AsyncIter { dest, .. } @@ -1389,7 +1519,9 @@ impl Op { | Self::IsMapping { dest, .. } | Self::MatchAttr { dest, .. } | Self::MethodStands { dest, .. } + | Self::BuiltinStands { dest, .. } | Self::AccessorStands { dest, .. } + | Self::FieldStands { dest, .. } | Self::DictShadows { dest, .. } | Self::IsMissing { dest, .. } | Self::MatchSlice { dest, .. } @@ -1403,6 +1535,7 @@ impl Op { | Self::IntCompare { dest, .. } | Self::ObjectBinary { dest, .. } | Self::ObjectCompare { dest, .. } + | Self::ObjectRichCompare { dest, .. } | Self::StrCompare { dest, .. } | Self::Truthy { dest, .. } | Self::FloatCompare { dest, .. } @@ -1435,6 +1568,7 @@ impl Op { | Self::MakeClosure { dest, .. } | Self::CallMethod { dest, .. } | Self::GetField { dest, .. } + | Self::FieldIsSet { dest, .. } | Self::GetAttr { dest, .. } | Self::SetAttr { dest, .. } | Self::BuildList { dest, .. } @@ -1465,6 +1599,7 @@ impl Op { | Self::DeleteAttr { dest, .. } | Self::ArrayPush { dest, .. } | Self::Extend { dest, .. } + | Self::MergeKeywords { dest, .. } | Self::CallUnpacked { dest, .. } | Self::PushHandled { dest, .. } => Some(dest), Self::CallNative { dest, .. } => dest.as_mut(), @@ -1473,8 +1608,10 @@ impl Op { | Self::FinishFrame { .. } | Self::RaiseObject { .. } | Self::PopHandled { .. } + | Self::Release { .. } | Self::Reraise { .. } | Self::LicenceHolds { .. } + | Self::RequireField { .. } | Self::SetField { .. } => None, } } @@ -1488,9 +1625,11 @@ impl Op { .. } => vec![manager, exception], Self::Assign { src, .. } + | Self::Move { src, .. } | Self::Box { src, .. } | Self::MethodStands { src, .. } | Self::AccessorStands { src, .. } + | Self::FieldStands { src, .. } | Self::LicenceHolds { src, .. } | Self::DictShadows { src, .. } | Self::IsMissing { src, .. } @@ -1549,6 +1688,7 @@ impl Op { | Self::IntCompare { lhs, rhs, .. } | Self::ObjectBinary { lhs, rhs, .. } | Self::ObjectCompare { lhs, rhs, .. } + | Self::ObjectRichCompare { lhs, rhs, .. } | Self::StrCompare { lhs, rhs, .. } | Self::StrConcat { lhs, rhs, .. } | Self::StrConcatInt { @@ -1571,7 +1711,10 @@ impl Op { all.extend(args.iter()); all } - Self::GetAttr { receiver, .. } | Self::GetField { receiver, .. } => vec![receiver], + Self::GetAttr { receiver, .. } + | Self::GetField { receiver, .. } + | Self::RequireField { receiver, .. } + | Self::FieldIsSet { receiver, .. } => vec![receiver], Self::ImportFrom { module, .. } => vec![module], Self::SetField { receiver, value, .. @@ -1612,6 +1755,7 @@ impl Op { Self::RaiseStandard { .. } | Self::FetchException { .. } | Self::LoadGlobal { .. } + | Self::BuiltinStands { .. } | Self::LoadEllipsis { .. } | Self::ModuleDict { .. } | Self::DeleteGlobal { .. } @@ -1655,6 +1799,12 @@ impl Op { Self::Extend { container, source, .. } => vec![container, source], + Self::MergeKeywords { + container, + source, + callee, + .. + } => vec![container, source, callee], Self::CallUnpacked { callee, args, @@ -1672,6 +1822,7 @@ impl Op { Self::Reraise { value } | Self::PushHandled { value, .. } | Self::PopHandled { value } + | Self::Release { value, .. } | Self::FinishFrame { value } | Self::RaiseWith { value, .. } => vec![value], } @@ -1690,9 +1841,11 @@ impl Op { .. } => vec![manager, exception], Self::Assign { src, .. } + | Self::Move { src, .. } | Self::Box { src, .. } | Self::MethodStands { src, .. } | Self::AccessorStands { src, .. } + | Self::FieldStands { src, .. } | Self::LicenceHolds { src, .. } | Self::DictShadows { src, .. } | Self::IsMissing { src, .. } @@ -1751,6 +1904,7 @@ impl Op { | Self::IntCompare { lhs, rhs, .. } | Self::ObjectBinary { lhs, rhs, .. } | Self::ObjectCompare { lhs, rhs, .. } + | Self::ObjectRichCompare { lhs, rhs, .. } | Self::StrCompare { lhs, rhs, .. } | Self::StrConcat { lhs, rhs, .. } | Self::StrConcatInt { @@ -1775,7 +1929,10 @@ impl Op { all.extend(args.iter_mut()); all } - Self::GetAttr { receiver, .. } | Self::GetField { receiver, .. } => vec![receiver], + Self::GetAttr { receiver, .. } + | Self::GetField { receiver, .. } + | Self::RequireField { receiver, .. } + | Self::FieldIsSet { receiver, .. } => vec![receiver], Self::ImportFrom { module, .. } => vec![module], Self::SetField { receiver, value, .. @@ -1816,6 +1973,7 @@ impl Op { Self::RaiseStandard { .. } | Self::FetchException { .. } | Self::LoadGlobal { .. } + | Self::BuiltinStands { .. } | Self::LoadEllipsis { .. } | Self::ModuleDict { .. } | Self::DeleteGlobal { .. } @@ -1859,6 +2017,12 @@ impl Op { Self::Extend { container, source, .. } => vec![container, source], + Self::MergeKeywords { + container, + source, + callee, + .. + } => vec![container, source, callee], Self::CallUnpacked { callee, args, @@ -1876,6 +2040,7 @@ impl Op { Self::Reraise { value } | Self::PushHandled { value, .. } | Self::PopHandled { value } + | Self::Release { value, .. } | Self::FinishFrame { value } | Self::RaiseWith { value, .. } => vec![value], } @@ -2025,6 +2190,8 @@ mod tests { class: "Held".to_string(), field: "tag".to_string(), value: Value::Int(1), + moves: false, + present: false, }, Op::CallNative { owner: Some("Held".to_string()), diff --git a/crates/by_ir/src/print.rs b/crates/by_ir/src/print.rs index 61bfa3052d..70081233e1 100644 --- a/crates/by_ir/src/print.rs +++ b/crates/by_ir/src/print.rs @@ -7,7 +7,9 @@ use std::fmt::Write; use crate::function::{Function, ModuleIr}; -use crate::ops::{LicenceKind, Mutation, Op, RegisterId, Terminator, UnaryOp, Value}; +use crate::ops::{ + Concatenation, LicenceKind, Mutation, Op, RegisterId, Terminator, UnaryOp, Value, +}; /// render a whole module pub fn print_module(module: &ModuleIr) -> String { @@ -148,6 +150,10 @@ fn print_op(function: &Function, op: &Op) -> String { value(subject), attribute.as_deref().unwrap_or("") ), + Op::BuiltinStands { + dest, + name: builtin, + } => format!("{} = builtin-stands {builtin}", name(*dest)), Op::MethodStands { dest, src, @@ -168,6 +174,16 @@ fn print_op(function: &Function, op: &Op) -> String { name(*dest), value(src) ), + Op::FieldStands { + dest, + src, + class, + field, + } => format!( + "{} = field-stands {} {class}.{field}", + name(*dest), + value(src) + ), Op::LicenceHolds { src, class, @@ -273,7 +289,8 @@ fn print_op(function: &Function, op: &Op) -> String { Op::IntCompare { dest, op, lhs, rhs } | Op::FloatCompare { dest, op, lhs, rhs } | Op::StrCompare { dest, op, lhs, rhs } - | Op::ObjectCompare { dest, op, lhs, rhs } => { + | Op::ObjectCompare { dest, op, lhs, rhs } + | Op::ObjectRichCompare { dest, op, lhs, rhs } => { format!( "{} = {} {} {}", name(*dest), @@ -399,6 +416,18 @@ fn print_op(function: &Function, op: &Op) -> String { if *mapping { "update" } else { "extend" }, value(source) ), + Op::MergeKeywords { + dest, + container, + source, + callee, + } => format!( + "{} = {} keywords {} for {}", + name(*dest), + value(container), + value(source), + value(callee) + ), Op::Unpack { dest, src, starred } => match starred { Some(index) => format!("{} = unpack {} star {index}", name(*dest), value(src)), None => format!("{} = unpack {}", name(*dest), value(src)), @@ -414,9 +443,14 @@ fn print_op(function: &Function, op: &Op) -> String { dest, lhs, value: src, + mutation, } => { + let form = match mutation { + Mutation::Fresh => "", + Mutation::InPlace => " in place", + }; format!( - "{} = str-concat-int {}, {}", + "{} = str-concat-int{form} {}, {}", name(*dest), value(lhs), value(src) @@ -572,12 +606,34 @@ fn print_op(function: &Function, op: &Op) -> String { class, field, } => format!("{} = {}.<{class}.{field}>", name(*dest), value(receiver)), + Op::RequireField { + receiver, + class, + field, + } => format!("require {}.<{class}.{field}>", value(receiver)), + Op::FieldIsSet { + dest, + receiver, + class, + field, + } => format!( + "{} = has {}.<{class}.{field}>", + name(*dest), + value(receiver) + ), Op::SetField { receiver, class, field, value: v, - } => format!("{}.<{class}.{field}> = {}", value(receiver), value(v)), + moves, + .. + } => format!( + "{}.<{class}.{field}> = {}{}", + value(receiver), + if *moves { "move " } else { "" }, + value(v) + ), Op::GetAttr { dest, receiver, @@ -681,6 +737,20 @@ fn print_op(function: &Function, op: &Op) -> String { format!("{} = push handled {}", name(*dest), value(v)) } Op::PopHandled { value: v } => format!("pop handled {}", value(v)), + Op::Release { value: v, path } => { + let mut text = format!("release {}", value(v)); + for index in path { + let _ = write!(text, "[{index}]"); + } + text + } + Op::Move { dest, src, path } => { + let mut text = format!("{} = move {}", name(*dest), value(src)); + for index in path { + let _ = write!(text, "[{index}]"); + } + text + } Op::RaiseObject { exception, cause } => match cause { Some(cause) => format!("raise {} from {}", value(exception), value(cause)), None => format!("raise {}", value(exception)), @@ -700,9 +770,20 @@ fn print_op(function: &Function, op: &Op) -> String { lhs, rhs, consumes_lhs, + concatenation, } => { let take = if *consumes_lhs { "move " } else { "" }; - format!("{} = {take}{} ++ {}", name(*dest), value(lhs), value(rhs)) + let operator = match concatenation { + Concatenation::Join => "++", + Concatenation::Operator(Mutation::Fresh) => "+", + Concatenation::Operator(Mutation::InPlace) => "+=", + }; + format!( + "{} = {take}{} {operator} {}", + name(*dest), + value(lhs), + value(rhs) + ) } Op::RaiseStandard { error, message } => { format!("raise {error:?}({message:?})") @@ -816,6 +897,7 @@ mod tests { coroutine_body: None, doc: None, takes_a_weak_reference: false, + nested: None, } } @@ -889,6 +971,7 @@ b2: coroutine_body: None, doc: None, takes_a_weak_reference: false, + nested: None, }; assert!(print_function(&function).contains("return 1.0")); } diff --git a/crates/by_ir/src/rtype.rs b/crates/by_ir/src/rtype.rs index 9dc722c700..af9bd77bb3 100644 --- a/crates/by_ir/src/rtype.rs +++ b/crates/by_ir/src/rtype.rs @@ -238,6 +238,44 @@ impl RType { /// the value a register of this type holds before it is assigned, and the /// value a fallible function returns on the error path + /// whether a value of this type is one reference to one object, which a single + /// release lets go of and an empty register spells as `NULL` + pub fn is_object_reference(&self) -> bool { + matches!( + self, + Self::Primitive( + Primitive::Object + | Primitive::Str + | Primitive::Bytes + | Primitive::List + | Primitive::Dict + | Primitive::Tuple + ) | Self::Instance { .. } + ) + } + + /// whether a value of this type is, or holds as an element, a reference to an + /// object — something that may have a finalizer, or hold one that does + pub fn holds_object_reference(&self) -> bool { + match self { + Self::Tuple(items) => items.iter().any(Self::holds_object_reference), + Self::Array(element) => element.holds_object_reference(), + _ => self.is_object_reference(), + } + } + + /// the type of the element `path` names inside a fixed-length tuple, following + /// elements outermost first. the empty path names the whole value + pub fn element(&self, path: &[usize]) -> Option<&Self> { + match path.split_first() { + None => Some(self), + Some((index, rest)) => match self { + Self::Tuple(items) => items.get(*index)?.element(rest), + _ => None, + }, + } + } + pub fn undefined(&self) -> String { match self { Self::Array(_) => "NULL".to_string(), diff --git a/crates/by_ir/src/verify.rs b/crates/by_ir/src/verify.rs index 17a0017129..1fc4983b86 100644 --- a/crates/by_ir/src/verify.rs +++ b/crates/by_ir/src/verify.rs @@ -8,7 +8,7 @@ //! this is the guard on the representation invariant. a pass that produces //! ill-typed BIR is a bug that would otherwise surface as miscompiled C. -use std::collections::{HashSet, VecDeque}; +use std::collections::{HashMap, HashSet, VecDeque}; use std::fmt; use crate::function::{Function, ModuleIr}; @@ -42,9 +42,10 @@ impl fmt::Display for VerifyError { /// verify every function in a module pub fn verify_module(module: &ModuleIr) -> Result<(), Vec> { + let bases = base_chain(module); let mut errors: Vec = module .all_functions() - .filter_map(|function| verify_in(function, Some(module)).err()) + .filter_map(|function| verify_in(function, Some(module), Some(&bases)).err()) .flatten() .collect(); errors.extend(check_accessor_coverage(module)); @@ -55,19 +56,53 @@ pub fn verify_module(module: &ModuleIr) -> Result<(), Vec> { } } +/// each class's in-module base, which is the whole of the layout chain +/// +/// an external base has no struct here, so nothing downstream of it is a free pointer +/// cast and the chain stops where one appears +fn base_chain(module: &ModuleIr) -> HashMap { + module + .classes + .iter() + .filter_map(|class| { + let base = class.base.as_ref()?.in_module()?; + Some((class.name.clone(), base.to_string())) + }) + .collect() +} + /// verify one function, on its own /// /// a check that needs to look at another function — a call's arguments against the /// callee's parameters — is skipped. [`verify_module`] is where those run pub fn verify(function: &Function) -> Result<(), Vec> { - verify_in(function, None) + verify_in(function, None, None) +} + +/// verify one function against the layout chain of the module being lowered +/// +/// the frontend verifies each function as it lowers it, before there is a `ModuleIr` to +/// verify against — but a receiver may name a base's field, and deciding that takes the +/// chain. so the chain is handed over on its own: everything else stays the single +/// function's business, and [`verify_module`] is still where the cross-function checks +/// run +pub fn verify_within( + function: &Function, + bases: &HashMap, +) -> Result<(), Vec> { + verify_in(function, None, Some(bases)) } /// verify one function, against the module it belongs to where that is known -fn verify_in(function: &Function, module: Option<&ModuleIr>) -> Result<(), Vec> { +fn verify_in( + function: &Function, + module: Option<&ModuleIr>, + bases: Option<&HashMap>, +) -> Result<(), Vec> { let mut verifier = Verifier { function, module, + bases, errors: Vec::new(), }; verifier.run(); @@ -208,6 +243,11 @@ struct Verifier<'a> { /// only a cross-function check needs it: a call's arguments are checked against /// the callee's parameters, which live in another `Function` module: Option<&'a ModuleIr>, + /// each class's in-module base, where the caller knows the layout chain + /// + /// separate from `module` because the frontend has the chain before it has a + /// module — see [`verify_within`] + bases: Option<&'a HashMap>, errors: Vec, } @@ -293,27 +333,39 @@ impl Verifier<'_> { else { return false; }; - let Some(module) = self.module else { + let Some(bases) = self.bases else { return false; }; - let mut current = Some(from.clone()); + let mut current = Some(from.as_str()); while let Some(name) = current { - if name == *declared { + if name == declared.as_str() { return true; } - // only an in-module base continues the chain: an external one has no struct - // here, so nothing downstream of it is a free pointer cast - current = module - .classes - .iter() - .find(|class| class.name == name) - .and_then(|class| class.base.as_ref()) - .and_then(|base| base.in_module()) - .map(str::to_owned); + current = bases.get(name).map(String::as_str); } false } + /// whether a field op naming `class` may take a receiver of representation `actual` + /// + /// the same upcast a call argument gets, for the same reason: the receiver is the + /// class the op names or one standing on it through this module's own layout chain, + /// and a subclass's struct begins with its base's. a class that adds no field of its + /// own is the shape that needs this — it is laid out with none at all, so the field + /// it reaches is always named against a base + fn names_the_receivers_layout(&self, actual: &RType, class: &str) -> bool { + match actual { + RType::Instance { class: got, .. } if got == class => true, + _ => self.upcasts( + actual, + &RType::Instance { + class: class.to_string(), + exact: false, + }, + ), + } + } + fn run(&mut self) { if self.function.blocks.is_empty() { self.error(None, "a function needs at least an entry block"); @@ -345,9 +397,10 @@ impl Verifier<'_> { /// /// `borrowed` means the frame never took a reference and so never gives one back: /// the emitter writes a plain store and the cleanup skips the register entirely. - /// that is only true of the four shapes the emitter recognises — a copy, a - /// narrowing test, an element read off a tuple, and a field read — each of which - /// leaves some other place still owning the value. every *other* operation + /// that is only true of the five shapes the emitter recognises — a copy, a + /// narrowing test, an element read off a tuple, a field read, and a box of `None`, + /// `True` or `False` — each of which leaves some other place still owning the value, + /// the last of them the interpreter itself. every *other* operation /// produces a reference of its own, and storing one into a register nothing /// releases leaks it. /// @@ -382,6 +435,9 @@ impl Verifier<'_> { } let lends = match op { Op::Assign { .. } | Op::TupleGet { .. } | Op::GetField { .. } => true, + Op::Box { src, .. } => { + matches!(src, Value::None | Value::Bool(_) | Value::Bit(_)) + } Op::Unbox { to, .. } => matches!( to, RType::Primitive(Primitive::Str | Primitive::List) | RType::Instance { .. } @@ -676,11 +732,14 @@ impl Verifier<'_> { } self.expect_dest(block, *dest, &RType::OBJECT, "a class pattern"); } + Op::BuiltinStands { dest, .. } => { + self.expect_dest(block, *dest, &RType::BIT, "a builtin test"); + } Op::MethodStands { dest, src, .. } => { self.expect(block, src, &RType::OBJECT, "a dispatch test"); self.expect_dest(block, *dest, &RType::BIT, "a dispatch test"); } - Op::AccessorStands { dest, src, .. } => { + Op::AccessorStands { dest, src, .. } | Op::FieldStands { dest, src, .. } => { // the test reads the receiver's type and stores the pointer nowhere, so // it borrows — which is why an emitted class's own pointer is taken as it // stands rather than through a `box`, the one widening that would @@ -789,6 +848,11 @@ impl Verifier<'_> { self.expect(block, rhs, &RType::OBJECT, op.symbol()); self.expect_dest(block, *dest, &RType::BIT, op.symbol()); } + Op::ObjectRichCompare { dest, lhs, rhs, op } => { + self.expect(block, lhs, &RType::OBJECT, op.symbol()); + self.expect(block, rhs, &RType::OBJECT, op.symbol()); + self.expect_dest(block, *dest, &RType::OBJECT, op.symbol()); + } Op::StrCompare { dest, lhs, rhs, op } => { self.expect(block, lhs, &RType::STR, op.symbol()); self.expect(block, rhs, &RType::STR, op.symbol()); @@ -978,6 +1042,17 @@ impl Verifier<'_> { self.expect(block, source, &RType::OBJECT, "extend"); self.expect_dest(block, *dest, &RType::BIT, "extend"); } + Op::MergeKeywords { + dest, + container, + source, + callee, + } => { + self.expect(block, container, &RType::OBJECT, "a keyword merge"); + self.expect(block, source, &RType::OBJECT, "a keyword merge"); + self.expect(block, callee, &RType::OBJECT, "a keyword merge"); + self.expect_dest(block, *dest, &RType::BIT, "a keyword merge"); + } Op::CallUnpacked { dest, callee, @@ -1142,7 +1217,19 @@ impl Verifier<'_> { }, "a cell read", ); - self.expect_dest(block, *dest, &RType::OBJECT, "a cell read"); + // a cell starts unset, so it is read into a representation with an unset + // value of its own: NULL for a single reference, the error value for a + // tagged `int` + let holds = self + .function + .register(*dest) + .is_some_and(|decl| decl.ty == RType::INT || decl.ty.is_object_reference()); + if !holds { + self.error( + Some(block), + format!("a cell read into {dest:?}, which has no unset value"), + ); + } } Op::NewInstance { dest, @@ -1262,10 +1349,12 @@ impl Verifier<'_> { class, .. } => { - // the receiver's representation has to *be* that class, or the + // the receiver's representation has to *be* that class, or a class + // under it — a subclass's struct begins with its base's, so the + // offset lands in the same place either way. anything else and the // offset would be read out of the wrong struct if let Some(actual) = self.operand_type(block, receiver) - && !matches!(actual, RType::Instance { class: ref got, .. } if got == class) + && !self.names_the_receivers_layout(&actual, class) { self.error( Some(block), @@ -1276,14 +1365,56 @@ impl Verifier<'_> { self.error(Some(block), format!("r{} is not declared", dest.0)); } } + Op::RequireField { + receiver, class, .. + } + | Op::FieldIsSet { + receiver, class, .. + } => { + if let Some(actual) = self.operand_type(block, receiver) + && !self.names_the_receivers_layout(&actual, class) + { + self.error( + Some(block), + format!("a field test on {class} has a {actual} receiver"), + ); + } + if let Op::FieldIsSet { dest, .. } = op { + self.expect_dest(block, *dest, &RType::BIT, "a field test"); + } + } Op::SetField { receiver, class, value, + moves, .. } => { + // a moving store empties its value's register, so the register has to own + // what it hands over, and be one an empty value can be written back into + if *moves { + let owns = match value { + Value::Register(id) => { + id.index() >= self.function.param_count + && self.function.register(*id).is_some_and(|decl| { + !decl.borrowed + && decl.name.is_none() + && (decl.ty == RType::INT || decl.ty.is_object_reference()) + }) + } + _ => false, + }; + if !owns { + self.error( + Some(block), + format!( + "a moving store of {value:?}, which owns no reference to hand over" + ), + ); + } + } if let Some(actual) = self.operand_type(block, receiver) - && !matches!(actual, RType::Instance { class: ref got, .. } if got == class) + && !self.names_the_receivers_layout(&actual, class) { self.error( Some(block), @@ -1446,6 +1577,64 @@ impl Verifier<'_> { Op::PopHandled { value } => { self.expect(block, value, &RType::OBJECT, "leaving a handler"); } + Op::Release { value, path } => { + let owns = match value { + Value::Register(id) => self.function.register(*id).is_some_and(|decl| { + !decl.borrowed + && decl + .ty + .element(path) + .is_some_and(RType::is_object_reference) + }), + _ => false, + }; + if !owns { + self.error( + Some(block), + format!( + "a release of {value:?} at {path:?}, which owns no single reference" + ), + ); + } + } + Op::Move { dest, src, path } => { + // the source hands over a reference it owns, and is left empty where it + // held it — so it has to be a temporary the frame owns, which nothing + // else expects to find still holding its value + let element = match src { + Value::Register(id) if id.index() >= self.function.param_count => self + .function + .register(*id) + .filter(|decl| !decl.borrowed && decl.name.is_none()) + .and_then(|decl| decl.ty.element(path)) + .filter(|ty| **ty == RType::INT || ty.is_object_reference()) + .cloned(), + _ => None, + }; + let Some(element) = element else { + self.error( + Some(block), + format!( + "a move of {src:?} at {path:?}, which owns no reference to hand over" + ), + ); + return; + }; + if self + .function + .register(*dest) + .is_some_and(|decl| decl.borrowed) + { + self.error( + Some(block), + format!( + "r{} is borrowed, so it cannot take over a reference", + dest.0 + ), + ); + } + self.expect_dest(block, *dest, &element, "a move"); + } Op::RaiseObject { exception, cause } => { self.expect(block, exception, &RType::OBJECT, "a raise"); if let Some(cause) = cause { @@ -1490,6 +1679,7 @@ impl Verifier<'_> { lhs, rhs, consumes_lhs, + .. } => { self.expect(block, lhs, &RType::STR, "str concatenation"); self.expect(block, rhs, &RType::STR, "str concatenation"); @@ -1535,7 +1725,9 @@ impl Verifier<'_> { } } } - Op::StrConcatInt { dest, lhs, value } => { + Op::StrConcatInt { + dest, lhs, value, .. + } => { self.expect(block, lhs, &RType::STR, "concatenating the str of an int"); self.expect(block, value, &RType::INT, "concatenating the str of an int"); // unlike `str-of-int` on its own this *is* a `str`: what the name @@ -1706,8 +1898,12 @@ impl Verifier<'_> { continue; // unreachable block: nothing to prove about it }; for op in &block.ops { - for operand in op.operands() { - self.check_read(id, operand, &state); + // a release of a register nothing has written yet lets go of nothing: + // every register starts empty, which is what makes that safe + if !matches!(op, Op::Release { .. }) { + for operand in op.operands() { + self.check_read(id, operand, &state); + } } if let Some(dest) = op.dest() && dest.index() < state.len() @@ -1768,7 +1964,7 @@ mod tests { use super::*; use crate::builder::FunctionBuilder; use crate::function::{BasicBlock, CallConvention, RegisterDecl}; - use crate::ops::{BinOp, CmpOp}; + use crate::ops::{BinOp, CmpOp, Concatenation, Mutation}; fn reg(name: &str, ty: RType) -> RegisterDecl { RegisterDecl { @@ -1820,6 +2016,7 @@ mod tests { coroutine_body: None, doc: None, takes_a_weak_reference: false, + nested: None, } } @@ -2010,6 +2207,7 @@ mod tests { coroutine_body: None, doc: None, takes_a_weak_reference: false, + nested: None, }; let errors = verify(&f).unwrap_err(); assert!( @@ -2061,6 +2259,7 @@ mod tests { coroutine_body: None, doc: None, takes_a_weak_reference: false, + nested: None, }; assert_eq!(verify(&f), Ok(())); } @@ -2113,6 +2312,7 @@ mod tests { coroutine_body: None, doc: None, takes_a_weak_reference: false, + nested: None, }; let errors = verify(&f).unwrap_err(); assert!( @@ -2216,6 +2416,7 @@ mod tests { coroutine_body: None, doc: None, takes_a_weak_reference: false, + nested: None, }; let errors = verify(&f).unwrap_err(); assert!(errors.iter().any(|e| e.message.contains("past the end"))); @@ -2505,6 +2706,7 @@ mod tests { lhs: Value::Register(head), rhs: Value::Register(tail), consumes_lhs: true, + concatenation: Concatenation::Operator(Mutation::Fresh), }); builder.terminate(Terminator::Return(Value::Register(out))); let mut function = builder.finish(); @@ -2522,6 +2724,7 @@ mod tests { lhs: Value::Register(out), rhs: Value::Register(tail), consumes_lhs: true, + concatenation: Concatenation::Operator(Mutation::Fresh), }; assert_eq!(verify(&function), Ok(())); } @@ -2538,6 +2741,7 @@ mod tests { lhs: Value::Register(held), rhs: Value::Register(tail), consumes_lhs: true, + concatenation: Concatenation::Operator(Mutation::Fresh), }); builder.terminate(Terminator::Return(Value::Register(out))); let mut function = builder.finish(); diff --git a/crates/by_irbuild/src/closures.rs b/crates/by_irbuild/src/closures.rs index 156732e9a2..66bb6f94f7 100644 --- a/crates/by_irbuild/src/closures.rs +++ b/crates/by_irbuild/src/closures.rs @@ -3,9 +3,10 @@ //! a closure needs somewhere for the captured values to live that outlives the //! frame that made it. that place is an object with a fixed layout and one field //! per capture — which is exactly a [`ClassIr`], so the whole native-class -//! machinery applies: a captured read is a `GetField` at a compile-time offset, the -//! nested function is a method whose receiver is the environment, and binding the -//! name is `PyCFunction_NewEx` with the environment as `self`. +//! machinery applies: a captured read is a `GetField` at a compile-time offset, and the +//! nested function is a method whose receiver is the environment. binding the name +//! makes a function object over the environment — see `By_MakeFunction` — because +//! python sees a function where the `def` stands, not a method of anything. //! //! ## what is captured //! @@ -324,6 +325,28 @@ fn nonlocal_names(def: &ast::StmtFunctionDef) -> Vec<&str> { .collect() } +/// every function nested in `body`, at any depth, that declares `name` `nonlocal` +/// +/// each is a frame that may write the cell `name` is, so each says something about what +/// the cell holds. one whose `nonlocal` reaches a nearer frame's own `name` is counted as +/// well, which can only widen what the cell is taken to hold +pub(crate) fn nonlocal_writers<'a>(body: &'a [Stmt], name: &str) -> Vec<&'a ast::StmtFunctionDef> { + let mut out = Vec::new(); + for stmt in crate::walk(body) { + match stmt { + Stmt::FunctionDef(def) => { + if nonlocal_names(def).contains(&name) { + out.push(def); + } + out.extend(nonlocal_writers(&def.body, name)); + } + Stmt::ClassDef(class) => out.extend(nonlocal_writers(&class.body, name)), + _ => {} + } + } + out +} + /// the names a function declares `global` fn global_names(def: &ast::StmtFunctionDef) -> Vec<&str> { crate::walk(&def.body) @@ -621,11 +644,15 @@ pub(crate) fn environment_name(enclosing: Option<&str>, owner: &str) -> String { /// else a nested function reads lives further up, and is reached through the chain /// rather than copied: giving it a field of its own here would shadow the real one /// with a copy that is never seeded +/// +/// `cell` is the representation a shared cell holds — see the caller, which is what can +/// ask every frame that writes one pub(crate) fn environment( name: &str, enclosing: Option<&str>, nested: &[Nested], representation: &impl Fn(&str) -> Option, + cell: &impl Fn(&str) -> by_ir::rtype::RType, owned: &HashSet, ) -> Lowered> { if nested.is_empty() { @@ -640,16 +667,17 @@ pub(crate) fn environment( if fields.iter().any(|field| field.name == *capture) || !owned.contains(capture) { continue; } - // a *shared* cell is always `object`: it starts unset, and NULL has to be - // distinguishable from every value it could hold. an unboxed zero would not be + // a *shared* cell starts unset, so it is held in a representation with an unset + // value of its own, distinct from every value it could hold let ty = if shared.contains(capture.as_str()) { - by_ir::rtype::RType::OBJECT + cell(capture) } else { representation(capture).ok_or_else(|| { Decline::new(format!("`{capture}` has no representation to capture")) })? }; fields.push(by_ir::function::FieldDecl { + cell: shared.contains(capture.as_str()), name: capture.clone(), ty, default: None, @@ -662,6 +690,7 @@ pub(crate) fn environment( // one cell if let Some(enclosing) = enclosing { fields.push(by_ir::function::FieldDecl { + cell: false, optional: false, defaulted_by: None, name: OUTER_FIELD.to_string(), @@ -676,6 +705,7 @@ pub(crate) fn environment( // still needs an object to hang the method on if fields.is_empty() { fields.push(by_ir::function::FieldDecl { + cell: false, name: "$empty".to_string(), ty: by_ir::rtype::RType::NONE, default: None, diff --git a/crates/by_irbuild/src/comprehension_scopes.rs b/crates/by_irbuild/src/comprehension_scopes.rs new file mode 100644 index 0000000000..64fc277fd9 --- /dev/null +++ b/crates/by_irbuild/src/comprehension_scopes.rs @@ -0,0 +1,214 @@ +//! the names a comprehension binds, kept apart from the frame it is lowered into +//! +//! a comprehension is lowered inline, as a loop in the function that holds it, but +//! python gives it a scope of its own: +//! +//! ```python +//! def f(xs): +//! x = 5 +//! out = [x + 1 for x in xs] +//! return x # 5, whatever `xs` held +//! ``` +//! +//! every part of the lowering that asks about a name — which register it is, what +//! representation it has, whether a closure captures it and whether that capture is a +//! shared cell — asks by the name's spelling. so rather than teach each of them about a +//! second kind of scope, each variable a comprehension's `for` clauses bind is given a +//! spelling of its own before any of them look, everywhere the comprehension's scope +//! can see it. no python identifier contains `$`, so the new spelling can meet nothing +//! the source wrote. +//! +//! what python's scoping rules say, and so what is renamed: +//! +//! - the first clause's iterable is evaluated in the enclosing scope, and every later +//! clause's iterable, every condition and the element inside the comprehension's +//! - a lambda inside reads the comprehension's variable unless a parameter of its own +//! has the name +//! - a walrus binds in the enclosing function, and python refuses one whose target is +//! a comprehension's variable, so a walrus target is never renamed +//! - a comprehension nested inside another binds its own variables, which hide the +//! outer one's of the same name +//! +//! a later clause's iterable reading a variable that clause or a later one binds reads +//! it before anything is bound, which python answers with `UnboundLocalError`. that +//! read is declined rather than given a spelling the error would then name + +use std::cell::{Cell, RefCell}; +use std::collections::HashMap; + +use ruff_python_ast::name::Name; +use ruff_python_ast::visitor::transformer::{Transformer, walk_expr, walk_parameters}; +use ruff_python_ast::{self as ast, Expr}; + +use crate::mapper::{Decline, Lowered}; + +/// the marker every spelling this gives out contains, and no source name can +const MARKER: char = '$'; + +/// `function`, with each comprehension variable spelled apart from every other name +pub(crate) fn scoped(function: &ast::StmtFunctionDef) -> Lowered { + let mut function = function.clone(); + let renamer = Renamer { + scopes: RefCell::new(Vec::new()), + next: Cell::new(0), + refused: RefCell::new(None), + }; + for stmt in &mut function.body { + renamer.visit_stmt(stmt); + } + match renamer.refused.into_inner() { + Some(reason) => Err(Decline::new(reason)), + None => Ok(function), + } +} + +/// what one scope does to a name it binds +#[derive(Clone)] +enum Binding { + /// the comprehension's variable, under its own spelling + Renamed(Name), + /// the same, before the clause that binds it has run + Unbound(Name), + /// a lambda parameter, which hides every scope outside it + Hidden, +} + +struct Renamer { + /// innermost last + scopes: RefCell>>, + next: Cell, + refused: RefCell>, +} + +impl Renamer { + fn lookup(&self, name: &Name) -> Option { + self.scopes + .borrow() + .iter() + .rev() + .find_map(|scope| scope.get(name).cloned()) + } + + fn comprehension(&self, generators: &mut [ast::Comprehension], rest: &mut [&mut Expr]) { + let Some((first, _)) = generators.split_first_mut() else { + return; + }; + self.visit_expr(&mut first.iter); + // every variable is the comprehension's from its first clause on, bound or not + let mut scope: HashMap = HashMap::new(); + for generator in generators.iter() { + target_names(&generator.target, &mut |name| { + if !name.contains(MARKER) && !scope.contains_key(name) { + let at = self.next.get(); + self.next.set(at + 1); + let spelling = Name::new(format!("{name}{MARKER}{at}")); + scope.insert(name.clone(), Binding::Unbound(spelling)); + } + }); + } + self.scopes.borrow_mut().push(scope); + for (clause, generator) in generators.iter_mut().enumerate() { + if clause > 0 { + self.visit_expr(&mut generator.iter); + } + self.target(&mut generator.target); + for condition in &mut generator.ifs { + self.visit_expr(condition); + } + } + for expr in rest { + self.visit_expr(expr); + } + self.scopes.borrow_mut().pop(); + } + + /// a clause's target, whose names are bound from here on + fn target(&self, target: &mut Expr) { + match target { + Expr::Name(name) => { + let mut scopes = self.scopes.borrow_mut(); + if let Some(scope) = scopes.last_mut() + && let Some(binding) = scope.get_mut(&name.id) + { + let spelling = match binding { + Binding::Renamed(spelling) | Binding::Unbound(spelling) => spelling.clone(), + Binding::Hidden => return, + }; + *binding = Binding::Renamed(spelling.clone()); + name.id = spelling; + } + } + Expr::Tuple(tuple) => tuple + .elts + .iter_mut() + .for_each(|element| self.target(element)), + Expr::List(list) => list + .elts + .iter_mut() + .for_each(|element| self.target(element)), + Expr::Starred(starred) => self.target(&mut starred.value), + // an attribute or a subscript is a store through a value the target reads + other => self.visit_expr(other), + } + } +} + +impl Transformer for Renamer { + fn visit_expr(&self, expr: &mut Expr) { + match expr { + Expr::ListComp(node) => self.comprehension(&mut node.generators, &mut [&mut node.elt]), + Expr::SetComp(node) => self.comprehension(&mut node.generators, &mut [&mut node.elt]), + Expr::Generator(node) => { + self.comprehension(&mut node.generators, &mut [&mut node.elt]); + } + Expr::DictComp(node) => match &mut node.key { + Some(key) => { + self.comprehension(&mut node.generators, &mut [key, &mut node.value]); + } + None => self.comprehension(&mut node.generators, &mut [&mut node.value]), + }, + Expr::Lambda(node) => { + // the defaults are read where the lambda is made, before its + // parameters exist + let mut hidden: HashMap = HashMap::new(); + if let Some(parameters) = &mut node.parameters { + walk_parameters(self, parameters); + for parameter in parameters.iter() { + hidden.insert(parameter.name().id.clone(), Binding::Hidden); + } + } + self.scopes.borrow_mut().push(hidden); + self.visit_expr(&mut node.body); + self.scopes.borrow_mut().pop(); + } + // python binds a walrus in the function, and refuses one naming a variable a + // comprehension around it binds + Expr::Named(node) => self.visit_expr(&mut node.value), + Expr::Name(node) => match self.lookup(&node.id) { + Some(Binding::Renamed(spelling)) => node.id = spelling, + Some(Binding::Unbound(_)) => { + self.refused.borrow_mut().get_or_insert_with(|| { + format!( + "a comprehension clause reads `{}` before the clause that binds it", + node.id + ) + }); + } + Some(Binding::Hidden) | None => {} + }, + _ => walk_expr(self, expr), + } + } +} + +fn target_names(target: &Expr, found: &mut impl FnMut(&Name)) { + match target { + Expr::Name(name) => found(&name.id), + Expr::Tuple(tuple) => tuple + .iter() + .for_each(|element| target_names(element, found)), + Expr::List(list) => list.iter().for_each(|element| target_names(element, found)), + Expr::Starred(starred) => target_names(&starred.value, found), + _ => {} + } +} diff --git a/crates/by_irbuild/src/generators.rs b/crates/by_irbuild/src/generators.rs index b3914ae691..c04503c548 100644 --- a/crates/by_irbuild/src/generators.rs +++ b/crates/by_irbuild/src/generators.rs @@ -189,10 +189,11 @@ pub(crate) fn state_names(function: &ast::StmtFunctionDef, locals: &[String]) -> out.push(iterator_field(index)); } let mut seen: HashSet = out.iter().cloned().collect(); + // `*args` and `**kwargs` included: they are parameters the body reads like any other for name in function .parameters - .iter_non_variadic_params() - .map(|parameter| parameter.parameter.name.to_string()) + .iter() + .map(|parameter| parameter.name().to_string()) .chain(locals.iter().cloned()) { if seen.insert(name.clone()) { @@ -369,6 +370,7 @@ pub(crate) fn park_live_registers( }; if declared.insert(*id) { fields.push(by_ir::function::FieldDecl { + cell: false, name: park_field(*id), ty: decl.ty.clone(), default: None, @@ -384,6 +386,8 @@ pub(crate) fn park_live_registers( class: class.to_string(), field: park_field(*id), value: Value::Register(*id), + moves: false, + present: false, })); } let moved = BlockId(function.blocks.len()); @@ -564,8 +568,8 @@ pub(crate) fn definitely_assigned(function: &ast::StmtFunctionDef) -> HashSet = function .parameters - .iter_non_variadic_params() - .map(|parameter| parameter.parameter.name.to_string()) + .iter() + .map(|parameter| parameter.name().to_string()) .collect(); let mut mentioned: HashSet = HashSet::new(); diff --git a/crates/by_irbuild/src/lib.rs b/crates/by_irbuild/src/lib.rs index d0bd293f63..22b8976e32 100644 --- a/crates/by_irbuild/src/lib.rs +++ b/crates/by_irbuild/src/lib.rs @@ -9,6 +9,7 @@ //! milestone and only the speed varies. mod closures; +mod comprehension_scopes; mod generators; pub mod mapper; pub mod shims; @@ -104,8 +105,8 @@ use by_ir::function::{ KeywordValue, ModuleIr, ModuleName, SlotAlias, qualify, }; use by_ir::ops::{ - BinOp, BlockId, CmpOp, Conversion, LicenceKind, Mutation, Op, RegisterId, StandardError, - Terminator, UnaryOp, Value, + BinOp, BlockId, CmpOp, Concatenation, Conversion, LicenceKind, Mutation, Op, RegisterId, + StandardError, Terminator, UnaryOp, Value, }; use by_ir::rtype::{Primitive, RType}; use mapper::{Decline, Layouts, Lowered, map_fixed_tuple, map_type, map_type_with}; @@ -114,7 +115,7 @@ use ruff_python_ast::{ helpers, }; use ruff_python_stdlib::identifiers::is_identifier; -use ruff_text_size::{Ranged, TextSize}; +use ruff_text_size::{Ranged, TextRange, TextSize}; use ty_python_semantic::ProgramEnvironment; use ty_python_semantic::types::{KnownClass, SpecialFormType, Type, TypeDefinition, TypeQualifier}; use ty_python_semantic::{HasType, SemanticModel}; @@ -597,6 +598,7 @@ pub fn build_module( directs: &no_directs, // a module-level frame is in no class body, so nothing it names is mangled owner: None, + frame: None, }; // pass one and a half: the direct editions, before any caller, because an `await` @@ -612,7 +614,7 @@ pub fn build_module( continue; } if let Ok(edition) = lower_direct_edition(unit, function) - .and_then(|mut edition| verify_one(&mut edition).map(|()| edition)) + .and_then(|mut edition| verify_one(&bases, &mut edition).map(|()| edition)) { directs.insert(function.name.to_string()); direct_editions.push(edition); @@ -627,7 +629,7 @@ pub fn build_module( // pass two: bodies, with the layouts available for stmt in suite { if let Stmt::ClassDef(class) = stmt { - match lower_class(unit, class).and_then(verified_class) { + match lower_class(unit, class).and_then(|lowered| verified_class(&bases, lowered)) { Ok((lowered, environments)) => { module.classes.push(lowered); // an environment a method needed is a sibling class, and the layout @@ -650,7 +652,7 @@ pub fn build_module( .extend(promoted_places(db, env, model, function, &layouts)); match defined_once(suite, function) .and_then(|()| lower_function(unit, function)) - .and_then(verified) + .and_then(|lowered| verified(&bases, lowered)) { Ok((lowered, environments)) => { module.functions.push(lowered); @@ -660,7 +662,9 @@ pub fn build_module( // boxed one still stands and every caller reaches that for positions in arrays.get(function.name.as_str()).into_iter().flatten() { if let Ok(edition) = lower_array_edition(unit, function, positions) - .and_then(|mut edition| verify_one(&mut edition).map(|()| edition)) + .and_then(|mut edition| { + verify_one(&bases, &mut edition).map(|()| edition) + }) { module.functions.push(edition); } @@ -1252,6 +1256,7 @@ fn lower_generator( let fields: Vec = names .into_iter() .map(|name| by_ir::function::FieldDecl { + cell: false, optional: false, defaulted_by: None, // the state number is never unset — the constructor writes 0 — so it is @@ -1328,6 +1333,7 @@ fn lower_generator( fields_are_parameters: true, dataclass: false, immutable: false, + environment: false, keywords: Vec::new(), }], )) @@ -1517,7 +1523,9 @@ fn lower_resume( zero_super: Err( "a `super()` with no arguments reads slot zero, which a generator's resume frame fills with its state", ), + unwound: Vec::new(), comprehensions: 0, + unnarrowed: None, language: unit.language, recheck_licences: unit.recheck_licences, environment: None, @@ -1622,6 +1630,8 @@ fn lower_resume( class: class.to_string(), field: generators::STATE_FIELD.to_string(), value: Value::Int(-1), + moves: false, + present: false, }); let nothing = lowering.builder.temp(RType::OBJECT); lowering.builder.push(Op::Box { @@ -1647,20 +1657,22 @@ fn lower_resume( /// the optimization passes are held to a stricter standard — a pass bug fails the /// build, because it is not user code that provoked it fn verified( + bases: &HashMap, lowered: (Function, Vec), ) -> Lowered<(Function, Vec)> { let (mut function, mut environments) = lowered; - verify_one(&mut function)?; + verify_one(bases, &mut function)?; for method in environments .iter_mut() .flat_map(|environment| environment.methods.iter_mut()) { - verify_one(method)?; + verify_one(bases, method)?; } Ok((function, environments)) } fn verified_class( + bases: &HashMap, lowered: (by_ir::function::ClassIr, Vec), ) -> Lowered<(by_ir::function::ClassIr, Vec)> { let (mut class, mut environments) = lowered; @@ -1669,16 +1681,16 @@ fn verified_class( .iter_mut() .flat_map(|environment| environment.methods.iter_mut()), ) { - verify_one(method)?; + verify_one(bases, method)?; } Ok((class, environments)) } -fn verify_one(function: &mut Function) -> Lowered<()> { +fn verify_one(bases: &HashMap, function: &mut Function) -> Lowered<()> { // a local some path reads before writing is compiled with a byte saying whether it // was written, so the flag has to be set before the verifier objects to the read by_ir::unbound_locals::mark(function); - by_ir::verify::verify(function).map_err(|errors| { + by_ir::verify::verify_within(function, bases).map_err(|errors| { let detail = errors .iter() .map(|error| error.message.clone()) @@ -2773,6 +2785,7 @@ fn lower_class<'a>( exported: true, name: class.name.to_string(), immutable, + environment: false, base, // neither written nor generated: a `data class` always gets one, a written // `__init__` is a method of its own, and an accessor block's storage is @@ -3368,6 +3381,7 @@ fn init_fields( continue; } fields.push(by_ir::function::FieldDecl { + cell: false, name: name.clone(), ty: ty.clone(), default: None, @@ -3392,6 +3406,7 @@ fn init_fields( .map(|(_, rtype)| rtype.clone()) .ok_or_else(|| Decline::new("an attribute assignment has no representation"))?; fields.push(by_ir::function::FieldDecl { + cell: false, name, ty, default: None, @@ -3408,6 +3423,7 @@ fn init_fields( continue; } fields.push(by_ir::function::FieldDecl { + cell: false, name: name.clone(), ty: ty.clone(), default: None, @@ -3424,6 +3440,7 @@ fn init_fields( continue; } fields.push(by_ir::function::FieldDecl { + cell: false, name: name.clone(), ty: ty.clone(), default: None, @@ -3515,6 +3532,7 @@ fn slot_fields( continue; } fields.push(by_ir::function::FieldDecl { + cell: false, name, ty: RType::OBJECT, default: None, @@ -5549,6 +5567,7 @@ fn class_fields( Decline::new("an accessor block's storage has no inferred type") })?; fields.push(by_ir::function::FieldDecl { + cell: false, name, ty: map_type_with(db, env, ty, layouts)?, default: Some(default), @@ -5783,6 +5802,7 @@ fn data_fields( .inferred_type(model) .ok_or_else(|| Decline::new("a field has no inferred type"))?; fields.push(by_ir::function::FieldDecl { + cell: false, name, ty: map_type_with(db, env, ty, layouts)?, default, @@ -6756,6 +6776,7 @@ fn lower_function_with_receiver( frame: Frame, ) -> Lowered<(Function, Vec)> { surface::gate_function(unit.language.source_type(), function)?; + let function = &comprehension_scopes::scoped(function)?; let Unit { env, db, @@ -6782,6 +6803,18 @@ fn lower_function_with_receiver( // applies them to the closure it just made — see `nested_def`. carrying them here // as well would apply them a second time, to the environment class's method let nested = matches!(receiver, Some(Receiver::Implicit(_))); + // what python calls this frame: a nested function after the frames it is written + // in, and anything else after its class and its own name + let python_name = match captures { + Some(entry) if entry.lambda.is_some() => "".to_string(), + _ => function.name.to_string(), + }; + let qualname = match (unit.frame, unit.owner) { + (Some(enclosing), _) if nested => format!("{enclosing}..{python_name}"), + (_, Some(class)) => format!("{class}.{python_name}"), + (_, None) => python_name.clone(), + }; + let named_by_its_frame = nested; let mut decorators = Vec::with_capacity(function.decorator_list.len()); let mut wrote_a_decorator = false; for decorator in function.decorator_list.iter().filter(|_| !nested) { @@ -6840,7 +6873,15 @@ fn lower_function_with_receiver( "a generator that shares a cell with the frame around it is not lowered yet", )); } - return lower_generator(unit, function, decorators, receiver, captures); + let (mut constructor, classes) = + lower_generator(unit, function, decorators, receiver, captures)?; + if named_by_its_frame { + constructor.nested = Some(by_ir::function::NestedName { + name: python_name, + qualname, + }); + } + return Ok((constructor, classes)); } let Signature { @@ -6963,11 +7004,37 @@ fn lower_function_with_receiver( } else { owned }; + // a shared cell holds what every frame writing it stores: this one, and each nested + // function at any depth that writes it through `nonlocal`. where they all store one + // representation with an unset value of its own — a tagged `int`'s error value, or a + // pointer's NULL — the cell is held that way, and anything else is an `object` + let cell = |name: &str| { + let mut stored = representation(name); + for writer in closures::nonlocal_writers(&function.body, name) { + let written = local_representations( + db, + env, + model, + &writer.body, + layouts, + unit.sealed, + unit.arrays, + ) + .into_iter() + .find(|(candidate, _)| candidate == name) + .map(|(_, rtype)| rtype); + stored = stored + .zip(written) + .map(|(left, right)| covering(&left, &right)); + } + stored.filter(has_an_unset_value).unwrap_or(RType::OBJECT) + }; let outer_environment = closures::environment( &frame_name, enclosing, &nested, &representation, + &cell, &frame_owned, )?; let (environment, outer_environment) = if split { @@ -6976,6 +7043,7 @@ fn lower_function_with_receiver( Some(&frame_name), &nested, &representation, + &cell, &bindings_here, )?; (closure, outer_environment) @@ -6991,6 +7059,7 @@ fn lower_function_with_receiver( for entry in &nested { for parameter in computed_default_names(&entry.def) { environment.fields.push(by_ir::function::FieldDecl { + cell: false, name: by_ir::function::receiver_default_field( entry.def.name.as_str(), ¶meter, @@ -7127,7 +7196,7 @@ fn lower_function_with_receiver( .is_some_and(|id| id.index() < params.len()); values.push(match locals.get(&field.name) { Some(&id) if is_parameter && shared.contains(&field.name) => { - Some(boxed_object(&mut builder, id)) + Some(seeded_cell(&mut builder, id, &field.ty)) } Some(&id) if !shared.contains(&field.name) => Some(Value::Register(id)), _ => None, @@ -7161,7 +7230,7 @@ fn lower_function_with_receiver( // other cell starts unset, so reading it before a write is // `UnboundLocalError`, exactly as python reports it Some(&id) if is_parameter && shared.contains(&field.name) => { - Some(boxed_object(&mut builder, id)) + Some(seeded_cell(&mut builder, id, &field.ty)) } Some(&id) if !shared.contains(&field.name) => Some(Value::Register(id)), _ => None, @@ -7297,7 +7366,9 @@ fn lower_function_with_receiver( zero_super, language: unit.language, recheck_licences: unit.recheck_licences, + unwound: Vec::new(), comprehensions: 0, + unnarrowed: None, generator: None, delegations: 0, contexts: 0, @@ -7339,6 +7410,12 @@ fn lower_function_with_receiver( let mut lowered = lowering.builder.finish(); lowered.binding = binding; + if named_by_its_frame { + lowered.nested = Some(by_ir::function::NestedName { + name: python_name, + qualname: qualname.clone(), + }); + } // the environment's methods are the nested bodies, lowered with the environment // as the receiver — so a captured read is a field read like any other let environments = match environment { @@ -7353,6 +7430,7 @@ fn lower_function_with_receiver( let inner_unit = Unit { layouts, methods, + frame: Some(&qualname), ..unit }; let mut lowered_methods = Vec::with_capacity(nested.len()); @@ -7396,6 +7474,7 @@ fn lower_function_with_receiver( fields_are_parameters: true, dataclass: false, immutable: false, + environment: true, }) .collect(); all.push(by_ir::function::ClassIr { @@ -7416,6 +7495,7 @@ fn lower_function_with_receiver( fields_are_parameters: true, dataclass: false, immutable: false, + environment: true, }); all.extend(inner_environments); all @@ -7515,6 +7595,10 @@ struct Unit<'a> { /// decides how python mangles a private name — see [`mangled`]. a nested frame /// inherits it, because the mangling follows the source and not the receiver owner: Option<&'a str>, + /// the `__qualname__` of the frame a nested function is written in, which names the + /// nested one — see [`by_ir::function::Function::nested`]. only a nested frame has + /// one: anything else is named by its class and its own name + frame: Option<&'a str>, } /// what a nested function reads through its environment @@ -7559,9 +7643,9 @@ enum Place { }, /// a *shared* cell — a field both frames write, so both see one value. /// - /// it starts unset, and reading it before a write is `UnboundLocalError`. that is - /// why a cell is always `object`: NULL has to be distinguishable from every value - /// it could hold, and an unboxed zero is not + /// it starts unset, and reading it before a write is `UnboundLocalError`. so a cell is + /// held in a representation with an unset value distinct from every value it could + /// hold — see [`has_an_unset_value`] Cell { receiver: RegisterId, class: String, @@ -7569,6 +7653,7 @@ enum Place { /// whether this frame *closes over* the name rather than owning it, which /// decides whether an unset read is `NameError` or `UnboundLocalError` free: bool, + ty: RType, }, /// a field of an environment further up the chain /// @@ -7645,7 +7730,14 @@ enum Cleanup { direct: Option, }, /// an `except` block's handled exception, put back on the way out - Handled(RegisterId), + /// + /// `exception` is what the block marked as handled and `previous` is what that + /// displaced. a suspension inside the block gives `previous` back to whoever + /// resumed the frame and takes `exception` back when it is resumed again + Handled { + previous: RegisterId, + exception: RegisterId, + }, } /// an assignment target with its location parts already evaluated @@ -8051,7 +8143,14 @@ fn signature( deferring.push(params.len()); RType::FLOAT } else { - map_type_with(db, env, ty, layouts)? + let rtype = map_type_with(db, env, ty, layouts)?; + // where `float` admits no `int` it still admits a subclass of + // `float`, which a `double` would strip of its type and its + // operators, so the boundary tests the call the same way + if receiver.is_none() && rtype == RType::FLOAT { + deferring.push(params.len()); + } + rtype } } }; @@ -8345,6 +8444,26 @@ fn is_reflected_or_augmented_operator(name: &str) -> bool { /// /// a cell holds objects, and a parameter that is already one is already what the cell /// wants — boxing it again is ill-formed, and the verifier says so +/// what a parameter that is a shared cell seeds the cell with: its own register where the +/// cell holds the parameter's representation, and its box where the cell is an `object` +fn seeded_cell( + builder: &mut by_ir::builder::FunctionBuilder, + id: RegisterId, + cell: &RType, +) -> Value { + if builder.register_type(id) == Some(cell) { + return Value::Register(id); + } + boxed_object(builder, id) +} + +/// whether a representation has a value of its own that says "unset", distinct from every +/// value it holds, which is what a cell needs before its first write: NULL for a single +/// reference, and the error value for a tagged `int` +fn has_an_unset_value(rtype: &RType) -> bool { + *rtype == RType::INT || rtype.is_object_reference() +} + fn boxed_object(builder: &mut by_ir::builder::FunctionBuilder, id: RegisterId) -> Value { if builder.register_type(id) == Some(&RType::OBJECT) { return Value::Register(id); @@ -8775,7 +8894,8 @@ fn local_representations( for stmt in walk(body) { // a comprehension's target is a local of this frame — the comprehension is - // desugared into it — so a closure inside one captures it like any other + // desugared into it, under a spelling of its own (see `comprehension_scopes`) — + // so a closure inside one captures it like any other let mut targets: Vec<(String, RType)> = Vec::new(); for expr in crate::closures::statement_expressions(stmt) { crate::closures::visit_expressions(expr, &mut |child| { @@ -9234,6 +9354,11 @@ struct Lowering<'a, 'db> { /// the exception each enclosing `except` block caught, innermost last — which is /// what a bare `raise` re-raises handling: Vec, + /// the stretches of [`Self::cleanups`] an early exit being lowered has already run + /// + /// they stay on the stack, because the path that falls through has to run them too, + /// so this is what says a handled exception among them has been put back already + unwound: Vec>, /// what a zero-argument `super()` in this frame stands for — the class the `def` /// is written in and the parameter python reads out of slot zero — or why this /// frame stands for nothing. see [`zero_argument_super`] @@ -9247,6 +9372,34 @@ struct Lowering<'a, 'db> { /// container forms back into the method's. so only the depth being zero says /// slot zero is the receiver whatever the interpreter turns out to be comprehensions: usize, + /// the expression whose value python asks nothing of, so a call's answer is handed + /// on as it is rather than narrowed to what the checker declared + /// + /// an expression statement throws its value away: `cb()` alone on a line, with `cb` + /// declared to answer an `int`, is not narrowed to one. `raise` asks only that the + /// value be an exception, which raising it checks: `raise Refused(...)` raises + /// whatever the name answered, a class rebound after import included. named by its + /// range, which no expression inside it shares + unnarrowed: Option, +} + +/// where [`Lowering::resolve_builtin`] reads the object a builtin's name resolves to +#[derive(Clone, Copy, PartialEq, Eq)] +enum Wanted { + /// on every path, for a use that calls it whether or not it is the builtin + Always, + /// only where it is not the builtin, which is the only place the use calls it + WhereRebound, +} + +/// a `range(...)` a loop may count over, as it stood before anything was counted +struct CountedRange { + /// what the name `range` resolved to + callee: Value, + /// whether that is the interpreter's own `range` + genuine: RegisterId, + start: Option<(Value, RType)>, + stop: (Value, RType), } /// the two halves of the `super(C, self)` a zero-argument `super()` is sugar for @@ -9269,6 +9422,187 @@ impl Lowering<'_, '_> { mangled(self.owner.as_deref(), written) } + /// the class whose storage keeps `name` for an instance of `class`, and what it keeps + /// there + /// + /// nearly every class carries its base's fields at the head of its own layout, so the + /// first rung answers and this is the lookup that was always done. the exception is a + /// class that adds no field of its own: it is laid out with *none at all*, because its + /// instances are its base's at the base's own offsets and there is no region past them + /// for it to own. an attribute such a class reaches has to be named against the base, + /// which is the same thing the base's own methods do — `logging.handlers` writes + /// `self.closeOnError` in both `SocketHandler.__init__` and the `DatagramHandler` + /// below it, and only the second has an empty layout to look in. + /// + /// answering `None` is not a decline: the caller has a property to try and the dynamic + /// form behind that, and [`holds_attribute`](Self::holds_attribute) is what decides + /// whether the attribute has anywhere to go at all + /// raise `AttributeError` where the instance lacks the field about to be read + /// + /// an optional field's own read asks for itself, and answers with a class-level value + /// where there is one + fn require_field(&mut self, receiver: &Value, owner: &str, held: &by_ir::function::FieldDecl) { + if held.optional || !held.tracks_absence() { + return; + } + self.builder.push(Op::RequireField { + receiver: receiver.clone(), + class: owner.to_string(), + field: held.name.clone(), + }); + } + + /// `receiver.name` read out of the layout, where `name` is a field `owner` holds + /// + /// a class something extends or changes can have the name answered by something other + /// than the field: a property or a `__getattribute__` on an interpreted subclass, or a + /// descriptor put on the class after import. so a receiver of such a class is read at + /// the offset only while [`Op::FieldStands`] says none of those has happened, and + /// through the attribute otherwise + fn field_read( + &mut self, + receiver: Value, + receiver_ty: &RType, + class: &str, + owner: String, + held: &by_ir::function::FieldDecl, + ) -> (Value, RType) { + let field_ty = held.ty.clone(); + let dest = self.builder.temp(field_ty.clone()); + let join = self.mutable.contains(class).then(|| { + let (hit, miss, join) = self.field_guard(&receiver, class, &held.name); + self.builder.switch_to(miss); + let object = self.widen_to_object(receiver.clone(), receiver_ty); + let answer = self.builder.temp(RType::OBJECT); + self.builder.push(Op::GetAttr { + dest: answer, + receiver: object, + name: held.name.clone(), + }); + // what the checker says the attribute is, asked of whatever answered for it + if field_ty == RType::OBJECT { + self.builder.assign(dest, Value::Register(answer)); + } else { + self.builder.push(Op::Unbox { + dest, + src: Value::Register(answer), + to: field_ty.clone(), + }); + } + self.builder.terminate(Terminator::Goto(join)); + self.builder.switch_to(hit); + join + }); + self.require_field(&receiver, &owner, held); + self.builder.push(Op::GetField { + dest, + receiver, + class: owner, + field: held.name.clone(), + }); + if let Some(join) = join { + self.builder.terminate(Terminator::Goto(join)); + self.builder.switch_to(join); + } + (Value::Register(dest), field_ty) + } + + /// `receiver.name = value` into the layout, where `name` is a field `owner` holds — + /// behind the test [`Self::field_read`] makes, for the same reasons + #[expect( + clippy::too_many_arguments, + reason = "the write and everything it is written through" + )] + fn field_write( + &mut self, + receiver: &Value, + receiver_ty: &RType, + class: &str, + owner: String, + held: &by_ir::function::FieldDecl, + value: Value, + ty: &RType, + ) -> Lowered<()> { + let join = if self.mutable.contains(class) { + let (hit, miss, join) = self.field_guard(receiver, class, &held.name); + self.builder.switch_to(miss); + // the value as it was handed over: whatever answers for the name decides + // what it accepts + let object = self.widen_to_object(receiver.clone(), receiver_ty); + let boxed = self.widen_to_object(value.clone(), ty); + let refused = self.builder.temp(RType::BIT); + self.builder.push(Op::SetAttr { + dest: refused, + receiver: object, + name: held.name.clone(), + value: boxed, + }); + self.builder.terminate(Terminator::Goto(join)); + self.builder.switch_to(hit); + Some(join) + } else { + None + }; + let value = self.coerce(value, ty, &held.ty)?; + self.builder.push(Op::SetField { + receiver: receiver.clone(), + class: owner, + field: held.name.clone(), + value, + moves: false, + present: false, + }); + if let Some(join) = join { + self.builder.terminate(Terminator::Goto(join)); + self.builder.switch_to(join); + } + Ok(()) + } + + /// the test in front of a field access on a receiver typed `class`, answering with the + /// block it goes to when the field stands, the one it goes to otherwise, and the one + /// both arms end in + fn field_guard( + &mut self, + receiver: &Value, + class: &str, + field: &str, + ) -> (BlockId, BlockId, BlockId) { + let hit = self.builder.new_block(); + let miss = self.builder.new_block(); + let join = self.builder.new_block(); + let stands = self.builder.temp(RType::BIT); + self.builder.push(Op::FieldStands { + dest: stands, + src: receiver.clone(), + class: class.to_string(), + field: field.to_string(), + }); + self.builder.terminate(Terminator::Branch { + cond: Value::Register(stands), + then_block: hit, + else_block: miss, + }); + (hit, miss, join) + } + + fn field_home(&self, class: &str, name: &str) -> Option<(String, by_ir::function::FieldDecl)> { + let mut current = class; + // bounded by the class count, the way every base walk here is: a chain that + // visits a class twice is a cycle, and the layouts never settle on one + for _ in 0..=self.bases.len() { + if let Some(held) = self + .layouts + .get(current) + .and_then(|fields| fields.iter().find(|field| field.name == name)) + { + return Some((current.to_string(), held.clone())); + } + current = self.bases.get(current).map(String::as_str)?; + } + None + } + /// whether an instance whose type is `class` — or, where the receiver is not exact, /// any emitted class under it — has somewhere to keep an attribute called `name` /// @@ -9468,6 +9802,8 @@ impl Lowering<'_, '_> { class, field: generators::STATE_FIELD.to_string(), value: Value::Int(-1), + moves: false, + present: false, }); // the frame is finished, and the value goes with the finish — which is // how it reaches whatever was driving the iteration, and how `await` @@ -9654,7 +9990,10 @@ impl Lowering<'_, '_> { Stmt::Raise(node) => self.raise_statement(node), Stmt::Expr(node) => { // an expression statement is only worth lowering for its effect - self.expression(&node.value).map(|_| ()) + let enclosing = self.unnarrowed.replace(node.value.range()); + let lowered = self.expression(&node.value).map(|_| ()); + self.unnarrowed = enclosing; + lowered } other => Err(Decline::new(format!( "{} is not lowered yet", @@ -9849,8 +10188,7 @@ impl Lowering<'_, '_> { self.builder.switch_to(body_block); if let Some(guard) = &case.guard { - let (cond, cond_ty) = self.expression(guard)?; - let cond = self.truthy(cond, &cond_ty); + let cond = self.condition(guard)?; let guarded = self.builder.new_block(); self.builder.terminate(Terminator::Branch { cond, @@ -10266,9 +10604,25 @@ impl Lowering<'_, '_> { // representation the field holds it in. an attribute the pattern names // any other way still goes out through the lookup, where *absent* is an // answer rather than an error + // an instance without the field does not match, the way python's lookup + // failing with `AttributeError` does not let field_read = |lowering: &mut Self, index: usize| { let receiver = held.clone()?; let (class, field, rtype) = reads.get(index)?.clone()?; + let has = lowering.builder.temp(RType::BIT); + lowering.builder.push(Op::FieldIsSet { + dest: has, + receiver: receiver.clone(), + class: class.clone(), + field: field.clone(), + }); + let present = lowering.builder.new_block(); + lowering.builder.terminate(Terminator::Branch { + cond: Value::Register(has), + then_block: present, + else_block: unmatched, + }); + lowering.builder.switch_to(present); let dest = lowering.builder.temp(rtype.clone()); lowering.builder.push(Op::GetField { dest, @@ -10416,8 +10770,7 @@ impl Lowering<'_, '_> { } fn if_statement(&mut self, node: &ast::StmtIf) -> Lowered<()> { - let (cond, cond_ty) = self.expression(&node.test)?; - let cond = self.truthy(cond, &cond_ty); + let cond = self.condition(&node.test)?; let then_block = self.builder.new_block(); let else_block = self.builder.new_block(); @@ -10448,8 +10801,7 @@ impl Lowering<'_, '_> { } // `elif:` — a fresh branch inside the else block Some(test) => { - let (cond, cond_ty) = self.expression(test)?; - let cond = self.truthy(cond, &cond_ty); + let cond = self.condition(test)?; let elif_then = self.builder.new_block(); let elif_else = self.builder.new_block(); self.builder.terminate(Terminator::Branch { @@ -10488,8 +10840,12 @@ impl Lowering<'_, '_> { fn while_statement(&mut self, node: &ast::StmtWhile) -> Lowered<()> { // `while i < len(A)` with a counting `i` and an `A` the body leaves alone puts - // every `A[i]` inside it in range, which is the one thing the checked read tests - let proven = counted_over(node); + // every `A[i]` inside it in range, which is the one thing the checked read tests. + // that is the builtin `len`'s answer: a name bound to something else is not + // asked here, and a `len` rebound at runtime refuses a buffer before the body + // can run — see [`Self::len_call`] + let proven = counted_over(node) + .filter(|_| !self.native_callees.contains("len") && !self.binds("len")); let header = self.builder.new_block(); let body = self.builder.new_block(); // the natural exit and the `break` exit are different blocks: `else` runs @@ -10500,8 +10856,7 @@ impl Lowering<'_, '_> { self.builder.terminate(Terminator::Goto(header)); self.builder.switch_to(header); - let (cond, cond_ty) = self.expression(&node.test)?; - let cond = self.truthy(cond, &cond_ty); + let cond = self.condition(&node.test)?; self.builder.terminate(Terminator::Branch { cond, then_block: body, @@ -10581,26 +10936,38 @@ impl Lowering<'_, '_> { /// the stack stays intact because the *fall-through* path still has to run them /// too — an early exit is an extra path, not a replacement for the normal one fn unwind(&mut self, depth: usize) -> Lowered<()> { - let pending: Vec = self.cleanups[depth..].iter().rev().cloned().collect(); - for cleanup in pending { - match cleanup { - Cleanup::Finally(body) => self.block(&body)?, - Cleanup::Handled(handled) => self.builder.push(Op::PopHandled { - value: Value::Register(handled), - }), - Cleanup::Context { - manager, - is_async, - direct, - } => { - let (manager, _) = self.read_place(&manager)?; - let ignored = self.builder.temp(RType::BIT); - if is_async { - let none = self.widen_to_object(Value::None, &RType::NONE); - self.await_exit(manager, none, ignored)?; - } else { - self.leave_context(manager, ignored, direct.as_deref()); - } + let end = self.cleanups.len(); + for at in (depth..end).rev() { + let Some(cleanup) = self.cleanups.get(at).cloned() else { + continue; + }; + self.unwound.push(at..end); + let ran = self.run_cleanup(cleanup); + self.unwound.pop(); + ran?; + } + Ok(()) + } + + /// one cleanup of an early exit, the ones above it having already run + fn run_cleanup(&mut self, cleanup: Cleanup) -> Lowered<()> { + match cleanup { + Cleanup::Finally(body) => self.block(&body)?, + Cleanup::Handled { previous, .. } => self.builder.push(Op::PopHandled { + value: Value::Register(previous), + }), + Cleanup::Context { + manager, + is_async, + direct, + } => { + let (manager, _) = self.read_place(&manager)?; + let ignored = self.builder.temp(RType::BIT); + if is_async { + let none = self.widen_to_object(Value::None, &RType::NONE); + self.await_exit(manager, none, ignored)?; + } else { + self.leave_context(manager, ignored, direct.as_deref()); } } } @@ -10658,6 +11025,12 @@ impl Lowering<'_, '_> { class: class.to_string(), exact: false, }; + // a manager held in its own class's register is already what the call takes + if let Value::Register(id) = &manager + && self.builder.register_type(*id) == Some(&ty) + { + return manager; + } let narrowed = self.builder.temp(ty.clone()); self.builder.push(Op::Unbox { dest: narrowed, @@ -10667,9 +11040,22 @@ impl Lowering<'_, '_> { Value::Register(narrowed) } + /// the manager as the object the protocol takes, whichever register holds it + fn manager_object(&mut self, manager: Value) -> Value { + let ty = match &manager { + Value::Register(id) => self.builder.register_type(*id).cloned(), + other => other.immediate_type(), + }; + match ty { + Some(ty) => self.widen_to_object(manager, &ty), + None => manager, + } + } + /// `__enter__`, reached directly where the class licenses it fn enter_context(&mut self, manager: Value, dest: RegisterId, direct: Option<&str>) { let Some(class) = direct else { + let manager = self.manager_object(manager); self.builder.push(Op::Enter { dest, manager }); return; }; @@ -10681,6 +11067,7 @@ impl Lowering<'_, '_> { "__enter__", 1, ) else { + let manager = self.manager_object(manager); self.builder.push(Op::Enter { dest, manager }); return; }; @@ -10719,6 +11106,7 @@ impl Lowering<'_, '_> { ) }); let (Some(class), Some(owner)) = (direct, owner) else { + let manager = self.manager_object(manager); let no_exception = self.widen_to_object(Value::None, &RType::NONE); self.builder.push(Op::ExitContext { dest, @@ -10774,12 +11162,19 @@ impl Lowering<'_, '_> { _ => None, }) }; - let manager = self.widen_to_object(manager, &manager_ty); // the manager is read again on both exits, so it lives in a register of its - // own rather than being re-evaluated + // own rather than being re-evaluated. where both halves are reached directly + // that register is the class's own, so neither half narrows back to it — but + // inside a generator it is parked in a field every suspension goes through, and + // a field there is an `object` + let (manager, held_ty) = if direct.is_some() && self.generator.is_none() { + (manager, manager_ty.clone()) + } else { + (self.widen_to_object(manager, &manager_ty), RType::OBJECT) + }; let held = self .builder - .local(format!("$manager{}", self.contexts), RType::OBJECT); + .local(format!("$manager{}", self.contexts), held_ty); self.contexts += 1; self.builder.assign(held, manager); // inside a generator the manager outlives the frame: the body suspends and @@ -10866,6 +11261,7 @@ impl Lowering<'_, '_> { // its own read: this block is reached after a suspension the other never // saw, and the register the normal exit used is stale by then let (raising, _) = self.read_place(&held)?; + let raising = self.manager_object(raising); // awaiting the exit suspends, and the reraise below still needs the // exception afterwards — so it goes into a field too let parked_exception = if node.is_async { @@ -10873,21 +11269,58 @@ impl Lowering<'_, '_> { } else { None }; - if node.is_async { - self.await_exit(raising, Value::Register(exception), suppressed)?; + // while `__exit__` decides, the block's exception is the one being handled: + // it is what `sys.exception()` answers inside `__exit__`, and what anything + // raised there chains onto. the exit gets a block of its own so that leaving + // it by raising puts the handled exception back first, as an `except` does + let handled = self.builder.temp(RType::OBJECT); + self.builder.push(Op::PushHandled { + dest: handled, + value: Value::Register(exception), + }); + let deciding = self.builder.new_block(); + let exit_raised = self.builder.new_block(); + self.builder.terminate(Terminator::Goto(deciding)); + self.builder.switch_to(deciding); + let outer = self.builder.set_error_target(Some(exit_raised)); + self.cleanups.push(Cleanup::Handled { + previous: handled, + exception, + }); + let decided = if node.is_async { + self.await_exit(raising, Value::Register(exception), suppressed) } else { self.builder.push(Op::ExitContext { dest: suppressed, manager: raising, exception: Value::Register(exception), }); - } + Ok(()) + }; + self.cleanups.pop(); + decided?; + self.builder.push(Op::PopHandled { + value: Value::Register(handled), + }); let reraise = self.builder.new_block(); self.builder.terminate(Terminator::Branch { cond: Value::Register(suppressed), then_block: after, else_block: reraise, }); + self.builder.set_error_target(outer); + + // what `__exit__` raised leaves in place of the block's exception + self.builder.switch_to(exit_raised); + let pending = self.builder.temp(RType::OBJECT); + self.builder.push(Op::FetchException { dest: pending }); + self.builder.push(Op::PopHandled { + value: Value::Register(handled), + }); + self.builder.push(Op::Reraise { + value: Value::Register(pending), + }); + self.builder.terminate(Terminator::Unreachable); self.builder.switch_to(reraise); let exception = match &parked_exception { Some(parked) => { @@ -10968,6 +11401,23 @@ impl Lowering<'_, '_> { self.builder.switch_to(handler_entry); let exception = self.builder.temp(RType::OBJECT); self.builder.push(Op::FetchException { dest: exception }); + // the exception is being *handled* from before any clause's class is evaluated, + // which is what an error raised by that evaluation chains onto and what anything + // it calls sees in `sys.exception()`. a raise inside the block — or inside + // anything it calls — chains onto it for the same reason + let handled = self.builder.temp(RType::OBJECT); + self.builder.push(Op::PushHandled { + dest: handled, + value: Value::Register(exception), + }); + // the clauses start in a block of their own, whose error edge leaves with the + // exception handled already + let clauses = self.builder.new_block(); + self.builder.terminate(Terminator::Goto(clauses)); + self.builder.switch_to(clauses); + // what evaluating a clause's class raised leaves in place of the exception, once + // the handled exception is put back and `finally` has run + let mut class_raised: Option = None; for handler in &node.handlers { let ast::ExceptHandler::ExceptHandler(handler) = handler; @@ -10980,8 +11430,18 @@ impl Lowering<'_, '_> { Some(class) => { // an ordinary expression, so a user-defined class, a tuple of // them and a *shadowed* builtin all take one path - let (value, ty) = self.expression(class)?; - let class = self.widen_to_object(value, &ty); + let raised = *class_raised.get_or_insert_with(|| self.builder.new_block()); + let outer = self.builder.set_error_target(Some(raised)); + let class = self + .expression(class) + .map(|(value, ty)| self.widen_to_object(value, &ty)); + let class = match class { + Ok(class) => class, + Err(decline) => { + self.builder.set_error_target(outer); + return Err(decline); + } + }; let test = self.builder.temp(RType::BIT); self.builder.push(Op::ExceptionMatches { dest: test, @@ -10993,6 +11453,7 @@ impl Lowering<'_, '_> { then_block: matched, else_block: next, }); + self.builder.set_error_target(outer); } } @@ -11001,13 +11462,6 @@ impl Lowering<'_, '_> { let target = self.binding(bound.as_str(), &RType::OBJECT); self.write_place(&target, Value::Register(exception), &RType::OBJECT)?; } - // from here the exception is being *handled*, which is what makes a raise - // inside the block — or inside anything it calls — chain onto it - let handled = self.builder.temp(RType::OBJECT); - self.builder.push(Op::PushHandled { - dest: handled, - value: Value::Register(exception), - }); // the body gets its own block for the same reason the `try` body does, // and its own error target so leaving by raising still unwinds @@ -11023,7 +11477,10 @@ impl Lowering<'_, '_> { self.cleanups .push(Cleanup::Finally(node.finalbody.to_vec())); } - self.cleanups.push(Cleanup::Handled(handled)); + self.cleanups.push(Cleanup::Handled { + previous: handled, + exception, + }); self.handling.push(exception); let body = self.block(&handler.body); self.handling.pop(); @@ -11066,13 +11523,31 @@ impl Lowering<'_, '_> { self.builder.switch_to(next); } - // nothing matched: run `finally`, then let the exception continue + // nothing matched: put the handled exception back, run `finally`, then let the + // exception continue + self.builder.push(Op::PopHandled { + value: Value::Register(handled), + }); self.block(&node.finalbody)?; self.builder.push(Op::Reraise { value: Value::Register(exception), }); self.builder.terminate(Terminator::Unreachable); + if let Some(class_raised) = class_raised { + self.builder.switch_to(class_raised); + let pending = self.builder.temp(RType::OBJECT); + self.builder.push(Op::FetchException { dest: pending }); + self.builder.push(Op::PopHandled { + value: Value::Register(handled), + }); + self.block(&node.finalbody)?; + self.builder.push(Op::Reraise { + value: Value::Register(pending), + }); + self.builder.terminate(Terminator::Unreachable); + } + self.builder.switch_to(after); if !after_reached { self.builder.terminate(Terminator::Unreachable); @@ -11082,8 +11557,7 @@ impl Lowering<'_, '_> { /// `assert cond` / `assert cond, "message"` fn assert_statement(&mut self, node: &ast::StmtAssert) -> Lowered<()> { - let (cond, cond_ty) = self.expression(&node.test)?; - let cond = self.truthy(cond, &cond_ty); + let cond = self.condition(&node.test)?; let fail = self.builder.new_block(); let ok = self.builder.new_block(); self.builder.terminate(Terminator::Branch { @@ -11138,13 +11612,9 @@ impl Lowering<'_, '_> { { return Ok(()); } - let (value, ty) = self.expression(exception)?; - let exception = self.widen_to_object(value, &ty); + let exception = self.unnarrowed_object(exception)?; let cause = match &node.cause { - Some(cause) => { - let (value, ty) = self.expression(cause)?; - Some(self.widen_to_object(value, &ty)) - } + Some(cause) => Some(self.unnarrowed_object(cause)?), None => None, }; self.builder.push(Op::RaiseObject { exception, cause }); @@ -11210,6 +11680,7 @@ impl Lowering<'_, '_> { Expr::Name(callee) if callee.id.as_str() == "range" && !self.native_callees.contains("range") + && !self.binds("range") && call.arguments.keywords.is_empty() => { Some(call) @@ -11275,44 +11746,93 @@ impl Lowering<'_, '_> { return self.for_over_iterable(node); }; + // python reads the name before the bounds, and calls whatever it read + let (callee, genuine) = self.resolve_builtin("range", Wanted::Always); // the bounds come first, because whether this is a counting loop at all // depends on their representations — and they are evaluated in this order // either way, so nothing is computed twice - let (start_value, start_ty) = match start { - None => (Value::Int(0), RType::INT), - Some(expr) => self.expression(expr)?, + let start = match start { + None => None, + Some(expr) => Some(self.expression(expr)?), }; let (stop_value, stop_ty) = self.expression(stop)?; + // the arguments as written, for the call python makes of whatever `range` is + let written = |lowering: &mut Self| { + let mut args = Vec::with_capacity(3); + if let Some((value, ty)) = &start { + args.push(lowering.widen_to_object(value.clone(), ty)); + } + args.push(lowering.widen_to_object(stop_value.clone(), &stop_ty)); + if step.is_some() { + args.push(lowering.widen_to_object(Value::Int(step_value), &RType::INT)); + } + args + }; + let (start_value, start_ty) = start.clone().unwrap_or((Value::Int(0), RType::INT)); if start_ty != RType::INT || stop_ty != RType::INT { - // the bounds are already evaluated, so the protocol path takes the - // `range` object built from them rather than the expression again - let start_value = self.widen_to_object(start_value, &start_ty); - let stop_value = self.widen_to_object(stop_value, &stop_ty); - let step_value = self.widen_to_object(Value::Int(step_value), &RType::INT); + // the bounds are already evaluated, so the protocol path takes the object + // the call builds from them rather than the expression again + let args = written(self); let iterable = self.builder.temp(RType::OBJECT); - self.builder.push(Op::CallPython { + self.builder.push(Op::CallValue { dest: iterable, - callee: "range".to_string(), - args: vec![start_value, stop_value, step_value], + callee, + args, }); return self.for_over_value(node, Value::Register(iterable), &RType::OBJECT); } let index = self.binding_register(target.id.as_str(), &RType::INT)?; - self.builder.assign(index, start_value); - // the stop bound is read once into its own register, so a call or a // mutated local cannot change the trip count mid-loop let limit = self.builder.temp(RType::INT); - self.builder.assign(limit, stop_value); + let iterator = self.builder.temp(RType::OBJECT); + // one body serves both ways of stepping: counting while `range` is the builtin, + // and the iteration protocol over what the call answered otherwise + let counting = self.builder.new_block(); + let protocol = self.builder.new_block(); let header = self.builder.new_block(); + let next = self.builder.new_block(); + let bind = self.builder.new_block(); let body = self.builder.new_block(); let step_block = self.builder.new_block(); + let count = self.builder.new_block(); let natural_exit = self.builder.new_block(); let after = self.builder.new_block(); + self.builder.terminate(Terminator::Branch { + cond: Value::Register(genuine), + then_block: counting, + else_block: protocol, + }); + + // each way writes what only the other reads as well, because nothing that checks + // the function can see the two never cross + self.builder.switch_to(counting); + self.builder.assign(index, start_value); + self.builder.assign(limit, stop_value.clone()); + self.builder.push(Op::Box { + dest: iterator, + src: Value::None, + }); self.builder.terminate(Terminator::Goto(header)); + self.builder.switch_to(protocol); + self.builder.assign(limit, Value::Int(0)); + let args = written(self); + let iterable = self.builder.temp(RType::OBJECT); + self.builder.push(Op::CallValue { + dest: iterable, + callee, + args, + }); + self.builder.push(Op::GetIter { + dest: iterator, + src: Value::Register(iterable), + cursor: None, + }); + self.builder.terminate(Terminator::Goto(next)); + self.builder.switch_to(header); let cond = self.builder.temp(RType::BIT); self.builder.push(Op::IntCompare { @@ -11327,6 +11847,34 @@ impl Lowering<'_, '_> { else_block: natural_exit, }); + self.builder.switch_to(next); + let raw = self.builder.temp(RType::OBJECT); + self.builder.push(Op::IterNext { + dest: raw, + iter: Value::Register(iterator), + cursor: None, + }); + let exhausted = self.builder.temp(RType::BIT); + self.builder.push(Op::IsNull { + dest: exhausted, + src: Value::Register(raw), + }); + self.builder.terminate(Terminator::Branch { + cond: Value::Register(exhausted), + then_block: natural_exit, + else_block: bind, + }); + + self.builder.switch_to(bind); + let item = self.builder.temp(RType::INT); + self.builder.push(Op::Unbox { + dest: item, + src: Value::Register(raw), + to: RType::INT, + }); + self.builder.assign(index, Value::Register(item)); + self.builder.terminate(Terminator::Goto(body)); + self.builder.switch_to(body); self.for_pattern(node)?; // `continue` jumps to the step, not the header, or the index never moves @@ -11337,6 +11885,13 @@ impl Lowering<'_, '_> { self.builder.terminate(Terminator::Goto(step_block)); self.builder.switch_to(step_block); + self.builder.terminate(Terminator::Branch { + cond: Value::Register(genuine), + then_block: count, + else_block: next, + }); + + self.builder.switch_to(count); self.builder.push(Op::IntBinary { dest: index, op: BinOp::Add, @@ -11678,6 +12233,22 @@ impl Lowering<'_, '_> { /// /// a register wins over a field: a generator's *parameters* are registers even /// where its locals are fields, and a closure's own parameters shadow a capture + /// whether `class`'s field `name` is a shared cell, which may be unset + fn field_is_cell(&self, class: &str, name: &str) -> bool { + self.layouts + .get(class) + .and_then(|fields| fields.iter().find(|field| field.name == name)) + .is_some_and(|field| field.cell) + } + + /// the representation `class` holds its field `name` in, as the layout says + fn field_type(&self, class: &str, name: &str) -> RType { + self.layouts + .get(class) + .and_then(|fields| fields.iter().find(|field| field.name == name)) + .map_or(RType::OBJECT, |field| field.ty.clone()) + } + fn place(&self, name: &str) -> Option { // asked first, and it has to be: a `global` declaration says this name is not // this frame's to bind, so nothing else may answer for it @@ -11700,6 +12271,7 @@ impl Lowering<'_, '_> { class: owned.class.clone(), name: name.to_string(), free: owned.free, + ty: self.field_type(&owned.class, name), }); } let captured = self.captures.as_ref()?; @@ -11709,6 +12281,7 @@ impl Lowering<'_, '_> { class: captured.class.clone(), name: name.to_string(), free: captured.free, + ty: self.field_type(&captured.class, name), }); } if !captured.names.contains(name) { @@ -12051,8 +12624,9 @@ impl Lowering<'_, '_> { class, name, free, + ty, } => { - let dest = self.builder.temp(RType::OBJECT); + let dest = self.builder.temp(ty.clone()); self.builder.push(Op::GetCell { dest, receiver: Value::Register(*receiver), @@ -12060,16 +12634,17 @@ impl Lowering<'_, '_> { field: name.clone(), free: *free, }); - Ok((Value::Register(dest), RType::OBJECT)) + Ok((Value::Register(dest), ty.clone())) } Place::Chained { path, name, ty } => { let (path, name, ty) = (path.clone(), name.clone(), ty.clone()); let outer = path.last().cloned().unwrap_or_default(); let receiver = self.enclosing_environment(&path); let dest = self.builder.temp(ty.clone()); - // an `object` field up the chain may be a cell, and the checked read is - // sound either way — a field that is never unset simply never fails it - if ty == RType::OBJECT { + // a cell up the chain is read with the test for unset, and so is any + // `object` field there, which is sound either way — a field that is never + // unset simply never fails it + if ty == RType::OBJECT || self.field_is_cell(&outer, &name) { self.builder.push(Op::GetCell { dest, receiver: Value::Register(receiver), @@ -12118,22 +12693,26 @@ impl Lowering<'_, '_> { class: class.clone(), field: name.clone(), value, + moves: false, + present: false, }); Ok(()) } - // a cell holds an `object`, always — see [`Place::Cell`] Place::Cell { receiver, class, name, + ty: cell_ty, .. } => { - let value = self.coerce(value, ty, &RType::OBJECT)?; + let value = self.coerce(value, ty, cell_ty)?; self.builder.push(Op::SetField { receiver: Value::Register(*receiver), class: class.clone(), field: name.clone(), value, + moves: false, + present: false, }); Ok(()) } @@ -12151,6 +12730,8 @@ impl Lowering<'_, '_> { class: outer, field: name, value, + moves: false, + present: false, }); Ok(()) } @@ -12239,18 +12820,9 @@ impl Lowering<'_, '_> { name, } => { if let RType::Instance { class, .. } = receiver_ty - && let Some(fields) = self.layouts.get(class) - && let Some(held) = fields.iter().find(|field| field.name == *name) + && let Some((owner, held)) = self.field_home(class, name) { - let (class, field_ty) = (class.clone(), held.ty.clone()); - let dest = self.builder.temp(field_ty.clone()); - self.builder.push(Op::GetField { - dest, - receiver: receiver.clone(), - class, - field: name.clone(), - }); - return Ok((Value::Register(dest), field_ty)); + return Ok(self.field_read(receiver.clone(), receiver_ty, class, owner, &held)); } // a `@property` whose getter this module lowered is called outright. the // protocol would have arrived at this same body, by way of a lookup on @@ -12334,18 +12906,9 @@ impl Lowering<'_, '_> { name, } => { if let RType::Instance { class, .. } = receiver_ty - && let Some(fields) = self.layouts.get(class) - && let Some(held) = fields.iter().find(|field| field.name == *name) + && let Some((owner, held)) = self.field_home(class, name) { - let (class, field_ty) = (class.clone(), held.ty.clone()); - let value = self.coerce(value, ty, &field_ty)?; - self.builder.push(Op::SetField { - receiver: receiver.clone(), - class, - field: name.clone(), - value, - }); - return Ok(()); + return self.field_write(receiver, receiver_ty, class, owner, &held, value, ty); } // the setter half, called outright, for the reason the read above gives. // a property with no setter is deliberately not here: it falls through to @@ -12948,6 +13511,8 @@ impl Lowering<'_, '_> { class: class.clone(), field: generators::KIND_FIELD.to_string(), value: Value::Int(kind as i64), + moves: false, + present: false, }); let state = i64::try_from(generator.resumptions.len()).unwrap_or(i64::MAX - 1) + 1; self.builder.push(Op::SetField { @@ -12955,11 +13520,52 @@ impl Lowering<'_, '_> { class: class.clone(), field: generators::STATE_FIELD.to_string(), value: Value::Int(state), + moves: false, + present: false, }); + // python hands a suspended frame's handled exception back to its caller and + // takes it again on resumption, so each side sees only its own. the frame may + // be resumed from somewhere else entirely, which is why what each block + // displaced is asked again rather than remembered + let handled: Vec<(RegisterId, RegisterId)> = self + .cleanups + .iter() + .enumerate() + .filter(|(at, _)| !self.unwound.iter().any(|run| run.contains(at))) + .filter_map(|(_, cleanup)| match cleanup { + Cleanup::Handled { + previous, + exception, + } => Some((*previous, *exception)), + _ => None, + }) + .collect(); + if let Some((outermost, _)) = handled.first() { + self.builder.push(Op::PopHandled { + value: Value::Register(*outermost), + }); + } let suspend_at = self.builder.current_block(); self.builder.terminate(Terminator::Return(value)); let resume_at = self.builder.new_block(); self.builder.switch_to(resume_at); + // before the `throw` below is raised, so an exception thrown in at the + // suspension chains onto the frame's own. in a block with no error edge: a + // handler is entered from before its block's first write, and it reads what + // these write + if !handled.is_empty() { + let outer = self.builder.set_error_target(None); + for (previous, exception) in handled { + self.builder.push(Op::PushHandled { + dest: previous, + value: Value::Register(exception), + }); + } + let resumed = self.builder.new_block(); + self.builder.terminate(Terminator::Goto(resumed)); + self.builder.set_error_target(outer); + self.builder.switch_to(resumed); + } if let Some(generator) = &mut self.generator { generator.resumptions.push(generators::Resumption { state, @@ -13017,11 +13623,16 @@ impl Lowering<'_, '_> { class, field: generators::THROWN_FIELD.to_string(), value: cleared, + moves: false, + present: false, }); // the block's error target is the enclosing handler, which is exactly where a - // `throw` at a suspension has to land - self.builder.push(Op::Reraise { - value: Value::Register(thrown), + // `throw` at a suspension has to land. it is *raised* rather than put back: + // python chains an exception thrown into a frame onto what that frame is + // handling + self.builder.push(Op::RaiseObject { + exception: Value::Register(thrown), + cause: None, }); self.builder.terminate(Terminator::Unreachable); @@ -13057,6 +13668,8 @@ impl Lowering<'_, '_> { class: class.clone(), field: field.clone(), value: Value::Register(iterator), + moves: false, + present: false, }); Ok(Place::Field { receiver: RegisterId(0), @@ -13173,6 +13786,8 @@ impl Lowering<'_, '_> { parameter.parameter.name.as_str(), ), value, + moves: false, + present: false, }); } Ok(()) @@ -13457,11 +14072,32 @@ impl Lowering<'_, '_> { map_type_with(self.db, env, ty, self.layouts) } + /// the representation lowering `expr` as a value answers in + /// + /// the checker's type, except for a comparison: that answers with the object its + /// method returned wherever the operands could have a method of their own, whatever + /// the checker was told the method returns + fn representation_of(&self, expr: &Expr) -> Lowered { + if let Expr::Compare(node) = expr { + let mut left = self.peek_type(&node.left)?; + let mut every_bool = true; + for (op, right) in node.ops.iter().zip(node.comparators.iter()) { + let right = self.peek_type(right)?; + every_bool &= Self::compares_to_a_bool(*op, &left, &right); + left = right; + } + if !every_bool { + return Ok(RType::OBJECT); + } + } + self.peek_type(expr) + } + /// the representation that holds every one of `exprs` fn unified_type(&self, exprs: &[&Expr]) -> Lowered { let mut found: Option = None; for expr in exprs { - let ty = self.peek_type(expr)?; + let ty = self.representation_of(expr)?; found = Some(match found { Some(existing) if existing != ty => RType::OBJECT, _ => ty, @@ -13508,8 +14144,7 @@ impl Lowering<'_, '_> { let result_ty = self.unified_type(&[&node.body, &node.orelse])?; let result = self.builder.temp(result_ty.clone()); - let (cond, cond_ty) = self.expression(&node.test)?; - let cond = self.truthy(cond, &cond_ty); + let cond = self.condition(&node.test)?; let then_block = self.builder.new_block(); let else_block = self.builder.new_block(); let join = self.builder.new_block(); @@ -13925,6 +14560,7 @@ impl Lowering<'_, '_> { lhs, rhs, consumes_lhs: false, + concatenation: Concatenation::Operator(mutation), }); } (RType::Primitive(Primitive::Float), RType::Primitive(Primitive::Float)) @@ -14014,6 +14650,11 @@ impl Lowering<'_, '_> { Value::Register(dest) } + /// a comparison whose answer is used as a value + /// + /// that answer is whatever the comparison method returned, which only an `int`, a + /// `float` or a `str` pair — and `is` and `in`, which are no method's — can be relied + /// on to make a `bool`. every other pair answers with the object fn compare(&mut self, node: &ast::ExprCompare) -> Lowered<(Value, RType)> { if node.ops.len() > 1 { return self.chained_compare(node); @@ -14023,9 +14664,81 @@ impl Lowering<'_, '_> { }; let left_value = self.expression(&node.left)?; let right_value = self.expression(right)?; - let bit = - self.emit_compare_of(*op, left_value, right_value, Some(&node.left), Some(right))?; - Ok((bit, RType::BIT)) + self.compare_value_of(*op, left_value, right_value, Some(&node.left), Some(right)) + } + + /// a condition: the truth of a value, which for a comparison is asked of the + /// comparison directly rather than of an object built to hold its answer + fn condition(&mut self, expr: &Expr) -> Lowered { + if let Expr::Compare(node) = expr { + if node.ops.len() > 1 { + return self.chained_condition(node); + } + if let ([op], [right]) = (node.ops.as_ref(), node.comparators.as_ref()) { + let left_value = self.expression(&node.left)?; + let right_value = self.expression(right)?; + return self.emit_compare_of( + *op, + left_value, + right_value, + Some(&node.left), + Some(right), + ); + } + } + let (value, ty) = self.expression(expr)?; + Ok(self.truthy(value, &ty)) + } + + /// whether a comparison of this pair answers with a `bool` whatever the operands are + fn compares_to_a_bool(op: AstCmpOp, lhs_ty: &RType, rhs_ty: &RType) -> bool { + matches!( + op, + AstCmpOp::In | AstCmpOp::NotIn | AstCmpOp::Is | AstCmpOp::IsNot + ) || matches!( + (lhs_ty, rhs_ty), + ( + RType::Primitive(Primitive::Int), + RType::Primitive(Primitive::Int) + ) | ( + RType::Primitive(Primitive::Float), + RType::Primitive(Primitive::Float) + ) | ( + RType::Primitive(Primitive::Str), + RType::Primitive(Primitive::Str) + ) + ) + } + + /// one comparison as a value — see [`Self::compare`] + fn compare_value_of( + &mut self, + op: AstCmpOp, + (lhs, lhs_ty): (Value, RType), + (rhs, rhs_ty): (Value, RType), + left: Option<&Expr>, + right: Option<&Expr>, + ) -> Lowered<(Value, RType)> { + if Self::compares_to_a_bool(op, &lhs_ty, &rhs_ty) { + let bit = self.emit_compare_of(op, (lhs, lhs_ty), (rhs, rhs_ty), left, right)?; + return Ok((bit, RType::BIT)); + } + // the class's own comparison, where the pair licenses reaching it directly and it + // is declared to answer a `bool`: its answer is that value as it stands + if let Some(call) = comparison_dunder(op).and_then(|name| { + self.operator_dunder(&lhs_ty, &rhs_ty, name).filter(|call| { + matches!(call.ret, RType::Primitive(Primitive::Bool | Primitive::Bit)) + }) + }) { + return self.direct_operator(call, (lhs, &lhs_ty), (rhs, &rhs_ty)); + } + let op = compare_op(op)?; + let lhs = self.widen_to_object(lhs, &lhs_ty); + let rhs = self.widen_to_object(rhs, &rhs_ty); + let dest = self.builder.temp(RType::OBJECT); + self.builder + .push(Op::ObjectRichCompare { dest, op, lhs, rhs }); + Ok((Value::Register(dest), RType::OBJECT)) } /// one comparison, choosing the representation from the operand pair @@ -14082,8 +14795,8 @@ impl Lowering<'_, '_> { } // the class's own comparison, where the pair licenses reaching it directly. a // body answering anything but a `bool` is left to the protocol: the value this - // produces is a bit, and collapsing something else to one is a decision - // `PyObject_RichCompareBool` is already making on its own terms + // produces is a bit, and collapsing something else to one is a decision the + // protocol path already makes for itself, with `PyObject_IsTrue` if let Some(call) = comparison_dunder(op).and_then(|name| { self.operator_dunder(&lhs_ty, &rhs_ty, name).filter(|call| { matches!(call.ret, RType::Primitive(Primitive::Bool | Primitive::Bit)) @@ -14218,7 +14931,14 @@ impl Lowering<'_, '_> { class: class.clone(), method: name.clone(), }); - let dest = self.builder.temp(ret.clone()); + // what the two arms join in: the body's own answer, unless the statement + // throws it away, when the shadow's answer is not asked to be one + let site = if self.leaves_unnarrowed(&Expr::Call(node.clone())) { + RType::OBJECT + } else { + ret.clone() + }; + let dest = self.builder.temp(site.clone()); let stored = self.builder.new_block(); let compiled = self.builder.new_block(); let join = self.builder.new_block(); @@ -14230,12 +14950,21 @@ impl Lowering<'_, '_> { self.builder.switch_to(compiled); self.licence_holds(&receiver, &class, &name, LicenceKind::Method); + let answered = if site == ret { + dest + } else { + self.builder.temp(ret.clone()) + }; self.builder.push(Op::CallNative { - dest: Some(dest), + dest: Some(answered), owner: Some(owner), callee: name.clone(), args: direct, }); + if answered != dest { + let widened = self.widen_to_object(Value::Register(answered), &ret); + self.builder.assign(dest, widened); + } self.builder.terminate(Terminator::Goto(join)); self.builder.switch_to(stored); @@ -14251,12 +14980,12 @@ impl Lowering<'_, '_> { name, args: boxed, }); - let narrowed = self.coerce(Value::Register(answer), &RType::OBJECT, &ret)?; + let narrowed = self.coerce(Value::Register(answer), &RType::OBJECT, &site)?; self.builder.assign(dest, narrowed); self.builder.terminate(Terminator::Goto(join)); self.builder.switch_to(join); - return Ok((Value::Register(dest), ret)); + return Ok((Value::Register(dest), site)); } // an override reached through a base-typed name — `shapes[i].area()`, where the @@ -14558,8 +15287,11 @@ impl Lowering<'_, '_> { /// the representation a call site hands on, whatever the callee turns out to be /// /// the checker's type for the call, where an object can be narrowed to it, and - /// `object` otherwise + /// `object` otherwise — or where nothing is asked of the value, see [`Self::unnarrowed`] fn call_result_type(&mut self, call: &Expr) -> Lowered { + if self.leaves_unnarrowed(call) { + return Ok(RType::OBJECT); + } let declared = self.peek_type(call)?; Ok(if self.narrowable_here(&declared) { declared @@ -14568,6 +15300,20 @@ impl Lowering<'_, '_> { }) } + /// whether `expr` is a value nothing is asked of — see [`Self::unnarrowed`] + fn leaves_unnarrowed(&self, expr: &Expr) -> bool { + self.unnarrowed == Some(expr.range()) + } + + /// `expr` as an object, with nothing asked of what it answers + fn unnarrowed_object(&mut self, expr: &Expr) -> Lowered { + let enclosing = self.unnarrowed.replace(expr.range()); + let lowered = self.expression(expr); + self.unnarrowed = enclosing; + let (value, ty) = lowered?; + Ok(self.widen_to_object(value, &ty)) + } + /// narrow a boxed call result to the representation the checker says it has /// /// the call proves nothing, so this is a *checked* unbox — the `arguments` @@ -14625,7 +15371,7 @@ impl Lowering<'_, '_> { let callee = self.callable(&node.func)?; let (args, args_ty) = self.display(&node.arguments.args, Display::Tuple)?; let args = self.widen_to_object(args, &args_ty); - let kwargs = self.keyword_dict(&node.arguments.keywords)?; + let kwargs = self.keyword_dict(&node.arguments.keywords, &callee)?; let dest = self.builder.temp(RType::OBJECT); self.builder.push(Op::CallUnpacked { dest, @@ -14656,7 +15402,14 @@ impl Lowering<'_, '_> { } /// a call's keywords as a dict, with `**` merging another mapping in - fn keyword_dict(&mut self, keywords: &[ast::Keyword]) -> Lowered> { + /// + /// every merge is the call's own rather than a display's, because a call refuses a + /// keyword it is given twice — whether the first came from a name or an earlier `**` + fn keyword_dict( + &mut self, + keywords: &[ast::Keyword], + callee: &Value, + ) -> Lowered> { if keywords.is_empty() { return Ok(None); } @@ -14681,12 +15434,12 @@ impl Lowering<'_, '_> { pairs: std::mem::take(&mut pairs), }); if started { - self.extend(accumulator, Value::Register(target), Display::Dict); + self.merge_keywords(accumulator, Value::Register(target), callee); } started = true; let (value, ty) = self.expression(&keyword.value)?; let source = self.widen_to_object(value, &ty); - self.extend(accumulator, source, Display::Dict); + self.merge_keywords(accumulator, source, callee); } } } @@ -14694,7 +15447,7 @@ impl Lowering<'_, '_> { if !pairs.is_empty() { let tail = self.builder.temp(RType::OBJECT); self.builder.push(Op::BuildDict { dest: tail, pairs }); - self.extend(accumulator, Value::Register(tail), Display::Dict); + self.merge_keywords(accumulator, Value::Register(tail), callee); } } else { self.builder.push(Op::BuildDict { @@ -14705,6 +15458,16 @@ impl Lowering<'_, '_> { Ok(Some(Value::Register(accumulator))) } + fn merge_keywords(&mut self, container: RegisterId, source: Value, callee: &Value) { + let status = self.builder.temp(RType::BIT); + self.builder.push(Op::MergeKeywords { + dest: status, + container: Value::Register(container), + source, + callee: callee.clone(), + }); + } + /// a comprehension, desugared to an empty container plus a loop that fills it /// /// the accumulator's method is called through the object protocol, which is @@ -14840,8 +15603,7 @@ impl Lowering<'_, '_> { // each `if` guard skips straight back to this loop's header for condition in &generator.ifs { - let (cond, cond_ty) = self.expression(condition)?; - let cond = self.truthy(cond, &cond_ty); + let cond = self.condition(condition)?; let kept = self.builder.new_block(); self.builder.terminate(Terminator::Branch { cond, @@ -14925,11 +15687,11 @@ impl Lowering<'_, '_> { // driving the iteration protocol here cost a `range` object, an iterator, a // `next` and an unbox *per element* — for a loop whose bounds are right there if let Expr::Name(name) = target - && let Some(count) = self.counting_range(&generator.iter)? + && let Some(range) = self.counting_range(&generator.iter)? { return self.comprehension_counted( name, - count, + range, generator, rest, kind, @@ -15003,8 +15765,7 @@ impl Lowering<'_, '_> { // each `if` guard skips straight back to this loop's header for condition in &generator.ifs { - let (cond, cond_ty) = self.expression(condition)?; - let cond = self.truthy(cond, &cond_ty); + let cond = self.condition(condition)?; let kept = self.builder.new_block(); self.builder.terminate(Terminator::Branch { cond, @@ -15021,9 +15782,9 @@ impl Lowering<'_, '_> { Ok(()) } - /// the `(start, stop)` of a `range` call this frame can count over, when the - /// expression is one — a computed step is left to the protocol - fn counting_range(&mut self, iter: &Expr) -> Lowered> { + /// a `range` call this frame can count over, when the expression is one — a + /// computed step is left to the protocol + fn counting_range(&mut self, iter: &Expr) -> Lowered> { let Expr::Call(call) = iter else { return Ok(None); }; @@ -15042,21 +15803,23 @@ impl Lowering<'_, '_> { [start, stop] => (Some(start), stop), _ => return Ok(None), }; - let stop = { - let (value, ty) = self.expression(stop)?; - self.coerce(value, &ty, &RType::INT)? - }; + // python reads the name before the bounds, and calls whatever it read + let (callee, genuine) = self.resolve_builtin("range", Wanted::WhereRebound); let start = match start { - None => Value::Int(0), - Some(start) => { - let (value, ty) = self.expression(start)?; - self.coerce(value, &ty, &RType::INT)? - } + None => None, + Some(start) => Some(self.expression(start)?), }; - Ok(Some((start, stop))) + let stop = self.expression(stop)?; + Ok(Some(CountedRange { + callee, + genuine, + start, + stop, + })) } - /// a comprehension clause over `range`, as a counting loop + /// a comprehension clause over `range`, as a counting loop while `range` is the + /// builtin and the iteration protocol over what the call answered otherwise #[expect( clippy::too_many_arguments, reason = "one clause of a comprehension needs all of its context" @@ -15064,27 +15827,77 @@ impl Lowering<'_, '_> { fn comprehension_counted( &mut self, name: &ast::ExprName, - (start, stop): (Value, Value), + range: CountedRange, generator: &ast::Comprehension, rest: &[ast::Comprehension], kind: &Comprehension<'_>, accumulator: RegisterId, accumulator_ty: &RType, ) -> Lowered<()> { + let CountedRange { + callee, + genuine, + start, + stop, + } = range; let counter = self.binding_register(name.id.as_str(), &RType::INT)?; - self.store(counter, start, &RType::INT)?; let bound = self .builder .local(format!("$stop{}", self.contexts), RType::INT); self.contexts += 1; - self.store(bound, stop, &RType::INT)?; + let iterator = self.builder.temp(RType::OBJECT); + let counting = self.builder.new_block(); + let protocol = self.builder.new_block(); let header = self.builder.new_block(); + let next = self.builder.new_block(); + let bind = self.builder.new_block(); let body = self.builder.new_block(); let step = self.builder.new_block(); + let count = self.builder.new_block(); let exit = self.builder.new_block(); + self.builder.terminate(Terminator::Branch { + cond: Value::Register(genuine), + then_block: counting, + else_block: protocol, + }); + + self.builder.switch_to(counting); + let first = match &start { + None => Value::Int(0), + Some((value, ty)) => self.coerce(value.clone(), ty, &RType::INT)?, + }; + let last = self.coerce(stop.0.clone(), &stop.1, &RType::INT)?; + self.store(counter, first, &RType::INT)?; + self.store(bound, last, &RType::INT)?; + // each way writes what only the other reads as well, because nothing that checks + // the function can see the two never cross + self.builder.push(Op::Box { + dest: iterator, + src: Value::None, + }); self.builder.terminate(Terminator::Goto(header)); + self.builder.switch_to(protocol); + self.store(bound, Value::Int(0), &RType::INT)?; + let mut args = Vec::with_capacity(2); + if let Some((value, ty)) = start { + args.push(self.widen_to_object(value, &ty)); + } + args.push(self.widen_to_object(stop.0, &stop.1)); + let iterable = self.builder.temp(RType::OBJECT); + self.builder.push(Op::CallValue { + dest: iterable, + callee, + args, + }); + self.builder.push(Op::GetIter { + dest: iterator, + src: Value::Register(iterable), + cursor: None, + }); + self.builder.terminate(Terminator::Goto(next)); + self.builder.switch_to(header); let more = self.builder.temp(RType::BIT); self.builder.push(Op::IntCompare { @@ -15099,11 +15912,37 @@ impl Lowering<'_, '_> { else_block: exit, }); + self.builder.switch_to(next); + let raw = self.builder.temp(RType::OBJECT); + self.builder.push(Op::IterNext { + dest: raw, + iter: Value::Register(iterator), + cursor: None, + }); + let exhausted = self.builder.temp(RType::BIT); + self.builder.push(Op::IsNull { + dest: exhausted, + src: Value::Register(raw), + }); + self.builder.terminate(Terminator::Branch { + cond: Value::Register(exhausted), + then_block: exit, + else_block: bind, + }); + + self.builder.switch_to(bind); + let item = self.builder.temp(RType::INT); + self.builder.push(Op::Unbox { + dest: item, + src: Value::Register(raw), + to: RType::INT, + }); + self.store(counter, Value::Register(item), &RType::INT)?; + self.builder.terminate(Terminator::Goto(body)); + self.builder.switch_to(body); - let mut inner = body; for condition in &generator.ifs { - let (cond, cond_ty) = self.expression(condition)?; - let cond = self.truthy(cond, &cond_ty); + let cond = self.condition(condition)?; let kept = self.builder.new_block(); self.builder.terminate(Terminator::Branch { cond, @@ -15111,13 +15950,18 @@ impl Lowering<'_, '_> { else_block: step, }); self.builder.switch_to(kept); - inner = kept; } - let _ = inner; self.comprehension_loop(rest, kind, accumulator, accumulator_ty)?; self.builder.terminate(Terminator::Goto(step)); self.builder.switch_to(step); + self.builder.terminate(Terminator::Branch { + cond: Value::Register(genuine), + then_block: count, + else_block: next, + }); + + self.builder.switch_to(count); let stepped = self.builder.temp(RType::INT); self.builder.push(Op::IntBinary { dest: stepped, @@ -15357,6 +16201,7 @@ impl Lowering<'_, '_> { lhs: previous, rhs: part, consumes_lhs: false, + concatenation: Concatenation::Join, }); Value::Register(dest) } @@ -15428,6 +16273,7 @@ impl Lowering<'_, '_> { lhs: prefix, rhs: piece, consumes_lhs: false, + concatenation: Concatenation::Join, }); Value::Register(dest) } @@ -15465,6 +16311,7 @@ impl Lowering<'_, '_> { lhs: Value::Str(debug.as_str().to_string()), rhs: Value::Register(dest), consumes_lhs: false, + concatenation: Concatenation::Join, }); Ok(Value::Register(joined)) } @@ -15495,18 +16342,10 @@ impl Lowering<'_, '_> { // one load at a compile-time offset, no hash lookup and no descriptor let name = self.attribute_name(&node.attr); if let RType::Instance { class, .. } = &receiver_ty - && let Some(fields) = self.layouts.get(class) - && let Some(held) = fields.iter().find(|field| field.name == name) + && let Some((owner, held)) = self.field_home(class, &name) { - let field_ty = held.ty.clone(); - let dest = self.builder.temp(field_ty.clone()); - self.builder.push(Op::GetField { - dest, - receiver, - class: class.clone(), - field: name, - }); - return Ok((Value::Register(dest), field_ty)); + let class = class.clone(); + return Ok(self.field_read(receiver, &receiver_ty, &class, owner, &held)); } // a `@property` this module lowered a getter for is called outright. the @@ -15595,6 +16434,20 @@ impl Lowering<'_, '_> { "a `super()` with no arguments needs a receiver this frame still holds", )); }; + // python fills the two arguments in only for the builtin `super`, and calls + // anything else the name resolves to with none + let (callee, genuine) = self.resolve_builtin("super", Wanted::Always); + let dest = self.builder.temp(RType::OBJECT); + let filled = self.builder.new_block(); + let bare = self.builder.new_block(); + let join = self.builder.new_block(); + self.builder.terminate(Terminator::Branch { + cond: Value::Register(genuine), + then_block: filled, + else_block: bare, + }); + + self.builder.switch_to(filled); let (receiver, receiver_ty) = self.read_place(&place)?; let receiver = self.widen_to_object(receiver, &receiver_ty); let class = self.builder.temp(RType::OBJECT); @@ -15602,17 +16455,31 @@ impl Lowering<'_, '_> { dest: class, class: owner, }); - let dest = self.builder.temp(RType::OBJECT); - self.builder.push(Op::CallPython { - dest, - callee: "super".to_string(), + let answer = self.builder.temp(RType::OBJECT); + self.builder.push(Op::CallValue { + dest: answer, + callee: callee.clone(), args: vec![Value::Register(class), receiver], }); + self.builder.assign(dest, Value::Register(answer)); + self.builder.terminate(Terminator::Goto(join)); + + self.builder.switch_to(bare); + let answer = self.builder.temp(RType::OBJECT); + self.builder.push(Op::CallValue { + dest: answer, + callee, + args: Vec::new(), + }); + self.builder.assign(dest, Value::Register(answer)); + self.builder.terminate(Terminator::Goto(join)); + + self.builder.switch_to(join); Ok((Value::Register(dest), RType::OBJECT)) } /// `a < b < c` — each operand evaluated once, and short-circuiting - fn chained_compare(&mut self, node: &ast::ExprCompare) -> Lowered<(Value, RType)> { + fn chained_condition(&mut self, node: &ast::ExprCompare) -> Lowered { let result = self.builder.temp(RType::BIT); let join = self.builder.new_block(); @@ -15636,7 +16503,40 @@ impl Lowering<'_, '_> { } self.builder.terminate(Terminator::Goto(join)); self.builder.switch_to(join); - Ok((Value::Register(result), RType::BIT)) + Ok(Value::Register(result)) + } + + /// `a < b < c` as a value: `a < b and b < c`, with `b` evaluated once — so the answer + /// is the first link's answer that is false, or the last link's + fn chained_compare(&mut self, node: &ast::ExprCompare) -> Lowered<(Value, RType)> { + let result_ty = self.representation_of(&Expr::Compare(node.clone()))?; + let result = self.builder.temp(result_ty.clone()); + let join = self.builder.new_block(); + + let mut left = self.expression(&node.left)?; + for (index, (op, right)) in node.ops.iter().zip(node.comparators.iter()).enumerate() { + let right = self.expression(right)?; + let (answer, answer_ty) = + self.compare_value_of(*op, left.clone(), right.clone(), None, None)?; + let stored = self.coerce(answer.clone(), &answer_ty, &result_ty)?; + self.builder.assign(result, stored); + + // every link but the last short-circuits on a false result + if index + 1 < node.ops.len() { + let truth = self.truthy(answer, &answer_ty); + let next = self.builder.new_block(); + self.builder.terminate(Terminator::Branch { + cond: truth, + then_block: next, + else_block: join, + }); + self.builder.switch_to(next); + } + left = right; + } + self.builder.terminate(Terminator::Goto(join)); + self.builder.switch_to(join); + Ok((Value::Register(result), result_ty)) } /// the builtins whose answer is about the frame that called them @@ -15681,8 +16581,31 @@ impl Lowering<'_, '_> { // arguments is a `TypeError` python has to raise rather than a namespace // question — and the interpreted definition raises it with python's wording "globals" if node.arguments.is_empty() => { + let (callee, genuine) = self.resolve_builtin("globals", Wanted::WhereRebound); let dest = self.builder.temp(RType::OBJECT); - self.builder.push(Op::ModuleDict { dest }); + let native = self.builder.new_block(); + let called = self.builder.new_block(); + let join = self.builder.new_block(); + self.builder.terminate(Terminator::Branch { + cond: Value::Register(genuine), + then_block: native, + else_block: called, + }); + self.builder.switch_to(native); + let namespace = self.builder.temp(RType::OBJECT); + self.builder.push(Op::ModuleDict { dest: namespace }); + self.builder.assign(dest, Value::Register(namespace)); + self.builder.terminate(Terminator::Goto(join)); + self.builder.switch_to(called); + let answer = self.builder.temp(RType::OBJECT); + self.builder.push(Op::CallValue { + dest: answer, + callee, + args: Vec::new(), + }); + self.builder.assign(dest, Value::Register(answer)); + self.builder.terminate(Terminator::Goto(join)); + self.builder.switch_to(join); Ok(Some((Value::Register(dest), RType::OBJECT))) } // `vars(x)` is `x.__dict__` and `dir(x)` is that object's names; only the @@ -16107,21 +17030,7 @@ impl Lowering<'_, '_> { && !self.binds("len") && node.arguments.keywords.is_empty() { - let [argument] = node.arguments.args.as_ref() else { - return Err(Decline::new("`len` takes exactly one argument")); - }; - let (value, ty) = self.expression(argument)?; - // an array knows its own length, and it is a field read rather than a - // call into the object protocol - if matches!(ty, RType::Array(_)) { - let dest = self.builder.temp(RType::INT); - self.builder.push(Op::ArrayLen { dest, array: value }); - return Ok((Value::Register(dest), RType::INT)); - } - let boxed = self.widen_to_object(value, &ty); - let dest = self.builder.temp(RType::INT); - self.builder.push(Op::Len { dest, src: boxed }); - return Ok((Value::Register(dest), RType::INT)); + return self.len_call(node); } // a closure this frame made itself is called at its native entry point: the @@ -16223,6 +17132,149 @@ impl Lowering<'_, '_> { self.native_call(node, name) } + /// whether `name` still resolves to the interpreter's own builtin, and what it + /// resolves to + /// + /// python reads the name before it evaluates anything the call is handed, so this + /// comes first. `wanted` says where the object itself is read: a use in a loop that + /// only calls it where it is not the builtin reads it only there, so the native + /// path pays for the test and nothing else, and holds `None` in its place + fn resolve_builtin(&mut self, name: &str, wanted: Wanted) -> (Value, RegisterId) { + let callee = self.builder.temp(RType::OBJECT); + if wanted == Wanted::Always { + self.builder.push(Op::LoadGlobal { + dest: callee, + name: name.to_string(), + }); + } + let genuine = self.builder.temp(RType::BIT); + self.builder.push(Op::BuiltinStands { + dest: genuine, + name: name.to_string(), + }); + if wanted == Wanted::Always { + return (Value::Register(callee), genuine); + } + let builtin = self.builder.new_block(); + let rebound = self.builder.new_block(); + let resolved = self.builder.new_block(); + self.builder.terminate(Terminator::Branch { + cond: Value::Register(genuine), + then_block: builtin, + else_block: rebound, + }); + self.builder.switch_to(builtin); + self.builder.push(Op::Box { + dest: callee, + src: Value::None, + }); + self.builder.terminate(Terminator::Goto(resolved)); + self.builder.switch_to(rebound); + self.builder.push(Op::LoadGlobal { + dest: callee, + name: name.to_string(), + }); + self.builder.terminate(Terminator::Goto(resolved)); + self.builder.switch_to(resolved); + (Value::Register(callee), genuine) + } + + /// `len(x)` on a name that resolves to a builtin statically: read directly while + /// it still is the builtin, and called otherwise + fn len_call(&mut self, node: &ast::ExprCall) -> Lowered<(Value, RType)> { + let [argument] = node.arguments.args.as_ref() else { + return Err(Decline::new("`len` takes exactly one argument")); + }; + // python reads `len` before the argument, and calls what it read even if the + // argument rebinds the name. reading a name this frame binds runs nothing, so + // there the object can be read where it is called, and the builtin's path is + // left with the test alone — which is what a loop guard pays every trip + let inert = matches!(argument, Expr::Name(name) if self.binds(name.id.as_str())); + let (early, genuine) = if inert { + let genuine = self.builder.temp(RType::BIT); + self.builder.push(Op::BuiltinStands { + dest: genuine, + name: "len".to_string(), + }); + (None, genuine) + } else { + let (callee, genuine) = self.resolve_builtin("len", Wanted::WhereRebound); + (Some(callee), genuine) + }; + let (value, ty) = self.expression(argument)?; + let call = Expr::Call(node.clone()); + let site = self.call_result_type(&call)?; + let dest = self.builder.temp(site.clone()); + let native = self.builder.new_block(); + let called = self.builder.new_block(); + let join = self.builder.new_block(); + self.builder.terminate(Terminator::Branch { + cond: Value::Register(genuine), + then_block: native, + else_block: called, + }); + + self.builder.switch_to(native); + let length = self.builder.temp(RType::INT); + // an array knows its own length, and it is a field read rather than a call + // into the object protocol + if matches!(ty, RType::Array(_)) { + self.builder.push(Op::ArrayLen { + dest: length, + array: value.clone(), + }); + } else { + let boxed = self.widen_to_object(value.clone(), &ty); + self.builder.push(Op::Len { + dest: length, + src: boxed, + }); + } + let answer = self.coerce(Value::Register(length), &RType::INT, &site)?; + self.builder.assign(dest, answer); + self.builder.terminate(Terminator::Goto(join)); + + self.builder.switch_to(called); + // a list held as a buffer has no list object to hand over, and building one + // would be a copy, which a `len` of its own could tell apart — so this refuses, + // loudly, rather than calling it with a different list + if matches!(ty, RType::Array(_)) { + self.builder.push(Op::RaiseStandard { + error: StandardError::RuntimeError, + message: "`len` no longer names the builtin, and this compiled function holds \ + the list as an unboxed buffer it cannot hand to anything else" + .to_string(), + }); + self.builder.terminate(Terminator::Unreachable); + self.builder.switch_to(join); + return Ok((Value::Register(dest), site)); + } + let callee = match early { + Some(callee) => callee, + None => { + let callee = self.builder.temp(RType::OBJECT); + self.builder.push(Op::LoadGlobal { + dest: callee, + name: "len".to_string(), + }); + Value::Register(callee) + } + }; + let boxed = self.widen_to_object(value, &ty); + let result = self.builder.temp(RType::OBJECT); + self.builder.push(Op::CallValue { + dest: result, + callee, + args: vec![boxed], + }); + let (answer, _) = self.narrow_call_result(result, &call)?; + self.builder.assign(dest, answer); + self.builder.terminate(Terminator::Goto(join)); + + self.builder.switch_to(join); + Ok((Value::Register(dest), site)) + } + /// a call that reaches a definition in this same unit at its native entry point /// /// the arguments are bound here rather than at runtime: the callee's signature says diff --git a/crates/by_irbuild/src/tests.rs b/crates/by_irbuild/src/tests.rs index d17e4e5315..e6ed64e7ae 100644 --- a/crates/by_irbuild/src/tests.rs +++ b/crates/by_irbuild/src/tests.rs @@ -194,6 +194,143 @@ fn a_function_returning_nothing_returns_none() { assert!(ir.contains("return None"), "{ir}"); } +#[test] +fn a_call_whose_value_is_discarded_is_not_narrowed() { + // `cb()` alone on a line throws its value away, so the declared `int` is not asked + // of it — while the same call used as a value still is + let ir = ir("\ +from collections.abc import Callable + +def discard(cb: Callable[[], int]) -> None: + cb() + +def keep(cb: Callable[[], int]) -> int: + return cb() +"); + let discard = &ir[ir.find("def discard").unwrap_or(0)..ir.find("def keep").unwrap_or(ir.len())]; + let keep = &ir[ir.find("def keep").unwrap_or(0)..]; + assert!(discard.contains("callobj cb()"), "{ir}"); + assert!(!discard.contains("unbox"), "{ir}"); + assert!(keep.contains("unbox"), "{ir}"); +} + +#[test] +fn a_raised_call_is_not_narrowed() { + // `raise` asks only that the value be an exception, which raising it checks, so what + // the call answered is raised as it is + let ir = ir("\ +class Refused(Exception): + pass + +def fail(text: str) -> int: + raise Refused(text) +"); + assert!(ir.contains("raise "), "{ir}"); + assert!(!ir.contains("unbox"), "{ir}"); +} + +#[test] +fn a_builtin_lowered_natively_is_tested_for_being_the_builtin() { + // `len` and `range` are looked up and compared on every use, and the native path + // is only one arm of the answer + let ir = ir("\ +def length(xs: list[int]) -> int: + return len(xs) + +def count(n: int) -> int: + total = 0 + for i in range(n): + total = total + i + return total +"); + assert!(ir.contains("= global len"), "{ir}"); + assert!( + ir.contains("builtin-stands") && ir.contains(" len\n"), + "{ir}" + ); + assert!(ir.contains("= global range"), "{ir}"); + assert!(ir.contains(" range\n"), "{ir}"); +} + +#[test] +fn a_field_read_tests_the_field_is_there_and_a_pattern_does_not_match_without_it() { + // an instance `__new__` made without `__init__` has no fields yet: a read raises, and + // a class pattern capturing one does not match + let ir = ir("\ +class P: + def __init__(self, n: int) -> None: + self.n = n + +def get(p: P) -> int: + return p.n + +def bump(p: P) -> None: + p.n += 1 + +def pick(value: object) -> int: + match value: + case P(n=k): + return k + case _: + return 0 +"); + let get = &ir[ir.find("def get").unwrap_or(0)..ir.find("def bump").unwrap_or(ir.len())]; + assert!(get.contains("require p."), "{ir}"); + let bump = &ir[ir.find("def bump").unwrap_or(0)..ir.find("def pick").unwrap_or(ir.len())]; + assert!(bump.contains("require p."), "{ir}"); + let pick = &ir[ir.find("def pick").unwrap_or(0)..]; + assert!(!pick.contains("require"), "{ir}"); +} + +#[test] +fn a_comparison_of_objects_answers_an_object_as_a_value_and_a_bit_as_a_condition() { + with_source( + "\ +def value(a: object, b: object) -> object: + return a > b + +def condition(a: object, b: object) -> int: + if a > b: + return 1 + return 0 + +def ints(a: int, b: int) -> object: + return a > b +", + |db, env, model, suite| { + let module = crate::build_module(db, env, model, suite, "app", crate::Language::Python); + assert!(module.declined.is_empty(), "{:?}", module.declined); + let named = |name: &str| { + module + .functions + .iter() + .find(|function| function.name == name) + .map(print_function) + .unwrap_or_default() + }; + let rich = |name: &str| { + module.functions.iter().any(|function| { + function.name == name + && has_op(function, |op| matches!(op, Op::ObjectRichCompare { .. })) + }) + }; + let bit = |name: &str| { + module.functions.iter().any(|function| { + function.name == name + && has_op(function, |op| matches!(op, Op::ObjectCompare { .. })) + }) + }; + assert!(rich("value") && !bit("value"), "{}", named("value")); + assert!( + bit("condition") && !rich("condition"), + "{}", + named("condition") + ); + assert!(!rich("ints") && !bit("ints"), "{}", named("ints")); + }, + ); +} + #[test] fn a_bool_condition_converts_to_a_bit() { let ir = ir("\ @@ -496,10 +633,11 @@ def before(a: str, b: object) -> bool: print_function(same) ); // only *both* operands being `str` settles whose comparison runs, so a - // gradual right-hand side stays on the protocol + // gradual right-hand side stays on the protocol — and answers with whatever + // the protocol answered let before = function("before"); assert!( - has_op(before, |op| matches!(op, Op::ObjectCompare { .. })), + has_op(before, |op| matches!(op, Op::ObjectRichCompare { .. })), "{}", print_function(before) ); @@ -835,6 +973,26 @@ def f(rows: list[list[int]]) -> object: ); } +#[test] +fn a_comprehension_clause_reading_a_variable_before_it_is_bound_declines() { + // `y` in the first clause's own iterable is the enclosing frame's, and in the second + // clause's iterable it is the comprehension's, which nothing has bound yet: python + // raises `UnboundLocalError` there + assert_eq!( + declines( + "\ +def outer(xs: list[list[int]], y: list[int]) -> object: + ok = [x for x in y] + return [x for x in xs for y in y] +" + ), + [( + "outer".to_string(), + "a comprehension clause reads `y` before the clause that binds it".to_string() + )] + ); +} + #[test] fn a_target_list_unpacks_into_a_fixed_tuple() { // one op with one destination, read back element by element — a second @@ -1059,6 +1217,62 @@ def f(n: int) -> int: ); } +#[test] +fn a_with_block_reaching_its_manager_directly_holds_it_as_its_own_class() { + // both halves of the protocol are this module's own methods, called directly. the + // manager is held in the class's register, so neither call narrows it back from an + // `object` it never needed to become + with_source( + "\ +class Guard: + depth: int + + def __init__(self): + self.depth = 0 + + def __enter__(self) -> \"Guard\": + self.depth = self.depth + 1 + return self + + def __exit__(self, kind: object, value: object, trace: object): + self.depth = self.depth - 1 + + +def run(guard: Guard) -> int: + with guard: + pass + return guard.depth +", + |db, env, model, suite| { + let module = + crate::build_module(db, env, model, suite, "app", crate::Language::BasedPython); + let run = module + .all_functions() + .find(|function| function.name == "run") + .expect("run is compiled"); + assert!( + !has_op(run, |op| matches!( + op, + Op::Unbox { + to: RType::Instance { .. }, + .. + } + )), + "{}", + print_function(run) + ); + assert!( + has_op( + run, + |op| matches!(op, Op::CallNative { callee, .. } if callee == "__exit__") + ), + "{}", + print_function(run) + ); + }, + ); +} + #[test] fn a_bare_raise_outside_a_handler_is_declined() { // it would need the *interpreter's* handled exception, which we never set @@ -2235,6 +2449,64 @@ class Derived(Base): ); } +/// a class that adds no field of its own still reaches the ones its base declares +/// +/// such a class is laid out with an *empty* layout — its instances are its base's, at the +/// base's offsets, with no region past them for it to own. so an attribute of its own +/// reaches nothing, and the field op has to name the base that keeps it. looking only in +/// the receiver's own layout sent every read and write of an inherited attribute out +/// through the dynamic form instead, which on an emitted instance is what stops the whole +/// module from ever narrowing a refusal +#[test] +fn a_class_that_adds_no_field_reaches_its_base_s_storage_directly() { + with_source( + "\ +class Held(list): + def __init__(self, *args): + super().__init__(*args) + self.notes = [] + + +class Restated(Held): + def __init__(self, *args): + super().__init__(*args) + self.notes = ['restated'] + + def first_note(self): + return self.notes[0] +", + |db, env, model, suite| { + let module = crate::build_module(db, env, model, suite, "app", crate::Language::Python); + assert!(module.declined.is_empty(), "{:?}", module.declined); + let restated = module + .classes + .iter() + .find(|class| class.name == "Restated") + .expect("Restated is emitted"); + assert!(restated.fields.is_empty(), "{:?}", restated.fields); + let method = |name: &str| { + restated + .methods + .iter() + .find(|candidate| candidate.name == name) + .unwrap_or_else(|| panic!("Restated.{name} is emitted")) + }; + // a class something extends is written and read through the attribute too, + // on the arm where the field does not stand — see `Op::FieldStands` — so what + // is asserted is that the field op is there, and that the test in front of it + // names the receiver's own class + let written = method("__init__"); + let body = print_function(written); + assert!(body.contains("self. ="), "{body}"); + assert!(body.contains("field-stands self Restated.notes"), "{body}"); + let read = method("first_note"); + let body = print_function(read); + assert!(body.contains("= self."), "{body}"); + assert!(body.contains("field-stands self Restated.notes"), "{body}"); + }, + ); +} + /// each method names its own receiver, and the layout has to read the one it wrote /// /// the field pass took slot zero off `__init__` and then matched *that* name in every @@ -4810,7 +5082,7 @@ def f(a: int) -> int: fn concatenating_two_strings_stays_a_string() { let ir = ir("def f(a: str, b: str) -> str:\n return a + b\n"); assert!(ir.contains("-> str"), "{ir}"); - assert!(ir.contains(" ++ "), "{ir}"); + assert!(ir.contains(" + "), "{ir}"); } #[test] @@ -5456,7 +5728,9 @@ def f(n: object) -> None: for i in range(n): pass "); - assert_eq!(ir.matches("pycall range").count(), 1, "{ir}"); + assert_eq!(ir.matches("callobj ").count(), 1, "{ir}"); + assert_eq!(ir.matches("= global range").count(), 1, "{ir}"); + assert!(ir.contains("callobj r2(n)"), "{ir}"); assert!(ir.contains("= iter "), "{ir}"); } @@ -6430,6 +6704,32 @@ def make_adder(n: int) -> object: ); } +#[test] +fn a_cell_two_frames_store_different_representations_in_is_an_object() { + // the nested frame stores a `str` where the enclosing one only ever stored an `int`, + // so no one representation holds both and the cell is an `object` — asked of every + // frame that writes it, not only of the one that owns it + with_source( + "\ +def counter() -> (() -> object): + n = 0 + def get() -> object: + nonlocal n + n = 'x' + return n + return get +", + |db, env, model, suite| { + let module = + crate::build_module(db, env, model, suite, "app", crate::Language::BasedPython); + assert!(module.declined.is_empty(), "{:?}", module.declined); + let environment = &module.classes[0]; + assert_eq!(environment.fields[0].name, "n"); + assert_eq!(environment.fields[0].ty, RType::OBJECT); + }, + ); +} + #[test] fn a_capture_either_frame_writes_becomes_a_shared_cell() { // python closes over the *variable*: the write after the `def` is visible through @@ -6448,10 +6748,10 @@ def counter() -> (() -> int): crate::build_module(db, env, model, suite, "app", crate::Language::BasedPython); assert!(module.declined.is_empty(), "{:?}", module.declined); let environment = &module.classes[0]; - // a cell is always `object`: it starts unset, and NULL has to be - // distinguishable from every value it could hold + // both frames only ever store an `int`, so the cell holds a tagged one, whose + // error value is what unset means assert_eq!(environment.fields[0].name, "n"); - assert_eq!(environment.fields[0].ty, RType::OBJECT); + assert_eq!(environment.fields[0].ty, RType::INT); // the *enclosing* frame writes the field too let outer = print_function(&module.functions[0]); @@ -6951,10 +7251,7 @@ def run(times: int, k: int) -> int: let outer = &module.functions[0]; let text = print_function(outer); assert!(text.contains("call run$env.step("), "{text}"); - assert!( - !has_op(outer, |op| matches!(op, Op::CallValue { .. })), - "{text}" - ); + assert!(!text.contains("callobj step("), "{text}"); }, ); } @@ -8241,10 +8538,14 @@ data class B(A): "B", "label", ); - // `class B` is the pivot and `r1` the boxed receiver: the two-argument form - // python's own compiler would have built from the frame + // `class B` is the pivot and `r4` the boxed receiver: the two-argument form + // python's own compiler would have built from the frame, taken while `super` is + // the builtin — and a `super` rebound at runtime is called with nothing, as python + // calls it assert!(ir.contains("= class B"), "{ir}"); - assert!(ir.contains("pycall super(r2, r1)"), "{ir}"); + assert!(ir.contains("r2 = builtin-stands super"), "{ir}"); + assert!(ir.contains("callobj r1(r5, r4)"), "{ir}"); + assert!(ir.contains("callobj r1()"), "{ir}"); } #[test] diff --git a/crates/by_opt/src/borrow.rs b/crates/by_opt/src/borrow.rs index 51c5b56beb..d839d19f3c 100644 --- a/crates/by_opt/src/borrow.rs +++ b/crates/by_opt/src/borrow.rs @@ -84,26 +84,33 @@ //! ## why none of them asks for a temporary written once //! //! the obvious way to say "this copy is the register's whole life" is to ask that the -//! register be written exactly once over the function and carry no name from the -//! source program. both stand in for something rather than saying it, and both are -//! wrong often enough to matter: +//! register be written exactly once over the function. that stands in for something +//! rather than saying it, and is wrong often enough to matter: `unswitch` runs before +//! this pass and emits a *second copy of every loop body*, reusing the same registers. +//! so no register written inside a duplicated loop is ever written once, however +//! plainly its copy dominates its uses. this is what was keeping a table subscripted +//! three times a trip — the key widened into an operand of its own each time, in the +//! hottest block of the program — paying a retain and a release per subscript //! -//! - `unswitch` runs before this pass and emits a *second copy of every loop body*, -//! reusing the same registers. so no register written inside a duplicated loop is -//! ever written once, however plainly its copy dominates its uses. this is what was -//! keeping a table subscripted three times a trip — the key widened into an operand -//! of its own each time, in the hottest block of the program — paying a retain and -//! a release per subscript -//! - `pair = Pair(i, i + 1)` in a loop body writes a register the source program -//! named, and a name says nothing at all about whether the copy still owns -//! -//! so the property those two stood for is stated instead: every write of the register +//! so the property that stood for is stated instead: every write of the register //! lends from the same source, at most one per block, no block reads the register //! without having written it first, no terminator reads it, and nothing writes over //! the source between the write and the last read. the first three together are what //! make each write's window its own block, which is the window the last one is stated //! against. //! +//! ## a name owns what it holds +//! +//! a register the source program named never borrows. python lets go of what a name +//! holds when the name is rebound or deleted, and a borrowed register lets go of +//! nothing: the value would go when its lender is next written, which is somewhere +//! else entirely. in a loop body that is before the new value is built +//! +//! ```python +//! while i < n: +//! held = Cell(i) # the previous `Cell` goes once this one is built +//! ``` +//! //! ## a chain of lends is one borrow each, not one borrow between them //! //! `key = unbox e` and then `r = key` is two lending writes, and only the first has a @@ -234,51 +241,63 @@ fn write_counts(function: &Function) -> HashMap { writes } -/// every register that only ever holds a literal, which may borrow rather than own +/// every register that only ever holds a constant, which may borrow rather than own /// /// a string or bytes literal is a module static the emitter builds once at import and -/// never gives back, so unlike a copy there is no window to reason about at all: the -/// value cannot go away while the frame is running, whatever happens in between. +/// never gives back, and `None`, `True` and `False` are the interpreter's own +/// singletons, so unlike a copy there is no window to reason about at all: the value +/// cannot go away while the frame is running, whatever happens in between. +/// +/// the condition is on every write rather than on there being one. a register whose +/// every write is a constant never owns anything, however many writes there are — and +/// `unswitch` gives each loop it duplicates a second copy of its body, so a register +/// written once in the source is written twice by here. /// /// what this is worth is a loop. `if part.startswith("w")` reads the literal into a -/// register of its own every trip, and today that is a retain and a release per trip -/// on a value that was never going anywhere +/// register of its own every trip, and a `with` block hands `__exit__` three `None`s, +/// and each was a retain and a release per trip on a value that was never going +/// anywhere fn constants(function: &Function) -> Vec { let writes = write_counts(function); + let mut constant_writes: BTreeMap = BTreeMap::new(); + for op in function.blocks.iter().flat_map(|block| &block.ops) { + if let Op::Assign { + dest, + src: Value::Str(_) | Value::Bytes(_), + } + | Op::Box { + dest, + src: Value::None | Value::Bool(_) | Value::Bit(_), + } = op + { + *constant_writes.entry(*dest).or_default() += 1; + } + } let mut candidates = Vec::new(); - for block in &function.blocks { - for op in &block.ops { - let Op::Assign { - dest, - src: Value::Str(_) | Value::Bytes(_), - } = op - else { - continue; - }; - // written once, unnamed, and refcounted — as for a copy, this is what - // makes the literal the register's whole life - if writes.get(dest) != Some(&1) { - continue; - } - let Some(decl) = function.register(*dest) else { - continue; - }; - if decl.name.is_some() || !decl.ty.is_refcounted() { - continue; - } - // a use in a terminator would hand a reference out of the frame - if function.blocks.iter().any(|block| { - block - .terminator - .operands() - .iter() - .any(|operand| reads(operand, *dest)) - }) { - continue; - } - candidates.push(*dest); + for (dest, count) in constant_writes { + if writes.get(&dest) != Some(&count) { + continue; + } + // unnamed and refcounted: a name says the source may rebind it to anything, and + // a value with no reference has nothing to save + let Some(decl) = function.register(dest) else { + continue; + }; + if decl.name.is_some() || !decl.ty.is_refcounted() { + continue; + } + // a use in a terminator would hand a reference out of the frame + if function.blocks.iter().any(|block| { + block + .terminator + .operands() + .iter() + .any(|operand| reads(operand, dest)) + }) { + continue; } + candidates.push(dest); } candidates } @@ -287,34 +306,37 @@ fn constants(function: &Function) -> Vec { fn field_reads(function: &Function) -> Vec { // a parameter is already borrowed by the frame, and a named local outlives any // single statement — only a temporary is a candidate - let writes = write_counts(function); - - let mut candidates = Vec::new(); - for block in &function.blocks { - for (index, op) in block.ops.iter().enumerate() { - let Op::GetField { dest, .. } = op else { - continue; - }; - // written once, unnamed, and refcounted — otherwise there is either - // nothing to save or no single window to reason about - if writes.get(dest) != Some(&1) { - continue; - } - let Some(decl) = function.register(*dest) else { - continue; - }; - if decl.name.is_some() || !decl.ty.is_refcounted() { - continue; - } - if reads_outside(function, block, *dest) { - continue; - } - if borrow_is_safe(function, block, index, *dest) { - candidates.push(*dest); - } - } - } - candidates + (function.param_count..function.registers.len()) + .map(RegisterId) + .filter(|register| { + function + .register(*register) + .is_some_and(|decl| decl.name.is_none() && decl.ty.is_refcounted()) + }) + // every write a field read, and each write's window its own block: `unswitch` + // copies a loop body and its copy writes the same registers, so a register may be + // written once in each of several blocks, and what the argument below needs is + // only that no block reads what another one left + .filter(|register| { + lender(function, *register, |op| match op { + Op::GetField { + receiver: Value::Register(receiver), + .. + } => Some(*receiver), + _ => None, + }) + .is_some() + }) + .filter(|register| { + function.blocks.iter().all(|block| { + block + .ops + .iter() + .position(|op| op.dest() == Some(*register)) + .is_none_or(|at| borrow_is_safe(function, block, at, *register)) + }) + }) + .collect() } /// every register filled only by copies of one other register, paired with the @@ -380,11 +402,11 @@ fn lending_writes( // a parameter is already borrowed by the frame, so there is nothing to save on it for index in function.param_count..function.registers.len() { let register = RegisterId(index); - // a register with nothing to save is not worth an answer either way - if function - .register(register) - .is_none_or(|decl| !decl.ty.is_refcounted()) - { + // a register with nothing to save is not worth an answer either way, and a name + // owns an object it holds + if function.register(register).is_none_or(|decl| { + !decl.ty.is_refcounted() || (decl.name.is_some() && decl.ty.holds_object_reference()) + }) { continue; } if let Some(source) = lender(function, register, lends) { @@ -540,6 +562,10 @@ fn borrow_is_safe( // field value first, and that `__del__` could free the very object // being written through Op::GetField { receiver, .. } if reads(receiver, register) => {} + // and asking whether the receiver has a field reads it no further than + // the load + Op::RequireField { receiver, .. } | Op::FieldIsSet { receiver, .. } + if reads(receiver, register) => {} // tagged integer arithmetic lowers to one of the `By_Int*` helpers, // and each of those either stays on a fast path over two tagged // shorts — which hold no reference to keep alive — or hands both @@ -570,25 +596,6 @@ fn borrow_is_safe( used } -/// whether any block but this one reads `register` -/// -/// the safety argument only covers one block's worth of straight-line code, so a -/// use anywhere else takes the borrow off the table -fn reads_outside(function: &Function, home: &BasicBlock, register: RegisterId) -> bool { - function - .blocks - .iter() - .filter(|block| !std::ptr::eq(*block, home)) - .any(|block| { - block - .ops - .iter() - .flat_map(Op::operands) - .chain(block.terminator.operands()) - .any(|operand| reads(operand, register)) - }) -} - fn reads(operand: &Value, register: RegisterId) -> bool { matches!(operand, Value::Register(id) if *id == register) } @@ -614,8 +621,13 @@ fn is_inert(function: &Function, op: &Op) -> bool { .is_some_and(|decl| !decl.ty.is_refcounted() || decl.borrowed) }; match op { - // pure double arithmetic touches no reference at all - Op::FloatBinary { .. } | Op::FloatCompare { .. } => true, + // pure double arithmetic touches no reference at all, and neither does asking + // whether an instance has a field: a load and a comparison, and a raise at worst, + // which leaves before any later use of what is borrowed + Op::FloatBinary { .. } + | Op::FloatCompare { .. } + | Op::RequireField { .. } + | Op::FieldIsSet { .. } => true, Op::Assign { dest, .. } | Op::TupleGet { dest, .. } | Op::GetField { dest, .. } => { plain(dest) } @@ -630,7 +642,7 @@ fn is_inert(function: &Function, op: &Op) -> bool { mod tests { use super::*; use by_ir::builder::FunctionBuilder; - use by_ir::ops::{BinOp, Terminator}; + use by_ir::ops::{BinOp, Concatenation, Mutation, Terminator}; use by_ir::rtype::{IntWidth, RType}; use by_ir::verify::verify; @@ -675,6 +687,57 @@ mod tests { builder.finish() } + /// `sum = self.x + k`, twice over: once in a loop body and once in the copy of it + /// `unswitch` makes, which writes the same registers. `read_first` puts a read of the + /// field's register ahead of the second copy's write + fn read_in_two_copies(read_first: bool) -> (Function, RegisterId) { + let mut builder = FunctionBuilder::new("f", RType::INT); + let receiver = builder.param("p", nested()); + let k = builder.param("k", RType::INT); + let read = builder.temp(RType::INT); + let sum = builder.temp(RType::INT); + let field = Op::GetField { + dest: read, + receiver: Value::Register(receiver), + class: "Holder".to_string(), + field: "count".to_string(), + }; + let add = Op::IntBinary { + dest: sum, + op: BinOp::Add, + lhs: Value::Register(read), + rhs: Value::Register(k), + }; + let copy = builder.new_block(); + builder.push(field.clone()); + builder.push(add.clone()); + builder.terminate(Terminator::Goto(copy)); + builder.switch_to(copy); + if read_first { + builder.push(add.clone()); + } + builder.push(field); + builder.push(add); + builder.terminate(Terminator::Return(Value::Register(sum))); + (builder.finish(), read) + } + + #[test] + fn a_field_read_in_each_copy_of_an_unswitched_loop_borrows() { + // each write's window is its own block, and each is safe, which is all the + // borrow rests on — that the register is written twice is not + let (function, read) = read_in_two_copies(false); + assert_eq!(field_reads(&function), vec![read]); + } + + #[test] + fn a_field_read_its_block_reads_before_writing_does_not_borrow() { + // the second block reads the register before writing it, so what it reads there + // is what the first block left, and nothing holds that any more + let (function, _) = read_in_two_copies(true); + assert_eq!(field_reads(&function), Vec::new()); + } + #[test] fn an_intermediate_in_a_chained_read_borrows() { let mut m = module(chain()); @@ -865,6 +928,8 @@ mod tests { class: "Holder".to_string(), field: "label".to_string(), value: Value::Str("x".to_string()), + moves: false, + present: false, }); builder.terminate(Terminator::Return(Value::None)); let mut m = module(builder.finish()); @@ -943,6 +1008,8 @@ mod tests { class: "Nest".to_string(), field: "other".to_string(), value: Value::Register(inner), + moves: false, + present: false, }); builder.terminate(Terminator::Return(Value::None)); let mut m = module(builder.finish()); @@ -1166,10 +1233,10 @@ mod tests { } #[test] - fn a_named_destination_borrows_a_copy() { - // `v = build()` in a loop body writes a register the source program named, and - // the name says nothing about whether the copy still owns: the write is the - // only one, and every read of it follows that write in the same block + fn a_named_destination_owns_its_copy() { + // the write is the only one and every read of it follows that write in the same + // block, which would let a temporary borrow — but a name holds its value until + // it is rebound, which the lender knows nothing about let mut builder = FunctionBuilder::new("f", RType::INT); let line = builder.param("line", RType::STR); let held = builder.local("held", RType::OBJECT); @@ -1182,7 +1249,41 @@ mod tests { builder.terminate(Terminator::Return(Value::Register(length))); let mut m = module(builder.finish()); run(&mut m); - assert!(m.functions[0].registers[held.index()].borrowed); + assert!(!m.functions[0].registers[held.index()].borrowed); + assert_eq!(verify(&m.functions[0]), Ok(())); + } + + #[test] + fn a_named_int_destination_borrows_an_element() { + // an `int` has no finalizer, so when python lets go of what the name held is + // not something a program can see, and `part` in `head, part = split(s)` may + // hold its element on loan from the tuple + let mut builder = FunctionBuilder::new("f", RType::INT); + let text = builder.param("s", RType::STR); + let pair = builder.temp(RType::Tuple(Box::new([RType::STR, RType::INT]))); + let part = builder.local("part", RType::INT); + let sum = builder.temp(RType::INT); + builder.push(Op::CallNative { + owner: None, + dest: Some(pair), + callee: "split".to_string(), + args: vec![Value::Register(text)], + }); + builder.push(Op::TupleGet { + dest: part, + src: Value::Register(pair), + index: 1, + }); + builder.push(Op::IntBinary { + dest: sum, + op: BinOp::Add, + lhs: Value::Register(part), + rhs: Value::Int(1), + }); + builder.terminate(Terminator::Return(Value::Register(sum))); + let mut m = module(builder.finish()); + run(&mut m); + assert!(m.functions[0].registers[part.index()].borrowed); assert_eq!(verify(&m.functions[0]), Ok(())); } @@ -1380,6 +1481,71 @@ mod tests { assert!(m.functions[0].registers[widened.index()].borrowed); } + #[test] + fn a_boxed_singleton_borrows_it() { + // `None`, `True` and `False` are the interpreter's own objects, which outlive + // every frame, so a register holding one owns nothing a release could drop + let mut builder = FunctionBuilder::new("f", RType::OBJECT); + let none = builder.temp(RType::OBJECT); + let yes = builder.temp(RType::OBJECT); + let answer = builder.temp(RType::OBJECT); + builder.push(Op::Box { + dest: none, + src: Value::None, + }); + builder.push(Op::Box { + dest: yes, + src: Value::Bool(true), + }); + builder.push(Op::CallPython { + dest: answer, + callee: "pick".to_string(), + args: vec![Value::Register(none), Value::Register(yes)], + }); + builder.terminate(Terminator::Return(Value::Register(answer))); + let mut m = module(builder.finish()); + run(&mut m); + assert!(m.functions[0].registers[none.index()].borrowed); + assert!(m.functions[0].registers[yes.index()].borrowed); + } + + #[test] + fn a_register_only_ever_written_with_a_constant_borrows_however_often() { + // `unswitch` gives a loop a second copy of its body, so a register the source + // writes once is written twice by the time this runs — and each write still + // lends. two arms writing the same singleton are the same shape + let mut builder = FunctionBuilder::new("f", RType::OBJECT); + let flag = builder.param("c", RType::BIT); + let none = builder.temp(RType::OBJECT); + let answer = builder.temp(RType::OBJECT); + let left = builder.new_block(); + let right = builder.new_block(); + let join = builder.new_block(); + builder.terminate(Terminator::Branch { + cond: Value::Register(flag), + then_block: left, + else_block: right, + }); + for arm in [left, right] { + builder.switch_to(arm); + builder.push(Op::Box { + dest: none, + src: Value::None, + }); + builder.terminate(Terminator::Goto(join)); + } + builder.switch_to(join); + builder.push(Op::CallPython { + dest: answer, + callee: "pick".to_string(), + args: vec![Value::Register(none)], + }); + builder.terminate(Terminator::Return(Value::Register(answer))); + let mut m = module(builder.finish()); + run(&mut m); + assert!(m.functions[0].registers[none.index()].borrowed); + } + #[test] fn a_returned_literal_does_not_borrow() { // as for a copy, the frame would be handing out a reference it never took @@ -1450,12 +1616,11 @@ mod tests { #[test] fn an_element_read_off_a_tuple_borrows() { // the tuple register owns both elements, so reading one is a copy of a place - // it holds — and the destination is a name, which is what `head, tail = ...` - // always produces + // it holds let mut builder = FunctionBuilder::new("f", RType::INT); let text = builder.param("s", RType::STR); let pair = builder.temp(pair_type()); - let head = builder.local("head", RType::STR); + let head = builder.temp(RType::STR); let length = builder.temp(RType::INT); call_pair(&mut builder, pair, text); builder.push(Op::TupleGet { @@ -1485,7 +1650,7 @@ mod tests { let mut builder = FunctionBuilder::new("f", RType::INT); let text = builder.param("s", RType::STR); let pair = builder.temp(pair_type()); - let head = builder.local("head", RType::STR); + let head = builder.temp(RType::STR); let length = builder.temp(RType::INT); let second = builder.new_block(); for block in [None, Some(second)] { @@ -1520,7 +1685,7 @@ mod tests { let mut builder = FunctionBuilder::new("f", RType::INT); let text = builder.param("s", RType::STR); let pair = builder.temp(pair_type()); - let head = builder.local("head", RType::STR); + let head = builder.temp(RType::STR); let length = builder.temp(RType::INT); let next = builder.new_block(); call_pair(&mut builder, pair, text); @@ -1546,7 +1711,7 @@ mod tests { let mut builder = FunctionBuilder::new("f", RType::INT); let text = builder.param("s", RType::STR); let pair = builder.temp(pair_type()); - let head = builder.local("head", RType::STR); + let head = builder.temp(RType::STR); let first = builder.temp(RType::INT); let second = builder.temp(RType::INT); builder.push(Op::Len { @@ -1576,7 +1741,7 @@ mod tests { let mut builder = FunctionBuilder::new("f", RType::INT); let text = builder.param("s", RType::STR); let pair = builder.temp(pair_type()); - let head = builder.local("head", RType::STR); + let head = builder.temp(RType::STR); let length = builder.temp(RType::INT); call_pair(&mut builder, pair, text); builder.push(Op::TupleGet { @@ -1601,7 +1766,7 @@ mod tests { let mut builder = FunctionBuilder::new("f", RType::STR); let text = builder.param("s", RType::STR); let pair = builder.temp(pair_type()); - let head = builder.local("head", RType::STR); + let head = builder.temp(RType::STR); call_pair(&mut builder, pair, text); builder.push(Op::TupleGet { dest: head, @@ -1621,7 +1786,7 @@ mod tests { let mut builder = FunctionBuilder::new("f", RType::INT); let text = builder.param("s", RType::STR); let pair = builder.temp(pair_type()); - let head = builder.local("head", RType::STR); + let head = builder.temp(RType::STR); let length = builder.temp(RType::INT); let next = builder.new_block(); call_pair(&mut builder, pair, text); @@ -1641,6 +1806,7 @@ mod tests { lhs: Value::Str("a".to_string()), rhs: Value::Str("b".to_string()), consumes_lhs: false, + concatenation: Concatenation::Operator(Mutation::Fresh), }); builder.push(Op::Len { dest: length, @@ -1658,7 +1824,7 @@ mod tests { let mut builder = FunctionBuilder::new("f", RType::INT); let text = builder.param("s", RType::STR); let pair = builder.temp(pair_type()); - let head = builder.local("head", RType::STR); + let head = builder.temp(RType::STR); let length = builder.temp(RType::INT); call_pair(&mut builder, pair, text); for index in [0, 1] { @@ -1686,7 +1852,7 @@ mod tests { let text = builder.param("s", RType::STR); let first = builder.temp(pair_type()); let second = builder.temp(pair_type()); - let head = builder.local("head", RType::STR); + let head = builder.temp(RType::STR); let length = builder.temp(RType::INT); let next = builder.new_block(); for (pair, last) in [(first, false), (second, true)] { @@ -1719,7 +1885,7 @@ mod tests { let mut builder = FunctionBuilder::new("f", RType::INT); let text = builder.param("s", RType::STR); let pair = builder.temp(pair_type()); - let head = builder.local("head", RType::STR); + let head = builder.temp(RType::STR); let length = builder.temp(RType::INT); call_pair(&mut builder, pair, text); builder.push(Op::TupleGet { @@ -1748,7 +1914,7 @@ mod tests { let mut builder = FunctionBuilder::new("f", RType::INT); let text = builder.param("s", RType::STR); let pair = builder.temp(pair_type()); - let head = builder.local("head", RType::STR); + let head = builder.temp(RType::STR); let widened = builder.temp(RType::OBJECT); let length = builder.temp(RType::INT); call_pair(&mut builder, pair, text); @@ -1785,7 +1951,7 @@ mod tests { let text = builder.param("s", RType::STR); let widened = builder.temp(RType::OBJECT); let pair = builder.temp(pair_type()); - let head = builder.local("head", RType::STR); + let head = builder.temp(RType::STR); let length = builder.temp(RType::INT); assert!(widened < head, "the copy has to be settled first"); call_pair(&mut builder, pair, text); @@ -1815,7 +1981,7 @@ mod tests { let mut builder = FunctionBuilder::new("f", RType::INT); let text = builder.param("s", RType::STR); let pair = builder.temp(pair_type()); - let head = builder.local("head", RType::STR); + let head = builder.temp(RType::STR); let widened = builder.temp(RType::OBJECT); let length = builder.temp(RType::INT); call_pair(&mut builder, pair, text); @@ -1843,7 +2009,7 @@ mod tests { let mut builder = FunctionBuilder::new("f", RType::INT); let text = builder.param("s", RType::STR); let held = builder.temp(RType::OBJECT); - let part = builder.local("part", to.clone()); + let part = builder.temp(to.clone()); let length = builder.temp(RType::INT); builder.push(Op::CallNative { owner: None, @@ -1943,7 +2109,7 @@ mod tests { let mut builder = FunctionBuilder::new("f", RType::FLOAT); let text = builder.param("s", RType::STR); let pair = builder.temp(RType::Tuple(Box::new([RType::FLOAT, RType::FLOAT]))); - let head = builder.local("head", RType::FLOAT); + let head = builder.temp(RType::FLOAT); let doubled = builder.temp(RType::FLOAT); call_pair(&mut builder, pair, text); builder.push(Op::TupleGet { diff --git a/crates/by_opt/src/copy_propagation.rs b/crates/by_opt/src/copy_propagation.rs index 36039441a4..97c09110ba 100644 --- a/crates/by_opt/src/copy_propagation.rs +++ b/crates/by_opt/src/copy_propagation.rs @@ -255,6 +255,7 @@ fn usage_counts(function: &Function) -> HashMap { fn retarget(op: &mut Op, new_dest: RegisterId) { match op { Op::Assign { dest, .. } + | Op::Move { dest, .. } | Op::IntBinary { dest, .. } | Op::FloatBinary { dest, .. } | Op::FloatObjectBinary { dest, .. } @@ -271,13 +272,16 @@ fn retarget(op: &mut Op, new_dest: RegisterId) { | Op::AsyncContext { dest, .. } | Op::IsMissing { dest, .. } | Op::MethodStands { dest, .. } + | Op::BuiltinStands { dest, .. } | Op::AccessorStands { dest, .. } + | Op::FieldStands { dest, .. } | Op::DictShadows { dest, .. } | Op::MatchSlice { dest, .. } | Op::IntCompare { dest, .. } | Op::FloatCompare { dest, .. } | Op::ObjectBinary { dest, .. } | Op::ObjectCompare { dest, .. } + | Op::ObjectRichCompare { dest, .. } | Op::StrCompare { dest, .. } | Op::Truthy { dest, .. } | Op::Len { dest, .. } @@ -332,6 +336,7 @@ fn retarget(op: &mut Op, new_dest: RegisterId) { | Op::DeleteAttr { dest, .. } | Op::ArrayPush { dest, .. } | Op::Extend { dest, .. } + | Op::MergeKeywords { dest, .. } | Op::CallUnpacked { dest, .. } | Op::StrConcat { dest, .. } | Op::StrConcatInt { dest, .. } @@ -340,6 +345,7 @@ fn retarget(op: &mut Op, new_dest: RegisterId) { | Op::IntToFloat { dest, .. } | Op::Unbox { dest, .. } | Op::TupleBuild { dest, .. } + | Op::FieldIsSet { dest, .. } | Op::TupleGet { dest, .. } => *dest = new_dest, Op::CallNative { dest, .. } => *dest = Some(new_dest), // every other arm here has no destination at all, so the caller never reaches @@ -350,8 +356,10 @@ fn retarget(op: &mut Op, new_dest: RegisterId) { | Op::FinishFrame { .. } | Op::RaiseObject { .. } | Op::PopHandled { .. } + | Op::Release { .. } | Op::Reraise { .. } | Op::LicenceHolds { .. } + | Op::RequireField { .. } | Op::SetField { .. } => {} } } diff --git a/crates/by_opt/src/dead_allocations.rs b/crates/by_opt/src/dead_allocations.rs index e48b0e434b..e9a149b129 100644 --- a/crates/by_opt/src/dead_allocations.rs +++ b/crates/by_opt/src/dead_allocations.rs @@ -78,19 +78,47 @@ fn drop_unread(function: &mut Function) -> bool { // del x # raises `UnboundLocalError` if `x = n` never ran // // nothing reads `x` there, and dropping the assignment made the first `del` raise + // + // and a store into a name is a binding python keeps until the name is rebound or + // deleted, whether or not anything reads it. where the value could be, or hold, + // an object with a finalizer, dropping the store changes when that runs: the + // value would be let go of along with whatever it was built from, rather than when + // the name is rebound + // + // held = Cell(1) + // held = Cell(2) # `Cell(1)` is let go of here + let finalizer_free = finalizer_free(function); let observable: HashSet = function .registers .iter() .enumerate() - .filter(|(_, decl)| decl.may_be_unassigned) + .filter(|(index, decl)| { + decl.may_be_unassigned + || (decl.name.is_some() && !finalizer_free.contains(&RegisterId(*index))) + }) .map(|(index, _)| RegisterId(index)) .collect(); + // a tuple holds what it was built from until it is let go of, and then lets go of + // its elements last to first. one nothing reads is let go of as soon as it is built, + // which for a display is after its last element is made — so without it, an element + // could go before the elements after it were made: + // + // (make(1), make(2)) # `make(1)`'s value outlives `make(2)` + // x = (make(1), make(2))[1] + let holds_its_elements = |op: &Op| { + matches!(op, Op::TupleBuild { .. } | Op::BuildTuple { .. }) + && op + .dest() + .is_some_and(|dest| !finalizer_free.contains(&dest)) + }; + let mut dropped = false; for block in &mut function.blocks { let before = block.ops.len(); block.ops.retain(|op| { !computes_only(op) + || holds_its_elements(op) || op .dest() .is_none_or(|dest| read.contains(&dest) || observable.contains(&dest)) @@ -99,3 +127,166 @@ fn drop_unread(function: &mut Function) -> bool { } dropped } + +/// the registers that can only ever hold a value whose release runs nothing +/// +/// a representation with no object reference in it is one, and so is an object every +/// write of which boxes such a value, or builds a tuple of them. that is `pair` in +/// `pair = (whole, part)` over two `int`s, once its reads have been folded into reads +/// of the elements. a tagged `int` counts as having no finalizer, as it does for the +/// release pass +fn finalizer_free(function: &Function) -> HashSet { + let mut free: HashSet = function + .registers + .iter() + .enumerate() + .filter(|(_, decl)| !decl.ty.holds_object_reference()) + .map(|(index, _)| RegisterId(index)) + .collect(); + loop { + let mut grew = false; + for index in function.param_count..function.registers.len() { + let register = RegisterId(index); + if free.contains(®ister) { + continue; + } + let mut writes = function + .blocks + .iter() + .flat_map(|block| &block.ops) + .filter(|op| op.dest() == Some(register)) + .peekable(); + if writes.peek().is_none() { + continue; + } + let value_free = |value: &Value| match value { + Value::Register(id) => free.contains(id), + _ => true, + }; + let builds_free = writes.all(|op| match op { + Op::Assign { src, .. } | Op::Box { src, .. } => value_free(src), + Op::BuildTuple { items, .. } | Op::TupleBuild { items, .. } => { + items.iter().all(value_free) + } + _ => false, + }); + if builds_free { + free.insert(register); + grew = true; + } + } + if !grew { + return free; + } + } +} + +#[cfg(test)] +mod tests { + use by_ir::builder::FunctionBuilder; + use by_ir::function::ModuleIr; + use by_ir::ops::{Op, Terminator, Value}; + use by_ir::rtype::RType; + + fn module(function: by_ir::function::Function) -> ModuleIr { + let mut module = ModuleIr::new("app"); + module.functions.push(function); + module + } + + fn call(dest: by_ir::ops::RegisterId, arg: by_ir::ops::RegisterId) -> Op { + Op::CallPython { + dest, + callee: "make".to_string(), + args: vec![Value::Register(arg)], + } + } + + #[test] + fn a_store_into_a_name_nothing_reads_is_kept() { + // `held = make(p)` with `held` never read: the name still holds the value until + // it is rebound, and the store is what lets the rebinding let go of it + let mut builder = FunctionBuilder::new("f", RType::OBJECT); + let p = builder.param("p", RType::OBJECT); + let made = builder.temp(RType::OBJECT); + let held = builder.local("held".to_string(), RType::OBJECT); + builder.push(call(made, p)); + builder.assign(held, Value::Register(made)); + builder.terminate(Terminator::Return(Value::Register(p))); + + let mut module = module(builder.finish()); + super::run(&mut module); + + assert_eq!(module.functions[0].blocks[0].ops.len(), 2); + } + + #[test] + fn a_store_into_a_temporary_nothing_reads_goes() { + let mut builder = FunctionBuilder::new("f", RType::OBJECT); + let p = builder.param("p", RType::OBJECT); + let made = builder.temp(RType::OBJECT); + let copy = builder.temp(RType::OBJECT); + builder.push(call(made, p)); + builder.assign(copy, Value::Register(made)); + builder.terminate(Terminator::Return(Value::Register(p))); + + let mut module = module(builder.finish()); + super::run(&mut module); + + assert_eq!(module.functions[0].blocks[0].ops.len(), 1); + } + + #[test] + fn a_name_nothing_reads_holding_only_ints_goes() { + // `pair = (whole, part)` over two `int`s, with its reads folded into reads of + // the elements: letting go of a tuple of boxed ints runs nothing, so when it + // happens cannot be told apart, and the tuple and both boxes go + let mut builder = FunctionBuilder::new("f", RType::INT); + let whole = builder.param("whole", RType::INT); + let part = builder.param("part", RType::INT); + let first = builder.temp(RType::OBJECT); + let second = builder.temp(RType::OBJECT); + let pair = builder.local("pair".to_string(), RType::OBJECT); + builder.push(Op::Box { + dest: first, + src: Value::Register(whole), + }); + builder.push(Op::Box { + dest: second, + src: Value::Register(part), + }); + builder.push(Op::BuildTuple { + dest: pair, + items: vec![Value::Register(first), Value::Register(second)], + }); + builder.terminate(Terminator::Return(Value::Register(whole))); + + let mut module = module(builder.finish()); + super::run(&mut module); + + assert!(module.functions[0].blocks[0].ops.is_empty()); + } + + #[test] + fn a_tuple_nothing_reads_holding_objects_stays() { + // `(make(p), make(p))` thrown away: the tuple is what keeps the first element + // alive while the second is made, and what lets both go, last to first + let mut builder = FunctionBuilder::new("f", RType::OBJECT); + let p = builder.param("p", RType::OBJECT); + let first = builder.temp(RType::OBJECT); + let second = builder.temp(RType::OBJECT); + let pair = builder.temp(RType::Tuple(Box::new([RType::OBJECT, RType::OBJECT]))); + builder.push(call(first, p)); + builder.push(call(second, p)); + builder.push(Op::TupleBuild { + dest: pair, + items: vec![Value::Register(first), Value::Register(second)], + }); + builder.terminate(Terminator::Return(Value::Register(p))); + + let mut module = module(builder.finish()); + super::run(&mut module); + + assert_eq!(module.functions[0].blocks[0].ops.len(), 3); + } +} diff --git a/crates/by_opt/src/dict_find.rs b/crates/by_opt/src/dict_find.rs index 50c8684ee3..65f439a421 100644 --- a/crates/by_opt/src/dict_find.rs +++ b/crates/by_opt/src/dict_find.rs @@ -13,14 +13,16 @@ //! //! ## the fact is checked at runtime, not inferred from a type //! -//! two ordinary things make asking twice observable. a dict *subclass* may have +//! three ordinary things make asking twice observable. a dict *subclass* may have //! overridden `__contains__` or `__getitem__`, and then how many times each is -//! called is the program's own business. and a key may have a `__hash__` that -//! counts its calls, in which case hashing once where the source hashes twice is a -//! different program. neither is a question this pass answers: the emitted helper -//! takes its single probe only for an exact dict keyed by an exact `str`, and -//! everything else goes through the protocol twice over in the order it would -//! have. so the pass needs no static type at all — it is a *shape* rewrite, and it +//! called is the program's own business. a key may have a `__hash__` that counts +//! its calls, in which case hashing once where the source hashes twice is a +//! different program. and a key already stored in the dict has its `__eq__` asked +//! by every probe that walks past it, which can count the probes or change the +//! dict between them. none is a question this pass answers: the emitted helper +//! takes its single probe only for an exact dict keyed by an exact `str` whose +//! table holds nothing but exact `str` keys, and everything else goes through the +//! protocol twice over in the order it would have. so the pass needs no static type at all — it is a *shape* rewrite, and it //! is correct over a list, a subclass or anything else that reaches it. //! //! ## why the branch has to be part of the shape diff --git a/crates/by_opt/src/field_proofs.rs b/crates/by_opt/src/field_proofs.rs new file mode 100644 index 0000000000..54ced1278f --- /dev/null +++ b/crates/by_opt/src/field_proofs.rs @@ -0,0 +1,403 @@ +//! drop the test before a field read that an earlier write or test already answered +//! +//! an instance `__new__` made without running `__init__` has none of the fields +//! `__init__` gives every other instance, so a read of one is preceded by a test that +//! raises `AttributeError` where the field is absent. but a field that is always +//! assigned cannot be taken away again once it has been — `del` of one is refused, and +//! only a field `__init__` might leave out can be deleted, whose read tests for itself. +//! so once a function has written a field of an object, or tested it, every later read +//! of that field of that same object finds it: +//! +//! ```python +//! def step(state: State) -> None: +//! state.a = state.b + state.c # `b` and `c` are tested +//! state.b = state.a - state.c # `a` was written and `c` tested: nothing is +//! ``` +//! +//! "that same object" is a register that has not been written in between. a register +//! written again may hold another object, and one written on some path into a block is +//! as good as written. a handler is entered from the middle of the blocks it covers, so +//! what it can rely on is only what held all through each of them + +use std::collections::{HashMap, HashSet}; + +use by_ir::function::{BasicBlock, Function, ModuleIr}; +use by_ir::ops::{BlockId, Op, RegisterId, Value}; + +/// a field of the object a register holds, which the function knows the object has +type Known = (RegisterId, String); + +/// each class's fields in layout order, with `None` for one this pass says nothing about +type Layouts = HashMap>>; + +pub(crate) fn run(module: &mut ModuleIr) { + // only a field nothing can take away again: an optional one can be deleted by + // anything the function calls, and a field no instance lacks has nothing to prove + let layouts: HashMap>> = module + .classes + .iter() + .map(|class| { + ( + class.name.clone(), + class + .fields + .iter() + .map(|field| { + (!field.optional && field.tracks_absence()).then(|| field.name.clone()) + }) + .collect(), + ) + }) + .collect(); + for function in module.all_functions_mut() { + prove(function, &layouts); + } +} + +/// what an operation lets the function know about fields afterwards +fn learned(op: &Op, layouts: &Layouts) -> Vec { + let permanent = |class: &str, field: &str| { + layouts + .get(class) + .is_some_and(|names| names.iter().flatten().any(|name| name == field)) + }; + match op { + Op::SetField { + receiver: Value::Register(receiver), + class, + field, + .. + } + | Op::RequireField { + receiver: Value::Register(receiver), + class, + field, + } if permanent(class, field) => vec![(*receiver, field.clone())], + Op::NewInstance { + dest, + class, + fields, + } => layouts + .get(class) + .into_iter() + .flat_map(|names| names.iter().zip(fields)) + .filter_map(|(name, value)| value.as_ref().and(name.as_ref())) + .map(|name| (*dest, name.clone())) + .collect(), + _ => Vec::new(), + } +} + +/// the register an operation leaves holding something else, if any +fn rebinds(op: &Op) -> Vec { + op.dest() + .into_iter() + .chain(op.unbinds()) + .chain(op.loop_cursor()) + .collect() +} + +/// `known` after `op` has run +fn step(known: &mut HashSet, op: &Op, layouts: &Layouts) { + for register in rebinds(op) { + known.retain(|(held, _)| *held != register); + } + known.extend(learned(op, layouts)); +} + +/// what holds at every point of a block, for a handler it can raise into +fn throughout(block: &BasicBlock, entry: &HashSet) -> HashSet { + let rebound: HashSet = block.ops.iter().flat_map(rebinds).collect(); + entry + .iter() + .filter(|(held, _)| !rebound.contains(held)) + .cloned() + .collect() +} + +fn prove(function: &mut Function, layouts: &Layouts) { + let count = function.blocks.len(); + if count == 0 { + return; + } + // `None` is "not reached yet", which is the whole set for the intersection below + let mut entry: Vec>> = vec![None; count]; + entry[0] = Some(HashSet::new()); + let mut changed = true; + while changed { + changed = false; + for index in 0..count { + let Some(start) = entry[index].clone() else { + continue; + }; + let block = &function.blocks[index]; + let mut known = start.clone(); + for op in &block.ops { + step(&mut known, op, layouts); + } + // the exception edge is the last of the successors, and it carries only what + // held all through the block + let mut normal = block.successors(); + if block.error_target.is_some() { + normal.pop(); + } + let mut flows: Vec<(BlockId, HashSet)> = normal + .into_iter() + .map(|successor| (successor, known.clone())) + .collect(); + if let Some(handler) = block.error_target { + flows.push((handler, throughout(block, &start))); + } + for (successor, facts) in flows { + let Some(slot) = entry.get_mut(successor.index()) else { + continue; + }; + let merged = match slot { + None => facts, + Some(existing) => existing.intersection(&facts).cloned().collect(), + }; + if slot.as_ref() != Some(&merged) { + *slot = Some(merged); + changed = true; + } + } + } + } + + for (index, block) in function.blocks.iter_mut().enumerate() { + let Some(mut known) = entry[index].clone() else { + continue; + }; + let mut kept = Vec::with_capacity(block.ops.len()); + for op in block.ops.drain(..) { + let op = match op { + Op::RequireField { + receiver: Value::Register(receiver), + ref field, + .. + } if known.contains(&(receiver, field.clone())) => continue, + Op::FieldIsSet { + dest, + receiver: Value::Register(receiver), + ref field, + .. + } if known.contains(&(receiver, field.clone())) => Op::Assign { + dest, + src: Value::Bit(true), + }, + Op::SetField { + receiver: receiver @ Value::Register(held), + class, + field, + value, + moves, + present: false, + } if known.contains(&(held, field.clone())) => Op::SetField { + receiver, + class, + field, + value, + moves, + present: true, + }, + other => other, + }; + step(&mut known, &op, layouts); + kept.push(op); + } + block.ops = kept; + } +} + +#[cfg(test)] +mod tests { + use by_ir::builder::FunctionBuilder; + use by_ir::function::{ClassIr, FieldDecl, ModuleIr}; + use by_ir::ops::{BinOp, Op, Terminator, Value}; + use by_ir::rtype::RType; + + fn class(fields: &[(&str, bool)]) -> ClassIr { + ClassIr { + name: "State".to_string(), + immutable: false, + environment: false, + exported: true, + base: None, + inherited_init: false, + fields_are_parameters: false, + dataclass: false, + generic: false, + declares_slots: false, + constants: Vec::new(), + properties: Vec::new(), + slot_aliases: Vec::new(), + fields: fields + .iter() + .map(|(name, optional)| FieldDecl { + cell: false, + name: (*name).to_string(), + ty: RType::INT, + default: None, + optional: *optional, + defaulted_by: None, + }) + .collect(), + decorators: Vec::new(), + methods: Vec::new(), + resume: None, + keywords: Vec::new(), + } + } + + fn state() -> RType { + RType::Instance { + class: "State".to_string(), + exact: false, + } + } + + fn require(receiver: by_ir::ops::RegisterId, field: &str) -> Op { + Op::RequireField { + receiver: Value::Register(receiver), + class: "State".to_string(), + field: field.to_string(), + } + } + + fn read(dest: by_ir::ops::RegisterId, receiver: by_ir::ops::RegisterId, field: &str) -> Op { + Op::GetField { + dest, + receiver: Value::Register(receiver), + class: "State".to_string(), + field: field.to_string(), + } + } + + fn requires(module: &ModuleIr) -> usize { + module.functions[0] + .blocks + .iter() + .flat_map(|block| &block.ops) + .filter(|op| matches!(op, Op::RequireField { .. })) + .count() + } + + #[test] + fn a_second_read_of_a_tested_field_is_not_tested_again() { + let mut builder = FunctionBuilder::new("f", RType::INT); + let s = builder.param("s", state()); + let first = builder.temp(RType::INT); + let second = builder.temp(RType::INT); + let sum = builder.temp(RType::INT); + builder.push(require(s, "b")); + builder.push(read(first, s, "b")); + builder.push(require(s, "b")); + builder.push(read(second, s, "b")); + builder.push(Op::IntBinary { + dest: sum, + op: BinOp::Add, + lhs: Value::Register(first), + rhs: Value::Register(second), + }); + builder.terminate(Terminator::Return(Value::Register(sum))); + let mut module = ModuleIr::new("app"); + module.classes.push(class(&[("b", false)])); + module.functions.push(builder.finish()); + + super::run(&mut module); + + assert_eq!(requires(&module), 1); + } + + #[test] + fn a_written_field_is_not_tested_and_not_marked_again() { + let mut builder = FunctionBuilder::new("f", RType::INT); + let s = builder.param("s", state()); + let value = builder.param("v", RType::INT); + let out = builder.temp(RType::INT); + let store = |present| Op::SetField { + receiver: Value::Register(s), + class: "State".to_string(), + field: "a".to_string(), + value: Value::Register(value), + moves: false, + present, + }; + builder.push(store(false)); + builder.push(require(s, "a")); + builder.push(read(out, s, "a")); + builder.push(store(false)); + builder.terminate(Terminator::Return(Value::Register(out))); + let mut module = ModuleIr::new("app"); + module.classes.push(class(&[("a", false)])); + module.functions.push(builder.finish()); + + super::run(&mut module); + + let ops = &module.functions[0].blocks[0].ops; + assert_eq!(requires(&module), 0, "{ops:?}"); + assert!( + matches!(ops[0], Op::SetField { present: false, .. }), + "{ops:?}" + ); + assert!( + matches!(ops[2], Op::SetField { present: true, .. }), + "{ops:?}" + ); + } + + #[test] + fn a_rebound_receiver_or_an_optional_field_is_tested_again() { + // the register may hold another object once written, and an optional field can be + // deleted by anything in between + let mut builder = FunctionBuilder::new("f", RType::INT); + let s = builder.param("s", state()); + let t = builder.param("t", state()); + let held = builder.local("held", state()); + let out = builder.temp(RType::INT); + builder.assign(held, Value::Register(s)); + builder.push(require(held, "a")); + builder.assign(held, Value::Register(t)); + builder.push(require(held, "a")); + builder.push(require(held, "maybe")); + builder.push(require(held, "maybe")); + builder.push(read(out, held, "a")); + builder.terminate(Terminator::Return(Value::Register(out))); + let mut module = ModuleIr::new("app"); + module.classes.push(class(&[("a", false), ("maybe", true)])); + module.functions.push(builder.finish()); + + super::run(&mut module); + + assert_eq!(requires(&module), 4); + } + + #[test] + fn a_test_in_a_loop_body_is_not_answered_by_the_trip_before() { + // the first trip reaches the test with nothing known, so it stays + let mut builder = FunctionBuilder::new("f", RType::INT); + let s = builder.param("s", state()); + let more = builder.param("more", RType::BIT); + let out = builder.temp(RType::INT); + let body = builder.new_block(); + let done = builder.new_block(); + builder.terminate(Terminator::Goto(body)); + builder.switch_to(body); + builder.push(require(s, "a")); + builder.push(read(out, s, "a")); + builder.terminate(Terminator::Branch { + cond: Value::Register(more), + then_block: body, + else_block: done, + }); + builder.switch_to(done); + builder.terminate(Terminator::Return(Value::Register(out))); + let mut module = ModuleIr::new("app"); + module.classes.push(class(&[("a", false)])); + module.functions.push(builder.finish()); + + super::run(&mut module); + + assert_eq!(requires(&module), 1); + } +} diff --git a/crates/by_opt/src/fold.rs b/crates/by_opt/src/fold.rs index e958b8f961..2273014dcf 100644 --- a/crates/by_opt/src/fold.rs +++ b/crates/by_opt/src/fold.rs @@ -953,6 +953,7 @@ mod tests { m.classes.push(by_ir::function::ClassIr { name: "Vec2".to_string(), immutable: true, + environment: false, resume: None, keywords: Vec::new(), exported: true, @@ -961,6 +962,7 @@ mod tests { fields_are_parameters: true, dataclass: false, fields: vec![by_ir::function::FieldDecl { + cell: false, name: "x".to_string(), ty: RType::FLOAT, default: None, @@ -1020,6 +1022,7 @@ mod tests { m.classes.push(by_ir::function::ClassIr { name: "Loose".to_string(), immutable: false, + environment: false, resume: None, keywords: Vec::new(), exported: true, @@ -1028,6 +1031,7 @@ mod tests { fields_are_parameters: true, dataclass: false, fields: vec![by_ir::function::FieldDecl { + cell: false, name: "x".to_string(), ty: RType::FLOAT, default: None, @@ -1078,6 +1082,7 @@ mod tests { m.classes.push(by_ir::function::ClassIr { name: "Vec2".to_string(), immutable: true, + environment: false, resume: None, keywords: Vec::new(), exported: true, @@ -1086,6 +1091,7 @@ mod tests { fields_are_parameters: true, dataclass: false, fields: vec![by_ir::function::FieldDecl { + cell: false, name: "x".to_string(), ty: RType::FLOAT, default: None, @@ -1138,6 +1144,7 @@ mod tests { m.classes.push(by_ir::function::ClassIr { name: "Vec2".to_string(), immutable: true, + environment: false, resume: None, keywords: Vec::new(), exported: true, @@ -1146,6 +1153,7 @@ mod tests { fields_are_parameters: true, dataclass: false, fields: vec![by_ir::function::FieldDecl { + cell: false, name: "x".to_string(), ty: RType::FLOAT, default: None, diff --git a/crates/by_opt/src/infallible.rs b/crates/by_opt/src/infallible.rs index 8bcdd725e1..f4882c89bb 100644 --- a/crates/by_opt/src/infallible.rs +++ b/crates/by_opt/src/infallible.rs @@ -132,6 +132,7 @@ fn op_can_fail(module: &ModuleIr, function: &by_ir::function::Function, op: &Op) // fixed tuple cannot raise on their own. neither does naming an emitted // class: the type object is this module's own, and already built Op::Assign { .. } + | Op::Move { .. } | Op::FloatCompare { .. } | Op::TupleGet { .. } | Op::LoadClass { .. } @@ -146,6 +147,7 @@ fn op_can_fail(module: &ModuleIr, function: &by_ir::function::Function, op: &Op) | Op::IntCompare { .. } | Op::ObjectBinary { .. } | Op::ObjectCompare { .. } + | Op::ObjectRichCompare { .. } | Op::StrCompare { .. } | Op::Truthy { .. } | Op::Len { .. } @@ -198,6 +200,7 @@ fn op_can_fail(module: &ModuleIr, function: &by_ir::function::Function, op: &Op) | Op::GetField { .. } | Op::PushHandled { .. } | Op::PopHandled { .. } + | Op::Release { .. } | Op::SetField { .. } => false, // the module namespace is a dict this module already holds — the extension's @@ -216,7 +219,7 @@ fn op_can_fail(module: &ModuleIr, function: &by_ir::function::Function, op: &Op) Op::CallUnpacked { .. } => true, // merging into a display drives an iterator or a mapping, either of which fails - Op::Extend { .. } => true, + Op::Extend { .. } | Op::MergeKeywords { .. } => true, // allocating, growing and indexing can all fail; reading the length is a // field read of a buffer we already hold @@ -248,10 +251,19 @@ fn op_can_fail(module: &ModuleIr, function: &by_ir::function::Function, op: &Op) Op::IsMapping { .. } => false, // an object with no `__aiter__` is a `TypeError`, and the call may raise Op::AsyncIter { .. } | Op::AsyncContext { .. } => true, + // a field the instance does not have is an `AttributeError` + Op::RequireField { .. } => true, + // and asking whether it has one is a load and a comparison + Op::FieldIsSet { .. } => false, + // the name is resolved, and one bound nowhere is a `NameError` + Op::BuiltinStands { .. } => true, // an attribute lookup reaches `__getattr__`, and `__match_args__` with it Op::MatchAttr { .. } => true, // a pointer comparison against a singleton - Op::IsMissing { .. } | Op::MethodStands { .. } | Op::AccessorStands { .. } => false, + Op::IsMissing { .. } + | Op::MethodStands { .. } + | Op::AccessorStands { .. } + | Op::FieldStands { .. } => false, // it either agrees and returns, or aborts the process. it never sets an // exception, so there is no error edge for a caller to take Op::LicenceHolds { .. } => false, diff --git a/crates/by_opt/src/lib.rs b/crates/by_opt/src/lib.rs index 6771031c24..62b2a7dfc5 100644 --- a/crates/by_opt/src/lib.rs +++ b/crates/by_opt/src/lib.rs @@ -11,9 +11,13 @@ pub(crate) mod copy_propagation; mod dead_allocations; pub(crate) mod dead_registers; pub(crate) mod dict_find; +mod field_proofs; pub(crate) mod fold; pub(crate) mod infallible; +mod liveness; pub(crate) mod refcount; +pub(crate) mod release_temporaries; +mod store_moves; pub(crate) mod str_append; mod str_concat_int; pub(crate) mod str_item_compare; @@ -36,6 +40,12 @@ const PASSES: &[Pass] = &[ name: "copy-propagation", run: copy_propagation::run, }, + // after copy propagation, which is what makes two reads of one object name one + // register, and before folding, which is what turns a test it answered into a jump + Pass { + name: "field-proofs", + run: field_proofs::run, + }, // folding runs after copy propagation, which is what turns a comparison's // temp into an immediate the branch can see Pass { @@ -136,6 +146,18 @@ const PASSES: &[Pass] = &[ name: "str-concat-int", run: str_concat_int::run, }, + // after every pass that decides what a register owns — the borrow pass above all — + // because where a temporary can be let go of rests on the final answer + Pass { + name: "release-temporaries", + run: release_temporaries::run, + }, + // after the release pass, whose release straight after a dying store is what a + // moving store replaces + Pass { + name: "store-moves", + run: store_moves::run, + }, // last: it reads the final shape of every block Pass { name: "refcount", @@ -303,6 +325,7 @@ mod tests { name: "Pair".to_string(), fields: vec![ by_ir::function::FieldDecl { + cell: false, name: "a".to_string(), ty: RType::FLOAT, default: None, @@ -310,6 +333,7 @@ mod tests { defaulted_by: None, }, by_ir::function::FieldDecl { + cell: false, name: "b".to_string(), ty: RType::FLOAT, default: None, @@ -329,6 +353,7 @@ mod tests { fields_are_parameters: true, dataclass: false, immutable: false, + environment: false, }); assert_eq!(optimize(&mut module), Ok(())); diff --git a/crates/by_opt/src/liveness.rs b/crates/by_opt/src/liveness.rs new file mode 100644 index 0000000000..9c5f5faf56 --- /dev/null +++ b/crates/by_opt/src/liveness.rs @@ -0,0 +1,122 @@ +//! which registers are still going to be read, at the start of each block +//! +//! an exception leaves a block from the middle, so what its handler reads is live at +//! every point of the block rather than only at its end. a pass that asks whether a +//! register is read again has to ask across that edge too, or a handler reads a value +//! the pass has already given away + +use std::collections::HashSet; +use std::hash::Hash; + +use by_ir::function::{BasicBlock, Function}; +use by_ir::ops::{BlockId, Op, RegisterId, Value}; + +/// the registers live on entry to each block +/// +/// `reads` says which registers an operation reads. most passes want +/// [`read_registers`]; one that knows a register can be read *through* another hands +/// in a wider answer +pub(crate) fn live_in( + function: &Function, + reads: &impl Fn(&Op) -> Vec, +) -> Vec> { + live_in_by(function, reads, &|register| vec![register]) +} + +/// [`live_in`], over any finer division of a register than the register itself +/// +/// `whole` names every part of a register, which is what a write overwrites and what a +/// terminator reads. a pass that follows the elements of a fixed-length tuple apart +/// answers with one part per element +pub(crate) fn live_in_by( + function: &Function, + reads: &impl Fn(&Op) -> Vec

, + whole: &impl Fn(RegisterId) -> Vec

, +) -> Vec> { + let mut live_in: Vec> = vec![HashSet::new(); function.blocks.len()]; + + let mut changed = true; + while changed { + changed = false; + for index in (0..function.blocks.len()).rev() { + let Some(block) = function.block(BlockId(index)) else { + continue; + }; + let mut live = live_out_by(block, &live_in, whole); + let error_live = error_live(block, &live_in); + for op in block.ops.iter().rev() { + live.extend(error_live.iter().cloned()); + if let Some(dest) = op.dest() { + for part in whole(dest) { + live.remove(&part); + } + } + live.extend(reads(op)); + } + + if live != live_in[index] { + live_in[index] = live; + changed = true; + } + } + } + + live_in +} + +/// what is live when `block` ends: its successors' needs, and what its terminator reads +pub(crate) fn live_out(block: &BasicBlock, live_in: &[HashSet]) -> HashSet { + live_out_by(block, live_in, &|register| vec![register]) +} + +/// [`live_out`], over the parts [`live_in_by`] divides a register into +pub(crate) fn live_out_by( + block: &BasicBlock, + live_in: &[HashSet

], + whole: &impl Fn(RegisterId) -> Vec

, +) -> HashSet

{ + let mut live: HashSet

= block + .successors() + .iter() + .filter_map(|successor| live_in.get(successor.index())) + .flatten() + .cloned() + .collect(); + live.extend( + block + .terminator + .operands() + .into_iter() + .filter_map(register) + .flat_map(whole), + ); + live +} + +/// what the block's handler reads, which is live at every point of the block +pub(crate) fn error_live(block: &BasicBlock, live_in: &[HashSet

]) -> HashSet

{ + block + .error_target + .and_then(|target| live_in.get(target.index())) + .cloned() + .unwrap_or_default() +} + +/// the registers an operation reads +/// +/// `del x` leaves its destination unbound, but it reads the value first — to release +/// it — so the reference is still needed up to that point +pub(crate) fn read_registers(op: &Op) -> Vec { + op.operands() + .into_iter() + .filter_map(register) + .chain(op.unbinds()) + .collect() +} + +pub(crate) fn register(value: &Value) -> Option { + match value { + Value::Register(id) => Some(*id), + _ => None, + } +} diff --git a/crates/by_opt/src/release_temporaries.rs b/crates/by_opt/src/release_temporaries.rs new file mode 100644 index 0000000000..dca5ee04ee --- /dev/null +++ b/crates/by_opt/src/release_temporaries.rs @@ -0,0 +1,774 @@ +//! drop a value nothing names as soon as nothing will read it again +//! +//! python keeps a value alive for exactly as long as something refers to it. a +//! temporary — the file `open(p, "w")` answers in `open(p, "w").write(s)`, or the +//! result of a call made only for its effect — is referred to by nothing once the +//! expression that made it is done with it, so python drops it there, and a finalizer +//! runs at that point: the file is closed, and so flushed, before the next line reads +//! it back. +//! +//! a register otherwise holds its reference until something writes over it or the +//! frame returns, which is late by however much of the function is left. so after the +//! last read of an unnamed temporary this pass releases it and leaves the register +//! empty: the release every exit makes of it then finds nothing, and so does the +//! release a later write makes of what the register held before. a named local is left +//! alone: it owns a reference of its own, which a later write or `del` of the name lets +//! go of, as python does when the name is rebound or deleted +//! +//! ## a tuple is let go of an element at a time +//! +//! a fixed-length tuple is held as a struct of its elements, each owning a reference of +//! its own, so each element is a temporary on its own account. `part = make(i)[1]` names +//! one element, and python drops the rest of the tuple as soon as the subscript is done +//! with it — so an element read off the tuple reads that element and no other, and the +//! elements nobody reads again are let go of, last to first as a tuple drops them, while +//! the named one stays +//! +//! ## a borrow reads through its lender +//! +//! the borrow pass lets a register hold a value on loan from one its defining operation +//! read — a copy's source, a field read's receiver, the tuple an element came out of. +//! the lender's reference is what keeps the borrowed value alive, so a read of the +//! borrower counts as a read of every register that could be lending to it, and a +//! lender stays until its borrowers are done +//! +//! ## a release can run anything +//! +//! dropping the last reference to a value runs its finalizer, which is arbitrary +//! python. the borrow pass proved each borrowed register's window free of anything that +//! could run python, so no release is placed inside one: it waits for the window to +//! close, which it always does before the block ends +//! +//! ## the error edge +//! +//! a handler can read a register the block it covers is otherwise done with, so what a +//! handler reads is live at every point of that block, exactly as it is for the +//! in-place append. and a value the block was holding when it raised is dropped once +//! the handler has taken the exception, which is where python's unwinding has dropped +//! it by + +use std::cmp::{Ordering, Reverse}; +use std::collections::{BTreeSet, HashMap, HashSet}; + +use by_ir::function::{BasicBlock, Function, ModuleIr}; +use by_ir::ops::{BlockId, Op, RegisterId, Value}; +use by_ir::rtype::RType; + +use crate::liveness; + +pub(crate) fn run(module: &mut ModuleIr) { + for function in module.all_functions_mut() { + release_temporaries(function); + } +} + +/// one reference this pass can let go of on its own: a whole register, or one element +/// of a fixed-length tuple a register holds as a struct, as a path of elements +/// outermost first +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +struct Place { + register: RegisterId, + path: Vec, +} + +/// the elements of one register go last to first, so that where several are let go of at +/// one point it is in the order a `tuple` drops its elements in +impl Ord for Place { + fn cmp(&self, other: &Self) -> Ordering { + (self.register, Reverse(&self.path)).cmp(&(other.register, Reverse(&other.path))) + } +} + +impl PartialOrd for Place { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +/// every reference a value of `ty` holds, as the path to each: one per element of a +/// fixed-length tuple, into nested ones too, and otherwise the value itself +fn leaves(ty: &RType, prefix: &mut Vec, out: &mut Vec>) { + match ty { + RType::Tuple(items) => { + for (index, item) in items.iter().enumerate() { + prefix.push(index); + leaves(item, prefix, out); + prefix.pop(); + } + } + _ => out.push(prefix.clone()), + } +} + +/// every place under `prefix` in `register`'s value +fn places_under(function: &Function, register: RegisterId, prefix: &[usize]) -> Vec { + let Some(ty) = function + .register(register) + .and_then(|decl| decl.ty.element(prefix)) + else { + return Vec::new(); + }; + let mut paths = Vec::new(); + leaves(ty, &mut prefix.to_vec(), &mut paths); + paths + .into_iter() + .map(|path| Place { register, path }) + .collect() +} + +/// every place a register's value is made of +fn places_of(function: &Function, register: RegisterId) -> Vec { + places_under(function, register, &[]) +} + +/// the places an operation reads: an element read off a tuple reads that element and +/// nothing else of it, and any other read of a register reads all of it +fn read_places(function: &Function, op: &Op) -> Vec { + if let Op::TupleGet { + src: Value::Register(src), + index, + .. + } = op + { + return places_under(function, *src, &[*index]); + } + liveness::read_registers(op) + .into_iter() + .flat_map(|register| places_of(function, register)) + .collect() +} + +fn release_temporaries(function: &mut Function) { + let candidates = candidates(function); + if candidates.is_empty() { + return; + } + // a terminator never reads a borrowed register — returning one would hand out a + // reference the frame does not own, and the borrow pass refuses it — so the + // lending below only has to be followed through operations. a function where that + // does not hold is left as it was rather than reasoned about + let borrowed: HashSet = function + .registers + .iter() + .enumerate() + .filter(|(_, decl)| decl.borrowed) + .map(|(index, _)| RegisterId(index)) + .collect(); + if function.blocks.iter().any(|block| { + block + .terminator + .operands() + .into_iter() + .filter_map(liveness::register) + .any(|register| borrowed.contains(®ister)) + }) { + return; + } + + let lenders = lenders(function, &borrowed); + let reads = |op: &Op| through_lenders(function, read_places(function, op), &lenders); + let whole = |register: RegisterId| places_of(function, register); + let live_in = liveness::live_in_by(function, &reads, &whole); + + let mut releases: BTreeSet<(BlockId, usize, Place)> = BTreeSet::new(); + for (index, block) in function.blocks.iter().enumerate() { + let id = BlockId(index); + let windows = borrow_windows(block, &borrowed); + let error_live = liveness::error_live(block, &live_in); + let mut live = liveness::live_out_by(block, &live_in, &whole); + for (position, op) in block.ops.iter().enumerate().rev() { + // an exception leaves from the middle of the block, so what the handler + // reads is live after every operation in it + live.extend(error_live.iter().cloned()); + let touched = reads(op) + .into_iter() + .chain(op.dest().into_iter().flat_map(whole)); + for place in touched { + if candidates.contains(&place) && !live.contains(&place) { + releases.insert((id, clear_of(&windows, position + 1), place)); + } + } + if let Some(dest) = op.dest() { + for place in whole(dest) { + live.remove(&place); + } + } + live.extend(reads(op)); + } + } + for (id, place, at) in entered_dead(function, &live_in, &candidates) { + if let Some(block) = function.block(id) { + let windows = borrow_windows(block, &borrowed); + releases.insert((id, clear_of(&windows, at), place)); + } + } + + // inserted from the back of each block, so an earlier position still means what it + // meant when it was chosen + for (id, at, place) in releases.into_iter().rev() { + if let Some(block) = function.blocks.get_mut(id.index()) { + let at = at.min(block.ops.len()); + block.ops.insert( + at, + Op::Release { + value: Value::Register(place.register), + path: place.path.into(), + }, + ); + } + } +} + +/// the places this pass may release: those of temporaries that own one reference apiece +/// +/// a tagged `int` is refcounted as well, but an `int` has no finalizer to run, so it is +/// left for the frame to let go of, inside a tuple as much as on its own +fn candidates(function: &Function) -> HashSet { + // a loop's cursor is read and written by the operation that advances it, outside + // the operands this pass follows, so it is never offered + let cursors: HashSet = function + .blocks + .iter() + .flat_map(|block| &block.ops) + .filter_map(Op::loop_cursor) + .collect(); + (function.param_count..function.registers.len()) + .map(RegisterId) + .filter(|register| !cursors.contains(register)) + .filter_map(|register| Some((register, function.register(register)?))) + .filter(|(_, decl)| decl.name.is_none() && !decl.borrowed) + .flat_map(|(register, decl)| { + places_of(function, register).into_iter().filter(|place| { + decl.ty + .element(&place.path) + .is_some_and(RType::is_object_reference) + }) + }) + .collect() +} + +/// where a borrowed register's value is on loan from +enum Loan { + /// the value at `at` inside `of`, which a place inside the borrower is a place + /// inside too: a copy of a register, or an element read off a tuple + View { of: RegisterId, at: Vec }, + /// somewhere among these + Among(Vec), +} + +/// where each borrowed register could be holding its value on loan from +/// +/// every write of it is followed, which over-counts rather than under-counts: a lender +/// held a little longer costs nothing, and one let go early is a value freed under a +/// register still reading it. an element read off a tuple borrows from that element +/// alone, which is what lets the rest of the tuple go +fn lenders(function: &Function, borrowed: &HashSet) -> HashMap> { + let mut lenders: HashMap> = HashMap::new(); + for op in function.blocks.iter().flat_map(|block| &block.ops) { + let Some(dest) = op.dest().filter(|dest| borrowed.contains(dest)) else { + continue; + }; + let loan = match op { + Op::TupleGet { + src: Value::Register(src), + index, + .. + } => Loan::View { + of: *src, + at: vec![*index], + }, + Op::Assign { + src: Value::Register(src), + .. + } => Loan::View { + of: *src, + at: Vec::new(), + }, + other => Loan::Among(read_places(function, other)), + }; + lenders.entry(dest).or_default().push(loan); + } + lenders +} + +/// `reads`, with every place a borrowed register among them could be lending from +/// +/// a place inside a view is a place inside what it views, where the two agree on the +/// layout. where they do not, the whole of the lender is read +fn through_lenders( + function: &Function, + reads: Vec, + lenders: &HashMap>, +) -> Vec { + let mut seen: HashSet = HashSet::new(); + let mut pending = reads; + while let Some(place) = pending.pop() { + if seen.contains(&place) { + continue; + } + for loan in lenders.get(&place.register).into_iter().flatten() { + match loan { + Loan::View { of, at } => { + let path: Vec = at.iter().chain(&place.path).copied().collect(); + let inside = function + .register(*of) + .and_then(|decl| decl.ty.element(&path)) + .is_some(); + if inside { + pending.push(Place { + register: *of, + path, + }); + } else { + pending.extend(places_of(function, *of)); + } + } + Loan::Among(places) => pending.extend(places.iter().cloned()), + } + } + seen.insert(place); + } + seen.into_iter().collect() +} + +/// the stretches of a block no release may be placed inside, as `(write, last read)` +/// +/// a release placed before operation `at` is inside a window when `write < at <= last +/// read`. a borrowed register the block reads without writing is held from the +/// block's start +fn borrow_windows( + block: &BasicBlock, + borrowed: &HashSet, +) -> Vec<(Option, usize)> { + borrowed + .iter() + .filter_map(|register| { + let reads = |op: &Op| { + op.operands() + .into_iter() + .filter_map(liveness::register) + .any(|read| read == *register) + }; + let last_read = block.ops.iter().rposition(reads)?; + let write = block.ops.iter().position(|op| op.dest() == Some(*register)); + Some((write.filter(|write| *write < last_read), last_read)) + }) + .collect() +} + +/// the first position at or after `at` that no window covers +fn clear_of(windows: &[(Option, usize)], mut at: usize) -> usize { + while let Some(&(_, last_read)) = windows + .iter() + .find(|(write, last_read)| write.is_none_or(|write| write < at) && at <= *last_read) + { + at = last_read + 1; + } + at +} + +/// the candidates a block is entered holding and does not need, with where to drop them +/// +/// a place one successor goes on to read and another does not is still holding a +/// value on the edge to the second. a handler is entered holding whatever the block it +/// covers had written, and it drops them once it has taken the exception: a finalizer +/// run while an exception is still pending would find it there +fn entered_dead( + function: &Function, + live_in: &[HashSet], + candidates: &HashSet, +) -> Vec<(BlockId, Place, usize)> { + let whole = |register: RegisterId| places_of(function, register); + let mut holding: Vec> = vec![HashSet::new(); function.blocks.len()]; + for block in &function.blocks { + let out = liveness::live_out_by(block, live_in, &whole); + for successor in block.successors() { + if let Some(slot) = holding.get_mut(successor.index()) { + slot.extend(out.iter().cloned()); + } + } + if let Some(handler) = block.error_target + && let Some(slot) = holding.get_mut(handler.index()) + { + slot.extend(block.ops.iter().flat_map(|op| { + read_places(function, op) + .into_iter() + .chain(op.dest().into_iter().flat_map(whole)) + })); + } + } + + let handlers: HashSet = function + .blocks + .iter() + .filter_map(|block| block.error_target) + .collect(); + let mut out = Vec::new(); + for (index, held) in holding.iter().enumerate() { + let id = BlockId(index); + let Some(block) = function.block(id) else { + continue; + }; + let at = if handlers.contains(&id) { + match block.ops.first() { + Some(Op::FetchException { .. }) => 1, + // a handler that does not begin by taking the exception is left alone + _ => continue, + } + } else { + 0 + }; + let needed = live_in.get(index); + let mut dropped: Vec = held + .iter() + .filter(|place| candidates.contains(place)) + .filter(|place| needed.is_none_or(|needed| !needed.contains(place))) + .cloned() + .collect(); + dropped.sort_unstable(); + out.extend(dropped.into_iter().map(|place| (id, place, at))); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + use by_ir::builder::FunctionBuilder; + use by_ir::ops::Terminator; + use by_ir::rtype::RType; + use by_ir::verify::verify; + + fn module(function: Function) -> ModuleIr { + ModuleIr { + name: by_ir::ModuleName::new("app"), + functions: vec![function], + declined: Vec::new(), + classes: Vec::new(), + gradual: Vec::new(), + promoted: Vec::new(), + lines: None, + fallback_source: None, + fallback_code: None, + shims: None, + verify_install: true, + } + } + + fn call(dest: RegisterId, callee: &str, arg: RegisterId) -> Op { + Op::CallPython { + dest, + callee: callee.to_string(), + args: vec![Value::Register(arg)], + } + } + + /// where in `block` each release sits, and of which register + fn releases(module: &ModuleIr, block: usize) -> Vec<(usize, RegisterId)> { + module.functions[0].blocks[block] + .ops + .iter() + .enumerate() + .filter_map(|(at, op)| match op { + Op::Release { + value: Value::Register(id), + path, + } if path.is_empty() => Some((at, *id)), + _ => None, + }) + .collect() + } + + /// every release in `block`: where it sits, of which register, and of which element + fn every_release(module: &ModuleIr, block: usize) -> Vec<(usize, RegisterId, Vec)> { + module.functions[0].blocks[block] + .ops + .iter() + .enumerate() + .filter_map(|(at, op)| match op { + Op::Release { + value: Value::Register(id), + path, + } => Some((at, *id, path.to_vec())), + _ => None, + }) + .collect() + } + + /// the layout `pair(i) -> tuple[Loud, Loud]` answers with + fn pair() -> RType { + RType::Tuple(Box::new([RType::OBJECT, RType::OBJECT])) + } + + /// a tuple of `items`, which retains each + fn build(dest: RegisterId, items: &[RegisterId]) -> Op { + Op::TupleBuild { + dest, + items: items.iter().copied().map(Value::Register).collect(), + } + } + + #[test] + fn a_tuple_nothing_reads_drops_its_elements_last_to_first() { + let mut builder = FunctionBuilder::new("f", RType::OBJECT); + let p = builder.param("p", RType::OBJECT); + let made = builder.temp(pair()); + builder.push(build(made, &[p, p])); + builder.terminate(Terminator::Return(Value::Register(p))); + + let mut module = module(builder.finish()); + run(&mut module); + + assert_eq!( + every_release(&module, 0), + vec![(1, made, vec![1]), (2, made, vec![0])], + "{:?}", + module.functions[0].blocks + ); + assert_eq!(verify(&module.functions[0]), Ok(())); + } + + #[test] + fn a_name_given_one_element_holds_a_reference_of_its_own() { + // `part = pair(p)[0]`: python drops the tuple as soon as the subscript is done, + // and the name goes on holding the element it was given. so the tuple lets go + // of the element nobody reads once it is built, and of the named one once it + // has been read + let mut builder = FunctionBuilder::new("f", RType::OBJECT); + let p = builder.param("p", RType::OBJECT); + let made = builder.temp(pair()); + let part = builder.local("part".to_string(), RType::OBJECT); + builder.push(build(made, &[p, p])); + builder.push(Op::TupleGet { + dest: part, + src: Value::Register(made), + index: 0, + }); + builder.terminate(Terminator::Return(Value::Register(part))); + + let mut module = module(builder.finish()); + run(&mut module); + + assert_eq!( + every_release(&module, 0), + vec![(1, made, vec![1]), (3, made, vec![0])], + "{:?}", + module.functions[0].blocks + ); + assert_eq!(verify(&module.functions[0]), Ok(())); + } + + #[test] + fn an_element_on_loan_is_held_until_the_borrower_is_done() { + let mut builder = FunctionBuilder::new("f", RType::OBJECT); + let p = builder.param("p", RType::OBJECT); + let made = builder.temp(pair()); + let first = builder.temp(RType::OBJECT); + let answer = builder.temp(RType::OBJECT); + builder.push(build(made, &[p, p])); + builder.push(Op::TupleGet { + dest: first, + src: Value::Register(made), + index: 0, + }); + builder.push(call(answer, "len", first)); + builder.terminate(Terminator::Return(Value::Register(answer))); + let mut function = builder.finish(); + function.registers[first.index()].borrowed = true; + + let mut module = module(function); + run(&mut module); + + // the element nobody reads goes as soon as the tuple is made, and the one on + // loan once the call reading it is done. the borrower owns nothing to let go of + assert_eq!( + every_release(&module, 0), + vec![(1, made, vec![1]), (4, made, vec![0])], + "{:?}", + module.functions[0].blocks + ); + assert_eq!(verify(&module.functions[0]), Ok(())); + } + + #[test] + fn a_tuple_on_loan_owns_nothing_to_let_go_of() { + // `nested(p)[0][1]`: the inner tuple is read off the outer one on loan, so its + // elements are the outer tuple's references. only the outer tuple lets go, and + // what the element read off the loan goes on reading is one element deep + let mut builder = FunctionBuilder::new("f", RType::OBJECT); + let p = builder.param("p", RType::OBJECT); + let inner = builder.temp(pair()); + let outer = builder.temp(RType::Tuple(Box::new([pair(), RType::OBJECT]))); + let lent = builder.temp(pair()); + let answer = builder.temp(RType::OBJECT); + let shown = builder.temp(RType::OBJECT); + builder.push(build(inner, &[p, p])); + builder.push(build(outer, &[inner, p])); + builder.push(Op::TupleGet { + dest: lent, + src: Value::Register(outer), + index: 0, + }); + builder.push(Op::TupleGet { + dest: answer, + src: Value::Register(lent), + index: 1, + }); + builder.push(call(shown, "str", answer)); + builder.terminate(Terminator::Return(Value::Register(shown))); + let mut function = builder.finish(); + function.registers[lent.index()].borrowed = true; + function.registers[answer.index()].borrowed = true; + + let mut module = module(function); + run(&mut module); + + // the inner tuple's first element is read by nothing past the loan, but no + // release goes inside a loan's window, so it waits for the call that reads its + // second, and the two then go last to first as the inner tuple would drop them + assert_eq!( + every_release(&module, 0), + vec![ + (2, inner, vec![1]), + (3, inner, vec![0]), + (4, outer, vec![1]), + (8, outer, vec![0, 1]), + (9, outer, vec![0, 0]), + ], + "{:?}", + module.functions[0].blocks + ); + assert_eq!(verify(&module.functions[0]), Ok(())); + } + + #[test] + fn a_temporary_is_released_after_its_last_read() { + let mut builder = FunctionBuilder::new("f", RType::OBJECT); + let p = builder.param("p", RType::OBJECT); + let made = builder.temp(RType::OBJECT); + let answer = builder.temp(RType::OBJECT); + builder.push(call(made, "repr", p)); + builder.push(call(answer, "len", made)); + builder.terminate(Terminator::Return(Value::Register(answer))); + + let mut module = module(builder.finish()); + run(&mut module); + + // the returned value is read by the return itself, so it is not released + assert_eq!( + releases(&module, 0), + vec![(2, made)], + "{:?}", + module.functions[0].blocks + ); + assert_eq!(verify(&module.functions[0]), Ok(())); + } + + #[test] + fn a_named_local_is_held_to_the_end() { + let mut builder = FunctionBuilder::new("f", RType::OBJECT); + let p = builder.param("p", RType::OBJECT); + let made = builder.local("made".to_string(), RType::OBJECT); + let answer = builder.temp(RType::OBJECT); + builder.push(call(made, "repr", p)); + builder.push(call(answer, "len", made)); + builder.terminate(Terminator::Return(Value::Register(answer))); + + let mut module = module(builder.finish()); + run(&mut module); + + assert_eq!(releases(&module, 0), Vec::new()); + } + + #[test] + fn a_lender_is_held_until_its_borrower_is_done() { + let mut builder = FunctionBuilder::new("f", RType::OBJECT); + let p = builder.param("p", RType::OBJECT); + let made = builder.temp(RType::OBJECT); + let copy = builder.temp(RType::OBJECT); + let answer = builder.temp(RType::OBJECT); + builder.push(call(made, "repr", p)); + builder.assign(copy, Value::Register(made)); + builder.push(call(answer, "len", copy)); + builder.terminate(Terminator::Return(Value::Register(answer))); + let mut function = builder.finish(); + function.registers[copy.index()].borrowed = true; + + let mut module = module(function); + run(&mut module); + + // `made` is last read directly by the copy, but the copy reads through it, so + // it goes after the copy's own last read — and the copy owns nothing to let go + assert_eq!( + releases(&module, 0), + vec![(3, made)], + "{:?}", + module.functions[0].blocks + ); + assert_eq!(verify(&module.functions[0]), Ok(())); + } + + #[test] + fn no_release_lands_inside_a_borrow_window() { + let mut builder = FunctionBuilder::new("f", RType::OBJECT); + let p = builder.param("p", RType::OBJECT); + let early = builder.temp(RType::OBJECT); + let lent = builder.temp(RType::OBJECT); + let copy = builder.temp(RType::OBJECT); + let shown = builder.temp(RType::OBJECT); + let answer = builder.temp(RType::OBJECT); + builder.push(call(early, "repr", p)); + builder.push(call(lent, "str", p)); + builder.assign(copy, Value::Register(lent)); + // the last read of `early`, inside the window the copy is borrowing across + builder.push(call(shown, "str", early)); + builder.push(call(answer, "len", copy)); + builder.terminate(Terminator::Return(Value::Register(answer))); + let mut function = builder.finish(); + function.registers[copy.index()].borrowed = true; + + let mut module = module(function); + run(&mut module); + + // both wait for the window to close after the copy's last read, at 4 + let placed = releases(&module, 0); + assert!( + placed.iter().all(|(at, _)| *at > 4), + "{placed:?} in {:?}", + module.functions[0].blocks + ); + assert!(placed.iter().any(|(_, id)| *id == early), "{placed:?}"); + assert!(placed.iter().any(|(_, id)| *id == lent), "{placed:?}"); + assert!(placed.iter().any(|(_, id)| *id == shown), "{placed:?}"); + assert_eq!(verify(&module.functions[0]), Ok(())); + } + + #[test] + fn a_value_one_successor_needs_is_dropped_entering_the_other() { + let mut builder = FunctionBuilder::new("f", RType::OBJECT); + let p = builder.param("p", RType::OBJECT); + let cond = builder.param("c", RType::BIT); + let made = builder.temp(RType::OBJECT); + let answer = builder.temp(RType::OBJECT); + let reads = builder.new_block(); + let skips = builder.new_block(); + builder.push(call(made, "repr", p)); + builder.terminate(Terminator::Branch { + cond: Value::Register(cond), + then_block: reads, + else_block: skips, + }); + builder.switch_to(reads); + builder.push(call(answer, "len", made)); + builder.terminate(Terminator::Return(Value::Register(answer))); + builder.switch_to(skips); + builder.terminate(Terminator::Return(Value::Register(p))); + + let mut module = module(builder.finish()); + run(&mut module); + + assert_eq!(releases(&module, reads.index()), vec![(1, made)]); + assert_eq!(releases(&module, skips.index()), vec![(0, made)]); + assert_eq!(verify(&module.functions[0]), Ok(())); + } +} diff --git a/crates/by_opt/src/store_moves.rs b/crates/by_opt/src/store_moves.rs new file mode 100644 index 0000000000..89a7ff7ee5 --- /dev/null +++ b/crates/by_opt/src/store_moves.rs @@ -0,0 +1,488 @@ +//! store into a field by handing the register's reference over, where the store is +//! the last thing that reads it +//! +//! a field store copies: it retains the value for the field, and the register goes on +//! owning its own reference until something writes over it or the frame returns. when +//! nothing reads the register again that is a retain and a release around nothing — +//! `self.depth = self.depth + 1` pays both on every call. so a store that is its value's +//! last use takes the register's reference instead, and leaves the register empty, which +//! every later release of it then finds +//! +//! ## what is left alone +//! +//! - a **parameter**, whose reference the caller owns rather than the frame +//! - a **borrowed** register, which owns nothing to hand over +//! - a value a **name** holds: python keeps a named value alive until the name is +//! rebound or the frame returns, so the field is not the only thing holding it, and +//! rewriting the field later must not be what frees it +//! - a value a handler could read: the store leaves the register empty, and a handler +//! entered from later in the block would find nothing there +//! +//! the release pass has already put a release straight after such a store — the store +//! was the value's last read — and that release is what the move replaces, so it goes +//! +//! ## a copy into another register moves the same way +//! +//! `held = Cell(i)` builds the value into a temporary and copies it into the name, and +//! `held, part = split(i)` reads each element off the tuple the call answered. the name +//! owns what it holds, so python can let go of it when the name is rebound — and the +//! temporary is let go of straight after the copy, which is a retain and a release +//! around nothing. so a copy or an element read followed by the release of the place it +//! read hands that place's reference over instead +//! +//! the release may sit a little after the store, past operations the release pass would +//! not put it inside. the move empties the place early, which nothing reads, and holds +//! the value in the destination instead — so the scan from the store to the release +//! gives up at anything that reads the place, writes or deletes its register, or writes +//! or deletes the destination, which would let go of the value in the meantime + +use by_ir::function::{Function, ModuleIr}; +use by_ir::ops::{BlockId, Op, RegisterId, Value}; +use by_ir::rtype::RType; + +use crate::liveness; + +pub(crate) fn run(module: &mut ModuleIr) { + for function in module.all_functions_mut() { + move_dying_stores(function); + } +} + +fn move_dying_stores(function: &mut Function) { + // a release of the stored register reads nothing: the move is what it would have + // been doing, and counting it as a read would keep every store from qualifying + let reads = |op: &Op| match op { + Op::Release { .. } => Vec::new(), + other => liveness::read_registers(other), + }; + let live_in = liveness::live_in(function, &reads); + + let mut moved: Vec<(BlockId, usize, RegisterId)> = Vec::new(); + for (index, block) in function.blocks.iter().enumerate() { + let error_live = liveness::error_live(block, &live_in); + let mut live = liveness::live_out(block, &live_in); + for (position, op) in block.ops.iter().enumerate().rev() { + live.extend(error_live.iter().copied()); + if let Op::SetField { + receiver, + value: Value::Register(value), + .. + } = op + && !live.contains(value) + && !error_live.contains(value) + && receiver != &Value::Register(*value) + && may_hand_over(function, *value) + { + moved.push((BlockId(index), position, *value)); + } + if let Some(dest) = op.dest() { + live.remove(&dest); + } + live.extend(reads(op)); + } + } + + for (block, position, value) in moved { + let Some(block) = function.blocks.get_mut(block.index()) else { + continue; + }; + if let Some(Op::SetField { moves, .. }) = block.ops.get_mut(position) { + *moves = true; + } + // the release the store's being the last read put here is now the move's job + let released = block.ops[position + 1..].iter().position(|op| { + matches!( + op, + Op::Release { value: Value::Register(id), path } if *id == value && path.is_empty() + ) + }); + if let Some(offset) = released { + block.ops.remove(position + 1 + offset); + } + } + + // after the field stores, whose positions were found before any release went + move_dying_copies(function); +} + +/// turn each copy or element read whose source place is released after it, with +/// nothing touching either in between, into a move +fn move_dying_copies(function: &mut Function) { + let mut moved: Vec<(usize, usize, usize)> = Vec::new(); + for (index, block) in function.blocks.iter().enumerate() { + for (position, op) in block.ops.iter().enumerate() { + let (dest, source, path) = match op { + Op::Assign { + dest, + src: Value::Register(source), + } => (*dest, *source, Vec::new()), + Op::TupleGet { + dest, + src: Value::Register(source), + index, + } => (*dest, *source, vec![*index]), + _ => continue, + }; + // a widening copy changes the representation the reference is held in, + // which a move does not + let same_type = function + .register(source) + .and_then(|decl| decl.ty.element(&path)) + .zip(function.register(dest)) + .is_some_and(|(element, decl)| *element == decl.ty); + if dest == source + || !same_type + || function.register(dest).is_none_or(|decl| decl.borrowed) + || !owns_place(function, source, &path) + { + continue; + } + let touches = |later: &Op| { + later.dest() == Some(source) + || later.unbinds() == Some(source) + || later.dest() == Some(dest) + || later.unbinds() == Some(dest) + || reads_place(later, source, &path) + }; + let Some(offset) = block.ops[position + 1..] + .iter() + .position(|later| touches(later) || releases_place(later, source, &path)) + else { + continue; + }; + let at = position + 1 + offset; + if releases_place(&block.ops[at], source, &path) { + moved.push((index, position, at)); + } + } + } + + for (index, position, _) in &moved { + let Some(op) = function + .blocks + .get_mut(*index) + .and_then(|block| block.ops.get_mut(*position)) + else { + continue; + }; + let replacement = match op { + Op::Assign { dest, src } => Op::Move { + dest: *dest, + src: src.clone(), + path: Box::default(), + }, + Op::TupleGet { dest, src, index } => Op::Move { + dest: *dest, + src: src.clone(), + path: Box::new([*index]), + }, + _ => continue, + }; + *op = replacement; + } + // the releases go last and back to front, so removing one cannot move another + let mut releases: Vec<(usize, usize)> = moved + .into_iter() + .map(|(index, _, release)| (index, release)) + .collect(); + releases.sort_unstable_by(|a, b| b.cmp(a)); + for (index, release) in releases { + if let Some(block) = function.blocks.get_mut(index) + && release < block.ops.len() + { + block.ops.remove(release); + } + } +} + +/// whether `register` owns the one reference at `path` inside it, which a move may take +fn owns_place(function: &Function, register: RegisterId, path: &[usize]) -> bool { + register.index() >= function.param_count + && function.register(register).is_some_and(|decl| { + !decl.borrowed + && decl.name.is_none() + && decl + .ty + .element(path) + .is_some_and(|ty| *ty == RType::INT || ty.is_object_reference()) + }) +} + +/// whether `op` is the release of exactly the place at `path` inside `register` +fn releases_place(op: &Op, register: RegisterId, path: &[usize]) -> bool { + matches!( + op, + Op::Release { value: Value::Register(id), path: released } + if *id == register && **released == *path + ) +} + +/// whether `op` reads anything at or around the place at `path` inside `register` +/// +/// an element read reads its own element and nothing else of the tuple, and a release +/// of a different element reads nothing +fn reads_place(op: &Op, register: RegisterId, path: &[usize]) -> bool { + let overlaps = |other: &[usize]| other.starts_with(path) || path.starts_with(other); + match op { + Op::TupleGet { + src: Value::Register(id), + index, + .. + } if *id == register => overlaps(&[*index]), + Op::Release { + value: Value::Register(id), + path: released, + } if *id == register => overlaps(released), + other => liveness::read_registers(other).contains(®ister), + } +} + +/// whether the register owns a reference of its own to give the field +fn may_hand_over(function: &Function, register: RegisterId) -> bool { + if register.index() < function.param_count { + return false; + } + function.register(register).is_some_and(|decl| { + !decl.borrowed + && decl.name.is_none() + && (decl.ty == RType::INT || decl.ty.is_object_reference()) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use by_ir::builder::FunctionBuilder; + use by_ir::ops::{BinOp, Terminator}; + + fn module(function: Function) -> ModuleIr { + ModuleIr { + name: by_ir::ModuleName::new("app"), + functions: vec![function], + declined: Vec::new(), + classes: Vec::new(), + gradual: Vec::new(), + promoted: Vec::new(), + lines: None, + fallback_source: None, + fallback_code: None, + shims: None, + verify_install: true, + } + } + + fn moves(module: &ModuleIr) -> Vec { + module.functions[0] + .blocks + .iter() + .flat_map(|block| &block.ops) + .filter_map(|op| match op { + Op::SetField { moves, .. } => Some(*moves), + _ => None, + }) + .collect() + } + + fn store(receiver: RegisterId, value: RegisterId) -> Op { + Op::SetField { + receiver: Value::Register(receiver), + class: "C".to_string(), + field: "depth".to_string(), + value: Value::Register(value), + moves: false, + present: false, + } + } + + #[test] + fn a_store_that_is_its_value_s_last_read_hands_the_reference_over() { + let mut builder = FunctionBuilder::new("f", RType::NONE); + let receiver = builder.param("self", RType::OBJECT); + let k = builder.param("k", RType::INT); + let sum = builder.temp(RType::INT); + builder.push(Op::IntBinary { + dest: sum, + op: BinOp::Add, + lhs: Value::Register(k), + rhs: Value::Int(1), + }); + builder.push(store(receiver, sum)); + builder.terminate(Terminator::Return(Value::None)); + + let mut module = module(builder.finish()); + run(&mut module); + + assert_eq!(moves(&module), vec![true]); + } + + #[test] + fn a_value_read_again_is_copied() { + let mut builder = FunctionBuilder::new("f", RType::INT); + let receiver = builder.param("self", RType::OBJECT); + let k = builder.param("k", RType::INT); + let sum = builder.temp(RType::INT); + builder.push(Op::IntBinary { + dest: sum, + op: BinOp::Add, + lhs: Value::Register(k), + rhs: Value::Int(1), + }); + builder.push(store(receiver, sum)); + builder.terminate(Terminator::Return(Value::Register(sum))); + + let mut module = module(builder.finish()); + run(&mut module); + + assert_eq!(moves(&module), vec![false]); + } + + #[test] + fn a_parameter_or_a_named_value_is_copied() { + let mut builder = FunctionBuilder::new("f", RType::NONE); + let receiver = builder.param("self", RType::OBJECT); + let k = builder.param("k", RType::INT); + let named = builder.local("n".to_string(), RType::INT); + builder.push(store(receiver, k)); + builder.push(Op::IntBinary { + dest: named, + op: BinOp::Add, + lhs: Value::Register(k), + rhs: Value::Int(1), + }); + builder.push(store(receiver, named)); + builder.terminate(Terminator::Return(Value::None)); + + let mut module = module(builder.finish()); + run(&mut module); + + assert_eq!(moves(&module), vec![false, false]); + } + + fn call(dest: RegisterId, arg: RegisterId) -> Op { + Op::CallPython { + dest, + callee: "make".to_string(), + args: vec![Value::Register(arg)], + } + } + + fn release(register: RegisterId, path: &[usize]) -> Op { + Op::Release { + value: Value::Register(register), + path: path.into(), + } + } + + #[test] + fn a_copy_released_straight_after_hands_its_reference_over() { + // `held = make(p)` arrives as a call into a temporary, a copy into the name, and + // the release of the temporary the copy was the last read of + let mut builder = FunctionBuilder::new("f", RType::OBJECT); + let p = builder.param("p", RType::OBJECT); + let made = builder.temp(RType::OBJECT); + let held = builder.local("held".to_string(), RType::OBJECT); + builder.push(call(made, p)); + builder.assign(held, Value::Register(made)); + builder.push(release(made, &[])); + builder.terminate(Terminator::Return(Value::Register(p))); + + let mut module = module(builder.finish()); + run(&mut module); + + let ops = &module.functions[0].blocks[0].ops; + assert_eq!(ops.len(), 2, "{ops:?}"); + assert_eq!( + ops[1], + Op::Move { + dest: held, + src: Value::Register(made), + path: Box::default(), + } + ); + assert_eq!(by_ir::verify::verify(&module.functions[0]), Ok(())); + } + + #[test] + fn an_element_read_released_after_hands_that_element_over() { + // `head, tail = split(p)`: each element is read off the tuple and let go of, and + // the release of the other element in between reads nothing of this one + let mut builder = FunctionBuilder::new("f", RType::OBJECT); + let p = builder.param("p", RType::OBJECT); + let pair = builder.temp(RType::Tuple(Box::new([RType::OBJECT, RType::OBJECT]))); + let head = builder.local("head".to_string(), RType::OBJECT); + builder.push(Op::TupleBuild { + dest: pair, + items: vec![Value::Register(p), Value::Register(p)], + }); + builder.push(release(pair, &[1])); + builder.push(Op::TupleGet { + dest: head, + src: Value::Register(pair), + index: 0, + }); + builder.push(release(pair, &[0])); + builder.terminate(Terminator::Return(Value::Register(head))); + + let mut module = module(builder.finish()); + run(&mut module); + + let ops = &module.functions[0].blocks[0].ops; + assert_eq!(ops.len(), 3, "{ops:?}"); + assert_eq!( + ops[2], + Op::Move { + dest: head, + src: Value::Register(pair), + path: Box::new([0]), + } + ); + assert_eq!(by_ir::verify::verify(&module.functions[0]), Ok(())); + } + + #[test] + fn a_copy_whose_destination_is_written_before_the_release_is_left_alone() { + // the second write lets go of what the first one stored, and a move would have + // left nothing else holding it until the release + let mut builder = FunctionBuilder::new("f", RType::OBJECT); + let p = builder.param("p", RType::OBJECT); + let made = builder.temp(RType::OBJECT); + let held = builder.local("held".to_string(), RType::OBJECT); + builder.push(call(made, p)); + builder.assign(held, Value::Register(made)); + builder.assign(held, Value::Register(p)); + builder.push(release(made, &[])); + builder.terminate(Terminator::Return(Value::Register(p))); + + let mut module = module(builder.finish()); + run(&mut module); + + let ops = &module.functions[0].blocks[0].ops; + assert!( + !ops.iter().any(|op| matches!(op, Op::Move { .. })), + "{ops:?}" + ); + } + + #[test] + fn a_copy_read_again_before_the_release_is_left_alone() { + let mut builder = FunctionBuilder::new("f", RType::OBJECT); + let p = builder.param("p", RType::OBJECT); + let made = builder.temp(RType::OBJECT); + let held = builder.local("held".to_string(), RType::OBJECT); + let again = builder.temp(RType::OBJECT); + builder.push(call(made, p)); + builder.assign(held, Value::Register(made)); + builder.push(call(again, made)); + builder.push(release(made, &[])); + builder.terminate(Terminator::Return(Value::Register(again))); + + let mut module = module(builder.finish()); + run(&mut module); + + let ops = &module.functions[0].blocks[0].ops; + assert!( + !ops.iter().any(|op| matches!(op, Op::Move { .. })), + "{ops:?}" + ); + } +} diff --git a/crates/by_opt/src/str_append.rs b/crates/by_opt/src/str_append.rs index 9c215318ca..2982f24b69 100644 --- a/crates/by_opt/src/str_append.rs +++ b/crates/by_opt/src/str_append.rs @@ -34,11 +34,11 @@ //! resizing. the runtime helper refuses this case too, so the hazard does not //! depend on this pass being right about it -use std::collections::HashSet; - use by_ir::function::{Function, ModuleIr}; use by_ir::ops::{BlockId, Op, RegisterId, Value}; +use crate::liveness; + pub(crate) fn run(module: &mut ModuleIr) { for function in module.all_functions_mut() { consume_dying_operands(function); @@ -46,23 +46,12 @@ pub(crate) fn run(module: &mut ModuleIr) { } fn consume_dying_operands(function: &mut Function) { - let live_in = live_in_sets(function); + let live_in = liveness::live_in(function, &liveness::read_registers); let mut consumed: Vec<(BlockId, usize)> = Vec::new(); for (index, block) in function.blocks.iter().enumerate() { - let error_live = block - .error_target - .and_then(|target| live_in.get(target.index())) - .cloned() - .unwrap_or_default(); - let mut live: HashSet = block - .successors() - .iter() - .filter_map(|successor| live_in.get(successor.index())) - .flatten() - .copied() - .collect(); - live.extend(block.terminator.operands().into_iter().filter_map(register)); + let error_live = liveness::error_live(block, &live_in); + let mut live = liveness::live_out(block, &live_in); for (position, op) in block.ops.iter().enumerate().rev() { // an exception leaves from the middle of the block, so what the handler @@ -91,11 +80,7 @@ fn consume_dying_operands(function: &mut Function) { if let Some(dest) = op.dest() { live.remove(&dest); } - // `del x` leaves its destination unbound, but it reads the value first — - // to release it — so the reference is still needed here and must not be - // handed to an append below - live.extend(op.unbinds()); - live.extend(op.operands().into_iter().filter_map(register)); + live.extend(liveness::read_registers(op)); } } @@ -120,64 +105,11 @@ fn may_hand_over(function: &Function, register: RegisterId) -> bool { .is_some_and(|decl| !decl.borrowed) } -/// the registers live on entry to each block -fn live_in_sets(function: &Function) -> Vec> { - let mut live_in: Vec> = vec![HashSet::new(); function.blocks.len()]; - - let mut changed = true; - while changed { - changed = false; - for index in (0..function.blocks.len()).rev() { - let Some(block) = function.block(BlockId(index)) else { - continue; - }; - let mut live: HashSet = block - .successors() - .iter() - .filter_map(|successor| live_in.get(successor.index())) - .flatten() - .copied() - .collect(); - live.extend(block.terminator.operands().into_iter().filter_map(register)); - - // an exception leaves from the middle of the block, so the handler's - // needs survive every kill below it - let error_live = block - .error_target - .and_then(|target| live_in.get(target.index())) - .cloned() - .unwrap_or_default(); - for op in block.ops.iter().rev() { - live.extend(error_live.iter().copied()); - if let Some(dest) = op.dest() { - live.remove(&dest); - } - live.extend(op.unbinds()); - live.extend(op.operands().into_iter().filter_map(register)); - } - - if live != live_in[index] { - live_in[index] = live; - changed = true; - } - } - } - - live_in -} - -fn register(value: &Value) -> Option { - match value { - Value::Register(id) => Some(*id), - _ => None, - } -} - #[cfg(test)] mod tests { use super::*; use by_ir::builder::FunctionBuilder; - use by_ir::ops::{CmpOp, Op, Terminator, Value}; + use by_ir::ops::{CmpOp, Concatenation, Mutation, Op, Terminator, Value}; use by_ir::rtype::RType; use by_ir::verify::verify; @@ -219,12 +151,14 @@ mod tests { lhs: Value::Str("a".to_string()), rhs: Value::Register(piece), consumes_lhs: false, + concatenation: Concatenation::Operator(Mutation::Fresh), }); builder.push(Op::StrConcat { dest: second, lhs: Value::Register(first), rhs: Value::Str("b".to_string()), consumes_lhs: false, + concatenation: Concatenation::Operator(Mutation::Fresh), }); builder.terminate(Terminator::Return(Value::Register(second))); @@ -247,6 +181,7 @@ mod tests { lhs: Value::Register(held), rhs: Value::Str("b".to_string()), consumes_lhs: false, + concatenation: Concatenation::Operator(Mutation::Fresh), }); // the second read is what makes the first one not the last builder.push(Op::StrConcat { @@ -254,6 +189,7 @@ mod tests { lhs: Value::Register(joined), rhs: Value::Register(held), consumes_lhs: false, + concatenation: Concatenation::Operator(Mutation::Fresh), }); builder.terminate(Terminator::Return(Value::Register(doubled))); @@ -277,6 +213,7 @@ mod tests { lhs: Value::Register(out), rhs: Value::Register(piece), consumes_lhs: false, + concatenation: Concatenation::Operator(Mutation::Fresh), }); builder.terminate(Terminator::Return(Value::Register(out))); @@ -302,6 +239,7 @@ mod tests { lhs: Value::Register(out), rhs: Value::Register(piece), consumes_lhs: false, + concatenation: Concatenation::Operator(Mutation::Fresh), }); builder.terminate(Terminator::Return(Value::Register(out))); builder.switch_to(handler); @@ -324,6 +262,7 @@ mod tests { lhs: Value::Register(text), rhs: Value::Str("b".to_string()), consumes_lhs: false, + concatenation: Concatenation::Operator(Mutation::Fresh), }); builder.terminate(Terminator::Return(Value::Register(joined))); @@ -369,6 +308,7 @@ mod tests { lhs: Value::Register(out), rhs: Value::Register(piece), consumes_lhs: false, + concatenation: Concatenation::Operator(Mutation::Fresh), }); builder.assign(out, Value::Register(grown)); builder.terminate(Terminator::Goto(header)); @@ -401,6 +341,7 @@ mod tests { lhs: Value::Register(held), rhs: Value::Str("b".to_string()), consumes_lhs: false, + concatenation: Concatenation::Operator(Mutation::Fresh), }); builder.terminate(Terminator::Return(Value::Register(joined))); builder.switch_to(handler); @@ -423,12 +364,14 @@ mod tests { lhs: Value::Str("a".to_string()), rhs: Value::Register(piece), consumes_lhs: false, + concatenation: Concatenation::Operator(Mutation::Fresh), }); builder.push(Op::StrConcat { dest: doubled, lhs: Value::Register(made), rhs: Value::Register(made), consumes_lhs: false, + concatenation: Concatenation::Operator(Mutation::Fresh), }); builder.terminate(Terminator::Return(Value::Register(doubled))); diff --git a/crates/by_opt/src/str_concat_int.rs b/crates/by_opt/src/str_concat_int.rs index 1ef3938fa0..17e8be408e 100644 --- a/crates/by_opt/src/str_concat_int.rs +++ b/crates/by_opt/src/str_concat_int.rs @@ -42,7 +42,7 @@ use std::collections::{HashMap, HashSet}; use by_ir::function::{Function, ModuleIr}; -use by_ir::ops::{BlockId, Op, RegisterId, Value}; +use by_ir::ops::{BlockId, Concatenation, Op, RegisterId, Value}; use by_ir::rtype::RType; pub(crate) fn run(module: &mut ModuleIr) { @@ -79,13 +79,27 @@ fn fuse(function: &mut Function) { let Some(block) = function.blocks.get_mut(chain.block.index()) else { continue; }; - let (Some(Op::StrOfInt { value, .. }), Some(Op::StrConcat { dest, lhs, .. })) = ( + let ( + Some(Op::StrOfInt { value, .. }), + Some(Op::StrConcat { + dest, + lhs, + concatenation: Concatenation::Operator(mutation), + .. + }), + ) = ( block.ops.get(chain.index).cloned(), block.ops.get(chain.index + 2).cloned(), - ) else { + ) + else { continue; }; - block.ops[chain.index] = Op::StrConcatInt { dest, lhs, value }; + block.ops[chain.index] = Op::StrConcatInt { + dest, + lhs, + value, + mutation, + }; block.ops.drain(chain.index + 1..chain.index + 3); } } @@ -104,7 +118,10 @@ fn chains(function: &Function) -> Vec { to, }, Op::StrConcat { - rhs, consumes_lhs, .. + rhs, + consumes_lhs, + concatenation: Concatenation::Operator(_), + .. }, ] = window else { @@ -196,7 +213,7 @@ fn wholly_owned(function: &Function, chains: &[Chain]) -> HashSet { mod tests { use by_ir::builder::FunctionBuilder; use by_ir::function::{Function, ModuleIr}; - use by_ir::ops::{Op, RegisterId, Terminator, Value}; + use by_ir::ops::{Concatenation, Mutation, Op, RegisterId, Terminator, Value}; use by_ir::rtype::RType; /// `prefix + str(i)`, in the shape the passes before this one leave it @@ -220,6 +237,7 @@ mod tests { lhs: Value::Str(prefix.to_string()), rhs: Value::Register(checked), consumes_lhs: false, + concatenation: Concatenation::Operator(Mutation::Fresh), }); wire(&mut builder, checked); builder.terminate(Terminator::Return(Value::Register(out))); @@ -294,6 +312,7 @@ mod tests { lhs: Value::Register(held), rhs: Value::Register(checked), consumes_lhs: true, + concatenation: Concatenation::Operator(Mutation::Fresh), }); builder.terminate(Terminator::Return(Value::Register(held))); let function = run_on(builder.finish()); diff --git a/crates/by_rt/include/by.h b/crates/by_rt/include/by.h index 0a9a2f6038..e15bf2e1c4 100644 --- a/crates/by_rt/include/by.h +++ b/crates/by_rt/include/by.h @@ -110,6 +110,12 @@ typedef size_t ByTagged; #define BY_SHORT_MAX (PY_SSIZE_T_MAX >> 1) #define BY_SHORT_MIN (-BY_SHORT_MAX - 1) +/* the compiler decides which integer literals are short when it writes the C, and it + * decides against this bound. a narrower word would take a literal the compiler called + * short and silently drop its top bits */ +_Static_assert(BY_SHORT_MAX == 4611686018427387903LL, + "the emitted C sorts integer literals against a 64-bit short range"); + /* a tagged integer is a machine word until it is not, and the not is rare — every * loop counter, index and accumulator in a real program stays short. telling the * compiler so is what keeps the slow path out of the straight line */ @@ -260,8 +266,41 @@ static inline PyObject *By_BoxInt(ByTagged x) { return o; } -/* takes a new reference to `o` when it cannot be represented as a short */ +/* an integer literal too wide to be a short, which the module built once at init + * + * the tag is added and no reference is taken: like a string literal, the module owns + * it and a use of it is borrowed. the value lies outside the short range by + * construction, which is the invariant every tagged pointer carries */ +BY_HOT ByTagged By_TaggedLiteral(PyObject *o) { + return ((ByTagged)(void *)o) | BY_INT_TAG; +} + +static inline void By_TypeError(const char *expected, PyObject *got) { + PyErr_Format(PyExc_TypeError, "expected %s, got %s", expected, + got == NULL ? "NULL" : Py_TYPE(got)->tp_name); +} + +/* how many digits an `int` has to have before no value of that length is a short + * + * python 3.12 onwards keeps the digit count in the object's header, beside the sign. + * the smallest magnitude with `n` digits is 2**(PyLong_SHIFT*(n-1)), and to be past + * every short it has to be *strictly* past 2**62, because -2**62 is itself one — so + * this is the first `n` for which that holds. it is four for 30-bit digits and six + * for 15-bit ones, which is why it is worked out rather than written down */ +#if PY_VERSION_HEX >= 0x030C0000 && !defined(Py_LIMITED_API) +#define BY_DIGITS_PAST_SHORT ((uintptr_t)((sizeof(Py_ssize_t) * 8 - 2) / PyLong_SHIFT + 2)) +#endif + +/* takes a new reference to `o` when it cannot be represented as a short + * + * anything that is not an `int` raises the `TypeError` an unbox of it raises, and the + * error path never clears: an operator on a big `int` subclass can answer with any + * object at all, and swallowing the failure here once tagged a `str` as an int */ static inline ByTagged By_TaggedFromLong(PyObject *o) { + if (BY_UNLIKELY(o == NULL || !PyLong_Check(o))) { + By_TypeError("int", o); + return BY_INT_ERROR; + } #if PY_VERSION_HEX >= 0x030C0000 /* almost every `int` a program unboxes is one that fits a single digit, and * python 3.12 onwards stores such a value in the object's own header rather @@ -273,24 +312,30 @@ static inline ByTagged By_TaggedFromLong(PyObject *o) { * `PyUnstable_Long_*` are cpython's own accessors for exactly this, and the * header defines them as inline functions over the macros that name them — * so this is the interpreter's own fast path rather than a guess about its - * layout. the type test is what makes it safe to take: a caller handing over - * something that is not an `int` at all still reaches the checked route - * below and gets the `TypeError` from it */ - if (BY_LIKELY(PyLong_Check(o) && PyUnstable_Long_IsCompact((PyLongObject *)o))) { + * layout */ + if (BY_LIKELY(PyUnstable_Long_IsCompact((PyLongObject *)o))) { return By_ShortFrom(PyUnstable_Long_CompactValue((PyLongObject *)o)); } +#endif +#ifdef BY_DIGITS_PAST_SHORT + /* and the other end: a value this long is never a short, so the checked + * conversion below would only find out what the header already says */ + if ((((PyLongObject *)o)->long_value.lv_tag >> _PyLong_NON_SIZE_BITS) + >= BY_DIGITS_PAST_SHORT) { + By_IncRef(o); + return ((ByTagged)(void *)o) | BY_INT_TAG; + } #endif { + /* an `int` is read without asking `__index__`, so the only failure left is one + * too wide for the word, and that is reported through `overflow` rather than + * raised */ int overflow = 0; - Py_ssize_t value = PyLong_AsLongLongAndOverflow(o, &overflow); - if (!overflow && value != -1) { - if (By_FitsShort((Py_ssize_t)value)) { - return By_ShortFrom((Py_ssize_t)value); - } - } else if (!overflow && !PyErr_Occurred() && By_FitsShort((Py_ssize_t)value)) { + long long value = PyLong_AsLongLongAndOverflow(o, &overflow); + if (value == -1 && PyErr_Occurred()) return BY_INT_ERROR; + if (!overflow && By_FitsShort((Py_ssize_t)value)) { return By_ShortFrom((Py_ssize_t)value); } - PyErr_Clear(); By_IncRef(o); return ((ByTagged)(void *)o) | BY_INT_TAG; } @@ -377,37 +422,92 @@ BY_COLD ByTagged By_IntSlowBitwise(ByTagged a, ByTagged b, char op) { return tagged; } -BY_HOT ByTagged By_IntAdd(ByTagged a, ByTagged b) { +/* the fast paths alone, for a caller that branches on them: each answers 1 with the + * result written where both operands are shorts and so is the result, and 0 where the + * slow path has to decide. a short is an even word and each of these results is one + * too, so what they write is never the error value — a caller only has to test for an + * error on the slow path, which is the only one that calls into cpython */ +static inline int By_IntAddShort(ByTagged a, ByTagged b, ByTagged *out) { if (BY_LIKELY(By_IsShort(a) && By_IsShort(b))) { Py_ssize_t x = (Py_ssize_t)a, y = (Py_ssize_t)b; /* wrap in the unsigned domain, where overflow is defined, then use the * sign test: overflow happened iff both operands differ in sign from the * result */ Py_ssize_t sum = (Py_ssize_t)((size_t)x + (size_t)y); - if (BY_LIKELY(((x ^ sum) & (y ^ sum)) >= 0)) return (ByTagged)sum; + if (BY_LIKELY(((x ^ sum) & (y ^ sum)) >= 0)) { + *out = (ByTagged)sum; + return 1; + } } - return By_IntSlowBinary(a, b, "+"); + return 0; } -BY_HOT ByTagged By_IntSub(ByTagged a, ByTagged b) { +static inline int By_IntSubShort(ByTagged a, ByTagged b, ByTagged *out) { if (BY_LIKELY(By_IsShort(a) && By_IsShort(b))) { Py_ssize_t x = (Py_ssize_t)a, y = (Py_ssize_t)b; Py_ssize_t diff = (Py_ssize_t)((size_t)x - (size_t)y); - if (BY_LIKELY(((x ^ y) & (x ^ diff)) >= 0)) return (ByTagged)diff; + if (BY_LIKELY(((x ^ y) & (x ^ diff)) >= 0)) { + *out = (ByTagged)diff; + return 1; + } } - return By_IntSlowBinary(a, b, "-"); + return 0; } /* a product of two values within this bound cannot leave the short range */ #define BY_MUL_SAFE (((Py_ssize_t)1) << ((sizeof(Py_ssize_t) * 8 - 4) / 2)) -BY_HOT ByTagged By_IntMul(ByTagged a, ByTagged b) { +static inline int By_IntMulShort(ByTagged a, ByTagged b, ByTagged *out) { if (BY_LIKELY(By_IsShort(a) && By_IsShort(b))) { Py_ssize_t x = By_ShortValue(a), y = By_ShortValue(b); if (x > -BY_MUL_SAFE && x < BY_MUL_SAFE && y > -BY_MUL_SAFE && y < BY_MUL_SAFE) { - return By_ShortFrom(x * y); + *out = By_ShortFrom(x * y); + return 1; } } + return 0; +} + +/* `& | ^` are exact on the shifted representation: (2a)&(2b) == 2(a&b) */ +static inline int By_IntAndShort(ByTagged a, ByTagged b, ByTagged *out) { + if (BY_LIKELY(By_IsShort(a) && By_IsShort(b))) { + *out = a & b; + return 1; + } + return 0; +} + +static inline int By_IntOrShort(ByTagged a, ByTagged b, ByTagged *out) { + if (BY_LIKELY(By_IsShort(a) && By_IsShort(b))) { + *out = a | b; + return 1; + } + return 0; +} + +static inline int By_IntXorShort(ByTagged a, ByTagged b, ByTagged *out) { + if (BY_LIKELY(By_IsShort(a) && By_IsShort(b))) { + *out = a ^ b; + return 1; + } + return 0; +} + +BY_HOT ByTagged By_IntAdd(ByTagged a, ByTagged b) { + ByTagged sum; + if (BY_LIKELY(By_IntAddShort(a, b, &sum))) return sum; + return By_IntSlowBinary(a, b, "+"); +} + +BY_HOT ByTagged By_IntSub(ByTagged a, ByTagged b) { + ByTagged diff; + if (BY_LIKELY(By_IntSubShort(a, b, &diff))) return diff; + return By_IntSlowBinary(a, b, "-"); +} + +BY_HOT ByTagged By_IntMul(ByTagged a, ByTagged b) { + ByTagged product; + if (BY_LIKELY(By_IntMulShort(a, b, &product))) return product; return By_IntSlowBinary(a, b, "*"); } @@ -454,24 +554,25 @@ BY_HOT ByTagged By_IntFloorDiv(ByTagged a, ByTagged b) { } Py_ssize_t x = By_ShortValue(a); /* the one case where the quotient leaves the range of the operands */ - if (!(x == PY_SSIZE_T_MIN && y == -1)) { + if (!(x == BY_SHORT_MIN && y == -1)) { return By_ShortFrom(By_FloorDivSsize(x, y)); } } return By_IntSlowBinary(a, b, "/"); } +/* `%` works on the tagged words themselves, with no shift out and back. a short is + * its value shifted left by one, so `(2x) % (2y)` is `2 (x % y)`, and the floor fix-up + * adds the divisor, `2y` — the answer is already a tagged short. the one remainder that + * overflows is by -1, and a tagged divisor is even, so it never is: only a zero divisor + * faults */ BY_HOT ByTagged By_IntMod(ByTagged a, ByTagged b) { if (BY_LIKELY(By_IsShort(a) && By_IsShort(b))) { - Py_ssize_t y = By_ShortValue(b); - if (y == 0) { + if (b == By_ShortFrom(0)) { By_ZeroDivision(PyNumber_Remainder, 0); return BY_INT_ERROR; } - Py_ssize_t x = By_ShortValue(a); - if (!(x == PY_SSIZE_T_MIN && y == -1)) { - return By_ShortFrom(By_ModSsize(x, y)); - } + return (ByTagged)By_ModSsize((Py_ssize_t)a, (Py_ssize_t)b); } return By_IntSlowBinary(a, b, "%"); } @@ -490,17 +591,19 @@ static inline double By_IntTrueDiv(ByTagged a, ByTagged b) { return value; } -/* `& | ^` are exact on the shifted representation: (2a)&(2b) == 2(a&b) */ BY_HOT ByTagged By_IntAnd(ByTagged a, ByTagged b) { - if (BY_LIKELY(By_IsShort(a) && By_IsShort(b))) return a & b; + ByTagged out; + if (BY_LIKELY(By_IntAndShort(a, b, &out))) return out; return By_IntSlowBitwise(a, b, '&'); } BY_HOT ByTagged By_IntOr(ByTagged a, ByTagged b) { - if (BY_LIKELY(By_IsShort(a) && By_IsShort(b))) return a | b; + ByTagged out; + if (BY_LIKELY(By_IntOrShort(a, b, &out))) return out; return By_IntSlowBitwise(a, b, '|'); } BY_HOT ByTagged By_IntXor(ByTagged a, ByTagged b) { - if (BY_LIKELY(By_IsShort(a) && By_IsShort(b))) return a ^ b; + ByTagged out; + if (BY_LIKELY(By_IntXorShort(a, b, &out))) return out; return By_IntSlowBitwise(a, b, '^'); } static inline ByTagged By_IntShl(ByTagged a, ByTagged b) { @@ -521,7 +624,8 @@ static inline ByTagged By_IntInvert(ByTagged a) { static inline ByTagged By_IntNeg(ByTagged a) { if (By_IsShort(a)) { Py_ssize_t x = By_ShortValue(a); - if (x != PY_SSIZE_T_MIN) return By_ShortFrom(-x); + /* the short range is one wider below zero than above it */ + if (x != BY_SHORT_MIN) return By_ShortFrom(-x); } return By_IntSub(By_ShortFrom(0), a); } @@ -529,6 +633,29 @@ static inline ByTagged By_IntNeg(ByTagged a) { /* ── int comparison ───────────────────────────────────────────────────────── */ BY_COLD char By_IntCompareSlow(ByTagged a, ByTagged b, int op) { +#if PY_VERSION_HEX >= 0x030C0000 && !defined(Py_LIMITED_API) + /* an exact `int` is only ever held behind a pointer when it does not fit a short, + * so against a short the two can never be equal and the pointer's sign alone says + * which is larger. a subclass is left to its own comparison, which is what makes + * this a fact about the representation rather than about the values */ + if (By_IsShort(a) != By_IsShort(b)) { + PyObject *big = By_LongOf(By_IsShort(a) ? b : a); + if (big != NULL && PyLong_CheckExact(big)) { + /* the low two bits of the tag are 0 for positive and 2 for negative; a + * value too big to be a short is never zero */ + int negative = (((PyLongObject *)big)->long_value.lv_tag & _PyLong_SIGN_MASK) == 2; + /* whether `a < b` */ + char below = (char)(By_IsShort(a) ? !negative : negative); + switch (op) { + case Py_EQ: return 0; + case Py_NE: return 1; + case Py_LT: + case Py_LE: return below; + default: return (char)!below; + } + } + } +#endif PyObject *left = By_BoxInt(a); if (left == NULL) return 2; PyObject *right = By_BoxInt(b); @@ -781,11 +908,6 @@ static inline PyObject *By_BoxNone(void) { return Py_None; } -static inline void By_TypeError(const char *expected, PyObject *got) { - PyErr_Format(PyExc_TypeError, "expected %s, got %s", expected, - got == NULL ? "NULL" : Py_TYPE(got)->tp_name); -} - /* unboxing is a *narrowing*, so it is always checked — this is the * representation invariant's inserted check, not an assumption */ static inline ByTagged By_UnboxInt(PyObject *o) { @@ -974,10 +1096,34 @@ static inline PyObject *By_ObjIShr(PyObject *a, PyObject *b) { static inline PyObject *By_ObjNeg(PyObject *o) { return PyNumber_Negative(o); } static inline PyObject *By_ObjInvert(PyObject *o) { return PyNumber_Invert(o); } -/* a comparison yields a bit, and 2 means an exception is set */ +/* `a b` through the abstract protocol, as a bit — and 2 means an exception is set + * + * this is `PyObject_RichCompareBool` with its one shortcut removed. that function + * answers `Py_EQ` on a pair of identical pointers `True` *before* it asks the type, + * on the documented guarantee that identity implies equality — a guarantee ordinary + * python does not make. a NaN is not equal to itself, and neither is an instance of + * a class whose `__eq__` says so, so a comparison lowered onto it disagreed with the + * interpreted twin while raising nothing and declining nothing. + * + * the shortcut is correct where the *interpreter* takes it, and it does take it in + * places this must not disturb: `x in xs` and `xs.index(x)` reach it through + * `PySequence_Contains` and `list.index`, and a dict lookup reaches it through + * `lookdict`. those are answered by cpython's own code here — [`By_Contains`] and an + * ordinary method call — so they keep it by construction and are not this helper's + * to decide. + * + * what is left is the rest of that function: compare, then collapse the answer to a + * bit. so the pointer test is the whole of what removing the shortcut costs. + * + * a NULL operand carries an exception set by whatever produced it, and the comparison + * would dereference it — [`By_StrCompare`] hands its own such pair straight here */ static inline char By_ObjCompare(PyObject *a, PyObject *b, int op) { - int result = PyObject_RichCompareBool(a, b, op); - return result < 0 ? 2 : (char)result; + if (BY_UNLIKELY(a == NULL || b == NULL)) return 2; + PyObject *result = PyObject_RichCompare(a, b, op); + if (result == NULL) return 2; + int truth = PyObject_IsTrue(result); + Py_DECREF(result); + return truth < 0 ? 2 : (char)truth; } /* the type python names in an `AttributeError` @@ -992,6 +1138,21 @@ static inline const char *By_TypeName(PyObject *o) { return dot == NULL ? name : dot + 1; } +/* raise python's `AttributeError` for an instance without a field of its layout + * + * out of line, because it is the rare branch beside every field read nothing proves + * assigned, and building the message there made those reads' functions too big to + * inline into their callers. `qualified` names the type in full, which is how a slot's + * descriptor words it */ +BY_COLD void By_FieldMissing(PyObject *o, const char *name, int qualified) { + if (qualified) { + PyErr_Format(PyExc_AttributeError, "'%T' object has no attribute '%s'", o, name); + } else { + PyErr_Format(PyExc_AttributeError, "'%s' object has no attribute '%s'", By_TypeName(o), + name); + } +} + /* python truthiness, which can raise from a user `__bool__` or `__len__` */ static inline char By_Truthy(PyObject *o) { int result = PyObject_IsTrue(o); @@ -1107,6 +1268,90 @@ static inline PyObject *By_FixedName(PyObject **slot, const char *name, Py_ssize * does before anything can read a global */ static PyObject *by_module_builtins = NULL; +/* ── one module object per process ─────────────────────────────────────────── + * + * a compiled module keeps its namespace, and every memo of a name in it, in statics the + * whole process shares. a second module object — `del sys.modules[name]` and an import, + * or an import in another interpreter — would have those statics pointed at its own + * namespace, and the first object's compiled code would read and write the second's. + * + * so the first module to finish executing is recorded in the interpreter that imported + * it, and a later import there hands that module back, as it does for a single-phase C + * extension. an import in any other interpreter is refused, since there is no second set + * of statics to give it. the record is keyed by this build's module definition, so two + * builds of one module name are never taken for each other */ + +/* the interpreter the module executed in, and NULL until it has */ +static PyInterpreterState *by_module_home = NULL; + +/* the interpreter-wide dict key the module is recorded under, new, or NULL on failure */ +static PyObject *By_ModuleRecordKey(PyModuleDef *def) { + return PyUnicode_FromFormat("by.module:%s:%p", def->m_name, (void *)def); +} + +/* the module this interpreter already holds for `def`, borrowed, or NULL with no error + * set where it holds none */ +static PyObject *By_RecordedModule(PyModuleDef *def) { + PyObject *registry = PyInterpreterState_GetDict(PyInterpreterState_Get()); + PyObject *key; + PyObject *found; + if (registry == NULL) return NULL; + key = By_ModuleRecordKey(def); + if (key == NULL) return NULL; + found = PyDict_GetItemWithError(registry, key); + Py_DECREF(key); + return found; +} + +/* `Py_mod_create`: the module already imported here, or a new one */ +static PyObject *By_CreateModule(PyObject *spec, PyModuleDef *def) { + PyInterpreterState *interp = PyInterpreterState_Get(); + PyObject *found; + PyObject *name; + PyObject *module; + if (by_module_home != NULL && by_module_home != interp) { + PyErr_Format(PyExc_ImportError, + "the compiled module '%s' is already imported in another interpreter, " + "and its state is shared by the whole process", + def->m_name); + return NULL; + } + found = By_RecordedModule(def); + if (found != NULL) return By_NewRef(found); + if (PyErr_Occurred()) return NULL; + name = PyObject_GetAttrString(spec, "name"); + if (name == NULL) return NULL; + module = PyModule_NewObject(name); + Py_DECREF(name); + return module; +} + +/* whether `module` is the one this interpreter already executed, whose exec slot is then + * run again only because the import machinery hands it back to be executed */ +static int By_ModuleExecuted(PyObject *module, PyModuleDef *def) { + PyObject *found = By_RecordedModule(def); + if (found == NULL && PyErr_Occurred()) return -1; + return found == module; +} + +/* record `module` as the one this interpreter holds, once its exec slot has succeeded */ +static int By_RecordModule(PyObject *module, PyModuleDef *def) { + PyObject *registry = PyInterpreterState_GetDict(PyInterpreterState_Get()); + PyObject *key; + int failed; + if (registry == NULL) { + PyErr_SetString(PyExc_ImportError, "the interpreter has no dict to record a module in"); + return -1; + } + key = By_ModuleRecordKey(def); + if (key == NULL) return -1; + failed = PyDict_SetItem(registry, key, module); + Py_DECREF(key); + if (failed < 0) return -1; + by_module_home = PyInterpreterState_Get(); + return 0; +} + /* bind that namespace, the way python binds one for an interpreted module * * python reaches a function's builtins through `__globals__['__builtins__']`, fixed @@ -3176,11 +3421,12 @@ typedef PyObject *(*ByFastKwCall)(PyObject *, PyObject *const *, Py_ssize_t, PyO * will never be able to serve — every instance of a class written in the interpreter * — is asked about once and then costs the same two comparisons as a hit. * - * that order matters for a second reason. the attribute lookup below is the one step - * here that another thread can run inside, and it sits *after* the type has been - * recorded and the answer cleared — so a site another thread arms in the middle of - * this one is left holding that thread's type against this one's version, which no - * receiver matches, rather than that thread's type against this one's body */ + * that order matters for a second reason. the lookup below runs no python of its own, + * but it was once an attribute read that could, and nothing here should rest on which + * it is: it sits *after* the type has been recorded and the answer cleared, so a site + * another thread arms in the middle of this one is left holding that thread's type + * against this one's version, which no receiver matches, rather than that thread's type + * against this one's body */ static void By_ArmMethodSite(ByMethodSite *site, PyTypeObject *tp, PyObject *name, Py_ssize_t nargs) { site->type = (PyObject *)tp; @@ -3192,18 +3438,18 @@ static void By_ArmMethodSite(ByMethodSite *site, PyTypeObject *tp, PyObject *nam if (tp->tp_flags & BY_INLINE_VALUES_FLAG) return; if (tp->tp_dictoffset < 0) return; site->dict_offset = tp->tp_dictoffset; - PyObject *found = PyObject_GetAttr((PyObject *)tp, name); - if (found == NULL) { - PyErr_Clear(); - return; - } - /* reading the type's own attribute hands back the descriptor rather than a bound - * method, so the builtin behind it is reachable through it */ - PyMethodDef *method = Py_IS_TYPE(found, &PyMethodDescr_Type) - ? ((PyMethodDescrObject *)found)->d_method - : NULL; - Py_DECREF(found); - if (method == NULL) return; + /* the entry an instance's lookup finds on the type, borrowed and with no descriptor + * run to find it. reading the attribute off the class instead runs a `__get__` of + * the entry's own, and what that hands back for the class need not be what it hands + * back for an instance */ + PyObject *found = _PyType_Lookup(tp, name); + if (found == NULL || !Py_IS_TYPE(found, &PyMethodDescr_Type)) return; + /* a method descriptor serves the instances of the type that defined it, and refuses + * anything else with a `TypeError`. `pop = list.pop` in a class of its own is that + * refusal, and calling the entry point directly would hand `list_pop` a receiver + * that is not a list */ + if (!PyType_IsSubtype(tp, PyDescr_TYPE(found))) return; + PyMethodDef *method = ((PyMethodDescrObject *)found)->d_method; /* the whole of the flags decides, with only the one bit that says nothing about the * arguments set aside. a mask naming the conventions this knows would *drop* the * bits it does not know rather than refuse them, and `METH_METHOD` is exactly such @@ -5813,6 +6059,91 @@ static inline char By_Contains(PyObject *container, PyObject *value, int negated return (char)(negated ? !found : found); } +/* whether a dict's keys table holds nothing but exact `str` keys + * + * cpython keeps a table of exact `str` keys in a form of its own, and turns it into the + * general form the moment any other key is stored. which form a table is in is + * `dk_kind`, a field of the private `PyDictKeysObject` head, so reading it trusts a + * layout no public header promises. that trust is earned twice over: only on the + * versions whose layout was read (3.13 and 3.14, with the GIL, where the head is + * `dk_refcnt`, `dk_log2_size`, `dk_log2_index_bytes`, `dk_kind`), and only once + * [`By_CheckDictKinds`] has built dicts whose form is known and found the field saying + * so. anything else answers 0, which is the answer that is never wrong */ +#if PY_VERSION_HEX >= 0x030D0000 && PY_VERSION_HEX < 0x030F0000 && !defined(Py_GIL_DISABLED) \ + && !defined(Py_LIMITED_API) +#define BY_DICT_KINDS 1 + +typedef struct { + Py_ssize_t dk_refcnt; + uint8_t dk_log2_size; + uint8_t dk_log2_index_bytes; + uint8_t dk_kind; +} ByDictKeysHead; + +/* 1 once the layout has been checked and found as read, 0 before that or if it was not */ +static char by_dict_kinds_checked = 0; + +static inline const ByDictKeysHead *By_DictKeysHead(PyObject *dict) { + return (const ByDictKeysHead *)((PyDictObject *)dict)->ma_keys; +} + +static inline char By_DictKeysAllStr(PyObject *dict) { + return by_dict_kinds_checked && By_DictKeysHead(dict)->dk_kind != 0; +} + +/* whether the head of `dict`'s table reads as a fresh table of `kind` would: a minimum + * sized table owned by this dict alone, or a table shared between instances */ +static char By_DictKeysReadAs(PyObject *dict, uint8_t kind) { + const ByDictKeysHead *head = By_DictKeysHead(dict); + if (head->dk_kind != kind) return 0; + /* a split table is shared by the class and every instance using it */ + if (kind == 2) return head->dk_refcnt >= 1; + return head->dk_refcnt == 1 && head->dk_log2_size == 3 && head->dk_log2_index_bytes == 3; +} + +/* build one dict of each form and check the private read names each correctly + * + * `{"a": 1}` is a table of exact `str` keys, `{1: 1}` a general one, and the `__dict__` + * of an instance of a fresh class a split one. a failure of any step, or any answer + * other than the one the layout predicts, leaves every lookup on the form that reads no + * private field, for as long as this module is loaded */ +static void By_CheckDictKinds(void) { + PyObject *text = NULL, *number = NULL, *type = NULL, *instance = NULL, *split = NULL; + PyObject *name = NULL, *one = NULL; + char held = 0; + if (by_dict_kinds_checked) return; + one = PyLong_FromLong(1); + name = PyUnicode_FromString("a"); + text = PyDict_New(); + number = PyDict_New(); + if (one == NULL || name == NULL || text == NULL || number == NULL) goto done; + if (PyDict_SetItem(text, name, one) < 0 || PyDict_SetItem(number, one, one) < 0) goto done; + type = PyObject_CallFunction((PyObject *)&PyType_Type, "s(){}", "by_dict_kinds"); + if (type == NULL) goto done; + instance = PyObject_CallNoArgs(type); + if (instance == NULL || PyObject_SetAttr(instance, name, one) < 0) goto done; + split = PyObject_GenericGetDict(instance, NULL); + if (split == NULL || !PyDict_CheckExact(split)) goto done; + held = By_DictKeysReadAs(text, 1) && By_DictKeysReadAs(number, 0) + && By_DictKeysReadAs(split, 2); +done: + if (PyErr_Occurred()) PyErr_Clear(); + Py_XDECREF(split); + Py_XDECREF(instance); + Py_XDECREF(type); + Py_XDECREF(number); + Py_XDECREF(text); + Py_XDECREF(name); + Py_XDECREF(one); + by_dict_kinds_checked = held; +} + +#else + +static inline void By_CheckDictKinds(void) {} + +#endif /* BY_DICT_KINDS */ + /* `k in d` answered by the very lookup `d[k]` would go on to make * * the two hash the same key and walk the same table, so where the second is only @@ -5822,32 +6153,30 @@ static inline char By_Contains(PyObject *container, PyObject *value, int negated * the caller tells the two apart with `PyErr_Occurred`, the way it already does * for an exhausted iterator * - * asking once has to be *earned* at runtime, because both of the things that would - * make asking twice observable are ordinary python. a dict subclass may have - * overridden `__contains__` or `__getitem__`, and then the number and order of - * those calls is the program's own business. a key may have a `__hash__` that - * counts how often it is called, and then hashing once where the source hashes - * twice is a different program. so the single probe is taken only for an exact - * dict keyed by an exact `str`, whose hash and equality are the interpreter's and - * have nothing to observe; everything else takes the protocol twice over, in the - * order it would have — and `__getitem__` only where `__contains__` said yes, - * which is the branch this stands in for */ + * asking once has to be *earned* at runtime, because everything that would make + * asking twice observable is ordinary python. a dict subclass may have overridden + * `__contains__` or `__getitem__`, and then the number and order of those calls is + * the program's own business. a key may have a `__hash__` that counts how often it is + * called, and then hashing once where the source hashes twice is a different program. + * and a key already *stored* in the dict, hashing like the probe, has its `__eq__` + * asked by every probe that walks past it — which can count, or delete the key or + * rewrite its value between the test and the read. so the single probe is taken only + * for an exact dict keyed by an exact `str` whose table holds nothing but exact `str` + * keys, where every hash and comparison is the interpreter's own; everything else + * takes the protocol twice over, in the order it would have — and `__getitem__` only + * where `__contains__` said yes, which is the branch this stands in for */ static inline PyObject *By_DictFind(PyObject *container, PyObject *key) { /* a null operand carries an exception already set by whatever produced it */ if (container == NULL || key == NULL) return NULL; - if (BY_LIKELY(PyDict_CheckExact(container) && PyUnicode_CheckExact(key))) { -#if PY_VERSION_HEX >= 0x030D0000 +#ifdef BY_DICT_KINDS + if (BY_LIKELY(PyDict_CheckExact(container) && PyUnicode_CheckExact(key) + && By_DictKeysAllStr(container))) { PyObject *value; /* -1 failed, 0 absent, 1 there — and absent leaves `value` NULL */ if (PyDict_GetItemRef(container, key, &value) < 0) return NULL; return value; -#else - PyObject *value = PyDict_GetItemWithError(container, key); - /* absent and failed are both NULL here, which is what this returns for - * both anyway — the exception state is what separates them */ - return value == NULL ? NULL : By_NewRef(value); -#endif } +#endif int found = PySequence_Contains(container, key); if (found <= 0) return NULL; return PyObject_GetItem(container, key); @@ -6059,6 +6388,42 @@ static inline int By_HoldFieldDefault(PyTypeObject *type, const char *name, PyOb return 0; } +/* work out, once at import, whether `type` still answers `name` with the descriptor its + * field `name` was published through — and record the version that held when it was + * + * a field is published either as a getset or, beside a class-level value, as a descriptor + * of this runtime's own; either carries the field's getter, which is what is compared. + * a type answering reads or writes with a hook of its own is refused, as an accessor is */ +static inline void By_ArmField(ByAccessorLicence *licence, PyObject *type, const char *name, + getter get) { + PyObject *key; + PyObject *found; + int published; + licence->version = 0u; + if (type == NULL || !PyType_Check(type)) return; + PyTypeObject *owner = (PyTypeObject *)type; + if (owner->tp_getattro != PyObject_GenericGetAttr) return; + if (owner->tp_setattro != PyObject_GenericSetAttr) return; + key = PyUnicode_InternFromString(name); + if (key == NULL) { + PyErr_Clear(); + return; + } + /* borrowed, and no descriptor is run to find it */ + found = _PyType_Lookup(owner, key); + Py_DECREF(key); + if (found == NULL) return; + published = (Py_IS_TYPE(found, &PyGetSetDescr_Type) + && ((PyGetSetDescrObject *)found)->d_getset->get == get) + || (Py_IS_TYPE(found, &By_FieldDefaultType) + && ((By_FieldDefaultObject *)found)->by_get == get); + if (!published) return; +#if PY_VERSION_HEX >= 0x030C0000 + if (!PyUnstable_Type_AssignVersionTag(owner)) return; +#endif + licence->version = owner->tp_version_tag; +} + /* ── an emitted instance's `__dict__` ───────────────────────────────────────── * * an emitted instance keeps its attributes in two places. the ones the class itself @@ -6097,6 +6462,11 @@ typedef struct { * never wrote one, and python's `__dict__` does not name a class attribute. every * other field reports its own absence by raising, which the read below reads */ By_FieldPresent present; + /* the field's own setter, which also deletes, and NULL where the class publishes + * none. a write through `__dict__` naming the field is a write of the field, and + * reaching it through the setter rather than the attribute is what keeps a subclass's + * `__setattr__` or a descriptor shadowing the name from running */ + setter set; } By_DictField; typedef struct { @@ -6263,88 +6633,178 @@ static int By_InstanceDict_clear(PyObject *selfobj) { return PyDict_Type.tp_clear(selfobj); } -/* a write goes through the attribute, which is what keeps the two halves one mapping: a - * name the layout knows lands in the layout and every other name lands in this mapping - * itself, which *is* the object's dict word — exactly as an assignment through the object - * would put it there, because that is what this does. +/* the layout field a key names, or NULL for a key the layout has no field for */ +static const By_DictField *By_InstanceDictFieldNamed(ByInstanceDictObject *self, PyObject *key) { + const By_DictField *field; + if (!PyUnicode_Check(key)) return NULL; + for (field = self->fields; field->name != NULL; field++) { + if (PyUnicode_CompareWithASCIIString(key, field->name) == 0) return field; + } + return NULL; +} + +/* one entry written into, or with a NULL value deleted from, the mapping an instance + * answers `__dict__` with * - * nothing is patched into the storage here. the layout half publishes back on its own - * from the field write, which is what keeps a mapping right when the object is written to - * some other way, and it publishes what the layout *stored* rather than what it was - * handed — an unboxed field takes a representation of its own */ + * python's instance dict is a plain dict, so a write through it runs no descriptor and + * no `__setattr__`: it is the entry and nothing else. a compiled instance keeps the names + * its layout declares in fields rather than in the dict, so a key naming one writes the + * field — through the field's own setter, which publishes what the layout stored back into + * this mapping, and which checks the value's representation as any write of the field + * does. every other key is an entry here, and this mapping is the object's dict word, so + * an attribute read finds it exactly as it finds one python stored */ static int By_InstanceDict_assign(PyObject *selfobj, PyObject *key, PyObject *value) { ByInstanceDictObject *self = (ByInstanceDictObject *)selfobj; - int failed; + const By_DictField *field; /* detached, so there is no object to write back to and this is a plain dict */ if (self->owner == NULL) { return value == NULL ? PyDict_DelItem(selfobj, key) : PyDict_SetItem(selfobj, key, value); } - failed = value == NULL ? PyObject_DelAttr(self->owner, key) - : PyObject_SetAttr(self->owner, key, value); - return failed < 0 ? -1 : 0; + field = By_InstanceDictFieldNamed(self, key); + if (field == NULL) { + return value == NULL ? PyDict_DelItem(selfobj, key) + : PyDict_SetItem(selfobj, key, value); + } + if (value == NULL) { + /* a field the instance has no value in is a key the mapping does not hold */ + int held = PyDict_Contains(selfobj, key); + if (held < 0) return -1; + if (held == 0) { + PyErr_SetObject(PyExc_KeyError, key); + return -1; + } + } + if (field->set == NULL) { + PyErr_Format(PyExc_AttributeError, "attribute '%s' of '%s' objects is not writable", + field->name, Py_TYPE(self->owner)->tp_name); + return -1; + } + return field->set(self->owner, value, NULL); } -/* every write through the attribute, one at a time, so that a name the layout knows - * reaches the layout — which is the whole point of the mapping writing back */ +/* one `key, value` pair of an update, as `By_InstanceDict_assign` writes it */ +static int By_InstanceDictWritePair(PyObject *selfobj, PyObject *key, PyObject *value) { + int failed; + Py_INCREF(key); + Py_INCREF(value); + failed = By_InstanceDict_assign(selfobj, key, value); + Py_DECREF(value); + Py_DECREF(key); + return failed; +} + +/* `dict.update`'s positional argument, written one entry at a time in the order python's + * own update writes them, so that an entry naming a field reaches the field + * + * a dict is walked as a dict, anything with `keys` is asked for its keys and then each + * value, and anything else is taken as pairs — with python's wording for a pair that is + * not one */ static int By_InstanceDictWriteAll(ByInstanceDictObject *self, PyObject *source) { - PyObject *items; - Py_ssize_t index; - Py_ssize_t count; - if (self->owner == NULL) return PyDict_Update((PyObject *)self, source); - items = PyMapping_Items(source); - if (items == NULL) { - PyErr_Clear(); - items = PySequence_List(source); + PyObject *selfobj = (PyObject *)self; + PyObject *keys_attr; + if (self->owner == NULL) { + PyObject *update = PyObject_GetAttrString((PyObject *)&PyDict_Type, "update"); + PyObject *done; + if (update == NULL) return -1; + done = PyObject_CallFunctionObjArgs(update, selfobj, source, NULL); + Py_DECREF(update); + Py_XDECREF(done); + return done == NULL ? -1 : 0; + } + if (PyDict_Check(source)) { + PyObject *items = PyDict_Items(source); + Py_ssize_t index; if (items == NULL) return -1; - } - count = PySequence_Size(items); - if (count < 0) { + for (index = 0; index < PyList_GET_SIZE(items); index++) { + PyObject *pair = PyList_GET_ITEM(items, index); + if (By_InstanceDictWritePair(selfobj, PyTuple_GET_ITEM(pair, 0), + PyTuple_GET_ITEM(pair, 1)) < 0) { + Py_DECREF(items); + return -1; + } + } Py_DECREF(items); - return -1; + return 0; } - for (index = 0; index < count; index++) { - PyObject *pair = PySequence_GetItem(items, index); + keys_attr = PyObject_GetAttrString(source, "keys"); + if (keys_attr != NULL) { + PyObject *keys = PyObject_CallNoArgs(keys_attr); + PyObject *iterator; PyObject *key; - PyObject *value; - int failed; - if (pair == NULL) { - Py_DECREF(items); - return -1; - } - key = PySequence_GetItem(pair, 0); - value = PySequence_GetItem(pair, 1); - Py_DECREF(pair); - if (key == NULL || value == NULL) { - Py_XDECREF(key); + int failed = 0; + Py_DECREF(keys_attr); + if (keys == NULL) return -1; + iterator = PyObject_GetIter(keys); + Py_DECREF(keys); + if (iterator == NULL) return -1; + while (!failed && (key = PyIter_Next(iterator)) != NULL) { + PyObject *value = PyObject_GetItem(source, key); + failed = value == NULL || By_InstanceDictWritePair(selfobj, key, value) < 0; Py_XDECREF(value); - Py_DECREF(items); - return -1; + Py_DECREF(key); } - failed = PyObject_SetAttr(self->owner, key, value) < 0; - Py_DECREF(key); - Py_DECREF(value); - if (failed) { - Py_DECREF(items); - return -1; + Py_DECREF(iterator); + return failed || PyErr_Occurred() ? -1 : 0; + } + if (!PyErr_ExceptionMatches(PyExc_AttributeError)) return -1; + PyErr_Clear(); + { + PyObject *iterator = PyObject_GetIter(source); + PyObject *item; + Py_ssize_t at = 0; + if (iterator == NULL) return -1; + while ((item = PyIter_Next(iterator)) != NULL) { + PyObject *pair = PySequence_Fast(item, ""); + int failed; + if (pair == NULL) { + if (PyErr_ExceptionMatches(PyExc_TypeError)) { + PyErr_Format(PyExc_TypeError, + "cannot convert dictionary update sequence element #%zd to a sequence", + at); + } + Py_DECREF(item); + Py_DECREF(iterator); + return -1; + } + if (PySequence_Fast_GET_SIZE(pair) != 2) { + PyErr_Format(PyExc_ValueError, + "dictionary update sequence element #%zd has length %zd; 2 is required", + at, PySequence_Fast_GET_SIZE(pair)); + Py_DECREF(pair); + Py_DECREF(item); + Py_DECREF(iterator); + return -1; + } + failed = By_InstanceDictWritePair(selfobj, PySequence_Fast_GET_ITEM(pair, 0), + PySequence_Fast_GET_ITEM(pair, 1)); + Py_DECREF(pair); + Py_DECREF(item); + if (failed < 0) { + Py_DECREF(iterator); + return -1; + } + at++; } + Py_DECREF(iterator); + return PyErr_Occurred() ? -1 : 0; } - Py_DECREF(items); - return 0; } -static PyObject *By_InstanceDict_update(PyObject *selfobj, PyObject *const *args, - Py_ssize_t nargs) { - if (nargs != 1) { - PyErr_SetString(PyExc_TypeError, "update() takes exactly one argument"); +/* `update(other=(), /, **kwargs)`: the positional argument first, then each keyword */ +static PyObject *By_InstanceDict_update(PyObject *selfobj, PyObject *args, PyObject *kwargs) { + PyObject *other = NULL; + if (!PyArg_UnpackTuple(args, "update", 0, 1, &other)) return NULL; + if (other != NULL && By_InstanceDictWriteAll((ByInstanceDictObject *)selfobj, other) < 0) { + return NULL; + } + if (kwargs != NULL && By_InstanceDictWriteAll((ByInstanceDictObject *)selfobj, kwargs) < 0) { return NULL; } - if (By_InstanceDictWriteAll((ByInstanceDictObject *)selfobj, args[0]) < 0) return NULL; Py_RETURN_NONE; } static PyObject *By_InstanceDict_ior(PyObject *selfobj, PyObject *other) { - if (!PyDict_Check(other)) Py_RETURN_NOTIMPLEMENTED; if (By_InstanceDictWriteAll((ByInstanceDictObject *)selfobj, other) < 0) return NULL; return By_NewRef(selfobj); } @@ -6365,8 +6825,7 @@ static PyObject *By_InstanceDict_clearmethod(PyObject *selfobj, PyObject *unused if (keys == NULL) return NULL; count = PyList_GET_SIZE(keys); for (index = 0; index < count; index++) { - if (PyObject_DelAttr(((ByInstanceDictObject *)selfobj)->owner, - PyList_GET_ITEM(keys, index)) < 0) { + if (By_InstanceDict_assign(selfobj, PyList_GET_ITEM(keys, index), NULL) < 0) { Py_DECREF(keys); return NULL; } @@ -6438,7 +6897,8 @@ static PyObject *By_InstanceDict_setdefault(PyObject *selfobj, PyObject *const * * one `dict` already has, reading the storage this keeps filled — which is the whole * reason for being a `dict` at all */ static PyMethodDef By_InstanceDict_methods[] = { - {"update", (PyCFunction)(void (*)(void))By_InstanceDict_update, METH_FASTCALL, NULL}, + {"update", (PyCFunction)(void (*)(void))By_InstanceDict_update, METH_VARARGS | METH_KEYWORDS, + NULL}, {"clear", By_InstanceDict_clearmethod, METH_NOARGS, NULL}, {"pop", (PyCFunction)(void (*)(void))By_InstanceDict_pop, METH_FASTCALL, NULL}, {"popitem", By_InstanceDict_popitem, METH_NOARGS, NULL}, @@ -7454,6 +7914,124 @@ static inline char By_Extend(PyObject *container, PyObject *source, int mapping) return PyErr_Occurred() ? 2 : 0; } +/* what python calls a callable in an error about the arguments it was handed + * + * `module.qualname()`, or `qualname()` for one of the builtins, or the object's `str` + * where it has no qualified name at all */ +BY_COLD PyObject *By_FunctionStr(PyObject *callable) { + PyObject *qualname = PyObject_GetAttrString(callable, "__qualname__"); + PyObject *module; + PyObject *result = NULL; + if (qualname == NULL) { + if (!PyErr_ExceptionMatches(PyExc_AttributeError)) return NULL; + PyErr_Clear(); + return PyObject_Str(callable); + } + module = PyObject_GetAttrString(callable, "__module__"); + if (module == NULL) { + if (!PyErr_ExceptionMatches(PyExc_AttributeError)) goto done; + PyErr_Clear(); + } else if (module != Py_None) { + int other = PyUnicode_Check(module) ? PyUnicode_CompareWithASCIIString(module, "builtins") + : 1; + if (other != 0) { + result = PyUnicode_FromFormat("%S.%S()", module, qualname); + goto done; + } + } + result = PyUnicode_FromFormat("%S()", qualname); +done: + Py_XDECREF(module); + Py_DECREF(qualname); + return result; +} + +/* the error a failed merge into a call's keywords reports, in python's words + * + * python rewords the two errors a merge raises with a single argument, whatever raised + * them: an `AttributeError` says the operand was no mapping, and a `KeyError` names the + * keyword given twice. anything else is left as it was raised */ +BY_COLD void By_KeywordsMergeError(PyObject *callee, PyObject *source) { + int attribute = PyErr_ExceptionMatches(PyExc_AttributeError); + PyObject *type, *value, *traceback, *args, *name; + if (!attribute && !PyErr_ExceptionMatches(PyExc_KeyError)) return; + PyErr_Fetch(&type, &value, &traceback); + PyErr_NormalizeException(&type, &value, &traceback); + args = value == NULL ? NULL : ((PyBaseExceptionObject *)value)->args; + if (args == NULL || !PyTuple_Check(args) || PyTuple_GET_SIZE(args) != 1) { + PyErr_Restore(type, value, traceback); + return; + } + name = By_FunctionStr(callee); + if (name != NULL) { + PyObject *key = PyTuple_GET_ITEM(args, 0); + if (attribute) { + PyErr_Format(PyExc_TypeError, "%U argument after ** must be a mapping, not %.200s", + name, Py_TYPE(source)->tp_name); + } else { + PyErr_Format(PyExc_TypeError, "%U got multiple values for keyword argument '%S'", + name, key); + } + Py_DECREF(name); + } + Py_XDECREF(type); + Py_XDECREF(value); + Py_XDECREF(traceback); +} + +/* one key of a `**` merged into a call's keywords, refusing one already there */ +static int By_MergeKeyword(PyObject *keywords, PyObject *key, PyObject *value) { + int present = PyDict_Contains(keywords, key); + if (present < 0) return -1; + if (present) { + PyObject *args = PyTuple_Pack(1, key); + if (args != NULL) { + PyErr_SetObject(PyExc_KeyError, args); + Py_DECREF(args); + } + return -1; + } + return PyDict_SetItem(keywords, key, value); +} + +/* `**source` in a call to `callee`, merged into the keywords built so far + * + * the merge python makes for a call, which differs from a display's in refusing a key + * that is already there. a dict that iterates as a dict is walked as one, whatever its + * `keys` says; anything else is asked for its keys and then for each value */ +static inline char By_MergeKeywords(PyObject *keywords, PyObject *source, PyObject *callee) { + int failed = 0; + if (keywords == NULL || source == NULL || callee == NULL) return 2; + if (PyDict_Check(source) && Py_TYPE(source)->tp_iter == PyDict_Type.tp_iter) { + Py_ssize_t at = 0; + PyObject *key, *value; + while (!failed && PyDict_Next(source, &at, &key, &value)) { + Py_INCREF(key); + Py_INCREF(value); + failed = By_MergeKeyword(keywords, key, value) < 0; + Py_DECREF(value); + Py_DECREF(key); + } + } else { + PyObject *keys = PyMapping_Keys(source); + PyObject *iterator = keys == NULL ? NULL : PyObject_GetIter(keys); + PyObject *key; + Py_XDECREF(keys); + failed = iterator == NULL; + while (!failed && (key = PyIter_Next(iterator)) != NULL) { + PyObject *value = PyObject_GetItem(source, key); + failed = value == NULL || By_MergeKeyword(keywords, key, value) < 0; + Py_XDECREF(value); + Py_DECREF(key); + } + Py_XDECREF(iterator); + failed = failed || PyErr_Occurred() != NULL; + } + if (!failed) return 0; + By_KeywordsMergeError(callee, source); + return 2; +} + /* unpack `value` into `count` slots, the way an assignment target list does * * `starred` is the index that collects the surplus into a list, or -1. this drives @@ -7627,15 +8205,26 @@ static inline void By_RaiseObject(PyObject *exception, PyObject *cause) { } /* put an exception back, for a handler that did not match or a bare re-raise + * + * the exception is restored as it stands, which is python's re-raise. raising it + * with `PyErr_SetObject` would chain it again onto whatever is being handled by + * now — and on the way out of an `except` block that is the exception from before + * the block, so the context the exception picked up where it was raised would be + * overwritten with an older one * * the operand is *borrowed*, as every helper's is: the register holding it belongs - * to the frame, which releases it on each exit path. `PyErr_SetObject` takes its - * own reference, so a retain here would be a second one nobody owns — which is - * exactly what leaked a `GeneratorExit` per abandoned generator, and the thrown - * exception per `throw` */ + * to the frame, which releases it on each exit path. the restore steals, so the + * reference handed over is a new one — keeping a second would be one nobody owns, + * which is exactly what leaked a `GeneratorExit` per abandoned generator, and the + * thrown exception per `throw` */ static inline void By_Reraise(PyObject *value) { if (value == NULL) return; - PyErr_SetObject((PyObject *)Py_TYPE(value), value); +#if PY_VERSION_HEX >= 0x030C0000 + PyErr_SetRaisedException(By_NewRef(value)); +#else + PyErr_Restore(By_NewRef((PyObject *)Py_TYPE(value)), By_NewRef(value), + PyException_GetTraceback(value)); +#endif } /* ── a compiled method, as a decorator sees it ──────────────────────────────── @@ -8031,10 +8620,37 @@ static inline PyObject *By_CursorStep(PyObject *it, int64_t *at) { /* ── str ──────────────────────────────────────────────────────────────────── */ +/* two strings joined as the pieces of an f-string are, asking neither anything */ static inline PyObject *By_StrConcat(PyObject *a, PyObject *b) { return PyUnicode_Concat(a, b); } +/* `a + b`, or `a += b`, over two operands that are not both exact `str`s + * + * a subclass may define `__add__`, `__radd__` or `__iadd__`, and python asks them the + * way it asks any operand's. what they answer is what the register holding a `str` is + * given, so an answer that is not one is refused rather than stored */ +BY_COLD PyObject *By_StrAddSlow(PyObject *a, PyObject *b, int in_place) { + PyObject *result = in_place ? PyNumber_InPlaceAdd(a, b) : PyNumber_Add(a, b); + if (result != NULL && BY_UNLIKELY(!PyUnicode_Check(result))) { + By_TypeError("str", result); + Py_DECREF(result); + return NULL; + } + return result; +} + +/* `a + b` or `a += b` where the checker says both are `str` + * + * the interpreter's own concatenation is the answer only for two exact `str`s, which + * have no operator methods a program can have replaced */ +static inline PyObject *By_StrAdd(PyObject *a, PyObject *b, int in_place) { + if (BY_LIKELY(PyUnicode_CheckExact(a) && PyUnicode_CheckExact(b))) { + return PyUnicode_Concat(a, b); + } + return By_StrAddSlow(a, b, in_place); +} + /* the widest decimal an ssize_t reaches, with room for a sign and a terminator */ #define BY_INT_DIGITS 22 @@ -8077,6 +8693,92 @@ static inline PyObject *By_ShortToStr(Py_ssize_t value) { return text; } +/* whether `fn` is the interpreter's own builtin `name` + * + * what a name resolves to is looked up on every call, and a native lowering of a builtin + * is only exact while the lookup answers with the builtin itself. that is a type the + * interpreter defines statically under that bare name, which is how a type's module + * comes out as `builtins`, or a C function the `builtins` module owns under that name. + * neither can be built from python, and a module attribute written from outside or a + * patched `builtins` is neither. + * + * the answer for the object last found to be the builtin is remembered in `genuine`, + * which holds a reference to it, so that no other object can come to live at the + * address it names and be taken for it */ +static inline char By_IsBuiltin(PyObject *fn, const char *name, PyObject **genuine) { + if (BY_LIKELY(fn == *genuine)) return 1; + if (PyType_Check(fn)) { + PyTypeObject *type = (PyTypeObject *)fn; + if ((type->tp_flags & Py_TPFLAGS_HEAPTYPE) || strcmp(type->tp_name, name) != 0) { + return 0; + } + } else if (PyCFunction_CheckExact(fn)) { + PyObject *owner = PyCFunction_GET_SELF(fn); + const char *module; + if (strcmp(((PyCFunctionObject *)fn)->m_ml->ml_name, name) != 0) return 0; + if (owner == NULL || !PyModule_CheckExact(owner)) return 0; + module = PyModule_GetName(owner); + if (module == NULL) { + PyErr_Clear(); + return 0; + } + if (strcmp(module, "builtins") != 0) return 0; + } else { + return 0; + } + Py_XSETREF(*genuine, Py_NewRef(fn)); + return 1; +} + +/* one site asking whether a name still resolves to the builtin of that name + * + * the answer is kept beside the lookup's memo and stands exactly as long as that does: + * any write to a namespace the answer could have come from moves the counter, and the + * next ask looks the name up again */ +typedef struct { + ByGlobalSite lookup; + char answer; + /* interned on the first ask, which is the slow one anyway */ + PyObject *name; + /* see [`By_IsBuiltin`] */ + PyObject *genuine; +} ByBuiltinSite; + +#define BY_BUILTIN_SITE_INIT { BY_GLOBAL_SITE_INIT, 0, NULL, NULL } + +/* resolve the name and work the answer out again. 2 is the `NameError` of a name bound + * nowhere, or a failure to intern it */ +static char By_ArmBuiltinSite(ByBuiltinSite *site, PyObject *dict, const char *builtin) { + PyObject *found; + if (site->name == NULL) { + site->name = By_InternedStr(builtin, (Py_ssize_t)strlen(builtin)); + if (site->name == NULL) return 2; + } +#ifdef BY_GLOBAL_SITES + found = By_ArmGlobalSite(&site->lookup, dict, site->name); +#else + found = By_LookupGlobal(dict, site->name); +#endif + if (found == NULL) return 2; + site->answer = By_IsBuiltin(found, builtin, &site->genuine); + Py_DECREF(found); + return site->answer; +} + +/* whether `builtin` resolves, through the module namespace and then builtins, to the + * interpreter's own builtin of that name + * + * no reference is taken on the way, and while nothing has been written to a namespace + * the answer is one comparison and a load: it is asked every trip round a loop */ +static inline char By_BuiltinStands(ByBuiltinSite *site, PyObject *dict, const char *builtin) { +#ifdef BY_GLOBAL_SITES + if (BY_LIKELY(by_globals != NULL && site->lookup.generation == by_globals->generation)) { + return site->answer; + } +#endif + return By_ArmBuiltinSite(site, dict, builtin); +} + /* `str(n)` for a tagged integer, given whatever the name `str` resolved to * * the resolution is the caller's and still happens every time, so a module that @@ -8142,32 +8844,26 @@ static inline PyObject *By_ConcatShortToStr(PyObject *left, Py_ssize_t value) { * is compared rather than assumed, so a module that rebinds `str` is obeyed, and a * tagged value that is not short holds an object whose `__str__` has to be asked. * - * the slow path checks what came back before concatenating it, because a rebound - * `str` may return anything at all — that is the check the unfused shape made - * between the two operations, in the same place and with the same message. the fast - * path needs none for the digits: it built them itself. - * - * it does test `left`, which the ir says is a `str` and every path into here should - * have made one. reading a header that is not a string's would decide how long the - * answer is from whatever the field happens to overlap, and getting *that* wrong is - * memory written past the end rather than a wrong answer — so the invariant is - * tested rather than trusted, and a left operand that fails goes the long way and is - * refused by `PyUnicode_Concat` exactly as it was before */ -static inline PyObject *By_StrConcatInt(PyObject *left, PyObject *fn, ByTagged n) { + * the slow path hands what came back straight to the operator, because a rebound + * `str` may return anything at all and python adds whatever it returned: a `str` + * subclass answers through its own `__radd__`, and anything else is refused in the + * operator's own words. the fast path needs no such care for the digits: it built + * them itself. + * + * it tests `left` for an exact `str`, both because a subclass may define `__add__` + * and because reading a header that is not a string's would decide how long the + * answer is from whatever the field happens to overlap — memory written past the end + * rather than a wrong answer */ +static inline PyObject *By_StrConcatInt(PyObject *left, PyObject *fn, ByTagged n, int in_place) { if (BY_LIKELY(fn == (PyObject *)&PyUnicode_Type && By_IsShort(n) - && PyUnicode_Check(left))) { + && PyUnicode_CheckExact(left))) { return By_ConcatShortToStr(left, By_ShortValue(n)); } { PyObject *right = By_StrOfInt(fn, n); PyObject *result; if (right == NULL) return NULL; - if (BY_UNLIKELY(!PyUnicode_Check(right))) { - By_TypeError("str", right); - Py_DECREF(right); - return NULL; - } - result = PyUnicode_Concat(left, right); + result = By_StrAdd(left, right, in_place); Py_DECREF(right); return result; } @@ -8202,6 +8898,24 @@ static inline PyObject *By_StrAppend(PyObject *left, PyObject *right) { return result; } +/* `By_StrAppend` for `a + b` or `a += b`, which asks a subclass's operator methods + * + * the in-place append is the interpreter's own concatenation, so it is taken for two + * exact `str`s alone, and the reference to `left` is consumed on every path here too */ +static inline PyObject *By_StrAddAppend(PyObject *left, PyObject *right, int in_place) { + if (BY_UNLIKELY(left == NULL || right == NULL)) { + Py_XDECREF(left); + return NULL; + } + if (BY_LIKELY(left != right && PyUnicode_CheckExact(left) && PyUnicode_CheckExact(right))) { + PyUnicode_Append(&left, right); /* NULLs `left` when it fails */ + return left; + } + PyObject *result = By_StrAdd(left, right, in_place); + Py_DECREF(left); + return result; +} + /* whether two exact `str`s hold the same text * * the same three tests `unicode_richcompare` makes, in the same order: a string is @@ -8249,15 +8963,17 @@ static inline char By_StrCompare(PyObject *a, PyObject *b, int op) { * comparison — twelve per cent, against six per cent gained on the inheritance * benchmark. so this one stays whole */ static inline ByTagged By_Len(PyObject *o) { - // the common containers know their own size in a field + // the common containers know their own size in a field, and hold at least a byte an + // item, so the size is below the short range on any address space there is if (PyList_CheckExact(o)) return By_ShortFrom(PyList_GET_SIZE(o)); if (PyUnicode_CheckExact(o)) return By_ShortFrom(PyUnicode_GET_LENGTH(o)); if (PyTuple_CheckExact(o)) return By_ShortFrom(PyTuple_GET_SIZE(o)); if (PyDict_CheckExact(o)) return By_ShortFrom(PyDict_GET_SIZE(o)); if (PyBytes_CheckExact(o)) return By_ShortFrom(PyBytes_GET_SIZE(o)); + // anything else answers `__len__` with whatever it likes, `range(2**62)` included Py_ssize_t length = PyObject_Length(o); if (length < 0) return BY_INT_ERROR; - return By_ShortFrom(length); + return By_IntFromI64((int64_t)length); } /* raise `cls(message)` — the shape `assert` and a bare `raise Cls(...)` need */ @@ -9084,28 +9800,19 @@ typedef struct { * same two comparisons as a hit */ static void By_ArmProtocolSite(ByProtocolSite *site, PyTypeObject *tp, PyObject *name) { PyObject *found; - int stands; site->type = (PyObject *)tp; site->version = tp->tp_version_tag; site->method = NULL; - /* a metaclass can answer the name with something the class's own version tag says - * nothing about: `PyType_Modified` on a metaclass reaches that metaclass's - * subclasses, and not the classes that are instances of it. refusing anything but a - * plain `type` also pins the lookup below to `type.__getattribute__`, so nothing of - * the class's own can run inside it */ - if (!Py_IS_TYPE(tp, &PyType_Type)) return; - found = PyObject_GetAttr((PyObject *)tp, name); - if (found == NULL) { - PyErr_Clear(); - return; - } - /* what is kept is borrowed, so it has to be the object the type's dict holds rather - * than something a `__get__` built on the way out and handed over. these two are - * the ones that give themselves back when the name is read off the class, where a - * `classmethod` or a `property` would build something */ - stands = PyFunction_Check(found) || Py_IS_TYPE(found, &PyMethodDescr_Type); - Py_DECREF(found); - if (!stands) return; + /* the entry the type's mro holds, which is what the interpreter looks a special + * method up as. it is borrowed from that dict and no descriptor runs to find it, so + * nothing a `__get__` builds on the way — a `functools.partialmethod` builds a new + * function every time — can be what is kept */ + found = _PyType_Lookup(tp, name); + if (found == NULL) return; + /* the two that bind to the manager by taking it as their first argument, so calling + * the entry with the manager in front is what binding it would have done. anything + * else binds some other way, or not at all, and is looked up in full every time */ + if (!PyFunction_Check(found) && !Py_IS_TYPE(found, &PyMethodDescr_Type)) return; #if PY_VERSION_HEX >= 0x030C0000 /* from 3.12 the tag is handed out on request rather than by whoever reads an * attribute, and a request is the only thing that reliably produces one */ @@ -9119,23 +9826,143 @@ static void By_ArmProtocolSite(ByProtocolSite *site, PyTypeObject *tp, PyObject #endif /* Py_GIL_DISABLED */ -/* `getattr(type(manager), name)` through a memo of what it last answered */ +/* a special method of `self`, looked up the way the interpreter looks one up for a + * protocol: on the type alone, past the instance and past any metaclass, and bound to + * `self` through the descriptor protocol + * + * `*prepend` says whether the answer still wants `self` in front of the arguments. a + * plain function and a method descriptor are handed back unbound, which saves building + * the bound method they would give; everything else comes back already bound, or is + * not a descriptor and is called as it is. NULL with no exception set is a name the type + * does not have */ +static inline PyObject *By_LookupSpecial(PyObject *self, PyObject *name, int *prepend) { + PyObject *found = _PyType_Lookup(Py_TYPE(self), name); + *prepend = 0; + if (found == NULL) return NULL; + if (PyFunction_Check(found) || Py_IS_TYPE(found, &PyMethodDescr_Type)) { + *prepend = 1; + return Py_NewRef(found); + } + descrgetfunc get = Py_TYPE(found)->tp_descr_get; + if (get == NULL) return Py_NewRef(found); + /* the entry is only borrowed, and `__get__` is arbitrary python */ + Py_INCREF(found); + PyObject *bound = get(found, self, (PyObject *)Py_TYPE(self)); + Py_DECREF(found); + return bound; +} + +/* [`By_LookupSpecial`] through a memo of what it last answered */ static inline PyObject *By_ProtocolMethod(ByProtocolSite *site, PyObject *manager, - PyObject *name) { + PyObject *name, int *prepend) { PyTypeObject *tp = Py_TYPE(manager); #ifndef Py_GIL_DISABLED if (BY_UNLIKELY((PyObject *)tp != site->type || tp->tp_version_tag != site->version)) { if (site->misses >= BY_PROTOCOL_SITE_MISSES) { - return PyObject_GetAttr((PyObject *)tp, name); + return By_LookupSpecial(manager, name, prepend); } site->misses++; By_ArmProtocolSite(site, tp, name); } - if (BY_LIKELY(site->method != NULL)) return By_NewRef(site->method); + if (BY_LIKELY(site->method != NULL)) { + *prepend = 1; + return By_NewRef(site->method); + } #else (void)site; #endif - return PyObject_GetAttr((PyObject *)tp, name); + return By_LookupSpecial(manager, name, prepend); +} + +/* call what [`By_ProtocolMethod`] answered with `args`, whose first slot holds the + * manager and is left out where the answer is already bound */ +static inline PyObject *By_CallProtocol(PyObject *method, int prepend, PyObject **args, + Py_ssize_t nargs) { + if (prepend) return PyObject_Vectorcall(method, args, (size_t)nargs, NULL); + return PyObject_Vectorcall(method, args + 1, + (size_t)(nargs - 1) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); +} + +/* whether a manager's type has `name`, the half of the protocol a `with` calls on the way + * out: 1 or 0, and -1 with an exception set + * + * python looks both halves up before it calls either, so a manager missing `__exit__` is + * refused before its `__enter__` runs. only the presence is asked here, and the half is + * looked up again where it is called. a site whose memo still stands for this type was + * armed by a call that got past this test already, and a type written to since is a + * miss */ +static inline int By_ManagerHas(ByProtocolSite *site, PyObject *manager, PyObject **cache, + const char *name, Py_ssize_t length) { + PyTypeObject *tp = Py_TYPE(manager); + PyObject *key; +#ifndef Py_GIL_DISABLED + if (BY_LIKELY((PyObject *)tp == site->type && tp->tp_version_tag == site->version + && site->method != NULL)) { + return 1; + } +#else + (void)site; +#endif + key = By_FixedName(cache, name, length); + if (key == NULL) return -1; + return _PyType_Lookup(tp, key) != NULL; +} + +/* the refusal for a manager whose type lacks `exit`, the half a `with` calls on the way + * out, where `has_enter` says whether it has the other half + * + * from 3.14 the half called on the way out is looked up first, and the message names + * whichever half is missing; before that the half called on the way in is looked up + * first, and only a missing exit is named */ +BY_COLD void By_ManagerMissingExit(PyObject *manager, const char *protocol, const char *exit, + int has_enter) { +#if PY_VERSION_HEX >= 0x030E0000 + (void)has_enter; + PyErr_Format(PyExc_TypeError, "'%s' object does not support the %s protocol (missed %s method)", + Py_TYPE(manager)->tp_name, protocol, exit); +#else + if (!has_enter) { + PyErr_Format(PyExc_TypeError, "'%s' object does not support the %s protocol", + Py_TYPE(manager)->tp_name, protocol); + return; + } + PyErr_Format(PyExc_TypeError, "'%s' object does not support the %s protocol (missed %s method)", + Py_TYPE(manager)->tp_name, protocol, exit); +#endif +} + +/* the refusal for a manager whose type has the half called on the way out and lacks + * `enter`, the one called on the way in */ +BY_COLD void By_ManagerMissingEnter(PyObject *manager, const char *protocol, const char *enter) { +#if PY_VERSION_HEX >= 0x030E0000 + PyErr_Format(PyExc_TypeError, "'%s' object does not support the %s protocol (missed %s method)", + Py_TYPE(manager)->tp_name, protocol, enter); +#else + (void)enter; + PyErr_Format(PyExc_TypeError, "'%s' object does not support the %s protocol", + Py_TYPE(manager)->tp_name, protocol); +#endif +} + +/* a manager's `enter` and its callable, after checking it has `exit` as python does before + * calling either — NULL with an exception set where it lacks one */ +static inline PyObject *By_ManagerEnter(ByProtocolSite *site, PyObject *manager, + PyObject **enter_name, const char *enter, + PyObject **exit_name, const char *exit, + const char *protocol, int *prepend) { + PyObject *name; + PyObject *method; + int has_exit = By_ManagerHas(site, manager, exit_name, exit, (Py_ssize_t)strlen(exit)); + if (has_exit < 0) return NULL; + name = By_FixedName(enter_name, enter, (Py_ssize_t)strlen(enter)); + if (name == NULL) return NULL; + if (!has_exit) { + By_ManagerMissingExit(manager, protocol, exit, _PyType_Lookup(Py_TYPE(manager), name) != NULL); + return NULL; + } + method = By_ProtocolMethod(site, manager, name, prepend); + if (method == NULL && !PyErr_Occurred()) By_ManagerMissingEnter(manager, protocol, enter); + return method; } /* `__aenter__` and `__aexit__`, which hand back *awaitables* rather than answers @@ -9145,19 +9972,14 @@ static inline PyObject *By_ProtocolMethod(ByProtocolSite *site, PyObject *manage */ static inline PyObject *By_AsyncEnter(ByProtocolSite *site, PyObject *manager) { static PyObject *by_aenter = NULL; - PyObject *name; + static PyObject *by_aexit = NULL; if (manager == NULL) return NULL; - name = By_FixedName(&by_aenter, "__aenter__", 10); - if (name == NULL) return NULL; - PyObject *method = By_ProtocolMethod(site, manager, name); - if (method == NULL) { - PyErr_Format(PyExc_TypeError, - "'%s' object does not support the asynchronous context manager protocol", - Py_TYPE(manager)->tp_name); - return NULL; - } + int prepend; + PyObject *method = By_ManagerEnter(site, manager, &by_aenter, "__aenter__", &by_aexit, + "__aexit__", "asynchronous context manager", &prepend); + if (method == NULL) return NULL; PyObject *args[1] = {manager}; - PyObject *result = PyObject_Vectorcall(method, args, 1, NULL); + PyObject *result = By_CallProtocol(method, prepend, args, 1); Py_DECREF(method); return result; } @@ -9169,8 +9991,10 @@ static inline PyObject *By_AsyncExit(ByProtocolSite *site, PyObject *manager, if (manager == NULL) return NULL; name = By_FixedName(&by_aexit, "__aexit__", 9); if (name == NULL) return NULL; - PyObject *method = By_ProtocolMethod(site, manager, name); + int prepend; + PyObject *method = By_ProtocolMethod(site, manager, name, &prepend); if (method == NULL) { + if (PyErr_Occurred()) return NULL; PyErr_Format(PyExc_TypeError, "'%s' object does not support the asynchronous context manager protocol " "(missed __aexit__ method)", @@ -9188,7 +10012,7 @@ static inline PyObject *By_AsyncExit(ByProtocolSite *site, PyObject *manager, if (found != NULL) traceback = found; } PyObject *args[4] = {manager, type, value, traceback}; - PyObject *result = PyObject_Vectorcall(method, args, 4, NULL); + PyObject *result = By_CallProtocol(method, prepend, args, 4); Py_DECREF(method); Py_XDECREF(found); return result; @@ -9196,18 +10020,14 @@ static inline PyObject *By_AsyncExit(ByProtocolSite *site, PyObject *manager, static inline PyObject *By_Enter(ByProtocolSite *site, PyObject *manager) { static PyObject *by_enter = NULL; - PyObject *name; + static PyObject *by_exit = NULL; if (manager == NULL) return NULL; - name = By_FixedName(&by_enter, "__enter__", 9); - if (name == NULL) return NULL; - PyObject *method = By_ProtocolMethod(site, manager, name); - if (method == NULL) { - PyErr_Format(PyExc_TypeError, "'%s' object does not support the context manager protocol", - Py_TYPE(manager)->tp_name); - return NULL; - } + int prepend; + PyObject *method = By_ManagerEnter(site, manager, &by_enter, "__enter__", &by_exit, + "__exit__", "context manager", &prepend); + if (method == NULL) return NULL; PyObject *args[1] = {manager}; - PyObject *result = PyObject_Vectorcall(method, args, 1, NULL); + PyObject *result = By_CallProtocol(method, prepend, args, 1); Py_DECREF(method); return result; } @@ -9224,8 +10044,17 @@ static inline int By_ExitContext(ByProtocolSite *site, PyObject *manager, if (manager == NULL) return -1; name = By_FixedName(&by_exit, "__exit__", 8); if (name == NULL) return -1; - PyObject *method = By_ProtocolMethod(site, manager, name); - if (method == NULL) return -1; + int prepend; + PyObject *method = By_ProtocolMethod(site, manager, name, &prepend); + if (method == NULL) { + if (!PyErr_Occurred()) { + PyErr_Format(PyExc_TypeError, + "'%s' object does not support the context manager protocol " + "(missed __exit__ method)", + Py_TYPE(manager)->tp_name); + } + return -1; + } /* `None` is the normal path just as NULL is: the frontend hands over a boxed `None`, and reading a traceback off it would be a wild pointer */ int raising = exception != NULL && exception != Py_None @@ -9238,7 +10067,7 @@ static inline int By_ExitContext(ByProtocolSite *site, PyObject *manager, if (found != NULL) traceback = found; } PyObject *args[4] = {manager, type, value, traceback}; - PyObject *result = PyObject_Vectorcall(method, args, 4, NULL); + PyObject *result = By_CallProtocol(method, prepend, args, 4); Py_DECREF(method); if (traceback != Py_None) Py_DECREF(traceback); if (result == NULL) return -1; @@ -9294,6 +10123,22 @@ static inline PyObject *By_DelegateStep(PyObject *inner, PyObject *sent, int *do return result; } +/* whether a generator's code carries `CO_ITERABLE_COROUTINE`, which `types.coroutine` + * sets to let `await` drive it: 1 or 0, and -1 with an exception set */ +static int By_GeneratorIsCoroutine(PyObject *generator) { + PyObject *code; + int flags; +#if PY_VERSION_HEX >= 0x030C0000 + code = (PyObject *)PyGen_GetCode((PyGenObject *)generator); +#else + code = PyObject_GetAttrString(generator, "gi_code"); +#endif + if (code == NULL) return -1; + flags = PyCode_Check(code) ? ((PyCodeObject *)code)->co_flags : 0; + Py_DECREF(code); + return (flags & CO_ITERABLE_COROUTINE) != 0; +} + /* the iterator a delegation drives: `iter(x)` for `yield from`, `x.__await__()` for * `await`. keeping them apart matters — awaiting an ordinary iterable is an error * @@ -9310,6 +10155,13 @@ static inline PyObject *By_AwaitIter(PyObject *awaitable) { /* a coroutine is already the thing to drive: its own `__await__` only hands * back a wrapper around itself */ if (PyCoro_CheckExact(awaitable)) return By_NewRef(awaitable); + /* and so is a generator `types.coroutine` marked as one, which has no `__await__` + * at all — python tells it by the flag on its code */ + if (PyGen_CheckExact(awaitable)) { + int marked = By_GeneratorIsCoroutine(awaitable); + if (marked < 0) return NULL; + if (marked) return By_NewRef(awaitable); + } type = Py_TYPE(awaitable); getter = type->tp_as_async == NULL ? NULL : type->tp_as_async->am_await; if (getter == NULL) { @@ -9444,7 +10296,7 @@ static inline PyObject *By_ReadCell(PyObject *value, const char *name, int free) * `UnboundLocalError`, one that closes over it sees a free variable and a * plain `NameError`. python distinguishes the two, wording included */ PyErr_Format(free ? PyExc_NameError : PyExc_UnboundLocalError, - free ? "cannot access free variable '%s' where it is not associated with a value" + free ? "cannot access free variable '%s' where it is not associated with a value in enclosing scope" : "cannot access local variable '%s' where it is not associated with a value", name); return NULL; @@ -9452,12 +10304,265 @@ static inline PyObject *By_ReadCell(PyObject *value, const char *name, int free) return By_NewRef(value); } -/* bind a method to a receiver, giving a callable. this is what a nested - * function's name is bound to: the receiver is its closure environment, and the - * fastcall convention hands it back as `self` on every call */ -static inline PyObject *By_MakeClosure(PyMethodDef *def, PyObject *env) { +/* read a shared cell held as a tagged `int`. its error value is what unset means, + * because a zero is a value like any other */ +static inline ByTagged By_ReadCellTagged(ByTagged value, const char *name, int free) { + if (value == BY_INT_ERROR) { + (void)By_ReadCell(NULL, name, free); + return BY_INT_ERROR; + } + By_IncRefTagged(value); + return value; +} + +/* the flag a type with a vectorcall slot sets, public from 3.12 */ +#ifndef Py_TPFLAGS_HAVE_VECTORCALL +#define Py_TPFLAGS_HAVE_VECTORCALL _Py_TPFLAGS_HAVE_VECTORCALL +#endif + +/* ── a compiled nested function ────────────────────────────────────────────── + * + * a nested function is a closure over its environment, and python's own is a + * `function`: a descriptor, so installed on a class it binds the receiver; a thing + * with a `__dict__` and a writable `__name__`, so `functools.wraps` can dress it; and + * a thing that says where it was written, `counter..step`, rather than what + * holds its captures. a `PyCFunction` over the environment is none of those, and the + * first is a silent wrong answer: a method installed from one never receives `self`. + * + * so a nested function is one of these. a call is `vectorcall` straight into the + * function's own boundary, which reads the environment off the object — there is no + * trampoline between the caller and the boundary. what the definition says about + * itself is in a static `ByFunctionSpec`, and a name, qualname, module or docstring + * only becomes an object when something reads it or writes over it. + * + * where the body reads its own name, the environment holds the function and the + * function holds the environment, so both are collected types: this one traverses + * the environment, and the environment traverses its cells. `tp_clear` leaves the + * environment alone, so a call reached from a finalizer during a collection still has + * one to read — clearing the environment's cells is what breaks the cycle. + * + * `copy` treats only `function` and `builtin_function_or_method` as atomic, so this + * says it is atomic itself: a copy of a function is the function. and pickling one + * pickles it by name, as python does, which for a nested function is the same refusal + * python gives, in its own words */ +typedef struct { + /* the boundary, handed the function object itself as the callable */ + vectorcallfunc call; + const char *name; + const char *qualname; + /* NULL where the definition has no docstring */ + const char *doc; + /* the module's namespace, where `__module__` is read from */ + PyObject **globals; +} ByFunctionSpec; + +typedef struct { + PyObject_HEAD + vectorcallfunc vectorcall; + PyObject *env; + const ByFunctionSpec *spec; + PyObject *name; + PyObject *qualname; + PyObject *module; + PyObject *doc; + PyObject *dict; +} ByFunctionObject; + +static void By_Function_dealloc(ByFunctionObject *self) { + PyObject_GC_UnTrack(self); + Py_CLEAR(self->env); + Py_CLEAR(self->name); + Py_CLEAR(self->qualname); + Py_CLEAR(self->module); + Py_CLEAR(self->doc); + Py_CLEAR(self->dict); + PyObject_GC_Del(self); +} + +static int By_Function_traverse(ByFunctionObject *self, visitproc visit, void *arg) { + Py_VISIT(self->env); + Py_VISIT(self->module); + Py_VISIT(self->doc); + Py_VISIT(self->dict); + return 0; +} + +static int By_Function_clear(ByFunctionObject *self) { + Py_CLEAR(self->module); + Py_CLEAR(self->doc); + Py_CLEAR(self->dict); + return 0; +} + +/* bound the way a python function is: through the type it is itself, and through + * an instance a method of that instance */ +static PyObject *By_Function_descr_get(PyObject *self, PyObject *obj, PyObject *type) { + (void)type; + if (obj == NULL || obj == Py_None) return By_NewRef(self); + return PyMethod_New(self, obj); +} + +static PyObject *By_Function_repr(ByFunctionObject *self) { + PyObject *qualname = self->qualname != NULL + ? By_NewRef(self->qualname) + : PyUnicode_FromString(self->spec->qualname); + if (qualname == NULL) return NULL; + PyObject *repr = PyUnicode_FromFormat("", qualname, (void *)self); + Py_DECREF(qualname); + return repr; +} + +/* one of the two names: what was written over it, or else what the definition says */ +static PyObject *By_Function_name_of(PyObject **slot, const char *spelled) { + if (*slot == NULL) { + *slot = PyUnicode_InternFromString(spelled); + if (*slot == NULL) return NULL; + } + return By_NewRef(*slot); +} + +static int By_Function_set_name_of(PyObject **slot, PyObject *value, const char *which) { + if (value == NULL || !PyUnicode_Check(value)) { + PyErr_Format(PyExc_TypeError, "%s must be set to a string object", which); + return -1; + } + Py_XSETREF(*slot, By_NewRef(value)); + return 0; +} + +static PyObject *By_Function_get_name(PyObject *self, void *closure) { + (void)closure; + ByFunctionObject *function = (ByFunctionObject *)self; + return By_Function_name_of(&function->name, function->spec->name); +} + +static int By_Function_set_name(PyObject *self, PyObject *value, void *closure) { + (void)closure; + return By_Function_set_name_of(&((ByFunctionObject *)self)->name, value, "__name__"); +} + +static PyObject *By_Function_get_qualname(PyObject *self, void *closure) { + (void)closure; + ByFunctionObject *function = (ByFunctionObject *)self; + return By_Function_name_of(&function->qualname, function->spec->qualname); +} + +static int By_Function_set_qualname(PyObject *self, PyObject *value, void *closure) { + (void)closure; + return By_Function_set_name_of(&((ByFunctionObject *)self)->qualname, value, + "__qualname__"); +} + +/* python takes a function's `__module__` from its globals' `__name__` where the `def` + * runs, and lets it be written over afterwards */ +static PyObject *By_Function_get_module(PyObject *self, void *closure) { + (void)closure; + ByFunctionObject *function = (ByFunctionObject *)self; + if (function->module == NULL) { + PyObject *globals = *function->spec->globals; + PyObject *name = globals == NULL ? NULL : PyDict_GetItemString(globals, "__name__"); + function->module = By_NewRef(name == NULL ? Py_None : name); + } + return By_NewRef(function->module); +} + +static int By_Function_set_module(PyObject *self, PyObject *value, void *closure) { + (void)closure; + Py_XSETREF(((ByFunctionObject *)self)->module, By_NewRef(value == NULL ? Py_None : value)); + return 0; +} + +static PyObject *By_Function_get_doc(PyObject *self, void *closure) { + (void)closure; + ByFunctionObject *function = (ByFunctionObject *)self; + if (function->doc == NULL) { + function->doc = function->spec->doc == NULL + ? By_NewRef(Py_None) + : PyUnicode_FromString(function->spec->doc); + if (function->doc == NULL) return NULL; + } + return By_NewRef(function->doc); +} + +static int By_Function_set_doc(PyObject *self, PyObject *value, void *closure) { + (void)closure; + Py_XSETREF(((ByFunctionObject *)self)->doc, By_NewRef(value == NULL ? Py_None : value)); + return 0; +} + +static PyObject *By_Function_self(PyObject *self, PyObject *unused) { + (void)unused; + return By_NewRef(self); +} + +/* pickled by name, as a python function is. the name of a nested one runs through + * ``, so pickle refuses it exactly as it refuses python's */ +static PyObject *By_Function_reduce(PyObject *self, PyObject *unused) { + (void)unused; + return By_Function_get_qualname(self, NULL); +} + +static PyMethodDef By_Function_methods[] = { + {"__copy__", By_Function_self, METH_NOARGS, NULL}, + {"__deepcopy__", By_Function_self, METH_O, NULL}, + {"__reduce__", By_Function_reduce, METH_NOARGS, NULL}, + {NULL, NULL, 0, NULL}, +}; + +static PyGetSetDef By_Function_getset[] = { + {"__name__", By_Function_get_name, By_Function_set_name, NULL, NULL}, + {"__qualname__", By_Function_get_qualname, By_Function_set_qualname, NULL, NULL}, + {"__module__", By_Function_get_module, By_Function_set_module, NULL, NULL}, + {"__doc__", By_Function_get_doc, By_Function_set_doc, NULL, NULL}, + {"__dict__", PyObject_GenericGetDict, PyObject_GenericSetDict, NULL, NULL}, + {NULL, NULL, NULL, NULL, NULL}, +}; + +static PyTypeObject By_FunctionType = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "by.function", + .tp_basicsize = sizeof(ByFunctionObject), + .tp_itemsize = 0, + .tp_dealloc = (destructor)By_Function_dealloc, + .tp_vectorcall_offset = offsetof(ByFunctionObject, vectorcall), + .tp_repr = (reprfunc)By_Function_repr, + .tp_call = PyVectorcall_Call, + .tp_getattro = PyObject_GenericGetAttr, + .tp_setattro = PyObject_GenericSetAttr, + /* a method descriptor as python's own function is one: `obj.f()` through the type + * calls it with `obj` in front rather than building a bound method first */ + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HAVE_VECTORCALL + | Py_TPFLAGS_METHOD_DESCRIPTOR, + .tp_traverse = (traverseproc)By_Function_traverse, + .tp_clear = (inquiry)By_Function_clear, + .tp_methods = By_Function_methods, + .tp_getset = By_Function_getset, + .tp_descr_get = By_Function_descr_get, + .tp_dictoffset = offsetof(ByFunctionObject, dict), + .tp_free = PyObject_GC_Del, +}; + +/* the function a `def` binds, over the environment the frame made for it */ +static inline PyObject *By_MakeFunction(const ByFunctionSpec *spec, PyObject *env) { + ByFunctionObject *self; if (env == NULL) return NULL; - return PyCFunction_NewEx(def, env, NULL); + self = PyObject_GC_New(ByFunctionObject, &By_FunctionType); + if (self == NULL) return NULL; + self->vectorcall = spec->call; + self->env = By_NewRef(env); + self->spec = spec; + self->name = NULL; + self->qualname = NULL; + self->module = NULL; + self->doc = NULL; + self->dict = NULL; + PyObject_GC_Track(self); + return (PyObject *)self; +} + +/* the environment a nested function's boundary reads its captures from */ +static inline PyObject *By_FunctionEnvironment(PyObject *callable) { + return ((ByFunctionObject *)callable)->env; } /* narrowing an object to a refcounted type is a test rather than a change of diff --git a/docs/basedpython/development/compilation/ir.md b/docs/basedpython/development/compilation/ir.md index ae8873c134..098b0c6f3e 100644 --- a/docs/basedpython/development/compilation/ir.md +++ b/docs/basedpython/development/compilation/ir.md @@ -216,7 +216,7 @@ the ops that exist, beyond the arithmetic and control-flow ones above: | `CallPython` / `CallMethod` | a call resolved by name, or on a receiver | | `LoadGlobal` | the module namespace then its builtins, resolved per read | | `ImportModule` / `ImportFrom` | `from x import y` — the module with a fromlist, then a name | -| `NewInstance` / `MakeClosure` | allocate an emitted class; bind a method to it | +| `NewInstance` / `MakeClosure` | allocate an emitted class; make a nested function over one | | `GetAttr` / `SetAttr` | the attribute protocol, for a receiver with no layout | | `GetIter` / `IterNext` | the iteration protocol | | `BuildList` / `Set` / `Tuple` / `Dict` | a display | diff --git a/docs/basedpython/development/compilation/runtime.md b/docs/basedpython/development/compilation/runtime.md index d349ce2314..4937e205eb 100644 --- a/docs/basedpython/development/compilation/runtime.md +++ b/docs/basedpython/development/compilation/runtime.md @@ -629,6 +629,67 @@ callers and falls back to attribute lookup for the interpreted subclass the same reason mypyc does — the fixed layout must be initialized. a `data class` satisfies this automatically +## where compiled code differs from python + +each of these is a tradeoff made on purpose, and each names the shape of program +that can tell the two builds apart + +### a rebound `len` and a list held as a buffer + +a list of unboxed values that never leaves its function is held as a buffer +rather than as a `list`. `len` of it is a field read while `len` names the +builtin; a module that rebinds the name has what it rebound called instead, as +python does. a buffer has no `list` to hand over, though, and building one would +be a copy that the replacement could tell apart from the list python would have +passed, by identity or by writing to it. so that one call raises `RuntimeError` +instead: + +```python +def total(n: int) -> float: + xs = [i * 0.5 for i in range(n)] + out = 0.0 + j = 0 + while j < len(xs): + out = out + xs[j] + j = j + 1 + return out + + +# from another module +mod.len = lambda xs: 1 +mod.total(3) # python: 0.0; compiled: RuntimeError +``` + +### one module object for the whole process + +a compiled module keeps its namespace, and every memo of a name in it, in state +the whole process shares, so it can stand behind one module object and no more. +importing it again in the same interpreter hands back the module the first import +made, as importing a C extension that keeps no per-module state does. python would +build a second module and run its body again: + +```python +import sys +import mod + +del sys.modules["mod"] +import mod as again + +again is mod # python: False; compiled: True +``` + +importing it in a second interpreter — a subinterpreter, legacy or isolated — is +refused with an `ImportError`, since there is no second copy of that state to give +it: + +```python +import _interpreters +import mod + +sub = _interpreters.create("legacy") +_interpreters.exec(sub, "import mod") # python: imports; compiled: ImportError +``` + ## debugging and inspection - **`#line` directives** in the generated C point at `.by` source. gdb, lldb, From e95be1f1453a8dda5f5ae1010766faad0f42e155 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Mon, 14 Sep 2026 12:32:44 +1000 Subject: [PATCH 02/16] native: make compiled generators, coroutines and async generators behave as python's state objects collected from creation and refused construction from python, throw/close/athrow with python's arguments and wording, delegation through yield from and await, a running frame refused, the RuntimeWarning for a coroutine never awaited, weak references, generator expressions and lambdas inside a resumable frame, finally blocks on the exception path, and a traceback entry for each compiled frame an exception passes --- Cargo.lock | 1 + crates/by_build/tests/differential.rs | 2154 ++++++++++++++++- crates/by_codegen_c/src/lib.rs | 1011 ++++++-- crates/by_ir/src/builder.rs | 91 +- crates/by_ir/src/function.rs | 39 +- crates/by_ir/src/ops.rs | 100 +- crates/by_ir/src/print.rs | 28 +- crates/by_ir/src/verify.rs | 29 +- crates/by_irbuild/Cargo.toml | 1 + crates/by_irbuild/src/closures.rs | 322 ++- crates/by_irbuild/src/generators.rs | 39 +- crates/by_irbuild/src/lib.rs | 917 ++++++- crates/by_irbuild/src/shims.rs | 17 +- crates/by_irbuild/src/tests.rs | 94 + crates/by_opt/src/copy_propagation.rs | 4 + crates/by_opt/src/infallible.rs | 11 +- crates/by_opt/src/unswitch.rs | 1 + crates/by_rt/include/by.h | 673 ++++- .../development/compilation/plan.md | 8 +- .../development/compilation/runtime.md | 103 +- 20 files changed, 5124 insertions(+), 519 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9c2cc0537c..7b0ac07ea8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -387,6 +387,7 @@ dependencies = [ "ruff_python_ast", "ruff_python_parser", "ruff_python_stdlib", + "ruff_source_file", "ruff_text_size", "thin-vec", "ty_project", diff --git a/crates/by_build/tests/differential.rs b/crates/by_build/tests/differential.rs index edfc6ecdd1..8759bb0395 100644 --- a/crates/by_build/tests/differential.rs +++ b/crates/by_build/tests/differential.rs @@ -508,6 +508,33 @@ def _after_raising(gen): gc.collect() return out +# a `throw` or a `close` landing on a frame after `steps` resumptions, what it answered, +# what one more resumption answers, and what the frames logged. `send(None)` rather +# than `next`, so a coroutine is driven the same way a generator is +def _thrown_at(make, steps, *args): + log = [] + gen = make(log) + out = [gen.send(None) for _ in range(steps)] + try: + out.append(('answered', gen.throw(*args))) + except BaseException as e: + out.append((type(e).__name__, str(e))) + try: + out.append(('then', gen.send(None))) + except BaseException as e: + out.append((type(e).__name__, str(e))) + return (out, log) + +def _closed_at(make, steps): + log = [] + gen = make(log) + out = [gen.send(None) for _ in range(steps)] + try: + out.append(('closed', gen.close())) + except BaseException as e: + out.append((type(e).__name__, str(e))) + return (out, log) + import asyncio def _run(coro): @@ -534,6 +561,266 @@ def _drained(gen): pass return gen +# every way python has of making or re-making an object only a frame may make: a +# generator, a coroutine, and the awaitable an async generator hands back. each has +# to be refused, and re-running `__init__` has to be `object`'s, which does nothing. +# the type's own name is masked because a compiled one is named after its function and +# spelled with its module +def _forged(obj): + kind = type(obj) + out = [] + for attempt in (lambda: kind.__new__(kind), kind, lambda: object.__new__(kind), + lambda: __import__('copy').copy(obj), + lambda: __import__('pickle').dumps(obj)): + try: + attempt() + out.append('made') + except TypeError as e: + out.append(str(e).replace(kind.__module__ + '.', '').replace(kind.__name__, 'T')) + out.append(obj.__init__()) + out.append([hasattr(obj, name) for name in ('$state', '$sent', '$resume', 'i', 'n')]) + return out + +# a suspended generator held only through a cycle of its own: the collector is the one +# thing that can free it, and freeing it closes it +def _cycled(make): + log = [] + box = [None] + gen = make(box, log) + box[0] = gen + next(gen) + del gen, box + gc.collect() + return log + +# how many references to `payload` a hundred abandoned cycles through a generator leave +def _cycle_references(make): + import sys + payload = object() + gc.collect() + before = sys.getrefcount(payload) + for _ in range(100): + box = [None] + gen = make(payload, box) + box[0] = gen + next(gen) + del gen, box + gc.collect() + return sys.getrefcount(payload) - before + +# a frame handed probes that each try to resume the frame running them, by every entry +# point its surface has. `make` builds the frame over the probes, and the frame is +# reachable through `box` by the time any probe runs +def _reentry_probes(box, surface): + name = surface[0] + if surface == 'async': + return [lambda: next(box[name].__anext__()), + lambda: next(box[name].asend(None)), + lambda: next(box[name].athrow(ValueError('x'))), + lambda: next(box[name].aclose())] + return [lambda: box[name].send(None), + lambda: box[name].send(1), + lambda: box[name].throw(ValueError('x')), + lambda: box[name].close()] + ([lambda: next(box[name])] if surface == 'gen' else []) + +def _reentered(make, surface): + box = {} + frame = make(_reentry_probes(box, surface)) + box[surface[0]] = frame + if surface == 'gen': + return list(frame) + if surface == 'async': + return _run(_drain(frame)) + return _sent_once(frame) + +# a frame re-entered from inside itself by iterating it, the way a generator reached through +# a list it holds is. the box is `[the generator, whether to re-enter]` +def _iterated_from_inside(make): + box = [None, True] + gen = make(box) + box[0] = gen + out = [] + try: + for value in gen: + out.append(value) + except BaseException as e: + out.append((type(e).__name__, str(e))) + return out + +# every argument shape `throw` takes, each into a fresh frame suspended inside a handler +# that reports what it caught. the three-argument form is deprecated, and the warning is +# part of the answer. `start` resumes the frame to its first suspension, and `step` is how +# a throw is made — the method itself, or an async generator's `athrow` driven by hand +def _throw_shapes(make, start, step): + import types + tb = None + try: + raise KeyError('with a traceback') + except KeyError as e: + tb = e.__traceback__ + out = [] + for args in [(ValueError, ValueError('old'), None), (ValueError, 'text'), + (ValueError, ('a', 'b')), (ValueError, KeyError('k')), (ValueError, None), + (ValueError('i'), None), (ValueError('i'), 'v'), (ValueError, None, 5), + (ValueError, None, tb), (ValueError('i'), None, tb), (), (1, 2, 3, 4), (5,), + (int,)]: + frame = make() + start(frame) + with warnings.catch_warnings(record=True) as seen: + warnings.simplefilter('always') + try: + answer = step(frame, args) + except BaseException as e: + answer = (type(e).__name__, str(e)) + out.append((answer, [(str(w.message), w.category.__name__) for w in seen])) + return out + +# an awaitable that suspends exactly once, so a coroutine awaiting it has a suspension +# to be thrown into +@_types.coroutine +def _suspends_once(): + yield 'suspended' + +# one step of an awaitable driven by hand, with no event loop to install hooks on an +# async generator: what a `yield` finished it with, or the exception it raised +def _stepped_by_hand(awaitable): + try: + next(awaitable) + except StopIteration as e: + return e.value + return 'suspended' + +# each step in turn, and what it answered or raised. a `StopIteration` is an answer: it is +# how an awaitable hands back what it finished with +def _steps(*steps): + out = [] + for step in steps: + try: + out.append(('answered', step())) + except StopIteration as e: + out.append(('finished', e.value)) + except BaseException as e: + out.append((type(e).__name__, str(e))) + return out + +# a manager class whose `__exit__` a block can rebind on the class while it runs, and the +# method to rebind it to. `__aexit__` too, for an `async with` +def _rebindable_manager(): + class Manager: + def __init__(self, log): + self.log = log + def __enter__(self): + return self + def __exit__(self, *args): + self.log.append('bound at entry') + return False + async def __aenter__(self): + return self + async def __aexit__(self, *args): + self.log.append('bound at entry') + return False + def later(self, *args): + self.log.append('rebound') + return True + async def later_async(self, *args): + self.log.append('rebound') + return True + return Manager, later, later_async + +# a frame made over a log, and the steps `script` answers for it, run in turn +def _against(make, script): + log = [] + frame = make(log) + return (_steps(*script(frame)), log) + +# what python says of a coroutine asked to be an iterator, with the type's own name +# masked, and closed afterwards so it leaves no warning behind +def _as_iterator(coro): + kind = type(coro) + out = [] + for attempt in (lambda: next(coro), lambda: iter(coro), lambda: coro.__next__()): + try: + attempt() + out.append('iterated') + except (TypeError, AttributeError) as e: + out.append((type(e).__name__, + str(e).replace(kind.__module__ + '.', '').replace(kind.__name__, 'T'))) + out.append(isinstance(coro, __import__('collections.abc').abc.Iterator)) + coro.close() + return out + +# the iterator `__await__` hands back, driven through every method it has +def _await_wrapper(coro): + wrapper = coro.__await__() + out = [type(wrapper).__name__, iter(wrapper) is wrapper, _stepped_by_hand(wrapper)] + for attempt in (lambda: wrapper.send(None), lambda: wrapper.throw(ValueError('w')), + lambda: wrapper.close()): + try: + out.append(attempt()) + except BaseException as e: + out.append((type(e).__name__, str(e))) + return out + +# a frame that ignores `GeneratorExit` and suspends again instead: closing it is refused +# and leaves it suspended, a later step resumes it, and dropping it closes it once more — +# which fails again, reported through the unraisable hook. `close` and `step` are how +# this surface is closed and resumed +def _ignored_close(make, start, close, step): + import sys + seen = [] + log = [] + hook = sys.unraisablehook + sys.unraisablehook = lambda u: seen.append( + (type(u.exc_value).__name__, str(u.exc_value), u.err_msg and u.err_msg.split(' <')[0])) + try: + frame = make(log) + start(frame) + out = [] + for attempt in (close, step, close): + try: + out.append(attempt(frame)) + except BaseException as e: + out.append((type(e).__name__, str(e))) + del frame + gc.collect() + return (out, log, seen) + finally: + sys.unraisablehook = hook + +# every warning a frame `make` builds and drops leaves behind, down to the line blamed. +# the source is asked only whether there is one, since its repr names an address +def _dropped_frame_warnings(make): + with warnings.catch_warnings(record=True) as seen: + warnings.simplefilter('always') + make() + gc.collect() + return [(str(w.message), w.category.__name__, _os.path.basename(w.filename), w.lineno, + w.source is not None) for w in seen] + +# a weak reference to a frame `make` builds: it reaches the frame while the frame lives, +# and once `finish` has run and the frame is dropped it is dead and its callback has run +def _weakly(make, finish): + import weakref + called = [] + frame = make() + ref = weakref.ref(frame, lambda r: called.append('callback')) + alive = ref() is frame + finish(frame) + del frame + gc.collect() + return (alive, ref() is None, called) + +# a call with one of the module's names rebound for its duration, answering what came +# back or what it raised +def _with_module_name(module, name, value, fn, *args): + module.__dict__[name] = value + try: + return fn(*args) + except BaseException as e: + return (type(e).__name__, str(e)) + finally: + module.__dict__.pop(name, None) + # await something that is not a coroutine — an `asend` awaitable, say — so a driver # that insists on one can still reach it async def _awaits(aw): @@ -803,6 +1090,15 @@ def _rebound(m): out.append(m.held(mgr)) return out +# the traceback entries an exception carries, for the frames written in the module: the +# function each names and its line. only the base name of the file, for the reason +# `_warned_into` gives, and the harness's own frames left out, since the snippet runs +# from a string +def _frames(e): + import traceback + return [(f.filename.replace('\\\\', '/').rsplit('/', 1)[-1], f.name, f.lineno) + for f in traceback.extract_tb(e.__traceback__) if f.filename != ''] + def _chain(e): out = [] while e is not None: @@ -6864,6 +7160,51 @@ def drop() -> None: assert_eq!(out, "ImportError True\n20\n20"); } +#[test] +fn importing_a_compiled_module_again_allocates_nothing_that_stays() { + // python forgets the state pointer of the module a create slot hands back and has the + // exec slot allocate a fresh one, even for a module asking for no state at all. handing + // the one module back import after import used to leave an allocation behind each time + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = diff_root().join("by_diff_reimportleak"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +def total(n: int) -> int: + return n + 1 +"; + let options = Options { + language: by_irbuild::Language::Python, + ..Options::default() + }; + if build_source(source, "by_diff_reimportleak", &toolchain, &dir, &options).is_err() { + eprintln!("skipping: no working C toolchain"); + return; + } + let out = run( + &python, + &dir, + "import gc, sys\n\ + import by_diff_reimportleak as first\n\ + assert first.__file__.endswith('.so'), first.__file__\n\ + def again():\n\ + \x20 del sys.modules['by_diff_reimportleak']\n\ + \x20 import by_diff_reimportleak\n\ + for _ in range(100):\n\ + \x20 again()\n\ + gc.collect()\n\ + before = sys.getallocatedblocks()\n\ + for _ in range(2000):\n\ + \x20 again()\n\ + gc.collect()\n\ + grown = sys.getallocatedblocks() - before\n\ + print('stable' if grown < 200 else f'grew by {grown}')\n\ + print(sys.modules['by_diff_reimportleak'] is first, first.total(2))\n", + ); + assert_eq!(out, "stable\nTrue 3"); +} + #[test] fn a_native_class_has_a_fixed_layout() { let Some((python, toolchain)) = environment() else { @@ -9972,6 +10313,56 @@ def guarded(words: list[str], index: int) -> str: assert_eq!(out, "stable\nheldxxxx"); } +#[test] +fn an_attribute_the_instance_is_given_answers_before_a_method_of_the_same_name() { + // python looks an instance's own attribute up before a function its class holds, so + // an encoder handed a `default` calls that one and an encoder handed nothing calls the + // method — whether the name is called straight away or read first + agree_python_with_declines( + "fieldshadowsmethod", + "\ +from collections.abc import Callable + + +class Encoder: + def __init__(self, default: Callable[[object], object] | None = None) -> None: + if default is not None: + self.default = default + + def default(self, o: object) -> object: + return 'method' + + def called(self) -> object: + return self.default(1) + + def read(self) -> object: + return self.default + + +class Base: + def label(self) -> str: + return 'base method' + + +class Labelled(Base): + def __init__(self, label: Callable[[], str]) -> None: + self.label = label + + def shown(self) -> str: + return self.label() +", + &[ + "m.Encoder().called()", + "m.Encoder(str).called()", + "m.Encoder().read().__name__", + "m.Encoder(str).read()", + "m.Encoder.default(m.Encoder(str), 2)", + "m.Labelled(lambda: 'given').shown()", + "m.Labelled(lambda: 'given').label()", + ], + ); +} + #[test] fn a_mutable_capture_agrees() { // python closes over the *variable*, so all of these depend on both frames seeing @@ -10035,6 +10426,159 @@ def accumulate(values: list[int]) -> int: ); } +#[test] +fn a_capture_read_in_any_evaluated_position_reaches_the_closure() { + // a nested function captures every enclosing name it reads, wherever in a statement + // the read stands: an `elif` test, a `match` subject, a guard or a value pattern, an + // `except` clause's class, and a default a `def` nested one level further evaluates + // in the frame around it. a read left out of the analysis was looked up in the + // module namespace instead, and raised `NameError` + agree_python( + "capturepositions", + "\ +from collections.abc import Callable + + +def elif_test(x: int) -> Callable[[int], int]: + def inner(y: int) -> int: + if y: + return 1 + elif x: + return 2 + else: + return 3 + return inner + + +def elif_lambda(x: int) -> Callable[[int], Callable[[], int]]: + def inner(y: int) -> Callable[[], int]: + if y: + return lambda: 1 + elif (f := lambda: x) and y == 0: + return f + return lambda: 3 + return inner + + +def match_subject(x: int) -> Callable[[], int]: + def inner() -> int: + match x: + case 1: + return 10 + case _: + return 0 + return inner + + +def match_guard(x: int) -> Callable[[int], int]: + def inner(y: int) -> int: + match y: + case 1 if x: + return 10 + case _: + return 0 + return inner + + +class Box: + def __init__(self, v: int) -> None: + self.v = v + + +def match_value(box: Box) -> Callable[[int], int]: + def inner(y: int) -> int: + match y: + case box.v: + return 10 + case _: + return 0 + return inner + + +def match_class(kind: type[Box]) -> Callable[[object], int]: + def inner(y: object) -> int: + match y: + case kind(v=v): + return v + case _: + return 0 + return inner + + +def except_class(kind: type[Exception]) -> Callable[[], str]: + def inner() -> str: + try: + raise KeyError('k') + except kind: + return 'caught' + except Exception: + return 'other' + return inner + + +def default_two_deep(x: int) -> Callable[[], Callable[[], int]]: + def middle() -> Callable[[], int]: + def inner(y: int = x) -> int: + return y + return inner + return middle + + +def elif_walrus(x: int) -> int: + if x == 0: + return 0 + elif (y := x + 1) > 0: + f = lambda: y + return f() + return -1 +", + &[ + "[m.elif_test(x)(y) for x in (0, 1) for y in (0, 1)]", + "[m.elif_lambda(x)(y)() for x in (0, 5) for y in (0, 1)]", + "[m.match_subject(x)() for x in (0, 1)]", + "[m.match_guard(x)(y) for x in (0, 1) for y in (0, 1)]", + "[m.match_value(m.Box(4))(y) for y in (3, 4)]", + "[m.match_class(m.Box)(y) for y in (m.Box(6), 6)]", + "[m.except_class(kind)() for kind in (KeyError, ValueError)]", + "m.default_two_deep(7)()()", + "[m.elif_walrus(x) for x in (0, 3)]", + ], + ); +} + +#[test] +fn a_capture_read_only_by_a_destructuring_statement_reaches_the_closure() { + // the value a `let` destructures is read in the frame the statement stands in, and + // so is the class its pattern names + agree( + "letcapturevalue", + &format!( + "{A_MATCHABLE_RECT}\ +def area(p: object) -> (() -> object): + def inner() -> object: + let (w, h) := p + return w * h + + return inner + + +def width(kind: type[Rect], r: object) -> (() -> object): + def inner() -> object: + let kind(w=w) := r else: + return 'no' + return w + + return inner +" + ), + &[ + "m.area((3, 4))()", + "m.width(m.Rect, m.Rect(5, 6))()", + "m.width(m.Rect, (5, 6))()", + ], + ); +} + /// a cell is read again on every call, however often the call is repeated /// /// a closure called in a loop is the shape a compiled build has the most to gain from @@ -10655,39 +11199,255 @@ def nested(n: int) -> object: } #[test] -fn async_await_agrees() { - agree_with_declines( - "asyncawait", +fn close_answers_what_the_frame_returned() { + // since 3.13 `close()` hands back the value a frame returns while it unwinds from + // the `GeneratorExit`, and `None` for every other way a close can end + agree_python( + "closevalue", "\ -async def plain(n: int) -> int: - return n * 2 +from collections.abc import Iterator -async def chained(n: int) -> int: - a = await plain(n) - b = await plain(a) - return b + 1 -async def nothing() -> None: - pass +def returning(log: list[str]) -> Iterator[int]: + try: + yield 1 + except GeneratorExit: + log.append('returning') + return 5 + yield 2 -async def forwards(n: int) -> int: - return await chained(n) + +def returning_none(log: list[str]) -> Iterator[int]: + try: + yield 1 + except GeneratorExit: + return None + + +def returning_a_tuple(log: list[str]) -> Iterator[int]: + try: + yield 1 + finally: + return (6, 7) + + +def plain(log: list[str]) -> Iterator[int]: + yield 1 + yield 2 + + +async def coroutine_returning(log: list[str]) -> int: + try: + await Suspend() + except GeneratorExit: + return 8 + return 0 + + +class Suspend: + def __await__(self) -> Iterator[None]: + yield None ", &[ - "__import__('asyncio').run(m.plain(5))", - "__import__('asyncio').run(m.chained(3))", - "__import__('asyncio').run(m.nothing())", - "__import__('asyncio').run(m.forwards(1))", - "[__import__('asyncio').run(m.plain(a)) for a in (0, -3, 10 ** 20)]", + "_closed_at(m.returning, 1)", + "_closed_at(m.returning, 0)", + "_closed_at(m.returning_none, 1)", + "_closed_at(m.returning_a_tuple, 1)", + "_closed_at(m.plain, 1)", + "_closed_at(m.plain, 2)", + "_closed_at(m.coroutine_returning, 1)", + "[(g := m.returning([]), next(g), g.close(), g.close())[2:]]", ], ); } #[test] -fn a_coroutine_reads_its_variadic_parameters() { - // `*args` and `**kwargs` are parameters like any other, and a coroutine's body reads - // them whether it suspends before doing so or never suspends at all — reached by - // `asyncio.run`, or awaited from another coroutine +fn a_throw_or_close_reaches_the_frame_a_delegation_is_waiting_on() { + // a frame suspended in `yield from` or `await` is suspended *inside* the iterator it + // delegates to, so `throw` goes to that iterator's own `throw` first and `close` + // closes it before the delegating frame is unwound. only what the inner iterator + // raises back reaches the delegating frame, at the suspension — and a + // `StopIteration` among those is the delegation finishing with its value + agree_python( + "delegatethrow", + "\ +from collections.abc import Generator, Iterator + + +def inner(log: list[str]) -> Iterator[int]: + try: + yield 1 + except ValueError as e: + log.append('inner caught ' + repr(e)) + yield 2 + finally: + log.append('inner finally') + + +def outer(log: list[str]) -> Iterator[int]: + try: + yield from inner(log) + log.append('delegation ended') + except TypeError as e: + log.append('outer caught ' + str(e)) + yield 3 + finally: + log.append('outer finally') + yield 4 + + +def returning(log: list[str]) -> Iterator[int]: + try: + yield 1 + except ValueError: + return 7 + return 0 + + +def takes_the_return(log: list[str]) -> Iterator[object]: + got = yield from returning(log) + log.append('got ' + repr(got)) + yield got + + +def over_a_list(log: list[str]) -> Iterator[object]: + try: + got = yield from [1, 2] + log.append('got ' + repr(got)) + except ValueError as e: + log.append('outer caught ' + repr(e)) + yield 5 + + +class Answering: + def __init__(self, log: list[str]) -> None: + self.log = log + + def __iter__(self) -> 'Answering': + return self + + def __next__(self) -> int: + return 1 + + def throw(self, kind: object, value: object = None, traceback: object = None) -> int: + self.log.append('throw got ' + repr(kind)) + if isinstance(kind, KeyError): + raise StopIteration('stopped') + return 9 + + def close(self) -> None: + self.log.append('closed') + + +def over_answering(log: list[str]) -> Iterator[object]: + got = yield from Answering(log) + log.append('got ' + repr(got)) + yield got + + +class Pending: + def __init__(self, log: list[str]) -> None: + self.log = log + + def __await__(self) -> Generator[str, None, str]: + try: + yield 'suspended' + except ValueError as e: + self.log.append('pending caught ' + str(e)) + return 'handled' + finally: + self.log.append('pending finally') + return 'done' + + +async def waiting(log: list[str]) -> str: + got = await Pending(log) + log.append('awaited ' + got) + return got + + +async def waits_on(log: list[str]) -> str: + try: + return await waiting(log) + finally: + log.append('outer finally') + + +def reentered(box: list[Iterator[object]]) -> Iterator[object]: + try: + yield 1 + except ValueError: + try: + next(box[0]) + except ValueError as e: + yield str(e) + + +def reenters(box: list[Iterator[object]]) -> Iterator[object]: + yield from reentered(box) +", + &[ + // the inner frame's handler catches it and yields on, and the outer frame + // stays suspended in the delegation + "_thrown_at(m.outer, 1, ValueError('v'))", + // what the inner frame cannot make an exception of, it raises back, and the + // outer frame's handler is the one that sees it + "_thrown_at(m.outer, 1, 5)", + "_thrown_at(m.outer, 1, ValueError, None, 5)", + "_thrown_at(m.outer, 1, KeyError('k'))", + "_thrown_at(m.outer, 2, ValueError('after'))", + "_closed_at(m.outer, 1)", + "_closed_at(m.outer, 2)", + "_thrown_at(m.takes_the_return, 1, ValueError('r'))", + "_thrown_at(m.over_a_list, 1, ValueError('l'))", + "_thrown_at(m.over_a_list, 1, StopIteration('l'))", + "_closed_at(m.over_a_list, 1)", + "_thrown_at(m.over_answering, 1, ValueError('a'))", + "_thrown_at(m.over_answering, 1, KeyError('a'))", + "_closed_at(m.over_answering, 1)", + "_thrown_at(m.waits_on, 1, ValueError('c'))", + "_thrown_at(m.waits_on, 1, KeyError('c'))", + "_closed_at(m.waits_on, 1)", + // the outer frame counts as running while the inner one handles the throw + "[(box := [], g := m.reenters(box), box.append(g), next(g), g.throw(ValueError('x')))[4]]", + ], + ); +} + +#[test] +fn async_await_agrees() { + agree_with_declines( + "asyncawait", + "\ +async def plain(n: int) -> int: + return n * 2 + +async def chained(n: int) -> int: + a = await plain(n) + b = await plain(a) + return b + 1 + +async def nothing() -> None: + pass + +async def forwards(n: int) -> int: + return await chained(n) +", + &[ + "__import__('asyncio').run(m.plain(5))", + "__import__('asyncio').run(m.chained(3))", + "__import__('asyncio').run(m.nothing())", + "__import__('asyncio').run(m.forwards(1))", + "[__import__('asyncio').run(m.plain(a)) for a in (0, -3, 10 ** 20)]", + ], + ); +} + +#[test] +fn a_coroutine_reads_its_variadic_parameters() { + // `*args` and `**kwargs` are parameters like any other, and a coroutine's body reads + // them whether it suspends before doing so or never suspends at all — reached by + // `asyncio.run`, or awaited from another coroutine agree_python( "asyncvariadic", "\ @@ -11517,30 +12277,43 @@ async def streamed() -> Any: } let emitted = std::fs::read_to_string(dir.join("by_diff_framekind_pin.c")) .expect("the generated C is written beside the extension"); - // each surface names itself, and none of them names another + // each surface names itself, and none of them names another. `send` is the one entry + // every surface has for (function, kind) in [ ("plain", "BY_FRAME_GENERATOR"), ("awaited", "BY_FRAME_COROUTINE"), ("streamed", "BY_FRAME_ASYNC_GENERATOR"), ] { - let symbol = format!("By_by_diff_framekind_pin_{function}_gen_Type_iternext"); - let iternext = emitted_function(&emitted, &symbol); - assert!(iternext.contains(kind), "{function}: {iternext}"); + let symbol = format!("By_by_diff_framekind_pin_{function}_gen_Type_send"); + let send = emitted_function(&emitted, &symbol); + assert!(send.contains(kind), "{function}: {send}"); for other in [ "BY_FRAME_GENERATOR", "BY_FRAME_COROUTINE", "BY_FRAME_ASYNC_GENERATOR", ] { assert_eq!( - iternext.contains(other), + send.contains(other), other == kind, - "{function} named {other}: {iternext}" + "{function} named {other}: {send}" ); } - // and it carries `None` in, because a resumption through this slot carries - // nothing — the store cannot be skipped or the last `send` survives it - assert!(iternext.contains("Py_None"), "{function}: {iternext}"); } + // and a step that carries nothing carries `None` in, because the store cannot be + // skipped or the last `send` survives it: a generator's `__next__`, and the step of + // the awaitable an async generator's `__anext__` hands back — which is its `send` + // with nothing sent. a coroutine has no such step of its own — it is not an iterator + for symbol in [ + "By_by_diff_framekind_pin_plain_gen_Type_iternext", + "By_by_diff_framekind_pin_streamed_gen_Type_asend_send", + ] { + let step = emitted_function(&emitted, symbol); + assert!(step.contains("Py_None"), "{symbol}: {step}"); + } + assert!( + !emitted.contains("By_by_diff_framekind_pin_awaited_gen_Type_iternext"), + "{emitted}" + ); } /// one emitted C function's body, from its definition to the closing brace @@ -12102,6 +12875,63 @@ def entered(mgr: object) -> object: ); } +#[test] +fn a_with_block_calls_the_exit_bound_as_it_was_entered() { + // python looks `__exit__` up as the block is entered, before `__enter__` runs, and the + // block calls that one however the class is changed while it runs — on the normal + // path, the raising one, and leaving early + agree_python( + "withexitbound", + "\ +from collections.abc import AsyncIterator, Callable, Iterator + + +def normal(make: Callable[[list[str]], object], later: object, log: list[str]) -> list[str]: + with make(log) as manager: # type: ignore + type(manager).__exit__ = later + return log + + +def raising(make: Callable[[list[str]], object], later: object, log: list[str]) -> list[str]: + try: + with make(log) as manager: # type: ignore + type(manager).__exit__ = later + raise KeyError('k') + except KeyError: + log.append('not suppressed') + return log + + +def leaving(make: Callable[[list[str]], object], later: object, log: list[str]) -> list[str]: + for _ in range(2): + with make(log) as manager: # type: ignore + type(manager).__exit__ = later + return log + return log + + +def suspended(make: Callable[[list[str]], object], later: object, log: list[str]) -> Iterator[int]: + with make(log) as manager: # type: ignore + yield 1 + type(manager).__exit__ = later + yield 2 + + +async def awaited(make: Callable[[list[str]], object], later: object, log: list[str]) -> list[str]: + async with make(log) as manager: # type: ignore + type(manager).__aexit__ = later + return log +", + &[ + "(lambda M, later, _: m.normal(M, later, []))(*_rebindable_manager())", + "(lambda M, later, _: m.raising(M, later, []))(*_rebindable_manager())", + "(lambda M, later, _: m.leaving(M, later, []))(*_rebindable_manager())", + "(lambda M, later, _: (lambda log: (list(m.suspended(M, later, log)), log))([]))(*_rebindable_manager())", + "(lambda M, _, later: _run(m.awaited(M, later, [])))(*_rebindable_manager())", + ], + ); +} + #[test] fn a_with_block_sees_its_manager_rebound_after_it_first_ran() { // a `with` site remembers what the two protocol names resolved to, so the answer @@ -12806,6 +13636,46 @@ def raised(fn: object) -> str: /// keep giving: the setter's body runs rather than a store landing beside it, the getter's /// body runs rather than the field it reads being taken directly, and a property with no /// setter still refuses the write in python's own wording +#[test] +fn a_property_whose_halves_make_closures_agrees() { + // a getter reading a generator expression over its own tokens, as `email`'s token + // lists do, and a setter and deleter that make closures of their own. every half is + // written `def value`, and each gets an environment of its own + agree_python( + "propclosures", + "\ +class Tokens: + def __init__(self, parts: list[str]) -> None: + self.parts = parts + self.log: list[str] = [] + + @property + def joined(self) -> str: + return ''.join(part.upper() for part in self.parts) + + @property + def value(self) -> int: + return sum(len(part) for part in self.parts if part) + + @value.setter + def value(self, n: int) -> None: + make = lambda i: 'x' * (i + n) + self.parts = [make(i) for i in range(2)] + + @value.deleter + def value(self) -> None: + self.log.append(','.join(part for part in self.parts)) + self.parts = [] +", + &[ + "m.Tokens(['a', 'bc']).joined", + "m.Tokens(['a', '', 'bc']).value", + "(lambda t: (setattr(t, 'value', 1), t.parts, t.value))(m.Tokens([]))", + "(lambda t: (delattr(t, 'value'), t.parts, t.log))(m.Tokens(['p', 'q']))", + ], + ); +} + #[test] fn a_property_reached_from_compiled_code_agrees() { agree_python( @@ -21688,9 +22558,11 @@ class Listc(A): "Lam", "a `super()` with no arguments reads the nested function's own slot zero, not the method's receiver" ), + // a generator expression is a generator function of its own, and no python + // folds one into the method around it ( "Genex", - "a `super()` in a comprehension reads that comprehension's own frame, which only python 3.12 and later fold into the method's" + "a `super()` with no arguments reads slot zero, which a generator's resume frame fills with its state" ), ( "Gen", @@ -22954,20 +23826,23 @@ def make_stream(step: int) -> Any: ); } +/// a generator that shares a cell with the frame around it reads and writes that one cell +/// +/// python closes over the variable, not the value, so a write on either side after the +/// generator was made is seen on the other — and in python every trip round a loop +/// rebinds one variable, so a generator made in a loop reads the loop's latest value #[test] -fn a_generator_sharing_a_cell_is_declined_and_still_runs() { - // a *shared* capture is a cell both frames write, which a copy cannot be — so - // this one stays interpreted rather than being quietly given a stale value - agree_with_declines( +fn a_generator_sharing_a_cell_with_its_frame_agrees() { + agree_python( "sharedcell", "\ -from typing import Any +from collections.abc import AsyncIterator, Awaitable, Callable, Iterator -def counter(n: int) -> Any: +def counter(n: int) -> tuple[Callable[[], Iterator[int]], Callable[[], int]]: seen = 0 - def gen() -> Any: + def gen() -> Iterator[int]: nonlocal seen i = 0 while i < n: @@ -22979,8 +23854,137 @@ def counter(n: int) -> Any: return seen return (gen, read) + + +def late_binding(xs: list[int], k: int) -> list[int]: + def g() -> Iterator[int]: + for x in xs: + yield x * k + + it = g() + k = 100 + return list(it) + + +def late_mid(xs: list[int], k: int) -> list[int]: + def g() -> Iterator[int]: + for x in xs: + yield x * k + + it = g() + out = [next(it)] + k = 100 + out.append(next(it)) + return out + + +def writes_back(n: int) -> list[int]: + total = 0 + + def g() -> Iterator[int]: + nonlocal total + for i in range(n): + total = total + i + yield i + + out = [] + for _ in g(): + out.append(total) + return out + + +def in_loop(xs: list[int], ks: list[int]) -> int: + t = 0 + for k in ks: + def g() -> Iterator[int]: + for x in xs: + yield x * k + + t = t + sum(g()) + return t + + +def made_in_loop(ks: list[int]) -> list[int]: + gens = [] + for k in ks: + def g() -> Iterator[int]: + yield k + + gens.append(g()) + return [next(it) for it in gens] + + +def two_deep(n: int) -> list[int]: + acc = 0 + + def middle() -> list[int]: + def inner() -> Iterator[int]: + nonlocal acc + for i in range(n): + acc = acc + i + yield acc + + return list(inner()) + + out = middle() + out.append(acc) + return out + + +def unbound() -> str: + def g() -> Iterator[int]: + yield v + + it = g() + try: + next(it) + except NameError as e: + return str(e) + v = 1 + return 'bound' + + +def summed() -> tuple[Callable[[int], Awaitable[int]], Callable[[], int]]: + total = 0 + + async def add(i: int) -> int: + nonlocal total + total = total + i + return total + + def read() -> int: + return total + + return (add, read) + + +def streaming(n: int) -> tuple[Callable[[], AsyncIterator[int]], Callable[[], int]]: + step = 1 + + async def doubled() -> AsyncIterator[int]: + nonlocal step + for i in range(n): + step = step * 2 + yield step + + def read() -> int: + return step + + return (doubled, read) ", - &["_counted(m.counter(3))", "_counted(m.counter(0))"], + &[ + "_counted(m.counter(3))", + "_counted(m.counter(0))", + "m.late_binding([1, 2], 3)", + "m.late_mid([1, 2], 3)", + "m.writes_back(4)", + "m.in_loop([1, 2, 3], [10, 20])", + "m.made_in_loop([1, 2, 3])", + "m.two_deep(4)", + "m.unbound()", + "(lambda pair: (_run(pair[0](3)), _run(pair[0](4)), pair[1]()))(m.summed())", + "(lambda pair: (_run(_drain(pair[0]())), pair[1]()))(m.streaming(3))", + ], ); } @@ -23047,6 +24051,45 @@ async def echoed(n: int) -> Any: ); } +#[test] +fn an_async_generator_delivers_a_tuple_or_an_exception_whole() { + // an item reaches the awaiting frame as the value of a `StopIteration`, and python + // builds that exception around the item rather than handing the item to its + // constructor — so a tuple is not spread into the exception's arguments and an + // exception instance is not taken for the exception being raised + agree_python( + "asyncgenwhole", + "\ +from typing import Any + + +async def pairs() -> Any: + yield (1, 2) + yield () + yield (3,) + yield ValueError('an item') + yield StopIteration(4) + + +async def answered() -> Any: + got = yield 0 + yield (got, got) + try: + yield 1 + except KeyError: + yield ('caught', 'it') +", + &[ + "_run(_drain(m.pairs()))", + "_run(_comprehended(m.pairs()))", + "[repr(v) for v in _run(_drain(m.pairs()))]", + "_run(_stepped(m.pairs()))", + "_run(_reasend(m.answered(), (5, 6), 0))", + "_run(_athrown(m.answered(), KeyError, 3))", + ], + ); +} + #[test] fn a_generator_or_coroutine_may_be_a_method() { // the state object holds `self` like any other parameter, so the body reads @@ -23099,30 +24142,99 @@ class Other: } #[test] -fn a_compiled_class_may_be_a_context_manager_or_an_async_iterator() { - // `__enter__` and its three relatives are not slots: python reaches them by an - // ordinary type lookup, which finds the method table without an adapter +fn an_async_generator_s_awaitables_are_coroutines_of_their_own() { + // `__anext__`, `asend`, `athrow` and `aclose` each hand back an awaitable with the + // coroutine methods: `send`, `throw` and `close`. while one of them is suspended at an + // `await` inside the generator, the generator is running, and no other awaitable may + // start stepping it; and an awaitable that has finished refuses to be stepped again agree_python( - "ctxclass", + "agenawaitables", "\ -from typing import Any - - -class Sync: - def __init__(self, log: list[Any]) -> None: - self.log = log +from collections.abc import AsyncGenerator, AsyncIterator - def __enter__(self) -> str: - self.log.append('enter') - return 'held' - def __exit__(self, kind: object, value: object, tb: object) -> bool: - self.log.append(('exit', kind)) - return False +class Suspend: + def __await__(self): # type: ignore + yield 'waiting' -class Async: - def __init__(self, log: list[Any]) -> None: +async def counted(log: list[str]) -> AsyncIterator[int]: + try: + await Suspend() + yield 1 + await Suspend() + yield 2 + except ValueError as e: + log.append('caught ' + str(e)) + yield 3 + finally: + log.append('finally') + + +async def echoed() -> AsyncGenerator[object, object]: + got = yield 0 + yield got + await Suspend() + yield 'after' +", + &[ + // a step that reaches an `await` is suspended there, and a second awaitable is + // refused while it is + "_against(m.counted, lambda g: [lambda: g.__anext__().send(None), \ + lambda: g.__anext__().send(None), lambda: next(g.asend(None)), \ + lambda: next(g.athrow(ValueError('x'))), lambda: next(g.aclose())])", + // the awaitable's own `send`, `throw` and `close` + "_against(m.counted, lambda g: (lambda a: [lambda: a.send(None), lambda: a.send(None), \ + lambda: a.send(None), lambda: a.send(None)])(g.__anext__()))", + "_against(m.counted, lambda g: (lambda a: [lambda: a.send(None), \ + lambda: a.throw(ValueError('t')), lambda: a.send(None)])(g.__anext__()))", + "_against(m.counted, lambda g: (lambda a: [lambda: a.send(None), lambda: a.close(), \ + lambda: a.send(None), lambda: a.close()])(g.__anext__()))", + "_against(m.counted, lambda g: [lambda: next(g.__anext__()), lambda: next(g.__anext__()), \ + lambda: (lambda a: (a.send(None), a.close()))(g.__anext__()), \ + lambda: next(g.__anext__())])", + // `asend`'s value rides in on the first step whatever that step sends + "_against(lambda log: m.echoed(), lambda g: [lambda: next(g.__anext__()), \ + lambda: g.asend('v').send(None), lambda: next(g.__anext__()), \ + lambda: next(g.__anext__())])", + // `athrow` refuses a first step that carries a value, and `aclose` is spent once + // it is done + "_against(lambda log: m.echoed(), lambda g: [lambda: next(g.__anext__()), \ + lambda: g.athrow(ValueError('e')).send(1), \ + lambda: g.athrow(KeyError('k')).send(None)])", + "_against(lambda log: m.echoed(), lambda g: (lambda t: [lambda: t.send(None), \ + lambda: t.send(None), lambda: next(g.aclose())])(g.aclose()))", + "_against(m.counted, lambda g: (lambda t: [lambda: next(g.__anext__()), \ + lambda: t.throw(KeyError('k')), lambda: t.send(None)])(g.athrow(ValueError('v'))))", + ], + ); +} + +#[test] +fn a_compiled_class_may_be_a_context_manager_or_an_async_iterator() { + // `__enter__` and its three relatives are not slots: python reaches them by an + // ordinary type lookup, which finds the method table without an adapter + agree_python( + "ctxclass", + "\ +from typing import Any + + +class Sync: + def __init__(self, log: list[Any]) -> None: + self.log = log + + def __enter__(self) -> str: + self.log.append('enter') + return 'held' + + def __exit__(self, kind: object, value: object, tb: object) -> bool: + self.log.append(('exit', kind)) + return False + + +class Async: + def __init__(self, log: list[Any]) -> None: self.log = log async def __aenter__(self) -> str: @@ -23617,6 +24729,650 @@ def held(manager: object, n: int) -> object: ); } +/// a resumable frame's state object is made by calling its function and by nothing else +/// +/// one python allocates for itself holds no iterator and no parameters, and one whose +/// `__init__` runs a second time has its fields replaced under a suspended frame — both +/// reach the frame's body with fields it never wrote +#[test] +fn python_cannot_make_or_remake_a_resumable_frame() { + agree_python( + "genforged", + "\ +from collections.abc import AsyncIterator, Iterator + + +def counted(n: int) -> Iterator[int]: + i = 0 + while i < n: + yield i + i = i + 1 + + +async def plus(n: int) -> int: + return n + 1 + + +async def stream(n: int) -> AsyncIterator[int]: + i = 0 + while i < n: + yield i + i = i + 1 +", + &[ + "(lambda g: (_forged(g), list(g)))(m.counted(3))", + "(lambda g: (next(g), _forged(g), list(g)))(m.counted(3))", + "(lambda c: (_forged(c), _sent_once(c)))(m.plus(4))", + "(lambda a: (_forged(a), _forged(a.__anext__()), _run(_drain(a))))(m.stream(2))", + ], + ); +} + +/// a generator the collector cannot walk keeps every cycle through it alive: the cycle is +/// never freed, and a generator suspended inside `try` never runs its `finally` +#[test] +fn a_generator_in_a_reference_cycle_is_collected() { + agree_python( + "gencycle", + "\ +from collections.abc import AsyncIterator, Iterator + + +def holds(box: list[object], log: list[str]) -> Iterator[int]: + try: + yield len(box) + yield 2 + finally: + log.append('finally') + + +def keeps(payload: object, box: list[object]) -> Iterator[object]: + held = [payload, box] + yield held + yield len(held) + + +async def plus(n: int) -> int: + return n + 1 + + +async def stream(n: int) -> AsyncIterator[int]: + yield n +", + &[ + "_cycled(lambda box, log: m.holds(box, log))", + "gc.is_tracked(m.holds([], []))", + "(lambda c: (gc.is_tracked(c), c.close()))(m.plus(1))", + "(lambda a: (gc.is_tracked(a), gc.is_tracked(a.__anext__())))(m.stream(1))", + "_cycle_references(lambda payload, box: m.keeps(payload, box))", + ], + ); +} + +/// a frame that is running cannot be resumed a second time from inside itself +/// +/// python refuses every entry point while the frame executes, so the resumption in the +/// middle of a body never happens: re-entered, a frame would start again from its own +/// suspension point while its registers are still in use +#[test] +fn a_running_frame_refuses_to_be_resumed() { + agree_python( + "genrunning", + "\ +from collections.abc import AsyncIterator, Callable, Iterator + + +def probed(probes: list[Callable[[], object]]) -> Iterator[object]: + for probe in probes: + try: + yield probe() + except ValueError as e: + yield ('ValueError', str(e)) + + +async def awaited(probes: list[Callable[[], object]]) -> list[object]: + out: list[object] = [] + for probe in probes: + try: + out.append(probe()) + except ValueError as e: + out.append(('ValueError', str(e))) + return out + + +async def streamed(probes: list[Callable[[], object]]) -> AsyncIterator[object]: + for probe in probes: + try: + yield probe() + except RuntimeError as e: + yield 'RuntimeError: ' + str(e) + + +def loopy(box: list[object], xs: list[int]) -> Iterator[int]: + for x in xs: + if box[1]: + box[1] = False + for _ in box[0]: # type: ignore + pass + yield x + + +def holder(box: list[object]) -> Iterator[str]: + s = 'abc' * (len(box) + 1) + yield s + if box[1]: + box[1] = False + for _ in box[0]: # type: ignore + pass + yield s + '!' + s = 'zz' + yield s + + +def drain(box: list[object]) -> str: + if box[1]: + box[1] = False + for _ in box[0]: # type: ignore + pass + return '!' + + +def borrow_list(box: list[object], n: int) -> Iterator[int]: + xs = [n, n + 1, n + 2] + yield 0 + yield len(xs) + len(drain(box)) + xs[0] + xs = [n * 100] + yield xs[0] +", + &[ + "_reentered(m.probed, 'gen')", + "_reentered(m.awaited, 'coro')", + "_reentered(m.streamed, 'async')", + "_iterated_from_inside(lambda box: m.loopy(box, [1, 2, 3, 4]))", + "_iterated_from_inside(m.holder)", + "_iterated_from_inside(lambda box: m.borrow_list(box, 7))", + ], + ); +} + +/// `throw` takes python's whole argument list: an exception, or a type with a value to +/// build it from and a traceback to hang on it +/// +/// the value is not a detail. `throw(ValueError, ValueError("old"))` raises that instance, +/// and one built from nothing in its place is a handler seeing a different exception +#[test] +fn throw_takes_a_value_and_a_traceback() { + agree_python( + "genthrowargs", + "\ +from collections.abc import AsyncIterator, Awaitable, Callable, Iterator + + +def caught() -> Iterator[str]: + try: + yield 'start' + except ValueError as e: + yield repr(e) + ' ' + repr(e.args) + + +async def awaited(step: Callable[[], Awaitable[object]]) -> str: + try: + await step() + except ValueError as e: + return repr(e) + ' ' + repr(e.args) + return 'nothing' + + +async def streamed() -> AsyncIterator[str]: + try: + yield 'start' + except ValueError as e: + yield repr(e) + ' ' + repr(e.args) +", + &[ + "_throw_shapes(m.caught, next, lambda g, args: g.throw(*args))", + "_throw_shapes(lambda: m.awaited(_suspends_once), lambda c: c.send(None), \ + lambda c, args: c.throw(*args))", + "_throw_shapes(lambda: _spend(m.awaited(lambda: _Ready(0))), lambda c: None, \ + lambda c, args: c.throw(*args))", + "_throw_shapes(m.streamed, lambda a: _stepped_by_hand(a.__anext__()), \ + lambda a, args: _stepped_by_hand(a.athrow(*args)))", + ], + ); +} + +/// a coroutine is awaitable and nothing else: `next()` of one is a `TypeError`, and the +/// iterator an `await` drives is the separate object `__await__` hands back +#[test] +fn a_coroutine_is_not_an_iterator() { + agree_python( + "coronext", + "\ +from collections.abc import Awaitable, Callable + + +async def plus(n: int) -> int: + return n + 1 + + +async def awaited(step: Callable[[], Awaitable[object]]) -> str: + try: + await step() + await step() + except ValueError as e: + return 'caught ' + repr(e) + return 'nothing' + + +async def chained(step: Callable[[], Awaitable[object]], n: int) -> str: + first = awaited(step) + second = plus(n) + return await first + str(await second) +", + &[ + "_as_iterator(m.plus(1))", + "_await_wrapper(m.plus(1))", + "_await_wrapper(m.awaited(_suspends_once))", + "_run(_awaits(m.plus(4)))", + "_run(_awaits(m.awaited(lambda: _Suspends(3))))", + "_run(m.chained(lambda: _Suspends(3), 2))", + ], + ); +} + +/// a frame that suspends again after `close()` threw `GeneratorExit` into it is still +/// suspended: the close is refused, the frame can be resumed, and finalizing it tries +/// the close a second time +#[test] +fn a_frame_that_ignores_generator_exit_stays_suspended() { + agree_python( + "genignoredexit", + "\ +from collections.abc import AsyncIterator, Awaitable, Callable, Iterator + + +def stubborn(log: list[str]) -> Iterator[int]: + while True: + try: + yield 1 + except GeneratorExit: + log.append('ignored') + + +async def stubborn_coroutine(log: list[str], step: Callable[[], Awaitable[object]]) -> None: + while True: + try: + await step() + except GeneratorExit: + log.append('ignored') + + +async def stubborn_stream(log: list[str]) -> AsyncIterator[int]: + while True: + try: + yield 1 + except GeneratorExit: + log.append('ignored') + + +async def once() -> AsyncIterator[int]: + yield 1 +", + &[ + "_ignored_close(m.stubborn, next, lambda g: g.close(), next)", + "_ignored_close(lambda log: m.stubborn_coroutine(log, _suspends_once), \ + lambda c: c.send(None), lambda c: c.close(), lambda c: c.send(None))", + "_ignored_close(m.stubborn_stream, lambda a: _stepped_by_hand(a.__anext__()), \ + lambda a: _stepped_by_hand(a.aclose()), lambda a: _stepped_by_hand(a.__anext__()))", + // a close that finished the frame leaves nothing for a second one to refuse + "(lambda a: [_stepped_by_hand(a.__anext__()), _stepped_by_hand(a.aclose()), \ + _stepped_by_hand(a.aclose()), _stepped_by_hand(a.athrow(ValueError('v')))])(m.once())", + ], + ); +} + +/// a coroutine dropped without ever being awaited is almost always a missing `await`, +/// and python says so with a `RuntimeWarning` naming the coroutine +#[test] +fn a_coroutine_never_awaited_warns() { + agree_python( + "coronever", + "\ +async def plus(n: int) -> int: + return n + 1 + + +class Holder: + async def method(self) -> int: + return 1 +", + &[ + "_dropped_frame_warnings(lambda: m.plus(1))", + "_dropped_frame_warnings(lambda: m.Holder().method())", + // one that was awaited, or closed, or started, has nothing to warn about + "_dropped_frame_warnings(lambda: _sent_once(m.plus(1)))", + "_dropped_frame_warnings(lambda: m.plus(1).close())", + "_dropped_frame_warnings(lambda: _run(m.plus(1)))", + ], + ); +} + +/// a generator, a coroutine and an async generator can each be referred to weakly +#[test] +fn a_resumable_frame_takes_a_weak_reference() { + agree_python( + "genweak", + "\ +from collections.abc import AsyncIterator, Iterator + + +def counted(n: int) -> Iterator[int]: + i = 0 + while i < n: + yield i + i = i + 1 + + +async def plus(n: int) -> int: + return n + 1 + + +async def stream(n: int) -> AsyncIterator[int]: + yield n +", + &[ + "_weakly(lambda: m.counted(3), list)", + "_weakly(lambda: m.counted(3), next)", + "_weakly(lambda: m.plus(1), _sent_once)", + "_weakly(lambda: m.stream(1), lambda a: _stepped_by_hand(a.__anext__()))", + ], + ); +} + +#[test] +fn a_generator_expression_or_lambda_inside_a_generator_or_coroutine_agrees() { + // the frame a generator expression is made in may itself be suspended, and what the + // expression reads out of it is read when the expression runs, not when it is made — + // so a local the frame writes afterwards is seen at its new value + agree_python( + "genexpingen", + "\ +from collections.abc import AsyncIterator, Callable, Iterator + + +def grouped(groups: list[list[str]]) -> Iterator[str]: + for group in groups: + if any(tag in {'replace', 'delete'} for tag in group): + yield 'changed' + yield ','.join(tag.upper() for tag in group) + + +def parameter(xs: list[int], k: int) -> Iterator[int]: + yield sum(x * k for x in xs) + yield max(x + k for x in xs) + + +def late(xs: list[int]) -> Iterator[object]: + k = 1 + g = (x * k for x in xs) + yield 'made' + k = 10 + yield list(g) + + +def suspended_between(xs: list[int]) -> Iterator[object]: + total = 0 + g = (x + total for x in xs) + yield next(g) + total = 100 + yield next(g) + + +def lambdas(xs: list[int]) -> Iterator[object]: + scale = 2 + f: Callable[[int], int] = lambda x: x * scale + yield f(3) + scale = 5 + yield [f(x) for x in xs] + + +async def awaited(xs: list[int]) -> int: + return sum(x for x in xs if x > 1) + + +async def streamed(xs: list[int]) -> AsyncIterator[str]: + for x in xs: + yield '-'.join(str(y) for y in range(x)) + + +def unset_first(xs: list[int]) -> Iterator[object]: + g = (x + later for x in xs) + yield 'made' + try: + yield next(g) + except NameError as e: + yield type(e).__name__ + later = 7 + yield list(g) + + +def nested_expressions(rows: list[list[int]]) -> Iterator[int]: + bias = 1 + yield sum(sum(y + bias for y in row) for row in rows) + + +def enclosing(k: int) -> Callable[[list[int]], Iterator[int]]: + def inner(xs: list[int]) -> Iterator[int]: + yield sum(x * k for x in xs) + return inner +", + &[ + "list(m.unset_first([1, 2]))", + "list(m.nested_expressions([[1, 2], [3]]))", + "list(m.enclosing(3)([1, 2]))", + "list(m.grouped([['keep'], ['replace', 'x'], []]))", + "list(m.parameter([1, 2, 3], 4))", + "list(m.late([1, 2]))", + "list(m.suspended_between([1, 2]))", + "list(m.lambdas([1, 2]))", + "_run(m.awaited([1, 2, 3]))", + "_run(_drain(m.streamed([1, 3])))", + ], + ); +} + +/// a generator expression is a generator: nothing in it runs until it is asked for a +/// value, it runs only as far as it is asked, and it is an iterator of its own +/// +/// the one place it is built at once is as the sole argument of `list`, `tuple`, `set`, +/// `frozenset`, `sorted` or `dict` while the name is still the builtin, which drains it +/// straight away — and a `StopIteration` escaping it there still becomes the +/// `RuntimeError` a generator's frame makes of one +#[test] +fn a_generator_expression_is_lazy() { + agree_python( + "genexplazy", + "\ +from collections.abc import Iterator + + +def deferred(xs: list[int]) -> tuple[int, list[int]]: + log: list[int] = [] + g = (log.append(x) for x in xs) + log.append(-1) + next(g) + return (len(log), log) + + +def short_circuits() -> bool: + return any(1.0 / x > 0.0 for x in [1.0, 0.0]) + + +def stepped(xs: list[int]) -> list[object]: + g = (x * 2 for x in xs) + first = next(g) + return [first, next(g), list(g), isinstance(g, Iterator), iter(g) is g] + + +def stored(xs: list[int]) -> Iterator[int]: + return (x + 1 for x in xs if x % 2 == 0) + + +def late(xs: list[int]) -> list[int]: + k = 1 + g = (x * k for x in xs) + k = 10 + return list(g) + + +def first_iterable_now(xs: object) -> str: + try: + (x for x in xs) # type: ignore + except TypeError as e: + return str(e) + return 'made' + + +def membership(ks: list[int], xs: list[int]) -> list[bool]: + out = [] + for k in ks: + out.append(any(x == k for x in xs)) + return out + + +def totals(xs: list[int]) -> list[object]: + return [sum(x * 2 for x in xs), min(x for x in xs), max(-x for x in xs), + all(x > 0 for x in xs), list(zip((x for x in xs), (x * x for x in xs)))] + + +def drained(xs: list[int]) -> list[object]: + return [list(x * 2 for x in xs), tuple(x for x in xs), sorted(-x for x in xs), + set(x % 2 for x in xs), frozenset(x % 3 for x in xs), + dict((x, x * x) for x in xs)] + + +def nested_clauses(xss: list[list[int]]) -> list[int]: + return list(y for xs in xss if xs for y in xs if y != 2) + + +def stops(xs: list[int]) -> object: + it = iter(xs) + return list(next(it) for _ in range(len(xs) + 1)) + + +def stops_lazily(xs: list[int]) -> object: + it = iter(xs) + return sum(next(it) for _ in range(len(xs) + 1)) + + +def walrus(xs: list[int]) -> tuple[int, int, int]: + d = 0 + g = ((d := x * 2) + d for x in xs) + before = d + first = next(g) + seen = d + return (before + first, seen, sum(g) + d) + + +def walrus_drained(xs: list[float]) -> tuple[list[float], float]: + out = list((d := x - 1.0) * d for x in xs) + return (out, d) + + +def in_elif(xs: list[int], k: int) -> str: + if k < 0: + return 'negative' + elif any(x == k for x in xs): + return 'found' + return 'missing' + + +def in_rows(rows: list[list[int]]) -> list[int]: + return [sum(y * 2 for y in row) for row in rows] +", + &[ + "m.deferred([1, 2, 3])", + "m.short_circuits()", + "m.stepped([1, 2, 3, 4])", + "(lambda g: (next(g), list(g)))(m.stored([1, 2, 3, 4, 6]))", + "m.late([1, 2])", + "m.first_iterable_now(5)", + "m.membership([1, 5, 3], [3, 4, 1])", + "m.totals([3, 1, 2])", + "m.drained([3, 1, 2])", + "m.nested_clauses([[1, 2], [], [2, 3]])", + "_escaped(m.stops, [1, 2])", + "_escaped(m.stops_lazily, [1, 2])", + // a walrus inside binds in the function the expression stands in + "m.walrus([1, 2, 3])", + "m.walrus_drained([1.5, 3.0])", + // wherever the statement holds it, and inside a comprehension's element + "[m.in_elif([1, 2], k) for k in (-1, 2, 5)]", + "m.in_rows([[1, 2], [], [3]])", + "_with_module_name(m, 'list', lambda g: (next(g), next(g)), m.walrus_drained, [2.0, 4.0])", + // a name that no longer holds the builtin is handed the generator itself + "[_with_module_name(m, name, lambda g: (name, next(g), next(g)), m.drained, [5, 6]) \ + for name in ('list', 'tuple', 'sorted', 'set', 'frozenset', 'dict')]", + "_with_module_name(m, 'any', lambda g: next(g), m.membership, [1], [2])", + ], + ); +} + +/// a closure environment is reachable from python through the collector, and has to be +/// no more constructible than a generator's state object +#[test] +fn python_cannot_make_or_remake_a_closure_environment() { + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = diff_root().join("by_diff_envforged"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +from collections.abc import Callable + + +def counter() -> Callable[[], int]: + seen = 0 + + def step() -> int: + nonlocal seen + seen = seen + 1 + return seen + + return step +"; + let options = Options { + language: by_irbuild::Language::Python, + require_native: true, + ..Options::default() + }; + if build_source(source, "by_diff_envforged", &toolchain, &dir, &options).is_err() { + eprintln!("skipping: no working C toolchain"); + return; + } + let out = run( + &python, + &dir, + "import gc, by_diff_envforged as m\n\ + step = m.counter()\n\ + step()\n\ + env = [o for o in gc.get_referents(step) if type(o).__name__.endswith('$env')]\n\ + print(len(env))\n\ + kind = type(env[0])\n\ + for attempt in (lambda: kind.__new__(kind), kind, lambda: object.__new__(kind)):\n\ + \x20 try:\n\ + \x20 attempt()\n\ + \x20 print('made')\n\ + \x20 except TypeError as e:\n\ + \x20 print(str(e).replace(kind.__module__ + '.', '').replace(kind.__name__, 'T'))\n\ + print(env[0].__init__(), step())\n", + ); + assert_eq!( + out, + "1\ncannot create 'T' instances\ncannot create 'T' instances\ncannot create 'T' instances\nNone 2" + ); +} + #[test] fn async_comprehensions_agree() { // the same machine `async for` is, driving the rest of the comprehension where @@ -29165,6 +30921,288 @@ def restored_after(log: list[str]) -> str: ); } +#[test] +fn a_finally_on_the_exception_path_sees_the_exception_it_unwinds() { + // python runs a `finally` that an exception is passing through as though the + // exception were being handled, so `sys.exception()` in the body answers that + // exception — not the one an enclosing handler holds, and not `None` — and an + // exception raised there chains onto it. the normal path sees what it saw before + agree_python( + "finallyhandled", + "\ +import sys +from collections.abc import Iterator + + +class Refused(Exception): + pass + + +def unmatched_inside(log: list[str]) -> int: + try: + raise KeyError('outer') + except KeyError: + try: + raise Refused('empty') + except ValueError: + return 1 + finally: + log.append(repr(sys.exception())) + return 0 + + +def plain_finally(log: list[str]) -> int: + try: + raise Refused('x') + finally: + log.append(repr(sys.exception())) + + +def normal_path(log: list[str]) -> int: + try: + raise KeyError('outer') + except KeyError: + try: + log.append('body') + finally: + log.append(repr(sys.exception())) + log.append(repr(sys.exception())) + return 0 + + +def nested_finally(log: list[str]) -> int: + try: + try: + raise Refused('inner') + finally: + log.append(repr(sys.exception())) + finally: + log.append(repr(sys.exception())) + + +def raised_in_finally(log: list[str]) -> int: + try: + raise Refused('first') + finally: + raise KeyError('second') + + +def after_the_finally(log: list[str]) -> int: + try: + try: + raise Refused('gone') + finally: + log.append(repr(sys.exception())) + except Refused: + pass + log.append(repr(sys.exception())) + return 0 + + +def returning_from_finally(log: list[str]) -> int: + try: + raise Refused('swallowed') + finally: + log.append(repr(sys.exception())) + return 2 + + +def yields_in_finally(log: list[str]) -> Iterator[str]: + try: + raise Refused('suspended') + finally: + yield repr(sys.exception()) + log.append(repr(sys.exception())) +", + &[ + "(lambda log: (repr(_capture(m.unmatched_inside, log)), log))([])", + "(lambda log: (repr(_capture(m.plain_finally, log)), log))([])", + "(lambda log: (m.normal_path(log), log))([])", + "(lambda log: (repr(_capture(m.nested_finally, log)), log))([])", + "_chain(_capture(m.raised_in_finally, []))", + "(lambda log: (m.after_the_finally(log), log))([])", + "(lambda log: (m.returning_from_finally(log), log, repr(sys.exception())))([])", + // a frame suspended in the `finally` hands the exception back to its caller + // while it is away, and takes it again when it resumes + "(lambda log: (g := m.yields_in_finally(log), next(g), repr(sys.exception()), \ + repr(_capture(next, g)), log)[1:])([])", + ], + ); +} + +#[test] +fn an_exception_s_traceback_names_every_compiled_frame_it_passed() { + // python adds a traceback entry for each frame an exception is raised in or passes + // through, naming the function and the line of the instruction that raised. a caught + // exception carries the entries from the raise to the handler; one that leaves the + // module carries the whole way out, with no entry for anything python did not run — + // the forwarder a module function is published through included. putting back an + // exception the frame already holds adds nothing + agree_python( + "tbentries", + "\ +import asyncio +from collections.abc import Callable, Iterator + + +class Refused(Exception): + pass + + +def parse(text: str) -> int: + n = len(text) + if n == 0: + raise Refused('empty') + return n + + +def caught(text: str) -> object: + try: + parse(text) + except Refused as e: + return _names(e) + return None + + +def _names(e: BaseException) -> list[tuple[str, int]]: + out = [] + tb = e.__traceback__ + while tb is not None: + out.append((tb.tb_frame.f_code.co_name, tb.tb_lineno)) + tb = tb.tb_next + return out + + +def propagates(text: str) -> int: + total = 0 + total = total + parse(text) + return total + + +def spread(xs: list[int], at: int) -> int: + first = 1 + got = xs[ + at + ] + first + return got + + +def recursed(n: int) -> int: + if n == 0: + return parse('') + return recursed(n - 1) + + +def re_raised(text: str) -> int: + try: + parse(text) + except Refused: + raise + return 0 + + +def raised_again(text: str) -> int: + try: + parse(text) + except Refused as e: + raise e + return 0 + + +def through_finally(text: str) -> int: + try: + return parse(text) + finally: + text = text + '!' + + +def while_handling(text: str) -> int: + try: + parse(text) + except Refused: + return int('x') + return 0 + + +def generated(text: str) -> Iterator[int]: + yield 1 + yield parse(text) + + +def suspended() -> Iterator[int]: + yield 1 + yield 2 + + +def lambda_of() -> Callable[[str], int]: + return lambda t: parse(t) + + +def nested_of() -> Callable[[str], int]: + def inner(t: str) -> int: + return parse(t) + return inner + + +def expressed(texts: list[str]) -> list[int]: + return list(parse(t) for t in texts) + + +def variadic(*texts: str) -> int: + return parse(texts[0]) + + +class Box: + def __init__(self, text: str) -> None: + self.text = text + + def measured(self) -> int: + return parse(self.text) + + @property + def size(self) -> int: + return parse(self.text) + + def __hidden(self) -> int: + return parse(self.text) + + def hidden(self) -> int: + return self.__hidden() + + +async def awaited(text: str) -> int: + await asyncio.sleep(0) + return parse(text) +", + &[ + "m.caught('')", + "m.caught('abc')", + "_frames(_capture(m.parse, ''))", + "_frames(_capture(m.propagates, ''))", + "_frames(_capture(m.spread, [1], 1))", + "_frames(_capture(m.recursed, 3))", + "_frames(_capture(m.re_raised, ''))", + "_frames(_capture(m.raised_again, ''))", + "_frames(_capture(m.through_finally, ''))", + "(lambda e: (_frames(e), _frames(e.__context__)))(_capture(m.while_handling, ''))", + "_frames(_capture(list, m.generated('')))", + // a `throw` raises at the suspension, which is where the frame's entry points + "(lambda g: (next(g), _frames(_capture(g.throw, ValueError('t')))))(m.suspended())", + "_frames(_capture(m.lambda_of(), ''))", + "_frames(_capture(m.nested_of(), ''))", + "_frames(_capture(m.expressed, ['a', '']))", + "_frames(_capture(m.variadic, ''))", + "_frames(_capture(m.Box('').measured))", + "_frames(_capture(getattr, m.Box(''), 'size'))", + "_frames(_capture(m.Box('').hidden))", + "_frames(_capture(asyncio.run, m.awaited('')))", + // what python raises binding a call has no frame of the callee's in it + "_frames(_capture(m.parse))", + "_frames(_capture(m.variadic))", + ], + ); +} + #[test] fn a_raise_raises_what_the_constructor_answered() { // `raise Refused(...)` raises whatever calling the name answered. python asks diff --git a/crates/by_codegen_c/src/lib.rs b/crates/by_codegen_c/src/lib.rs index 47c4ed5a55..6209cbcdc7 100644 --- a/crates/by_codegen_c/src/lib.rs +++ b/crates/by_codegen_c/src/lib.rs @@ -114,6 +114,16 @@ fn frame_kind(surface: Surface) -> &'static str { } } +/// the accessor a resumable frame's `throw` and `close` ask for the iterator a suspension +/// delegates to, or `NULL` for a frame with no delegation to ask about +fn delegate_accessor(resume: &by_ir::function::Resumption, type_name: &str) -> String { + if resume.delegations.is_empty() { + "NULL".to_string() + } else { + format!("{type_name}_delegate") + } +} + fn mangle_member(name: &str) -> String { by_ir::function::FieldDecl { cell: false, @@ -307,7 +317,11 @@ pub fn emit_module(module: &ModuleIr) -> String { // function's spec, and the spec names its wrapper. interleaving per class made that // a forward reference for class in &module.classes { - for method in &class.methods { + for method in class + .methods + .iter() + .filter(|method| !class.resumes_through(method)) + { let _ = writeln!( out, "static PyObject *{}({});", @@ -339,6 +353,9 @@ pub fn emit_module(module: &ModuleIr) -> String { for method in &class.methods { out.push_str(&emit_function(module, method)); out.push('\n'); + if class.resumes_through(method) { + continue; + } // a method's wrapper takes the receiver from the `self` slot rather // than from the argument vector, which is how `METH_FASTCALL` on a // type presents it. a `staticmethod` has nothing there and binds every @@ -572,41 +589,135 @@ fn emit_collected_instance(module: &ModuleIr, class: &ClassIr) -> String { ) } -/// `tp_traverse` and `tp_clear` for a closure environment +/// a place in a frame's state that may hold an object reference +struct HeldReference { + /// the C lvalue holding it + slot: String, + /// whether the slot is a tagged `int`, which is only a reference while it is not short + tagged: bool, +} + +/// every place a value of `ty` at `expr` may hold an object reference /// -/// a nested function that reads its own name holds the environment it reads it out of, -/// and the environment holds the function, so counting alone never frees either — the -/// collector has to be able to walk the cycle, and clearing the environment's fields is -/// what breaks it. the type is static, so there is no reference to it to report -fn emit_collected_environment(module: &ModuleIr, class: &ClassIr) -> String { +/// a frame's state is whatever its registers held, so a field can be any representation a +/// register has: a fixed-length tuple holds its elements inline, and a tagged `int` holds a +/// real `int` once it leaves the short range. an unboxed buffer is left out, because what +/// it holds is unboxed values rather than objects +fn held_references(ty: &RType, expr: &str) -> Vec { + match ty { + RType::Primitive(Primitive::Int) => vec![HeldReference { + slot: expr.to_string(), + tagged: true, + }], + RType::Tuple(items) => items + .iter() + .enumerate() + .flat_map(|(index, item)| held_references(item, &format!("{expr}.f{index}"))) + .collect(), + RType::Array(_) => Vec::new(), + _ if ty.is_refcounted() => vec![HeldReference { + slot: expr.to_string(), + tagged: false, + }], + _ => Vec::new(), + } +} + +/// `tp_traverse` for a frame's state — a closure environment or a generator's state +/// object — over every reference its fields hold +/// +/// the type is static, so there is no reference to it to report +fn emit_frame_traverse(module: &ModuleIr, class: &ClassIr) -> String { let struct_name = class.struct_name(module.name.dotted()); let type_name = class.type_name(module.name.dotted()); let mut visits = String::new(); - let mut clears = String::new(); - for field in class.fields.iter().filter(|field| collectable(field)) { - let _ = writeln!( - out_slot(&mut visits), - " Py_VISIT(self->{});", - field.member() - ); - let _ = writeln!( - out_slot(&mut clears), - " Py_CLEAR(self->{});", - field.member() - ); + if class.resume.is_some() { + visits.push_str(" Py_VISIT(self->by_returned);\n"); + } + for field in &class.fields { + for held in held_references(&field.ty, &format!("self->{}", field.member())) { + let slot = held.slot; + if held.tagged { + let _ = writeln!( + visits, + " if (!By_IsShort({slot})) Py_VISIT(By_LongOf({slot}));" + ); + } else { + let _ = writeln!(visits, " Py_VISIT({slot});"); + } + } } format!( "static int {type_name}_traverse({struct_name} *self, visitproc visit, void *arg) {{\n\ \x20 (void)self; (void)visit; (void)arg;\n\ {visits}\ - \x20 return 0;\n}}\n\n\ - static int {type_name}_clear({struct_name} *self) {{\n\ - \x20 (void)self;\n\ - {clears}\ \x20 return 0;\n}}\n\n" ) } +/// `tp_traverse` and `tp_clear` for a closure environment +/// +/// a nested function that reads its own name holds the environment it reads it out of, +/// and the environment holds the function, so counting alone never frees either — the +/// collector has to be able to walk the cycle, and clearing the environment's fields is +/// what breaks it +fn emit_collected_environment(module: &ModuleIr, class: &ClassIr) -> String { + let struct_name = class.struct_name(module.name.dotted()); + let type_name = class.type_name(module.name.dotted()); + format!( + "{}static int {type_name}_clear({struct_name} *self) {{\n\ + \x20 (void)self;\n\ + {}\ + \x20 return 0;\n}}\n\n", + emit_frame_traverse(module, class), + emit_frame_clears(class) + ) +} + +/// `tp_traverse` and `tp_clear` for a generator's, coroutine's or async generator's +/// state object +/// +/// a suspended frame is as able to sit in a cycle as any other object — a generator that +/// holds a list the generator is in, say — and python frees that cycle by finalizing the +/// generator, which closes it and runs the `finally` it is suspended in. a type the +/// collector cannot walk keeps the cycle, and every object in it, for good. +/// +/// the clear runs after the finalizer has closed the frame, and it leaves the frame +/// finished: the fields it releases are the ones a resumption would read, and a finished +/// frame reads none of them +fn emit_collected_generator(module: &ModuleIr, class: &ClassIr) -> String { + let struct_name = class.struct_name(module.name.dotted()); + let type_name = class.type_name(module.name.dotted()); + format!( + "{}static int {type_name}_clear({struct_name} *self) {{\n\ + \x20 By_FinishGenerator(&self->{state});\n\ + \x20 Py_CLEAR(self->by_returned);\n\ + {}\ + \x20 return 0;\n}}\n\n", + emit_frame_traverse(module, class), + emit_frame_clears(class), + state = mangle_member(GENERATOR_STATE) + ) +} + +/// the releases a frame's `tp_clear` makes, one for each object reference a traverse +/// reports +/// +/// a tagged `int` is left alone: the `int` it may hold cannot be part of a cycle, so +/// releasing it breaks nothing, and a field still holding a valid representation is one +/// less state to reason about +fn emit_frame_clears(class: &ClassIr) -> String { + let mut clears = String::new(); + for field in &class.fields { + for held in held_references(&field.ty, &format!("self->{}", field.member())) { + if !held.tagged { + let _ = writeln!(clears, " Py_CLEAR({});", held.slot); + } + } + } + clears +} + /// `tp_dealloc`, `tp_traverse` and `tp_clear` for a class whose fields sit past a /// base's instance, and for a base such a class stands on /// @@ -764,6 +875,23 @@ fn emit_class_struct(module: &ModuleIr, class: &ClassIr) -> String { // parks across a suspension in it: it is written once, on the way out if class.resume.is_some() { out.push_str(" PyObject *by_returned;\n"); + // the weak references to the frame: python's generators, coroutines and async + // generators all take them + out.push_str(" PyObject *by_weakrefs;\n"); + } + // python's `ag_closed`: an async generator that has been asked to `aclose()` once + // answers every later `aclose()` and `athrow()` as already ended, whether or not the + // first close finished its frame + if class + .resume + .as_ref() + .is_some_and(|resume| resume.surface == Surface::AsyncGenerator) + { + out.push_str(" char by_closed;\n"); + // python's `ag_running_async`: an awaitable of this generator is stepping it, and + // stays stepping while the frame is suspended at an `await` — so no other + // awaitable may start until that one has finished + out.push_str(" char by_running_async;\n"); } for field in own_fields(module, class) { let _ = writeln!(out, " {} {};", ctype(module, &field.ty), field.member()); @@ -1010,6 +1138,8 @@ fn emit_class_type(module: &ModuleIr, class: &ClassIr) -> String { out.push_str(&emit_collected_instance(module, class)); } else if class.environment { out.push_str(&emit_collected_environment(module, class)); + } else if class.resume.is_some() { + out.push_str(&emit_collected_generator(module, class)); } // dealloc releases each refcounted field, then the object let _ = writeln!( @@ -1031,11 +1161,24 @@ fn emit_class_type(module: &ModuleIr, class: &ClassIr) -> String { // a subclass writes a dealloc of its own, so it has to make the call for the // finalizer it inherited — which is why this asks the whole chain rather than // only this class + // before the finalizer, as python's own generators do: a weak reference's callback + // runs while the object is still whole, and it sees a reference that is already dead + if class.resume.is_some() { + out.push_str( + " if (self->by_weakrefs != NULL) PyObject_ClearWeakRefs((PyObject *)self);\n", + ); + } if class.resume.is_some() || finalizes(module, class) { out.push_str( " if (PyObject_CallFinalizerFromDealloc((PyObject *)self) < 0) return;\n", ); } + // a frame's finalizer closes it, and a close that resurrects the object has to + // leave it tracked — so a generator comes off the collector's list only once + // the finalizer has said it is really going + if class.resume.is_some() { + out.push_str(" PyObject_GC_UnTrack(self);\n"); + } if keeps_a_dict { let _ = writeln!(out, " By_ReleaseInstanceDict(&self->{BY_DICT_MEMBER});"); } @@ -1059,7 +1202,7 @@ fn emit_class_type(module: &ModuleIr, class: &ClassIr) -> String { // a class with nothing to initialize publishes no `__init__` at all, so that // `object.__init__` is what a construction reaches — exactly as in the source - if !initializes(module, class) { + if !initializes(module, class) || made_only_by_its_frame(module, class) { out.push_str(&emit_class_members(module, class)); return out; } @@ -1924,6 +2067,38 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { .find(|method| method.name == resume.method) .map(|method| method.native_symbol(module.name.dotted())) .unwrap_or_default(); + // the iterator a suspension inside `yield from` or `await` waits on, which `throw` + // and `close` reach before the frame. a frame that never delegates has none to + // name, and says so with a null accessor rather than one that always answers NULL + let delegate = delegate_accessor(resume, &type_name); + if !resume.delegations.is_empty() { + let mut cases = String::new(); + for delegation in &resume.delegations { + let Some(field) = class + .fields + .iter() + .find(|field| field.name == delegation.field) + else { + continue; + }; + let _ = writeln!( + cases, + "\x20 case {}: return (PyObject *)self->{};", + delegation.state, + field.member() + ); + } + let _ = writeln!( + out, + "static PyObject *{type_name}_delegate(PyObject *selfobj) {{\n\ + \x20 {struct_name} *self = ({struct_name} *)selfobj;\n\ + \x20 switch (By_ShortValue(self->{state})) {{\n\ + {cases}\ + \x20 default: return NULL;\n\ + \x20 }}\n}}", + state = mangle_member(crate::GENERATOR_STATE) + ); + } let _ = writeln!( out, "static PyObject *{type_name}_send(PyObject *selfobj, PyObject *const *args, Py_ssize_t nargs) {{\n\ @@ -1938,19 +2113,17 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { sent = mangle_member(crate::GENERATOR_SENT), state = mangle_member(crate::GENERATOR_STATE) ); - // `close` throws `GeneratorExit` in, which runs every enclosing `finally`, then - // marks the machine exhausted whatever came back + // `close` throws `GeneratorExit` in, which runs every enclosing `finally`. the + // frame finishes itself on the way out, and one that suspends again instead is + // still suspended — python refuses that close and a later step resumes it let _ = writeln!( out, "static PyObject *{type_name}_close(PyObject *selfobj, PyObject *const *args, Py_ssize_t nargs) {{\n\ \x20 (void)args; (void)nargs;\n\ \x20 {struct_name} *self = ({struct_name} *)selfobj;\n\ - \x20 int by_r = By_CloseGenerator(selfobj, &self->{sent}, &self->{thrown},\n\ - \x20 &self->by_returned, &self->{state}, {frame},\n\ - \x20 (PyObject *(*)(PyObject *)){symbol});\n\ - \x20 By_FinishGenerator(&self->{state});\n\ - \x20 if (by_r < 0) return NULL;\n\ - \x20 Py_RETURN_NONE;\n}}", + \x20 return By_CloseGenerator(selfobj, &self->{sent}, &self->{thrown},\n\ + \x20 &self->by_returned, &self->{state}, {frame},\n\ + \x20 {delegate}, (PyObject *(*)(PyObject *)){symbol});\n}}", sent = mangle_member(crate::GENERATOR_SENT), thrown = mangle_member(crate::GENERATOR_THROWN), state = mangle_member(crate::GENERATOR_STATE) @@ -1960,15 +2133,13 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { let _ = writeln!( out, "static PyObject *{type_name}_throw(PyObject *selfobj, PyObject *const *args, Py_ssize_t nargs) {{\n\ - \x20 if (nargs < 1) {{\n\ - \x20 PyErr_SetString(PyExc_TypeError, \"throw() takes at least one argument\");\n\ - \x20 return NULL;\n\ - \x20 }}\n\ + \x20 if (By_CountThrowArguments(\"throw\", nargs) < 0\n\ + \x20 || By_WarnThrowSignature(\"throw\", nargs) < 0) return NULL;\n\ \x20 return By_ThrowInto(selfobj, &(({struct_name} *)selfobj)->{sent},\n\ \x20 &(({struct_name} *)selfobj)->{thrown},\n\ \x20 &(({struct_name} *)selfobj)->by_returned,\n\ - \x20 &(({struct_name} *)selfobj)->{state}, {frame}, args[0],\n\ - \x20 (PyObject *(*)(PyObject *)){symbol});\n}}", + \x20 &(({struct_name} *)selfobj)->{state}, {frame}, args, nargs,\n\ + \x20 {delegate}, (PyObject *(*)(PyObject *)){symbol});\n}}", sent = mangle_member(crate::GENERATOR_SENT), thrown = mangle_member(crate::GENERATOR_THROWN), state = mangle_member(crate::GENERATOR_STATE) @@ -1980,10 +2151,28 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { // by closing it, which resumes the frame *by raising* at the suspension and so // runs every `finally` and every `__exit__` the body was inside. without this a // context manager the frame never left would simply never be exited - if class.resume.is_some() { + if let Some(resume) = &class.resume { + // a coroutine that never ran is not unwound, it is reported: python takes one + // dropped before its first step for a missing `await` + let unawaited = if resume.surface == Surface::Coroutine { + format!( + "\x20 if ((({struct_name} *)selfobj)->{state} == By_ShortFrom(0)) {{\n\ + \x20 PyObject *by_type, *by_value, *by_tb;\n\ + \x20 PyErr_Fetch(&by_type, &by_value, &by_tb);\n\ + \x20 By_WarnUnawaitedCoroutine(selfobj, {qualname});\n\ + \x20 PyErr_Restore(by_type, by_value, by_tb);\n\ + \x20 return;\n\ + \x20 }}\n", + state = mangle_member(crate::GENERATOR_STATE), + qualname = c_string(&resume.qualname) + ) + } else { + String::new() + }; let _ = writeln!( out, "static void {type_name}_finalize(PyObject *selfobj) {{\n\ + {unawaited}\ \x20 /* only a *suspended* frame has anything to unwind: state 0 never\n\ \x20 * started and -1 already finished, and resuming either would run\n\ \x20 * the body a second time */\n\ @@ -1991,7 +2180,7 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { \x20 PyObject *by_type, *by_value, *by_tb;\n\ \x20 PyErr_Fetch(&by_type, &by_value, &by_tb);\n\ \x20 PyObject *by_r = {type_name}_close(selfobj, NULL, 0);\n\ - \x20 if (by_r == NULL) PyErr_WriteUnraisable(selfobj);\n\ + \x20 if (by_r == NULL) By_CloseUnraisable(selfobj);\n\ \x20 else Py_DECREF(by_r);\n\ \x20 PyErr_Restore(by_type, by_value, by_tb);\n}}", state = mangle_member(crate::GENERATOR_STATE) @@ -2066,16 +2255,20 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { .find(|method| method.name == resume.method) .map(|method| method.native_symbol(module.name.dotted())) .unwrap_or_default(); - let _ = writeln!( - out, - "static PyObject *{type_name}_iternext(PyObject *self) {{\n\ + // only a generator is an iterator: a coroutine is driven through its send + // slot, and an async generator through the awaitable `__anext__` hands back + if resume.surface == Surface::Generator { + let _ = writeln!( + out, + "static PyObject *{type_name}_iternext(PyObject *self) {{\n\ \x20 return By_StepGenerator(self, &(({struct_name} *)self)->{sent},\n\ \x20 &(({struct_name} *)self)->by_returned,\n\ \x20 &(({struct_name} *)self)->{state}, {frame}, Py_None,\n\ \x20 (PyObject *(*)(PyObject *)){symbol});\n}}", - sent = mangle_member(crate::GENERATOR_SENT), - state = mangle_member(crate::GENERATOR_STATE) - ); + sent = mangle_member(crate::GENERATOR_SENT), + state = mangle_member(crate::GENERATOR_STATE) + ); + } // the slot `PyIter_Send` prefers, and the reason a `return` is reported by // writing it down rather than by raising: an `await` that completes gets its // answer without an exception being built and immediately unpacked again. @@ -2105,6 +2298,7 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { .map(by_ir::FieldDecl::member) .unwrap_or_default(); let dotted = module.name.dotted(); + let delegate = delegate_accessor(resume, &type_name); if resume.surface == Surface::AsyncGenerator { // `__anext__` hands back an awaitable rather than an item, because the // body may `await` before it reaches its next `yield`. one `resume` @@ -2114,144 +2308,324 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { "typedef struct {{\n\ \x20 PyObject_HEAD\n\ \x20 {struct_name} *by_gen;\n\ - \x20 /* 0 anext, 1 aclose, 2 asend, 3 athrow. the value is what the\n\ - \x20 * last two carry, consumed on the first step so a resumption\n\ - \x20 * after it behaves like a plain one */\n\ + \x20 /* 0 anext, 1 aclose, 2 asend, 3 athrow. what the last two carry: the value `asend`\n\ + \x20 * sends in on its first step, and the arguments `athrow` builds its exception from */\n\ \x20 char by_mode;\n\ - \x20 PyObject *by_value;\n}} {type_name}_asend;\n\ + \x20 /* python's awaitable states: 0 not yet stepped, 1 being stepped, 2 finished */\n\ + \x20 char by_state;\n\ + \x20 PyObject *by_value;\n\ + }} {type_name}_asend;\n\ static PyTypeObject {type_name}_asend_type;\n\ static void {type_name}_asend_dealloc(PyObject *self) {{\n\ + \x20 PyObject_GC_UnTrack(self);\n\ \x20 Py_XDECREF((PyObject *)(({type_name}_asend *)self)->by_gen);\n\ \x20 Py_XDECREF((({type_name}_asend *)self)->by_value);\n\ - \x20 Py_TYPE(self)->tp_free(self);\n}}\n\ + \x20 Py_TYPE(self)->tp_free(self);\n\ + }}\n\ + static int {type_name}_asend_traverse(PyObject *self, visitproc visit, void *arg) {{\n\ + \x20 Py_VISIT((PyObject *)(({type_name}_asend *)self)->by_gen);\n\ + \x20 Py_VISIT((({type_name}_asend *)self)->by_value);\n\ + \x20 return 0;\n\ + }}\n\ static PyObject *{type_name}_asend_await(PyObject *self) {{\n\ - \x20 return By_NewRef(self);\n}}\n\ + \x20 return By_NewRef(self);\n\ + }}\n\ + /* what a step of the frame hands the awaitable: a yield finishes the awaitable with its\n\ + \x20* item, an await is passed on to whatever drives it, and the frame leaving ends the\n\ + \x20* iteration. either of the last two leaves the generator free for another awaitable */\n\ + static PyObject *{type_name}_unwrap({struct_name} *by_gen, PyObject *by_step) {{\n\ + \x20 if (by_step == NULL) {{\n\ + \x20 if (!PyErr_Occurred()) PyErr_SetNone(PyExc_StopAsyncIteration);\n\ + \x20 By_EndAsyncIteration();\n\ + \x20 if (PyErr_ExceptionMatches(PyExc_StopAsyncIteration)\n\ + \x20 || PyErr_ExceptionMatches(PyExc_GeneratorExit)) {{\n\ + \x20 by_gen->by_closed = 1;\n\ + \x20 }}\n\ + \x20 by_gen->by_running_async = 0;\n\ + \x20 return NULL;\n\ + \x20 }}\n\ + \x20 if (by_gen->{kind} != By_ShortFrom(1)) return by_step;\n\ + \x20 By_RaiseWith(PyExc_StopIteration, by_step);\n\ + \x20 Py_DECREF(by_step);\n\ + \x20 by_gen->by_running_async = 0;\n\ + \x20 return NULL;\n\ + }}\n\ + /* the message a second awaitable is refused with while one is stepping the generator */\n\ + static void {type_name}_refuse_running({type_name}_asend *by_self) {{\n\ + \x20 by_self->by_state = 2;\n\ + \x20 PyErr_Format(PyExc_RuntimeError, \"%s(): asynchronous generator is already running\",\n\ + \x20 by_self->by_mode == 1 ? \"aclose\" : by_self->by_mode == 3 ? \"athrow\" : \"anext\");\n\ + }}\n\ + static void {type_name}_refuse_reuse({type_name}_asend *by_self) {{\n\ + \x20 PyErr_SetString(PyExc_RuntimeError, by_self->by_mode == 1 || by_self->by_mode == 3\n\ + \x20 ? \"cannot reuse already awaited aclose()/athrow()\"\n\ + \x20 : \"cannot reuse already awaited __anext__()/asend()\");\n\ + }}\n\ + /* python's `throw` of the generator itself, which an awaitable's `throw` is */\n\ + static PyObject *{type_name}_gen_throw({struct_name} *by_gen, PyObject *const *args,\n\ + \x20 Py_ssize_t nargs) {{\n\ + \x20 if (By_CountThrowArguments(\"throw\", nargs) < 0 || By_WarnThrowSignature(\"throw\", nargs) < 0)\n\ + \x20 return NULL;\n\ + \x20 return By_ThrowInto((PyObject *)by_gen, &by_gen->{sent}, &by_gen->{thrown},\n\ + \x20 &by_gen->by_returned, &by_gen->{state}, {frame}, args, nargs,\n\ + \x20 {delegate}, (PyObject *(*)(PyObject *)){symbol});\n\ + }}\n\ + static PyObject *{type_name}_asend_send(PyObject *self, PyObject *by_arg) {{\n\ + \x20 {type_name}_asend *by_self = ({type_name}_asend *)self;\n\ + \x20 {struct_name} *by_gen = by_self->by_gen;\n\ + \x20 if (by_self->by_mode == 1 || by_self->by_mode == 3) {{\n\ + \x20 /* `aclose()` and `athrow()` */\n\ + \x20 int by_closing = by_self->by_mode == 1;\n\ + \x20 PyObject *by_step;\n\ + \x20 if (by_self->by_state == 2) {{\n\ + \x20 {type_name}_refuse_reuse(by_self);\n\ + \x20 return NULL;\n\ + \x20 }}\n\ + \x20 if (By_ShortValue(by_gen->{state}) == -1) {{\n\ + \x20 by_self->by_state = 2;\n\ + \x20 PyErr_SetNone(PyExc_StopIteration);\n\ + \x20 return NULL;\n\ + \x20 }}\n\ + \x20 if (by_arg == NULL) by_arg = Py_None;\n\ + \x20 if (by_self->by_state == 0) {{\n\ + \x20 if (by_gen->by_running_async) {{\n\ + \x20 {type_name}_refuse_running(by_self);\n\ + \x20 return NULL;\n\ + \x20 }}\n\ + \x20 if (by_gen->by_closed) {{\n\ + \x20 by_self->by_state = 2;\n\ + \x20 PyErr_SetNone(PyExc_StopAsyncIteration);\n\ + \x20 return NULL;\n\ + \x20 }}\n\ + \x20 if (by_arg != Py_None) {{\n\ + \x20 PyErr_SetString(PyExc_RuntimeError,\n\ + \x20 \"can't send non-None value to a just-started coroutine\");\n\ + \x20 return NULL;\n\ + \x20 }}\n\ + \x20 by_self->by_state = 1;\n\ + \x20 by_gen->by_running_async = 1;\n\ + \x20 if (by_closing) {{\n\ + \x20 by_gen->by_closed = 1;\n\ + \x20 PyObject *by_exit = PyExc_GeneratorExit;\n\ + \x20 by_step = By_ThrowInto((PyObject *)by_gen, &by_gen->{sent}, &by_gen->{thrown},\n\ + \x20 &by_gen->by_returned, &by_gen->{state}, {frame},\n\ + \x20 &by_exit, 1, {delegate},\n\ + \x20 (PyObject *(*)(PyObject *)){symbol});\n\ + \x20 }} else {{\n\ + \x20 /* what `athrow` was called with, kept whole: python builds the exception\n\ + \x20 * out of it at this step rather than at the call */\n\ + \x20 PyObject *by_carried = by_self->by_value;\n\ + \x20 if (By_CountThrowArguments(\"athrow\", PyTuple_GET_SIZE(by_carried)) < 0) return NULL;\n\ + \x20 by_step = By_ThrowInto((PyObject *)by_gen, &by_gen->{sent}, &by_gen->{thrown},\n\ + \x20 &by_gen->by_returned, &by_gen->{state}, {frame},\n\ + \x20 &PyTuple_GET_ITEM(by_carried, 0),\n\ + \x20 PyTuple_GET_SIZE(by_carried), {delegate},\n\ + \x20 (PyObject *(*)(PyObject *)){symbol});\n\ + \x20 by_step = {type_name}_unwrap(by_gen, by_step);\n\ + \x20 if (by_step != NULL) return by_step;\n\ + \x20 by_gen->by_running_async = 0;\n\ + \x20 by_self->by_state = 2;\n\ + \x20 return NULL;\n\ + \x20 }}\n\ + \x20 }} else {{\n\ + \x20 by_step = By_StepGenerator((PyObject *)by_gen, &by_gen->{sent}, &by_gen->by_returned,\n\ + \x20 &by_gen->{state}, {frame}, by_arg,\n\ + \x20 (PyObject *(*)(PyObject *)){symbol});\n\ + \x20 if (!by_closing) return {type_name}_unwrap(by_gen, by_step);\n\ + \x20 }}\n\ + \x20 /* `aclose()` from here on: a yield is the frame refusing to close, and the frame\n\ + \x20 * leaving is the close finishing */\n\ + \x20 if (by_step != NULL && by_gen->{kind} != By_ShortFrom(1)) return by_step;\n\ + \x20 by_gen->by_running_async = 0;\n\ + \x20 by_self->by_state = 2;\n\ + \x20 if (by_step != NULL) {{\n\ + \x20 Py_DECREF(by_step);\n\ + \x20 PyErr_SetString(PyExc_RuntimeError, \"async generator ignored GeneratorExit\");\n\ + \x20 return NULL;\n\ + \x20 }}\n\ + \x20 By_EndAsyncIteration();\n\ + \x20 if (PyErr_ExceptionMatches(PyExc_StopAsyncIteration)\n\ + \x20 || PyErr_ExceptionMatches(PyExc_GeneratorExit)) {{\n\ + \x20 PyErr_Clear();\n\ + \x20 PyErr_SetNone(PyExc_StopIteration);\n\ + \x20 }}\n\ + \x20 return NULL;\n\ + \x20 }}\n\ + \x20 /* `__anext__()` and `asend()` */\n\ + \x20 if (by_self->by_state == 2) {{\n\ + \x20 {type_name}_refuse_reuse(by_self);\n\ + \x20 return NULL;\n\ + \x20 }}\n\ + \x20 if (by_self->by_state == 0) {{\n\ + \x20 if (by_gen->by_running_async) {{\n\ + \x20 {type_name}_refuse_running(by_self);\n\ + \x20 return NULL;\n\ + \x20 }}\n\ + \x20 if (by_arg == NULL || by_arg == Py_None) by_arg = by_self->by_value;\n\ + \x20 by_self->by_state = 1;\n\ + \x20 }}\n\ + \x20 by_gen->by_running_async = 1;\n\ + \x20 PyObject *by_step = By_StepGenerator((PyObject *)by_gen, &by_gen->{sent}, &by_gen->by_returned,\n\ + \x20 &by_gen->{state}, {frame},\n\ + \x20 by_arg != NULL ? by_arg : Py_None,\n\ + \x20 (PyObject *(*)(PyObject *)){symbol});\n\ + \x20 by_step = {type_name}_unwrap(by_gen, by_step);\n\ + \x20 if (by_step == NULL) by_self->by_state = 2;\n\ + \x20 return by_step;\n\ + }}\n\ static PyObject *{type_name}_asend_next(PyObject *self) {{\n\ + \x20 return {type_name}_asend_send(self, NULL);\n\ + }}\n\ + static PyObject *{type_name}_asend_throw(PyObject *self, PyObject *const *args, Py_ssize_t nargs) {{\n\ \x20 {type_name}_asend *by_self = ({type_name}_asend *)self;\n\ \x20 {struct_name} *by_gen = by_self->by_gen;\n\ - \x20 if (by_self->by_mode == 1) {{\n\ - \x20 PyObject *by_done = {type_name}_close((PyObject *)by_gen, NULL, 0);\n\ - \x20 if (by_done == NULL) return NULL;\n\ - \x20 PyErr_SetObject(PyExc_StopIteration, by_done);\n\ - \x20 Py_DECREF(by_done);\n\ + \x20 int by_closing = by_self->by_mode == 1;\n\ + \x20 if (by_self->by_state == 2) {{\n\ + \x20 {type_name}_refuse_reuse(by_self);\n\ + \x20 return NULL;\n\ + \x20 }}\n\ + \x20 if (by_self->by_state == 0) {{\n\ + \x20 if (by_gen->by_running_async) {{\n\ + \x20 {type_name}_refuse_running(by_self);\n\ + \x20 return NULL;\n\ + \x20 }}\n\ + \x20 by_self->by_state = 1;\n\ + \x20 by_gen->by_running_async = 1;\n\ + \x20 }}\n\ + \x20 PyObject *by_step = {type_name}_gen_throw(by_gen, args, nargs);\n\ + \x20 if (!by_closing) {{\n\ + \x20 by_step = {type_name}_unwrap(by_gen, by_step);\n\ + \x20 if (by_step == NULL) {{\n\ + \x20 by_gen->by_running_async = 0;\n\ + \x20 by_self->by_state = 2;\n\ + \x20 }}\n\ + \x20 return by_step;\n\ + \x20 }}\n\ + \x20 if (by_step != NULL && by_gen->{kind} == By_ShortFrom(1)) {{\n\ + \x20 by_gen->by_running_async = 0;\n\ + \x20 by_self->by_state = 2;\n\ + \x20 Py_DECREF(by_step);\n\ + \x20 PyErr_SetString(PyExc_RuntimeError, \"async generator ignored GeneratorExit\");\n\ \x20 return NULL;\n\ \x20 }}\n\ - \x20 PyObject *by_carried = by_self->by_value;\n\ - \x20 by_self->by_value = NULL;\n\ - \x20 PyObject *by_step;\n\ - \x20 /* an `athrow` into a frame that has already finished neither\n\ - \x20 * resumes it nor re-raises: python ends the await with `None`.\n\ - \x20 * one that never *started* is the other case, and raises at the\n\ - \x20 * call site — which is what `By_ThrowInto` does with it */\n\ - \x20 if (by_carried != NULL && by_self->by_mode == 3\n\ - \x20 && By_ShortValue(by_gen->{state}) < 0) {{\n\ - \x20 Py_DECREF(by_carried);\n\ + \x20 if (by_step != NULL) return by_step;\n\ + \x20 by_gen->by_running_async = 0;\n\ + \x20 by_self->by_state = 2;\n\ + \x20 By_EndAsyncIteration();\n\ + \x20 if (PyErr_ExceptionMatches(PyExc_StopAsyncIteration)\n\ + \x20 || PyErr_ExceptionMatches(PyExc_GeneratorExit)) {{\n\ + \x20 PyErr_Clear();\n\ \x20 PyErr_SetNone(PyExc_StopIteration);\n\ + \x20 }}\n\ + \x20 return NULL;\n\ + }}\n\ + static PyObject *{type_name}_asend_close(PyObject *self, PyObject *by_unused) {{\n\ + \x20 (void)by_unused;\n\ + \x20 if ((({type_name}_asend *)self)->by_state == 2) Py_RETURN_NONE;\n\ + \x20 PyObject *by_exit = PyExc_GeneratorExit;\n\ + \x20 PyObject *by_step = {type_name}_asend_throw(self, &by_exit, 1);\n\ + \x20 if (by_step != NULL) {{\n\ + \x20 Py_DECREF(by_step);\n\ + \x20 PyErr_SetString(PyExc_RuntimeError, \"coroutine ignored GeneratorExit\");\n\ \x20 return NULL;\n\ \x20 }}\n\ - \x20 if (by_carried != NULL && by_self->by_mode == 3) {{\n\ - \x20 by_step = By_ThrowInto((PyObject *)by_gen, &by_gen->{sent},\n\ - \x20 &by_gen->{thrown}, &by_gen->by_returned,\n\ - \x20 &by_gen->{state}, {frame}, by_carried,\n\ - \x20 (PyObject *(*)(PyObject *)){symbol});\n\ - \x20 Py_DECREF(by_carried);\n\ - \x20 }} else {{\n\ - \x20 /* `__anext__` carries nothing, which is `None` and not\n\ - \x20 * whatever the last `asend` left standing */\n\ - \x20 by_step = By_StepGenerator((PyObject *)by_gen, &by_gen->{sent},\n\ - \x20 &by_gen->by_returned, &by_gen->{state},\n\ - \x20 {frame},\n\ - \x20 by_carried != NULL ? by_carried : Py_None,\n\ - \x20 (PyObject *(*)(PyObject *)){symbol});\n\ - \x20 Py_XDECREF(by_carried);\n\ + \x20 if (PyErr_ExceptionMatches(PyExc_StopIteration)\n\ + \x20 || PyErr_ExceptionMatches(PyExc_StopAsyncIteration)\n\ + \x20 || PyErr_ExceptionMatches(PyExc_GeneratorExit)) {{\n\ + \x20 PyErr_Clear();\n\ + \x20 Py_RETURN_NONE;\n\ \x20 }}\n\ - \x20 if (by_step == NULL) return By_EndAsyncIteration();\n\ - \x20 if (by_gen->{kind} != By_ShortFrom(1)) return by_step;\n\ - \x20 /* a yield finishes *this* await, carrying the item */\n\ - \x20 PyErr_SetObject(PyExc_StopIteration, by_step);\n\ - \x20 Py_DECREF(by_step);\n\ - \x20 return NULL;\n}}\n\ + \x20 return NULL;\n\ + }}\n\ + static PyMethodDef {type_name}_asend_methods[] = {{\n\ + \x20 {{\"send\", (PyCFunction){type_name}_asend_send, METH_O, NULL}},\n\ + \x20 {{\"throw\", (PyCFunction)(void(*)(void)){type_name}_asend_throw, METH_FASTCALL, NULL}},\n\ + \x20 {{\"close\", (PyCFunction){type_name}_asend_close, METH_NOARGS, NULL}},\n\ + \x20 {{NULL, NULL, 0, NULL}}\n\ + }};\n\ static PyAsyncMethods {type_name}_asend_async = {{\n\ - \x20 .am_await = {type_name}_asend_await,\n}};\n\ + \x20 .am_await = {type_name}_asend_await,\n\ + }};\n\ static PyTypeObject {type_name}_asend_type = {{\n\ \x20 PyVarObject_HEAD_INIT(NULL, 0)\n\ - \x20 .tp_name = \"{dotted}.{}.ascend\",\n\ + \x20 .tp_name = \"{dotted}.{0}.ascend\",\n\ \x20 .tp_basicsize = sizeof({type_name}_asend),\n\ \x20 .tp_dealloc = (destructor){type_name}_asend_dealloc,\n\ - \x20 .tp_flags = Py_TPFLAGS_DEFAULT,\n\ + \x20 .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC\n\ + \x20 | Py_TPFLAGS_DISALLOW_INSTANTIATION,\n\ + \x20 .tp_traverse = {type_name}_asend_traverse,\n\ \x20 .tp_as_async = &{type_name}_asend_async,\n\ + \x20 .tp_iter = PyObject_SelfIter,\n\ \x20 .tp_iternext = {type_name}_asend_next,\n\ - \x20 .tp_new = PyType_GenericNew,\n}};\n\ + \x20 .tp_methods = {type_name}_asend_methods,\n\ + }};\n\ static PyObject *{type_name}_aiter(PyObject *self) {{\n\ - \x20 return By_NewRef(self);\n}}\n\ - static PyObject *{type_name}_anext(PyObject *self) {{\n\ + \x20 return By_NewRef(self);\n\ + }}\n\ + static PyObject *{type_name}_await_step(PyObject *self, char by_mode, PyObject *by_value) {{\n\ \x20 {type_name}_asend *by_send =\n\ - \x20 PyObject_New({type_name}_asend, &{type_name}_asend_type);\n\ - \x20 if (by_send == NULL) return NULL;\n\ - \x20 by_send->by_gen = ({struct_name} *)By_NewRef(self);\n\ - \x20 by_send->by_mode = 0;\n\ - \x20 by_send->by_value = NULL;\n\ - \x20 return (PyObject *)by_send;\n}}\n\ - static PyObject *{type_name}_await_step(PyObject *self, char by_mode,\n\ - \x20 PyObject *by_value) {{\n\ - \x20 {type_name}_asend *by_send =\n\ - \x20 PyObject_New({type_name}_asend, &{type_name}_asend_type);\n\ + \x20 PyObject_GC_New({type_name}_asend, &{type_name}_asend_type);\n\ \x20 if (by_send == NULL) return NULL;\n\ \x20 by_send->by_gen = ({struct_name} *)By_NewRef(self);\n\ \x20 by_send->by_mode = by_mode;\n\ - \x20 by_send->by_value = By_NewRef(by_value);\n\ - \x20 return (PyObject *)by_send;\n}}\n\ + \x20 by_send->by_state = 0;\n\ + \x20 by_send->by_value = by_value == NULL ? NULL : By_NewRef(by_value);\n\ + \x20 PyObject_GC_Track(by_send);\n\ + \x20 return (PyObject *)by_send;\n\ + }}\n\ + static PyObject *{type_name}_anext(PyObject *self) {{\n\ + \x20 return {type_name}_await_step(self, 0, NULL);\n\ + }}\n\ static PyObject *{type_name}_do_asend(PyObject *self, PyObject *const *args,\n\ - \x20 Py_ssize_t nargs) {{\n\ + \x20 Py_ssize_t nargs) {{\n\ \x20 if (nargs != 1) {{\n\ \x20 PyErr_SetString(PyExc_TypeError, \"asend() takes exactly one argument\");\n\ \x20 return NULL;\n\ \x20 }}\n\ - \x20 /* `asend(None)` *is* `__anext__`: nothing to carry in */\n\ - \x20 if (args[0] == Py_None) return {type_name}_anext(self);\n\ - \x20 return {type_name}_await_step(self, 2, args[0]);\n}}\n\ + \x20 return {type_name}_await_step(self, 2, args[0]);\n\ + }}\n\ static PyObject *{type_name}_do_athrow(PyObject *self, PyObject *const *args,\n\ - \x20 Py_ssize_t nargs) {{\n\ - \x20 if (nargs < 1) {{\n\ - \x20 PyErr_SetString(PyExc_TypeError, \"athrow() takes at least one argument\");\n\ - \x20 return NULL;\n\ - \x20 }}\n\ - \x20 return {type_name}_await_step(self, 3, args[0]);\n}}\n\ + \x20 Py_ssize_t nargs) {{\n\ + \x20 if (By_WarnThrowSignature(\"athrow\", nargs) < 0) return NULL;\n\ + \x20 PyObject *by_args = PyTuple_New(nargs);\n\ + \x20 if (by_args == NULL) return NULL;\n\ + \x20 for (Py_ssize_t by_i = 0; by_i < nargs; by_i++)\n\ + \x20 PyTuple_SET_ITEM(by_args, by_i, By_NewRef(args[by_i]));\n\ + \x20 PyObject *by_step = {type_name}_await_step(self, 3, by_args);\n\ + \x20 Py_DECREF(by_args);\n\ + \x20 return by_step;\n\ + }}\n\ static PyObject *{type_name}_aclose(PyObject *self, PyObject *const *args,\n\ - \x20 Py_ssize_t nargs) {{\n\ + \x20 Py_ssize_t nargs) {{\n\ \x20 (void)args; (void)nargs;\n\ - \x20 {type_name}_asend *by_send =\n\ - \x20 PyObject_New({type_name}_asend, &{type_name}_asend_type);\n\ - \x20 if (by_send == NULL) return NULL;\n\ - \x20 by_send->by_gen = ({struct_name} *)By_NewRef(self);\n\ - \x20 by_send->by_mode = 1;\n\ - \x20 by_send->by_value = NULL;\n\ - \x20 return (PyObject *)by_send;\n}}\n\ + \x20 return {type_name}_await_step(self, 1, NULL);\n\ + }}\n\ static PyAsyncMethods {type_name}_async = {{\n\ \x20 .am_aiter = {type_name}_aiter,\n\ - \x20 .am_anext = {type_name}_anext,\n}};", + \x20 .am_anext = {type_name}_anext,\n\ + }};", class.name, kind = kind_member, sent = mangle_member(crate::GENERATOR_SENT), thrown = mangle_member(crate::GENERATOR_THROWN), state = mangle_member(crate::GENERATOR_STATE), + dotted = dotted, ); format!( " .tp_as_async = &{type_name}_async,\n .tp_finalize = {type_name}_finalize,\n" ) } else if resume.surface == Surface::Coroutine { - // a coroutine is awaitable, not iterable: `__await__` hands back an - // iterator, and `for x in coro()` has to stay a `TypeError` + // a coroutine is awaitable and is not an iterator: `__await__` hands back + // one wrapped around it, `next(coro)` is a `TypeError`, and so is + // `for x in coro()`. an await drives it through the send slot let _ = writeln!( out, - "static PyObject *{type_name}_await(PyObject *self) {{\n\ - \x20 return By_NewRef(self);\n}}\n\ - static PyAsyncMethods {type_name}_async = {{\n\ - \x20 .am_await = {type_name}_await,\n\ + "static PyAsyncMethods {type_name}_async = {{\n\ + \x20 .am_await = By_CoroutineAwait,\n\ \x20 .am_send = {type_name}_send_slot,\n}};" ); format!( - " .tp_as_async = &{type_name}_async,\n .tp_iternext = {type_name}_iternext,\n .tp_finalize = {type_name}_finalize,\n" + " .tp_as_async = &{type_name}_async,\n .tp_finalize = {type_name}_finalize,\n" ) } else { // a generator answers the send slot too — a `yield from` reaches it the @@ -2276,16 +2650,31 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { // a static struct is what a class no name reaches gets: a generator's state or a // closure's environment, neither of which anything can ask about if !heap_type(module, class) { - let flags = if class.environment { + let (refused, construction) = if made_only_by_its_frame(module, class) { + (" | Py_TPFLAGS_DISALLOW_INSTANTIATION", String::new()) + } else { + ( + "", + format!( + " .tp_init = {type_name}_init,\n .tp_new = PyType_GenericNew,\n" + ), + ) + }; + let weakrefs = if class.resume.is_some() { + format!(" .tp_weaklistoffset = offsetof({struct_name}, by_weakrefs),\n") + } else { + String::new() + }; + let flags = if class.environment || class.resume.is_some() { format!( - " .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,\n .tp_traverse = (traverseproc){type_name}_traverse,\n .tp_clear = (inquiry){type_name}_clear,\n" + " .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC{refused},\n .tp_traverse = (traverseproc){type_name}_traverse,\n .tp_clear = (inquiry){type_name}_clear,\n" ) } else { - " .tp_flags = Py_TPFLAGS_DEFAULT,\n".to_string() + format!(" .tp_flags = Py_TPFLAGS_DEFAULT{refused},\n") }; let _ = write!( out, - "static PyTypeObject {type_name} = {{\n PyVarObject_HEAD_INIT(NULL, 0)\n .tp_name = \"{dotted}.{}\",\n .tp_basicsize = sizeof({struct_name}),\n .tp_itemsize = 0,\n .tp_dealloc = (destructor){type_name}_dealloc,\n{flags}{iterator}{dunders} .tp_methods = {type_name}_methods,\n .tp_getset = {type_name}_getset,\n .tp_init = {type_name}_init,\n .tp_new = PyType_GenericNew,\n }};\n\ + "static PyTypeObject {type_name} = {{\n PyVarObject_HEAD_INIT(NULL, 0)\n .tp_name = \"{dotted}.{}\",\n .tp_basicsize = sizeof({struct_name}),\n .tp_itemsize = 0,\n .tp_dealloc = (destructor){type_name}_dealloc,\n{flags}{weakrefs}{iterator}{dunders} .tp_methods = {type_name}_methods,\n .tp_getset = {type_name}_getset,\n{construction} }};\n\ ", class.name ); @@ -2735,6 +3124,26 @@ fn initializes(module: &ModuleIr, class: &ClassIr) -> bool { !mutable_type(module, class) || !class.inherited_init } +/// whether an instance of this class may only come from the compiled frame that owns it +/// +/// a generator's state object and a closure's environment are allocated by that frame +/// and filled field by field, with no constructor in between. python has no name for +/// either type but can still reach one — `type(gen)`, or `gc.get_referents` of a +/// closure — and an instance it allocated, or re-initialised under a suspended frame, +/// would reach the body with fields the frame never wrote: +/// +/// ```python +/// g = counted(3) +/// kind = type(g) +/// next(kind.__new__(kind)) # an iterator field that was never filled +/// ``` +/// +/// so the type refuses instantiation the way python's own `generator` does, and has no +/// `__init__` for a second call to reach +fn made_only_by_its_frame(module: &ModuleIr, class: &ClassIr) -> bool { + !class.exported && !heap_type(module, class) +} + /// whether this class keeps its fields *after* a base's instance rather than inside one /// /// PEP 697: a negative `basicsize` asks for that much room past whatever the base @@ -4583,27 +4992,25 @@ fn dunder_initializers(class: &ClassIr, type_name: &str) -> String { } fn iterator_slots(class: &ClassIr, type_name: &str) -> Vec<(String, String)> { - if class.resume.is_none() { + let Some(resume) = &class.resume else { return Vec::new(); - } - let mut slots = vec![ - ( - "Py_tp_iternext".to_string(), - format!("{type_name}_iternext"), - ), - ( - "Py_tp_finalize".to_string(), - format!("{type_name}_finalize"), - ), - ]; - if class - .resume - .as_ref() - .is_some_and(|resume| resume.surface == Surface::Coroutine) - { - slots.push(("Py_am_await".to_string(), format!("{type_name}_await"))); - } else { - slots.push(("Py_tp_iter".to_string(), "PyObject_SelfIter".to_string())); + }; + let mut slots = vec![( + "Py_tp_finalize".to_string(), + format!("{type_name}_finalize"), + )]; + match resume.surface { + Surface::Generator => { + slots.push(( + "Py_tp_iternext".to_string(), + format!("{type_name}_iternext"), + )); + slots.push(("Py_tp_iter".to_string(), "PyObject_SelfIter".to_string())); + } + Surface::Coroutine => { + slots.push(("Py_am_await".to_string(), "By_CoroutineAwait".to_string())); + } + Surface::AsyncGenerator => {} } slots } @@ -4892,6 +5299,7 @@ fn emit_function(module: &ModuleIr, function: &Function) -> String { // not run, which leaves the emitter its conservative answer let mut at_error: Option> = entered.as_ref().map(|_| HashSet::new()); + let mut sites = TracebackSites::default(); for (index, block) in function.blocks.iter().enumerate() { // a label immediately before a declaration is invalid in C89 and merely // ugly later; the empty statement keeps it valid everywhere @@ -4901,11 +5309,38 @@ fn emit_function(module: &ModuleIr, function: &Function) -> String { .as_ref() .and_then(|sets| sets.get(index).cloned()) .unwrap_or_default(); + // until the block says otherwise, its ops came from where the block starts + let mut line = block + .position + .map(|position| position.offset) + .or_else(|| block.range.or(function.range).map(|(start, _)| start)); + let mut in_generator_expression = block + .position + .is_some_and(|position| position.generator_expression); for op in &block.ops { - let mut fragment = guard_unassigned(function, &op.operands(), block.error_target); - fragment.push_str(&emit_op(module, function, op, block.error_target)); - if let Some(at_error) = at_error.as_mut() { - note_shared_error_jump(&fragment, &written, op.dest(), at_error); + if let Op::Line { position } = op { + line = Some(position.offset); + in_generator_expression = position.generator_expression; + continue; + } + let edge = if adds_a_traceback_entry(op) { + sites.edge(module, line, in_generator_expression, block.error_target) + } else { + ErrorEdge::straight(block.error_target) + }; + let mut fragment = guard_unassigned(function, &op.operands(), edge); + fragment.push_str(&emit_op(module, function, op, edge)); + sites.note(&fragment, edge); + if let Some(at_error) = at_error.as_mut() + && edge.target.is_none() + { + note_shared_error_jump( + &fragment, + &error_label(edge), + &written, + op.dest(), + at_error, + ); } out.push_str(&fragment); out.push_str(&mark_assigned(function, op)); @@ -4913,19 +5348,23 @@ fn emit_function(module: &ModuleIr, function: &Function) -> String { written.insert(dest); } } - let mut fragment = - guard_unassigned(function, &block.terminator.operands(), block.error_target); + let edge = sites.edge(module, line, in_generator_expression, block.error_target); + let mut fragment = guard_unassigned(function, &block.terminator.operands(), edge); fragment.push_str(&emit_terminator( module, function, &block.terminator, block.owned_at_exit.as_deref(), )); - if let Some(at_error) = at_error.as_mut() { - note_shared_error_jump(&fragment, &written, None, at_error); + sites.note(&fragment, edge); + if let Some(at_error) = at_error.as_mut() + && edge.target.is_none() + { + note_shared_error_jump(&fragment, &error_label(edge), &written, None, at_error); } out.push_str(&fragment); } + out.push_str(&sites.emit(module, function)); if function.convention.can_fail() { let at_error = at_error.map(|set| { @@ -5002,11 +5441,12 @@ fn written_on_entry(function: &Function) -> Option>> { /// plain counts as written, which releases more rather than less fn note_shared_error_jump( fragment: &str, + label: &str, written: &HashSet, dest: Option, at_error: &mut HashSet, ) { - let Some(jump) = fragment.rfind(&format!("goto {};", error_label(None))) else { + let Some(jump) = fragment.rfind(&format!("goto {label};")) else { return; }; at_error.extend(written.iter().copied()); @@ -5019,6 +5459,105 @@ fn note_shared_error_jump( } } +/// whether python adds a traceback entry for the frame where `op` fails +/// +/// every exception raised in a frame or passing through one gets an entry naming the frame +/// and the line, including one a `throw` raises at a suspension. what does not is putting +/// back an exception the frame already holds — a bare `raise`, or a handler that did not +/// match — which python re-raises with the traceback it already has. and a finish is not a +/// failure at all, only a way out. +/// +/// an exception leaving a generator expression drained in this frame is the one exception +/// to the first rule: it left the generator's frame, and python adds this frame's entry as +/// it passes out of the call that drained it +fn adds_a_traceback_entry(op: &Op) -> bool { + !matches!(op, Op::Reraise { .. } | Op::FinishFrame { .. }) +} + +/// the traceback entries one function's error edges pass through +/// +/// a compiled frame is not a python frame, so nothing adds an entry for it as an exception +/// leaves it. each failing operation instead jumps first to a stub that adds one — naming +/// the function and the line the operation was lowered from — and then on to where it was +/// going. the stubs sit after the body, one for each line and target that some operation +/// really jumps to, so the path that does not fail is the one it always was. each keeps its +/// code object once it has built one; the frame is new every time, as python's is +#[derive(Default)] +struct TracebackSites { + /// the line, the frame and the target of each stub, in the order the stubs were first + /// used. the frame is `true` for a generator expression drained in this one + used: Vec<(usize, bool, Option)>, + /// the stub the last edge named, until it is used + offered: Option<(usize, bool, Option)>, +} + +impl TracebackSites { + /// the edge an operation lowered from `offset` takes to `target` + /// + /// a module with no line table has nothing to name, and its edges go straight on + fn edge( + &mut self, + module: &ModuleIr, + offset: Option, + generator_expression: bool, + target: Option, + ) -> ErrorEdge { + let (Some(lines), Some(offset)) = (&module.lines, offset) else { + return ErrorEdge::straight(target); + }; + let key = (lines.line(offset), generator_expression, target); + let site = self + .used + .iter() + .position(|used| *used == key) + .unwrap_or(self.used.len()); + self.offered = Some(key); + ErrorEdge { + target, + site: Some(site), + } + } + + /// keep the stub `edge` names when `fragment` really jumps to it + fn note(&mut self, fragment: &str, edge: ErrorEdge) { + if let Some(site) = edge.site + && site == self.used.len() + && let Some(key) = self.offered + && fragment.contains(&format!("goto {};", error_label(edge))) + { + self.used.push(key); + } + } + + /// the stubs, written after the body + fn emit(&self, module: &ModuleIr, function: &Function) -> String { + let path = module + .lines + .as_ref() + .map(|lines| lines.path.as_str()) + .unwrap_or_default(); + let mut out = String::new(); + for (site, (line, generator_expression, target)) in self.used.iter().enumerate() { + let name = if *generator_expression { + "" + } else { + function.frame_name.as_str() + }; + let _ = writeln!( + out, + "by_tb{site}: {{\n\ + \x20 static ByTracebackSite by_site = {{NULL, {}, {}, {line}}};\n\ + \x20 By_TracebackHere(&by_site, by_module_dict);\n\ + \x20 goto {};\n}}", + c_string(path), + c_string(name), + error_label(ErrorEdge::straight(*target)) + ); + } + out + } +} + /// release the registers an exit in `block` owns /// /// `live` is the refcount pass's answer for this block, when it ran. without it @@ -5265,14 +5804,11 @@ fn object_expr(function: &Function, value: &Value) -> String { /// the fields a class publishes as attributes /// -/// a closure environment publishes none. python has no name for it, and a cell's unset -/// value is not one a getter could hand out +/// a class python has no name for publishes none: a closure environment's cells and a +/// generator's parked locals are the frame's own, and a cell's unset value is not one a +/// getter could hand out fn published_fields(class: &ClassIr) -> &[by_ir::function::FieldDecl] { - if class.environment { - &[] - } else { - &class.fields - } + if !class.exported { &[] } else { &class.fields } } /// the entries the emitted method table carries *before* the class's own methods @@ -5428,7 +5964,7 @@ fn assign_checked( function: &Function, dest: RegisterId, expr: &str, - error_target: Option, + error_target: ErrorEdge, ) -> String { let Some(decl) = function.register(dest) else { return String::new(); @@ -5456,7 +5992,7 @@ fn protocol_site(body: &str) -> String { /// /// the operations that build an argument vector emit their own block, so they reach /// this point rather than going through [`assign_checked`]. the rule is the same one -fn commit_checked(function: &Function, dest: RegisterId, error_target: Option) -> String { +fn commit_checked(function: &Function, dest: RegisterId, error_target: ErrorEdge) -> String { let Some(decl) = function.register(dest) else { return String::new(); }; @@ -5476,7 +6012,7 @@ fn commit_checked(function: &Function, dest: RegisterId, error_target: Option) -> String { +fn resolve_str(error_target: ErrorEdge) -> String { let label = error_label(error_target); format!( " {{ static PyObject *by_g_str = NULL; PyObject *by_fn = NULL;\n \ @@ -5499,7 +6035,7 @@ fn global_namespace_op( dest: RegisterId, name: &str, call: &dyn Fn(&str) -> String, - error_target: Option, + error_target: ErrorEdge, ) -> String { let slot = format!("by_g_{}", mangle(name)); let mut out = format!(" {{ static PyObject *{slot} = NULL;\n"); @@ -5529,11 +6065,7 @@ fn global_namespace_op( /// every read of a flagged register is guarded, not only the ones the analysis found /// reachable while unwritten: a read after a write finds the byte set, so guarding it /// costs a predicted branch and asks nothing of the emitter about control flow -fn guard_unassigned( - function: &Function, - values: &[&Value], - error_target: Option, -) -> String { +fn guard_unassigned(function: &Function, values: &[&Value], error_target: ErrorEdge) -> String { let registers: Vec = values .iter() .filter_map(|value| match value { @@ -5549,7 +6081,7 @@ fn guard_unassigned( fn guard_unassigned_registers( function: &Function, registers: &[RegisterId], - error_target: Option, + error_target: ErrorEdge, ) -> String { let mut out = String::new(); for id in registers { @@ -5590,12 +6122,7 @@ fn mark_assigned(function: &Function, op: &Op) -> String { ) } -fn emit_op( - module: &ModuleIr, - function: &Function, - op: &Op, - error_target: Option, -) -> String { +fn emit_op(module: &ModuleIr, function: &Function, op: &Op, error_target: ErrorEdge) -> String { match op { Op::Assign { dest, src } => { let Some(decl) = function.register(*dest) else { @@ -5725,24 +6252,29 @@ fn emit_op( Op::AsyncContext { dest, manager, + exit, exception, - } => { - let expr = match exception { - Some(exception) => format!( - "By_AsyncExit(&by_ps, {}, {})", + } => match exit.as_ref().zip(exception.as_ref()) { + Some((exit, exception)) => { + let expr = format!( + "By_AsyncExit({}, {}, {})", value_expr(manager), + value_expr(exit), value_expr(exception) - ), - None => format!("By_AsyncEnter(&by_ps, {})", value_expr(manager)), - }; - protocol_site(&assign_checked( - module, - function, - *dest, - &expr, - error_target, - )) - } + ); + assign_checked(module, function, *dest, &expr, error_target) + } + None => { + let expr = format!("By_AsyncEnter(&by_ps, {})", value_expr(manager)); + protocol_site(&assign_checked( + module, + function, + *dest, + &expr, + error_target, + )) + } + }, Op::AsyncIter { dest, src, next } => { let expr = format!("By_AsyncIter({}, {})", value_expr(src), i32::from(*next)); assign_checked(module, function, *dest, &expr, error_target) @@ -6696,16 +7228,36 @@ fn emit_op( error_target, )) } + Op::BindExit { + dest, + manager, + is_async, + } => { + let call = format!( + "By_BindExit(&by_ps, {}, {})", + value_expr(manager), + i32::from(*is_async) + ); + protocol_site(&assign_checked( + module, + function, + *dest, + &call, + error_target, + )) + } Op::ExitContext { dest, manager, + exit, exception, } => { // -1 means `__exit__` itself raised, which is the error path. 0 and 1 are // "re-raise" and "suppressed", and both are ordinary control flow let mut out = format!( - " {{ int by_r = By_ExitContext(&by_ps, {}, {});\n", + " {{ int by_r = By_ExitContext({}, {}, {});\n", value_expr(manager), + value_expr(exit), value_expr(exception) ); let _ = writeln!( @@ -6714,7 +7266,7 @@ fn emit_op( error_label(error_target) ); let _ = writeln!(out, " {} = (char)by_r; }}", local(*dest)); - protocol_site(&out) + out } Op::DelegateIter { dest, @@ -6768,7 +7320,7 @@ fn emit_op( function.name ); debug_assert!( - error_target.is_none(), + error_target.target.is_none(), "a finish in `{}` still stands under a handler, which would catch it", function.name ); @@ -6779,7 +7331,7 @@ fn emit_op( \x20 {receiver}->by_returned = by_t; }}\n\ \x20 goto {};\n", value_expr(value), - error_label(None) + error_label(ErrorEdge::straight(None)) ) } Op::GetCell { @@ -7589,6 +8141,11 @@ fn emit_op( value_expr(value), error_label(error_target) ), + Op::LeaveGenerator { value } => format!( + " By_Reraise({});\n By_ConvertStopIteration(BY_FRAME_GENERATOR);\n goto {};\n", + value_expr(value), + error_label(error_target) + ), Op::GetIter { dest, src, cursor } => { let mut out = match cursor { Some(_) => { @@ -7628,6 +8185,12 @@ fn emit_op( let expr = format!("(char)({} == NULL)", value_expr(src)); assign_owned(module, function, *dest, &expr) } + // read back by the function's emitter, which names the line in a traceback entry + Op::Line { .. } => String::new(), + Op::StopIterationValue { dest, src } => { + let expr = format!("By_StopIterationValue({})", value_expr(src)); + assign_owned(module, function, *dest, &expr) + } Op::Len { dest, src } => { let expr = format!("By_Len({})", value_expr(src)); assign_checked(module, function, *dest, &expr, error_target) @@ -7750,7 +8313,7 @@ fn emit_container( items: &[Value], builder: &str, count: usize, - error_target: Option, + error_target: ErrorEdge, ) -> String { let mut out = String::new(); let argv = if items.is_empty() { @@ -7775,10 +8338,36 @@ fn emit_container( /// the label a failing operation jumps to: a handler block, or the function's /// own error exit -fn error_label(error_target: Option) -> String { +/// where a failing operation goes: the block's error target, by way of the traceback +/// entry python adds for this frame where the failure is a new one +#[derive(Clone, Copy)] +struct ErrorEdge { + /// the handler, or the function's own exit where `None` + target: Option, + /// the traceback entry the edge passes through first — see [`TracebackSites`] + site: Option, +} + +impl ErrorEdge { + /// an edge that goes straight to `target`, adding no entry + fn straight(target: Option) -> Self { + Self { target, site: None } + } +} + +fn error_label(error_target: ErrorEdge) -> String { match error_target { - Some(block) => format!("b{}", block.0), - None => "by_error".to_string(), + ErrorEdge { + site: Some(site), .. + } => format!("by_tb{site}"), + ErrorEdge { + target: Some(block), + site: None, + } => format!("b{}", block.0), + ErrorEdge { + target: None, + site: None, + } => "by_error".to_string(), } } diff --git a/crates/by_ir/src/builder.rs b/crates/by_ir/src/builder.rs index 2e6e88c84f..a09a535fc4 100644 --- a/crates/by_ir/src/builder.rs +++ b/crates/by_ir/src/builder.rs @@ -6,7 +6,7 @@ //! otherwise account for most verifier failures. use crate::function::{BasicBlock, CallConvention, Decorator, Function, RegisterDecl}; -use crate::ops::{BlockId, Op, RegisterId, Terminator, Value}; +use crate::ops::{BlockId, Op, Position, RegisterId, Terminator, Value}; use crate::rtype::RType; /// builds one function @@ -17,6 +17,7 @@ use crate::rtype::RType; )] pub struct FunctionBuilder { name: String, + frame_name: String, param_count: usize, ret: RType, convention: CallConvention, @@ -45,13 +46,26 @@ pub struct FunctionBuilder { block_ranges: Vec>, doc: Option, takes_a_weak_reference: bool, + /// where the ops pushed from here on were written: the byte offset, and the line the + /// frontend worked out for it + location: Option<(u32, u32)>, + /// whether the ops pushed from here on run as a drained generator expression's body — + /// see [`Op::Line`] + generator_expression: bool, + /// per block, where its first op was written + block_positions: Vec>, + /// per block, the line of the last position recorded in it, and whether that was in + /// a generator expression + block_lines: Vec>, } impl FunctionBuilder { /// start a function. the entry block is created and made current pub fn new(name: impl Into, ret: RType) -> Self { + let name = name.into(); Self { - name: name.into(), + frame_name: name.clone(), + name, param_count: 0, ret, convention: CallConvention::Native, @@ -75,9 +89,45 @@ impl FunctionBuilder { block_ranges: vec![None], doc: None, takes_a_weak_reference: false, + location: None, + generator_expression: false, + block_positions: vec![None], + block_lines: vec![None], } } + /// the name python gives the frame the body runs in — see [`Function::frame_name`] + pub fn frame_name(&mut self, name: impl Into) { + self.frame_name = name.into(); + } + + /// say that the ops pushed from here on were written at `offset`, on `line`, and + /// hand back the location this replaces so the caller can put it back + /// + /// the first op pushed into a block records it as the block's position, and an + /// [`Op::Line`] is written ahead of a later op wherever that op's line differs from + /// the last one its block recorded, so a run of ops from one line carries nothing + /// however many expressions it was lowered from + pub fn locate(&mut self, offset: u32, line: u32) -> Option<(u32, u32)> { + self.location.replace((offset, line)) + } + + /// where the ops pushed from here on were written, as [`Self::locate`] last said + pub fn location(&self) -> Option<(u32, u32)> { + self.location + } + + /// say whether the ops pushed from here on run as the body of a generator expression + /// drained in this frame, handing back what this replaces + pub fn in_generator_expression(&mut self, inside: bool) -> bool { + std::mem::replace(&mut self.generator_expression, inside) + } + + /// put back a location [`Self::locate`] handed out + pub fn relocate(&mut self, location: Option<(u32, u32)>) { + self.location = location; + } + /// record that the body takes a weak reference of a value it did not make itself — /// see [`Function::takes_a_weak_reference`] pub fn takes_a_weak_reference(&mut self) -> &mut Self { @@ -153,6 +203,8 @@ impl FunctionBuilder { self.blocks.push(None); self.pending.push(Vec::new()); self.block_ranges.push(None); + self.block_positions.push(None); + self.block_lines.push(None); id } @@ -232,6 +284,34 @@ impl FunctionBuilder { if self.is_sealed(self.current) { return; } + // a copy of a temporary or of an immediate cannot fail — only a named local can be + // read unbound — and it is where a value computed on one line is stored by a + // statement written on another. it takes no position of its own, so the value and + // the store stay next to each other for the passes that fold the two into one + let cannot_fail = matches!( + &op, + Op::Assign { src, .. } + if !matches!(src, Value::Register(id) + if self.registers.get(id.index()).is_none_or(|decl| decl.name.is_some())) + ); + if !cannot_fail + && let Some((offset, line)) = self.location + && let Some(last) = self.block_lines.get_mut(self.current.index()) + && *last != Some((line, self.generator_expression)) + { + let position = Position { + offset, + generator_expression: self.generator_expression, + }; + if last.is_none() { + if let Some(slot) = self.block_positions.get_mut(self.current.index()) { + *slot = Some(position); + } + } else if let Some(ops) = self.pending.get_mut(self.current.index()) { + ops.push(Op::Line { position }); + } + *last = Some((line, self.generator_expression)); + } if let Some(ops) = self.pending.get_mut(self.current.index()) { ops.push(op); } @@ -285,6 +365,7 @@ impl FunctionBuilder { if let Some(slot) = self.blocks.get_mut(self.current.index()) { *slot = Some(BasicBlock { range: None, + position: None, ops, terminator, owned_at_exit: None, @@ -313,15 +394,18 @@ impl FunctionBuilder { .into_iter() .zip(self.pending) .zip(self.block_ranges) - .map(|((block, ops), range)| { + .zip(self.block_positions) + .map(|(((block, ops), range), position)| { let mut block = block.unwrap_or(BasicBlock { ops, terminator: Terminator::Unreachable, owned_at_exit: None, error_target: None, range: None, + position: None, }); block.range = range; + block.position = position; block }) .collect(); @@ -329,6 +413,7 @@ impl FunctionBuilder { posonly: self.posonly, kwonly: self.kwonly, name: self.name, + frame_name: self.frame_name, param_count: self.param_count, ret: self.ret, convention: self.convention, diff --git a/crates/by_ir/src/function.rs b/crates/by_ir/src/function.rs index d92dfd2fb4..f83b8592eb 100644 --- a/crates/by_ir/src/function.rs +++ b/crates/by_ir/src/function.rs @@ -92,6 +92,9 @@ pub struct BasicBlock { /// their meaning, and a copied block reports the source it was copied from, /// which is the source it still is pub range: Option<(u32, u32)>, + /// where the block's first op was written, which a failing op names in its traceback + /// entry until an [`Op::Line`] in the block says otherwise + pub position: Option, /// where a *failing* operation in this block jumps. /// /// `None` is the function's own error exit. inside a `try` it is the handler, @@ -122,6 +125,7 @@ impl BasicBlock { owned_at_exit: None, error_target: None, range: None, + position: None, } } } @@ -249,6 +253,12 @@ impl Binding { pub struct Function { /// the name as written in source, used for the python-visible surface pub name: String, + /// the name python gives the frame this body runs in, which a traceback entry names + /// + /// the definition's own name, which [`Self::name`] stops being once a lowering + /// renames the body — an accessor's `$get`, a generator's `$resume`, a private + /// method's mangling — and `` or `` for a body python gives no name + pub frame_name: String, /// the number of leading registers that are parameters pub param_count: usize, pub ret: RType, @@ -645,6 +655,22 @@ pub struct Resumption { pub method: String, /// which surface the state object presents to python pub surface: Surface, + /// the `__qualname__` of the function whose frame this is, which python names the + /// frame by when it reports one — a coroutine dropped without being awaited, say + pub qualname: String, + /// the suspensions made inside a `yield from` or an `await`, each with the field the + /// iterator it delegates to is kept in. python's `throw` and `close` reach that + /// iterator before the frame itself + pub delegations: Vec, +} + +/// one suspension of a resumable frame that waits on an inner iterator +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Delegation { + /// the value `$state` holds while the frame is suspended there + pub state: i64, + /// the field of the state object holding the inner iterator + pub field: String, } /// what a resumable frame looks like from python @@ -892,11 +918,14 @@ impl ClassIr { /// descriptor under the same name, filled before the assignment and dropped by it /// /// a nested function is left out too: its environment is its receiver, but python - /// never sees it as a method of one — see [`Function::nested`] + /// never sees it as a method of one — see [`Function::nested`]. and so is the method + /// a generator or coroutine resumes through, which only its own slots may call: from + /// python it would run the frame with none of the bookkeeping a step does pub fn table_methods(&self) -> impl Iterator { self.methods.iter().filter(|method| { method.name != "__new__" && method.nested.is_none() + && !self.resumes_through(method) && !self .properties .iter() @@ -904,6 +933,13 @@ impl ClassIr { }) } + /// whether `method` is the one this class's resumable frame steps through + pub fn resumes_through(&self, method: &Function) -> bool { + self.resume + .as_ref() + .is_some_and(|resume| resume.method == method.name) + } + /// the C identifier for the instance struct pub fn struct_name(&self, module: &str) -> String { format!("By_{}_{}", mangle(module), mangle(&self.name)) @@ -1390,6 +1426,7 @@ mod tests { kwarg: false, range: None, name: "add".to_string(), + frame_name: "add".to_string(), param_count: 2, ret: RType::INT, convention: CallConvention::NativeInfallible, diff --git a/crates/by_ir/src/ops.rs b/crates/by_ir/src/ops.rs index 5b3ce49723..c5165497c5 100644 --- a/crates/by_ir/src/ops.rs +++ b/crates/by_ir/src/ops.rs @@ -553,6 +553,9 @@ pub enum Op { AsyncContext { dest: RegisterId, manager: Value, + /// the `__aexit__` [`Self::BindExit`] looked up where the block was entered, + /// present exactly when `exception` is + exit: Option, exception: Option, }, /// `__aiter__` on `src`, or `__anext__` when `next` @@ -727,6 +730,15 @@ pub enum Op { }, /// `with EXPR`: the manager's `__enter__` Enter { dest: RegisterId, manager: Value }, + /// the `__exit__` — or with `is_async`, the `__aexit__` — a `with` calls on its way + /// out, looked up as the block is entered and before `__enter__` runs, which is where + /// python binds it: a class rebinding the method while the block runs does not + /// change which one the block calls + BindExit { + dest: RegisterId, + manager: Value, + is_async: bool, + }, /// `__exit__`, on the normal path or with a live exception. /// /// `dest` is a bit: set when the exception was *suppressed*, so the caller @@ -734,6 +746,8 @@ pub enum Op { ExitContext { dest: RegisterId, manager: Value, + /// what [`Self::BindExit`] looked up where the block was entered + exit: Value, /// the live exception, or `None` on the normal path exception: Value, }, @@ -1127,6 +1141,13 @@ pub enum Op { }, /// put a fetched exception back, for an unmatched handler or a bare re-raise Reraise { value: Value }, + /// put a fetched exception back as it leaves a generator's frame, which is a + /// re-raise that pep 479 has something to say about: a `StopIteration` becomes the + /// `RuntimeError` python makes of one, with the original as its cause. + /// + /// only a generator expression built at once needs it, where the loop the + /// generator's frame would have run stands in the frame around it instead + LeaveGenerator { value: Value }, /// `iter(o)` /// /// `cursor` names a machine-integer register the loop keeps its position in, and @@ -1158,6 +1179,20 @@ pub enum Op { /// whether an object register is null, as a bit. this is how an exhausted /// iterator is tested, so it must not itself be treated as an error IsNull { dest: RegisterId, src: Value }, + /// the value an exception carries when it is a `StopIteration`, and null when `src` is + /// anything else. it asks and raises nothing: a delegation reads an exception thrown + /// in at its suspension through this, to tell the inner iterator finishing from an + /// error + StopIterationValue { dest: RegisterId, src: Value }, + /// the ops that follow were written at `position`, until the next of these + /// + /// python names the line of the instruction that raised in a traceback, and one + /// statement can span several lines. so rather than a span per op, which every pass + /// would have to carry, a block records where its first op was written, and the + /// frontend writes one of these wherever the line changes after that. it does nothing, + /// reads nothing and cannot fail — codegen reads it back to say which line a failing + /// op's traceback entry names + Line { position: Position }, /// the length of a sized object, as a tagged int Len { dest: RegisterId, src: Value }, /// concatenate two strings @@ -1308,6 +1343,7 @@ impl Op { | Self::ImportModule { .. } | Self::ImportFrom { .. } | Self::Enter { .. } + | Self::BindExit { .. } | Self::ExitContext { .. } | Self::DelegateIter { .. } | Self::DelegateStep { .. } @@ -1343,9 +1379,12 @@ impl Op { | Self::Move { .. } | Self::RaiseObject { .. } | Self::Reraise { .. } + | Self::LeaveGenerator { .. } | Self::GetIter { .. } | Self::IterNext { .. } | Self::IsNull { .. } + | Self::StopIterationValue { .. } + | Self::Line { .. } | Self::Len { .. } | Self::StrConcat { .. } | Self::StrOfInt { .. } @@ -1447,6 +1486,7 @@ impl Op { | Self::NewInstance { dest, .. } | Self::GetCell { dest, .. } | Self::Enter { dest, .. } + | Self::BindExit { dest, .. } | Self::ExitContext { dest, .. } | Self::DelegateIter { dest, .. } | Self::DelegateStep { dest, .. } @@ -1471,6 +1511,7 @@ impl Op { | Self::GetIter { dest, .. } | Self::IterNext { dest, .. } | Self::IsNull { dest, .. } + | Self::StopIterationValue { dest, .. } | Self::StrConcat { dest, .. } | Self::TupleGet { dest, .. } | Self::Unpack { dest, .. } @@ -1495,8 +1536,10 @@ impl Op { | Self::PopHandled { .. } | Self::Release { .. } | Self::Reraise { .. } + | Self::LeaveGenerator { .. } | Self::LicenceHolds { .. } | Self::RequireField { .. } + | Self::Line { .. } | Self::SetField { .. } => None, } } @@ -1562,6 +1605,7 @@ impl Op { | Self::NewInstance { dest, .. } | Self::GetCell { dest, .. } | Self::Enter { dest, .. } + | Self::BindExit { dest, .. } | Self::ExitContext { dest, .. } | Self::DelegateIter { dest, .. } | Self::DelegateStep { dest, .. } @@ -1586,6 +1630,7 @@ impl Op { | Self::GetIter { dest, .. } | Self::IterNext { dest, .. } | Self::IsNull { dest, .. } + | Self::StopIterationValue { dest, .. } | Self::StrConcat { dest, .. } | Self::TupleGet { dest, .. } | Self::Unpack { dest, .. } @@ -1610,8 +1655,10 @@ impl Op { | Self::PopHandled { .. } | Self::Release { .. } | Self::Reraise { .. } + | Self::LeaveGenerator { .. } | Self::LicenceHolds { .. } | Self::RequireField { .. } + | Self::Line { .. } | Self::SetField { .. } => None, } } @@ -1621,9 +1668,10 @@ impl Op { match self { Self::AsyncContext { manager, + exit: Some(exit), exception: Some(exception), .. - } => vec![manager, exception], + } => vec![manager, exit, exception], Self::Assign { src, .. } | Self::Move { src, .. } | Self::Box { src, .. } @@ -1640,6 +1688,12 @@ impl Op { exception: None, .. } + | Self::AsyncContext { + manager: src, + exit: None, + .. + } + | Self::BindExit { manager: src, .. } | Self::MatchAttr { subject: src, class: None, @@ -1654,6 +1708,7 @@ impl Op { | Self::StrOfInt { value: src, .. } | Self::GetIter { src, .. } | Self::IsNull { src, .. } + | Self::StopIterationValue { src, .. } | Self::TupleGet { src, .. } | Self::Unpack { src, .. } | Self::ToTuple { src, .. } => vec![src], @@ -1761,7 +1816,8 @@ impl Op { | Self::DeleteGlobal { .. } | Self::DeleteLocal { .. } | Self::LoadClass { .. } - | Self::ImportModule { .. } => Vec::new(), + | Self::ImportModule { .. } + | Self::Line { .. } => Vec::new(), Self::Warn { message, category, .. } => match category { @@ -1770,8 +1826,11 @@ impl Op { }, Self::Enter { manager, .. } => vec![manager], Self::ExitContext { - manager, exception, .. - } => vec![manager, exception], + manager, + exit, + exception, + .. + } => vec![manager, exit, exception], Self::DelegateIter { src, .. } => vec![src], Self::DelegateStep { inner, sent, .. } => vec![inner, sent], Self::ArrayNew { items, .. } => items.iter().collect(), @@ -1820,6 +1879,7 @@ impl Op { None => vec![exception], }, Self::Reraise { value } + | Self::LeaveGenerator { value } | Self::PushHandled { value, .. } | Self::PopHandled { value } | Self::Release { value, .. } @@ -1837,9 +1897,10 @@ impl Op { match self { Self::AsyncContext { manager, + exit: Some(exit), exception: Some(exception), .. - } => vec![manager, exception], + } => vec![manager, exit, exception], Self::Assign { src, .. } | Self::Move { src, .. } | Self::Box { src, .. } @@ -1856,6 +1917,12 @@ impl Op { exception: None, .. } + | Self::AsyncContext { + manager: src, + exit: None, + .. + } + | Self::BindExit { manager: src, .. } | Self::MatchAttr { subject: src, class: None, @@ -1870,6 +1937,7 @@ impl Op { | Self::StrOfInt { value: src, .. } | Self::GetIter { src, .. } | Self::IsNull { src, .. } + | Self::StopIterationValue { src, .. } | Self::TupleGet { src, .. } | Self::Unpack { src, .. } | Self::ToTuple { src, .. } => vec![src], @@ -1979,7 +2047,8 @@ impl Op { | Self::DeleteGlobal { .. } | Self::DeleteLocal { .. } | Self::LoadClass { .. } - | Self::ImportModule { .. } => Vec::new(), + | Self::ImportModule { .. } + | Self::Line { .. } => Vec::new(), Self::Warn { message, category, .. } => match category { @@ -1988,8 +2057,11 @@ impl Op { }, Self::Enter { manager, .. } => vec![manager], Self::ExitContext { - manager, exception, .. - } => vec![manager, exception], + manager, + exit, + exception, + .. + } => vec![manager, exit, exception], Self::DelegateIter { src, .. } => vec![src], Self::DelegateStep { inner, sent, .. } => vec![inner, sent], Self::ArrayNew { items, .. } => items.iter_mut().collect(), @@ -2038,6 +2110,7 @@ impl Op { None => vec![exception], }, Self::Reraise { value } + | Self::LeaveGenerator { value } | Self::PushHandled { value, .. } | Self::PopHandled { value } | Self::Release { value, .. } @@ -2047,6 +2120,17 @@ impl Op { } } +/// where lowered code was written — see [`Op::Line`] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Position { + /// the byte offset in the source + pub offset: u32, + /// whether the code runs as the body of a generator expression drained where it + /// stands, in the frame around it. python runs that body in a frame of its own, named + /// ``, and its traceback entry comes before the frame's + pub generator_expression: bool, +} + /// how a block ends. a block has exactly one, so control flow is total #[derive(Debug, Clone, PartialEq)] pub enum Terminator { diff --git a/crates/by_ir/src/print.rs b/crates/by_ir/src/print.rs index 70081233e1..785dfa3681 100644 --- a/crates/by_ir/src/print.rs +++ b/crates/by_ir/src/print.rs @@ -120,6 +120,7 @@ fn print_op(function: &Function, op: &Op) -> String { dest, manager, exception, + .. } => format!( "{} = {} {}", name(*dest), @@ -495,14 +496,26 @@ fn print_op(function: &Function, op: &Op) -> String { Op::Enter { dest, manager } => { format!("{} = enter {}", name(*dest), value(manager)) } + Op::BindExit { + dest, + manager, + is_async, + } => format!( + "{} = bind {} {}", + name(*dest), + if *is_async { "aexit" } else { "exit" }, + value(manager) + ), Op::ExitContext { dest, manager, + exit, exception, } => format!( - "{} = exit {} with {}", + "{} = exit {} through {} with {}", name(*dest), value(manager), + value(exit), value(exception) ), Op::DelegateIter { @@ -756,6 +769,7 @@ fn print_op(function: &Function, op: &Op) -> String { None => format!("raise {}", value(exception)), }, Op::Reraise { value: v } => format!("reraise {}", value(v)), + Op::LeaveGenerator { value: v } => format!("leave generator {}", value(v)), Op::GetIter { dest, src, cursor } => match cursor { Some(cursor) => format!("{} = iter {} @{}", name(*dest), value(src), name(*cursor)), None => format!("{} = iter {}", name(*dest), value(src)), @@ -765,6 +779,16 @@ fn print_op(function: &Function, op: &Op) -> String { None => format!("{} = next {}", name(*dest), value(iter)), }, Op::IsNull { dest, src } => format!("{} = {} is null", name(*dest), value(src)), + Op::Line { position } => { + if position.generator_expression { + format!("line @{} in a generator expression", position.offset) + } else { + format!("line @{}", position.offset) + } + } + Op::StopIterationValue { dest, src } => { + format!("{} = stop_iteration_value {}", name(*dest), value(src)) + } Op::StrConcat { dest, lhs, @@ -857,6 +881,7 @@ mod tests { kwarg: false, range: None, name: "abs".to_string(), + frame_name: "abs".to_string(), param_count: 1, ret: RType::INT, convention: CallConvention::NativeInfallible, @@ -956,6 +981,7 @@ b2: kwarg: false, range: None, name: "f".to_string(), + frame_name: "f".to_string(), param_count: 0, ret: RType::FLOAT, convention: CallConvention::NativeInfallible, diff --git a/crates/by_ir/src/verify.rs b/crates/by_ir/src/verify.rs index 1fc4983b86..7f1862d9b4 100644 --- a/crates/by_ir/src/verify.rs +++ b/crates/by_ir/src/verify.rs @@ -704,12 +704,22 @@ impl Verifier<'_> { Op::AsyncContext { dest, manager, + exit, exception, } => { self.expect(block, manager, &RType::OBJECT, "an async context manager"); if let Some(exception) = exception { self.expect(block, exception, &RType::OBJECT, "an async context manager"); } + if let Some(exit) = exit { + self.expect(block, exit, &RType::OBJECT, "an async context manager"); + } + if exit.is_some() != exception.is_some() { + self.error( + Some(block), + "an `__aexit__` call names an exit and an exception together or neither", + ); + } self.expect_dest(block, *dest, &RType::OBJECT, "an async context manager"); } Op::AsyncIter { dest, src, .. } => { @@ -1641,7 +1651,7 @@ impl Verifier<'_> { self.expect(block, cause, &RType::OBJECT, "a raise cause"); } } - Op::Reraise { value } => { + Op::Reraise { value } | Op::LeaveGenerator { value } => { self.expect(block, value, &RType::OBJECT, "a re-raise"); } Op::GetIter { dest, src, cursor } => { @@ -1662,6 +1672,12 @@ impl Verifier<'_> { self.expect(block, src, &RType::OBJECT, "a null test"); self.expect_dest(block, *dest, &RType::BIT, "a null test"); } + // a position reads and writes nothing, so there is nothing to check + Op::Line { .. } => {} + Op::StopIterationValue { dest, src } => { + self.expect(block, src, &RType::OBJECT, "a `StopIteration`'s value"); + self.expect_dest(block, *dest, &RType::OBJECT, "a `StopIteration`'s value"); + } Op::Len { dest, src } => { // a length is defined on anything with `__len__`, so the operand // is widened to `object` by the frontend before it gets here @@ -1740,12 +1756,18 @@ impl Verifier<'_> { self.expect(block, manager, &RType::OBJECT, "a context manager"); self.expect_dest(block, *dest, &RType::OBJECT, "`__enter__`"); } + Op::BindExit { dest, manager, .. } => { + self.expect(block, manager, &RType::OBJECT, "a context manager"); + self.expect_dest(block, *dest, &RType::OBJECT, "a context exit"); + } Op::ExitContext { dest, manager, + exit, exception, } => { self.expect(block, manager, &RType::OBJECT, "a context manager"); + self.expect(block, exit, &RType::OBJECT, "a context exit"); self.expect(block, exception, &RType::OBJECT, "a context exit"); self.expect_dest(block, *dest, &RType::BIT, "`__exit__`"); } @@ -2001,6 +2023,7 @@ mod tests { kwarg: false, range: None, name: "add".to_string(), + frame_name: "add".to_string(), param_count: 2, ret: RType::INT, convention: CallConvention::NativeInfallible, @@ -2192,6 +2215,7 @@ mod tests { kwarg: false, range: None, name: "cond".to_string(), + frame_name: "cond".to_string(), param_count: 2, ret: RType::INT, convention: CallConvention::NativeInfallible, @@ -2244,6 +2268,7 @@ mod tests { kwarg: false, range: None, name: "cond".to_string(), + frame_name: "cond".to_string(), param_count: 2, ret: RType::INT, convention: CallConvention::NativeInfallible, @@ -2292,6 +2317,7 @@ mod tests { kwarg: false, range: None, name: "loop".to_string(), + frame_name: "loop".to_string(), param_count: 2, ret: RType::INT, convention: CallConvention::NativeInfallible, @@ -2396,6 +2422,7 @@ mod tests { kwarg: false, range: None, name: "pick".to_string(), + frame_name: "pick".to_string(), param_count: 2, ret: RType::INT, convention: CallConvention::NativeInfallible, diff --git a/crates/by_irbuild/Cargo.toml b/crates/by_irbuild/Cargo.toml index 7ecf849cde..4254e1c62a 100644 --- a/crates/by_irbuild/Cargo.toml +++ b/crates/by_irbuild/Cargo.toml @@ -16,6 +16,7 @@ ruff_db = { workspace = true } ruff_python_ast = { workspace = true } ruff_python_parser = { workspace = true } ruff_python_stdlib = { workspace = true } +ruff_source_file = { workspace = true } thin-vec = { workspace = true } ruff_text_size = { workspace = true } ty_project = { workspace = true, features = ["testing"] } diff --git a/crates/by_irbuild/src/closures.rs b/crates/by_irbuild/src/closures.rs index 66bb6f94f7..6122500992 100644 --- a/crates/by_irbuild/src/closures.rs +++ b/crates/by_irbuild/src/closures.rs @@ -35,6 +35,7 @@ use std::collections::{HashMap, HashSet}; use ruff_python_ast::visitor::{self, Visitor}; use ruff_python_ast::{self as ast, Expr, Stmt}; +use ruff_text_size::Ranged; use crate::mapper::{Decline, Lowered}; @@ -54,6 +55,8 @@ pub(crate) struct Nested { pub(crate) def: ast::StmtFunctionDef, /// the lambda this was synthesized from, by source range pub(crate) lambda: Option, + /// the generator expression this was synthesized from, by source range + pub(crate) generator_expression: Option, /// the enclosing names the body reads, in a stable order pub(crate) captures: Vec, /// the captures that either frame *writes*, so both must see one cell @@ -74,11 +77,10 @@ pub(crate) fn nested_functions( let mut out = Vec::new(); // anywhere in the body, not only at the top level: a `def` inside a loop is the // case the shared-cell rule exists for - let mut definitions: Vec<(ast::StmtFunctionDef, Option)> = - Vec::new(); + let mut definitions: Vec<(ast::StmtFunctionDef, Synthesized)> = Vec::new(); for stmt in crate::walk(body) { if let Stmt::FunctionDef(def) = stmt { - definitions.push((def.clone(), None)); + definitions.push((def.clone(), Synthesized::Written)); } // a lambda in any expression position is a nested function with a generated // name, so the whole closure machinery applies to it unchanged @@ -86,10 +88,19 @@ pub(crate) fn nested_functions( visit_expressions(expr, &mut |child| { if let Expr::Lambda(lambda) = child { let name = format!("$lambda{}", definitions.len()); - definitions.push((synthesize(lambda, &name), Some(lambda.range))); + definitions + .push((synthesize(lambda, &name), Synthesized::Lambda(lambda.range))); } }); } + // and so is a generator expression, which python compiles to exactly that + for generator in generator_expressions(stmt) { + let name = format!("{GENERATOR_EXPRESSION}{}", definitions.len()); + definitions.push(( + synthesize_generator(generator, &name), + Synthesized::GeneratorExpression(generator.range), + )); + } } // two `def`s of one name in a scope — the `try` / `except` pair is the common @@ -106,7 +117,7 @@ pub(crate) fn nested_functions( } } - for (def, lambda) in definitions { + for (def, synthesized) in definitions { let def = &def; let own = own_names(def); let mut captures: Vec = Vec::new(); @@ -129,8 +140,14 @@ pub(crate) fn nested_functions( .filter(|name| !per_iteration.contains(name.as_str())) .cloned() .collect(); - // and a nested `nonlocal` write makes it shared even if it is never read - for name in nonlocal_names(def) { + // and a nested `nonlocal` write makes it shared even if it is never read. so does + // a walrus inside a generator expression, which python binds in the frame the + // expression stands in rather than in the generator's own + let walrus = match synthesized { + Synthesized::GeneratorExpression(_) => walrus_targets(&def.body), + _ => Vec::new(), + }; + for name in nonlocal_names(def).into_iter().chain(walrus) { if bound.contains(name) { if !captures.iter().any(|capture| capture == name) { captures.push(name.to_string()); @@ -142,7 +159,14 @@ pub(crate) fn nested_functions( } out.push(Nested { def: def.clone(), - lambda, + lambda: match synthesized { + Synthesized::Lambda(range) => Some(range), + _ => None, + }, + generator_expression: match synthesized { + Synthesized::GeneratorExpression(range) => Some(range), + _ => None, + }, captures, shared, }); @@ -260,6 +284,175 @@ pub(crate) fn bound_only_by_their_def(body: &[Stmt]) -> HashSet { .collect() } +/// what a nested function was written as +#[derive(Clone, Copy)] +enum Synthesized { + Written, + Lambda(ruff_text_size::TextRange), + GeneratorExpression(ruff_text_size::TextRange), +} + +/// the prefix of the name a generator expression's function is lowered under +pub(crate) const GENERATOR_EXPRESSION: &str = "$genexpr"; + +/// the parameter a generator expression's function takes its first iterator in +/// +/// python evaluates the first clause's iterable, and calls `iter` on it, where the +/// expression stands — so `(x for x in 5)` raises there, and a later rebinding of the +/// name the iterable was read from changes nothing. what the frame receives is the +/// iterator, as python's own `.0` parameter. it is not a python identifier, so nothing +/// the source wrote can name it +pub(crate) const GENERATOR_ITERATOR: &str = "$iter"; + +/// the generator expressions a statement makes where it stands +/// +/// every expression the statement itself holds, an `elif`'s test and a `match`'s guard +/// included, and none of the statements nested in its body — each of those is a +/// statement of its own. not one inside another generator expression, or inside a +/// lambda's body: each of those belongs to the frame it is written in. the first +/// clause's iterable of one is read here, so a generator expression in there is this +/// frame's too +/// +/// nor an asynchronous one, which python makes an async generator of. that is left for +/// the expression itself to decline +fn generator_expressions(stmt: &Stmt) -> Vec<&ast::ExprGenerator> { + struct Find<'a> { + found: Vec<&'a ast::ExprGenerator>, + } + impl<'a> Visitor<'a> for Find<'a> { + fn visit_body(&mut self, _body: &'a [Stmt]) {} + + fn visit_expr(&mut self, expr: &'a Expr) { + match expr { + Expr::Generator(node) => { + if lowers_as_a_generator(node) { + self.found.push(node); + } + if let Some(first) = node.generators.first() { + self.visit_expr(&first.iter); + } + } + Expr::Lambda(node) => { + for default in node + .parameters + .iter() + .flat_map(|parameters| parameters.iter()) + .filter_map(ast::AnyParameterRef::default) + { + self.visit_expr(default); + } + } + _ => visitor::walk_expr(self, expr), + } + } + } + let mut find = Find { found: Vec::new() }; + visitor::walk_stmt(&mut find, stmt); + find.found +} + +/// the names a walrus anywhere in `body` binds +fn walrus_targets(body: &[Stmt]) -> Vec<&str> { + let mut out = Vec::new(); + for stmt in crate::walk(body) { + for expr in statement_expressions(stmt) { + visit_expressions(expr, &mut |child| { + if let Expr::Named(node) = child + && let Expr::Name(name) = node.target.as_ref() + { + out.push(name.id.as_str()); + } + }); + } + } + out +} + +/// whether a generator expression is one python makes an ordinary generator of +pub(crate) fn lowers_as_a_generator(node: &ast::ExprGenerator) -> bool { + let mut plain = node.generators.iter().all(|clause| !clause.is_async); + let mut look = |child: &Expr| { + if matches!(child, Expr::Await(_) | Expr::Yield(_) | Expr::YieldFrom(_)) { + plain = false; + } + }; + visit_expressions(&node.elt, &mut look); + for (index, clause) in node.generators.iter().enumerate() { + if index > 0 { + visit_expressions(&clause.iter, &mut look); + } + for condition in &clause.ifs { + visit_expressions(condition, &mut look); + } + } + plain +} + +/// the generator function python compiles a generator expression into +/// +/// one `for` per clause, each clause's `if`s nested inside its `for`, and a `yield` of +/// the element innermost. the first `for` iterates [`GENERATOR_ITERATOR`], which the +/// function takes rather than declares: see there. every node the source wrote is +/// cloned, which keeps its identity, so the semantic model still answers for each +fn synthesize_generator(node: &ast::ExprGenerator, name: &str) -> ast::StmtFunctionDef { + let range = node.range; + let mut body = Stmt::Expr(ast::StmtExpr { + node_index: ruff_python_ast::AtomicNodeIndex::NONE, + range, + value: Box::new(Expr::Yield(ast::ExprYield { + node_index: ruff_python_ast::AtomicNodeIndex::NONE, + range, + value: Some(node.elt.clone()), + })), + }); + for (index, clause) in node.generators.iter().enumerate().rev() { + for condition in clause.ifs.iter().rev() { + body = Stmt::If(ast::StmtIf { + node_index: ruff_python_ast::AtomicNodeIndex::NONE, + range, + pattern: None, + test: Box::new(condition.clone()), + body: thin_vec::thin_vec![body], + elif_else_clauses: Vec::new(), + }); + } + let iter = if index == 0 { + Expr::Name(ast::ExprName { + node_index: ruff_python_ast::AtomicNodeIndex::NONE, + range: clause.iter.range(), + id: ruff_python_ast::name::Name::new_static(GENERATOR_ITERATOR), + ctx: ast::ExprContext::Load, + }) + } else { + clause.iter.clone() + }; + body = Stmt::For(ast::StmtFor { + node_index: ruff_python_ast::AtomicNodeIndex::NONE, + range, + is_async: false, + target: Box::new(clause.target.clone()), + pattern: None, + iter: Box::new(iter), + body: thin_vec::thin_vec![body], + orelse: thin_vec::ThinVec::new(), + }); + } + ast::StmtFunctionDef { + node_index: ruff_python_ast::AtomicNodeIndex::NONE, + range, + is_async: false, + decorator_list: thin_vec::ThinVec::new(), + name: ast::Identifier::new(name, range), + type_params: None, + parameters: Box::new(ast::Parameters::default()), + returns: None, + raises: None, + is_asserts_return: false, + body: thin_vec::thin_vec![body], + is_trailing_lambda: false, + } +} + /// a `StmtFunctionDef` equivalent to a lambda: its parameters, and its body as a /// single `return` fn synthesize(lambda: &ast::ExprLambda, name: &str) -> ast::StmtFunctionDef { @@ -299,6 +492,8 @@ fn own_names(def: &ast::StmtFunctionDef) -> HashSet<&str> { if let Some(kwarg) = &def.parameters.kwarg { out.insert(kwarg.name.as_str()); } + // a generator expression's first iterator is a parameter no source wrote + out.insert(GENERATOR_ITERATOR); out.extend(written_names(&def.body)); // a name declared `global` here resolves in the module namespace whether this body // writes it or only reads it, so an enclosing local of the same name must never be @@ -381,11 +576,15 @@ pub(crate) fn written_names(body: &[Stmt]) -> Vec<&str> { out.push(name.id.as_str()); } } - Stmt::For(node) => { - if let Expr::Name(name) = node.target.as_ref() { + // every name the target binds, an unpacking's included: `for a, b in pairs` + // binds `a` and `b` as surely as `for a in xs` binds `a` + Stmt::For(node) => visit_expressions(&node.target, &mut |child| { + if let Expr::Name(name) = child + && name.ctx == ast::ExprContext::Store + { out.push(name.id.as_str()); } - } + }), Stmt::FunctionDef(node) => out.push(node.name.as_str()), // a `case` pattern binds in the frame the `match` stands in, and it binds // without being an assignment — so none of the arms above reaches it @@ -461,7 +660,20 @@ fn read_names(body: &[Stmt]) -> Vec<&str> { out } -/// the expressions a statement evaluates, excluding assignment *targets* +/// the expressions a statement evaluates in the frame it stands in, excluding assignment +/// *targets* +/// +/// every position python evaluates, because every consumer asks a question that has to be +/// answered for all of them: which names a nested function captures, which lambdas the +/// frame makes, which names a walrus binds, whether a buffer escapes, how many delegations +/// a generator suspends in. a test left out here was once the `elif` test, and a nested +/// function reading a name only there resolved it as a global and raised `NameError`. +/// +/// the bodies of compound statements are not expressions of this one — [`crate::walk`] +/// reaches their statements — and neither is what a nested `def` or `class` runs in a frame +/// of its own. what such a statement evaluates where it stands is: its decorators, a +/// `def`'s defaults and a `class`'s bases. an annotation is not, because nothing the +/// frame runs evaluates one pub(crate) fn statement_expressions(stmt: &Stmt) -> Vec<&Expr> { match stmt { Stmt::Return(node) => node.value.iter().map(AsRef::as_ref).collect(), @@ -469,9 +681,42 @@ pub(crate) fn statement_expressions(stmt: &Stmt) -> Vec<&Expr> { Stmt::Assign(node) => vec![node.value.as_ref()], Stmt::AnnAssign(node) => node.value.iter().map(AsRef::as_ref).collect(), Stmt::AugAssign(node) => vec![node.target.as_ref(), node.value.as_ref()], - Stmt::If(node) => vec![node.test.as_ref()], + Stmt::If(node) => std::iter::once(node.test.as_ref()) + .chain( + node.elif_else_clauses + .iter() + .filter_map(|clause| clause.test.as_ref()), + ) + .collect(), Stmt::While(node) => vec![node.test.as_ref()], - Stmt::For(node) => vec![node.iter.as_ref()], + Stmt::For(node) => { + let mut all = vec![node.iter.as_ref()]; + if let Some(pattern) = &node.pattern { + all.extend(pattern_expressions(pattern)); + } + all + } + Stmt::Let(node) => { + let mut all = vec![node.value.as_ref()]; + all.extend(pattern_expressions(&node.pattern)); + all + } + Stmt::Match(node) => { + let mut all = vec![node.subject.as_ref()]; + for case in &node.cases { + all.extend(pattern_expressions(&case.pattern)); + all.extend(case.guard.as_deref()); + } + all + } + Stmt::Try(node) => node + .handlers + .iter() + .filter_map(|handler| { + let ast::ExceptHandler::ExceptHandler(handler) = handler; + handler.type_.as_deref() + }) + .collect(), Stmt::Raise(node) => node .exc .iter() @@ -484,8 +729,55 @@ pub(crate) fn statement_expressions(stmt: &Stmt) -> Vec<&Expr> { all } Stmt::With(node) => node.items.iter().map(|item| &item.context_expr).collect(), - _ => Vec::new(), + Stmt::FunctionDef(node) => node + .decorator_list + .iter() + .map(|decorator| &decorator.expression) + .chain( + node.parameters + .iter() + .filter_map(ast::AnyParameterRef::default), + ) + .collect(), + Stmt::ClassDef(node) => node + .decorator_list + .iter() + .map(|decorator| &decorator.expression) + .chain( + node.arguments + .iter() + .flat_map(|arguments| arguments.iter_source_order()) + .map(|argument| match argument { + ast::ArgOrKeyword::Arg(expr) => expr, + ast::ArgOrKeyword::Keyword(keyword) => &keyword.value, + }), + ) + .collect(), + Stmt::Delete(_) + | Stmt::TypeAlias(_) + | Stmt::Import(_) + | Stmt::ImportFrom(_) + | Stmt::Global(_) + | Stmt::Nonlocal(_) + | Stmt::Pass(_) + | Stmt::Break(_) + | Stmt::Continue(_) + | Stmt::IpyEscapeCommand(_) => Vec::new(), + } +} + +/// the expressions a pattern evaluates as it matches: a value pattern's value, a mapping +/// pattern's keys, and the class a class pattern names +fn pattern_expressions(pattern: &ast::Pattern) -> Vec<&Expr> { + struct Collect<'a>(Vec<&'a Expr>); + impl<'a> Visitor<'a> for Collect<'a> { + fn visit_expr(&mut self, expr: &'a Expr) { + self.0.push(expr); + } } + let mut collect = Collect(Vec::new()); + collect.visit_pattern(pattern); + collect.0 } /// the expressions a statement's assignment *targets* evaluate diff --git a/crates/by_irbuild/src/generators.rs b/crates/by_irbuild/src/generators.rs index c04503c548..2ff28b594f 100644 --- a/crates/by_irbuild/src/generators.rs +++ b/crates/by_irbuild/src/generators.rs @@ -37,9 +37,25 @@ //! each of those gets a field written at the suspension and read back at the //! resumption — see [`park_live_registers`] //! +//! ## a generator that is also a closure +//! +//! a nested generator copies a capture into its state object where the environment +//! holds it as a value no frame writes. a *cell* — a name some frame writes — is not +//! copied, because a copy is a second cell the two frames stop agreeing about: the state +//! object keeps the environment itself in `$outer`, and the body reads and writes the +//! cell there, walking the same chain a nested function walks +//! +//! ## a closure inside a generator +//! +//! the other way round needs no environment of its own: the state object already holds +//! every local as a field, so a generator expression or a lambda the body makes is a +//! method of the state object, and reads a local there when it runs +//! //! ## what is declined, and why it would be wrong rather than slow //! -//! - a generator that is also a closure — one object cannot be two environments +//! - a `def` nested inside a generator: the environment it would close over is a +//! register of the frame that makes it, and a generator's registers do not survive a +//! suspension — see [`check`] use std::collections::{BTreeSet, HashSet}; @@ -51,6 +67,15 @@ use crate::mapper::{Decline, Lowered}; /// the field holding which resumption point to enter at pub(crate) const STATE_FIELD: &str = "$state"; +/// what `$state` holds while the frame is running +/// +/// the dispatch writes it as the first thing a resumption does, and every way the frame +/// leaves writes over it: a suspension its resumption point, a finish `-1`. python +/// refuses to resume a frame that is running — a generator whose body iterates the +/// generator itself raises `ValueError: generator already executing` — and resuming it +/// here would start the body again from its last suspension while the registers of the +/// run in progress are still in use. the runtime reads it as `BY_FRAME_RUNNING` +pub(crate) const RUNNING_STATE: i64 = -2; /// the field holding the value passed to `send`, which is what `yield` evaluates to pub(crate) const SENT_FIELD: &str = "$sent"; /// the field holding an exception `throw` or `close` wants raised *at* the suspension @@ -268,11 +293,12 @@ fn for_loops(body: &[Stmt]) -> usize { .into_iter() .map(|stmt| match stmt { Stmt::For(node) => 1 + usize::from(node.is_async), - // each context manager is parked too: the body suspends and `__exit__` - // still has to run, whether the resumption returns or raises - // an `async with` item also awaits `__aenter__` and `__aexit__`, and - // the exit is awaited on both the normal and the raising path - Stmt::With(node) => node.items.len() * if node.is_async { 5 } else { 1 }, + // each context manager is parked too, and so is the exit bound as its block + // was entered: the body suspends and `__exit__` still has to run, whether the + // resumption returns or raises. an `async with` item also awaits + // `__aenter__` and `__aexit__`, and the exit is awaited on both the normal + // and the raising path + Stmt::With(node) => node.items.len() * if node.is_async { 6 } else { 2 }, _ => 0, }) .sum(); @@ -413,6 +439,7 @@ pub(crate) fn park_live_registers( terminator: std::mem::replace(&mut block.terminator, Terminator::Goto(moved)), owned_at_exit: None, range: block.range, + position: block.position.take(), error_target: block.error_target.take(), }; function.blocks.push(rest); diff --git a/crates/by_irbuild/src/lib.rs b/crates/by_irbuild/src/lib.rs index 22b8976e32..f7a36fc359 100644 --- a/crates/by_irbuild/src/lib.rs +++ b/crates/by_irbuild/src/lib.rs @@ -599,6 +599,7 @@ pub fn build_module( // a module-level frame is in no class body, so nothing it names is mangled owner: None, frame: None, + environment_suffix: None, }; // pass one and a half: the direct editions, before any caller, because an `await` @@ -1184,6 +1185,16 @@ fn span(range: ruff_text_size::TextRange) -> (u32, u32) { (range.start().to_u32(), range.end().to_u32()) } +/// the name python gives the frame `function` runs in: `` and `` for the +/// bodies python names that way, and the definition's own name for everything else +fn frame_name(function: &ast::StmtFunctionDef, captures: Option<&closures::Nested>) -> String { + match captures { + Some(entry) if entry.lambda.is_some() => "".to_string(), + Some(entry) if entry.generator_expression.is_some() => "".to_string(), + _ => function.name.to_string(), + } +} + /// lower a generator: a constructor, and a state class whose `$resume` method is /// the body as a state machine fn lower_generator( @@ -1192,6 +1203,7 @@ fn lower_generator( decorators: Vec, receiver: Option>, captures: Option<&closures::Nested>, + qualname: &str, ) -> Lowered<(Function, Vec)> { let Unit { env, @@ -1225,17 +1237,44 @@ fn lower_generator( unit.arrays, ); representations.retain(|(name, _)| !declared_global.contains(name)); + // a capture is copied into the state object where the environment holds it as a + // value neither frame writes. every other one — a cell some frame writes, or a name + // an environment further up holds — is reached through the environment itself, which + // the state object keeps in `$outer`: the frame outlives the call that made it, and a + // copy of a cell is a second cell the two frames stop agreeing about + let environment_class = match receiver { + Some(Receiver::Implicit(RType::Instance { class, .. })) if captures.is_some() => { + Some(class.clone()) + } + _ => None, + }; + let (captured, chained): (Vec, Vec) = captures + .map(|nested| nested.captures.clone()) + .unwrap_or_default() + .into_iter() + .partition(|name| { + !captures.is_some_and(|nested| nested.shared.contains(name)) + && environment_class + .as_ref() + .and_then(|class| layouts.get(class)) + .is_some_and(|fields| { + fields + .iter() + .any(|field| field.name == *name && !field.cell) + }) + }); + // a name the generator writes `nonlocal` is not one of its own locals + representations.retain(|(name, _)| !chained.contains(name)); let locals: Vec = representations .iter() .map(|(name, _)| name.clone()) .collect(); - let captured: Vec = captures - .map(|nested| nested.captures.clone()) - .unwrap_or_default(); // the constructor seeds every one of them, so they are as assigned as a parameter let mut assigned = generators::definitely_assigned(function); assigned.extend(captured.iter().cloned()); - let parameters = signature(db, env, model, function, layouts, receiver, &[])?.params; + let mut signed = signature(db, env, model, function, layouts, receiver, &[])?; + takes_the_first_iterator(captures, &mut signed); + let parameters = signed.params; let representation = |name: &str| { parameters .iter() @@ -1245,6 +1284,10 @@ fn lower_generator( }; let names = { let mut names = generators::state_names(function, &locals); + if captures.is_some_and(|nested| nested.generator_expression.is_some()) { + assigned.insert(closures::GENERATOR_ITERATOR.to_string()); + names.push(closures::GENERATOR_ITERATOR.to_string()); + } let extra: Vec = captured .iter() .filter(|name| !names.contains(name)) @@ -1253,7 +1296,7 @@ fn lower_generator( names.extend(extra); names }; - let fields: Vec = names + let mut fields: Vec = names .into_iter() .map(|name| by_ir::function::FieldDecl { cell: false, @@ -1272,24 +1315,135 @@ fn lower_generator( default: None, }) .collect(); + // the field a name reached through the environment walks from, named as an + // environment's own link to the one enclosing it so that the walk is the same one + if let Some(environment) = environment_class.as_ref().filter(|_| !chained.is_empty()) { + assigned.insert(closures::OUTER_FIELD.to_string()); + fields.push(by_ir::function::FieldDecl { + cell: false, + optional: false, + defaulted_by: None, + name: closures::OUTER_FIELD.to_string(), + ty: RType::Instance { + class: environment.clone(), + exact: false, + }, + default: None, + }); + } + // the closures the frame makes — a generator expression, a lambda — are methods of the + // state object, which holds every local as a field already. a closure reads a local + // there when it runs rather than when it was made, which is python's answer for a name + // the frame goes on writing after it made the closure; a name nothing writes is copied + let state = RType::Instance { + class: class.clone(), + exact: false, + }; + let written: HashSet = representations + .iter() + .map(|(name, _)| name.clone()) + .collect(); + let bound: HashSet = parameters + .iter() + .map(|(name, _)| name.clone()) + .chain(written.iter().cloned()) + .chain( + captures + .into_iter() + .flat_map(|nested| nested.captures.iter().cloned()), + ) + .collect(); + let never_written: HashSet = bound.difference(&written).cloned().collect(); + let per_iteration = if unit.language.unique_loop_bindings() { + closures::loop_targets(&function.body) + } else { + HashSet::new() + }; + let nested = + closures::nested_functions(&function.body, &bound, &never_written, &per_iteration)?; + for entry in &nested { + // a loop binding is frozen per closure, which takes an environment allocated at + // each one — and the state object is allocated once + if entry + .captures + .iter() + .any(|name| per_iteration.contains(name)) + { + return Err(Decline::new( + "a closure inside a generator over a loop binding is not lowered yet", + )); + } + // a name the closure shares with the frame is read as a cell, which needs a + // representation with an unset value + if let Some(name) = entry.shared.iter().find(|name| { + fields + .iter() + .any(|field| field.name == **name && !has_an_unset_value(&field.ty)) + }) { + return Err(Decline::new(format!( + "a closure inside a generator shares `{name}`, whose representation has no unset value to read a cell with" + ))); + } + } let mut layouts_with_state = layouts.clone(); layouts_with_state.insert(class.clone(), fields.clone()); - let (resume, parked) = lower_resume( + let mut methods_with_state = unit.methods.clone(); + if !nested.is_empty() { + let table = nested + .iter() + .filter_map(|entry| { + let mut signature = signature( + db, + env, + model, + &entry.def, + &layouts_with_state, + Some(Receiver::Implicit(&state)), + &[], + ) + .ok()?; + resumable_return(&entry.def, &mut signature); + takes_the_first_iterator(Some(entry), &mut signature); + Some((entry.def.name.to_string(), signature)) + }) + .collect(); + methods_with_state.insert(class.clone(), table); + } + let environment = (!nested.is_empty()).then(|| Closures { + class: class.clone(), + register: RegisterId(0), + lambdas: nested + .iter() + .filter_map(|entry| { + entry + .lambda + .or(entry.generator_expression) + .map(|range| (span(range), entry.def.name.to_string())) + }) + .collect(), + ready: HashSet::new(), + settled: closures::bound_only_by_their_def(&function.body), + outer: None, + per_closure: None, + }); + let (resume, parked, delegations) = lower_resume( Unit { layouts: &layouts_with_state, + methods: &methods_with_state, ..unit }, function, + &frame_name(function, captures), &class, &fields, &assigned, + environment, )?; // which registers had to be parked is only known once the body is lowered, so the // slots they took join the layout here — before the constructor, which seeds one // value per field and would otherwise leave the last of them off - let mut fields = fields; fields.extend(parked); layouts_with_state.insert(class.clone(), fields.clone()); let constructor = lower_generator_constructor( @@ -1303,8 +1457,33 @@ fn lower_generator( decorators, receiver, &captured, + captures, )?; + // the closures' own bodies, with the state object as their receiver + let mut methods = vec![resume]; + let mut classes = Vec::new(); + for entry in &nested { + let (mut method, inner) = lower_function_with_receiver( + Unit { + layouts: &layouts_with_state, + methods: &methods_with_state, + frame: Some(qualname), + environment_suffix: None, + ..unit + }, + &entry.def, + Some(Receiver::Implicit(&state)), + Some(entry), + &[], + Frame::AsWritten, + )?; + method.owner = Some(class.clone()); + method.exported = false; + methods.push(method); + classes.extend(inner); + } + Ok(( constructor, vec![by_ir::function::ClassIr { @@ -1320,6 +1499,8 @@ fn lower_generator( (true, false) => by_ir::function::Surface::Coroutine, _ => by_ir::function::Surface::Generator, }, + qualname: qualname.to_string(), + delegations, }), fields, decorators: Vec::new(), @@ -1327,7 +1508,7 @@ fn lower_generator( slot_aliases: Vec::new(), generic: false, properties: Vec::new(), - methods: vec![resume], + methods, base: None, inherited_init: false, fields_are_parameters: true, @@ -1335,11 +1516,18 @@ fn lower_generator( immutable: false, environment: false, keywords: Vec::new(), - }], + }] + .into_iter() + .chain(classes) + .collect(), )) } /// the function the *call* runs: allocate the state object, seed the parameters +#[expect( + clippy::too_many_arguments, + reason = "one call site, and every argument is part of what the state object is seeded from" +)] fn lower_generator_constructor( unit: Unit<'_>, function: &ast::StmtFunctionDef, @@ -1348,6 +1536,7 @@ fn lower_generator_constructor( decorators: Vec, receiver: Option>, captured: &[String], + nested: Option<&closures::Nested>, ) -> Lowered { let Unit { env, @@ -1356,6 +1545,8 @@ fn lower_generator_constructor( layouts, .. } = unit; + let mut signed = signature(db, env, model, function, layouts, receiver, &[])?; + takes_the_first_iterator(nested, &mut signed); let Signature { params, defaults, @@ -1366,9 +1557,10 @@ fn lower_generator_constructor( deferring, computed_defaults, .. - } = signature(db, env, model, function, layouts, receiver, &[])?; + } = signed; let mut builder = FunctionBuilder::new(function.name.to_string(), RType::OBJECT); + builder.frame_name(frame_name(function, nested)); builder.at(span(function.range)); builder.decorators(decorators); // this is the function the name holds, so it is the one `__doc__` is asked of — @@ -1399,6 +1591,9 @@ fn lower_generator_constructor( _ => String::new(), }; let mut seeded: HashMap = HashMap::new(); + if let Some(env) = environment { + seeded.insert(closures::OUTER_FIELD.to_string(), env); + } for name in captured { let Some(env) = environment else { continue }; let ty = layouts @@ -1463,10 +1658,16 @@ fn lower_generator_constructor( fn lower_resume( unit: Unit<'_>, function: &ast::StmtFunctionDef, + python_name: &str, class: &str, fields: &[by_ir::function::FieldDecl], assigned: &HashSet, -) -> Lowered<(Function, Vec)> { + environment: Option, +) -> Lowered<( + Function, + Vec, + Vec, +)> { let Unit { db, model, @@ -1478,6 +1679,7 @@ fn lower_resume( } = unit; let mut builder = FunctionBuilder::new(generators::RESUME_METHOD.to_string(), RType::OBJECT); + builder.frame_name(python_name); builder.at(span(function.range)); let receiver = builder.param( "$gen".to_string(), @@ -1495,6 +1697,7 @@ fn lower_resume( builder.switch_to(entry); let mut lowering = Lowering { + lines: model.line_index(), arrays: unit.arrays, directs: unit.directs, in_range: Vec::new(), @@ -1528,7 +1731,7 @@ fn lower_resume( unnarrowed: None, language: unit.language, recheck_licences: unit.recheck_licences, - environment: None, + environment, captures: Some(Captured { class: class.to_string(), receiver, @@ -1547,13 +1750,15 @@ fn lower_resume( // a generator's state field is the frame's *own* local, parked free: false, }), - // a generator that shares a cell with the frame around it is turned down above, - // and its own state fields are reached as ordinary captures + // a cell the generator shares with the frame around it lives in that frame's + // environment, reached through `$outer`, and its own state fields are reached as + // ordinary captures owned_cells: None, generator: Some(Generator { class: class.to_string(), resumptions: Vec::new(), iterators: 0, + delegations: Vec::new(), }), delegations: 0, contexts: 0, @@ -1566,10 +1771,10 @@ fn lower_resume( lowering.builder.terminate(Terminator::Goto(exhausted)); } - let resumptions = lowering + let (resumptions, delegations) = lowering .generator .take() - .map(|generator| generator.resumptions) + .map(|generator| (generator.resumptions, generator.delegations)) .unwrap_or_default(); // the property a *direct* edition rests on, checked against the machine that was @@ -1595,6 +1800,16 @@ fn lower_resume( class: class.to_string(), field: generators::STATE_FIELD.to_string(), }); + // from here until the frame suspends or finishes it is running, and every way of + // resuming it refuses — see `generators::RUNNING_STATE` + lowering.builder.push(Op::SetField { + receiver: Value::Register(receiver), + class: class.to_string(), + field: generators::STATE_FIELD.to_string(), + value: Value::Int(generators::RUNNING_STATE), + moves: false, + present: false, + }); let mut targets = vec![(0i64, entry)]; targets.extend(resumptions.iter().map(|point| (point.state, point.resume))); for (index, (value, block)) in targets.iter().enumerate() { @@ -1647,7 +1862,7 @@ fn lower_resume( let parked = generators::park_live_registers(&mut lowered, class, &resumptions)?; lowered.owner = Some(class.to_string()); lowered.exported = false; - Ok((lowered, parked)) + Ok((lowered, parked, delegations)) } /// a lowering that does not verify is a *decline*, not a build failure @@ -2479,6 +2694,14 @@ fn lower_class<'a>( deleted, }, )?; + // a class that adds no storage of its own keeps its base's fields in the base's layout + let inherited = base + .as_ref() + .and_then(ClassBase::in_module) + .and_then(|base| layouts.get(base)) + .map(Vec::as_slice) + .unwrap_or_default(); + no_field_shadows_a_method(db, env, model, suite, class, layouts, &fields, inherited)?; // the `@property` groups, worked out before any method is lowered: the two halves of // a pair are both written `def value`, so a pass that took them one at a time would // see a name defined twice rather than the single attribute python builds out of them. @@ -4410,6 +4633,13 @@ fn lower_accessor<'a>( class: &str, half: Half, ) -> Lowered<(Function, Vec)> { + // an environment is named after the `def` it belongs to, and every half of a + // property is written `def value` — so each half's is named after the half as well, + // or two halves that both make closures would ask for one class between them + let unit = Unit { + environment_suffix: Some(half.suffix()), + ..unit + }; let (mut method, produced) = lower_method(unit, accessor, class)?; method.decorators.clear(); // a boundary that hands the call on takes the twin off the interpreted class by @@ -4421,13 +4651,6 @@ fn lower_accessor<'a>( half.written_as() ))); } - // an environment class is named after the `def`, which both halves share - if !produced.is_empty() { - return Err(Decline::new(format!( - "a property `{}` that makes closures is not lowered yet", - half.written_as() - ))); - } method.name.push_str(half.suffix()); Ok((method, produced)) } @@ -5609,6 +5832,94 @@ fn class_fields( Ok(fields) } +/// refuse a class where a field and a method of the class, or of a base of ours, share a +/// name +/// +/// python answers an attribute the instance holds before a function its class holds +/// under the same name, and the function where the instance holds none: +/// +/// ```python +/// class Encoder: +/// def __init__(self, default=None): +/// if default is not None: +/// self.default = default +/// +/// def default(self, o): ... +/// +/// def encode(self, o): +/// return self.default(o) # the one it was handed, or the method +/// ``` +/// +/// a layout field's descriptor and the method table's entry would both be published under +/// the name, and a method call reaches the method by its class alone — so the call would +/// answer the method for an encoder handed a `default`, and a read the field's absence for +/// one handed nothing. a `property` is not among these: an assignment to one runs its +/// setter, and a field beside one is refused where properties are gathered +#[expect( + clippy::too_many_arguments, + reason = "one call site, and the base chain is resolved the way the layout resolves it" +)] +fn no_field_shadows_a_method( + db: &dyn ty_python_semantic::Db, + env: &ProgramEnvironment<'_>, + model: &SemanticModel<'_>, + suite: &[Stmt], + class: &ast::StmtClassDef, + layouts: &Layouts, + fields: &[by_ir::function::FieldDecl], + inherited: &[by_ir::function::FieldDecl], +) -> Lowered<()> { + let mut current = Some(class); + // a chain of classes is finite, and a cycle among them is not a program python runs + let mut visited: HashSet<&str> = HashSet::new(); + while let Some(owner) = current + && visited.insert(owner.name.as_str()) + { + for statement in &owner.body { + let Stmt::FunctionDef(method) = statement else { + continue; + }; + if method + .decorator_list + .iter() + .any(|decorator| is_property_decorator(&decorator.expression)) + { + continue; + } + let name = mangled(Some(&owner.name), method.name.as_str()); + if fields + .iter() + .chain(inherited) + .any(|field| field.name == name) + { + return Err(Decline::new(format!( + "`{}` is both a method and an attribute the instance is given, and python answers the attribute where the instance holds one", + method.name + ))); + } + } + current = match base_class(db, env, model, suite, owner, layouts)? { + Some(ClassBase::InModule(name)) => suite.iter().find_map(|statement| match statement { + Stmt::ClassDef(candidate) if candidate.name.as_str() == name => Some(candidate), + _ => None, + }), + _ => None, + }; + } + Ok(()) +} + +/// whether a decorator makes a `property`, or one of the halves added to one +fn is_property_decorator(expression: &Expr) -> bool { + match expression { + Expr::Name(name) => name.id.as_str() == "property", + Expr::Attribute(attribute) => { + matches!(attribute.attr.as_str(), "setter" | "getter" | "deleter") + } + _ => false, + } +} + /// whether a `__del__` this class writes is one the deallocation would ever reach /// /// `tp_finalize` is reached from `tp_dealloc`, and the dealloc that would reach it @@ -6291,6 +6602,19 @@ fn resumable_return(function: &ast::StmtFunctionDef, signature: &mut Signature) } } +/// the parameter a generator expression's function takes its first iterator in, which +/// no source wrote and so no signature read off the ast has +/// +/// see [`closures::GENERATOR_ITERATOR`] +fn takes_the_first_iterator(nested: Option<&closures::Nested>, signature: &mut Signature) { + if nested.is_some_and(|nested| nested.generator_expression.is_some()) { + signature + .params + .push((closures::GENERATOR_ITERATOR.to_string(), RType::OBJECT)); + signature.defaults.push(None); + } +} + /// the `(index, array)` a `while` loop counts over, when it counts over one /// /// the guard has to be exactly `i < len(A)`, `i` has to be a counter the body only @@ -6805,10 +7129,7 @@ fn lower_function_with_receiver( let nested = matches!(receiver, Some(Receiver::Implicit(_))); // what python calls this frame: a nested function after the frames it is written // in, and anything else after its class and its own name - let python_name = match captures { - Some(entry) if entry.lambda.is_some() => "".to_string(), - _ => function.name.to_string(), - }; + let python_name = frame_name(function, captures); let qualname = match (unit.frame, unit.owner) { (Some(enclosing), _) if nested => format!("{enclosing}..{python_name}"), (_, Some(class)) => format!("{class}.{python_name}"), @@ -6862,19 +7183,8 @@ fn lower_function_with_receiver( // state object and hand it back. the body becomes a method of that object if frame == Frame::AsWritten && (generators::is_generator(&function.body) || function.is_async) { - // a nested one keeps its captures in the *state* object rather than reaching - // back through the environment: the frame outlives the call that made it, and - // a copy is what a capture already is. a *shared* one is a cell both frames - // write, which a copy cannot be - if let Some(captured) = captures - && !captured.shared.is_empty() - { - return Err(Decline::new( - "a generator that shares a cell with the frame around it is not lowered yet", - )); - } let (mut constructor, classes) = - lower_generator(unit, function, decorators, receiver, captures)?; + lower_generator(unit, function, decorators, receiver, captures, &qualname)?; if named_by_its_frame { constructor.nested = Some(by_ir::function::NestedName { name: python_name, @@ -6997,7 +7307,11 @@ fn lower_function_with_receiver( // neither of these has one let frame_name = closures::environment_name( enclosing.or_else(|| unit.owner.filter(|_| binding != Binding::Instance)), - &function.name, + &format!( + "{}{}", + function.name, + unit.environment_suffix.unwrap_or_default() + ), ); let frame_owned: HashSet = if split { owned.difference(&bindings_here).cloned().collect() @@ -7100,6 +7414,7 @@ fn lower_function_with_receiver( ) .ok()?; resumable_return(&entry.def, &mut signature); + takes_the_first_iterator(Some(entry), &mut signature); Some((entry.def.name.to_string(), signature)) }) .collect(); @@ -7109,6 +7424,7 @@ fn lower_function_with_receiver( let methods = &methods_with_env; let mut builder = FunctionBuilder::new(function.name.to_string(), ret.clone()); + builder.frame_name(python_name.clone()); builder.at(span(function.range)); builder.decorators(decorators); builder.doc(docstring(&function.body)?); @@ -7338,6 +7654,7 @@ fn lower_function_with_receiver( let zero_super = zero_argument_super(function, declared_receiver); let mut lowering = Lowering { + lines: model.line_index(), arrays: unit.arrays, directs: unit.directs, in_range: Vec::new(), @@ -7383,6 +7700,7 @@ fn lower_function_with_receiver( .filter_map(|entry| { entry .lambda + .or(entry.generator_expression) .map(|range| (span(range), entry.def.name.to_string())) }) .collect(), @@ -7431,6 +7749,7 @@ fn lower_function_with_receiver( layouts, methods, frame: Some(&qualname), + environment_suffix: None, ..unit }; let mut lowered_methods = Vec::with_capacity(nested.len()); @@ -7599,6 +7918,11 @@ struct Unit<'a> { /// nested one — see [`by_ir::function::Function::nested`]. only a nested frame has /// one: anything else is named by its class and its own name frame: Option<&'a str>, + /// what the environment of the frame being lowered is named after beyond its `def`, + /// where the `def`'s name is not enough to tell it apart — see [`lower_accessor`]. a + /// frame nested inside takes its name from this one's environment, so it is never + /// handed down + environment_suffix: Option<&'a str>, } /// what a nested function reads through its environment @@ -7711,6 +8035,34 @@ enum Suspension { Yielded = 1, } +/// the delegation a suspension waits inside +/// +/// python raises an exception thrown into a frame suspended in `yield from` or `await` at +/// the delegation, where a `StopIteration` is not an error at all: it is the inner +/// iterator finishing, and its value is what the expression evaluates to. that is how an +/// inner iterator's `throw` that ends it hands back a result +#[derive(Clone, Copy)] +struct Delegated<'a> { + /// the register the expression's value is left in + result: RegisterId, + /// where the lowering continues once the delegation has finished + finished: BlockId, + /// where the inner iterator is kept across the suspension + iterator: &'a Place, +} + +/// how a `with` block reaches the half of the protocol it calls on its way out +#[derive(Clone)] +enum ContextExit { + /// straight to the `__exit__` of the class the manager is known to be, which the + /// compiler emitted as an immutable type — see [`Lowering::protocol_owner`]. nothing + /// can rebind the method, so where it is reached from does not matter + Direct(String), + /// through the `__exit__` or `__aexit__` [`Op::BindExit`] looked up as the block was + /// entered, kept where it survives the body + Bound(Place), +} + /// something an early exit has to run before it transfers control #[derive(Clone)] enum Cleanup { @@ -7725,9 +8077,7 @@ enum Cleanup { Context { manager: Place, is_async: bool, - /// the class whose `__exit__` this reaches, where the block's manager was - /// known to be one the compiler emitted — see [`Lowering::protocol_owner`] - direct: Option, + exit: ContextExit, }, /// an `except` block's handled exception, put back on the way out /// @@ -7798,6 +8148,8 @@ struct Generator { resumptions: Vec, /// how many `for` loops have taken an iterator field iterators: usize, + /// the suspensions lowered so far that wait inside a delegation + delegations: Vec, } /// the closure environment a frame allocates @@ -7805,8 +8157,8 @@ struct Closures { class: String, /// the register holding the instance register: RegisterId, - /// `lambda range -> generated method name`, so the *expression* can find the - /// method the closure analysis made for it + /// `lambda or generator expression range -> generated method name`, so the + /// *expression* can find the method the closure analysis made for it lambdas: HashMap<(u32, u32), String>, /// the nested functions whose `def` has been lowered, so the environment is /// live and a call to one can go straight to its native entry point @@ -9287,6 +9639,8 @@ fn named_parameters( struct Lowering<'a, 'db> { db: &'db dyn ty_python_semantic::Db, model: &'a SemanticModel<'db>, + /// the source's lines, which say what line each op was lowered from + lines: ruff_source_file::LineIndex, builder: FunctionBuilder, locals: HashMap, /// the names this frame declares `global`, which live in the module namespace @@ -9383,6 +9737,11 @@ struct Lowering<'a, 'db> { unnarrowed: Option, } +/// the builtins that run whatever they are handed to its end before they answer, so a +/// generator expression handed to one alone can be built at once — see +/// [`Lowering::drained_generator_expression`] +const DRAINING_BUILTINS: &[&str] = &["list", "tuple", "set", "frozenset", "sorted", "dict"]; + /// where [`Lowering::resolve_builtin`] reads the object a builtin's name resolves to #[derive(Clone, Copy, PartialEq, Eq)] enum Wanted { @@ -9662,15 +10021,27 @@ impl Lowering<'_, '_> { } fn block(&mut self, body: &[Stmt]) -> Lowered<()> { - for stmt in body { - self.statement(stmt)?; - } - Ok(()) + // what the statement around a body goes on to do once the body has run — a + // `with`'s exit, a loop's next step — was written on that statement's own line + let around = self.builder.location(); + let lowered = body.iter().try_for_each(|stmt| self.statement(stmt)); + self.builder.relocate(around); + lowered + } + + /// say that what is lowered from here on was written where `node` starts + fn locate(&mut self, node: ruff_text_size::TextRange) -> Option<(u32, u32)> { + let line = self.lines.line_index(node.start()).get(); + self.builder.locate( + node.start().to_u32(), + u32::try_from(line).unwrap_or(u32::MAX), + ) } fn statement(&mut self, stmt: &Stmt) -> Lowered<()> { // the first statement lowered into a block decides that block's `#line` self.builder.block_at(span(stmt.range())); + self.locate(stmt.range()); match stmt { Stmt::Pass(_) => Ok(()), // a declaration, not an action: the capture analysis already read it, and @@ -10959,15 +11330,15 @@ impl Lowering<'_, '_> { Cleanup::Context { manager, is_async, - direct, + exit, } => { let (manager, _) = self.read_place(&manager)?; let ignored = self.builder.temp(RType::BIT); if is_async { let none = self.widen_to_object(Value::None, &RType::NONE); - self.await_exit(manager, none, ignored)?; + self.await_exit(manager, &exit, none, ignored)?; } else { - self.leave_context(manager, ignored, direct.as_deref()); + self.leave_context(manager, ignored, &exit)?; } } } @@ -11094,27 +11465,30 @@ impl Lowering<'_, '_> { /// `__exit__` suppress an exception that is *there*, so on this path a truthy /// answer suppresses nothing, and `dest` is the "not suppressed" bit either way. /// that is what [`by_rt`]'s `By_ExitContext` does with it too - fn leave_context(&mut self, manager: Value, dest: RegisterId, direct: Option<&str>) { - let owner = direct.and_then(|class| { - self.protocol_owner( - &RType::Instance { - class: class.to_string(), - exact: false, - }, - "__exit__", - 4, - ) - }); - let (Some(class), Some(owner)) = (direct, owner) else { - let manager = self.manager_object(manager); + fn leave_context( + &mut self, + manager: Value, + dest: RegisterId, + exit: &ContextExit, + ) -> Lowered<()> { + let direct = match exit { + ContextExit::Direct(class) => self + .protocol_owner( + &RType::Instance { + class: class.clone(), + exact: false, + }, + "__exit__", + 4, + ) + .map(|owner| (class.clone(), owner)), + ContextExit::Bound(_) => None, + }; + let Some((class, owner)) = direct else { let no_exception = self.widen_to_object(Value::None, &RType::NONE); - self.builder.push(Op::ExitContext { - dest, - manager, - exception: no_exception, - }); - return; + return self.exit_through_protocol(manager, dest, exit, no_exception); }; + let class = class.as_str(); let ret = self .methods .get(&owner) @@ -11133,6 +11507,40 @@ impl Lowering<'_, '_> { args, }); self.builder.assign(dest, Value::Bit(false)); + Ok(()) + } + + /// `__exit__` called through the object protocol with `exception`, or `None` on the + /// normal path, answering in `dest` whether it suppressed the exception + fn exit_through_protocol( + &mut self, + manager: Value, + dest: RegisterId, + exit: &ContextExit, + exception: Value, + ) -> Lowered<()> { + let manager = self.manager_object(manager); + let bound = match exit { + ContextExit::Bound(place) => self.read_place(place)?.0, + // an immutable class, so the method bound here is the one binding it at the + // entry would have found + ContextExit::Direct(_) => { + let bound = self.builder.temp(RType::OBJECT); + self.builder.push(Op::BindExit { + dest: bound, + manager: manager.clone(), + is_async: false, + }); + Value::Register(bound) + } + }; + self.builder.push(Op::ExitContext { + dest, + manager, + exit: bound, + exception, + }); + Ok(()) } /// `with EXPR as VAR: BLOCK` @@ -11180,6 +11588,22 @@ impl Lowering<'_, '_> { // inside a generator the manager outlives the frame: the body suspends and // `__exit__` still has to run, whether the resumption returns or raises let held = self.park_iterator(held)?; + // python binds the exit as the block is entered, before `__enter__` runs, so a + // class rebinding it while the block runs does not change which one is called + let exit = match &direct { + Some(class) => ContextExit::Direct(class.clone()), + None => { + let (live, _) = self.read_place(&held)?; + let manager = self.manager_object(live); + let bound = self.builder.temp(RType::OBJECT); + self.builder.push(Op::BindExit { + dest: bound, + manager, + is_async: node.is_async, + }); + ContextExit::Bound(self.park_iterator(bound)?) + } + }; let entered = self.builder.temp(RType::OBJECT); let (live, _) = self.read_place(&held)?; @@ -11190,6 +11614,7 @@ impl Lowering<'_, '_> { self.builder.push(Op::AsyncContext { dest: awaitable, manager: live, + exit: None, exception: None, }); let (value, ty) = self.delegate_value(Value::Register(awaitable), true)?; @@ -11221,7 +11646,7 @@ impl Lowering<'_, '_> { self.cleanups.push(Cleanup::Context { manager: held.clone(), is_async: node.is_async, - direct: direct.clone(), + exit: exit.clone(), }); let body_result = if rest.is_empty() { self.block(&node.body) @@ -11247,9 +11672,9 @@ impl Lowering<'_, '_> { let (live, _) = self.read_place(&held)?; if node.is_async { let no_exception = self.widen_to_object(Value::None, &RType::NONE); - self.await_exit(live, no_exception, ignored)?; + self.await_exit(live, &exit, no_exception, ignored)?; } else { - self.leave_context(live, ignored, direct.as_deref()); + self.leave_context(live, ignored, &exit)?; } self.builder.terminate(Terminator::Goto(after)); @@ -11288,14 +11713,9 @@ impl Lowering<'_, '_> { exception, }); let decided = if node.is_async { - self.await_exit(raising, Value::Register(exception), suppressed) + self.await_exit(raising, &exit, Value::Register(exception), suppressed) } else { - self.builder.push(Op::ExitContext { - dest: suppressed, - manager: raising, - exception: Value::Register(exception), - }); - Ok(()) + self.exit_through_protocol(raising, suppressed, &exit, Value::Register(exception)) }; self.cleanups.pop(); decided?; @@ -11514,7 +11934,7 @@ impl Lowering<'_, '_> { self.builder.push(Op::PopHandled { value: Value::Register(handled), }); - self.block(&node.finalbody)?; + self.finally_while_raising(&node.finalbody, pending)?; self.builder.push(Op::Reraise { value: Value::Register(pending), }); @@ -11528,7 +11948,7 @@ impl Lowering<'_, '_> { self.builder.push(Op::PopHandled { value: Value::Register(handled), }); - self.block(&node.finalbody)?; + self.finally_while_raising(&node.finalbody, exception)?; self.builder.push(Op::Reraise { value: Value::Register(exception), }); @@ -11541,7 +11961,7 @@ impl Lowering<'_, '_> { self.builder.push(Op::PopHandled { value: Value::Register(handled), }); - self.block(&node.finalbody)?; + self.finally_while_raising(&node.finalbody, pending)?; self.builder.push(Op::Reraise { value: Value::Register(pending), }); @@ -11555,6 +11975,65 @@ impl Lowering<'_, '_> { Ok(()) } + /// a `finally` body on the path of an exception that is still propagating + /// + /// python runs it with that exception as the one being handled: `sys.exception()` + /// answers it, a bare `raise` re-raises it, and anything raised in the body chains + /// onto it. the handled exception is put back on every way out — falling off the end, + /// an early exit through the cleanups, and a raise, which gets a block of its own for + /// that. a body that falls off the end leaves the caller in a block that can still + /// re-raise `pending` + fn finally_while_raising(&mut self, finalbody: &[Stmt], pending: RegisterId) -> Lowered<()> { + if finalbody.is_empty() { + return Ok(()); + } + let previous = self.builder.temp(RType::OBJECT); + self.builder.push(Op::PushHandled { + dest: previous, + value: Value::Register(pending), + }); + let body = self.builder.new_block(); + let raised = self.builder.new_block(); + self.builder.terminate(Terminator::Goto(body)); + self.builder.switch_to(body); + let enclosing = self.builder.set_error_target(Some(raised)); + self.cleanups.push(Cleanup::Handled { + previous, + exception: pending, + }); + self.handling.push(pending); + let ran = self.block(finalbody); + self.handling.pop(); + self.cleanups.pop(); + let fell_through = ran.is_ok() && !self.builder.is_sealed(self.builder.current_block()); + let after = self.builder.new_block(); + if fell_through { + self.builder.push(Op::PopHandled { + value: Value::Register(previous), + }); + self.builder.terminate(Terminator::Goto(after)); + } + self.builder.set_error_target(enclosing); + ran?; + + self.builder.switch_to(raised); + let replaced = self.builder.temp(RType::OBJECT); + self.builder.push(Op::FetchException { dest: replaced }); + self.builder.push(Op::PopHandled { + value: Value::Register(previous), + }); + self.builder.push(Op::Reraise { + value: Value::Register(replaced), + }); + self.builder.terminate(Terminator::Unreachable); + + self.builder.switch_to(after); + if !fell_through { + self.builder.terminate(Terminator::Unreachable); + } + Ok(()) + } + /// `assert cond` / `assert cond, "message"` fn assert_statement(&mut self, node: &ast::StmtAssert) -> Lowered<()> { let cond = self.condition(&node.test)?; @@ -13350,13 +13829,21 @@ impl Lowering<'_, '_> { fn await_exit( &mut self, manager: Value, + exit: &ContextExit, exception: Value, suppressed: RegisterId, ) -> Lowered<()> { + let ContextExit::Bound(bound) = exit else { + return Err(Decline::new( + "an `async with` whose `__aexit__` was not bound as the block was entered", + )); + }; + let (bound, _) = self.read_place(bound)?; let awaitable = self.builder.temp(RType::OBJECT); self.builder.push(Op::AsyncContext { dest: awaitable, manager, + exit: Some(bound), exception: Some(exception), }); let (answer, ty) = self.delegate_value(Value::Register(awaitable), true)?; @@ -13437,12 +13924,18 @@ impl Lowering<'_, '_> { let header = self.builder.new_block(); let forward = self.builder.new_block(); + let returned = self.builder.new_block(); let finished = self.builder.new_block(); + let unwind = self.builder.new_block(); let result = self .builder .local(format!("$delegated{}", self.delegations), RType::OBJECT); self.delegations += 1; self.builder.terminate(Terminator::Goto(header)); + // python drops the inner iterator the moment the delegation ends, whichever way it + // ends, so one abandoned by an exception is finalized before any handler of this + // frame runs rather than whenever the frame itself goes + let enclosing = self.builder.set_error_target(Some(unwind)); // each trip: read the inner iterator back, send in whatever `send` gave us, // and either forward a value or take the result @@ -13471,19 +13964,62 @@ impl Lowering<'_, '_> { }); self.builder.terminate(Terminator::Branch { cond: Value::Register(done), - then_block: finished, + then_block: returned, else_block: forward, }); self.builder.switch_to(forward); - self.suspend(Value::Register(step), Suspension::Awaited)?; + self.suspend( + Value::Register(step), + Suspension::Awaited, + Some(Delegated { + result, + finished, + iterator: &parked, + }), + )?; self.builder.terminate(Terminator::Goto(header)); - self.builder.switch_to(finished); + self.builder.switch_to(returned); self.builder.assign(result, Value::Register(step)); + self.builder.terminate(Terminator::Goto(finished)); + self.builder.set_error_target(enclosing); + + self.builder.switch_to(unwind); + let pending = self.builder.temp(RType::OBJECT); + self.builder.push(Op::FetchException { dest: pending }); + self.clear_parked(&parked); + self.builder.push(Op::Reraise { + value: Value::Register(pending), + }); + self.builder.terminate(Terminator::Unreachable); + + self.builder.switch_to(finished); + self.clear_parked(&parked); Ok((Value::Register(result), RType::OBJECT)) } + /// let go of what a parked field holds, by writing `None` over it + fn clear_parked(&mut self, parked: &Place) { + if let Place::Field { + receiver, + class, + name, + .. + } = parked + { + let cleared = self.widen_to_object(Value::None, &RType::NONE); + self.builder.push(Op::SetField { + receiver: Value::Register(*receiver), + class: class.clone(), + field: name.clone(), + value: cleared, + moves: false, + present: false, + }); + } + } + /// what `send` last passed in fn read_sent(&mut self) -> Lowered { let Some(generator) = &self.generator else { @@ -13501,7 +14037,16 @@ impl Lowering<'_, '_> { } /// suspend with `value`, and continue in a fresh resumption block - fn suspend(&mut self, value: Value, kind: Suspension) -> Lowered<()> { + /// + /// `delegated` is the delegation a `yield from` or an `await` suspends inside, which + /// changes what an exception thrown in at the suspension means — see + /// [`Delegated`] + fn suspend( + &mut self, + value: Value, + kind: Suspension, + delegated: Option>, + ) -> Lowered<()> { let Some(generator) = &self.generator else { return Err(Decline::new("a suspension outside a generator")); }; @@ -13572,6 +14117,12 @@ impl Lowering<'_, '_> { suspend: suspend_at, resume: resume_at, }); + if let Some(Place::Field { name, .. }) = delegated.as_ref().map(|d| d.iterator) { + generator.delegations.push(by_ir::function::Delegation { + state, + field: name.clone(), + }); + } } // `throw` and `close` resume *by raising*, and the raise has to happen here — @@ -13626,6 +14177,32 @@ impl Lowering<'_, '_> { moves: false, present: false, }); + if let Some(Delegated { + result, finished, .. + }) = delegated + { + let value = self.builder.temp(RType::OBJECT); + self.builder.push(Op::StopIterationValue { + dest: value, + src: Value::Register(thrown), + }); + let other = self.builder.temp(RType::BIT); + self.builder.push(Op::IsNull { + dest: other, + src: Value::Register(value), + }); + let ended = self.builder.new_block(); + let raised = self.builder.new_block(); + self.builder.terminate(Terminator::Branch { + cond: Value::Register(other), + then_block: raised, + else_block: ended, + }); + self.builder.switch_to(ended); + self.builder.assign(result, Value::Register(value)); + self.builder.terminate(Terminator::Goto(finished)); + self.builder.switch_to(raised); + } // the block's error target is the enclosing handler, which is exactly where a // `throw` at a suspension has to land. it is *raised* rather than put back: // python chains an exception thrown into a frame onto what that frame is @@ -13694,13 +14271,61 @@ impl Lowering<'_, '_> { self.widen_to_object(value, &ty) } }; - self.suspend(value, Suspension::Yielded)?; + self.suspend(value, Suspension::Yielded, None)?; // the expression's own value is what `send` passed in, or `None` for plain // iteration let sent = self.read_sent()?; Ok((sent, RType::OBJECT)) } + /// a generator expression: the generator its function makes, handed the iterator of + /// its first clause's iterable, which is evaluated here + /// + /// nothing else runs until the generator is asked for a value — not the element, not + /// a condition, not a later clause's iterable — and the generator runs only as far as + /// it is asked. building the values at once instead is a different program wherever + /// the consumer stops early, as `any` does, or wherever the generator is kept and + /// resumed later + fn generator_expression(&mut self, node: &ast::ExprGenerator) -> Lowered<(Value, RType)> { + if !closures::lowers_as_a_generator(node) { + return Err(Decline::new( + "an asynchronous generator expression is not lowered yet", + )); + } + self.refresh_environment()?; + let Some(environment) = &self.environment else { + return Err(Decline::new( + "a generator expression is not lowered where the frame has no environment \ + to make its generator over", + )); + }; + let Some(method) = environment.lambdas.get(&span(node.range)).cloned() else { + return Err(Decline::new( + "a generator expression with no generated function", + )); + }; + let (class, register) = (environment.class.clone(), environment.register); + let Some(first) = node.generators.first() else { + return Err(Decline::new("a generator expression with no clause")); + }; + let (iterable, iterable_ty) = self.expression(&first.iter)?; + let iterable = self.widen_to_object(iterable, &iterable_ty); + let iterator = self.builder.temp(RType::OBJECT); + self.builder.push(Op::GetIter { + dest: iterator, + src: iterable, + cursor: None, + }); + let dest = self.builder.temp(RType::OBJECT); + self.builder.push(Op::CallNative { + dest: Some(dest), + owner: Some(class), + callee: method, + args: vec![Value::Register(register), Value::Register(iterator)], + }); + Ok((Value::Register(dest), RType::OBJECT)) + } + /// a lambda: the closure over this frame's environment, by the generated name fn lambda(&mut self, node: &ast::ExprLambda) -> Lowered<(Value, RType)> { self.refresh_environment()?; @@ -14166,7 +14791,20 @@ impl Lowering<'_, '_> { } /// lower an expression, returning its operand and representation + /// lower `expr`, marking what it lowers to with the line it was written on + /// + /// the line is put back afterwards, so what the enclosing expression goes on to do + /// with the value is marked with the enclosing expression's own line — python names + /// the line of the instruction that raised, and a call spread over several lines + /// raises on the line the call starts on, not the line its last argument was on fn expression(&mut self, expr: &Expr) -> Lowered<(Value, RType)> { + let enclosing = self.locate(expr.range()); + let lowered = self.expression_here(expr); + self.builder.relocate(enclosing); + lowered + } + + fn expression_here(&mut self, expr: &Expr) -> Lowered<(Value, RType)> { match expr { Expr::NumberLiteral(node) => match &node.value { ast::Number::Int(value) => { @@ -14330,15 +14968,7 @@ impl Lowering<'_, '_> { Expr::ListComp(node) => { self.comprehension(&node.generators, &Comprehension::List(&node.elt)) } - // a generator expression is lazy, and the consumers that matter — `sum`, - // `any`, `max` — drain it immediately. building the list is the same - // answer for those, so it is only a decline where the laziness is - // observable: an infinite one, or a side effect ordered against the - // consumer. neither is expressible without a `yield`, which a genexp has - // no way to write - Expr::Generator(node) => { - self.comprehension(&node.generators, &Comprehension::List(&node.elt)) - } + Expr::Generator(node) => self.generator_expression(node), Expr::SetComp(node) => { self.comprehension(&node.generators, &Comprehension::Set(&node.elt)) } @@ -17033,6 +17663,16 @@ impl Lowering<'_, '_> { return self.len_call(node); } + if DRAINING_BUILTINS.contains(&name) + && !self.native_callees.contains(name) + && !self.binds(name) + && node.arguments.keywords.is_empty() + && let [Expr::Generator(generator)] = node.arguments.args.as_ref() + && closures::lowers_as_a_generator(generator) + { + return self.drained_generator_expression(node, name, generator); + } + // a closure this frame made itself is called at its native entry point: the // environment is in a register right here, so there is nothing to look up // and nothing to box @@ -17132,6 +17772,85 @@ impl Lowering<'_, '_> { self.native_call(node, name) } + /// `list(x for x in xs)`, or the same handed to one of the other builtins that drain + /// what they are given before they answer + /// + /// while the name is the builtin the generator would be run to its end at once, so + /// the loop its frame would run is run here instead and the builtin is handed the + /// list — which for `list` is the answer itself. two things keep that the same + /// program: the name is read before anything else, as python reads it, and a + /// `StopIteration` leaving the loop becomes the `RuntimeError` a generator's frame + /// makes of one. where the name holds anything else it is handed the generator + fn drained_generator_expression( + &mut self, + node: &ast::ExprCall, + name: &str, + generator: &ast::ExprGenerator, + ) -> Lowered<(Value, RType)> { + let builds_the_answer = name == "list"; + let (callee, genuine) = self.resolve_builtin( + name, + if builds_the_answer { + Wanted::WhereRebound + } else { + Wanted::Always + }, + ); + let dest = self.builder.temp(RType::OBJECT); + let at_once = self.builder.new_block(); + let handed_over = self.builder.new_block(); + let join = self.builder.new_block(); + self.builder.terminate(Terminator::Branch { + cond: Value::Register(genuine), + then_block: at_once, + else_block: handed_over, + }); + + self.builder.switch_to(at_once); + let leaving = self.builder.new_block(); + let drained = self.builder.new_block(); + let previous = self.builder.set_error_target(Some(leaving)); + let around = self.builder.in_generator_expression(true); + let built = self.comprehension(&generator.generators, &Comprehension::List(&generator.elt)); + if built.is_ok() { + self.builder.terminate(Terminator::Goto(drained)); + } + self.builder.in_generator_expression(around); + self.builder.set_error_target(previous); + let (list, list_ty) = built?; + self.builder.switch_to(leaving); + let exception = self.builder.temp(RType::OBJECT); + self.builder.push(Op::FetchException { dest: exception }); + self.builder.push(Op::LeaveGenerator { + value: Value::Register(exception), + }); + self.builder.terminate(Terminator::Unreachable); + self.builder.switch_to(drained); + let list = self.widen_to_object(list, &list_ty); + if builds_the_answer { + self.builder.assign(dest, list); + } else { + self.builder.push(Op::CallValue { + dest, + callee: callee.clone(), + args: vec![list], + }); + } + self.builder.terminate(Terminator::Goto(join)); + + self.builder.switch_to(handed_over); + let (made, _) = self.generator_expression(generator)?; + self.builder.push(Op::CallValue { + dest, + callee, + args: vec![made], + }); + self.builder.terminate(Terminator::Goto(join)); + + self.builder.switch_to(join); + self.narrow_call_result(dest, &Expr::Call(node.clone())) + } + /// whether `name` still resolves to the interpreter's own builtin, and what it /// resolves to /// diff --git a/crates/by_irbuild/src/shims.rs b/crates/by_irbuild/src/shims.rs index b38ad311c2..ecefa0376b 100644 --- a/crates/by_irbuild/src/shims.rs +++ b/crates/by_irbuild/src/shims.rs @@ -155,7 +155,7 @@ pub fn shims(module: &ModuleIr, twin: &str) -> Option { // a forwarder is bound under a number here and takes the definition's name only // once it is on its way into the namespace let prefix = format!("{installer}_"); - let mut source = format!("\n\ndef {installer}({prefix}natives, {prefix}g):\n"); + let mut source = format!("\n\ndef {installer}({prefix}natives, {prefix}g, {prefix}caught):\n"); // the forwarder is a different object from the definition python would have // built, so everything that definition would have been asked about is carried // onto it — and the definition itself is left reachable through `__wrapped__`, @@ -207,12 +207,23 @@ pub fn shims(module: &ModuleIr, twin: &str) -> Option { }; // a cell of its own per forwarder. one shared name would be a single cell // every one of them closed over, so they would all end up calling whichever - // native was assigned last + // native was assigned last. + // + // an exception leaving the native has an entry for every compiled frame it passed + // through, the function's own among them, and then gains one more for the + // forwarder's frame — which python's program never had. the forwarder takes that + // one back off and re-raises, which adds nothing: the entry at the head is always + // its own, because it is the last frame the exception passed. `try` costs nothing + // on the way through, and it is written on the `def`'s next line with its body so + // no instruction is spent marking a line of its own let _ = write!( source, " {prefix}n{slot} = {prefix}natives[{slot}]\n\ \x20 def {prefix}f{slot}({params}):\n\ - \x20 return {prefix}n{slot}({arguments})\n\ + \x20 try: return {prefix}n{slot}({arguments})\n\ + \x20 except {prefix}caught as {prefix}e:\n\ + \x20 {prefix}e.__traceback__ = {prefix}e.__traceback__.tb_next\n\ + \x20 raise\n\ \x20 {prefix}adopt({prefix}f{slot}, {:?})\n", function.name ); diff --git a/crates/by_irbuild/src/tests.rs b/crates/by_irbuild/src/tests.rs index e6ed64e7ae..a85b59f2a5 100644 --- a/crates/by_irbuild/src/tests.rs +++ b/crates/by_irbuild/src/tests.rs @@ -973,6 +973,52 @@ def f(rows: list[list[int]]) -> object: ); } +#[test] +fn a_generator_expression_is_built_at_once_only_for_a_draining_builtin() { + // `list` runs the generator to its end before it answers, so while the name is the + // builtin the loop runs in this frame — with a `StopIteration` leaving it converted as + // it would leave the generator's frame. `sum` is handed the generator itself + with_source( + "\ +def drained(xs: list[int]) -> list[int]: + return list(x * 2 for x in xs) + +def summed(xs: list[int]) -> int: + return sum(x * 2 for x in xs) +", + |db, env, model, suite| { + let module = + crate::build_module(db, env, model, suite, "app", crate::Language::BasedPython); + assert!(module.declined.is_empty(), "{:?}", module.declined); + let function = |name: &str| { + module + .all_functions() + .find(|function| function.name == name) + .expect("the function is compiled") + }; + let makes_the_generator = |op: &Op| { + matches!(op, Op::CallNative { callee, .. } + if callee.starts_with(crate::closures::GENERATOR_EXPRESSION)) + }; + let drained = function("drained"); + assert!( + has_op(drained, |op| matches!(op, Op::BuildList { .. })) + && has_op(drained, |op| matches!(op, Op::LeaveGenerator { .. })) + && has_op(drained, makes_the_generator), + "{}", + print_function(drained) + ); + let summed = function("summed"); + assert!( + !has_op(summed, |op| matches!(op, Op::BuildList { .. })) + && has_op(summed, makes_the_generator), + "{}", + print_function(summed) + ); + }, + ); +} + #[test] fn a_comprehension_clause_reading_a_variable_before_it_is_bound_declines() { // `y` in the first clause's own iterable is the enclosing frame's, and in the second @@ -1786,6 +1832,54 @@ class Tagged: ); } +#[test] +fn a_method_that_is_also_an_attribute_the_instance_is_given_declines() { + // python answers an instance's own attribute before a function its class holds under + // the same name, so `json.encoder.JSONEncoder` reads the `default` its constructor was + // handed where there was one and the method where there was not. a layout field and a + // method table cannot both stand under one name, and a method call reaches the method + // by the class alone — so either the call or the read would answer the other one + for source in [ + "\ +class Encoder: + def __init__(self, default: object = None) -> None: + if default is not None: + self.default = default + + def default(self, o: object) -> object: + return o +", + "\ +class Base: + def read(self) -> int: + return 1 + + +class Reader(Base): + def __init__(self) -> None: + self.read = len +", + "\ +class Base: + def __init__(self) -> None: + self.read = len + + +class Reader(Base): + def read(self) -> int: + return 1 +", + ] { + let reasons = declines(source); + assert!( + reasons + .iter() + .any(|(_, reason)| reason.contains("is both a method and an attribute")), + "{reasons:?}" + ); + } +} + #[test] fn a_dunder_that_is_both_a_class_level_value_and_a_field_still_declines() { // a dunder in that shape is two mechanisms over one name: the type slot the assignment diff --git a/crates/by_opt/src/copy_propagation.rs b/crates/by_opt/src/copy_propagation.rs index 97c09110ba..feac47089d 100644 --- a/crates/by_opt/src/copy_propagation.rs +++ b/crates/by_opt/src/copy_propagation.rs @@ -301,6 +301,7 @@ fn retarget(op: &mut Op, new_dest: RegisterId) { | Op::NewInstance { dest, .. } | Op::GetCell { dest, .. } | Op::Enter { dest, .. } + | Op::BindExit { dest, .. } | Op::ExitContext { dest, .. } | Op::DelegateIter { dest, .. } | Op::DelegateStep { dest, .. } @@ -308,6 +309,7 @@ fn retarget(op: &mut Op, new_dest: RegisterId) { | Op::GetIter { dest, .. } | Op::IterNext { dest, .. } | Op::IsNull { dest, .. } + | Op::StopIterationValue { dest, .. } | Op::CallMethod { dest, .. } | Op::GetAttr { dest, .. } | Op::GetField { dest, .. } @@ -358,8 +360,10 @@ fn retarget(op: &mut Op, new_dest: RegisterId) { | Op::PopHandled { .. } | Op::Release { .. } | Op::Reraise { .. } + | Op::LeaveGenerator { .. } | Op::LicenceHolds { .. } | Op::RequireField { .. } + | Op::Line { .. } | Op::SetField { .. } => {} } } diff --git a/crates/by_opt/src/infallible.rs b/crates/by_opt/src/infallible.rs index f4882c89bb..617f5ebec8 100644 --- a/crates/by_opt/src/infallible.rs +++ b/crates/by_opt/src/infallible.rs @@ -171,6 +171,7 @@ fn op_can_fail(module: &ModuleIr, function: &by_ir::function::Function, op: &Op) | Op::GetCell { .. } | Op::RaiseWith { .. } | Op::Enter { .. } + | Op::BindExit { .. } | Op::ExitContext { .. } | Op::DelegateIter { .. } | Op::DelegateStep { .. } @@ -191,10 +192,13 @@ fn op_can_fail(module: &ModuleIr, function: &by_ir::function::Function, op: &Op) | Op::SetItem { .. } | Op::Format { .. } => true, - // a null test reads a pointer, and taking or matching a pending exception + // a null test reads a pointer, a `StopIteration`'s value is a field of it, a + // position is not an operation at all, and taking or matching a pending exception // touches only the thread state — none of them can fail // a field read or write is a struct access at a known offset Op::IsNull { .. } + | Op::StopIterationValue { .. } + | Op::Line { .. } | Op::FetchException { .. } | Op::ExceptionMatches { .. } | Op::GetField { .. } @@ -238,7 +242,10 @@ fn op_can_fail(module: &ModuleIr, function: &by_ir::function::Function, op: &Op) // a raise always leaves through the error path, and so does a finish — it // hands back nothing, so it takes the same exit even though no exception // is set - Op::Reraise { .. } | Op::RaiseObject { .. } | Op::FinishFrame { .. } => true, + Op::Reraise { .. } + | Op::LeaveGenerator { .. } + | Op::RaiseObject { .. } + | Op::FinishFrame { .. } => true, // the container protocol reaches `__contains__`, or iterates Op::Contains { .. } => true, diff --git a/crates/by_opt/src/unswitch.rs b/crates/by_opt/src/unswitch.rs index 39aa642796..9f7eb137e7 100644 --- a/crates/by_opt/src/unswitch.rs +++ b/crates/by_opt/src/unswitch.rs @@ -92,6 +92,7 @@ fn unswitch_one(function: &mut Function) -> bool { }, owned_at_exit: None, range: function.blocks[header.index()].range, + position: function.blocks[header.index()].position, error_target: None, }); // the preheader is excluded along with the loop: its own `otherwise` edge *is* an diff --git a/crates/by_rt/include/by.h b/crates/by_rt/include/by.h index e15bf2e1c4..2d14f1eb55 100644 --- a/crates/by_rt/include/by.h +++ b/crates/by_rt/include/by.h @@ -29,6 +29,9 @@ /* `PyMarshal_ReadObjectFromString`, which reads the interpreted twin's code object * back. `Python.h` does not pull this one in */ #include +/* `PyFrame_New`, which a traceback entry for a compiled frame is hung off. `Python.h` + * does not pull this one in either */ +#include #include #include #include @@ -1317,7 +1320,17 @@ static PyObject *By_CreateModule(PyObject *spec, PyModuleDef *def) { return NULL; } found = By_RecordedModule(def); - if (found != NULL) return By_NewRef(found); + if (found != NULL) { + /* python forgets the state pointer of whatever module a create slot hands back — + * `PyModule_FromDefAndSpec2` writes NULL over it the moment this returns, 3.11 + * through 3.15 — and the exec slot then allocates a fresh one. the state the module + * was given at its first import would be left behind at every import after it, so + * it is let go of here, just before python forgets it. this definition asks for no + * state, so nothing reads it in between */ + void *state = PyModule_GetState(found); + if (state != NULL) PyMem_Free(state); + return By_NewRef(found); + } if (PyErr_Occurred()) return NULL; name = PyObject_GetAttrString(spec, "name"); if (name == NULL) return NULL; @@ -7739,6 +7752,36 @@ static inline PyObject *By_Format(PyObject *value, PyObject *spec, int conversio * state, a test of whether it matches, and a way to put it back when it does not */ +/* one place in a compiled function a traceback entry names: the file, the function and + * the line, and the code object built for them the first time an exception passed */ +typedef struct { + PyCodeObject *code; + const char *file; + const char *name; + int line; +} ByTracebackSite; + +/* add the traceback entry python adds for a frame an exception is raised in or passes + * through + * + * a compiled function has no frame, so it hangs the entry off one made for the purpose: an + * empty code object naming the file, the function and the line, built once for the site + * and kept, and a new frame over it in the module's namespace. python's own entry is made + * the same way, from the frame it already has. a failure to build either leaves the + * exception as it was, one entry short */ +BY_COLD void By_TracebackHere(ByTracebackSite *site, PyObject *globals) { + PyObject *type, *value, *traceback; + PyErr_Fetch(&type, &value, &traceback); + if (site->code == NULL) site->code = PyCode_NewEmpty(site->file, site->name, site->line); + PyFrameObject *frame = site->code == NULL || globals == NULL + ? NULL + : PyFrame_New(PyThreadState_Get(), site->code, globals, NULL); + PyErr_Restore(type, value, traceback); + if (frame == NULL) return; + PyTraceBack_Here(frame); + Py_DECREF(frame); +} + /* take the pending exception. returns the value, or NULL when nothing is set */ static inline PyObject *By_FetchException(void) { PyObject *type = NULL, *value = NULL, *traceback = NULL; @@ -8523,9 +8566,10 @@ static inline PyObject *By_BuildForwarders(PyObject *module, PyObject *dict, PyM { /* the namespace is handed in rather than taken with `globals()`, because a * module that defines a function called `globals` would have made that name - * a local of the installer before the installer ever ran */ - PyObject *args[2] = {natives, staging}; - result = PyObject_Vectorcall(install, args, 2, NULL); + * a local of the installer before the installer ever ran. `BaseException` is + * handed in for the same reason: a forwarder catches it by that name */ + PyObject *args[3] = {natives, staging, PyExc_BaseException}; + result = PyObject_Vectorcall(install, args, 3, NULL); } Py_DECREF(install); Py_DECREF(natives); @@ -9022,6 +9066,15 @@ static inline PyObject *By_TakeReturn(PyObject **returned) { #define BY_FRAME_COROUTINE 1 #define BY_FRAME_ASYNC_GENERATOR 2 +/* what `$state` holds while the frame runs: its resume writes this before anything else, + * and a suspension or a finish writes over it. the frontend's `RUNNING_STATE` */ +#define BY_FRAME_RUNNING -2 + +/* whether a frame's resume is on the stack right now */ +static inline int By_FrameRunning(ByTagged state) { + return state == By_ShortFrom(BY_FRAME_RUNNING); +} + /* what python calls this surface in a message it writes about one */ static inline const char *By_FrameNoun(int frame) { return frame == BY_FRAME_COROUTINE ? "coroutine" @@ -9029,6 +9082,13 @@ static inline const char *By_FrameNoun(int frame) { : "generator"; } +/* refuse to resume a frame whose resume is on the stack, which is its own body asking */ +static inline int By_RefuseRunning(ByTagged state, int frame) { + if (BY_LIKELY(!By_FrameRunning(state))) return 0; + PyErr_Format(PyExc_ValueError, "%s already executing", By_FrameNoun(frame)); + return -1; +} + /* refuse a resumption python itself would not have performed * * two refusals, one at each end of a machine's life, and they are the same question: @@ -9046,11 +9106,15 @@ static inline const char *By_FrameNoun(int frame) { * closing a spent coroutine is how a caller says it is done with it, and `throw` into * an *unstarted* coroutine is exempt too — the thrown exception simply propagates. * + * and a frame that is *running* can take nothing at all: its body is what is asking. + * every surface refuses that, before either of the two above. + * * `state` is 0 before the frame first runs and -1 once it has left for good, so * anything above 0 is a real suspension point with a `yield` to resume. `arg` is NULL * where the caller carries no sent value at all, as a `throw` does */ static inline int By_RefuseResumption(ByTagged state, int frame, PyObject *arg) { if (!By_IsShort(state)) return 0; + if (By_RefuseRunning(state, frame) < 0) return -1; Py_ssize_t at = (Py_ssize_t)By_ShortValue(state); if (at == 0 && arg != NULL && arg != Py_None) { PyErr_Format(PyExc_TypeError, "can't send non-None value to a just-started %s", @@ -9156,42 +9220,254 @@ static inline PyObject *By_StepGenerator(PyObject *self, PyObject **sent, PyObje return By_TakeReturn(returned); } -/* `throw(exc)`: raise it *at the suspension point*. +/* the argument count `throw` and `athrow` take, in the words python uses for both * - * the exception goes into the state object's `$thrown` field, and the resumption - * point raises it — which is what lets a `yield` inside `try` enter its own handler - * rather than the exception appearing at the generator's entry. + * `throw` asks at the call. `athrow` asks when the awaitable it hands back is first + * stepped, because it keeps its arguments whole until then */ +static inline int By_CountThrowArguments(const char *method, Py_ssize_t nargs) { + if (nargs < 1) { + PyErr_Format(PyExc_TypeError, "%s expected at least 1 argument, got %zd", method, nargs); + return -1; + } + if (nargs > 3) { + PyErr_Format(PyExc_TypeError, "%s expected at most 3 arguments, got %zd", method, nargs); + return -1; + } + return 0; +} + +/* the warning the `(type, value, traceback)` form carries, deprecated since 3.12 and + * given at the call on every surface */ +static inline int By_WarnThrowSignature(const char *method, Py_ssize_t nargs) { +#if PY_VERSION_HEX >= 0x030C0000 + if (nargs > 1 + && PyErr_WarnFormat(PyExc_DeprecationWarning, 1, + "the (type, exc, tb) signature of %s() is deprecated, use the " + "single-arg signature instead.", + method) < 0) { + return -1; + } +#else + (void)method; + (void)nargs; +#endif + return 0; +} + +/* the exception `throw(type, value, traceback)` raises, built as python builds it * - * rejecting the argument never reaches the frame at all, and python leaves a - * generator resumable after a `throw` it refused to make sense of. a machine with no - * suspension point does not reach the frame either, but a throw does finish it — see - * below. otherwise it is the resumption that decides, and a body that catches what - * was thrown leaves the machine usable. + * a class is instantiated from the value the way a `raise` of the pair would be: an + * instance of it is kept, a tuple is the argument list, anything else is the one + * argument. an instance takes no value of its own. a traceback given is the one the + * exception carries when it is raised at the suspension. * - * the resumption raises instead of producing a value, so nothing rides in on `$sent` - * — it is parked as `None` all the same, because leaving the last `send`'s value - * standing is what would let a later `yield` read it again */ -static inline PyObject *By_ThrowInto(PyObject *self, PyObject **sent, PyObject **thrown, - PyObject **returned, ByTagged *state, int frame, - PyObject *exception, - PyObject *(*resume)(PyObject *)) { - if (thrown == NULL) return NULL; - /* a spent coroutine refuses a throw for the same reason it refuses a send, and - * before the argument is examined: python answers the reuse rather than whatever - * was being thrown. no sent value rides in on a throw, so the just-started half - * cannot fire here */ - if (By_RefuseResumption(*state, frame, NULL) < 0) return NULL; - PyObject *instance = NULL; - if (PyExceptionInstance_Check(exception)) { - instance = By_NewRef(exception); - } else if (PyExceptionClass_Check(exception)) { - instance = PyObject_CallNoArgs(exception); - if (instance == NULL) return NULL; + * the answer is a new reference to an exception instance, or NULL with the refusal set */ +static inline PyObject *By_ThrownException(PyObject *const *args, Py_ssize_t nargs) { + PyObject *type = args[0]; + PyObject *value = nargs > 1 ? args[1] : NULL; + PyObject *traceback = nargs > 2 ? args[2] : NULL; + if (traceback == Py_None) { + traceback = NULL; + } else if (traceback != NULL && !PyTraceBack_Check(traceback)) { + PyErr_SetString(PyExc_TypeError, "throw() third argument must be a traceback object"); + return NULL; + } + PyObject *instance; + if (PyExceptionClass_Check(type)) { + PyObject *built_type = By_NewRef(type); + PyObject *built = value == NULL ? NULL : By_NewRef(value); + PyObject *built_tb = traceback == NULL ? NULL : By_NewRef(traceback); + /* a constructor that raises leaves *its* exception in the triple, and that is + * what python throws in instead */ + PyErr_NormalizeException(&built_type, &built, &built_tb); + Py_XDECREF(built_type); + Py_XDECREF(built_tb); + if (built == NULL || !PyExceptionInstance_Check(built)) { + Py_XDECREF(built); + if (!PyErr_Occurred()) PyErr_BadInternalCall(); + return NULL; + } + instance = built; + } else if (PyExceptionInstance_Check(type)) { + if (value != NULL && value != Py_None) { + PyErr_SetString(PyExc_TypeError, "instance exception may not have a separate value"); + return NULL; + } + instance = By_NewRef(type); } else { /* `throw` words this differently from `raise`, and names what it was given */ PyErr_Format(PyExc_TypeError, "exceptions must be classes or instances deriving from BaseException, not %s", - Py_TYPE(exception)->tp_name); + Py_TYPE(type)->tp_name); + return NULL; + } + if (traceback != NULL && PyException_SetTraceback(instance, traceback) < 0) { + Py_DECREF(instance); + return NULL; + } + return instance; +} + +/* raise `instance` *at* a suspended frame's suspension point, taking the reference + * + * the exception goes into the state object's `$thrown` field, and the resumption point + * raises it — which is what lets a `yield` inside `try` enter its own handler rather than + * the exception appearing at the generator's entry. + * + * the resumption raises instead of producing a value, so nothing rides in on `$sent` — it + * is parked as `None` all the same, because leaving the last `send`'s value standing is + * what would let a later `yield` read it again */ +static inline PyObject *By_ResumeRaising(PyObject *self, PyObject **sent, PyObject **thrown, + PyObject **returned, ByTagged *state, int frame, + PyObject *instance, PyObject *(*resume)(PyObject *)) { + PyObject *old = *thrown; + *thrown = instance; + Py_XDECREF(old); + return By_StepGenerator(self, sent, returned, state, frame, Py_None, resume); +} + +/* the attribute `name` of `o`, or NULL with nothing raised where it has none */ +static inline int By_OptionalAttr(PyObject *o, PyObject *name, PyObject **found) { +#if PY_VERSION_HEX >= 0x030D0000 + return PyObject_GetOptionalAttr(o, name, found); +#else + return _PyObject_LookupAttr(o, name, found); +#endif +} + +/* python's `gen_close_iter`: close the iterator a frame is delegating to, before the frame + * itself is unwound. an iterator with no `close` has nothing to close, and one whose + * `close` cannot even be looked up is reported and passed over */ +static inline int By_CloseDelegate(PyObject *delegate) { + static PyObject *by_close = NULL; + PyObject *name = By_FixedName(&by_close, "close", 5); + if (name == NULL) return -1; + PyObject *close; + if (By_OptionalAttr(delegate, name, &close) < 0) { +#if PY_VERSION_HEX >= 0x030E0000 + PyErr_FormatUnraisable("Exception ignored while closing generator %R", delegate); +#else + PyErr_WriteUnraisable(delegate); +#endif + close = NULL; + } + if (close == NULL) return 0; + PyObject *result = PyObject_CallNoArgs(close); + Py_DECREF(close); + if (result == NULL) return -1; + Py_DECREF(result); + return 0; +} + +/* `throw` into a frame suspended in `yield from` or `await`, which python hands to the + * iterator the frame is delegating to rather than to the frame. the answer is NULL with + * `*forwarded` 0 where the delegation has nothing to do with the throw and it is raised at + * the suspension as an ordinary one would be. + * + * `GeneratorExit` closes the inner iterator instead — except in an async generator, whose + * `aclose` has to let what it awaits work through the exit — and anything else goes to + * the inner iterator's own `throw`, with the arguments exactly as they came: the frame + * never makes an exception of them. an inner iterator with no `throw` leaves the throw to + * the frame. what the inner iterator yields back is what this `throw` answers, and the + * frame stays suspended where it was; what it raises back is raised at the suspension. + * + * the frame counts as running for as long as the inner iterator has it, so a body that + * reaches back into the frame is refused as python refuses it */ +static inline PyObject *By_ThrowIntoDelegate(PyObject *self, PyObject **sent, PyObject **thrown, + PyObject **returned, ByTagged *state, int frame, + PyObject *const *args, Py_ssize_t nargs, + PyObject *delegate, + PyObject *(*resume)(PyObject *), int *forwarded) { + static PyObject *by_throw = NULL; + ByTagged suspended = *state; + *forwarded = 1; + Py_INCREF(delegate); + if (frame != BY_FRAME_ASYNC_GENERATOR + && PyErr_GivenExceptionMatches(args[0], PyExc_GeneratorExit)) { + *state = By_ShortFrom(BY_FRAME_RUNNING); + int closed = By_CloseDelegate(delegate); + *state = suspended; + Py_DECREF(delegate); + if (closed == 0) { + *forwarded = 0; + return NULL; + } + } else { + PyObject *name = By_FixedName(&by_throw, "throw", 5); + PyObject *method = NULL; + if (name == NULL || By_OptionalAttr(delegate, name, &method) < 0) { + Py_DECREF(delegate); + return NULL; + } + if (method == NULL) { + Py_DECREF(delegate); + *forwarded = 0; + return NULL; + } + /* python hands a generator or coroutine of its own the three arguments without + * going through `throw`, so the deprecated form warns once, at the outermost + * frame, however deep the delegation. asking the method would warn again: that + * generator is handed the exception it would have built out of them instead */ + PyObject *answer; + if (nargs > 1 && (PyGen_CheckExact(delegate) || PyCoro_CheckExact(delegate))) { + PyObject *instance = By_ThrownException(args, nargs); + if (instance == NULL) { + answer = NULL; + } else { + *state = By_ShortFrom(BY_FRAME_RUNNING); + answer = PyObject_CallOneArg(method, instance); + *state = suspended; + Py_DECREF(instance); + } + } else { + *state = By_ShortFrom(BY_FRAME_RUNNING); + answer = PyObject_Vectorcall(method, args, (size_t)nargs, NULL); + *state = suspended; + } + Py_DECREF(method); + Py_DECREF(delegate); + if (answer != NULL) return answer; + } + PyObject *raised = By_FetchException(); + if (raised == NULL) return NULL; + return By_ResumeRaising(self, sent, thrown, returned, state, frame, raised, resume); +} + +/* `throw(...)`: raise it *at the suspension point*. + * + * the arguments are `throw`'s own, already counted. a frame suspended in a delegation + * hands them to the iterator it delegates to — see `By_ThrowIntoDelegate` — and + * `delegate` is what says which iterator that is, where there is one: it answers NULL + * for a suspension at a `yield`, and is NULL itself for a frame with no delegations. + * + * otherwise python builds the exception out of the arguments before it asks anything of + * the frame, so a refusal of the arguments comes first, and it never reaches the frame at + * all: python leaves a generator resumable after a `throw` it refused to make sense of. a + * machine with no suspension point does not reach the frame either, but a throw does + * finish it — see below. otherwise it is the resumption that decides, and a body that + * catches what was thrown leaves the machine usable */ +static inline PyObject *By_ThrowInto(PyObject *self, PyObject **sent, PyObject **thrown, + PyObject **returned, ByTagged *state, int frame, + PyObject *const *args, Py_ssize_t nargs, + PyObject *(*delegate)(PyObject *), + PyObject *(*resume)(PyObject *)) { + if (thrown == NULL) return NULL; + PyObject *inner = delegate != NULL && By_IsShort(*state) && By_ShortValue(*state) > 0 + ? delegate(self) + : NULL; + if (inner != NULL) { + int forwarded; + PyObject *answer = By_ThrowIntoDelegate(self, sent, thrown, returned, state, frame, + args, nargs, inner, resume, &forwarded); + if (forwarded) return answer; + } + PyObject *instance = By_ThrownException(args, nargs); + if (instance == NULL) return NULL; + /* a spent coroutine refuses a throw for the same reason it refuses a send: python + * answers the reuse rather than whatever was being thrown. no sent value rides in on + * a throw, so the just-started half cannot fire here */ + if (By_RefuseResumption(*state, frame, NULL) < 0) { + Py_DECREF(instance); return NULL; } /* a machine with no suspension point has nowhere to raise *at*: one that never @@ -9215,23 +9491,83 @@ static inline PyObject *By_ThrowInto(PyObject *self, PyObject **sent, PyObject * Py_DECREF(instance); return NULL; } - PyObject *old = *thrown; - *thrown = instance; - Py_XDECREF(old); - return By_StepGenerator(self, sent, returned, state, frame, Py_None, resume); + return By_ResumeRaising(self, sent, thrown, returned, state, frame, instance, resume); } -/* `close()`: throw `GeneratorExit` in and accept the three legal outcomes. +/* the value a `StopIteration` carries, or NULL where `exception` is not one * - * exhausting, re-raising `GeneratorExit`, or being already finished are all a clean - * close. *yielding* is not — cpython calls that a `RuntimeError`. + * a `StopIteration` raised at a frame's delegation is the delegation *finishing*: python + * takes its value as what `yield from` or `await` evaluates to, which is how an inner + * iterator's `throw` that ends it hands back a result */ +static inline PyObject *By_StopIterationValue(PyObject *exception) { + if (exception == NULL || !PyExceptionInstance_Check(exception) + || !PyErr_GivenExceptionMatches(exception, PyExc_StopIteration)) { + return NULL; + } + return By_NewRef(((PyStopIterationObject *)exception)->value); +} + +/* report a close that failed while finalizing a frame, which has nowhere to raise * - * the `StopIteration` accepted below is the frame's own *end*, which is the only kind - * that can still be standing here: one the body raised has already become a - * `RuntimeError` on its way out, and comes back as the failure it is */ -static inline int By_CloseGenerator(PyObject *self, PyObject **sent, PyObject **thrown, - PyObject **returned, ByTagged *state, int frame, - PyObject *(*resume)(PyObject *)) { + * 3.14 says what it was doing when it reports one; earlier versions give the object and + * nothing more */ +static inline void By_CloseUnraisable(PyObject *self) { +#if PY_VERSION_HEX >= 0x030E0000 + PyErr_FormatUnraisable("Exception ignored while closing generator %R", self); +#else + PyErr_WriteUnraisable(self); +#endif +} + +/* python's `RuntimeWarning` for a coroutine dropped before it ever ran + * + * issued through `warnings.warn` rather than a C warning, because python's own hands the + * coroutine over as the warning's `source` — which is what `tracemalloc` hangs the + * allocation traceback off. the stack level is the frame the coroutine was dropped in, + * the same one python's `stacklevel=2` reaches from the helper it warns through. a failure + * to warn has nowhere to go and is reported as unraisable, as python reports it */ +static inline void By_WarnUnawaitedCoroutine(PyObject *coroutine, const char *qualname) { + PyObject *warnings = PyImport_ImportModule("warnings"); + PyObject *warn = warnings == NULL ? NULL : PyObject_GetAttrString(warnings, "warn"); + Py_XDECREF(warnings); + PyObject *message = + warn == NULL ? NULL : PyUnicode_FromFormat("coroutine '%s' was never awaited", qualname); + PyObject *level = message == NULL ? NULL : PyLong_FromLong(1); + PyObject *result = NULL; + if (level != NULL) { + PyObject *args[] = {message, PyExc_RuntimeWarning, level, coroutine}; + result = PyObject_Vectorcall(warn, args, 4, NULL); + } + Py_XDECREF(level); + Py_XDECREF(message); + Py_XDECREF(warn); + if (result == NULL) { + PyErr_WriteUnraisable(coroutine); + return; + } + Py_DECREF(result); +} + +/* `close()`: throw `GeneratorExit` in and accept the three legal outcomes, answering + * what `close()` returns. + * + * exhausting, re-raising `GeneratorExit`, or being already finished are all a clean + * close. *yielding* is not — cpython calls that a `RuntimeError`, and the frame stays + * suspended where it yielded, so a later step resumes it and finalizing it closes it again. + * + * since 3.13 a frame that *returns* while it unwinds hands that value back as `close()`'s + * answer, and every other clean close answers `None`. the `StopIteration` the value rides + * on is the frame's own *end*, which is the only kind that can still be standing here: one + * the body raised has already become a `RuntimeError` on its way out, and comes back as the + * failure it is. + * + * the throw is python's own `GeneratorExit`, so a frame suspended in a delegation closes + * the iterator it delegates to first — see `By_ThrowIntoDelegate` */ +static inline PyObject *By_CloseGenerator(PyObject *self, PyObject **sent, PyObject **thrown, + PyObject **returned, ByTagged *state, int frame, + PyObject *(*delegate)(PyObject *), + PyObject *(*resume)(PyObject *)) { + if (By_RefuseRunning(*state, frame) < 0) return NULL; /* a machine with no suspension point has nothing to unwind, and closing one runs * no body at all — not even a `finally` the body has not reached yet. asking * `By_ThrowInto` would give the right answer for a finished frame and the wrong @@ -9239,22 +9575,33 @@ static inline int By_CloseGenerator(PyObject *self, PyObject **sent, PyObject ** * `GeneratorExit` it had no way to see */ if (By_ShortValue(*state) <= 0) { By_FinishGenerator(state); - return 0; + Py_RETURN_NONE; } - PyObject *exit = PyObject_CallNoArgs(PyExc_GeneratorExit); - if (exit == NULL) return -1; - PyObject *result = By_ThrowInto(self, sent, thrown, returned, state, frame, exit, resume); - Py_DECREF(exit); + PyObject *exit = PyExc_GeneratorExit; + PyObject *result = By_ThrowInto(self, sent, thrown, returned, state, frame, &exit, 1, + delegate, resume); if (result != NULL) { Py_DECREF(result); - PyErr_SetString(PyExc_RuntimeError, "generator ignored GeneratorExit"); - return -1; + PyErr_Format(PyExc_RuntimeError, "%s ignored GeneratorExit", By_FrameNoun(frame)); + return NULL; } - if (PyErr_ExceptionMatches(PyExc_StopIteration) || PyErr_ExceptionMatches(PyExc_GeneratorExit)) { + if (PyErr_ExceptionMatches(PyExc_GeneratorExit)) { PyErr_Clear(); - return 0; + Py_RETURN_NONE; } - return -1; + if (PyErr_ExceptionMatches(PyExc_StopIteration)) { +#if PY_VERSION_HEX >= 0x030D0000 + PyObject *ended = By_FetchException(); + if (ended == NULL) return NULL; + PyObject *value = By_StopIterationValue(ended); + Py_DECREF(ended); + return value; +#else + PyErr_Clear(); + Py_RETURN_NONE; +#endif + } + return NULL; } /* the parameter a keyword names, or -1 when none does. @@ -9984,37 +10331,64 @@ static inline PyObject *By_AsyncEnter(ByProtocolSite *site, PyObject *manager) { return result; } -static inline PyObject *By_AsyncExit(ByProtocolSite *site, PyObject *manager, - PyObject *exception) { +/* the half of the protocol a `with` calls on its way out — `__exit__`, or `__aexit__` + * for `async with` — looked up as the block is entered, which is where python binds it + * + * the answer is what [`By_CallExit`] calls with. a plain function is kept as it is and + * called with the manager in front, as the lookup does for every call it makes; anything + * a descriptor handed back is already bound, and is kept in a one-element tuple so the + * exit knows not to add the manager. a function is never a tuple, so the two cannot be + * taken for each other */ +static inline PyObject *By_BindExit(ByProtocolSite *site, PyObject *manager, int is_async) { + static PyObject *by_exit = NULL; static PyObject *by_aexit = NULL; - PyObject *name; + static PyObject *by_enter = NULL; + static PyObject *by_aenter = NULL; if (manager == NULL) return NULL; - name = By_FixedName(&by_aexit, "__aexit__", 9); + PyObject *name = is_async ? By_FixedName(&by_aexit, "__aexit__", 9) + : By_FixedName(&by_exit, "__exit__", 8); if (name == NULL) return NULL; int prepend; PyObject *method = By_ProtocolMethod(site, manager, name, &prepend); if (method == NULL) { if (PyErr_Occurred()) return NULL; - PyErr_Format(PyExc_TypeError, - "'%s' object does not support the asynchronous context manager protocol " - "(missed __aexit__ method)", - Py_TYPE(manager)->tp_name); + PyObject *enter = is_async ? By_FixedName(&by_aenter, "__aenter__", 10) + : By_FixedName(&by_enter, "__enter__", 9); + if (enter == NULL) return NULL; + By_ManagerMissingExit(manager, is_async ? "asynchronous context manager" : "context manager", + is_async ? "__aexit__" : "__exit__", + _PyType_Lookup(Py_TYPE(manager), enter) != NULL); return NULL; } + if (prepend) return method; + PyObject *bound = PyTuple_Pack(1, method); + Py_DECREF(method); + return bound; +} + +/* call what [`By_BindExit`] bound, with `args` holding the manager in its first slot */ +static inline PyObject *By_CallExit(PyObject *exit, PyObject **args, Py_ssize_t nargs) { + if (PyTuple_CheckExact(exit)) return By_CallProtocol(PyTuple_GET_ITEM(exit, 0), 0, args, nargs); + return By_CallProtocol(exit, 1, args, nargs); +} + +/* the arguments an exit is called with: the exception's type, value and traceback, or three + * `None`s on the normal path. the traceback is a new reference where there is one */ +static inline void By_ExitArguments(PyObject *exception, PyObject **args) { int raising = exception != NULL && exception != Py_None && PyExceptionInstance_Check(exception); - PyObject *type = raising ? (PyObject *)Py_TYPE(exception) : Py_None; - PyObject *value = raising ? exception : Py_None; - PyObject *traceback = Py_None; - PyObject *found = NULL; - if (raising) { - found = PyException_GetTraceback(exception); - if (found != NULL) traceback = found; - } - PyObject *args[4] = {manager, type, value, traceback}; - PyObject *result = By_CallProtocol(method, prepend, args, 4); - Py_DECREF(method); - Py_XDECREF(found); + args[0] = raising ? (PyObject *)Py_TYPE(exception) : Py_None; + args[1] = raising ? exception : Py_None; + PyObject *found = raising ? PyException_GetTraceback(exception) : NULL; + args[2] = found != NULL ? found : Py_None; +} + +static inline PyObject *By_AsyncExit(PyObject *manager, PyObject *exit, PyObject *exception) { + if (manager == NULL || exit == NULL) return NULL; + PyObject *args[4] = {manager}; + By_ExitArguments(exception, args + 1); + PyObject *result = By_CallExit(exit, args, 4); + if (args[3] != Py_None) Py_DECREF(args[3]); return result; } @@ -10037,39 +10411,16 @@ static inline PyObject *By_Enter(ByProtocolSite *site, PyObject *manager) { * returns 1 when the exception was *suppressed*, 0 when it was not, and -1 when * `__exit__` itself raised. the caller re-raises on 0, which is what makes * `with` transparent to an exception it does not swallow */ -static inline int By_ExitContext(ByProtocolSite *site, PyObject *manager, - PyObject *exception) { - static PyObject *by_exit = NULL; - PyObject *name; - if (manager == NULL) return -1; - name = By_FixedName(&by_exit, "__exit__", 8); - if (name == NULL) return -1; - int prepend; - PyObject *method = By_ProtocolMethod(site, manager, name, &prepend); - if (method == NULL) { - if (!PyErr_Occurred()) { - PyErr_Format(PyExc_TypeError, - "'%s' object does not support the context manager protocol " - "(missed __exit__ method)", - Py_TYPE(manager)->tp_name); - } - return -1; - } +static inline int By_ExitContext(PyObject *manager, PyObject *exit, PyObject *exception) { + if (manager == NULL || exit == NULL) return -1; /* `None` is the normal path just as NULL is: the frontend hands over a boxed `None`, and reading a traceback off it would be a wild pointer */ int raising = exception != NULL && exception != Py_None && PyExceptionInstance_Check(exception); - PyObject *type = raising ? (PyObject *)Py_TYPE(exception) : Py_None; - PyObject *value = raising ? exception : Py_None; - PyObject *traceback = Py_None; - if (raising) { - PyObject *found = PyException_GetTraceback(exception); - if (found != NULL) traceback = found; - } - PyObject *args[4] = {manager, type, value, traceback}; - PyObject *result = By_CallProtocol(method, prepend, args, 4); - Py_DECREF(method); - if (traceback != Py_None) Py_DECREF(traceback); + PyObject *args[4] = {manager}; + By_ExitArguments(exception, args + 1); + PyObject *result = By_CallExit(exit, args, 4); + if (args[3] != Py_None) Py_DECREF(args[3]); if (result == NULL) return -1; int suppressed = PyObject_IsTrue(result); Py_DECREF(result); @@ -10139,6 +10490,115 @@ static int By_GeneratorIsCoroutine(PyObject *generator) { return (flags & CO_ITERABLE_COROUTINE) != 0; } +/* the iterator `__await__` hands back for a compiled coroutine: python's + * `coroutine_wrapper` + * + * a coroutine is awaitable and is not an iterator — `next(coro)` is a `TypeError` — but + * pep 492 says `__await__` owes an iterator, so it hands back this object wrapped + * around the coroutine. every step forwards: the coroutine's send slot for `__next__` + * and for a caller that sends, and its own `send`, `throw` and `close` methods by name. + * + * an `await` compiled here never builds one — see `By_AwaitIter` — and neither does + * python's own for its own coroutines. it exists for the other callers of `__await__`: + * an interpreted `await`, and anything that asks for the method directly */ +typedef struct { + PyObject_HEAD + PyObject *coroutine; +} ByCoroutineWrapper; + +static void By_CoroutineWrapper_dealloc(PyObject *self) { + PyObject_GC_UnTrack(self); + Py_XDECREF(((ByCoroutineWrapper *)self)->coroutine); + Py_TYPE(self)->tp_free(self); +} + +static int By_CoroutineWrapper_traverse(PyObject *self, visitproc visit, void *arg) { + Py_VISIT(((ByCoroutineWrapper *)self)->coroutine); + return 0; +} + +static PySendResult By_CoroutineWrapper_send_slot(PyObject *self, PyObject *arg, + PyObject **result) { + PyObject *coroutine = ((ByCoroutineWrapper *)self)->coroutine; + return Py_TYPE(coroutine)->tp_as_async->am_send(coroutine, arg, result); +} + +/* python's `gen_iternext`: a return of `None` ends the iteration with no exception at all, + * and any other return value rides out on the `StopIteration` */ +static PyObject *By_CoroutineWrapper_next(PyObject *self) { + PyObject *result = NULL; + PySendResult outcome = By_CoroutineWrapper_send_slot(self, Py_None, &result); + if (outcome != PYGEN_RETURN) return result; + if (result != Py_None) By_RaiseWith(PyExc_StopIteration, result); + Py_DECREF(result); + return NULL; +} + +static PyObject *By_CoroutineWrapper_send(PyObject *self, PyObject *arg) { + PyObject *name = PyUnicode_InternFromString("send"); + if (name == NULL) return NULL; + PyObject *result = PyObject_CallMethodOneArg(((ByCoroutineWrapper *)self)->coroutine, name, arg); + Py_DECREF(name); + return result; +} + +static PyObject *By_CoroutineWrapper_throw(PyObject *self, PyObject *const *args, + Py_ssize_t nargs) { + if (By_CountThrowArguments("throw", nargs) < 0) return NULL; + PyObject *name = PyUnicode_InternFromString("throw"); + if (name == NULL) return NULL; + PyObject *forwarded[4] = {((ByCoroutineWrapper *)self)->coroutine}; + for (Py_ssize_t i = 0; i < nargs; i++) forwarded[i + 1] = args[i]; + PyObject *result = PyObject_VectorcallMethod(name, forwarded, (size_t)nargs + 1, NULL); + Py_DECREF(name); + return result; +} + +static PyObject *By_CoroutineWrapper_close(PyObject *self, PyObject *unused) { + (void)unused; + PyObject *name = PyUnicode_InternFromString("close"); + if (name == NULL) return NULL; + PyObject *result = PyObject_CallMethodNoArgs(((ByCoroutineWrapper *)self)->coroutine, name); + Py_DECREF(name); + return result; +} + +static PyMethodDef By_CoroutineWrapper_methods[] = { + {"send", By_CoroutineWrapper_send, METH_O, NULL}, + {"throw", (PyCFunction)(void (*)(void))By_CoroutineWrapper_throw, METH_FASTCALL, NULL}, + {"close", By_CoroutineWrapper_close, METH_NOARGS, NULL}, + {NULL, NULL, 0, NULL}, +}; + +static PyAsyncMethods By_CoroutineWrapper_async = { + .am_send = By_CoroutineWrapper_send_slot, +}; + +static PyTypeObject By_CoroutineWrapperType = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "by.coroutine_wrapper", + .tp_basicsize = sizeof(ByCoroutineWrapper), + .tp_itemsize = 0, + .tp_dealloc = By_CoroutineWrapper_dealloc, + .tp_as_async = &By_CoroutineWrapper_async, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_DISALLOW_INSTANTIATION, + .tp_traverse = By_CoroutineWrapper_traverse, + .tp_iter = PyObject_SelfIter, + .tp_iternext = By_CoroutineWrapper_next, + .tp_methods = By_CoroutineWrapper_methods, +}; + +/* `__await__` of every compiled coroutine, one function for all of them so that an await + * can recognise one by its slot */ +static PyObject *By_CoroutineAwait(PyObject *coroutine) { + if (PyType_Ready(&By_CoroutineWrapperType) < 0) return NULL; + ByCoroutineWrapper *wrapper = PyObject_GC_New(ByCoroutineWrapper, &By_CoroutineWrapperType); + if (wrapper == NULL) return NULL; + wrapper->coroutine = By_NewRef(coroutine); + PyObject_GC_Track(wrapper); + return (PyObject *)wrapper; +} + /* the iterator a delegation drives: `iter(x)` for `yield from`, `x.__await__()` for * `await`. keeping them apart matters — awaiting an ordinary iterable is an error * @@ -10164,6 +10624,9 @@ static inline PyObject *By_AwaitIter(PyObject *awaitable) { } type = Py_TYPE(awaitable); getter = type->tp_as_async == NULL ? NULL : type->tp_as_async->am_await; + /* a compiled coroutine is driven through its own send slot, as python drives its own + * coroutines, rather than through the wrapper `__await__` would build around it */ + if (getter == By_CoroutineAwait) return By_NewRef(awaitable); if (getter == NULL) { /* `_PyCoro_GetAwaitableIter` raises this, and it is core-only — so the wording * is carried, and 3.14 rewrote it */ diff --git a/docs/basedpython/development/compilation/plan.md b/docs/basedpython/development/compilation/plan.md index 32610f1854..e22c417bca 100644 --- a/docs/basedpython/development/compilation/plan.md +++ b/docs/basedpython/development/compilation/plan.md @@ -22,7 +22,7 @@ short for that reason: | stack depth on deep recursion | python's limit | the C stack's | native frames are not python frames | | `f.__code__` for a compiled function | the definition's | the forwarder's | the module publishes a `function` that forwards to the native, and that is its code | | `f.__wrapped__` for a compiled function | absent | the interpreted definition | which is where `inspect` reads the signature, the source and the file from | -| a traceback through a compiled function | the module's file and the line | `` | the frame that exists is the forwarder's; the native has none | +| a traceback through a compiled function | the module's file and the line | the same, with no column positions | each entry hangs off a frame made for it, with no bytecode behind it | | `is` against a compiled function's own name | identity | `False` where the body captured it | the module body runs before the compiled name replaces it, and what it captured stays | | a `data class` constructor's argument types | unchecked (unless `--soundness all`) | checked | a compiled field is unboxed, so the check is mandatory | | assigning a method onto a class from outside it | works | `TypeError` at the assignment | an emitted class is sealed unless the source decorates it | @@ -49,8 +49,10 @@ answered quietly rather than raising. cpython offers no way to make a `PyCFuncti bind, so a module publishes a real `function` that forwards to the native instead. it answers for the definition about everything it is asked — its name, docstring, module, defaults, annotations, and, through `__wrapped__`, its signature, source and -file — and what is left over is the three rows above: its own code object, the -`__wrapped__` that carries the rest, and the frame a traceback finds +file — and what is left over is the two rows above: its own code object, and the +`__wrapped__` that carries the rest. the forwarder's frame is still a python frame, +and it takes its own entry back off an exception passing out through it, so a +traceback names the compiled function and never the forwarder the sealed-class row is where `functools.total_ordering(SomeClass)` from another module lands. it fills in the comparisons a class is missing by assigning them, and an diff --git a/docs/basedpython/development/compilation/runtime.md b/docs/basedpython/development/compilation/runtime.md index 4937e205eb..f9fc662b0f 100644 --- a/docs/basedpython/development/compilation/runtime.md +++ b/docs/basedpython/development/compilation/runtime.md @@ -479,9 +479,13 @@ path without the exceptional one weighing on it ### tracebacks a compiled frame is not a python frame, so a naive traceback would skip it. -compiled functions push a lightweight entry recording the function and the -current `.by` line, so a traceback through compiled code shows the same file and -line numbers the interpreted build would +python adds an entry for each frame an exception is raised in or passes through, and +a compiled function adds the same entry on the path its failure takes: the file, the +function and the line the failing operation was written on. the entry is cold — the +code object behind it is built the first time an exception passes that point and +kept, and only the frame it hangs off is made each time — so a function that does not +raise pays nothing for it. putting back an exception the frame already holds, as a +bare `raise` does, adds no entry, as in python `by run` already rewrites tracebacks from transpiled `.py` lines back to `.by` lines. compiled frames carry `.by` lines directly, so the same rendering path @@ -690,6 +694,99 @@ sub = _interpreters.create("legacy") _interpreters.exec(sub, "import mod") # python: imports; compiled: ImportError ``` +### a generator is not python's `generator` + +a compiled generator, coroutine or async generator is an object of a type of its +own, named after the function it belongs to, rather than an instance of python's +`generator`, `coroutine` or `async_generator`. it answers every step python's does — +`next`, `send`, `throw`, `close`, `await`, `async for`, running `finally` blocks when +it is closed, collected or dropped — but it carries none of the introspection a +frame object gives python's: + +```python +import inspect +from collections.abc import Iterator + + +def counted(n: int) -> Iterator[int]: + yield n + + +g = counted(1) +type(g).__name__ # python: 'generator'; compiled: 'counted$gen' +inspect.isgenerator(g) # python: True; compiled: False +g.gi_frame # python: a frame; compiled: AttributeError +``` + +the same holds for `gi_running`, `gi_code`, `gi_yieldfrom`, `cr_frame`, `cr_await`, +`ag_frame`, and the object's own `__name__` and `__qualname__`. a message python words +after the type's name names the compiled type instead, so +`type(g)()` refuses with `cannot create 'mod.counted$gen' instances` where python +says `cannot create 'generator' instances` + +### the three-argument `throw` through an interpreted generator + +a compiled frame suspended in `yield from` or `await` hands a `throw` on to the +iterator it is waiting on, as python does. python hands a generator or coroutine of +its own the arguments of the deprecated `throw(type, value, traceback)` form without +calling its `throw` method, which would warn about the deprecated form a second +time. a compiled frame cannot reach that generator any other way, so it builds the +exception out of the three arguments and hands the generator that one exception. +the generator builds the same exception itself when it is the frame that takes the +throw, so the two builds differ only where it is waiting on something in turn — an +iterator with a `throw` of its own is then handed one argument rather than three: + +```python +# an interpreted module +class Inner: + def __iter__(self): + return self + + def __next__(self): + return 1 + + def throw(self, *args): + return len(args) + + +def middle(): + yield from Inner() + + +# the compiled module +def outer(make: Callable[[], Iterator[object]]) -> Iterator[object]: + yield from make() + + +g = outer(middle) +next(g) +g.throw(ValueError, ValueError("x"), None) # python: 3; compiled: 1 +``` + +for the same reason, arguments the exception cannot be built from — a traceback +argument that is not a traceback, say — are refused at the compiled frame, where +python hands them on unchecked to whatever the interpreted generator is waiting on + +### a traceback entry's frame + +a traceback through compiled code names the same files, functions and lines python's +does, but the frame each compiled entry hangs off was made for the entry rather than +being the frame the function ran in. its code object has no bytecode, so it carries no +column positions and a formatted traceback prints no `~~^^` markers under the line; +the frame holds no locals; and its code object's `co_firstlineno` is the entry's own +line and `co_qualname` its bare name: + +```python +import traceback + +try: + mod.parse("") +except ValueError as e: + tb = e.__traceback__.tb_next + tb.tb_frame.f_locals # python: {'text': '', ...}; compiled: {} + traceback.print_exception(e) # python underlines the failing call; compiled does not +``` + ## debugging and inspection - **`#line` directives** in the generated C point at `.by` source. gdb, lldb, From 0096d2ec6e6c33d07dd521ef6a42ded8744b5aa8 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Mon, 14 Sep 2026 22:13:34 +1000 Subject: [PATCH 03/16] native: follow a rebound module function and python's recursion limit a compiled call notices a module function rebound, deleted or patched from outside, and honours a reassigned __code__ or defaults; a compiled recursion counts against the recursion limit and the thread's stack. compile.bind-functions-early and compile.follow-recursion-limit turn each back off, and the development docs say what each leaves --- crates/by_build/src/lib.rs | 21 + crates/by_build/tests/differential.rs | 524 ++++++++++++++ crates/by_build/tests/end_to_end.rs | 5 + crates/by_codegen_c/src/lib.rs | 333 ++++++++- crates/by_ir/src/call_graph.rs | 189 ++++++ crates/by_ir/src/function.rs | 7 + crates/by_ir/src/lib.rs | 1 + crates/by_ir/src/ops.rs | 62 +- crates/by_ir/src/print.rs | 38 ++ crates/by_ir/src/verify.rs | 32 + crates/by_irbuild/src/lib.rs | 283 +++++++- crates/by_irbuild/src/tests.rs | 11 +- crates/by_opt/src/coalesce.rs | 1 + crates/by_opt/src/copy_propagation.rs | 5 + crates/by_opt/src/dead_registers.rs | 1 + crates/by_opt/src/dict_find.rs | 1 + crates/by_opt/src/fold.rs | 1 + crates/by_opt/src/infallible.rs | 39 +- crates/by_opt/src/lib.rs | 2 + crates/by_opt/src/refcount.rs | 1 + crates/by_opt/src/release_temporaries.rs | 1 + crates/by_opt/src/store_moves.rs | 1 + crates/by_opt/src/str_append.rs | 1 + crates/by_opt/src/str_item_compare.rs | 1 + crates/by_opt/src/unbox_counters.rs | 1 + crates/by_opt/src/unswitch.rs | 1 + crates/by_rt/include/by.h | 640 +++++++++++++++++- crates/ty/docs/configuration.md | 82 +++ crates/ty/src/by_commands.rs | 24 +- crates/ty/src/lib.rs | 2 + crates/ty/tests/by_e2e.rs | 108 +++ crates/ty/tests/cli/config_option.rs | 2 +- crates/ty_project/src/metadata/options.rs | 73 ++ .../development/compilation/index.md | 5 +- .../development/compilation/plan.md | 34 +- .../development/compilation/runtime.md | 165 ++++- ty.schema.json | 31 + 37 files changed, 2648 insertions(+), 81 deletions(-) create mode 100644 crates/by_ir/src/call_graph.rs diff --git a/crates/by_build/src/lib.rs b/crates/by_build/src/lib.rs index badf621531..26ef087c88 100644 --- a/crates/by_build/src/lib.rs +++ b/crates/by_build/src/lib.rs @@ -219,6 +219,22 @@ pub struct Options { /// figure becomes an upper bound without anyone noticing. the check runs once per /// module at import rather than per call pub verify_install: bool, + /// count compiled calls against python's recursion limit, as well as against the + /// stack. + /// + /// on by default, so a compiled recursion raises `RecursionError` at the depth the + /// interpreted one does. off, a recursion is stopped only where the stack would run + /// out, which saves the count on every call inside a cycle of compiled calls and + /// leaves `sys.setrecursionlimit` unobserved by them. neither setting can crash + pub follow_recursion_limit: bool, + /// have a compiled call to a function the module defines go straight to its native + /// entry, without asking whether the name still holds that function. + /// + /// off by default, so a module function rebound, deleted or patched from outside — + /// `mock.patch("mod.fn")` — or given another `__code__` or other defaults reaches the + /// module's own compiled callers as it reaches python's. on, they go on calling the + /// function the module was compiled with, which saves the question on every such call + pub bind_functions_early: bool, } impl Default for Options { @@ -231,6 +247,8 @@ impl Default for Options { fallback: None, recheck_licences: false, verify_install: true, + follow_recursion_limit: true, + bind_functions_early: false, } } } @@ -241,6 +259,7 @@ impl Options { by_irbuild::LowerOptions { language: self.language, recheck_licences: self.recheck_licences, + bind_functions_early: self.bind_functions_early, } } } @@ -308,6 +327,7 @@ fn finish( } module.verify_install = options.verify_install; + module.follow_recursion_limit = options.follow_recursion_limit; if options.require_native && !module.declined.is_empty() { bail!( @@ -590,6 +610,7 @@ mod tests { fallback_code: None, shims: None, verify_install: true, + follow_recursion_limit: true, }; let dir = std::env::temp_dir().join("by_build_refuses_test"); let _ = fs::remove_dir_all(&dir); diff --git a/crates/by_build/tests/differential.rs b/crates/by_build/tests/differential.rs index 8759bb0395..4afd7f5387 100644 --- a/crates/by_build/tests/differential.rs +++ b/crates/by_build/tests/differential.rs @@ -921,6 +921,57 @@ def _capture(fn, *args): return e return None +# the deepest argument `fn` still answers for under a recursion limit of `limit`. +# every frame between here and `fn` is the same in both legs, so the number is the +# count of frames `fn` itself spends +def _deepest(fn, limit): + import sys + saved = sys.getrecursionlimit() + sys.setrecursionlimit(limit) + try: + n = 0 + while True: + try: + fn(n) + except RecursionError: + return n - 1 + n += 1 + finally: + sys.setrecursionlimit(saved) + +# a code object `fn.__code__` can be swapped for, answering `value` however it is called. +# python refuses a code object whose free variables do not match the function's, and a +# compiled function is published as a forwarder closing over its native entry — so the +# replacement closes over as many names as `fn` does, which is none for the interpreted leg +def _code_answering(fn, value): + names = fn.__code__.co_freevars + lines = ['def outer():'] + lines += [' %s = None' % name for name in names] + lines += [' def inner(*args, **kwargs):'] + lines += [' %s' % name for name in names] + lines += [' return %r' % (value,)] + lines += [' return inner'] + scope = {} + exec('\\n'.join(lines), scope) + return scope['outer']().__code__ + +# what `fn` answers while `target.name` is patched to `value`, and after the patch is +# taken back, which is the everyday shape of rebinding a module's function from outside +def _patched(target, name, value, fn, *args): + from unittest import mock + with mock.patch.object(target, name, value): + inside = fn(*args) + return (inside, fn(*args)) + +# what `fn` answers when it runs on a thread of its own, with the stack a thread is given +def _on_a_thread(fn): + import threading + out = [] + thread = threading.Thread(target=lambda: out.append(fn())) + thread.start() + thread.join() + return out[0] if out else 'the thread died' + # call `fn` from a frame standing in a builtins namespace of its own: the real one # with `overrides` applied over it. what a *callee* resolves must not move, because # python binds a function's builtins to the module that defined it — so this changes @@ -1244,6 +1295,56 @@ fn agree_inner(tag: &str, source: &str, calls: &[&str], allow_declines: bool) { ); } +/// build `source` under `options` and hand back what each of `calls` printed from the +/// compiled leg alone +/// +/// for the tradeoffs the build makes on purpose, where the interpreted leg is not the +/// expected value. `None` where there is no toolchain to build with +fn compiled_answers( + tag: &str, + source: &str, + options: Options, + calls: &[&str], +) -> Option> { + let (python, toolchain) = environment()?; + let dir = diff_root().join(format!("by_diff_{tag}_c")); + let _ = std::fs::remove_dir_all(&dir); + let module = format!("by_diff_{tag}"); + let options = Options { + recheck_licences: recheck_licences(), + ..options + }; + let built = match build_source(source, module.as_str(), &toolchain, &dir, &options) { + Ok(built) => built, + Err(error) => { + assert!( + missing_toolchain(&error), + "{tag} failed to build: {error:#}" + ); + eprintln!("skipping {tag}: no working C toolchain ({error})"); + return None; + } + }; + assert!( + built.declined.is_empty(), + "{tag} declined functions the test expects to be compiled: {:?}", + built.declined + ); + Some( + calls + .iter() + .map(|call| { + let body = format!( + "import {module} as m\n\ + assert m.__file__.endswith(('.so', '.pyd')), m.__file__\n\ + {CAPTURE_HELPER}print(repr({call}))\n" + ); + run(&python, &dir, &body) + }) + .collect(), + ) +} + /// whether this run has every licensed call re-ask the lookup it skipped /// /// on unless `BY_LICENCE_RECHECK=0` says otherwise. every case in this file is a @@ -1725,6 +1826,429 @@ def sum_squares(n: int) -> int: ); } +/// every shape of recursion a compiled module can make: a function calling itself, two +/// calling each other, a method, a float body that nothing in it can make fail, a +/// nested function, a generator delegating to itself through `yield from` and through +/// a `for` loop, a recursion entered from another compiled function, and dunders that +/// reach themselves again through the object protocol +const RECURSIONS: &str = "\ +from collections.abc import Callable, Iterator + + +def depth(n: int) -> int: + if n == 0: + return 0 + return depth(n - 1) + 1 + + +def ping(n: int) -> int: + if n == 0: + return 0 + return pong(n - 1) + 1 + + +def pong(n: int) -> int: + if n == 0: + return 0 + return ping(n - 1) + 1 + + +class Deep: + def down(self, n: int) -> int: + if n == 0: + return 0 + return self.down(n - 1) + 1 + + +def method_depth(n: int) -> int: + return Deep().down(n) + + +def float_depth(n: float) -> float: + if n <= 0.0: + return 0.0 + return float_depth(n - 1.0) + 1.0 + + +def nested_depth(n: int) -> int: + def inner(k: int) -> int: + if k == 0: + return 0 + return inner(k - 1) + 1 + + return inner(n) + + +def delegated(n: int) -> Iterator[int]: + if n == 0: + yield 0 + return + yield from delegated(n - 1) + + +def looped(n: int) -> Iterator[int]: + if n == 0: + yield 0 + return + for value in looped(n - 1): + yield value + + +def entered(n: int) -> int: + return depth(n) + + +def through(n: int, f: Callable[[int], int]) -> int: + if n == 0: + return 0 + return f(n - 1) + 1 + + +class Chain: + def __init__(self, n: int) -> None: + self.n = n + + def __getitem__(self, k: int) -> int: + if k == 0: + return 0 + return self[k - 1] + 1 + + def __len__(self) -> int: + if self.n == 0: + return 0 + self.n = self.n - 1 + return len(self) + 1 +"; + +#[test] +fn a_native_recursion_past_the_stack_raises_rather_than_crashing() { + // a compiled frame is a C frame, and nothing counted them: each of these ran off + // the end of the stack and the process died with a segfault where python raises + // `RecursionError`. a crash is the one answer no build may give + agree_python( + "recdeep", + RECURSIONS, + &[ + "_capture(m.depth, 10**6)", + "_capture(m.ping, 10**6)", + "_capture(m.method_depth, 10**6)", + "_capture(m.float_depth, 10.0**6)", + "_capture(m.nested_depth, 10**6)", + "_capture(m.entered, 10**6)", + "_capture(lambda: sum(m.delegated(10**5)))", + "_capture(lambda: sum(m.looped(10**5)))", + "_capture(lambda: m.Chain(1)[10**5])", + "_capture(lambda: len(m.Chain(10**5)))", + // the exception leaves the recursion as an ordinary one: a handler further + // out catches it and the module goes on answering + "(_capture(m.depth, 10**6).__class__.__name__, m.depth(10))", + ], + ); +} + +#[test] +fn a_native_recursion_follows_the_recursion_limit() { + // python raises at the frame the limit allows and not one later, and a compiled + // recursion counts its calls against the same limit. a published function is still + // entered through a python frame, which the interpreter counts, so the native call + // behind it counts nothing more — counting it as well raised one frame early. + // + // `nested_depth` is left out on purpose: its first call to `inner` is a native call + // that goes round no cycle of native calls, and such a call is not counted, so the + // compiled leg reaches one frame deeper. runtime.md lists that difference + agree_python( + "reclimit", + RECURSIONS, + &[ + "_deepest(m.depth, 100)", + "_deepest(m.ping, 100)", + "_deepest(m.method_depth, 100)", + "_deepest(lambda n: m.float_depth(float(n)), 100)", + "_deepest(m.entered, 100)", + "_deepest(lambda n: sum(m.delegated(n)), 100)", + "_deepest(lambda n: sum(m.looped(n)), 100)", + "_deepest(lambda n: m.through(n, m.depth), 100)", + "_deepest(m.Deep().down, 100)", + // not `Chain.__getitem__`: 3.14 lets a python `__getitem__` it has + // specialised the subscript for go thirty frames past the limit, so the + // interpreter answers differently from itself on 3.13 + "_deepest(lambda n: len(m.Chain(n)), 100)", + ], + ); +} + +#[test] +fn a_raised_recursion_limit_still_stops_at_the_stack() { + // python frames live on the heap, so python answers these where a native frame + // cannot: the limit allows it and the C stack does not. the compiled leg raises + // instead, and the same holds on a thread whose stack is smaller than the main one + let Some(answers) = compiled_answers( + "recstack", + RECURSIONS, + Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + &[ + "(__import__('sys').setrecursionlimit(10**7), type(_capture(m.depth, 10**6)).__name__)[1]", + "(__import__('sys').setrecursionlimit(10**7), type(_capture(m.ping, 10**6)).__name__)[1]", + "(__import__('sys').setrecursionlimit(10**7), type(_capture(lambda: sum(m.looped(10**6)))).__name__)[1]", + "_on_a_thread(lambda: type(_capture(m.depth, 10**6)).__name__)", + ], + ) else { + return; + }; + assert_eq!( + answers, + [ + "'RecursionError'", + "'RecursionError'", + "'RecursionError'", + "'RecursionError'" + ] + ); +} + +#[test] +fn a_build_not_following_the_recursion_limit_still_stops_at_the_stack() { + // the configuration that counts no frames against the limit leaves a lowered limit + // unobserved, which is the tradeoff it makes, and still never crashes + let Some(answers) = compiled_answers( + "recstackonly", + RECURSIONS, + Options { + language: by_irbuild::Language::Python, + follow_recursion_limit: false, + ..Options::default() + }, + &[ + "(__import__('sys').setrecursionlimit(100), m.depth(200))[1]", + "(__import__('sys').setrecursionlimit(100), m.method_depth(200))[1]", + "(__import__('sys').setrecursionlimit(100), sum(m.looped(200)))[1]", + "type(_capture(m.depth, 10**6)).__name__", + "type(_capture(m.ping, 10**6)).__name__", + "type(_capture(lambda: sum(m.delegated(10**6)))).__name__", + ], + ) else { + return; + }; + assert_eq!( + answers, + [ + "200", + "200", + "0", + "'RecursionError'", + "'RecursionError'", + "'RecursionError'" + ] + ); +} + +/// module functions whose defaults and code a program reaches in and replaces +const REASSIGNED: &str = "\ +def scaled(a: int, *, scale: int = 1) -> int: + return a * scale + + +def shifted(a: int, b: int = 1) -> int: + return a + b + + +def summed(a: int, b: int) -> int: + return a + b + + +def call_scaled() -> int: + return scaled(2) + + +def call_shifted() -> int: + return shifted(2) + + +def call_summed(a: int) -> int: + return summed(a, 3) + + +def loop_shifted(n: int) -> int: + total = 0 + i = 0 + while i < n: + total = total + shifted(i) + i = i + 1 + return total + + +async def added(a: int, b: int = 1) -> int: + return a + b + + +async def await_added() -> int: + return await added(2) +"; + +#[test] +fn a_reassigned_default_reaches_python_and_compiled_callers() { + // python binds a missing argument from the function object's `__defaults__` and + // `__kwdefaults__` as the call is made, so assigning either changes every later + // call. the published function forwards to a wrapper with the defaults compiled + // in, and a compiled caller filled them in itself — both kept answering with the + // old ones + agree_python( + "defaults", + REASSIGNED, + &[ + "(setattr(m.scaled, '__kwdefaults__', {'scale': 10}), m.scaled(2), m.call_scaled())", + "(setattr(m.shifted, '__defaults__', (100,)), m.shifted(2), m.call_shifted())", + "(m.loop_shifted(3), setattr(m.shifted, '__defaults__', (100,)), m.loop_shifted(3))", + // a function that had none takes the ones it is given + "(setattr(m.summed, '__defaults__', (5,)), m.summed(1), m.call_summed(1))", + // and one whose defaults are taken away refuses the call that needed them + "(setattr(m.shifted, '__defaults__', None), repr(_capture(m.shifted, 2)), repr(_capture(m.call_shifted)))", + "(setattr(m.scaled, '__kwdefaults__', None), repr(_capture(m.scaled, 2)), repr(_capture(m.call_scaled)))", + // a coroutine a compiled caller awaits in the same expression runs its body + // directly, and awaits what the function object hands back instead + "(asyncio.run(m.await_added()), setattr(m.added, '__defaults__', (100,)), asyncio.run(m.await_added()))", + ], + ); +} + +#[test] +fn a_swapped_code_object_reaches_compiled_callers() { + // the published function runs whatever code it holds, so a python caller already + // saw a swapped `__code__` — a compiled caller went straight to the native entry + // and never looked + agree_python( + "codeswap", + REASSIGNED, + &[ + "(m.call_summed(1), setattr(m.summed, '__code__', _code_answering(m.summed, 100)), m.summed(1, 2), m.call_summed(1))", + "(m.loop_shifted(3), setattr(m.shifted, '__code__', _code_answering(m.shifted, 7)), m.loop_shifted(3))", + ], + ); +} + +/// module functions a program rebinds, deletes and patches from outside +const REBOUND: &str = "\ +def add(a: int, b: int) -> int: + return a + b + + +def run(n: int) -> int: + total = 0 + i = 0 + while i < n: + total = add(total, i) + i = i + 1 + return total + + +def depth(n: int) -> int: + if n == 0: + return 0 + return depth(n - 1) + 1 + + +def scaled(a: int, *, by: int = 2) -> int: + return a * by + + +def call_scaled(n: int) -> int: + return scaled(n, by=3) +"; + +#[test] +fn a_rebound_module_function_reaches_compiled_callers() { + // python calls whatever the name holds when the call is made. a compiled caller + // went straight to the native entry, so a function replaced from outside — by an + // assignment, through the namespace, or by `mock.patch` — went on answering as + // the original, and a deleted one answered where python raises `NameError` + agree_python( + "rebound", + REBOUND, + &[ + "(m.run(3), setattr(m, 'add', lambda a, b: 100), m.run(3))", + "(m.run(3), m.__dict__.__setitem__('add', lambda a, b: a - b), m.run(3))", + "(m.run(3), delattr(m, 'add'), repr(_capture(m.run, 3)))", + "_patched(m, 'add', lambda a, b: 7, m.run, 3)", + // putting the function back makes the direct call right again + "(lambda saved: (setattr(m, 'add', lambda a, b: 100), m.run(2), setattr(m, 'add', saved), m.run(3)))(m.add)", + // a recursion calls the function its name holds on each call it makes + "(lambda held: (setattr(m, 'depth', lambda n: 1000), held(3)))(m.depth)[1]", + // keywords reach the replacement by name + "(setattr(m, 'scaled', lambda a, by: a * 100 + by), m.call_scaled(5))", + // a write to any other name leaves the direct call standing + "(setattr(m, 'unrelated', 5), m.__dict__.pop('unrelated'), m.run(3))", + ], + ); +} + +#[test] +fn a_build_binding_functions_early_goes_on_calling_what_it_compiled() { + // the configuration that assumes the module's functions are never rebound asks + // nothing at the call, which is the tradeoff it makes: a replacement reaches a call + // made from python and not one the module's own compiled code makes + let Some(answers) = compiled_answers( + "reboundearly", + REBOUND, + Options { + language: by_irbuild::Language::Python, + bind_functions_early: true, + ..Options::default() + }, + &[ + "(setattr(m, 'add', lambda a, b: 100), m.add(1, 2), m.run(3))", + "(delattr(m, 'add'), m.run(3))", + "(setattr(m.add, '__code__', _code_answering(m.add, 7)), m.run(3))", + // the function's own python entry still binds from the defaults it holds + "(setattr(m.scaled, '__kwdefaults__', {'by': 10}), m.scaled(1))", + ], + ) else { + return; + }; + assert_eq!( + answers, + ["(None, 100, 3)", "(None, 3)", "(None, 3)", "(None, 10)"] + ); +} + +#[test] +fn a_rebound_function_handed_an_unboxed_buffer_is_refused_loudly() { + // a list the caller holds as a buffer has no list object to hand a replacement, and + // a copy would be a different list — so the call raises, as a rebound `len` on a + // buffer does, rather than calling the replacement with a copy + let Some(answers) = compiled_answers( + "reboundbuffer", + "\ +def total(xs: list[float]) -> float: + out = 0.0 + i = 0 + while i < len(xs): + out = out + xs[i] + i = i + 1 + return out + + +def built(n: int) -> float: + xs = [0.0] + i = 0 + while i < n: + xs.append(i * 0.5) + i = i + 1 + return total(xs) +", + Options::default(), + &[ + "m.built(3)", + "(setattr(m, 'total', lambda xs: len(xs)), type(_capture(m.built, 3)).__name__)[1]", + ], + ) else { + return; + }; + assert_eq!(answers, ["1.5", "'RuntimeError'"]); +} + #[test] fn for_loops_over_range_agree() { agree( diff --git a/crates/by_build/tests/end_to_end.rs b/crates/by_build/tests/end_to_end.rs index d629a15061..0cdb60189c 100644 --- a/crates/by_build/tests/end_to_end.rs +++ b/crates/by_build/tests/end_to_end.rs @@ -153,6 +153,7 @@ fn arith_module() -> ModuleIr { fallback_code: None, shims: None, verify_install: true, + follow_recursion_limit: true, } } @@ -220,6 +221,7 @@ fn fib_module() -> ModuleIr { fallback_code: None, shims: None, verify_install: true, + follow_recursion_limit: true, } } @@ -371,6 +373,7 @@ fn division_floors_like_python_and_raises_on_zero() { fallback_code: None, shims: None, verify_install: true, + follow_recursion_limit: true, }; let Some(dir) = built(&module, &toolchain, "divzero") else { return; @@ -430,6 +433,7 @@ fn floats_are_unboxed_and_exclude_int() { fallback_code: None, shims: None, verify_install: true, + follow_recursion_limit: true, }; let Some(dir) = built(&module, &toolchain, "float") else { return; @@ -504,6 +508,7 @@ fn calls_between_compiled_functions_stay_native() { fallback_code: None, shims: None, verify_install: true, + follow_recursion_limit: true, }; let Some(dir) = built(&module, &toolchain, "call") else { return; diff --git a/crates/by_codegen_c/src/lib.rs b/crates/by_codegen_c/src/lib.rs index 6209cbcdc7..e21108fbc3 100644 --- a/crates/by_codegen_c/src/lib.rs +++ b/crates/by_codegen_c/src/lib.rs @@ -163,7 +163,12 @@ pub fn emit_module(module: &ModuleIr) -> String { if rechecks_licences(module) { out.push_str("#define BY_LICENCE_RECHECK 1\n"); } + if !module.follow_recursion_limit { + out.push_str("#define BY_RECURSION_STACK_ONLY 1\n"); + } out.push_str("#include \"by.h\"\n\n"); + let recursive = by_ir::call_graph::recursive_functions(module); + RECURSIVE.with_borrow_mut(|slot| slot.clone_from(&recursive)); // a native function resolves an unowned name through this, the way // `LOAD_GLOBAL` does. declared here because the bodies below read it; set // once, from the exec slot @@ -311,6 +316,16 @@ pub fn emit_module(module: &ModuleIr) -> String { for function in module.all_functions() { let _ = writeln!(out, "{};", signature(module, function)); + if is_recursive(function) { + let _ = writeln!(out, "{};", counted_signature(module, function)); + } + } + for name in published_functions(module) { + let _ = writeln!( + out, + "static ByFunctionSite {} = BY_FUNCTION_SITE_INIT;", + function_site(name) + ); } // the wrappers are declared before the method tables that name them, and the // tables before the bodies — because a `MakeClosure` in a body names a nested @@ -5195,7 +5210,35 @@ fn undefined(module: &ModuleIr, ty: &RType) -> String { } } +/// the signature the body is emitted under +/// +/// a function on a cycle of native calls is handed the depth its caller counted as its +/// first argument, under a symbol of its own — see [`counted_signature`] for the entry +/// every other caller keeps fn signature(module: &ModuleIr, function: &Function) -> String { + let mut params: Vec = function + .params() + .iter() + .enumerate() + .map(|(index, decl)| format!("{} {}", ctype(module, &decl.ty), local(RegisterId(index)))) + .collect(); + let symbol = if is_recursive(function) { + params.insert(0, "ByDepth by_depth".to_string()); + depth_symbol(module, function) + } else { + function.native_symbol(module.name.dotted()) + }; + let params = if params.is_empty() { + "void".to_string() + } else { + params.join(", ") + }; + format!("static {} {symbol}({params})", ctype(module, &function.ret)) +} + +/// the entry a function on a cycle keeps under its ordinary symbol, for every caller +/// that is not itself on a cycle: it looks the depth up and counts the call +fn counted_signature(module: &ModuleIr, function: &Function) -> String { let params = if function.param_count == 0 { "void".to_string() } else { @@ -5210,13 +5253,80 @@ fn signature(module: &ModuleIr, function: &Function) -> String { .join(", ") }; format!( - "static {} {}({})", + "static inline {} {}({params})", ctype(module, &function.ret), function.native_symbol(module.name.dotted()), - params ) } +/// the module functions published under their own names through a forwarder, each of +/// which a compiled caller reaches directly while [`Op::FunctionStands`] says it may +fn published_functions(module: &ModuleIr) -> Vec<&str> { + module + .shims + .as_ref() + .map(|shims| shims.functions.iter().map(String::as_str).collect()) + .unwrap_or_default() +} + +/// whether any compiled call asks whether the function published under `name` still +/// stands, which is what the namespace has to be watched for +fn asks_whether_it_stands(module: &ModuleIr, name: &str) -> bool { + module + .all_functions() + .flat_map(|function| &function.blocks) + .flat_map(|block| &block.ops) + .any(|op| matches!(op, Op::ResolveFunction { name: asked, .. } if asked == name)) +} + +/// the static a published function's [`Op::FunctionStands`] reads +fn function_site(name: &str) -> String { + format!("by_fs_{}", mangle(name)) +} + +/// the symbol a function on a cycle's body is emitted under +fn depth_symbol(module: &ModuleIr, function: &Function) -> String { + format!( + "byd{}", + function + .native_symbol(module.name.dotted()) + .trim_start_matches("by") + ) +} + +/// the counted entry itself: take a frame from the depth, run the body, give it back +fn emit_counted_entry(module: &ModuleIr, function: &Function) -> String { + let arguments = std::iter::once("by_depth".to_string()) + .chain((0..function.param_count).map(|index| local(RegisterId(index)))) + .collect::>() + .join(", "); + format!( + "{signature} {{ + ByDepth by_depth = By_DepthHere(); + if (BY_UNLIKELY(By_DepthEnter(by_depth))) return {undefined}; + {ret} by_result = {body}({arguments}); + By_DepthLeave(by_depth); + return by_result; + }} +", + signature = counted_signature(module, function), + undefined = undefined(module, &function.ret), + ret = ctype(module, &function.ret), + body = depth_symbol(module, function), + ) +} + +thread_local! { + /// the qualified names of the module's functions on a cycle of native calls, for the + /// duration of one emission + static RECURSIVE: std::cell::RefCell> = + std::cell::RefCell::new(HashSet::new()); +} + +fn is_recursive(function: &Function) -> bool { + RECURSIVE.with_borrow(|recursive| recursive.contains(&function.qualified_name())) +} + fn local(id: RegisterId) -> String { format!("r{}", id.0) } @@ -5391,6 +5501,9 @@ fn emit_function(module: &ModuleIr, function: &Function) -> String { } out.push_str("}\n"); + if is_recursive(function) { + out.push_str(&emit_counted_entry(module, function)); + } out } @@ -6320,6 +6433,59 @@ fn emit_op(module: &ModuleIr, function: &Function, op: &Op, error_target: ErrorE local(*dest), ) } + // a name this module publishes no forwarder for is never reached directly: its + // resolution is always the object under the name, and its test always says no + Op::ResolveFunction { dest, name } => { + let Some(decl) = function.register(*dest) else { + return String::new(); + }; + let target = local(*dest); + let release = dec_ref(&decl.ty, &target).unwrap_or_default(); + let label = error_label(error_target); + if published_functions(module).contains(&name.as_str()) { + // NULL is the answer that says the native entry stands, so a failure is + // told apart by an address no object can have + format!( + " {{ PyObject *by_t = By_ResolveFunction(&{site}, by_module_dict, {name});\n \ + if (BY_UNLIKELY(by_t == BY_RESOLVE_FAILED)) goto {label};\n \ + {release} {target} = by_t; }}\n", + site = function_site(name), + name = c_string(name), + ) + } else { + format!( + " {{ static PyObject *by_rf = NULL;\n \ + if (by_rf == NULL) by_rf = By_InternedStr({}, {});\n \ + if (by_rf == NULL) goto {label};\n \ + PyObject *by_t = By_LookupGlobal(by_module_dict, by_rf);\n \ + if (by_t == NULL) goto {label};\n \ + {release} {target} = by_t; }}\n", + c_string(name), + name.len(), + ) + } + } + Op::FunctionStands { dest, src, name } => { + if published_functions(module).contains(&name.as_str()) { + format!( + " {} = (char)BY_LIKELY({} == NULL && By_FunctionStands(&{}));\n", + local(*dest), + value_expr(src), + function_site(name) + ) + } else { + format!(" {} = 0;\n", local(*dest)) + } + } + Op::FunctionCallee { dest, src, name } => { + let forwarder = if published_functions(module).contains(&name.as_str()) { + format!("{}.forwarder", function_site(name)) + } else { + "NULL".to_string() + }; + let expr = format!("By_FunctionCallee({}, {forwarder})", value_expr(src)); + assign_checked(module, function, *dest, &expr, error_target) + } Op::MethodStands { dest, src, @@ -6773,19 +6939,47 @@ fn emit_op(module: &ModuleIr, function: &Function, op: &Op, error_target: ErrorE }) .collect::>() .join(", "); + // a call from one function on a cycle to another hands on the depth it was + // given and counts itself around the call, which is the whole of what makes + // a recursion stop where python's does. every other caller takes the counted + // entry, which looks the depth up for itself + let counted = is_recursive(function) && target.is_some_and(is_recursive); let symbol = match target { + Some(target) if counted => depth_symbol(module, target), Some(target) => target.native_symbol(module.name.dotted()), None => format!("by_{}_{}", mangle(module.name.dotted()), mangle(callee)), }; - let call = format!("{symbol}({args})"); + let call = match (counted, target) { + (true, Some(target)) => { + let args = if args.is_empty() { + "by_depth".to_string() + } else { + format!("by_depth, {args}") + }; + format!( + "({{ {ret} by_called = {symbol}({args}); By_DepthLeave(by_depth); by_called; }})", + ret = ctype(module, &target.ret) + ) + } + _ => format!("{symbol}({args})"), + }; + let enter = if counted { + format!( + " if (BY_UNLIKELY(By_DepthEnter(by_depth))) goto {};\n", + error_label(error_target) + ) + } else { + String::new() + }; let fallible = target.is_none_or(|target| target.convention.can_fail()); - match dest { + let body = match dest { Some(dest) if fallible => { assign_checked(module, function, *dest, &call, error_target) } Some(dest) => assign_owned(module, function, *dest, &call), None => format!(" (void){call};\n"), - } + }; + enter + &body } Op::IntToFloat { dest, src } => { let expr = format!("By_TaggedToDouble({})", value_expr(src)); @@ -7571,6 +7765,46 @@ fn emit_op(module: &ModuleIr, function: &Function, op: &Op, error_target: ErrorE ); assign_checked(module, function, *dest, &call, error_target) } + Op::CallThrough { + dest, + callee, + args, + keywords, + } => { + // the arguments go through a variadic helper rather than an array in this + // frame: an array is what gives a function a stack protector, and this is + // the rare arm of a call whose common arm should not pay for one + let mut out = String::new(); + let names = if keywords.is_empty() { + "NULL".to_string() + } else { + let _ = writeln!( + out, + " {{ static PyObject *by_kwnames = NULL;\n \ + if (by_kwnames == NULL) by_kwnames = By_KeywordNames({}, {});\n \ + if (by_kwnames == NULL) goto {};", + keywords.len(), + keywords + .iter() + .map(|keyword| c_string(keyword)) + .collect::>() + .join(", "), + error_label(error_target), + ); + "by_kwnames".to_string() + }; + if keywords.is_empty() { + out.push_str(" {\n"); + } + let arguments = std::iter::once(value_expr(callee)) + .chain([names, (args.len() - keywords.len()).to_string()]) + .chain(args.iter().map(value_expr)) + .collect::>() + .join(", "); + let _ = writeln!(out, " PyObject *by_t = By_CallThrough({arguments});"); + out.push_str(&commit_checked(function, *dest, error_target)); + out + } Op::CallValue { dest, callee, args } => { let argv = args.iter().map(value_expr).collect::>().join(", "); let mut out = String::new(); @@ -8107,6 +8341,22 @@ fn emit_op(module: &ModuleIr, function: &Function, op: &Op, error_target: ErrorE for index in path { let _ = write!(target, ".f{index}"); } + // what a function's resolution holds is NULL wherever the native entry stood + // in, which is the path that matters, and saying so outright keeps that path + // free of a call to a release clang has outlined from a body grown large with + // the call made in the entry's place + let resolution = path.is_empty() + && function + .blocks + .iter() + .flat_map(|block| &block.ops) + .any(|op| matches!(op, Op::ResolveFunction { dest, .. } if dest == id)); + if resolution { + return format!( + " {{ PyObject *by_old = {target}; {target} = NULL; \ + if (by_old != NULL) Py_DECREF(by_old); }}\n" + ); + } dec_ref(ty, "by_old").map_or_else(String::new, |release| { format!( " {{ {} by_old = {target}; {target} = {}; {release} }}\n", @@ -8527,6 +8777,19 @@ fn emit_wrapper(module: &ModuleIr, function: &Function, is_method: bool) -> Stri if !is_method { out.push_str(" (void)self;\n"); } + // the defaults written into this wrapper are the ones the function was compiled + // with, and once the published function's own have been reassigned they are not + // the ones python would bind — see `By_CallMoved` + if function.owner.is_none() + && function.nested.is_none() + && published_functions(module).contains(&function.name.as_str()) + { + let _ = writeln!( + out, + " if (BY_UNLIKELY({site}.flags.moved)) return By_CallMoved(&{site}, args, nargs, kwnames);", + site = function_site(&function.name) + ); + } // python counts a method's `self` in an arity error, and a nested function has no // receiver it can see let counted = i32::from(is_method && function.nested.is_none()); @@ -8668,16 +8931,46 @@ fn emit_wrapper(module: &ModuleIr, function: &Function, is_method: bool) -> Stri ); } - let args = (0..function.param_count) + let mut args: Vec = (0..function.param_count) .map(|index| format!("a{index}")) - .collect::>() - .join(", "); + .collect(); + // the interpreter pushes no frame for a call into this wrapper, and python would have + // pushed one for the body: a method called from python, a dunder a slot reaches, a + // nested function called as a value. so the call is counted here, as the frame python + // would have counted — and without it a recursion that goes round through the object + // protocol, `self[k - 1]` inside `__getitem__`, ran off the stack. a module-level + // function is the exception: the forwarder the module publishes is a python frame, + // already counted, and counting again raised `RecursionError` a frame early. its body + // is still handed the depth where it is on a cycle, so the calls round the cycle count + let counts_the_call = function.owner.is_some() || function.nested.is_some(); + let body = if is_recursive(function) { + depth_symbol(module, function) + } else { + function.native_symbol(module.name.dotted()) + }; + if is_recursive(function) { + let depth = if counts_the_call { + "by_depth" + } else { + "By_DepthHere()" + }; + args.insert(0, depth.to_string()); + } + let arguments = args.join(", "); + if counts_the_call { + out.push_str( + " ByDepth by_depth = By_DepthHere();\n\ + \x20 if (BY_UNLIKELY(By_DepthEnter(by_depth))) goto by_wrap_error;\n", + ); + } let _ = writeln!( out, - " {} by_result = {}({args});", + " {} by_result = {body}({arguments});", ctype(module, &function.ret), - function.native_symbol(module.name.dotted()) ); + if counts_the_call { + out.push_str(" By_DepthLeave(by_depth);\n"); + } // a resume hands back nothing both when the frame returned and when it raised, and // the second is the only one a python caller can be given. anything reaching the // step through this wrapper rather than through the iterator protocol still gets @@ -10095,9 +10388,22 @@ fn emit_module_init(module: &ModuleIr) -> String { format!( " {{ int by_installed = By_InstallForwarders(dict, by_built, by_forwarded, {}, {});\n\ \x20 Py_DECREF(by_built);\n\ - \x20 if (by_installed < 0) return -1; }}\n", + \x20 if (by_installed < 0) return -1; }}\n{}", shims.functions.len(), - c_string(&shims.installer) + c_string(&shims.installer), + shims + .functions + .iter() + .fold(String::new(), |mut arms, name| { + let _ = writeln!( + arms, + " if (By_ArmFunctionSite(&{}, dict, {}, {}) < 0) return -1;", + function_site(name), + c_string(name), + i32::from(asks_whether_it_stands(module, name)) + ); + arms + }) ), ), None => (String::new(), String::new()), @@ -10211,6 +10517,7 @@ mod tests { fallback_code: None, shims: None, verify_install: true, + follow_recursion_limit: true, } } @@ -11479,6 +11786,7 @@ mod tests { fallback_code: None, shims: None, verify_install: true, + follow_recursion_limit: true, }; let c = emit_module(&module); // the forward declaration precedes the body, so take the last split @@ -11843,6 +12151,7 @@ mod tests { fallback_code: None, shims: None, verify_install: true, + follow_recursion_limit: true, }; let c = emit_module(&module); assert!(c.contains("PyMODINIT_FUNC PyInit_app(void)")); diff --git a/crates/by_ir/src/call_graph.rs b/crates/by_ir/src/call_graph.rs new file mode 100644 index 0000000000..47df3a46c6 --- /dev/null +++ b/crates/by_ir/src/call_graph.rs @@ -0,0 +1,189 @@ +//! which compiled functions can reach themselves through native calls +//! +//! a native call pushes a C frame and no python one, so nothing the interpreter +//! counts stands between a function and the stack running out. a chain of such calls +//! with no cycle in it is as deep as the module is long, and the frames around it are +//! counted by whatever entered it; a cycle is as deep as its arguments say. so a +//! function on a cycle is the one place a depth has to be counted, and whether it is on +//! one is a question about the call graph alone — what the arguments are typed as has +//! nothing to add + +use std::collections::{HashMap, HashSet}; + +use crate::function::{ModuleIr, qualify}; +use crate::ops::Op; + +/// the qualified name of every function that lies on a cycle of native calls, +/// including one that calls itself +pub fn recursive_functions(module: &ModuleIr) -> HashSet { + let names: Vec = module + .all_functions() + .map(crate::function::Function::qualified_name) + .collect(); + let index: HashMap<&str, usize> = names + .iter() + .enumerate() + .map(|(at, name)| (name.as_str(), at)) + .collect(); + let edges: Vec> = module + .all_functions() + .map(|function| { + let mut callees: Vec = function + .blocks + .iter() + .flat_map(|block| &block.ops) + .filter_map(|op| match op { + Op::CallNative { owner, callee, .. } => index + .get(qualify(owner.as_deref(), callee).as_str()) + .copied(), + _ => None, + }) + .collect(); + callees.sort_unstable(); + callees.dedup(); + callees + }) + .collect(); + + let mut recursive = HashSet::new(); + for component in strongly_connected(&edges) { + let on_a_cycle = match component.as_slice() { + [only] => edges[*only].contains(only), + _ => true, + }; + if on_a_cycle { + recursive.extend(component.into_iter().map(|at| names[at].clone())); + } + } + recursive +} + +/// tarjan's strongly connected components, iteratively +/// +/// a module is free to be one long chain of calls, and walking it recursively would +/// put the compiler itself at the mercy of the stack this analysis exists to protect +fn strongly_connected(edges: &[Vec]) -> Vec> { + const UNVISITED: usize = usize::MAX; + let count = edges.len(); + let mut order = vec![UNVISITED; count]; + let mut low = vec![0; count]; + let mut on_stack = vec![false; count]; + let mut stack: Vec = Vec::new(); + let mut components = Vec::new(); + let mut next = 0; + + for root in 0..count { + if order[root] != UNVISITED { + continue; + } + // each entry is a node and how many of its edges have been followed + let mut walk: Vec<(usize, usize)> = vec![(root, 0)]; + order[root] = next; + low[root] = next; + next += 1; + stack.push(root); + on_stack[root] = true; + while let Some(frame) = walk.last_mut() { + let node = frame.0; + if let Some(&target) = edges[node].get(frame.1) { + frame.1 += 1; + if order[target] == UNVISITED { + order[target] = next; + low[target] = next; + next += 1; + stack.push(target); + on_stack[target] = true; + walk.push((target, 0)); + } else if on_stack[target] { + low[node] = low[node].min(order[target]); + } + continue; + } + walk.pop(); + if let Some(&(parent, _)) = walk.last() { + low[parent] = low[parent].min(low[node]); + } + if low[node] == order[node] { + let mut component = Vec::new(); + while let Some(member) = stack.pop() { + on_stack[member] = false; + component.push(member); + if member == node { + break; + } + } + components.push(component); + } + } + } + components +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::builder::FunctionBuilder; + use crate::ops::{Terminator, Value}; + use crate::rtype::RType; + + fn calling(name: &str, callees: &[&str]) -> crate::function::Function { + let mut builder = FunctionBuilder::new(name, RType::NONE); + for callee in callees { + builder.push(Op::CallNative { + dest: None, + owner: None, + callee: (*callee).to_string(), + args: Vec::new(), + }); + } + builder.terminate(Terminator::Return(Value::None)); + builder.finish() + } + + fn module_of(functions: Vec) -> ModuleIr { + let mut module = ModuleIr::new("app"); + module.functions = functions; + module + } + + #[test] + fn a_function_calling_itself_is_on_a_cycle() { + let module = module_of(vec![ + calling("depth", &["depth"]), + calling("entry", &["depth"]), + ]); + assert_eq!( + recursive_functions(&module), + HashSet::from(["depth".to_string()]) + ); + } + + #[test] + fn functions_calling_each_other_are_on_one_cycle_and_a_chain_is_not() { + let module = module_of(vec![ + calling("ping", &["pong"]), + calling("pong", &["ping", "leaf"]), + calling("leaf", &[]), + calling("top", &["middle"]), + calling("middle", &["leaf"]), + ]); + assert_eq!( + recursive_functions(&module), + HashSet::from(["ping".to_string(), "pong".to_string()]) + ); + } + + #[test] + fn a_long_chain_of_calls_is_walked_without_recursion() { + let names: Vec = (0..200_000).map(|at| format!("f{at}")).collect(); + let functions = names + .iter() + .enumerate() + .map(|(at, name)| match names.get(at + 1) { + Some(next) => calling(name, &[next.as_str()]), + None => calling(name, &["f0"]), + }) + .collect(); + assert_eq!(recursive_functions(&module_of(functions)).len(), 200_000); + } +} diff --git a/crates/by_ir/src/function.rs b/crates/by_ir/src/function.rs index f83b8592eb..524f33ed3c 100644 --- a/crates/by_ir/src/function.rs +++ b/crates/by_ir/src/function.rs @@ -1047,6 +1047,12 @@ pub struct ModuleIr { /// else here can see one — see `By_VerifyClass` in the runtime header for the four /// questions it asks and why each of them is a wrong answer already shipped pub verify_install: bool, + /// whether a call inside a cycle of native calls counts against python's recursion + /// limit as well as against the stack. + /// + /// the stack is watched either way, so no setting lets a recursion run off the end of + /// it — see `by_build::Options::follow_recursion_limit` + pub follow_recursion_limit: bool, } /// what the artefact has to do to publish a module's forwarders @@ -1284,6 +1290,7 @@ impl ModuleIr { fallback_code: None, shims: None, verify_install: true, + follow_recursion_limit: true, } } diff --git a/crates/by_ir/src/lib.rs b/crates/by_ir/src/lib.rs index 023f9dfeeb..e026b2bd65 100644 --- a/crates/by_ir/src/lib.rs +++ b/crates/by_ir/src/lib.rs @@ -8,6 +8,7 @@ //! recorded here rather than re-derived by reaching back into ty. pub mod builder; +pub mod call_graph; pub mod function; pub mod ops; pub mod print; diff --git a/crates/by_ir/src/ops.rs b/crates/by_ir/src/ops.rs index c5165497c5..e92c67b875 100644 --- a/crates/by_ir/src/ops.rs +++ b/crates/by_ir/src/ops.rs @@ -434,6 +434,33 @@ pub enum Op { /// wanted where it is not the builtin. what it costs at runtime is in /// `By_BuiltinStands` BuiltinStands { dest: RegisterId, name: String }, + /// what a call to the module function `name` has to go through where its native + /// entry may not stand in for the function object, read where python reads the name + /// + /// python calls whatever function object the name holds when the call is made, and + /// binds a missing argument from the defaults that object holds then, so the native + /// entry with its compiled defaults is only the same call while the object is the + /// one this module published and nothing about it has been reassigned. while that + /// holds this is NULL and costs one load; otherwise it is the object under the name, + /// resolved as a global read is, which raises `NameError` for a name bound nowhere — + /// see `By_ResolveFunction` + ResolveFunction { dest: RegisterId, name: String }, + /// whether the call [`Self::ResolveFunction`] read `src` for may go straight to the + /// native entry: the read found nothing to call, and nothing the arguments did since + /// has moved the function + FunctionStands { + dest: RegisterId, + src: Value, + name: String, + }, + /// the object a call [`Self::FunctionStands`] turned away goes through: the one + /// [`Self::ResolveFunction`] read, or the function this module published under the + /// name where the read found it still standing + FunctionCallee { + dest: RegisterId, + src: Value, + name: String, + }, /// whether a read or a write of `name` on `src` may go straight to the property /// half this module emitted for `class`, rather than round the descriptor protocol /// @@ -939,6 +966,19 @@ pub enum Op { callee: Value, args: Vec, }, + /// a call through the function object a direct call to a module function was + /// turned away from — see [`Self::FunctionStands`] + /// + /// the same call as [`Self::CallValue`], with keywords: the last `keywords.len()` + /// arguments are passed under those names. it is the rare arm of a hot call, so it + /// is written to cost that call nothing, where a call through a value is written + /// to be fast itself + CallThrough { + dest: RegisterId, + callee: Value, + args: Vec, + keywords: Vec, + }, /// `receiver.name(args)` through the object protocol CallMethod { dest: RegisterId, @@ -1309,6 +1349,9 @@ impl Op { | Self::MatchAttr { .. } | Self::MethodStands { .. } | Self::BuiltinStands { .. } + | Self::ResolveFunction { .. } + | Self::FunctionStands { .. } + | Self::FunctionCallee { .. } | Self::AccessorStands { .. } | Self::FieldStands { .. } | Self::DictShadows { .. } @@ -1358,6 +1401,7 @@ impl Op { | Self::DeleteGlobal { .. } | Self::DeleteLocal { .. } | Self::CallValue { .. } + | Self::CallThrough { .. } | Self::CallMethod { .. } | Self::GetAttr { .. } | Self::SetAttr { .. } @@ -1444,6 +1488,9 @@ impl Op { | Self::MatchAttr { dest, .. } | Self::MethodStands { dest, .. } | Self::BuiltinStands { dest, .. } + | Self::ResolveFunction { dest, .. } + | Self::FunctionStands { dest, .. } + | Self::FunctionCallee { dest, .. } | Self::AccessorStands { dest, .. } | Self::FieldStands { dest, .. } | Self::DictShadows { dest, .. } @@ -1474,6 +1521,7 @@ impl Op { | Self::ImportModule { dest, .. } | Self::ImportFrom { dest, .. } | Self::CallValue { dest, .. } + | Self::CallThrough { dest, .. } | Self::LoadGlobal { dest, .. } | Self::MakeSlice { dest, .. } | Self::LoadEllipsis { dest } @@ -1563,6 +1611,9 @@ impl Op { | Self::MatchAttr { dest, .. } | Self::MethodStands { dest, .. } | Self::BuiltinStands { dest, .. } + | Self::ResolveFunction { dest, .. } + | Self::FunctionStands { dest, .. } + | Self::FunctionCallee { dest, .. } | Self::AccessorStands { dest, .. } | Self::FieldStands { dest, .. } | Self::DictShadows { dest, .. } @@ -1593,6 +1644,7 @@ impl Op { | Self::ImportModule { dest, .. } | Self::ImportFrom { dest, .. } | Self::CallValue { dest, .. } + | Self::CallThrough { dest, .. } | Self::LoadGlobal { dest, .. } | Self::MakeSlice { dest, .. } | Self::LoadEllipsis { dest } @@ -1676,6 +1728,8 @@ impl Op { | Self::Move { src, .. } | Self::Box { src, .. } | Self::MethodStands { src, .. } + | Self::FunctionStands { src, .. } + | Self::FunctionCallee { src, .. } | Self::AccessorStands { src, .. } | Self::FieldStands { src, .. } | Self::LicenceHolds { src, .. } @@ -1752,7 +1806,7 @@ impl Op { | Self::FloatCompare { lhs, rhs, .. } => vec![lhs, rhs], Self::Unary { operand, .. } => vec![operand], Self::CallNative { args, .. } | Self::CallPython { args, .. } => args.iter().collect(), - Self::CallValue { callee, args, .. } => { + Self::CallValue { callee, args, .. } | Self::CallThrough { callee, args, .. } => { let mut all = vec![callee]; all.extend(args.iter()); all @@ -1811,6 +1865,7 @@ impl Op { | Self::FetchException { .. } | Self::LoadGlobal { .. } | Self::BuiltinStands { .. } + | Self::ResolveFunction { .. } | Self::LoadEllipsis { .. } | Self::ModuleDict { .. } | Self::DeleteGlobal { .. } @@ -1905,6 +1960,8 @@ impl Op { | Self::Move { src, .. } | Self::Box { src, .. } | Self::MethodStands { src, .. } + | Self::FunctionStands { src, .. } + | Self::FunctionCallee { src, .. } | Self::AccessorStands { src, .. } | Self::FieldStands { src, .. } | Self::LicenceHolds { src, .. } @@ -1983,7 +2040,7 @@ impl Op { Self::CallNative { args, .. } | Self::CallPython { args, .. } => { args.iter_mut().collect() } - Self::CallValue { callee, args, .. } => { + Self::CallValue { callee, args, .. } | Self::CallThrough { callee, args, .. } => { let mut all = vec![callee]; all.extend(args.iter_mut()); all @@ -2042,6 +2099,7 @@ impl Op { | Self::FetchException { .. } | Self::LoadGlobal { .. } | Self::BuiltinStands { .. } + | Self::ResolveFunction { .. } | Self::LoadEllipsis { .. } | Self::ModuleDict { .. } | Self::DeleteGlobal { .. } diff --git a/crates/by_ir/src/print.rs b/crates/by_ir/src/print.rs index 785dfa3681..e8bc71c238 100644 --- a/crates/by_ir/src/print.rs +++ b/crates/by_ir/src/print.rs @@ -155,6 +155,28 @@ fn print_op(function: &Function, op: &Op) -> String { dest, name: builtin, } => format!("{} = builtin-stands {builtin}", name(*dest)), + Op::ResolveFunction { + dest, + name: function, + } => format!("{} = resolve-function {function}", name(*dest)), + Op::FunctionStands { + dest, + src, + name: function, + } => format!( + "{} = function-stands {function} {}", + name(*dest), + value(src) + ), + Op::FunctionCallee { + dest, + src, + name: function, + } => format!( + "{} = function-callee {function} {}", + name(*dest), + value(src) + ), Op::MethodStands { dest, src, @@ -604,6 +626,21 @@ fn print_op(function: &Function, op: &Op) -> String { let args = args.iter().map(value).collect::>().join(", "); format!("{} = pycall {}({})", name(*dest), callee, args) } + Op::CallThrough { + dest, + callee, + args, + keywords, + } => { + let args = args.iter().map(value).collect::>().join(", "); + format!( + "{} = callthrough {}({}) keywords {}", + name(*dest), + value(callee), + args, + keywords.join(", ") + ) + } Op::CallMethod { dest, receiver, @@ -963,6 +1000,7 @@ b2: fallback_code: None, shims: None, verify_install: true, + follow_recursion_limit: true, }; let text = print_module(&module); assert!(text.starts_with("module app\n")); diff --git a/crates/by_ir/src/verify.rs b/crates/by_ir/src/verify.rs index 7f1862d9b4..abb1858d81 100644 --- a/crates/by_ir/src/verify.rs +++ b/crates/by_ir/src/verify.rs @@ -745,6 +745,17 @@ impl Verifier<'_> { Op::BuiltinStands { dest, .. } => { self.expect_dest(block, *dest, &RType::BIT, "a builtin test"); } + Op::ResolveFunction { dest, .. } => { + self.expect_dest(block, *dest, &RType::OBJECT, "a function resolution"); + } + Op::FunctionStands { dest, src, .. } => { + self.expect(block, src, &RType::OBJECT, "a function test"); + self.expect_dest(block, *dest, &RType::BIT, "a function test"); + } + Op::FunctionCallee { dest, src, .. } => { + self.expect(block, src, &RType::OBJECT, "a function callee"); + self.expect_dest(block, *dest, &RType::OBJECT, "a function callee"); + } Op::MethodStands { dest, src, .. } => { self.expect(block, src, &RType::OBJECT, "a dispatch test"); self.expect_dest(block, *dest, &RType::BIT, "a dispatch test"); @@ -1327,6 +1338,24 @@ impl Verifier<'_> { self.expect(block, module, &RType::OBJECT, "an import"); self.expect_dest(block, *dest, &RType::OBJECT, "an import"); } + Op::CallThrough { + dest, + callee, + args, + keywords, + } => { + self.expect(block, callee, &RType::OBJECT, "a call through a function"); + for arg in args { + self.expect(block, arg, &RType::OBJECT, "a call argument"); + } + if keywords.len() > args.len() { + self.error( + Some(block), + "a call names more keywords than it passes arguments", + ); + } + self.expect_dest(block, *dest, &RType::OBJECT, "a call through a function"); + } Op::CallValue { dest, callee, args } => { self.expect(block, callee, &RType::OBJECT, "a call through a value"); for arg in args { @@ -2466,6 +2495,7 @@ mod tests { fallback_code: None, shims: None, verify_install: true, + follow_recursion_limit: true, }; let errors = verify_module(&module).unwrap_err(); assert_eq!(errors.len(), 1); @@ -2504,6 +2534,7 @@ mod tests { fallback_code: None, shims: None, verify_install: true, + follow_recursion_limit: true, }; let errors = verify_module(&module).unwrap_err(); assert!( @@ -2800,6 +2831,7 @@ mod tests { fallback_code: None, shims: None, verify_install: true, + follow_recursion_limit: true, }; assert_eq!(verify_module(&module), Ok(())); diff --git a/crates/by_irbuild/src/lib.rs b/crates/by_irbuild/src/lib.rs index f7a36fc359..0838326c18 100644 --- a/crates/by_irbuild/src/lib.rs +++ b/crates/by_irbuild/src/lib.rs @@ -44,6 +44,14 @@ pub struct LowerOptions { /// off by default, because the lookup it re-asks is the whole cost a licence /// exists to avoid pub recheck_licences: bool, + /// have a call to a function this module defines go straight to its native entry + /// without asking whether the name still holds that function — see + /// [`by_ir::ops::Op::FunctionStands`] + /// + /// off by default: a module whose function is rebound, patched or deleted from + /// outside, or given another `__code__` or other defaults, is then no longer obeyed + /// by its own compiled callers + pub bind_functions_early: bool, } impl From for LowerOptions { @@ -51,6 +59,7 @@ impl From for LowerOptions { Self { language, recheck_licences: false, + bind_functions_early: false, } } } @@ -136,6 +145,7 @@ pub fn build_module( let LowerOptions { language, recheck_licences, + bind_functions_early, } = options.into(); let mut module = ModuleIr::new(module_name); @@ -581,6 +591,7 @@ pub fn build_module( accessors: &accessors, language, recheck_licences, + bind_functions_early, db, env, model, @@ -1731,6 +1742,7 @@ fn lower_resume( unnarrowed: None, language: unit.language, recheck_licences: unit.recheck_licences, + bind_functions_early: unit.bind_functions_early, environment, captures: Some(Captured { class: class.to_string(), @@ -7683,6 +7695,7 @@ fn lower_function_with_receiver( zero_super, language: unit.language, recheck_licences: unit.recheck_licences, + bind_functions_early: unit.bind_functions_early, unwound: Vec::new(), comprehensions: 0, unnarrowed: None, @@ -7862,6 +7875,8 @@ struct Unit<'a> { /// whether each licensed direct call re-asks the lookup it skips — see /// [`LowerOptions::recheck_licences`] recheck_licences: bool, + /// see [`LowerOptions::bind_functions_early`] + bind_functions_early: bool, db: &'a dyn ty_python_semantic::Db, model: &'a SemanticModel<'a>, native_callees: &'a HashSet, @@ -9701,6 +9716,8 @@ struct Lowering<'a, 'db> { /// whether each licensed direct call re-asks the lookup it skips — see /// [`LowerOptions::recheck_licences`] recheck_licences: bool, + /// see [`LowerOptions::bind_functions_early`] + bind_functions_early: bool, ret: RType, /// `(continue target, break target, cleanup depth)` for each enclosing loop, /// innermost last. the depth is what `break` unwinds to @@ -9742,6 +9759,16 @@ struct Lowering<'a, 'db> { /// [`Lowering::drained_generator_expression`] const DRAINING_BUILTINS: &[&str] = &["list", "tuple", "set", "frozenset", "sorted", "dict"]; +/// what a call reaching a native entry is for, which decides what the call made in its +/// place does with the function object's answer +#[derive(Clone, Copy, PartialEq, Eq)] +enum Delivery { + /// the call's own result + Called, + /// the value a coroutine's body returns, awaited in the same expression + Awaited, +} + /// where [`Lowering::resolve_builtin`] reads the object a builtin's name resolves to #[derive(Clone, Copy, PartialEq, Eq)] enum Wanted { @@ -13898,7 +13925,7 @@ impl Lowering<'_, '_> { return Ok(None); } let direct = generators::direct_name(name); - self.native_call(call, &direct).map(Some) + self.native_call(call, &direct, Delivery::Awaited).map(Some) } fn delegate(&mut self, source: &Expr, awaitable: bool) -> Lowered<(Value, RType)> { @@ -17769,7 +17796,7 @@ impl Lowering<'_, '_> { return self.narrow_call_result(dest, &Expr::Call(node.clone())); } - self.native_call(node, name) + self.native_call(node, name, Delivery::Called) } /// `list(x for x in xs)`, or the same handed to one of the other builtins that drain @@ -18004,7 +18031,12 @@ impl Lowering<'_, '_> { /// an [unboxed edition](lower_array_edition) and the [direct edition]( /// lower_direct_edition) of a coroutine are both other spellings of the same /// definition - fn native_call(&mut self, node: &ast::ExprCall, name: &str) -> Lowered<(Value, RType)> { + fn native_call( + &mut self, + node: &ast::ExprCall, + name: &str, + delivery: Delivery, + ) -> Lowered<(Value, RType)> { let env = &self.model.program_environment(); // a caller already holding buffers reaches the callee's unboxed edition, which // takes them as they are. anything else — a list from python, a name that is @@ -18111,39 +18143,238 @@ impl Lowering<'_, '_> { return Ok((Value::Register(dest), result_ty)); } + // python reads the name before it evaluates anything the call is handed, and + // calls what it read even where an argument rebinds the name. so the question of + // whether the native entry may stand in for the function object comes first, and + // the object itself is read only where it may not — see [`Op::FunctionStands`] + let guarded = match node.func.as_ref() { + Expr::Name(written) + if !self.bind_functions_early + && self.native_callees.contains(written.id.as_str()) => + { + let resolved = self.builder.temp(RType::OBJECT); + self.builder.push(Op::ResolveFunction { + dest: resolved, + name: written.id.to_string(), + }); + Some((resolved, written.id.to_string())) + } + _ => None, + }; + + // every argument once, in the order python evaluates them, before either arm + // decides what to make of it + let mut positional: Vec<(Value, RType)> = Vec::with_capacity(node.arguments.args.len()); + for arg in &node.arguments.args { + positional.push(self.expression(arg)?); + } + let mut keyworded: Vec<(String, Value, RType)> = Vec::new(); + for keyword in &node.arguments.keywords { + // a `**` keyword never reaches here: the whole call goes through + // [`Self::call_unpacked`] instead + let Some(keyword_name) = &keyword.arg else { + continue; + }; + let (value, ty) = self.expression(&keyword.value)?; + keyworded.push((keyword_name.to_string(), value, ty)); + } + + let dest = self.builder.temp(result_ty.clone()); + let Some((resolved, written)) = guarded else { + let args = self.bind_native_arguments( + name, + ¶ms, + &names, + &defaults, + (vararg, kwarg), + (posonly, positional_limit), + &positional, + &keyworded, + )?; + self.builder.push(Op::CallNative { + dest: Some(dest), + owner: None, + callee: name.to_string(), + args, + }); + return Ok((Value::Register(dest), result_ty)); + }; + + let stands = self.builder.temp(RType::BIT); + self.builder.push(Op::FunctionStands { + dest: stands, + src: Value::Register(resolved), + name: written.clone(), + }); + let native = self.builder.new_block(); + let called = self.builder.new_block(); + let join = self.builder.new_block(); + self.builder.terminate(Terminator::Branch { + cond: Value::Register(stands), + then_block: native, + else_block: called, + }); + + self.builder.switch_to(native); + let args = self.bind_native_arguments( + name, + ¶ms, + &names, + &defaults, + (vararg, kwarg), + (posonly, positional_limit), + &positional, + &keyworded, + )?; + let answer = self.builder.temp(result_ty.clone()); + self.builder.push(Op::CallNative { + dest: Some(answer), + owner: None, + callee: name.to_string(), + args, + }); + self.builder.assign(dest, Value::Register(answer)); + self.builder.terminate(Terminator::Goto(join)); + + self.builder.switch_to(called); + // a list held as a buffer has no list object to hand over, and building one + // would be a copy the function under the name could tell apart from the list + // python would have passed — so this refuses, loudly, as a rebound `len` does + if positional + .iter() + .map(|(_, ty)| ty) + .chain(keyworded.iter().map(|(_, _, ty)| ty)) + .any(|ty| matches!(ty, RType::Array(_))) + { + self.builder.push(Op::RaiseStandard { + error: StandardError::RuntimeError, + message: format!( + "`{written}` no longer names the function this module compiled, and this \ + compiled function holds a list it passes as an unboxed buffer it \ + cannot hand to anything else" + ), + }); + self.builder.terminate(Terminator::Unreachable); + self.builder.switch_to(join); + return Ok((Value::Register(dest), result_ty)); + } + let callee = self.builder.temp(RType::OBJECT); + self.builder.push(Op::FunctionCallee { + dest: callee, + src: Value::Register(resolved), + name: written, + }); + let mut boxed: Vec = positional + .iter() + .map(|(value, ty)| self.widen_to_object(value.clone(), ty)) + .collect(); + let mut keywords = Vec::with_capacity(keyworded.len()); + for (keyword_name, value, ty) in &keyworded { + keywords.push(keyword_name.clone()); + boxed.push(self.widen_to_object(value.clone(), ty)); + } + let result = self.builder.temp(RType::OBJECT); + self.builder.push(Op::CallThrough { + dest: result, + callee: Value::Register(callee), + args: boxed, + keywords, + }); + // the entry a direct `await` reaches runs the coroutine's body to its end, so + // what the function under the name hands back has to be awaited in its place + let (answer, answer_ty) = match delivery { + Delivery::Called => (Value::Register(result), RType::OBJECT), + Delivery::Awaited => self.delegate_value(Value::Register(result), true)?, + }; + let answer = self.widen_to_object(answer, &answer_ty); + let answer = self.narrow_answer(answer, &result_ty)?; + self.builder.assign(dest, answer); + self.builder.terminate(Terminator::Goto(join)); + + self.builder.switch_to(join); + Ok((Value::Register(dest), result_ty)) + } + + /// what a function object answered, in the representation its native entry would + /// have answered in + /// + /// the checked narrowing a call's result gets anywhere else, and for a tuple held in + /// registers the unpacking an assignment to a target list makes, each slot narrowed + /// the same way + fn narrow_answer(&mut self, answer: Value, to: &RType) -> Lowered { + let RType::Tuple(slots) = to else { + return self.coerce(answer, &RType::OBJECT, to); + }; + let objects: Box<[RType]> = vec![RType::OBJECT; slots.len()].into(); + let unpacked = self.builder.temp(RType::Tuple(objects)); + self.builder.push(Op::Unpack { + dest: unpacked, + src: answer, + starred: None, + }); + let mut items = Vec::with_capacity(slots.len()); + for (index, slot) in slots.iter().enumerate() { + let item = self.builder.temp(RType::OBJECT); + self.builder.push(Op::TupleGet { + dest: item, + src: Value::Register(unpacked), + index, + }); + items.push(self.narrow_answer(Value::Register(item), slot)?); + } + let built = self.builder.temp(to.clone()); + self.builder.push(Op::TupleBuild { dest: built, items }); + Ok(Value::Register(built)) + } + + /// the argument list a native entry takes, from arguments already evaluated + /// + /// a keyword goes to the position its name has in the callee's signature, an + /// unsupplied parameter takes its default, and whatever a variadic callee takes as + /// extra is packed into the tuple and the dict its body expects + #[expect( + clippy::too_many_arguments, + reason = "the halves of one signature, taken apart once by the caller" + )] + fn bind_native_arguments( + &mut self, + name: &str, + params: &[RType], + names: &[String], + defaults: &[Option], + (vararg, kwarg): (bool, bool), + (posonly, positional_limit): (usize, usize), + positional: &[(Value, RType)], + keyworded: &[(String, Value, RType)], + ) -> Lowered> { + let packed_count = usize::from(vararg) + usize::from(kwarg); + let named = names.len().saturating_sub(packed_count); let mut slots: Vec> = vec![None; params.len().max(names.len())]; let mut extra_positional: Vec = Vec::new(); let mut extra_keywords: Vec = Vec::new(); - for (index, arg) in node.arguments.args.iter().enumerate() { + for (index, (value, ty)) in positional.iter().enumerate() { if index >= positional_limit { if !vararg { return Err(Decline::new("too many arguments for the callee")); } - let (value, ty) = self.expression(arg)?; - extra_positional.push(self.widen_to_object(value, &ty)); + extra_positional.push(self.widen_to_object(value.clone(), ty)); continue; } - let (value, ty) = self.expression(arg)?; let value = match params.get(index) { - Some(param) => self.coerce(value, &ty, param)?, - None => value, + Some(param) => self.coerce(value.clone(), ty, param)?, + None => value.clone(), }; match slots.get_mut(index) { Some(slot) => *slot = Some(value), None => return Err(Decline::new("too many arguments for the callee")), } } - for keyword in &node.arguments.keywords { - // a `**` keyword never reaches here: the whole call goes through - // [`Self::call_unpacked`] instead - let Some(keyword_name) = &keyword.arg else { - continue; - }; + for (keyword_name, value, ty) in keyworded { // a positional-only parameter is not reachable by name let position = names .iter() .take(named) - .position(|param| param == keyword_name.as_str()) + .position(|param| param == keyword_name) .filter(|index| *index >= posonly); let Some(index) = position else { // a `**kwargs` parameter takes it; without one it is an error @@ -18152,13 +18383,12 @@ impl Lowering<'_, '_> { "`{name}` has no parameter `{keyword_name}`" ))); } - let (value, ty) = self.expression(&keyword.value)?; - let value = self.widen_to_object(value, &ty); + let value = self.widen_to_object(value.clone(), ty); // the name goes into the dict as the literal it is. a register // holding it would be filled from the module's interned string on // every call, so a keyword passed in a loop retained and released // one object per trip for the sake of a name that never changes - extra_keywords.push(Value::Str(keyword_name.to_string())); + extra_keywords.push(Value::Str(keyword_name.clone())); extra_keywords.push(value); continue; }; @@ -18167,10 +18397,9 @@ impl Lowering<'_, '_> { "`{name}` got two values for `{keyword_name}`" ))); } - let (value, ty) = self.expression(&keyword.value)?; let value = match params.get(index) { - Some(param) => self.coerce(value, &ty, param)?, - None => value, + Some(param) => self.coerce(value.clone(), ty, param)?, + None => value.clone(), }; slots[index] = Some(value); } @@ -18217,15 +18446,7 @@ impl Lowering<'_, '_> { }); args.push(Value::Register(dest)); } - - let dest = self.builder.temp(result_ty.clone()); - self.builder.push(Op::CallNative { - dest: Some(dest), - owner: None, - callee: name.to_string(), - args, - }); - Ok((Value::Register(dest), result_ty)) + Ok(args) } } diff --git a/crates/by_irbuild/src/tests.rs b/crates/by_irbuild/src/tests.rs index a85b59f2a5..ea772a47da 100644 --- a/crates/by_irbuild/src/tests.rs +++ b/crates/by_irbuild/src/tests.rs @@ -7754,7 +7754,15 @@ async def chained(n: int) -> int: ", ); assert!(text.contains("call step$direct(r"), "{text}"); - assert!(!text.contains("awaititer"), "{text}"); + // the object is built only in the arm a moved or rebound `step` sends the call + // through, where the function under the name is called and what it hands back + // awaited — the arm the direct call stands in for + let (direct, turned_away) = text + .split_once("callthrough") + .expect("a call the direct entry is turned away from goes through the name"); + assert!(direct.contains("function-stands step"), "{text}"); + assert!(!direct.contains("awaititer"), "{text}"); + assert!(turned_away.contains("awaititer"), "{text}"); assert!( emitted.iter().any(|name| name == "step$direct"), "{emitted:?}" @@ -11391,6 +11399,7 @@ fn lowered_with_recheck(source: &str, recheck_licences: bool) -> by_ir::function crate::LowerOptions { language: crate::Language::BasedPython, recheck_licences, + bind_functions_early: false, }, ) }) diff --git a/crates/by_opt/src/coalesce.rs b/crates/by_opt/src/coalesce.rs index d46115e9c9..6a47238e9d 100644 --- a/crates/by_opt/src/coalesce.rs +++ b/crates/by_opt/src/coalesce.rs @@ -129,6 +129,7 @@ mod tests { fallback_code: None, shims: None, verify_install: true, + follow_recursion_limit: true, } } diff --git a/crates/by_opt/src/copy_propagation.rs b/crates/by_opt/src/copy_propagation.rs index feac47089d..603528f9e8 100644 --- a/crates/by_opt/src/copy_propagation.rs +++ b/crates/by_opt/src/copy_propagation.rs @@ -273,6 +273,9 @@ fn retarget(op: &mut Op, new_dest: RegisterId) { | Op::IsMissing { dest, .. } | Op::MethodStands { dest, .. } | Op::BuiltinStands { dest, .. } + | Op::FunctionStands { dest, .. } + | Op::ResolveFunction { dest, .. } + | Op::FunctionCallee { dest, .. } | Op::AccessorStands { dest, .. } | Op::FieldStands { dest, .. } | Op::DictShadows { dest, .. } @@ -288,6 +291,7 @@ fn retarget(op: &mut Op, new_dest: RegisterId) { | Op::StrOfInt { dest, .. } | Op::CallPython { dest, .. } | Op::CallValue { dest, .. } + | Op::CallThrough { dest, .. } | Op::LoadGlobal { dest, .. } | Op::MakeSlice { dest, .. } | Op::LoadEllipsis { dest } @@ -390,6 +394,7 @@ mod tests { fallback_code: None, shims: None, verify_install: true, + follow_recursion_limit: true, } } diff --git a/crates/by_opt/src/dead_registers.rs b/crates/by_opt/src/dead_registers.rs index c4de447bb1..953ec1a7b3 100644 --- a/crates/by_opt/src/dead_registers.rs +++ b/crates/by_opt/src/dead_registers.rs @@ -128,6 +128,7 @@ mod tests { fallback_code: None, shims: None, verify_install: true, + follow_recursion_limit: true, } } diff --git a/crates/by_opt/src/dict_find.rs b/crates/by_opt/src/dict_find.rs index 65f439a421..aabd15a1ac 100644 --- a/crates/by_opt/src/dict_find.rs +++ b/crates/by_opt/src/dict_find.rs @@ -402,6 +402,7 @@ mod tests { fallback_code: None, shims: None, verify_install: true, + follow_recursion_limit: true, } } diff --git a/crates/by_opt/src/fold.rs b/crates/by_opt/src/fold.rs index 2273014dcf..4d9f9b3b4c 100644 --- a/crates/by_opt/src/fold.rs +++ b/crates/by_opt/src/fold.rs @@ -573,6 +573,7 @@ mod tests { fallback_code: None, shims: None, verify_install: true, + follow_recursion_limit: true, } } diff --git a/crates/by_opt/src/infallible.rs b/crates/by_opt/src/infallible.rs index 617f5ebec8..04b3537e56 100644 --- a/crates/by_opt/src/infallible.rs +++ b/crates/by_opt/src/infallible.rs @@ -27,23 +27,27 @@ use by_ir::rtype::{Primitive, RType}; /// mark every function that provably cannot fail /// /// this is a fixed point over the call graph: a function is infallible when -/// every one of its own operations is, *and* every function it calls is. so a -/// pair of mutually recursive float functions converges to infallible, and one -/// division anywhere in a cycle makes the whole cycle fallible. +/// every one of its own operations is, *and* every function it calls is. a function +/// on a cycle of calls never is: the call that goes round the cycle counts the +/// recursion depth, and can raise when the count runs out pub(crate) fn run(module: &mut ModuleIr) { let names: Vec = module .all_functions() .map(Function::qualified_name) .collect(); - // start optimistic and remove: the greatest fixed point is what makes a - // recursive function infallible rather than assuming the worst about itself + // start optimistic and remove let mut infallible: HashSet<&str> = names.iter().map(String::as_str).collect(); + // a call around a cycle of native calls is where the depth is counted, and a count + // that runs out raises `RecursionError` — so no function on a cycle is infallible, + // however little its own operations can do + let recursive = by_ir::call_graph::recursive_functions(module); + let mut own_ops_can_fail: HashMap = HashMap::new(); let mut callees: HashMap> = HashMap::new(); for function in module.all_functions() { - let mut can_fail = false; + let mut can_fail = recursive.contains(&function.qualified_name()); let mut called = Vec::new(); for block in &function.blocks { for op in &block.ops { @@ -157,6 +161,7 @@ fn op_can_fail(module: &ModuleIr, function: &by_ir::function::Function, op: &Op) | Op::StrConcatInt { .. } | Op::CallPython { .. } | Op::CallValue { .. } + | Op::CallThrough { .. } | Op::LoadGlobal { .. } // building a slice allocates, and allocation raises | Op::MakeSlice { .. } @@ -264,6 +269,10 @@ fn op_can_fail(module: &ModuleIr, function: &by_ir::function::Function, op: &Op) Op::FieldIsSet { .. } => false, // the name is resolved, and one bound nowhere is a `NameError` Op::BuiltinStands { .. } => true, + // the flag is read, and nothing is looked up + Op::FunctionStands { .. } => false, + // a name bound nowhere is a `NameError` + Op::ResolveFunction { .. } | Op::FunctionCallee { .. } => true, // an attribute lookup reaches `__getattr__`, and `__match_args__` with it Op::MatchAttr { .. } => true, // a pointer comparison against a singleton @@ -328,6 +337,7 @@ mod tests { fallback_code: None, shims: None, verify_install: true, + follow_recursion_limit: true, } } @@ -460,9 +470,9 @@ mod tests { } #[test] - fn mutual_recursion_converges_to_infallible() { - // the greatest fixed point is the point of starting optimistic: a - // least-fixed-point pass would call each of these fallible forever + fn a_cycle_of_calls_is_fallible_however_little_its_bodies_can_do() { + // nothing in either body can raise, but a call around the cycle is where the + // recursion depth is counted, and running out of it raises `RecursionError` let build = |name: &str, other: &str| { let mut builder = FunctionBuilder::new(name, RType::FLOAT); let x = builder.param("x", RType::FLOAT); @@ -476,10 +486,15 @@ mod tests { builder.terminate(Terminator::Return(Value::Register(out))); builder.finish() }; - let mut m = module(vec![build("ping", "pong"), build("pong", "ping")]); + let mut m = module(vec![ + build("ping", "pong"), + build("pong", "ping"), + build("entry", "ping"), + ]); run(&mut m); - assert_eq!(convention(&m, "ping"), CallConvention::NativeInfallible); - assert_eq!(convention(&m, "pong"), CallConvention::NativeInfallible); + assert_eq!(convention(&m, "ping"), CallConvention::Native); + assert_eq!(convention(&m, "pong"), CallConvention::Native); + assert_eq!(convention(&m, "entry"), CallConvention::Native); } #[test] diff --git a/crates/by_opt/src/lib.rs b/crates/by_opt/src/lib.rs index 62b2a7dfc5..075e8f7e52 100644 --- a/crates/by_opt/src/lib.rs +++ b/crates/by_opt/src/lib.rs @@ -226,6 +226,7 @@ mod tests { fallback_code: None, shims: None, verify_install: true, + follow_recursion_limit: true, }; assert!(optimize(&mut module).is_ok()); let fixed = |id: by_ir::ops::RegisterId| { @@ -264,6 +265,7 @@ mod tests { fallback_code: None, shims: None, verify_install: true, + follow_recursion_limit: true, }; assert!(optimize(&mut module).is_ok()); // both passes fired: the copy is gone and the function is infallible diff --git a/crates/by_opt/src/refcount.rs b/crates/by_opt/src/refcount.rs index 410214077b..5a530b0e0b 100644 --- a/crates/by_opt/src/refcount.rs +++ b/crates/by_opt/src/refcount.rs @@ -113,6 +113,7 @@ mod tests { fallback_code: None, shims: None, verify_install: true, + follow_recursion_limit: true, } } diff --git a/crates/by_opt/src/release_temporaries.rs b/crates/by_opt/src/release_temporaries.rs index dca5ee04ee..a2f2574405 100644 --- a/crates/by_opt/src/release_temporaries.rs +++ b/crates/by_opt/src/release_temporaries.rs @@ -452,6 +452,7 @@ mod tests { fallback_code: None, shims: None, verify_install: true, + follow_recursion_limit: true, } } diff --git a/crates/by_opt/src/store_moves.rs b/crates/by_opt/src/store_moves.rs index 89a7ff7ee5..3e9c200d21 100644 --- a/crates/by_opt/src/store_moves.rs +++ b/crates/by_opt/src/store_moves.rs @@ -268,6 +268,7 @@ mod tests { fallback_code: None, shims: None, verify_install: true, + follow_recursion_limit: true, } } diff --git a/crates/by_opt/src/str_append.rs b/crates/by_opt/src/str_append.rs index 2982f24b69..fff80f58b6 100644 --- a/crates/by_opt/src/str_append.rs +++ b/crates/by_opt/src/str_append.rs @@ -126,6 +126,7 @@ mod tests { fallback_code: None, shims: None, verify_install: true, + follow_recursion_limit: true, } } diff --git a/crates/by_opt/src/str_item_compare.rs b/crates/by_opt/src/str_item_compare.rs index 72f157801b..f82936682b 100644 --- a/crates/by_opt/src/str_item_compare.rs +++ b/crates/by_opt/src/str_item_compare.rs @@ -159,6 +159,7 @@ mod tests { fallback_code: None, shims: None, verify_install: true, + follow_recursion_limit: true, } } diff --git a/crates/by_opt/src/unbox_counters.rs b/crates/by_opt/src/unbox_counters.rs index 8643381b4d..b583310afd 100644 --- a/crates/by_opt/src/unbox_counters.rs +++ b/crates/by_opt/src/unbox_counters.rs @@ -326,6 +326,7 @@ mod tests { fallback_code: None, shims: None, verify_install: true, + follow_recursion_limit: true, } } diff --git a/crates/by_opt/src/unswitch.rs b/crates/by_opt/src/unswitch.rs index 9f7eb137e7..e88a7319ed 100644 --- a/crates/by_opt/src/unswitch.rs +++ b/crates/by_opt/src/unswitch.rs @@ -384,6 +384,7 @@ mod tests { fallback_code: None, shims: None, verify_install: true, + follow_recursion_limit: true, } } diff --git a/crates/by_rt/include/by.h b/crates/by_rt/include/by.h index 2d14f1eb55..f00cd401e1 100644 --- a/crates/by_rt/include/by.h +++ b/crates/by_rt/include/by.h @@ -36,6 +36,7 @@ #include #include #include +#include /* the floor, restated for the compiler * @@ -1888,6 +1889,213 @@ static inline PyObject *By_Warn(PyObject *message, PyObject *category, return Py_NewRef(Py_None); } +/* ── recursion depth ────────────────────────────────────────────────────────── + * + * a compiled call pushes a C frame and no python frame, so nothing the interpreter + * counts stands between a compiled recursion and the end of the stack: `depth(10**6)` + * segfaulted where python raises `RecursionError`. two things are watched instead, at + * every call that can go round a cycle of compiled calls and at every resumption of a + * compiled generator or coroutine. + * + * the stack itself, always. python frames live on the heap, and a program that raises + * the recursion limit is allowed as deep as its memory goes, where a C frame cannot go + * past the stack its thread was given. so the stack is a limit of its own, and the + * thing that turns running out of it into `RecursionError` rather than a crash. + * + * python's recursion limit, unless the build was told not to follow it. the count is + * the interpreter's own `py_recursion_remaining`, taken and given back exactly as a + * python frame takes and gives it back, so `sys.setrecursionlimit` reaches a compiled + * recursion and python frames and compiled ones share one budget. a build that defines + * `BY_RECURSION_STACK_ONLY` counts nothing and watches the stack alone. + * + * `ByDepth` is what a function on a cycle is handed by its caller, so a call round the + * cycle reads nothing the callee could not have been given: the thread state and the + * stack limit are looked up once, where the cycle is entered */ + +/* the lowest address a compiled frame may start below the top of, per thread + * + * zero until the thread first asks. a quarter of the thread's stack is left under it, + * which is the room whatever runs past the last check — an acyclic chain of compiled + * calls, the interpreter raising the error and building its traceback, a C api the + * last frame called — has to finish in */ +#if defined(_MSC_VER) && !defined(__clang__) +static __declspec(thread) uintptr_t by_stack_limit = 0; +#else +static _Thread_local uintptr_t by_stack_limit = 0; +#endif + +#if defined(__APPLE__) +#include +#elif defined(__FreeBSD__) +#include +#include +#elif defined(__linux__) && (defined(__GLIBC__) || defined(__BIONIC__)) +#include +#elif defined(_WIN32) +/* declared rather than taken from `windows.h`, whose macros collide with names in + * this header. it is `kernel32`'s, which every windows process links */ +__declspec(dllimport) void __stdcall GetCurrentThreadStackLimits(uintptr_t *low, uintptr_t *high); +#endif + +/* where this frame is on the stack. every platform a python wheel is built for grows + * its stack downwards, which is what every comparison below assumes */ +BY_HOT uintptr_t By_StackHere(void) { +#if defined(__GNUC__) || defined(__clang__) + return (uintptr_t)__builtin_frame_address(0); +#else + char here = 0; + return (uintptr_t)&here; +#endif +} + +/* ask the platform how big this thread's stack is, once + * + * musl is left to the estimate: its answer for the main thread is the part of the stack + * already mapped rather than the part the thread may grow into, which cpython found + * imposes a limit far below the real one. an estimate is also what a platform with no + * way to ask gets — the stack a thread is typically given, measured down from the + * frame that asked first, which is not far below the top of the stack */ +BY_COLD uintptr_t By_ReadStackLimit(void) { + uintptr_t low = 0; + uintptr_t size = 0; +#if defined(__APPLE__) + pthread_t self = pthread_self(); + size = (uintptr_t)pthread_get_stacksize_np(self); + low = (uintptr_t)pthread_get_stackaddr_np(self) - size; +#elif defined(__FreeBSD__) || (defined(__linux__) && (defined(__GLIBC__) || defined(__BIONIC__))) + pthread_attr_t attr; + void *address = NULL; + size_t length = 0; + size_t guard = 0; + int failed; +#if defined(__FreeBSD__) + failed = pthread_attr_init(&attr); + if (failed == 0) failed = pthread_attr_get_np(pthread_self(), &attr); +#else + failed = pthread_getattr_np(pthread_self(), &attr); +#endif + if (failed == 0) { + failed = pthread_attr_getstack(&attr, &address, &length); + if (failed == 0 && pthread_attr_getguardsize(&attr, &guard) != 0) guard = 0; + pthread_attr_destroy(&attr); + } + if (failed == 0 && length > guard) { + low = (uintptr_t)address + guard; + size = (uintptr_t)(length - guard); + } +#elif defined(_WIN32) + uintptr_t high = 0; + GetCurrentThreadStackLimits(&low, &high); + size = high - low; +#endif + if (size == 0) { + size = (uintptr_t)1 << 20; + low = By_StackHere() - size; + } + by_stack_limit = low + size / 4; + return by_stack_limit; +} + +static inline uintptr_t By_StackLimit(void) { + uintptr_t limit = by_stack_limit; + if (BY_UNLIKELY(limit == 0)) limit = By_ReadStackLimit(); + return limit; +} + +/* the thread's own count of the frames it may still push, which 3.12 split out of the + * count it had shared with C calls */ +#if PY_VERSION_HEX >= 0x030C0000 +#define BY_PY_RECURSION_REMAINING py_recursion_remaining +#else +#define BY_PY_RECURSION_REMAINING recursion_remaining +#endif + +BY_COLD char By_StackExhausted(void) { + PyErr_SetString(PyExc_RecursionError, "maximum recursion depth exceeded"); + return 1; +} + +#ifdef BY_RECURSION_STACK_ONLY + +typedef uintptr_t ByDepth; + +static inline ByDepth By_DepthHere(void) { return By_StackLimit(); } + +/* nonzero, with `RecursionError` set, where the frame about to be pushed would start + * too near the end of the stack */ +BY_HOT char By_DepthEnter(ByDepth depth) { + if (BY_UNLIKELY(By_StackHere() < depth)) return By_StackExhausted(); + return 0; +} + +BY_HOT void By_DepthLeave(ByDepth depth) { (void)depth; } + +#else + +typedef struct { + PyThreadState *thread; + uintptr_t stack_limit; +} ByDepth; + +static inline ByDepth By_DepthHere(void) { + ByDepth depth; +#if PY_VERSION_HEX >= 0x030D0000 + depth.thread = PyThreadState_GetUnchecked(); +#else + depth.thread = _PyThreadState_UncheckedGet(); +#endif + depth.stack_limit = By_StackLimit(); + return depth; +} + +/* the count ran out: the interpreter's `_Py_CheckRecursiveCallPy`, on a frame whose + * count has already been taken + * + * while the thread is already raising one of these it is allowed fifty frames more, as + * python allows them. python aborts the process past those; this raises again instead, + * which is the one place it parts from the interpreter, and a crash is not an answer. + * a refusal gives the count back, because the frame it was taken for is never pushed */ +BY_COLD char By_RecursionLimitReached(PyThreadState *thread) { + if (thread->recursion_headroom && thread->BY_PY_RECURSION_REMAINING >= -50) return 0; + thread->BY_PY_RECURSION_REMAINING++; + thread->recursion_headroom++; + PyErr_SetString(PyExc_RecursionError, "maximum recursion depth exceeded"); + thread->recursion_headroom--; + return 1; +} + +/* take one frame from the count and check the stack, as python does on pushing a frame. + * nonzero, with `RecursionError` set and nothing taken, where either has run out */ +BY_HOT char By_DepthEnter(ByDepth depth) { + if (BY_UNLIKELY(By_StackHere() < depth.stack_limit)) return By_StackExhausted(); + if (BY_UNLIKELY(--depth.thread->BY_PY_RECURSION_REMAINING < 0)) { + return By_RecursionLimitReached(depth.thread); + } + return 0; +} + +/* give back the frame an `By_DepthEnter` that succeeded took */ +BY_HOT void By_DepthLeave(ByDepth depth) { depth.thread->BY_PY_RECURSION_REMAINING++; } + +#endif /* BY_RECURSION_STACK_ONLY */ + +/* resume a compiled generator or coroutine's frame, counted as python counts pushing + * the frame it would have resumed + * + * this is a cycle the call graph cannot see: `yield from f(n - 1)` makes the next + * generator with a native call and then steps it through the iterator protocol, and the + * interpreter counts none of that — so a generator delegating to itself ran off the + * stack. NULL with `RecursionError` set where the frame could not be pushed, which the + * caller finishes the generator on, as python closes one whose frame could not be */ +static inline PyObject *By_ResumeCounted(PyObject *self, PyObject *(*resume)(PyObject *)) { + ByDepth depth = By_DepthHere(); + PyObject *result; + if (BY_UNLIKELY(By_DepthEnter(depth))) return NULL; + result = resume(self); + By_DepthLeave(depth); + return result; +} + /* a global read that remembers what the name resolved to last time * * `By_LookupGlobal` is two dict probes on every trip, and for a name that resolves to @@ -2111,6 +2319,434 @@ static inline PyObject *By_LookupGlobalSite(ByGlobalSite *site, PyObject *dict, #endif } +/* ── a module function a compiled caller reaches directly ───────────────────── + * + * a compiled call to a function the module defines goes straight to its native entry, + * with the defaults it was compiled with filled in. python instead calls whatever the + * function object under that name is when the call is made, running the code it holds + * and binding a missing argument from the defaults it holds, so the direct call is only + * the same program while the published function is still the one the module installed + * and still holds what it was installed with. + * + * `ByFunctionSite` is that question, asked once per function rather than per call. two + * watchers keep its answer. a dict watcher is told of every write to the namespace, and + * one to the function's name takes the answer back until a call finds the published + * function under the name again. a function watcher is told when the function object's + * code, defaults or keyword defaults are reassigned, and from then on `moved` is set for + * good: the compiled callers call the function object, and the function's own python + * entry hands the call to the interpreted definition, given the defaults the published + * function holds now. an edit made *inside* the `__kwdefaults__` dict is not a + * reassignment and nothing reports it, so that one is not seen */ + +/* what a watcher registered by any compiled module writes. its layout is fixed, because + * the watcher is registered once per interpreter by whichever module gets there first, + * and that module may have been built by another version of this header */ +typedef struct { + /* a compiled caller may go straight to the native entry */ + char stands; + /* the published function's code or defaults were reassigned */ + char moved; +} ByFunctionFlags; + +/* one name a dict watcher answers for: the namespace it is a name in, and the flags a + * write to it takes back. the namespace is borrowed, and a compiled module's lives as long + * as the process does. fixed for the same reason the flags are */ +typedef struct { + PyObject *dict; + ByFunctionFlags *flags; +} ByFunctionName; + +typedef struct { + ByFunctionFlags flags; + /* the function this module published under the name, and the interpreted + * definition it forwards for. both strong, and both for the life of the process: + * the module stands behind one module object and no more */ + PyObject *forwarder; + PyObject *twin; + /* the name, interned when the site is armed */ + PyObject *name; + /* what the dict watcher holds for this site */ + ByFunctionName entry; + /* both watchers answer for this site, so a call that finds the published function + * under its name again may go straight to the native entry again */ + char watched; +} ByFunctionSite; + +#define BY_FUNCTION_SITE_INIT { { 0, 0 }, NULL, NULL, NULL, { NULL, NULL }, 0 } + +#if !defined(Py_GIL_DISABLED) && PY_VERSION_HEX >= 0x030C0000 +#define BY_FUNCTION_SITES 1 +#endif + +#ifdef BY_FUNCTION_SITES + +/* the interpreter's one function watcher and one namespace watcher for every compiled + * module, and what they answer for. + * + * `published` is a dict from the function to a capsule holding its flags. the key is the + * function object itself, which the dict keeps alive, so a function that dies can never + * be mistaken for one born at its address. `names` is a dict from a name to a list of + * capsules, one per namespace publishing a function under that name */ +typedef struct { + int watcher; + int dict_watcher; + PyObject *published; + PyObject *names; + /* one bit per name `names` holds, at its hash modulo 64. a write under a name whose + * bit is clear cannot be one of them, which is nearly every write, and is passed over + * without looking anything up */ + uint64_t hashes; +} ByFunctionWatch; + +#define BY_FUNCTION_WATCH_KEY "_by_function_watch_v1" +#define BY_FUNCTION_FLAGS_CAPSULE "by.function_flags" +#define BY_FUNCTION_NAME_CAPSULE "by.function_name" + +static ByFunctionWatch *by_function_watch = NULL; +/* the registration this module's callback reads — see `by_globals_watched` for why it is + * not the same variable as the one this module's sites read */ +static ByFunctionWatch *by_function_watch_registered = NULL; + +/* a function python tells every watcher about. creation and destruction happen to every + * function in the process and say nothing about a published one, so they are passed over + * before anything is looked up */ +static int By_FunctionChanged(PyFunction_WatchEvent event, PyFunctionObject *func, + PyObject *new_value) { + (void)new_value; + if (event != PyFunction_EVENT_MODIFY_CODE && event != PyFunction_EVENT_MODIFY_DEFAULTS + && event != PyFunction_EVENT_MODIFY_KWDEFAULTS) { + return 0; + } + ByFunctionWatch *watch = by_function_watch_registered; + if (watch == NULL) return 0; + /* a function hashes by identity, so looking it up runs no python and cannot fail + * for a reason of its own; an exception already standing is left as it was */ + PyObject *type, *value, *traceback; + PyErr_Fetch(&type, &value, &traceback); + PyObject *found = PyDict_GetItemWithError(watch->published, (PyObject *)func); + if (found != NULL) { + ByFunctionFlags *flags = (ByFunctionFlags *)PyCapsule_GetPointer(found, BY_FUNCTION_FLAGS_CAPSULE); + if (flags != NULL) { + flags->moved = 1; + flags->stands = 0; + } + } + PyErr_Clear(); + PyErr_Restore(type, value, traceback); + return 0; +} + +/* take back the answer of every site `names` holds for a function published in `dict` + * under `key`, or under any name at all where `key` is NULL */ +static void By_TakeBackNames(PyObject *names, PyObject *dict, PyObject *key) { + PyObject *lists = NULL; + Py_ssize_t cursor = 0; + if (key != NULL) { + PyObject *list = PyDict_GetItemWithError(names, key); + if (list == NULL || !PyList_Check(list)) return; + for (Py_ssize_t at = 0; at < PyList_GET_SIZE(list); at++) { + ByFunctionName *name = (ByFunctionName *)PyCapsule_GetPointer( + PyList_GET_ITEM(list, at), BY_FUNCTION_NAME_CAPSULE); + if (name != NULL && name->dict == dict) name->flags->stands = 0; + } + return; + } + while (PyDict_Next(names, &cursor, NULL, &lists)) { + if (!PyList_Check(lists)) continue; + for (Py_ssize_t at = 0; at < PyList_GET_SIZE(lists); at++) { + ByFunctionName *name = (ByFunctionName *)PyCapsule_GetPointer( + PyList_GET_ITEM(lists, at), BY_FUNCTION_NAME_CAPSULE); + if (name != NULL && name->dict == dict) name->flags->stands = 0; + } + } +} + +/* a write to a namespace some compiled module publishes functions in + * + * every write to a watched namespace comes through here, so the common one — a name no + * function is published under — is one lookup of an exact `str` that misses. a key that + * is not an exact `str` can still stand for a name, through a hash and an equality of its + * own, so a write under one takes back every answer the namespace gave. a write that + * empties the namespace, or throws it away, does the same. the lookups here run no python + * and cannot fail, so an exception already standing is left alone */ +static int By_FunctionNameWritten(PyDict_WatchEvent event, PyObject *dict, PyObject *key, + PyObject *new_value) { + (void)event; + (void)new_value; + ByFunctionWatch *watch = by_function_watch_registered; + if (watch == NULL) return 0; + if (key != NULL && PyUnicode_CheckExact(key)) { + /* a key already in a dict has its hash cached, and an exact `str` has no hash of + * its own to run for one that does not, so this cannot fail */ + Py_hash_t hash = ((PyASCIIObject *)key)->hash; + if (hash == -1) hash = PyObject_Hash(key); + if (!(watch->hashes & ((uint64_t)1 << ((uint64_t)hash & 63u)))) return 0; + By_TakeBackNames(watch->names, dict, key); + return 0; + } + By_TakeBackNames(watch->names, dict, NULL); + return 0; +} + +/* find this interpreter's watchers, registering them the first time. like the dict + * watcher's counter they are never freed: a site in any module may still be armed by them */ +static void By_FindFunctionWatch(void) { + PyInterpreterState *interpreter; + PyObject *state; + PyObject *capsule; + ByFunctionWatch *shared; + if (by_function_watch != NULL) return; + interpreter = PyInterpreterState_Get(); + if (interpreter == NULL) return; + state = PyInterpreterState_GetDict(interpreter); + if (state == NULL) return; + capsule = PyDict_GetItemString(state, BY_FUNCTION_WATCH_KEY); + if (capsule != NULL) { + by_function_watch = (ByFunctionWatch *)PyCapsule_GetPointer(capsule, BY_FUNCTION_WATCH_KEY); + if (by_function_watch == NULL) PyErr_Clear(); + return; + } + PyErr_Clear(); + shared = (ByFunctionWatch *)PyMem_RawMalloc(sizeof(ByFunctionWatch)); + if (shared == NULL) return; + shared->hashes = 0u; + shared->published = PyDict_New(); + shared->names = PyDict_New(); + if (shared->published == NULL || shared->names == NULL) { + PyErr_Clear(); + Py_XDECREF(shared->published); + Py_XDECREF(shared->names); + PyMem_RawFree(shared); + return; + } + shared->watcher = PyFunction_AddWatcher(By_FunctionChanged); + if (shared->watcher < 0) { + PyErr_Clear(); + Py_DECREF(shared->published); + Py_DECREF(shared->names); + PyMem_RawFree(shared); + return; + } + shared->dict_watcher = PyDict_AddWatcher(By_FunctionNameWritten); + if (shared->dict_watcher < 0) { + PyErr_Clear(); + (void)PyFunction_ClearWatcher(shared->watcher); + PyErr_Clear(); + Py_DECREF(shared->published); + Py_DECREF(shared->names); + PyMem_RawFree(shared); + return; + } + by_function_watch_registered = shared; + capsule = PyCapsule_New(shared, BY_FUNCTION_WATCH_KEY, NULL); + if (capsule == NULL) { + PyErr_Clear(); + by_function_watch = shared; + return; + } + if (PyDict_SetItemString(state, BY_FUNCTION_WATCH_KEY, capsule) < 0) PyErr_Clear(); + Py_DECREF(capsule); + by_function_watch = shared; +} + +#endif /* BY_FUNCTION_SITES */ + +/* take the function this module has just published under `name` in hand, and start + * being told about it + * + * a site that cannot be watched is left not standing, which sends every compiled call + * through the function object: slower, and the same program. a build with no function + * watchers — before 3.12, or free-threaded — is always in that state. `called` says + * whether any compiled call asks the site: one nobody asks needs no word of writes to its + * name, only of what happens to the function its own python entry stands for */ +static int By_ArmFunctionSite(ByFunctionSite *site, PyObject *dict, const char *name, int called) { + if (site->name == NULL) site->name = By_InternedStr(name, (Py_ssize_t)strlen(name)); + if (site->name == NULL) return -1; + PyObject *forwarder = PyDict_GetItemWithError(dict, site->name); + if (forwarder == NULL) { + if (!PyErr_Occurred()) { + PyErr_Format(PyExc_ImportError, "this module published nothing under '%s'", name); + } + return -1; + } + site->forwarder = Py_NewRef(forwarder); + if (PyFunction_Check(forwarder)) { + PyObject *twin = PyObject_GetAttrString(forwarder, "__wrapped__"); + if (twin == NULL) { + PyErr_Clear(); + } else if (PyFunction_Check(twin)) { + site->twin = twin; + } else { + Py_DECREF(twin); + } + } +#ifdef BY_FUNCTION_SITES + By_FindFunctionWatch(); + if (by_function_watch != NULL && site->twin != NULL) { + ByFunctionWatch *watch = by_function_watch; + PyObject *flags = PyCapsule_New(&site->flags, BY_FUNCTION_FLAGS_CAPSULE, NULL); + if (flags == NULL) return -1; + int failed = PyDict_SetItem(watch->published, forwarder, flags) < 0; + Py_DECREF(flags); + if (failed) return -1; + if (!called) return 0; + if (PyDict_Watch(watch->dict_watcher, dict) < 0) { + /* a namespace that will not be watched is one whose writes no site hears + * about, so nothing is answered for it */ + PyErr_Clear(); + return 0; + } + site->entry.dict = dict; + site->entry.flags = &site->flags; + PyObject *entry = PyCapsule_New(&site->entry, BY_FUNCTION_NAME_CAPSULE, NULL); + if (entry == NULL) return -1; + PyObject *list = PyDict_GetItemWithError(watch->names, site->name); + if (list == NULL) { + if (PyErr_Occurred()) { + Py_DECREF(entry); + return -1; + } + list = PyList_New(0); + if (list == NULL || PyDict_SetItem(watch->names, site->name, list) < 0) { + Py_XDECREF(list); + Py_DECREF(entry); + return -1; + } + Py_DECREF(list); + } + failed = PyList_Append(list, entry) < 0; + Py_DECREF(entry); + if (failed) return -1; + watch->hashes |= (uint64_t)1 << ((uint64_t)PyObject_Hash(site->name) & 63u); + site->watched = 1; + site->flags.stands = 1; + } +#endif + return 0; +} + +/* what `By_ResolveFunction` answers where the read failed. an address nothing can be, so + * NULL is left free to say the native entry stands */ +static const char by_resolve_failed = 0; +#define BY_RESOLVE_FAILED ((PyObject *)(void *)&by_resolve_failed) + +/* the object under a published function's name, read the way a global read is */ +BY_COLD PyObject *By_ResolveFunctionSlow(ByFunctionSite *site, PyObject *dict, const char *name) { + PyObject *found; + /* a call made while the module is still being imported reaches a site that is not + * armed yet, and the name is whatever the body has left there so far */ + if (site->name == NULL) { + site->name = By_InternedStr(name, (Py_ssize_t)strlen(name)); + if (site->name == NULL) return BY_RESOLVE_FAILED; + } + /* the name holds the function this module published, and nothing since has moved it: + * both watchers are listening again from here, so the direct call stands again */ + if (site->watched && !site->flags.moved) { + PyObject *held = PyDict_GetItemWithError(dict, site->name); + if (held == site->forwarder) { + site->flags.stands = 1; + return NULL; + } + if (held == NULL && PyErr_Occurred()) return BY_RESOLVE_FAILED; + } + found = By_LookupGlobal(dict, site->name); + return found != NULL ? found : BY_RESOLVE_FAILED; +} + +/* NULL where the native entry may stand in for the function object, the object under the + * name as a new reference where it may not, or `BY_RESOLVE_FAILED` with the error set */ +static inline PyObject *By_ResolveFunction(ByFunctionSite *site, PyObject *dict, const char *name) { + if (BY_LIKELY(site->flags.stands)) return NULL; + return By_ResolveFunctionSlow(site, dict, name); +} + +/* whether a compiled call may go straight to the native entry: nothing can fail, and a + * call that may not goes through the function object, which raises what python raises */ +static inline char By_FunctionStands(ByFunctionSite *site) { + return site->flags.stands; +} + +/* the object a call the native entry was turned away from goes through, as a new + * reference: what the name was read as, or where the read found the published function + * still standing, that function — whose code or defaults the arguments may since have + * moved, which python would then run */ +static inline PyObject *By_FunctionCallee(PyObject *resolved, PyObject *published) { + PyObject *callee = resolved != NULL ? resolved : published; + if (callee == NULL) { + PyErr_SetString(PyExc_SystemError, "a compiled call has no function to go through"); + return NULL; + } + return Py_NewRef(callee); +} + +/* a tuple of interned keyword names, built once per call site that passes them */ +BY_COLD PyObject *By_KeywordNames(Py_ssize_t count, ...) { + PyObject *names = PyTuple_New(count); + va_list given; + if (names == NULL) return NULL; + va_start(given, count); + for (Py_ssize_t at = 0; at < count; at++) { + const char *name = va_arg(given, const char *); + PyObject *interned = By_InternedStr(name, (Py_ssize_t)strlen(name)); + if (interned == NULL) { + va_end(given); + Py_DECREF(names); + return NULL; + } + PyTuple_SET_ITEM(names, at, interned); + } + va_end(given); + return names; +} + +/* `callee(*positional, **keywords)` with the arguments handed over one at a time: the + * `nargs` positional ones, then one per name in `kwnames` */ +BY_COLD PyObject *By_CallThrough(PyObject *callee, PyObject *kwnames, Py_ssize_t nargs, ...) { + PyObject *small[8]; + PyObject **argv = small; + Py_ssize_t total = nargs + (kwnames != NULL ? PyTuple_GET_SIZE(kwnames) : 0); + va_list given; + PyObject *result; + if (total > (Py_ssize_t)(sizeof(small) / sizeof(small[0]))) { + argv = (PyObject **)PyMem_Malloc((size_t)total * sizeof(PyObject *)); + if (argv == NULL) return PyErr_NoMemory(); + } + va_start(given, nargs); + for (Py_ssize_t at = 0; at < total; at++) argv[at] = va_arg(given, PyObject *); + va_end(given); + result = PyObject_Vectorcall(callee, argv, (size_t)nargs, kwnames); + if (argv != small) PyMem_Free(argv); + return result; +} + +/* the published function's own python entry, once its code or defaults have moved + * + * the compiled wrapper has the defaults it was compiled with written into it, so the call + * goes to the interpreted definition instead, carrying the defaults and keyword defaults + * the published function holds now — which is exactly what binding against the published + * function would have used */ +BY_COLD PyObject *By_CallMoved(ByFunctionSite *site, PyObject *const *args, Py_ssize_t nargs, + PyObject *kwnames) { + PyObject *twin = site->twin; + PyObject *forwarder = site->forwarder; + if (twin == NULL || forwarder == NULL || !PyFunction_Check(forwarder)) { + PyErr_SetString(PyExc_SystemError, + "a compiled function whose defaults moved has no interpreted definition"); + return NULL; + } + PyObject *defaults = PyFunction_GetDefaults(forwarder); + if (defaults != PyFunction_GetDefaults(twin) + && PyFunction_SetDefaults(twin, defaults ? defaults : Py_None) < 0) { + return NULL; + } + PyObject *keywords = PyFunction_GetKwDefaults(forwarder); + if (keywords != PyFunction_GetKwDefaults(twin) + && PyFunction_SetKwDefaults(twin, keywords ? keywords : Py_None) < 0) { + return NULL; + } + return PyObject_Vectorcall(twin, args, (size_t)nargs, kwnames); +} + /* whether a type spec can be built on this tuple of bases * * `PyType_FromSpecWithBases` gives the type it builds `type` as its own, so any base @@ -9211,7 +9847,7 @@ static inline PyObject *By_StepGenerator(PyObject *self, PyObject **sent, PyObje PyObject *(*resume)(PyObject *)) { if (By_RefuseResumption(*state, frame, arg) < 0) return NULL; By_ParkSent(sent, arg); - PyObject *result = resume(self); + PyObject *result = By_ResumeCounted(self, resume); if (result != NULL) return result; By_FinishGenerator(state); /* an empty `$returned` is what says the frame left by *raising* rather than by @@ -10731,7 +11367,7 @@ static inline PySendResult By_SendGenerator(PyObject *self, PyObject **sent, PyObject *step; if (By_RefuseResumption(*state, frame, arg) < 0) return PYGEN_ERROR; By_ParkSent(sent, arg); - step = resume(self); + step = By_ResumeCounted(self, resume); if (step != NULL) { *result = step; return PYGEN_NEXT; diff --git a/crates/ty/docs/configuration.md b/crates/ty/docs/configuration.md index cef194bf2a..9ac13a56d4 100644 --- a/crates/ty/docs/configuration.md +++ b/crates/ty/docs/configuration.md @@ -1114,6 +1114,88 @@ explicitly to ship fewer. --- +## `compile` + +### `bind-functions-early` + +Whether a compiled call to a function defined in the same module goes straight to +the function the module was compiled with. + +Python calls whatever the name holds when the call is made, so by default a compiled +module asks, on each such call, whether the name still holds the function it +published and whether that function's `__code__`, `__defaults__` and +`__kwdefaults__` are still the ones it was published with. A function rebound, +deleted or patched from outside the module — `mock.patch("app.helper")` included — +then reaches the module's own compiled code, as it reaches python's. + +Turning this on skips the question, which is worth the most on code that makes many +small calls. A compiled caller then goes on calling the function the module was +compiled with whatever the name holds, so a program that rebinds, deletes or patches +one of the module's functions, or reassigns its `__code__` or defaults, can tell the +difference. Calls made from python still reach whatever the name holds. + +**Default value**: `false` + +**Type**: `bool` + +**Example usage**: + +=== "pyproject.toml" + + ```toml + [tool.ty.compile] + bind-functions-early = true + ``` + +=== "ty.toml" + + ```toml + [compile] + bind-functions-early = true + ``` + +--- + +### `follow-recursion-limit` + +Whether a compiled recursion counts its calls against python's recursion limit. + +A compiled call pushes no python frame, so the build counts one wherever the +interpreter would have: on a call that goes round a cycle of compiled calls, on a +compiled method or dunder the interpreter calls, and on each step of a compiled +generator. With the count on, `sys.setrecursionlimit` reaches compiled code and a +recursion raises `RecursionError` at the depth python raises it. + +Turning it off saves the count on each of those calls, which is most of what a +deep recursion costs, and a compiled recursion then runs until the stack is close +to running out instead. A program that lowers the recursion limit, or catches +`RecursionError` at a particular depth, can tell the difference. + +Either way the stack itself is watched, and a recursion too deep for it raises +`RecursionError` rather than crashing. + +**Default value**: `true` + +**Type**: `bool` + +**Example usage**: + +=== "pyproject.toml" + + ```toml + [tool.ty.compile] + follow-recursion-limit = false + ``` + +=== "ty.toml" + + ```toml + [compile] + follow-recursion-limit = false + ``` + +--- + ## `editor` ### `common-aliases` diff --git a/crates/ty/src/by_commands.rs b/crates/ty/src/by_commands.rs index 10677d83e0..3e4a8a51d5 100644 --- a/crates/ty/src/by_commands.rs +++ b/crates/ty/src/by_commands.rs @@ -286,11 +286,12 @@ pub(crate) fn cmd_run( // the entry module is the exception, and it has to be: `runpy` needs a // code object to run something as `__main__`, and an extension has none. // so the entry stays interpreted and everything it imports is native - let options = by_build::Options { + let mut options = by_build::Options { fallback: Some(config), language: by_irbuild::Language::default(), ..by_build::Options::default() }; + project.apply_compile_options(&mut options); // no added context: every way `probe` fails already names the interpreter and says // what about it was refused, and a blanket "could not read its build configuration" // on top of them told a user to go and install headers when the real answer was that @@ -760,6 +761,26 @@ impl ResolvedProject { } } + /// What the project's `compile` options ask of a native build. + /// + /// Destructured, so an option added to the group and never carried across is a + /// compile error here rather than a setting that silently does nothing. + fn apply_compile_options(&self, options: &mut by_build::Options) { + let Some(ty_project::metadata::options::CompileOptions { + follow_recursion_limit, + bind_functions_early, + }) = self.metadata.options().compile.as_ref() + else { + return; + }; + if let Some(follow) = follow_recursion_limit { + options.follow_recursion_limit = *follow; + } + if let Some(early) = bind_functions_early { + options.bind_functions_early = *early; + } + } + /// The environment the project configures, as an absolute path. fn configured_environment(&self) -> Option { let sys_root = SystemPath::from_std_path(&self.root)?; @@ -1137,6 +1158,7 @@ pub(crate) fn cmd_compile( // built against one abi is unimportable by another, and `by run --compiled` // imports what this wrote let project = ResolvedProject::discover(&cwd)?; + project.apply_compile_options(&mut options); let python = discover_interpreter(None, &project)?.path; // see the note on the other `probe` call: its own errors are self-describing, and the // context that used to sit here misreported a version refusal as a missing header diff --git a/crates/ty/src/lib.rs b/crates/ty/src/lib.rs index d223fe0cf9..ac19499a8a 100644 --- a/crates/ty/src/lib.rs +++ b/crates/ty/src/lib.rs @@ -200,6 +200,8 @@ fn run_command(command: Command) -> anyhow::Result { fallback: None, language: by_irbuild::Language::default(), verify_install: !no_verify_install, + follow_recursion_limit: true, + bind_functions_early: false, }, }, ), diff --git a/crates/ty/tests/by_e2e.rs b/crates/ty/tests/by_e2e.rs index bae179f899..99e7c2d198 100644 --- a/crates/ty/tests/by_e2e.rs +++ b/crates/ty/tests/by_e2e.rs @@ -732,6 +732,114 @@ async def total(s: str, n: int) -> int: ); } +#[test] +fn compile_reads_whether_to_follow_the_recursion_limit_from_the_project() { + // the option is only worth anything if the command a project actually runs reads + // it: a module compiled with the count left out says so to the header it includes + let emitted = |name: &str, table: &str| -> Option { + let dir = cli_root().join(name); + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).unwrap(); + fs::write( + dir.join("pyproject.toml"), + format!("[project]\nname=\"s\"\nversion=\"0\"\nrequires-python=\">=3.13\"\n{table}"), + ) + .unwrap(); + fs::write( + dir.join("deep.py"), + "def depth(n: int) -> int:\n if n == 0:\n return 0\n return depth(n - 1) + 1\n", + ) + .unwrap(); + let result = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") + .args(["compile", "--emit-c-only", "deep.py"]) + .current_dir(&dir) + .output() + .expect("failed to spawn by"); + if refused_for_python_version(&result) { + return None; + } + assert!( + result.status.success(), + "by exited with error:\n{}", + String::from_utf8_lossy(&result.stderr) + ); + Some(fs::read_to_string(dir.join("build").join("deep.c")).expect("the C is readable")) + }; + let Some(counted) = emitted("by_cli_recursion_default", "") else { + return; + }; + assert!( + counted.contains("By_DepthEnter"), + "the recursion is guarded" + ); + assert!(!counted.contains("BY_RECURSION_STACK_ONLY")); + let Some(stack_only) = emitted( + "by_cli_recursion_stack_only", + "\n[tool.ty.compile]\nfollow-recursion-limit = false\n", + ) else { + return; + }; + assert!(stack_only.contains("#define BY_RECURSION_STACK_ONLY 1")); + assert!( + stack_only.contains("By_DepthEnter"), + "the stack is still watched" + ); +} + +#[test] +fn compile_reads_whether_to_bind_functions_early_from_the_project() { + // a compiled call asks whether the name still holds the function unless the project + // says the module's functions are never rebound, and then it asks nothing + let emitted = |name: &str, table: &str| -> Option { + let dir = cli_root().join(name); + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).unwrap(); + fs::write( + dir.join("pyproject.toml"), + format!("[project]\nname=\"s\"\nversion=\"0\"\nrequires-python=\">=3.13\"\n{table}"), + ) + .unwrap(); + fs::write( + dir.join("calls.py"), + "def add(a: int, b: int) -> int:\n return a + b\n\n\ndef twice(a: int) -> int:\n return add(a, a)\n", + ) + .unwrap(); + let result = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") + .args(["compile", "--emit-c-only", "calls.py"]) + .current_dir(&dir) + .output() + .expect("failed to spawn by"); + if refused_for_python_version(&result) { + return None; + } + assert!( + result.status.success(), + "by exited with error:\n{}", + String::from_utf8_lossy(&result.stderr) + ); + Some(fs::read_to_string(dir.join("build").join("calls.c")).expect("the C is readable")) + }; + let Some(asked) = emitted("by_cli_binding_default", "") else { + return; + }; + assert!( + asked.contains("By_ResolveFunction(&by_fs_add"), + "the call asks" + ); + let Some(early) = emitted( + "by_cli_binding_early", + "\n[tool.ty.compile]\nbind-functions-early = true\n", + ) else { + return; + }; + assert!( + !early.contains("By_ResolveFunction"), + "the call asks nothing" + ); +} + #[test] fn run_executes_module() { let dir = tempfile::tempdir().expect("tempdir"); diff --git a/crates/ty/tests/cli/config_option.rs b/crates/ty/tests/cli/config_option.rs index 5bd32f5454..4717110b6a 100644 --- a/crates/ty/tests/cli/config_option.rs +++ b/crates/ty/tests/cli/config_option.rs @@ -123,7 +123,7 @@ fn cli_config_args_invalid_option() -> anyhow::Result<()> { | 1 | bad-option=true | ^^^^^^^^^^ - unknown field `bad-option`, expected one of `type-checking-preset`, `environment`, `src`, `rules`, `terminal`, `analysis`, `experimental`, `run`, `build`, `lowering`, `editor`, `overrides` + unknown field `bad-option`, expected one of `type-checking-preset`, `environment`, `src`, `rules`, `terminal`, `analysis`, `experimental`, `run`, `build`, `lowering`, `compile`, `editor`, `overrides` Usage: by diff --git a/crates/ty_project/src/metadata/options.rs b/crates/ty_project/src/metadata/options.rs index bd9d464d42..718fcebfd7 100644 --- a/crates/ty_project/src/metadata/options.rs +++ b/crates/ty_project/src/metadata/options.rs @@ -149,6 +149,12 @@ pub struct Options { #[option_group] pub lowering: Option, + /// Configures what the native modules `by compile` builds may assume about the + /// program that runs them. + #[serde(skip_serializing_if = "Option::is_none")] + #[option_group] + pub compile: Option, + /// Configures the parts of the editor experience that type checking does not decide. #[serde(skip_serializing_if = "Option::is_none")] #[option_group] @@ -1805,6 +1811,73 @@ impl Combine for FloatLiteralLowering { } } +#[derive( + Debug, + Default, + Clone, + Eq, + PartialEq, + Hash, + Combine, + Serialize, + Deserialize, + OptionsMetadata, + get_size2::GetSize, +)] +#[serde(rename_all = "kebab-case", deny_unknown_fields)] +#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] +pub struct CompileOptions { + /// Whether a compiled recursion counts its calls against python's recursion limit. + /// + /// A compiled call pushes no python frame, so the build counts one wherever the + /// interpreter would have: on a call that goes round a cycle of compiled calls, on a + /// compiled method or dunder the interpreter calls, and on each step of a compiled + /// generator. With the count on, `sys.setrecursionlimit` reaches compiled code and a + /// recursion raises `RecursionError` at the depth python raises it. + /// + /// Turning it off saves the count on each of those calls, which is most of what a + /// deep recursion costs, and a compiled recursion then runs until the stack is close + /// to running out instead. A program that lowers the recursion limit, or catches + /// `RecursionError` at a particular depth, can tell the difference. + /// + /// Either way the stack itself is watched, and a recursion too deep for it raises + /// `RecursionError` rather than crashing. + #[serde(skip_serializing_if = "Option::is_none")] + #[option( + default = r#"true"#, + value_type = "bool", + example = r#" + follow-recursion-limit = false + "# + )] + pub follow_recursion_limit: Option, + + /// Whether a compiled call to a function defined in the same module goes straight to + /// the function the module was compiled with. + /// + /// Python calls whatever the name holds when the call is made, so by default a compiled + /// module asks, on each such call, whether the name still holds the function it + /// published and whether that function's `__code__`, `__defaults__` and + /// `__kwdefaults__` are still the ones it was published with. A function rebound, + /// deleted or patched from outside the module — `mock.patch("app.helper")` included — + /// then reaches the module's own compiled code, as it reaches python's. + /// + /// Turning this on skips the question, which is worth the most on code that makes many + /// small calls. A compiled caller then goes on calling the function the module was + /// compiled with whatever the name holds, so a program that rebinds, deletes or patches + /// one of the module's functions, or reassigns its `__code__` or defaults, can tell the + /// difference. Calls made from python still reach whatever the name holds. + #[serde(skip_serializing_if = "Option::is_none")] + #[option( + default = r#"false"#, + value_type = "bool", + example = r#" + bind-functions-early = true + "# + )] + pub bind_functions_early: Option, +} + #[derive( Debug, Default, diff --git a/docs/basedpython/development/compilation/index.md b/docs/basedpython/development/compilation/index.md index cecf8b4cc5..ff39d62ca6 100644 --- a/docs/basedpython/development/compilation/index.md +++ b/docs/basedpython/development/compilation/index.md @@ -166,7 +166,10 @@ and each assumption is one the user opts into explicitly: | 3 | sealed unit | `api.lock` is the complete public surface | cross-module inlining, monomorphization, dead-code elimination, LTO | tier 2 is the default for a project, tier 1 for a single module compiled out of -a larger interpreted program. tier 3 requires a current +a larger interpreted program. the one tier-2 assumption the build makes today is opt-in +on its own: a call to one of a module's own functions is late-bound unless the project +sets `bind-functions-early` in its `[tool.ty.compile]` table — see +[runtime](runtime.md#a-module-function-rebound-from-outside). tier 3 requires a current [api lockfile](../../features/api-lock.md) — the lockfile stops being only a review artifact and becomes the ABI contract. see [optimizations](optimizations.md#the-lockfile-as-a-closed-world-boundary) diff --git a/docs/basedpython/development/compilation/plan.md b/docs/basedpython/development/compilation/plan.md index e22c417bca..4770c25edd 100644 --- a/docs/basedpython/development/compilation/plan.md +++ b/docs/basedpython/development/compilation/plan.md @@ -9,23 +9,23 @@ python is a smaller problem for us than it is for mypyc, because basedpython has already ruled out most of what makes compilation observable. the table below is short for that reason: -| behaviour | interpreted | compiled | why | -| ----------------------------------------------- | ------------------------------------ | ---------------------------------- | ------------------------------------------------------------------------------------- | -| monkey-patching a module function | works | tier 3: no | early binding, gated on `api.lock` | -| adding an attribute not in the class body | stored, unless `__slots__` | the same, on 3.13 and above | an instance dict beside the layout; below 3.13 it is refused | -| `is` on a fixed-length tuple | identity | unspecified | unboxed tuples have no identity | -| `type(x)` where `x: int` held a `bool` | `bool` | `int` | the tagged form has no room for it | -| `is` on a small `int` or interned `str` | identity | unspecified | unboxed and re-boxed values differ | -| a wrong annotation on non-compiled code | `TypeError` (soundness checks) | `TypeError` | same mechanism, already shipped | -| `__dict__` on an instance | present unless `__slots__` | present, over the whole state | a real `dict` every write updates; `del` of a *field* raises | -| `if __name__ == "__main__"` | n/a | n/a | the entry point is [`main`](../../features/main-function.md) | -| stack depth on deep recursion | python's limit | the C stack's | native frames are not python frames | -| `f.__code__` for a compiled function | the definition's | the forwarder's | the module publishes a `function` that forwards to the native, and that is its code | -| `f.__wrapped__` for a compiled function | absent | the interpreted definition | which is where `inspect` reads the signature, the source and the file from | -| a traceback through a compiled function | the module's file and the line | the same, with no column positions | each entry hangs off a frame made for it, with no bytecode behind it | -| `is` against a compiled function's own name | identity | `False` where the body captured it | the module body runs before the compiled name replaces it, and what it captured stays | -| a `data class` constructor's argument types | unchecked (unless `--soundness all`) | checked | a compiled field is unboxed, so the check is mandatory | -| assigning a method onto a class from outside it | works | `TypeError` at the assignment | an emitted class is sealed unless the source decorates it | +| behaviour | interpreted | compiled | why | +| ----------------------------------------------- | ------------------------------------ | ------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------- | +| monkey-patching a module function | works | works, unless the project binds early | a compiled call asks whether the name still holds the function; see [runtime](runtime.md#a-module-function-rebound-from-outside) | +| adding an attribute not in the class body | stored, unless `__slots__` | the same, on 3.13 and above | an instance dict beside the layout; below 3.13 it is refused | +| `is` on a fixed-length tuple | identity | unspecified | unboxed tuples have no identity | +| `type(x)` where `x: int` held a `bool` | `bool` | `int` | the tagged form has no room for it | +| `is` on a small `int` or interned `str` | identity | unspecified | unboxed and re-boxed values differ | +| a wrong annotation on non-compiled code | `TypeError` (soundness checks) | `TypeError` | same mechanism, already shipped | +| `__dict__` on an instance | present unless `__slots__` | present, over the whole state | a real `dict` every write updates; `del` of a *field* raises | +| `if __name__ == "__main__"` | n/a | n/a | the entry point is [`main`](../../features/main-function.md) | +| stack depth on deep recursion | python's limit | python's limit, and the C stack's | a native frame lives on the stack; see [runtime](runtime.md#how-deep-a-recursion-goes) | +| `f.__code__` for a compiled function | the definition's | the forwarder's | the module publishes a `function` that forwards to the native, and that is its code | +| `f.__wrapped__` for a compiled function | absent | the interpreted definition | which is where `inspect` reads the signature, the source and the file from | +| a traceback through a compiled function | the module's file and the line | the same, with no column positions | each entry hangs off a frame made for it, with no bytecode behind it | +| `is` against a compiled function's own name | identity | `False` where the body captured it | the module body runs before the compiled name replaces it, and what it captured stays | +| a `data class` constructor's argument types | unchecked (unless `--soundness all`) | checked | a compiled field is unboxed, so the check is mandatory | +| assigning a method onto a class from outside it | works | `TypeError` at the assignment | an emitted class is sealed unless the source decorates it | the `is`-related rows are the only genuine losses, and the first three are the same ones mypyc takes. the fourth — `is` against a function's own name — is ours: diff --git a/docs/basedpython/development/compilation/runtime.md b/docs/basedpython/development/compilation/runtime.md index f9fc662b0f..3191e3989b 100644 --- a/docs/basedpython/development/compilation/runtime.md +++ b/docs/basedpython/development/compilation/runtime.md @@ -506,7 +506,9 @@ are early-bound: a reference from a compiled function reads a static slot rather than doing a namespace lookup a non-`let` module global keeps late binding, and a builtin is late-bound too — a -module that rebinds `str` is obeyed. each call site remembers what its name last +module that rebinds `str` is obeyed. so is a call to one of the module's own functions, +which goes straight to its native entry only while the name still holds the function the +module published, and while nothing about that function has been reassigned. each call site remembers what its name last resolved to, and re-derives it whenever any namespace has been written to since, which a dict watcher on those namespaces is what says. so the binding is still looked up rather than assumed, and looking it up again is what a rebinding costs @@ -638,6 +640,78 @@ the same reason mypyc does — the fixed layout must be initialized. a each of these is a tradeoff made on purpose, and each names the shape of program that can tell the two builds apart +### a module function rebound from outside + +a compiled call to a function the same module defines goes straight to its native entry +while the name holds the function the module published, and through whatever the name +holds once it does not, so a function rebound, deleted or patched from another module +reaches the module's own compiled callers: + +```python +def add(a: int, b: int) -> int: + return a + b + + +def run(n: int) -> int: + total = 0 + i = 0 + while i < n: + total = add(total, i) + i = i + 1 + return total + + +# from another module +from unittest import mock + +with mock.patch.object(mod, "add", lambda a, b: 7): + mod.run(3) # 7 on both +``` + +the question costs a load and a test on each such call, which is most of what a call +with a one-line body costs — `calls` takes 43% more instructions than a build that does +not ask — and a write to any name in the module costs a few dozen instructions more, +where the module is told which name it was. `bind-functions-early = true` in the +project's `[tool.ty.compile]` table makes the closed-world assumption instead: a compiled +caller goes on calling the function the module was compiled with, so a program that +rebinds, deletes or patches one of the module's functions, or reassigns its `__code__` or +its defaults, can tell. a call made from python still reaches whatever the name holds, and +the function's own python entry still binds from the defaults it holds: + +```python +# built with `bind-functions-early = true` +mod.add = lambda a, b: 100 +mod.add(1, 2) # 100 on both +mod.run(3) # python: 100; compiled: 3 + +del mod.add +mod.run(3) # python: NameError; compiled: 3 +``` + +a call that hands over a list the caller holds as a buffer cannot be made through a +replacement, for the reason a rebound `len` cannot be handed one, and raises +`RuntimeError` the same way: + +```python +def total(xs: list[float]) -> float: + out = 0.0 + i = 0 + while i < len(xs): + out = out + xs[i] + i = i + 1 + return out + + +def built(n: int) -> float: + xs = [i * 0.5 for i in range(n)] + return total(xs) + + +# from another module +mod.total = lambda xs: len(xs) +mod.built(3) # python: 3; compiled: RuntimeError +``` + ### a rebound `len` and a list held as a buffer a list of unboxed values that never leaves its function is held as a buffer @@ -664,6 +738,95 @@ mod.len = lambda xs: 1 mod.total(3) # python: 0.0; compiled: RuntimeError ``` +### how deep a recursion goes + +a compiled call pushes a C frame rather than a python one, so a compiled module counts +the frames python would have pushed itself, against the interpreter's own recursion +limit: on a call that goes round a cycle of compiled calls, on a compiled method, dunder +or nested function the interpreter calls, and on each step of a compiled generator or +coroutine. a recursion raises `RecursionError` at the depth python raises it, and +`sys.setrecursionlimit` reaches it. + +a C frame lives on the thread's stack, where a python frame lives on the heap, so the +stack is watched as well, and a compiled recursion raises `RecursionError` once three +quarters of the stack is used, whatever the limit says. a program that raises the +limit far enough can tell: + +```python +def depth(n: int) -> int: + if n == 0: + return 0 + return depth(n - 1) + 1 + + +# from another module +import sys + +sys.setrecursionlimit(10**7) +mod.depth(500_000) # python: 500000; compiled: RecursionError +``` + +a compiled call that goes round no cycle of compiled calls is not counted: it can only +be as deep as the module is long, and counting every call would put a cost on every +call. so a recursion reached through one reaches the limit one frame later for each +such call in front of it: + +```python +def nested_depth(n: int) -> int: + def inner(k: int) -> int: + if k == 0: + return 0 + return inner(k - 1) + 1 + + return inner(n) # this call is not counted + + +# from another module +sys.setrecursionlimit(100) +mod.nested_depth(98) # python: RecursionError; compiled: 98 +``` + +counting costs most on a body that does little but recurse — half again the +instructions of a plain `fib` — so `follow-recursion-limit = false` in the project's +`[tool.ty.compile]` table leaves the count out and watches the stack alone. a compiled +recursion then runs until the stack is close to running out, and a program that lowers +the limit, or catches `RecursionError` at a depth it expects, can tell: + +```python +sys.setrecursionlimit(100) +mod.depth(200) # python: RecursionError; compiled with the count off: 200 +``` + +neither setting lets a recursion crash the process + +### a function's defaults edited in place + +python binds a missing argument from the function object's `__defaults__` and +`__kwdefaults__` as each call is made. a compiled module function is told when either is +reassigned, or its `__code__` is, and from then on every call to it — from python or from +compiled code — goes through the function object and the interpreted definition, given +the defaults the function holds then. an edit made *inside* the `__kwdefaults__` dict is +not a reassignment and nothing reports it, so until the function has been told of one the +defaults it was compiled with go on standing in: + +```python +def scaled(a: int, *, scale: int = 1) -> int: + return a * scale + + +# from another module +mod.scaled.__kwdefaults__["scale"] = 10 +mod.scaled(2) # python: 20; compiled: 2 + +mod.scaled.__kwdefaults__ = {"scale": 10} +mod.scaled(2) # 20 on both, and every edit to that dict is seen from here on +``` + +watching the dict itself would put a test on every call for a program that almost never +edits it. a function compiled from a module is also published as a closure over its +native entry, so a `__code__` it is given has to close over as many names: +`mod.scaled.__code__ = (lambda a: a).__code__` raises `ValueError` where python takes it + ### one module object for the whole process a compiled module keeps its namespace, and every memo of a name in it, in state diff --git a/ty.schema.json b/ty.schema.json index 5d6a1d7271..2454eb75a5 100644 --- a/ty.schema.json +++ b/ty.schema.json @@ -24,6 +24,17 @@ } ] }, + "compile": { + "description": "Configures what the native modules `by compile` builds may assume about the\nprogram that runs them.", + "anyOf": [ + { + "$ref": "#/definitions/CompileOptions" + }, + { + "type": "null" + } + ] + }, "editor": { "description": "Configures the parts of the editor experience that type checking does not decide.", "anyOf": [ @@ -366,6 +377,26 @@ "type": "string" } }, + "CompileOptions": { + "type": "object", + "properties": { + "bind-functions-early": { + "description": "Whether a compiled call to a function defined in the same module goes straight to\nthe function the module was compiled with.\n\nPython calls whatever the name holds when the call is made, so by default a compiled\nmodule asks, on each such call, whether the name still holds the function it\npublished and whether that function's `__code__`, `__defaults__` and\n`__kwdefaults__` are still the ones it was published with. A function rebound,\ndeleted or patched from outside the module — `mock.patch(\"app.helper\")` included —\nthen reaches the module's own compiled code, as it reaches python's.\n\nTurning this on skips the question, which is worth the most on code that makes many\nsmall calls. A compiled caller then goes on calling the function the module was\ncompiled with whatever the name holds, so a program that rebinds, deletes or patches\none of the module's functions, or reassigns its `__code__` or defaults, can tell the\ndifference. Calls made from python still reach whatever the name holds.", + "type": [ + "boolean", + "null" + ] + }, + "follow-recursion-limit": { + "description": "Whether a compiled recursion counts its calls against python's recursion limit.\n\nA compiled call pushes no python frame, so the build counts one wherever the\ninterpreter would have: on a call that goes round a cycle of compiled calls, on a\ncompiled method or dunder the interpreter calls, and on each step of a compiled\ngenerator. With the count on, `sys.setrecursionlimit` reaches compiled code and a\nrecursion raises `RecursionError` at the depth python raises it.\n\nTurning it off saves the count on each of those calls, which is most of what a\ndeep recursion costs, and a compiled recursion then runs until the stack is close\nto running out instead. A program that lowers the recursion limit, or catches\n`RecursionError` at a particular depth, can tell the difference.\n\nEither way the stack itself is watched, and a recursion too deep for it raises\n`RecursionError` rather than crashing.", + "type": [ + "boolean", + "null" + ] + } + }, + "additionalProperties": false + }, "EditorOptions": { "type": "object", "properties": { From 0275608d05db6020f359d85ac2fea8cfee3d3a65 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 15 Sep 2026 01:55:41 +1000 Subject: [PATCH 04/16] native: guard loops once on the way in, from an analysis of what can run python add an analysis of which operations can run python code over the native call graph, and use it to ask once, on the way into a loop, whether a builtin or module function still stands and whether an int field is set; read an extended class's fields through one licence per class --- crates/by_build/tests/differential.rs | 353 ++++++ crates/by_codegen_c/src/lib.rs | 785 ++++++++++-- crates/by_ir/src/ops.rs | 76 ++ crates/by_ir/src/print.rs | 33 + crates/by_ir/src/verify.rs | 68 +- crates/by_irbuild/src/lib.rs | 136 ++- crates/by_irbuild/src/tests.rs | 8 +- crates/by_opt/src/copy_propagation.rs | 4 + crates/by_opt/src/guard_loops.rs | 1139 ++++++++++++++++++ crates/by_opt/src/infallible.rs | 6 +- crates/by_opt/src/lib.rs | 8 + crates/by_opt/src/runs_python.rs | 1582 +++++++++++++++++++++++++ crates/by_rt/include/by.h | 200 +++- 13 files changed, 4228 insertions(+), 170 deletions(-) create mode 100644 crates/by_opt/src/guard_loops.rs create mode 100644 crates/by_opt/src/runs_python.rs diff --git a/crates/by_build/tests/differential.rs b/crates/by_build/tests/differential.rs index 4afd7f5387..edffa3b178 100644 --- a/crates/by_build/tests/differential.rs +++ b/crates/by_build/tests/differential.rs @@ -921,6 +921,14 @@ def _capture(fn, *args): return e return None +# a subclass of `base` whose `name` rebinds `len` in `module` to answer `n`, then answers +# as `answer` would +def _sub(module, base, name, n, answer): + def method(self, *args): + module.__dict__['len'] = lambda o: n + return answer(self, *args) + return type(base.__name__ + 'Sub', (base,), {name: method}) + # the deepest argument `fn` still answers for under a recursion limit of `limit`. # every frame between here and `fn` is the same in both legs, so the number is the # count of frames `fn` itself spends @@ -2184,6 +2192,84 @@ fn a_rebound_module_function_reaches_compiled_callers() { ); } +#[test] +fn a_function_rebound_while_a_loop_or_a_recursion_runs_is_called_from_the_next_call_on() { + // a loop or a function may ask whether a module function still stands once on the way + // in, because only python code can rebind it or swap its code. these are the ways python + // code runs while one does: a call in the body, an `int` subclass whose own arithmetic or + // comparison the body reaches, and a callback at some depth of a recursion. each rebinds + // or swaps the function partway, and python calls what the name holds from the next call + agree_python( + "reboundwhile", + "\ +import sys +from collections.abc import Callable + +namespace = sys.modules[__name__].__dict__ + + +def add(a: int, b: int) -> int: + return a + b + + +def run(n: int) -> int: + total = 0 + i = 0 + while i < n: + total = add(total, i) + i = i + 1 + return total + + +def run_with(n: int, hook: Callable[[int], None]) -> int: + total = 0 + i = 0 + while i < n: + if i % 2 == 1: + hook(i) + total = add(total, i) + i = i + 1 + return total + + +def depth(n: int) -> int: + if n < 1: + return 0 + return depth(n - 1) + 1 + + +def depth_with(n: int, hook: Callable[[int], None]) -> int: + if n < 1: + return 0 + hook(n) + return depth_with(n - 1, hook) + 1 + + +def restored(name: str, call: Callable[[], object]) -> object: + saved = namespace[name] + try: + return call() + except Exception as error: + return repr(error) + finally: + namespace[name] = saved +", + &[ + "[m.run(5), m.depth(5), m.run_with(6, lambda i: None), m.depth_with(4, lambda n: None)]", + "m.restored('add', lambda: m.run_with(6, lambda i: m.namespace.__setitem__('add', lambda a, b: a * 100) if i == 3 else None))", + "m.restored('add', lambda: m.run_with(6, lambda i: m.namespace.pop('add') if i == 3 else None))", + "m.restored('add', lambda: m.run_with(6, lambda i: setattr(m.add, '__code__', _code_answering(m.add, 7)) if i == 3 else None))", + "m.restored('depth_with', lambda: m.depth_with(6, lambda n: m.namespace.__setitem__('depth_with', lambda k, h: 1000) if n == 3 else None))", + // an `int` subclass runs python in the comparison and the subtraction the body + // reaches before each call + "m.restored('depth', lambda: m.depth(type('I', (int,), {'__lt__': lambda s, o: (m.namespace.__setitem__('depth', lambda k: 500), False)[1], '__sub__': lambda s, o: 2})(2 ** 70)))", + "m.restored('depth', lambda: m.depth(type('I', (int,), {'__sub__': lambda s, o: (m.namespace.__setitem__('depth', lambda k: 50), 2)[1]})(2 ** 70)))", + // a recursion limit lowered from a callback partway through + "(lambda saved: (sys := __import__('sys'), [type(_capture(m.depth_with, 400, lambda n: sys.setrecursionlimit(120) if n == 390 else None)).__name__, sys.setrecursionlimit(saved)][0])[1])(__import__('sys').getrecursionlimit())", + ], + ); +} + #[test] fn a_build_binding_functions_early_goes_on_calling_what_it_compiled() { // the configuration that assumes the module's functions are never rebound asks @@ -15032,6 +15118,66 @@ fn a_field_of_a_class_something_extends_is_read_by_name_where_the_name_is_taken_ agree_python("extfield", EXTENDED_FIELD, &calls); } +#[test] +fn a_property_and_a_field_of_an_extended_class_agree_whatever_changes_under_the_loop() { + // an accessor and a field of a class something extends are reached directly only while + // the receiver is exactly the class and the class is as it was compiled — every other + // receiver, and the class after anything is written to it, goes through the protocol. + // what changes is asked from inside the loop too: a getter that replaces the property + // partway, a field the class takes over with a property, an interpreted subclass whose + // reads go the long way round, and an instance whose field was never set + agree_python( + "extloop", + "\ +class Cell: + def __init__(self) -> None: + self._v = 0 + self._w = 5 + + @property + def v(self) -> int: + return self._v + + @v.setter + def v(self, given: int) -> None: + self._v = given + + def width(self) -> int: + return self._w + + +class Sized(Cell): + def extra(self) -> int: + return 1 + + +def run(cell: Cell, n: int) -> int: + i = 0 + while i < n: + cell.v = cell.v + 1 + i = i + 1 + return cell.v + cell.width() + + +class Swapping(Cell): + def __init__(self) -> None: + self._v = 0 + self._w = 5 + self.count = 0 +", + &[ + "[m.run(m.Cell(), 10), m.run(m.Sized(), 10)]", + // a getter that replaces the property on the class after a few reads + "(lambda c: (setattr(m.Cell, 'v', property(lambda s: (s._v, setattr(m.Cell, 'v', property(lambda t: 10000, lambda t, x: setattr(t, '_v', x))) if s._v == 3 else None)[0], lambda s, x: setattr(s, '_v', x))), m.run(c, 10))[-1])(m.Swapping())", + // the field taken over by a property once the loop has run + "(lambda c: (m.run(c, 3), setattr(m.Cell, '_w', property(lambda s: 42)), m.run(c, 3))[1:])(m.Cell.__new__(m.Cell).__class__())", + // an interpreted subclass, whose reads go by name + "m.run(type('S', (m.Cell,), {})(), 2)", + "[repr(_capture(m.run, m.Cell.__new__(m.Cell), 2)), _frames(_capture(m.run, m.Cell.__new__(m.Cell), 2))]", + ], + ); +} + #[test] fn a_subclass_overriding_one_half_of_a_property_agrees() { agree_python( @@ -36092,6 +36238,118 @@ def patched(name: str, value: object, call: Callable[[], object]) -> object: } } +#[test] +fn a_builtin_rebound_while_a_loop_runs_is_called_from_the_next_call_on() { + // a loop may ask whether `len` is still the builtin once on the way in, because only + // python code can rebind it. these are the ways python code runs while a loop does: a + // call in the body, on every trip or only some; a `str`, `list` or `int` subclass whose + // `__len__`, `__getitem__` or comparison the loop reaches; and a rebinding in `builtins` + // rather than in the module. each rebinds `len` partway through, and python calls the + // new one from the next call on — in the same trip where the call comes after + agree_python( + "rebuiltinloop", + "\ +import builtins +import sys +from collections.abc import Callable + +namespace = sys.modules[__name__].__dict__ + + +def rebind_at(trip: int, where: object = None) -> Callable[[int], None]: + def hook(i: int) -> None: + if i == trip: + target = namespace if where is None else builtins.__dict__ + target['len'] = lambda o: 2 + return hook + + +def restored(call: Callable[[], object]) -> object: + original = builtins.len + try: + return call() + except Exception as error: + return repr(error) + finally: + namespace.pop('len', None) + builtins.len = original + + +def every_trip(s: str, hook: Callable[[int], None]) -> list[int]: + seen = [] + i = 0 + while i < len(s): + hook(i) + seen.append(len(s)) + i = i + 1 + return seen + + +def some_trips(s: str, hook: Callable[[int], None]) -> int: + total = 0 + i = 0 + while i < len(s): + if i % 2 == 1: + hook(i) + total = total + len(s) + i = i + 1 + return total + + +def scan(line: str) -> int: + best = 0 + run = 0 + i = 0 + while i < len(line): + if line[i] == ' ': + if run > best: + best = run + run = 0 + else: + run = run + 1 + i = i + 1 + return best + + +def measured(xs: list[float], ys: list[float]) -> float: + out = 0.0 + i = 0 + while i < len(xs): + out = out + xs[i] * ys[i] + i = i + 1 + return out + + +def bounded(n: int) -> int: + total = 0 + i = 0 + while i < n: + total = total + len('abc') + i = i + 1 + return total +", + &[ + // `_sub(m, base, name, n, answer)` is a subclass the interpreter builds, whose `name` + // rebinds `len` to answer `n` and then answers as `answer` would + "m.restored(lambda: m.every_trip('abcde', m.rebind_at(1)))", + "m.restored(lambda: m.every_trip('abcde', m.rebind_at(1, 'builtins')))", + "m.restored(lambda: m.some_trips('abcdefgh', m.rebind_at(3)))", + "m.restored(lambda: m.every_trip('abc', m.rebind_at(9)))", + "m.restored(lambda: m.scan('ab cde f'))", + "m.restored(lambda: m.scan(_sub(m, str, '__len__', 4, str.__len__)('ab cde f')))", + "m.restored(lambda: m.scan(_sub(m, str, '__getitem__', 3, str.__getitem__)('ab cde fghij')))", + "m.restored(lambda: m.measured([1.0, 2.0, 3.0], [4.0, 5.0, 6.0]))", + "m.restored(lambda: m.measured(_sub(m, list, '__getitem__', 1, list.__getitem__)([1.0, 2.0, 3.0]), [4.0, 5.0, 6.0]))", + "m.restored(lambda: m.measured([1.0, 2.0, 3.0], _sub(m, list, '__getitem__', 1, list.__getitem__)([4.0, 5.0, 6.0])))", + "m.restored(lambda: m.bounded(3))", + "m.restored(lambda: m.bounded(_sub(m, int, '__gt__', 7, lambda self, other: other < 3)(2 ** 70)))", + "m.restored(lambda: m.scan(''))", + // a name bound nowhere is python's `NameError` at the first call, not before it + "m.restored(lambda: (builtins.__dict__.pop('len'), m.scan('ab c')))", + ], + ); +} + #[test] fn a_special_method_a_descriptor_builds_is_looked_up_as_python_looks_it_up() { // a `with` statement and a method call each remember what the name resolved to on @@ -36263,6 +36521,98 @@ def nonempty(xs: object) -> bool: ); } +#[test] +fn an_unset_int_field_read_where_it_is_consumed_raises_where_python_does() { + // an `int` field `__init__` always assigns is still absent on an instance `__new__` + // made, and a read of it raises `AttributeError` — from the frame and the line the read + // is written in, before anything after it runs. the read is asked in several shapes: + // one that keeps its own reference, one whose value goes straight into arithmetic, one + // inside an accessor, a `with` block's `__enter__` and `__exit__`, and a value too big + // to be a short, which a read keeps a reference to + agree_python( + "unsetconsumed", + "\ +class State: + def __init__(self) -> None: + self.a = 0 + self.b = 1 + self.c = 2 + + +def step(state: State, n: int) -> int: + i = 0 + while i < n: + state.a = state.b + state.c + state.b = state.a - state.c + state.c = state.c + 1 + i = i + 1 + return state.a + state.b + state.c + + +class Cell: + def __init__(self) -> None: + self._v = 0 + + @property + def v(self) -> int: + return self._v + + @v.setter + def v(self, given: int) -> None: + self._v = given + + +def bump(cell: Cell, n: int) -> int: + i = 0 + while i < n: + cell.v = cell.v + 1 + i = i + 1 + return cell.v + + +class Guard: + depth: int + + def __init__(self) -> None: + self.depth = 0 + + def __enter__(self) -> 'Guard': + self.depth = self.depth + 1 + return self + + def __exit__(self, kind: object, value: object, trace: object) -> None: + self.depth = self.depth - 1 + + +def guarded(guard: Guard, n: int) -> int: + total = 0 + i = 0 + while i < n: + with guard: + total = total + guard.depth + i = i + 1 + return total +", + &[ + "[m.step(m.State(), 3), m.bump(m.Cell(), 3), m.guarded(m.Guard(), 3)]", + "_frames(_capture(m.step, m.State.__new__(m.State), 3))", + "repr(_capture(m.step, m.State.__new__(m.State), 3))", + // each field set in turn, so each read in the body is the first to find one missing + "[repr(_capture(m.step, s, 2)) for s in [(lambda s: (setattr(s, 'b', 1), s)[1])(m.State.__new__(m.State)), (lambda s: (setattr(s, 'c', 1), s)[1])(m.State.__new__(m.State))]]", + "[_frames(_capture(m.step, s, 2)) for s in [(lambda s: (setattr(s, 'b', 1), s)[1])(m.State.__new__(m.State)), (lambda s: (setattr(s, 'c', 1), s)[1])(m.State.__new__(m.State))]]", + "(lambda s: (setattr(s, 'b', 2 ** 70), setattr(s, 'c', 2 ** 70), m.step(s, 3))[-1])(m.State.__new__(m.State))", + "_frames(_capture(m.bump, m.Cell.__new__(m.Cell), 3))", + "repr(_capture(lambda: m.Cell.__new__(m.Cell).v))", + "(lambda c: (setattr(c, 'v', 2 ** 70), m.bump(c, 3))[-1])(m.Cell.__new__(m.Cell))", + "_frames(_capture(m.guarded, m.Guard.__new__(m.Guard), 3))", + "(lambda g: (setattr(g, 'depth', 2 ** 70), m.guarded(g, 3))[-1])(m.Guard.__new__(m.Guard))", + // the field is missing only after `__enter__` found it: a `__exit__` reached on + // an instance whose field was never there + "_frames(_capture(lambda: m.Guard.__exit__(m.Guard.__new__(m.Guard), None, None, None)))", + ], + ); +} + #[test] fn an_instance_made_without_init_has_no_fields_until_they_are_assigned() { // `C.__new__(C)` makes an instance `__init__` never ran on, and each attribute @@ -37721,6 +38071,9 @@ fn a_licensed_call_re_asks_its_lookup_under_the_mode() { "{checked}" ); assert!(checked.contains("By_RecheckAccessor("), "{checked}"); + // `Money` is extended, so its getter reads `_amount` under a licence of its own kind + assert!(checked.contains("By_RecheckField("), "{checked}"); + assert!(checked.contains("\"_amount\""), "{checked}"); assert!( checked.contains("By_RecheckMethod((PyObject *)"), "{checked}" diff --git a/crates/by_codegen_c/src/lib.rs b/crates/by_codegen_c/src/lib.rs index e21108fbc3..3bb6ca3d10 100644 --- a/crates/by_codegen_c/src/lib.rs +++ b/crates/by_codegen_c/src/lib.rs @@ -296,18 +296,16 @@ pub fn emit_module(module: &ModuleIr) -> String { dispatch_licence(module, &class, &method) ); } - for (class, name) in accessor_licences(module) { - let _ = writeln!( - out, - "static ByAccessorLicence {} = BY_ACCESSOR_LICENCE_INIT;", - accessor_licence(module, &class, &name) - ); - } - for (class, field) in field_licences(module) { + let licensed: BTreeSet = accessor_licences(module) + .into_iter() + .chain(field_licences(module)) + .map(|(class, _)| class) + .collect(); + for class in licensed { let _ = writeln!( out, "static ByAccessorLicence {} = BY_ACCESSOR_LICENCE_INIT;", - field_licence(module, &class, &field) + class_licence(module, &class) ); } if !module.classes.is_empty() { @@ -3574,8 +3572,12 @@ fn answers_for_its_classes(module: &ModuleIr) -> bool { } // any write of a name the class does not hold, and the one read that // is never a method or a class-level constant somewhere else - Op::SetAttr { receiver, name, .. } | Op::GetAttr { receiver, name, .. } - if matches!(op, Op::SetAttr { .. }) || name == "__dict__" => + Op::SetAttr { receiver, name, .. } + | Op::WriteAttribute { receiver, name, .. } + | Op::GetAttr { receiver, name, .. } + | Op::ReadAttribute { receiver, name, .. } + if matches!(op, Op::SetAttr { .. } | Op::WriteAttribute { .. }) + || name == "__dict__" => { if let Some(class) = instance_held(&boxed, receiver) && let Some(owner) = class_named(module, class) @@ -3823,25 +3825,24 @@ fn dispatch_licences(module: &ModuleIr) -> BTreeSet<(String, String)> { wanted } -/// the static holding one site's licence to reach a compiled property half without the -/// descriptor protocol — see `By_ArmAccessor` -fn accessor_licence(module: &ModuleIr, class: &str, name: &str) -> String { - format!( - "by_prop_stands_{}_{}_{}", - mangle(module.name.dotted()), - mangle(class), - mangle(name) - ) -} - -/// the static holding the licence to read and write `class`'s `field` at its offset — see -/// `By_ArmField` -fn field_licence(module: &ModuleIr, class: &str, field: &str) -> String { +/// the one licence every property half and field of `class` is reached under +/// +/// each member's own licence is the same two facts — the receiver is exactly `class`, and +/// `class` still has the version it had when the member was found as compiled — so every +/// member armed without a write to the class in between records the same version. one +/// licence stands for all of them, and stands only where every one was found as compiled: +/// a member that was not refuses the class's licence, which sends each access of the class +/// through the protocol, and that is slower and the same program. +/// +/// the reason to share is what the C compiler can do with it. a getter that reads a field +/// asks the property's licence at the call and the field's inside the body; with one +/// licence the second test is the first test again, and once the body is written in place +/// it goes +fn class_licence(module: &ModuleIr, class: &str) -> String { format!( - "by_field_stands_{}_{}_{}", + "by_class_stands_{}_{}", mangle(module.name.dotted()), - mangle(class), - mangle(field) + mangle(class) ) } @@ -5233,7 +5234,95 @@ fn signature(module: &ModuleIr, function: &Function) -> String { } else { params.join(", ") }; - format!("static {} {symbol}({params})", ctype(module, &function.ret)) + let linkage = match in_place(function) { + InPlace::Always => "__attribute__((always_inline)) static inline", + InPlace::Offered => "static inline", + InPlace::Left => "static", + }; + format!( + "{linkage} {} {symbol}({params})", + ctype(module, &function.ret) + ) +} + +/// what a body's callers are told about writing it out in place +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum InPlace { + /// written out wherever it is called + Always, + /// offered, for the C compiler to weigh + Offered, + /// left to the C compiler's own judgement + Left, +} + +/// the most operations a body may have and still be offered to its callers to write out +/// in place +const OFFERED_OPERATIONS: usize = 16; + +/// the most operations a body may have and still be written out wherever it is called +const ALWAYS_OPERATIONS: usize = 6; + +/// how a body is offered to its callers to write out in place: not at all where it loops or +/// is on a cycle of calls, always where it is a few operations, and as a hint up to a handful +/// +/// the C compiler weighs a body against every caller, and a loop the optimizer has +/// duplicated calls each body twice. an accessor or a `with` block's `__exit__` that one +/// caller inlined stopped being inlined once a test on its cold path made it a little +/// larger, and the loop around the call paid for the call on every trip. a body of a few +/// operations is what an accessor, an `__exit__` or a one-line method is, and writing it out +/// costs no more than the call it replaces — and it is what lets the test a getter asks be +/// seen to be the test its caller asked a moment before +fn in_place(function: &Function) -> InPlace { + if is_recursive(function) || has_a_back_edge(function) { + return InPlace::Left; + } + let operations = function + .blocks + .iter() + .flat_map(|block| &block.ops) + .filter(|op| !matches!(op, Op::Line { .. })) + .count(); + if operations <= ALWAYS_OPERATIONS { + InPlace::Always + } else if operations <= OFFERED_OPERATIONS { + InPlace::Offered + } else { + InPlace::Left + } +} + +/// whether some block can reach itself +fn has_a_back_edge(function: &Function) -> bool { + // blocks finished without meeting one still on the path + let count = function.blocks.len(); + let mut done = vec![false; count]; + let mut on_path = vec![false; count]; + for start in 0..count { + if done[start] { + continue; + } + let mut path: Vec<(usize, Vec)> = + vec![(start, function.blocks[start].successors())]; + on_path[start] = true; + while let Some((id, pending)) = path.last_mut() { + match pending.pop() { + Some(next) if on_path.get(next.index()).copied().unwrap_or(false) => return true, + Some(next) if !done.get(next.index()).copied().unwrap_or(true) => { + on_path[next.index()] = true; + let successors = function.blocks[next.index()].successors(); + path.push((next.index(), successors)); + } + Some(_) => {} + None => { + done[*id] = true; + on_path[*id] = false; + path.pop(); + } + } + } + } + false } /// the entry a function on a cycle keeps under its ordinary symbol, for every caller @@ -5427,19 +5516,73 @@ fn emit_function(module: &ModuleIr, function: &Function) -> String { let mut in_generator_expression = block .position .is_some_and(|position| position.generator_expression); - for op in &block.ops { + // a test for a field that is read straight after it, which the read asks in its + // own place instead, or which the arithmetic straight after that asks on its slow + // path — see [`tested_read`] and [`unset_test`] + let mut tested_by_read: Option = None; + let mut tested_by_arithmetic: Vec = Vec::new(); + for (at, op) in block.ops.iter().enumerate() { if let Op::Line { position } = op { line = Some(position.offset); in_generator_expression = position.generator_expression; continue; } - let edge = if adds_a_traceback_entry(op) { - sites.edge(module, line, in_generator_expression, block.error_target) - } else { - ErrorEdge::straight(block.error_target) + let edge = match tested_by_read.take() { + Some(edge) => edge, + None if adds_a_traceback_entry(op) => { + sites.edge(module, line, in_generator_expression, block.error_target) + } + None => ErrorEdge::straight(block.error_target), }; + if matches!(op, Op::RequireField { .. }) + && block + .ops + .get(at + 1) + .and_then(|read| tested_read(module, function, op, read, edge)) + .is_some() + { + tested_by_read = Some(edge); + continue; + } + if let (Some(read), Some(arithmetic)) = (block.ops.get(at + 1), block.ops.get(at + 2)) + && let Some(test) = unset_test(module, function, op, read, arithmetic) + { + tested_by_arithmetic.push(test); + continue; + } let mut fragment = guard_unassigned(function, &op.operands(), edge); - fragment.push_str(&emit_op(module, function, op, edge)); + // the read in between leaves the tests waiting for the arithmetic after it + let unset = if matches!(op, Op::GetField { .. }) { + String::new() + } else { + std::mem::take(&mut tested_by_arithmetic).concat() + }; + match at + .checked_sub(1) + .and_then(|before| block.ops.get(before)) + .and_then(|require| tested_read(module, function, require, op, edge)) + { + Some(read) => fragment.push_str(&read), + None if !unset.is_empty() && matches!(op, Op::IntBinary { .. }) => { + fragment.push_str( + &split_int_binary_c( + function, + op, + edge, + &unset.replace("{label}", &error_label(edge)), + ) + .unwrap_or_default(), + ); + } + None => { + // a test waiting for arithmetic that is not this op is asked here after + // all, rather than lost + if !unset.is_empty() { + fragment.push_str(&unset.replace("{label}", &error_label(edge))); + } + fragment.push_str(&emit_op(module, function, op, edge)); + } + } sites.note(&fragment, edge); if let Some(at_error) = at_error.as_mut() && edge.target.is_none() @@ -5587,6 +5730,150 @@ fn adds_a_traceback_entry(op: &Op) -> bool { !matches!(op, Op::Reraise { .. } | Op::FinishFrame { .. }) } +/// a read of a tagged `int` field that asks, in its own place, the question the +/// [`Op::RequireField`] in front of it would have asked +/// +/// an unset field holds `BY_INT_ERROR`, which is odd, so it is never a short. a read that +/// retains what it loads already tests whether the value is a short, to know whether there +/// is an object to retain — and the unset test can go inside that arm, where a loop over +/// short values never reaches it. nothing happens between the two that anything could +/// observe, so the `AttributeError` is raised at the point python raises it +fn tested_read( + module: &ModuleIr, + function: &Function, + require: &Op, + read: &Op, + edge: ErrorEdge, +) -> Option { + let ( + Op::RequireField { + receiver, + class, + field, + }, + Op::GetField { + dest, + receiver: read_from, + class: read_class, + field: read_field, + }, + ) = (require, read) + else { + return None; + }; + if receiver != read_from || class != read_class || field != read_field { + return None; + } + let decl = field_decl(module, class, field)?; + let target = function.register(*dest)?; + if decl.optional + || decl.has_presence_byte() + || !decl.tracks_absence() + || decl.ty != RType::INT + || target.ty != RType::INT + || target.borrowed + { + return None; + } + let owner = class_named(module, class)?; + let fields = receiver_fields(module, class, field, receiver); + Some(format!( + " {{ ByTagged by_t = {fields}->{member}; + \ + if (BY_UNLIKELY(!By_IsShort(by_t))) {{ + \ + if (BY_UNLIKELY(by_t == BY_INT_ERROR)) {{ {missing} goto {label}; }} + \ + By_XIncRef(By_LongOf(by_t)); + \ + }} + \ + By_DecRefTagged({target}); {target} = by_t; }} +", + member = mangle_member(field), + missing = missing_attribute( + module, + owner, + &format!("(PyObject *){}", value_expr(receiver)), + field + ), + label = error_label(edge), + target = local(*dest), + )) +} + +/// the test an [`Op::RequireField`] asks, written for the slow path of the arithmetic that +/// consumes the read straight after it — with `{label}` where the jump goes +/// +/// a read that retains nothing loads the field and tests nothing, and the next thing to look +/// at the value is the arithmetic's own test that both operands are shorts. an unset field +/// holds `BY_INT_ERROR`, which is odd, so that test already sends it to the slow path, and +/// the unset test belongs there. nothing happens between the read and the arithmetic, so +/// the `AttributeError` is still raised before anything python could observe +fn unset_test( + module: &ModuleIr, + function: &Function, + require: &Op, + read: &Op, + arithmetic: &Op, +) -> Option { + let ( + Op::RequireField { + receiver, + class, + field, + }, + Op::GetField { + dest, + receiver: read_from, + class: read_class, + field: read_field, + }, + Op::IntBinary { + op, + lhs, + rhs, + dest: result, + }, + ) = (require, read, arithmetic) + else { + return None; + }; + if receiver != read_from || class != read_class || field != read_field { + return None; + } + let operand = Value::Register(*dest); + if (*lhs != operand && *rhs != operand) + || *result == *dest + || function.value_type(&Value::Register(*result)) != Some(RType::INT) + || split_int_binary(*op, lhs, rhs).is_none() + { + return None; + } + let decl = field_decl(module, class, field)?; + let target = function.register(*dest)?; + if decl.optional + || decl.has_presence_byte() + || !decl.tracks_absence() + || decl.ty != RType::INT + || target.ty != RType::INT + || !target.borrowed + { + return None; + } + let owner = class_named(module, class)?; + Some(format!( + "\x20 if (BY_UNLIKELY({} == BY_INT_ERROR)) {{ {} goto {{label}}; }}\n", + local(*dest), + missing_attribute( + module, + owner, + &format!("(PyObject *){}", value_expr(receiver)), + field + ), + )) +} + /// the traceback entries one function's error edges pass through /// /// a compiled frame is not a python frame, so nothing adds an entry for it as an exception @@ -6313,21 +6600,18 @@ fn emit_op(module: &ModuleIr, function: &Function, op: &Op, error_target: ErrorE } // an operation whose fast path cannot fail is written out with its two paths // apart, so the error test sits on the slow one alone — see `By_IntAddShort` - if function.value_type(&Value::Register(*dest)) == Some(RType::INT) - && let Some((fast, slow)) = split_int_binary(*op, lhs, rhs) - { - let target = local(*dest); - return format!( - " {{ ByTagged by_t;\n\ - \x20 if (BY_UNLIKELY(!{fast}({}, {}, &by_t))) {{\n\ - \x20 by_t = {slow};\n\ - \x20 if (BY_UNLIKELY(by_t == BY_INT_ERROR)) goto {};\n\ - \x20 }}\n\ - \x20 By_DecRefTagged({target}); {target} = by_t; }}\n", - value_expr(lhs), - value_expr(rhs), - error_label(error_target) - ); + if let Some(split) = split_int_binary_c( + function, + &Op::IntBinary { + dest: *dest, + op: *op, + lhs: lhs.clone(), + rhs: rhs.clone(), + }, + error_target, + "", + ) { + return split; } let call = match op { BinOp::Add => "By_IntAdd", @@ -6433,6 +6717,47 @@ fn emit_op(module: &ModuleIr, function: &Function, op: &Op, error_target: ErrorE local(*dest), ) } + // the type tests first: they cost a load and a compare, where a cold site looks a + // name up + Op::LoopGuardsHold { + dest, + builtins, + functions, + exact, + } => { + let mut out = String::from(" { char by_t = 1;\n"); + for value in exact { + let test = match function.value_type(value) { + Some(ty) if ty == RType::INT => format!("By_IsExactInt({})", value_expr(value)), + Some(ty) if ty == RType::STR => { + format!("PyUnicode_CheckExact({})", value_expr(value)) + } + _ => format!("PyList_CheckExact({})", value_expr(value)), + }; + let _ = writeln!(out, " by_t = by_t && {test};"); + } + for name in builtins { + let site = format!("by_ls_{}", mangle(name)); + let _ = writeln!( + out, + " static ByBuiltinSite {site} = BY_BUILTIN_SITE_INIT;\n \ + by_t = by_t && By_BuiltinStandsOnEntry(&{site}, by_module_dict, {});", + c_string(name) + ); + } + // a function this module publishes no forwarder for never stands, as its own + // question never does + for name in functions { + let stands = if published_functions(module).contains(&name.as_str()) { + format!("By_FunctionStands(&{})", function_site(name)) + } else { + "0".to_string() + }; + let _ = writeln!(out, " by_t = by_t && {stands};"); + } + let _ = writeln!(out, " {} = by_t; }}", local(*dest)); + out + } // a name this module publishes no forwarder for is never reached directly: its // resolution is always the object under the name, and its test always says no Op::ResolveFunction { dest, name } => { @@ -6465,6 +6790,10 @@ fn emit_op(module: &ModuleIr, function: &Function, op: &Op, error_target: ErrorE ) } } + Op::FunctionStood { dest, .. } => format!( + " {{ PyObject *by_old = {target}; {target} = NULL; Py_XDECREF(by_old); }}\n", + target = local(*dest) + ), Op::FunctionStands { dest, src, name } => { if published_functions(module).contains(&name.as_str()) { format!( @@ -6503,28 +6832,13 @@ fn emit_op(module: &ModuleIr, function: &Function, op: &Op, error_target: ErrorE dispatch_licence(module, class, method), ) } + // a property half and a field are asked about under the one licence their class + // holds — see [`class_licence`] Op::AccessorStands { - dest, - src, - class, - name, - } => { - let Some(owner) = class_named(module, class) else { - return format!(" {} = 0;\n", local(*dest)); - }; - format!( - " {} = By_AccessorStands((PyObject *){}, {}_OBJ, &{});\n", - local(*dest), - value_expr(src), - owner.type_name(module.name.dotted()), - accessor_licence(module, class, name), - ) + dest, src, class, .. } - Op::FieldStands { - dest, - src, - class, - field, + | Op::FieldStands { + dest, src, class, .. } => { let Some(owner) = class_named(module, class) else { return format!(" {} = 0;\n", local(*dest)); @@ -6534,7 +6848,7 @@ fn emit_op(module: &ModuleIr, function: &Function, op: &Op, error_target: ErrorE local(*dest), value_expr(src), owner.type_name(module.name.dotted()), - field_licence(module, class, field), + class_licence(module, class), ) } Op::LicenceHolds { @@ -6601,6 +6915,20 @@ fn emit_op(module: &ModuleIr, function: &Function, op: &Op, error_target: ErrorE c_string(member), ) } + LicenceKind::Field => { + // the getter the field's descriptor carries, which is what arming the + // licence found under the name + let Some(publisher) = publishing_class(module, owner, member) else { + return refuse("no class in the chain publishes the field"); + }; + format!( + " By_RecheckField((PyObject *){}, {type_name}_OBJ, {}, {}, {}_get_{member});\n", + value_expr(src), + c_string(class), + c_string(member), + publisher.type_name(module.name.dotted()), + ) + } } } Op::DictShadows { @@ -8127,6 +8455,66 @@ fn emit_op(module: &ModuleIr, function: &Function, op: &Op, error_target: ErrorE out.push_str(&commit_checked(function, *dest, error_target)); out } + Op::ReadAttribute { + dest, + receiver, + name, + } => { + let Some(decl) = function.register(*dest) else { + return String::new(); + }; + let reader = match &decl.ty { + RType::Primitive(Primitive::Int) => "By_ReadAttrInt", + RType::Primitive(Primitive::Float) => "By_ReadAttrFloat", + RType::Primitive(Primitive::Bool) => "By_ReadAttrBool", + RType::Primitive(Primitive::Str) => "By_ReadAttrStr", + _ => "By_ReadAttrObject", + }; + let slot = format!("by_a_{}", mangle(name)); + format!( + " {{ static PyObject *{slot} = NULL;\n{} }}\n", + assign_checked( + module, + function, + *dest, + &format!( + "{reader}((PyObject *){}, &{slot}, {})", + value_expr(receiver), + c_string_sized(name) + ), + error_target + ) + ) + } + Op::WriteAttribute { + dest, + receiver, + name, + value, + } => { + let writer = match function.value_type(value) { + Some(RType::Primitive(Primitive::Int)) => "By_WriteAttrInt", + Some(RType::Primitive(Primitive::Float)) => "By_WriteAttrFloat", + Some(RType::Primitive(Primitive::Bool)) => "By_WriteAttrBool", + _ => "By_WriteAttrObject", + }; + let slot = format!("by_a_{}", mangle(name)); + format!( + " {{ static PyObject *{slot} = NULL;\n{} }}\n", + assign_checked( + module, + function, + *dest, + &format!( + "{writer}((PyObject *){}, &{slot}, {}, {})", + value_expr(receiver), + c_string_sized(name), + value_expr(value) + ), + error_target + ) + ) + } Op::SetAttr { dest, receiver, @@ -8717,6 +9105,36 @@ fn defer_tests(function: &Function, receiver: bool) -> Vec { /// is one too — so the fast path never produces the error value, and only the slow call, /// which is the one that reaches cpython, needs testing for it. `//` and `%` are not here: /// a zero divisor fails on the fast path +/// a tagged `IntBinary` whose fast path cannot fail, written with its two paths apart, or +/// `None` where it has no such form. `unset` is C the slow path runs first +fn split_int_binary_c( + function: &Function, + op: &Op, + error_target: ErrorEdge, + unset: &str, +) -> Option { + let Op::IntBinary { dest, op, lhs, rhs } = op else { + return None; + }; + if function.value_type(&Value::Register(*dest)) != Some(RType::INT) { + return None; + } + let (fast, slow) = split_int_binary(*op, lhs, rhs)?; + let target = local(*dest); + Some(format!( + " {{ ByTagged by_t;\n\ + \x20 if (BY_UNLIKELY(!{fast}({}, {}, &by_t))) {{\n\ + {unset}\ + \x20 by_t = {slow};\n\ + \x20 if (BY_UNLIKELY(by_t == BY_INT_ERROR)) goto {};\n\ + \x20 }}\n\ + \x20 By_DecRefTagged({target}); {target} = by_t; }}\n", + value_expr(lhs), + value_expr(rhs), + error_label(error_target) + )) +} + fn split_int_binary(op: BinOp, lhs: &Value, rhs: &Value) -> Option<(&'static str, String)> { let (l, r) = (value_expr(lhs), value_expr(rhs)); Some(match op { @@ -10283,17 +10701,17 @@ fn emit_module_init(module: &ModuleIr) -> String { .collect(); // for the reason the dispatch licences are armed here, and one more: the property is // put into the type's dict by `class_init`, so nothing can be asked about it before - // that has run - let arm_accessors: String = accessor_licences(module) - .iter() - .filter_map(|(class, name)| { - let owner = class_named(module, class)?; + // that has run. every member a class licenses is armed into one scratch licence and + // joined into the class's — see [`class_licence`] + let mut arm_members: BTreeMap<&str, String> = BTreeMap::new(); + for (class, name) in accessor_licences(module) { + let arm = class_named(module, &class).and_then(|owner| { let property = owner .properties .iter() - .find(|property| property.name == *name)?; + .find(|property| property.name == name)?; let type_name = owner.type_name(module.name.dotted()); - let symbol = property_symbol(&type_name, name); + let symbol = property_symbol(&type_name, &name); // the two halves as `By_PublishProperty` was handed them, so that the licence // asks about the very descriptors the type was given let mut halves = property_halves(owner, property).map(|(half, body)| match body { @@ -10303,27 +10721,62 @@ fn emit_module_init(module: &ModuleIr) -> String { let get = halves.next()?; let set = halves.next()?; Some(format!( - " By_ArmAccessor(&{}, {type_name}_OBJ, {}, {get}, {set});\n", - accessor_licence(module, class, name), - c_string(name), + " By_ArmAccessor(&by_member, {type_name}_OBJ, {}, {get}, {set});\n", + c_string(&name), )) - }) - .collect(); + }); + let Some(owner) = class_named(module, &class) else { + continue; + }; + let _ = writeln!( + arm_members.entry(owner.name.as_str()).or_default(), + "{} By_JoinLicence(&by_version, &by_first, {});", + arm.as_deref().unwrap_or(""), + if arm.is_some() { + "by_member.version" + } else { + "0u" + }, + ); + } // the field's getter is what the descriptor standing under the name has to carry, // whichever of the two descriptors a field is published through - let arm_fields: String = field_licences(module) - .iter() - .filter_map(|(class, field)| { - let owner = class_named(module, class)?; - let type_name = owner.type_name(module.name.dotted()); - let publisher = publishing_class(module, owner, field)?.type_name(module.name.dotted()); - Some(format!( - " By_ArmField(&{}, {type_name}_OBJ, {}, {publisher}_get_{field});\n", - field_licence(module, class, field), - c_string(field), - )) - }) - .collect(); + for (class, field) in field_licences(module) { + let Some(owner) = class_named(module, &class) else { + continue; + }; + let type_name = owner.type_name(module.name.dotted()); + let arm = publishing_class(module, owner, &field).map(|publisher| { + format!( + " By_ArmField(&by_member, {type_name}_OBJ, {}, {}_get_{field});\n", + c_string(&field), + publisher.type_name(module.name.dotted()), + ) + }); + let _ = writeln!( + arm_members.entry(owner.name.as_str()).or_default(), + "{} By_JoinLicence(&by_version, &by_first, {});", + arm.as_deref().unwrap_or(""), + if arm.is_some() { + "by_member.version" + } else { + "0u" + }, + ); + } + let mut arm_accessors = String::new(); + for (class, members) in &arm_members { + let _ = writeln!( + arm_accessors, + " {{ ByAccessorLicence by_member = BY_ACCESSOR_LICENCE_INIT;\n \ + unsigned int by_version = 0u;\n \ + int by_first = 1;\n\ + {members} \ + {}.version = by_version; }}", + class_licence(module, class) + ); + } + let arm_fields = String::new(); // a class holding instance memory back holds it for as long as it might be asked // for another instance, which is as long as the module stands. handing it back when // the module goes is what keeps the whole scheme from being a leak @@ -10608,7 +11061,135 @@ mod tests { fn a_native_signature_uses_the_mangled_symbol() { let module = module_with(add()); let c = emit_module(&module); - assert!(c.contains("static ByTagged by_app_add(ByTagged r0, ByTagged r1)")); + assert!(c.contains("ByTagged by_app_add(ByTagged r0, ByTagged r1)")); + } + + /// `class Cell: v: int` with the field always assigned, and one method of it + fn cell(method: Function) -> ModuleIr { + let mut module = module_with(add()); + module.classes.push(ClassIr { + name: "Cell".to_string(), + immutable: true, + environment: false, + exported: true, + base: None, + inherited_init: false, + fields_are_parameters: true, + dataclass: false, + generic: false, + declares_slots: false, + constants: Vec::new(), + properties: Vec::new(), + slot_aliases: Vec::new(), + fields: vec![by_ir::function::FieldDecl { + cell: false, + name: "v".to_string(), + ty: RType::INT, + default: None, + optional: false, + defaulted_by: None, + }], + decorators: Vec::new(), + methods: vec![method], + resume: None, + keywords: Vec::new(), + }); + module + } + + /// `return self.v + 1`, with the field read into a register the borrow pass lends + /// where `borrowed` + fn reads_v_plus_one(borrowed: bool) -> Function { + let mut builder = FunctionBuilder::new("next", RType::INT); + let receiver = builder.param( + "self", + RType::Instance { + class: "Cell".to_string(), + exact: true, + }, + ); + let read = builder.temp(RType::INT); + let sum = builder.temp(RType::INT); + builder.push(Op::RequireField { + receiver: Value::Register(receiver), + class: "Cell".to_string(), + field: "v".to_string(), + }); + builder.push(Op::GetField { + dest: read, + receiver: Value::Register(receiver), + class: "Cell".to_string(), + field: "v".to_string(), + }); + builder.push(Op::IntBinary { + dest: sum, + op: BinOp::Add, + lhs: Value::Register(read), + rhs: Value::Int(1), + }); + builder.terminate(Terminator::Return(Value::Register(sum))); + let mut function = builder.finish(); + function.owner = Some("Cell".to_string()); + function.registers[read.index()].borrowed = borrowed; + function + } + + #[test] + fn an_unset_field_read_that_retains_asks_inside_the_arm_that_retains() { + // the value is only ever an unset field's error word when it is not a short, and + // a read that keeps a reference already asks that + let c = emit_module(&cell(reads_v_plus_one(false))); + let body = body_of(&c, "by_app_Cell_next(By_app_Cell * r0) {"); + let short = body + .find("if (BY_UNLIKELY(!By_IsShort(by_t))) {") + .unwrap_or_else(|| panic!("the read asks whether it has a short: {body}")); + let unset = body + .find("if (BY_UNLIKELY(by_t == BY_INT_ERROR)) { By_FieldMissing(") + .unwrap_or_else(|| panic!("the read asks whether the field is set: {body}")); + assert!(short < unset, "{body}"); + assert_eq!(body.matches("By_FieldMissing").count(), 1, "{body}"); + } + + #[test] + fn an_unset_field_read_into_arithmetic_asks_on_the_slow_path() { + // a read that keeps nothing leaves the value to the addition, whose test that both + // operands are shorts already sends an unset field's odd error word the slow way + let c = emit_module(&cell(reads_v_plus_one(true))); + let body = body_of(&c, "by_app_Cell_next(By_app_Cell * r0) {"); + let fast = body + .find("if (BY_UNLIKELY(!By_IntAddShort(") + .unwrap_or_else(|| panic!("the addition has a fast path: {body}")); + let unset = body + .find("if (BY_UNLIKELY(r1 == BY_INT_ERROR)) { By_FieldMissing(") + .unwrap_or_else(|| panic!("the slow path asks whether the field is set: {body}")); + let slow = body + .find("By_IntSlowBinary(") + .unwrap_or_else(|| panic!("the addition has a slow path: {body}")); + assert!(fast < unset && unset < slow, "{body}"); + assert_eq!(body.matches("By_FieldMissing").count(), 1, "{body}"); + } + + #[test] + fn a_small_body_with_no_loop_is_offered_to_be_written_in_place() { + let c = emit_module(&module_with(add())); + assert!( + c.contains("static inline ByTagged by_app_add(ByTagged r0, ByTagged r1)"), + "{c}" + ); + + // the same body going round a loop is not + let mut looping = add(); + let back = looping.blocks.len(); + let mut again = looping.blocks[0].clone(); + again.ops.clear(); + again.terminator = Terminator::Goto(BlockId(0)); + looping.blocks[0].terminator = Terminator::Goto(BlockId(back)); + looping.blocks.push(again); + let c = emit_module(&module_with(looping)); + assert!( + c.contains("static ByTagged by_app_add(ByTagged r0, ByTagged r1)"), + "{c}" + ); } #[test] @@ -11791,7 +12372,7 @@ mod tests { let c = emit_module(&module); // the forward declaration precedes the body, so take the last split let use_body = c - .rsplit("static ByTagged by_app_use") + .rsplit("static inline ByTagged by_app_use") .next() .expect("the caller is emitted"); assert!(use_body.contains("by_app_pure()"), "{use_body}"); @@ -11805,7 +12386,7 @@ mod tests { // releasing them here *and* in the wrapper was a double-decref that only // survived because small ints are unrefcounted let body = c - .split("static ByTagged by_app_add(ByTagged r0, ByTagged r1) {") + .split("static inline ByTagged by_app_add(ByTagged r0, ByTagged r1) {") .nth(1) .and_then(|rest| rest.split("static PyObject").next()) .expect("the body is emitted"); @@ -11874,7 +12455,7 @@ mod tests { builder.terminate(Terminator::Return(Value::Register(a))); let c = emit_module(&module_with(builder.finish())); let body = c - .split("static PyObject * by_app_first(PyObject * r0) {") + .split("static inline PyObject * by_app_first(PyObject * r0) {") .nth(1) .and_then(|rest| rest.split("static PyObject *byw").next()) .expect("the body is emitted"); @@ -11909,7 +12490,7 @@ mod tests { builder.terminate(Terminator::Return(Value::Register(n))); let c = emit_module(&module_with(builder.finish())); let body = c - .split("static ByTagged by_app_halve(ByTagged r0) {") + .split("static inline ByTagged by_app_halve(ByTagged r0) {") .nth(1) .and_then(|rest| rest.split("static PyObject").next()) .expect("the body is emitted"); @@ -12130,7 +12711,7 @@ mod tests { function.exported = false; let c = emit_module(&module_with(function)); assert!(!c.contains("byw_app_add")); - assert!(c.contains("static ByTagged by_app_add")); + assert!(c.contains("static inline ByTagged by_app_add")); } #[test] diff --git a/crates/by_ir/src/ops.rs b/crates/by_ir/src/ops.rs index e92c67b875..c47202ec82 100644 --- a/crates/by_ir/src/ops.rs +++ b/crates/by_ir/src/ops.rs @@ -214,6 +214,10 @@ pub enum LicenceKind { /// descriptor rather than calling a half, since calling one would run a body the /// program is already running Accessor, + /// a field read or written at its offset in the layout rather than through the + /// descriptor the class publishes for it. like a property, the descriptor is a data + /// descriptor, so the lookup is on the type alone + Field, } /// which error class a [`Op::RaiseStandard`] raises @@ -434,6 +438,24 @@ pub enum Op { /// wanted where it is not the builtin. what it costs at runtime is in /// `By_BuiltinStands` BuiltinStands { dest: RegisterId, name: String }, + /// whether a loop may be run as the copy of itself that asks none of its + /// [`Self::BuiltinStands`] questions about `builtins` and none of its + /// [`Self::ResolveFunction`] questions about `functions` again, and reads each register + /// of `exact` as the builtin its representation names + /// + /// asked once on the way into the loop. every one of `builtins` has to resolve to the + /// builtin, every one of `functions` has to still stand for its native entry, and every one of `exact` has to hold exactly an `int` where it is tagged, a + /// `str` where it is a `str` and a `list` where it is anything else, and not a subclass — neither of which anything but python code can undo, and the copy leaves + /// for the loop as written wherever it reaches an operation that can run python. so + /// the answer is a bit and never an error: a lookup that fails answers no, and the loop + /// as written asks it again where python would and raises what python raises. a build + /// on which another thread can write a namespace while this one runs answers no + LoopGuardsHold { + dest: RegisterId, + builtins: Vec, + functions: Vec, + exact: Vec, + }, /// what a call to the module function `name` has to go through where its native /// entry may not stand in for the function object, read where python reads the name /// @@ -445,6 +467,13 @@ pub enum Op { /// resolved as a global read is, which raises `NameError` for a name bound nowhere — /// see `By_ResolveFunction` ResolveFunction { dest: RegisterId, name: String }, + /// what [`Self::ResolveFunction`] answers where `name` is known to still stand for its + /// native entry: nothing to call + /// + /// written where a loop or a function was entered through a test that the function + /// stands and nothing since can have run python — see `by_opt`'s `guard_loops`. every + /// read of the register afterwards is the read it would have been + FunctionStood { dest: RegisterId, name: String }, /// whether the call [`Self::ResolveFunction`] read `src` for may go straight to the /// native entry: the read found nothing to call, and nothing the arguments did since /// has moved the function @@ -1047,6 +1076,29 @@ pub enum Op { name: String, value: Value, }, + /// `receiver.name` through the object protocol, narrowed to the representation `dest` + /// holds — the arm a licensed field or property read takes where the licence does not + /// stand + /// + /// the same as a [`Self::GetAttr`] of an object and a [`Self::Unbox`] of what it answered, + /// done as one call the C keeps out of line. the arm is the rare one of a test asked on + /// every trip, and as two operations it gave the function a register, a retain, a + /// release and two error tests of its own — enough that the C compiler stopped writing + /// an accessor out in place at all + ReadAttribute { + dest: RegisterId, + receiver: Value, + name: String, + }, + /// `receiver.name = value` through the object protocol, with `value` boxed the way its + /// representation boxes — the write [`Self::ReadAttribute`] is the read of. `dest` is the + /// status bit, 2 on failure + WriteAttribute { + dest: RegisterId, + receiver: Value, + name: String, + value: Value, + }, /// a list display, from already-boxed elements BuildList { dest: RegisterId, items: Vec }, /// a set display @@ -1349,7 +1401,9 @@ impl Op { | Self::MatchAttr { .. } | Self::MethodStands { .. } | Self::BuiltinStands { .. } + | Self::LoopGuardsHold { .. } | Self::ResolveFunction { .. } + | Self::FunctionStood { .. } | Self::FunctionStands { .. } | Self::FunctionCallee { .. } | Self::AccessorStands { .. } @@ -1405,6 +1459,8 @@ impl Op { | Self::CallMethod { .. } | Self::GetAttr { .. } | Self::SetAttr { .. } + | Self::ReadAttribute { .. } + | Self::WriteAttribute { .. } | Self::BuildList { .. } | Self::BuildSet { .. } | Self::BuildTuple { .. } @@ -1488,7 +1544,9 @@ impl Op { | Self::MatchAttr { dest, .. } | Self::MethodStands { dest, .. } | Self::BuiltinStands { dest, .. } + | Self::LoopGuardsHold { dest, .. } | Self::ResolveFunction { dest, .. } + | Self::FunctionStood { dest, .. } | Self::FunctionStands { dest, .. } | Self::FunctionCallee { dest, .. } | Self::AccessorStands { dest, .. } @@ -1544,6 +1602,8 @@ impl Op { | Self::FieldIsSet { dest, .. } | Self::GetAttr { dest, .. } | Self::SetAttr { dest, .. } + | Self::ReadAttribute { dest, .. } + | Self::WriteAttribute { dest, .. } | Self::BuildList { dest, .. } | Self::BuildSet { dest, .. } | Self::BuildTuple { dest, .. } @@ -1611,7 +1671,9 @@ impl Op { | Self::MatchAttr { dest, .. } | Self::MethodStands { dest, .. } | Self::BuiltinStands { dest, .. } + | Self::LoopGuardsHold { dest, .. } | Self::ResolveFunction { dest, .. } + | Self::FunctionStood { dest, .. } | Self::FunctionStands { dest, .. } | Self::FunctionCallee { dest, .. } | Self::AccessorStands { dest, .. } @@ -1667,6 +1729,8 @@ impl Op { | Self::FieldIsSet { dest, .. } | Self::GetAttr { dest, .. } | Self::SetAttr { dest, .. } + | Self::ReadAttribute { dest, .. } + | Self::WriteAttribute { dest, .. } | Self::BuildList { dest, .. } | Self::BuildSet { dest, .. } | Self::BuildTuple { dest, .. } @@ -1718,6 +1782,7 @@ impl Op { /// every operand this operation reads pub fn operands(&self) -> Vec<&Value> { match self { + Self::LoopGuardsHold { exact, .. } => exact.iter().collect(), Self::AsyncContext { manager, exit: Some(exit), @@ -1821,6 +1886,7 @@ impl Op { all } Self::GetAttr { receiver, .. } + | Self::ReadAttribute { receiver, .. } | Self::GetField { receiver, .. } | Self::RequireField { receiver, .. } | Self::FieldIsSet { receiver, .. } => vec![receiver], @@ -1830,6 +1896,9 @@ impl Op { } => vec![receiver, value], Self::SetAttr { receiver, value, .. + } + | Self::WriteAttribute { + receiver, value, .. } => vec![receiver, value], Self::StoreGlobal { value, .. } => vec![value], Self::TupleBuild { items, .. } @@ -1866,6 +1935,7 @@ impl Op { | Self::LoadGlobal { .. } | Self::BuiltinStands { .. } | Self::ResolveFunction { .. } + | Self::FunctionStood { .. } | Self::LoadEllipsis { .. } | Self::ModuleDict { .. } | Self::DeleteGlobal { .. } @@ -1950,6 +2020,7 @@ impl Op { /// silently skipped pub fn operands_mut(&mut self) -> Vec<&mut Value> { match self { + Self::LoopGuardsHold { exact, .. } => exact.iter_mut().collect(), Self::AsyncContext { manager, exit: Some(exit), @@ -2055,6 +2126,7 @@ impl Op { all } Self::GetAttr { receiver, .. } + | Self::ReadAttribute { receiver, .. } | Self::GetField { receiver, .. } | Self::RequireField { receiver, .. } | Self::FieldIsSet { receiver, .. } => vec![receiver], @@ -2064,6 +2136,9 @@ impl Op { } => vec![receiver, value], Self::SetAttr { receiver, value, .. + } + | Self::WriteAttribute { + receiver, value, .. } => vec![receiver, value], Self::StoreGlobal { value, .. } => vec![value], Self::TupleBuild { items, .. } @@ -2100,6 +2175,7 @@ impl Op { | Self::LoadGlobal { .. } | Self::BuiltinStands { .. } | Self::ResolveFunction { .. } + | Self::FunctionStood { .. } | Self::LoadEllipsis { .. } | Self::ModuleDict { .. } | Self::DeleteGlobal { .. } diff --git a/crates/by_ir/src/print.rs b/crates/by_ir/src/print.rs index e8bc71c238..7fbdef85e9 100644 --- a/crates/by_ir/src/print.rs +++ b/crates/by_ir/src/print.rs @@ -155,10 +155,26 @@ fn print_op(function: &Function, op: &Op) -> String { dest, name: builtin, } => format!("{} = builtin-stands {builtin}", name(*dest)), + Op::LoopGuardsHold { + dest, + builtins, + functions, + exact, + } => format!( + "{} = loop-guards-hold builtins [{}] functions [{}] exact [{}]", + name(*dest), + builtins.join(", "), + functions.join(", "), + exact.iter().map(value).collect::>().join(", ") + ), Op::ResolveFunction { dest, name: function, } => format!("{} = resolve-function {function}", name(*dest)), + Op::FunctionStood { + dest, + name: function, + } => format!("{} = function-stood {function}", name(*dest)), Op::FunctionStands { dest, src, @@ -216,6 +232,7 @@ fn print_op(function: &Function, op: &Op) -> String { let kind = match kind { LicenceKind::Method => "method", LicenceKind::Accessor => "accessor", + LicenceKind::Field => "field", }; format!("licence-holds {kind} {} {class}.{member}", value(src)) } @@ -689,6 +706,22 @@ fn print_op(function: &Function, op: &Op) -> String { receiver, name: attr, } => format!("{} = {}.{attr}", name(*dest), value(receiver)), + Op::ReadAttribute { + dest, + receiver, + name: attr, + } => format!("{} = read {}.{attr}", name(*dest), value(receiver)), + Op::WriteAttribute { + dest, + receiver, + name: attr, + value: v, + } => format!( + "{} = write ({}.{attr} = {})", + name(*dest), + value(receiver), + value(v) + ), Op::SetAttr { dest, receiver, diff --git a/crates/by_ir/src/verify.rs b/crates/by_ir/src/verify.rs index abb1858d81..23e371f27d 100644 --- a/crates/by_ir/src/verify.rs +++ b/crates/by_ir/src/verify.rs @@ -217,6 +217,21 @@ fn rendered_registers(rendered: &str) -> Vec { /// /// exactly one pair qualifies: a `bit` is a `bool` whose error case has been ruled /// out, and both are the same 0-or-1 byte. anything wider needs a real `Box` +/// whether [`Op::ReadAttribute`] narrows to, and [`Op::WriteAttribute`] boxes, this +/// representation +pub fn attribute_representation(ty: &RType) -> bool { + matches!( + ty, + RType::Primitive( + Primitive::Object + | Primitive::Int + | Primitive::Float + | Primitive::Bool + | Primitive::Str + ) + ) +} + fn free_widening(from: &RType, to: &RType) -> bool { match (from, to) { // a comparison result is already a valid bool byte @@ -611,6 +626,16 @@ impl Verifier<'_> { } } + /// an attribute's receiver is an object, or an instance of an emitted class, which + /// reaches `PyObject *` by a cast + fn expect_attribute_receiver(&mut self, block: BlockId, receiver: &Value) { + if let Some(actual) = self.operand_type(block, receiver) + && !matches!(actual, RType::Instance { .. }) + { + self.expect(block, receiver, &RType::OBJECT, "attribute access"); + } + } + fn expect_dest(&mut self, block: BlockId, dest: RegisterId, expected: &RType, what: &str) { match self.function.register(dest) { None => self.error(Some(block), format!("r{} is not declared", dest.0)), @@ -745,7 +770,23 @@ impl Verifier<'_> { Op::BuiltinStands { dest, .. } => { self.expect_dest(block, *dest, &RType::BIT, "a builtin test"); } - Op::ResolveFunction { dest, .. } => { + Op::LoopGuardsHold { dest, exact, .. } => { + for value in exact { + if let Some(actual) = self.operand_type(block, value) + && actual != RType::INT + && actual != RType::STR + && actual != RType::OBJECT + && actual != RType::LIST + { + self.error( + Some(block), + format!("a loop's entry test asks whether an {actual} is exact"), + ); + } + } + self.expect_dest(block, *dest, &RType::BIT, "a loop's entry test"); + } + Op::ResolveFunction { dest, .. } | Op::FunctionStood { dest, .. } => { self.expect_dest(block, *dest, &RType::OBJECT, "a function resolution"); } Op::FunctionStands { dest, src, .. } => { @@ -1466,6 +1507,31 @@ impl Verifier<'_> { self.expect(block, receiver, &RType::OBJECT, "attribute access"); self.expect_dest(block, *dest, &RType::OBJECT, "attribute access"); } + Op::ReadAttribute { dest, receiver, .. } => { + self.expect_attribute_receiver(block, receiver); + if let Some(decl) = self.function.register(*dest) + && !attribute_representation(&decl.ty) + { + self.error( + Some(block), + format!("an attribute read cannot narrow to {}", decl.ty), + ); + } + } + Op::WriteAttribute { + dest, + receiver, + value, + .. + } => { + self.expect_attribute_receiver(block, receiver); + if let Some(ty) = self.operand_type(block, value) + && !attribute_representation(&ty) + { + self.error(Some(block), format!("an attribute write cannot box {ty}")); + } + self.expect_dest(block, *dest, &RType::BIT, "attribute assignment"); + } Op::SetAttr { dest, receiver, diff --git a/crates/by_irbuild/src/lib.rs b/crates/by_irbuild/src/lib.rs index 0838326c18..b9a258bf2d 100644 --- a/crates/by_irbuild/src/lib.rs +++ b/crates/by_irbuild/src/lib.rs @@ -2144,7 +2144,9 @@ fn prune_unbuildable( { return Some(weakly_referenced(callee)); } - Op::CallMethod { name, .. } | Op::GetAttr { name, .. } + Op::CallMethod { name, .. } + | Op::GetAttr { name, .. } + | Op::ReadAttribute { name, .. } if blamed_methods.contains(name.as_str()) => { return Some(format!( @@ -9858,25 +9860,11 @@ impl Lowering<'_, '_> { let join = self.mutable.contains(class).then(|| { let (hit, miss, join) = self.field_guard(&receiver, class, &held.name); self.builder.switch_to(miss); - let object = self.widen_to_object(receiver.clone(), receiver_ty); - let answer = self.builder.temp(RType::OBJECT); - self.builder.push(Op::GetAttr { - dest: answer, - receiver: object, - name: held.name.clone(), - }); // what the checker says the attribute is, asked of whatever answered for it - if field_ty == RType::OBJECT { - self.builder.assign(dest, Value::Register(answer)); - } else { - self.builder.push(Op::Unbox { - dest, - src: Value::Register(answer), - to: field_ty.clone(), - }); - } + self.attribute_read(receiver.clone(), receiver_ty, &held.name, dest, &field_ty); self.builder.terminate(Terminator::Goto(join)); self.builder.switch_to(hit); + self.licence_holds(&receiver, class, &held.name, LicenceKind::Field); join }); self.require_field(&receiver, &owner, held); @@ -9914,17 +9902,10 @@ impl Lowering<'_, '_> { self.builder.switch_to(miss); // the value as it was handed over: whatever answers for the name decides // what it accepts - let object = self.widen_to_object(receiver.clone(), receiver_ty); - let boxed = self.widen_to_object(value.clone(), ty); - let refused = self.builder.temp(RType::BIT); - self.builder.push(Op::SetAttr { - dest: refused, - receiver: object, - name: held.name.clone(), - value: boxed, - }); + self.attribute_write(receiver.clone(), receiver_ty, &held.name, value.clone(), ty); self.builder.terminate(Terminator::Goto(join)); self.builder.switch_to(hit); + self.licence_holds(receiver, class, &held.name, LicenceKind::Field); Some(join) } else { None @@ -9945,6 +9926,79 @@ impl Lowering<'_, '_> { Ok(()) } + /// `receiver.name` through the object protocol into `dest`, narrowed to `ty` — the arm + /// a licensed access takes where its licence does not stand + /// + /// one [`Op::ReadAttribute`] where it narrows to `ty`, which keeps the whole arm to one + /// call; a lookup and an unbox otherwise + fn attribute_read( + &mut self, + receiver: Value, + receiver_ty: &RType, + name: &str, + dest: RegisterId, + ty: &RType, + ) { + if by_ir::verify::attribute_representation(ty) + && (matches!(receiver_ty, RType::Instance { .. }) || *receiver_ty == RType::OBJECT) + { + self.builder.push(Op::ReadAttribute { + dest, + receiver, + name: name.to_string(), + }); + return; + } + let object = self.widen_to_object(receiver, receiver_ty); + let answer = self.builder.temp(RType::OBJECT); + self.builder.push(Op::GetAttr { + dest: answer, + receiver: object, + name: name.to_string(), + }); + if *ty == RType::OBJECT { + self.builder.assign(dest, Value::Register(answer)); + } else { + self.builder.push(Op::Unbox { + dest, + src: Value::Register(answer), + to: ty.clone(), + }); + } + } + + /// `receiver.name = value` through the object protocol, `value` held as `ty` — the write + /// [`Self::attribute_read`] is the read of + fn attribute_write( + &mut self, + receiver: Value, + receiver_ty: &RType, + name: &str, + value: Value, + ty: &RType, + ) { + let refused = self.builder.temp(RType::BIT); + if by_ir::verify::attribute_representation(ty) + && (matches!(receiver_ty, RType::Instance { .. }) || *receiver_ty == RType::OBJECT) + { + self.builder.push(Op::WriteAttribute { + dest: refused, + receiver, + name: name.to_string(), + value, + }); + return; + } + let object = self.widen_to_object(receiver, receiver_ty); + let boxed = self.widen_to_object(value, ty); + self.builder.push(Op::SetAttr { + dest: refused, + receiver: object, + name: name.to_string(), + value: boxed, + }); + } + /// the test in front of a field access on a receiver typed `class`, answering with the /// block it goes to when the field stands, the one it goes to otherwise, and the one /// both arms end in @@ -12965,29 +13019,12 @@ impl Lowering<'_, '_> { }); self.builder.terminate(Terminator::Goto(join)); self.builder.switch_to(miss); - // the reference the protocol call needs is taken here rather than before the - // test, so the arm that does not need one does not pay for it - let object = self.widen_to_object(receiver, receiver_ty); - let answer = self.builder.temp(RType::OBJECT); - self.builder.push(Op::GetAttr { - dest: answer, - receiver: object, - name: name.to_string(), - }); // the protocol arm is where an override reached through a base-typed name // arrives, and it answers with an object. narrowing it to what the getter this // module compiled would have answered is the checked unbox every other call // result takes: the checker says the attribute is a `ret`, and a receiver that // makes that false is told so rather than read as one - if ret == RType::OBJECT { - self.builder.assign(dest, Value::Register(answer)); - } else { - self.builder.push(Op::Unbox { - dest, - src: Value::Register(answer), - to: ret.clone(), - }); - } + self.attribute_read(receiver, receiver_ty, name, dest, &ret); self.builder.terminate(Terminator::Goto(join)); self.builder.switch_to(join); (Value::Register(dest), ret) @@ -13036,16 +13073,7 @@ impl Lowering<'_, '_> { }); self.builder.terminate(Terminator::Goto(join)); self.builder.switch_to(miss); - // as the read does: the reference the protocol call needs is the slow arm's cost - let object = self.widen_to_object(receiver, receiver_ty); - let boxed = self.widen_to_object(value, param); - let refused = self.builder.temp(RType::BIT); - self.builder.push(Op::SetAttr { - dest: refused, - receiver: object, - name: name.to_string(), - value: boxed, - }); + self.attribute_write(receiver, receiver_ty, name, value, param); self.builder.terminate(Terminator::Goto(join)); self.builder.switch_to(join); } diff --git a/crates/by_irbuild/src/tests.rs b/crates/by_irbuild/src/tests.rs index ea772a47da..d46a050b91 100644 --- a/crates/by_irbuild/src/tests.rs +++ b/crates/by_irbuild/src/tests.rs @@ -9477,10 +9477,14 @@ def read(base: Base) -> int: )), "{text}" ); - // both arms, and the protocol is the one nothing proved its way off + // both arms, and the protocol is the one nothing proved its way off — asked as one + // read the C keeps out of line assert!(text.contains("call Base.v$get"), "{text}"); assert!( - has_op(read, |op| matches!(op, Op::GetAttr { .. })), + has_op( + read, + |op| matches!(op, Op::ReadAttribute { name, .. } if name == "v") + ), "{text}" ); }); diff --git a/crates/by_opt/src/copy_propagation.rs b/crates/by_opt/src/copy_propagation.rs index 603528f9e8..82d92fdcf2 100644 --- a/crates/by_opt/src/copy_propagation.rs +++ b/crates/by_opt/src/copy_propagation.rs @@ -273,8 +273,10 @@ fn retarget(op: &mut Op, new_dest: RegisterId) { | Op::IsMissing { dest, .. } | Op::MethodStands { dest, .. } | Op::BuiltinStands { dest, .. } + | Op::LoopGuardsHold { dest, .. } | Op::FunctionStands { dest, .. } | Op::ResolveFunction { dest, .. } + | Op::FunctionStood { dest, .. } | Op::FunctionCallee { dest, .. } | Op::AccessorStands { dest, .. } | Op::FieldStands { dest, .. } @@ -316,6 +318,8 @@ fn retarget(op: &mut Op, new_dest: RegisterId) { | Op::StopIterationValue { dest, .. } | Op::CallMethod { dest, .. } | Op::GetAttr { dest, .. } + | Op::ReadAttribute { dest, .. } + | Op::WriteAttribute { dest, .. } | Op::GetField { dest, .. } | Op::SetAttr { dest, .. } | Op::BuildList { dest, .. } diff --git a/crates/by_opt/src/guard_loops.rs b/crates/by_opt/src/guard_loops.rs new file mode 100644 index 0000000000..87bce7396b --- /dev/null +++ b/crates/by_opt/src/guard_loops.rs @@ -0,0 +1,1139 @@ +//! asking whether a builtin still stands once on the way into a loop, not on every trip +//! +//! a call to `len` is lowered natively only where the name still resolves to the builtin, +//! and python resolves it on every call — so the lowering asks on every call too: +//! +//! ```python +//! def longest_run(line: str) -> int: +//! i = 0 +//! while i < len(line): # is `len` still the builtin? +//! ... +//! ``` +//! +//! the question is cheap, but it is not the cost. its answer decides between the native +//! length and a call through whatever the name holds, and the two arms merge again before +//! the comparison, which keeps the C compiler from folding anything across them. +//! +//! only python code can write a namespace. so where the answer was yes on the way into +//! the loop, and nothing since can have run python, it is still yes. the loop is +//! duplicated: one copy exactly as it stands, and one that answers yes without asking, with +//! a test on the way in that picks between them. the test also asks whether each parameter +//! the loop reads is exactly an `int` or a `str`, since an exact value is what lets `len(s)` +//! or `i < n` run without reaching a method a subclass wrote. +//! +//! the copy is left for the loop as written right after any operation that can run python +//! — see [`crate::runs_python`] — continuing at the very next operation of the original, +//! which asks every question again. so the copy never answers for a moment python code has +//! been able to change, and a loop whose body always runs python pays one test on the way +//! in and runs as it did + +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; + +use by_ir::function::{BasicBlock, Function, ModuleIr, RegisterDecl}; +use by_ir::ops::{BlockId, Op, RegisterId, Terminator, Value}; +use by_ir::rtype::RType; + +use crate::runs_python::{Effects, Guard, Held, Kind, answer_yes}; + +/// how many loops in one function may be duplicated +/// +/// the same bound `unswitch` keeps, for the same reason: a copy of a loop that holds a loop +/// is a copy of that loop too +const MAX_PER_FUNCTION: usize = 4; + +/// the largest loop worth duplicating, in blocks +const MAX_BODY: usize = 32; + +pub(crate) fn run(module: &mut ModuleIr) { + // copies only ever drop questions and leave for the original, so what a call can run + // is the same after as before + let effects = Effects::of(module); + for function in module.all_functions_mut() { + // only the blocks the function started with are asked about. a block split off a + // loop is entered from its copy's exits, and taking one for a loop's header would + // put a test on the way back from every exit + let mut versioned = 0; + for index in 0..function.blocks.len() { + if versioned == MAX_PER_FUNCTION { + break; + } + if let Some(candidate) = candidate(&effects, function, BlockId(index)) + && version(&effects, function, candidate) + { + versioned += 1; + } + } + if versioned == 0 { + version_entry(&effects, function); + } + } +} + +/// duplicate a whole function with no loop in it, entered through the test where it starts +/// +/// the same copy a loop gets, for the calls a function makes once each: a function that +/// calls another module function asks whether that one still stands at every call, and +/// a recursive one asks it at every level. with nothing between the entry and the call +/// able to run python — once the parameters it reads are known to be exact — the +/// answer on the way in is the answer at the call +fn version_entry(effects: &Effects, function: &mut Function) -> bool { + let entry = Function::entry(); + let count = function.blocks.len(); + if count == 0 || count > MAX_BODY || goes_round(function) { + return false; + } + let all: Vec = (0..count).map(BlockId).collect(); + let (builtins, functions) = guards(function, &all); + if builtins.is_empty() && functions.is_empty() { + return false; + } + let exact = exact_parameters(function, &all); + + // the entry block's operations move to a block of their own, which the entry then + // falls into — so the entry has an edge to redirect through the test, as a loop does + let saved = function.blocks[entry.index()].clone(); + let header = BlockId(count); + function.blocks.push(saved.clone()); + let start = &mut function.blocks[entry.index()]; + start.ops.clear(); + start.terminator = Terminator::Goto(header); + start.error_target = None; + + let body: Vec = (1..=count).map(BlockId).collect(); + let held = effects.held_at_entry(function); + let versioned = version( + effects, + function, + Candidate { + header, + body, + builtins, + functions, + exact, + entry: held, + region: Region::Entry, + }, + ); + if !versioned { + function.blocks.truncate(count); + function.blocks[entry.index()] = saved; + } + versioned +} + +/// whether any block can reach itself +fn goes_round(function: &Function) -> bool { + let all: Vec = (0..function.blocks.len()).map(BlockId).collect(); + // with a block that cannot be on any cycle standing in for the header, a cycle among + // the rest is a cycle at all + let unreachable_header = BlockId(function.blocks.len()); + holds_a_loop(function, &all, unreachable_header) +} + +/// a loop worth duplicating +struct Candidate { + header: BlockId, + /// every block on a path from the header back to itself, the header included + body: Vec, + /// the builtins the loop asks about + builtins: Vec, + /// the module functions the loop asks about + functions: Vec, + /// the parameters the loop reads that the entry test asks to be exact + exact: Vec<(RegisterId, Kind)>, + /// what is known on every edge into the header from outside the loop + entry: Held, + region: Region, +} + +/// what is duplicated +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Region { + /// a loop, entered through the test from outside it + Loop, + /// a whole function with no loop in it, entered through the test when it is called + Entry, +} + +/// duplicate the loop, and enter it through the test, reporting whether it was worth it +fn version(effects: &Effects, function: &mut Function, candidate: Candidate) -> bool { + let Candidate { + header, + body, + builtins, + functions, + exact, + entry, + region, + } = candidate; + + // the copies land at the end, so every existing block keeps its id + let first_copy = function.blocks.len(); + let copies: BTreeMap = body + .iter() + .enumerate() + .map(|(offset, id)| (*id, BlockId(function.blocks.len() + offset))) + .collect(); + let standing: BTreeSet = builtins + .iter() + .cloned() + .map(Guard::Builtin) + .chain(functions.iter().cloned().map(Guard::Function)) + .collect(); + let mut answered: Vec<(BlockId, usize)> = Vec::new(); + for id in &body { + let mut block = function.blocks[id.index()].clone(); + map_edges(&mut block, true, |target| { + copies.get(&target).copied().unwrap_or(target) + }); + let copy = BlockId(function.blocks.len()); + answered.extend( + answer_yes(&mut block, &standing) + .into_iter() + .map(|index| (copy, index)), + ); + function.blocks.push(block); + } + let fast_header = copies[&header]; + + // where the copy has to leave: straight after the first operation in a block that can + // run python. the facts are those of the copy before any block was cut short, which + // has every path the cut copy has and more, so they hold for it too + let mut held = entry; + for (register, kind) in &exact { + held.assume_exact(*register, *kind); + } + held.assume_standing(&standing); + let copy_ids: HashSet = copies.values().copied().collect(); + let entries = Held::solve(function, fast_header, held, |id| copy_ids.contains(&id)); + let mut exits: Vec<(BlockId, BlockId, usize)> = Vec::new(); + for (original, copy) in &copies { + let Some(mut known) = entries.get(copy).cloned() else { + continue; + }; + let block = &function.blocks[copy.index()]; + for (index, op) in block.ops.iter().enumerate() { + if effects.op_runs_python(function, op, &known) { + exits.push((*copy, *original, index + 1)); + break; + } + known.step(function, op); + } + } + + // a copy that leaves on every trip round would cost a test on the way in and win + // nothing, and then the loop is left as it stands + let leaving: HashSet = exits.iter().map(|(copy, _, _)| *copy).collect(); + let worth_it = match region { + Region::Loop => comes_round(function, fast_header, ©_ids, &leaving), + // a function is worth it where the copy reaches a question it no longer asks + // before it leaves + Region::Entry => { + let before_leaving = reached_before_leaving(function, fast_header, ©_ids, &leaving); + answered.iter().any(|(copy, index)| { + before_leaving.contains(copy) + && exits + .iter() + .find(|(leaving, _, _)| leaving == copy) + .is_none_or(|(_, _, at)| index < at) + }) + } + }; + if !worth_it { + function.blocks.truncate(first_copy); + return false; + } + + // the original is split where the copy leaves it, latest point first so that an + // earlier split of the same block splits what is left in front of the later one + exits.sort_by(|a, b| (a.1, b.2).cmp(&(b.1, a.2))); + let mut tails: HashMap<(BlockId, usize), BlockId> = HashMap::new(); + for (_, original, at) in &exits { + if tails.contains_key(&(*original, *at)) { + continue; + } + let tail = split(function, *original, *at); + tails.insert((*original, *at), tail); + } + for (copy, original, at) in &exits { + let block = &mut function.blocks[copy.index()]; + block.ops.truncate(*at); + block.terminator = Terminator::Goto(tails[&(*original, *at)]); + } + + // a copy the cut copy no longer reaches would only be dead C + let reached = reachable(function, fast_header); + for copy in copies.values() { + if !reached.contains(copy) { + let block = &mut function.blocks[copy.index()]; + block.ops.clear(); + block.terminator = Terminator::Unreachable; + block.error_target = None; + } + } + + let answer = RegisterId(function.registers.len()); + function.registers.push(RegisterDecl { + name: None, + ty: RType::BIT, + borrowed: false, + may_be_unassigned: false, + }); + let preheader = BlockId(function.blocks.len()); + function.blocks.push(BasicBlock { + ops: vec![Op::LoopGuardsHold { + dest: answer, + builtins, + functions, + exact: exact + .iter() + .map(|(register, _)| Value::Register(*register)) + .collect(), + }], + terminator: Terminator::Branch { + cond: Value::Register(answer), + then_block: fast_header, + else_block: header, + }, + owned_at_exit: None, + range: function.blocks[header.index()].range, + position: function.blocks[header.index()].position, + error_target: None, + }); + + // every edge that entered the loop from outside now enters through the test. the loop, + // its copy, the blocks split off it and the test itself are all inside + let inside: HashSet = body + .iter() + .copied() + .chain(copy_ids) + .chain(tails.values().copied()) + .chain([preheader]) + .collect(); + for id in 0..function.blocks.len() { + if inside.contains(&BlockId(id)) { + continue; + } + map_edges(&mut function.blocks[id], false, |target| { + if target == header { preheader } else { target } + }); + } + true +} + +/// every block of `copies` reached from `header` without passing through a block the copy +/// leaves from, `header` and the blocks it leaves from included +fn reached_before_leaving( + function: &Function, + header: BlockId, + copies: &HashSet, + leaving: &HashSet, +) -> HashSet { + let mut seen = HashSet::from([header]); + let mut queue = vec![header]; + while let Some(id) = queue.pop() { + if leaving.contains(&id) { + continue; + } + for next in function.blocks[id.index()].successors() { + if copies.contains(&next) && seen.insert(next) { + queue.push(next); + } + } + } + seen +} + +/// whether the copy entered at `header` can come back round to it without leaving: every +/// block on the way is one of `copies`, and none of them is one the copy leaves from +fn comes_round( + function: &Function, + header: BlockId, + copies: &HashSet, + leaving: &HashSet, +) -> bool { + let mut seen = HashSet::new(); + let mut queue = vec![header]; + while let Some(id) = queue.pop() { + if leaving.contains(&id) { + continue; + } + for next in function.blocks[id.index()].successors() { + if next == header { + return true; + } + if copies.contains(&next) && seen.insert(next) { + queue.push(next); + } + } + } + false +} + +/// the loop `header` heads, if it is one worth duplicating +fn candidate(effects: &Effects, function: &Function, header: BlockId) -> Option { + if header == Function::entry() { + return None; + } + let body = natural_loop(function, header); + // a loop holding another is left to the inner one, whose copy is the one that saves a + // question on every trip: the outer copy would only ever save the inner loop's first + if body.len() < 2 || body.len() > MAX_BODY || holds_a_loop(function, &body, header) { + return None; + } + { + let (builtins, functions) = guards(function, &body); + if builtins.is_empty() && functions.is_empty() { + return None; + } + + let outside: Vec = (0..function.blocks.len()) + .map(BlockId) + .filter(|id| !body.contains(id)) + .filter(|id| function.blocks[id.index()].successors().contains(&header)) + .collect(); + // a handler entered from outside would carry an exception into the test, and a + // loop already entered through a test of its own has been duplicated once + if outside.is_empty() + || outside.iter().any(|id| { + let block = &function.blocks[id.index()]; + block.error_target == Some(header) || enters_by_test(block, header) + }) + { + return None; + } + let entries = Held::solve( + function, + Function::entry(), + effects.held_at_entry(function), + |_| true, + ); + let mut entry: Option = None; + for id in &outside { + let Some(mut known) = entries.get(id).cloned() else { + continue; + }; + for op in &function.blocks[id.index()].ops { + known.step(function, op); + } + known.step_terminator(&function.blocks[id.index()].terminator); + entry = Some(match entry { + None => known, + Some(existing) => existing.meet(&known), + }); + } + let entry = entry?; + let exact = exact_parameters(function, &body); + Some(Candidate { + header, + body, + builtins, + functions, + exact, + entry, + region: Region::Loop, + }) + } +} + +/// the builtins and the module functions the blocks of `body` ask about +fn guards(function: &Function, body: &[BlockId]) -> (Vec, Vec) { + let mut builtins: Vec = Vec::new(); + let mut functions: Vec = Vec::new(); + for op in body.iter().flat_map(|id| &function.blocks[id.index()].ops) { + match op { + Op::BuiltinStands { name, .. } if !builtins.contains(name) => { + builtins.push(name.clone()); + } + Op::ResolveFunction { name, .. } if !functions.contains(name) => { + functions.push(name.clone()); + } + _ => {} + } + } + (builtins, functions) +} + +/// the parameters `body` reads that the entry test asks to be exact, and as what +fn exact_parameters(function: &Function, body: &[BlockId]) -> Vec<(RegisterId, Kind)> { + let unwritten = unwritten_parameters(function); + let mut exact: Vec<(RegisterId, Kind)> = Vec::new(); + for op in body.iter().flat_map(|id| &function.blocks[id.index()].ops) { + // a register only the object protocol reads is asked to be a `list` where it is + // measured or indexed: that is what an exact `list` answers without asking + let measured = match op { + Op::Len { src, .. } => Some(src), + Op::GetItem { container, .. } => Some(container), + _ => None, + }; + for operand in op.operands() { + let Value::Register(register) = operand else { + continue; + }; + let kind = match function.register(*register).map(|decl| &decl.ty) { + Some(ty) if *ty == RType::INT => Kind::Int, + Some(ty) if *ty == RType::STR => Kind::Str, + Some(ty) + if (*ty == RType::OBJECT || *ty == RType::LIST) + && measured == Some(operand) => + { + Kind::List + } + _ => continue, + }; + if unwritten.contains(register) && !exact.iter().any(|(held, _)| held == register) { + exact.push((*register, kind)); + } + } + } + exact +} + +/// the parameters nothing in the function writes, which hold what the caller passed for +/// the whole call +fn unwritten_parameters(function: &Function) -> HashSet { + let mut unwritten: HashSet = (0..function.param_count).map(RegisterId).collect(); + for block in &function.blocks { + for op in &block.ops { + for register in op + .dest() + .into_iter() + .chain(op.unbinds()) + .chain(op.loop_cursor()) + { + unwritten.remove(®ister); + } + if let Op::Move { + src: Value::Register(register), + .. + } = op + { + unwritten.remove(register); + } + } + if let Some(register) = block.terminator.dest() { + unwritten.remove(®ister); + } + } + unwritten +} + +/// move everything from `at` on in `id` into a block of its own, which `id` then falls +/// into, and name the new block +fn split(function: &mut Function, id: BlockId, at: usize) -> BlockId { + let tail = BlockId(function.blocks.len()); + let block = &mut function.blocks[id.index()]; + let rest = block.ops.split_off(at); + // a failing operation names the line of the last marker before it, and the tail starts + // wherever the operations left in front of it had got to + let position = block + .ops + .iter() + .rev() + .find_map(|op| match op { + Op::Line { position } => Some(*position), + _ => None, + }) + .or(block.position); + let terminator = std::mem::replace(&mut block.terminator, Terminator::Goto(tail)); + let moved = BasicBlock { + ops: rest, + terminator, + owned_at_exit: None, + range: block.range, + position, + error_target: block.error_target, + }; + function.blocks.push(moved); + tail +} + +/// whether `block` is the test a duplicated loop is entered through +fn enters_by_test(block: &BasicBlock, header: BlockId) -> bool { + matches!(block.terminator, Terminator::Branch { else_block, .. } if else_block == header) + && matches!(block.ops.last(), Some(Op::LoopGuardsHold { .. })) +} + +/// point every edge of `block` somewhere else, the error edge too where `errors` +fn map_edges(block: &mut BasicBlock, errors: bool, map: impl Fn(BlockId) -> BlockId) { + match &mut block.terminator { + Terminator::Goto(target) => *target = map(*target), + Terminator::Branch { + then_block, + else_block, + .. + } => { + *then_block = map(*then_block); + *else_block = map(*else_block); + } + Terminator::NarrowShort { + fits, otherwise, .. + } => { + *fits = map(*fits); + *otherwise = map(*otherwise); + } + Terminator::Return(_) | Terminator::Unreachable => {} + } + if errors && let Some(target) = &mut block.error_target { + *target = map(*target); + } +} + +/// whether the blocks of `body` other than `header` go round a cycle of their own +fn holds_a_loop(function: &Function, body: &[BlockId], header: BlockId) -> bool { + let inside: HashSet = body.iter().copied().filter(|id| *id != header).collect(); + // a block is done once everything reachable from it inside has been seen without + // coming back to a block still on the path + let mut done: HashSet = HashSet::new(); + for start in body.iter().filter(|id| **id != header) { + if done.contains(start) { + continue; + } + let mut path: Vec<(BlockId, Vec)> = + vec![(*start, successors_within(function, *start, &inside))]; + let mut on_path: HashSet = HashSet::from([*start]); + while let Some((id, pending)) = path.last_mut() { + match pending.pop() { + Some(next) if on_path.contains(&next) => return true, + Some(next) if !done.contains(&next) => { + on_path.insert(next); + let successors = successors_within(function, next, &inside); + path.push((next, successors)); + } + Some(_) => {} + None => { + done.insert(*id); + on_path.remove(id); + path.pop(); + } + } + } + } + false +} + +fn successors_within(function: &Function, id: BlockId, inside: &HashSet) -> Vec { + function.blocks[id.index()] + .successors() + .into_iter() + .filter(|next| inside.contains(next)) + .collect() +} + +/// the loop `header` heads: `header`, and every block that reaches one of its back edges +/// without passing through `header` again +/// +/// a back edge is one into `header` from a block every path from the entry reaches through +/// `header`. an edge from anywhere else enters the loop, so a loop inside another is its +/// own loop, entered from the outer one's blocks, rather than the whole of the outer one +fn natural_loop(function: &Function, header: BlockId) -> Vec { + let everywhere = reachable_avoiding(function, Function::entry(), None); + let around = reachable_avoiding(function, Function::entry(), Some(header)); + let latches: Vec = everywhere + .iter() + .copied() + .filter(|id| !around.contains(id)) + .filter(|id| function.blocks[id.index()].successors().contains(&header)) + .collect(); + if latches.is_empty() { + return Vec::new(); + } + let mut body: HashSet = HashSet::from([header]); + let mut queue = latches; + while let Some(id) = queue.pop() { + if !body.insert(id) { + continue; + } + for (from, block) in function.blocks.iter().enumerate() { + let from = BlockId(from); + if !body.contains(&from) && block.successors().contains(&id) { + queue.push(from); + } + } + } + let mut body: Vec = body.into_iter().collect(); + body.sort_unstable(); + body +} + +/// the blocks reachable from `start` without passing through `avoiding` +fn reachable_avoiding( + function: &Function, + start: BlockId, + avoiding: Option, +) -> HashSet { + let mut seen = HashSet::new(); + if Some(start) == avoiding { + return seen; + } + seen.insert(start); + let mut queue = vec![start]; + while let Some(id) = queue.pop() { + for next in function.blocks[id.index()].successors() { + if Some(next) != avoiding && seen.insert(next) { + queue.push(next); + } + } + } + seen +} + +/// the blocks reachable from `start`, `start` included +fn reachable(function: &Function, start: BlockId) -> HashSet { + let mut seen = HashSet::from([start]); + let mut queue = vec![start]; + while let Some(id) = queue.pop() { + for next in function + .block(id) + .map(BasicBlock::successors) + .unwrap_or_default() + { + if seen.insert(next) { + queue.push(next); + } + } + } + seen +} + +#[cfg(test)] +mod tests { + use by_ir::builder::FunctionBuilder; + use by_ir::function::{Function, ModuleIr}; + use by_ir::ops::{BinOp, CmpOp, Op, Terminator, Value}; + use by_ir::rtype::{IntWidth, RType}; + use by_ir::verify::verify; + + /// `i = 0; while i < len(s): [if i == 3: hook()]; i = i + 1; return i`, lowered the + /// way the frontend lowers a `len` that may have been rebound + fn counting(call_in_body: bool) -> Function { + let mut builder = FunctionBuilder::new("count", RType::fixed(IntWidth::I64)); + let s = builder.param("s", RType::STR); + let hook = builder.param("hook", RType::OBJECT); + let i = builder.local("i", RType::fixed(IntWidth::I64)); + let stands = builder.temp(RType::BIT); + let widened = builder.temp(RType::OBJECT); + let length = builder.temp(RType::INT); + let found = builder.temp(RType::OBJECT); + let answered = builder.temp(RType::OBJECT); + let more = builder.temp(RType::BIT); + let called = builder.temp(RType::OBJECT); + builder.assign(i, Value::Fixed(0)); + let header = builder.new_block(); + let native = builder.new_block(); + let rebound = builder.new_block(); + let guard = builder.new_block(); + let body = builder.new_block(); + let exit = builder.new_block(); + builder.terminate(Terminator::Goto(header)); + + builder.switch_to(header); + builder.push(Op::BuiltinStands { + dest: stands, + name: "len".to_string(), + }); + builder.terminate(Terminator::Branch { + cond: Value::Register(stands), + then_block: native, + else_block: rebound, + }); + + builder.switch_to(native); + builder.assign(widened, Value::Register(s)); + builder.push(Op::Len { + dest: length, + src: Value::Register(widened), + }); + builder.terminate(Terminator::Goto(guard)); + + builder.switch_to(rebound); + builder.push(Op::LoadGlobal { + dest: found, + name: "len".to_string(), + }); + builder.push(Op::CallValue { + dest: answered, + callee: Value::Register(found), + args: vec![Value::Register(s)], + }); + builder.push(Op::Unbox { + dest: length, + src: Value::Register(answered), + to: RType::INT, + }); + builder.terminate(Terminator::Goto(guard)); + + builder.switch_to(guard); + builder.push(Op::IntCompare { + dest: more, + op: CmpOp::Lt, + lhs: Value::Register(i), + rhs: Value::Register(length), + }); + builder.terminate(Terminator::Branch { + cond: Value::Register(more), + then_block: body, + else_block: exit, + }); + + builder.switch_to(body); + if call_in_body { + let third = builder.temp(RType::BIT); + let calling = builder.new_block(); + let step = builder.new_block(); + builder.push(Op::IntCompare { + dest: third, + op: CmpOp::Eq, + lhs: Value::Register(i), + rhs: Value::Fixed(3), + }); + builder.terminate(Terminator::Branch { + cond: Value::Register(third), + then_block: calling, + else_block: step, + }); + builder.switch_to(calling); + builder.push(Op::CallValue { + dest: called, + callee: Value::Register(hook), + args: Vec::new(), + }); + builder.terminate(Terminator::Goto(step)); + builder.switch_to(step); + } + builder.push(Op::IntBinary { + dest: i, + op: BinOp::Add, + lhs: Value::Register(i), + rhs: Value::Fixed(1), + }); + builder.terminate(Terminator::Goto(header)); + + builder.switch_to(exit); + builder.terminate(Terminator::Return(Value::Register(i))); + builder.finish() + } + + fn run(function: Function) -> Function { + let mut module = ModuleIr::new("app"); + module.functions.push(function); + super::run(&mut module); + let function = module.functions.remove(0); + assert_eq!(verify(&function), Ok(())); + function + } + + fn entry_tests(function: &Function) -> Vec<&Op> { + function + .blocks + .iter() + .flat_map(|block| &block.ops) + .filter(|op| matches!(op, Op::LoopGuardsHold { .. })) + .collect() + } + + /// the blocks reachable from the block the entry test sends a yes to + fn fast_copy(function: &Function) -> Vec { + let Some(Terminator::Branch { then_block, .. }) = function + .blocks + .iter() + .find(|block| matches!(block.ops.last(), Some(Op::LoopGuardsHold { .. }))) + .map(|block| block.terminator.clone()) + else { + return Vec::new(); + }; + let mut seen = vec![then_block]; + let mut queue = vec![then_block]; + while let Some(id) = queue.pop() { + for next in function.blocks[id.index()].successors() { + if !seen.contains(&next) { + seen.push(next); + queue.push(next); + } + } + } + seen + } + + #[test] + fn a_loop_that_runs_no_python_asks_once_on_the_way_in() { + let function = run(counting(false)); + let tests = entry_tests(&function); + assert_eq!(tests.len(), 1, "{function:?}"); + let Op::LoopGuardsHold { + builtins, exact, .. + } = tests[0] + else { + return; + }; + assert_eq!(builtins, &["len".to_string()]); + // the `str` is asked to be exact, which is what lets `len` answer without asking it + assert_eq!(exact.len(), 1); + + // the copy the test says yes to asks nothing, and never reaches the rebound arm + let copy = fast_copy(&function); + let ops: Vec<&Op> = copy + .iter() + .flat_map(|id| &function.blocks[id.index()].ops) + .collect(); + assert!( + !ops.iter() + .any(|op| matches!(op, Op::BuiltinStands { .. } | Op::CallValue { .. })), + "{ops:?}" + ); + } + + #[test] + fn the_copy_leaves_for_the_loop_as_written_straight_after_a_call() { + let function = run(counting(true)); + assert_eq!(entry_tests(&function).len(), 1, "{function:?}"); + let copy = fast_copy(&function); + // the copy's block holding the call ends with it, and falls into a block of the + // original, which asks whether `len` stands before it measures again + let Some(leaving) = copy.iter().find(|id| { + function.blocks[id.index()] + .ops + .iter() + .any(|op| matches!(op, Op::CallValue { .. })) + }) else { + panic!("the copy lost its call: {function:?}"); + }; + let block = &function.blocks[leaving.index()]; + assert!( + matches!(block.ops.last(), Some(Op::CallValue { .. })), + "{block:?}" + ); + assert!( + copy.iter().any(|id| function.blocks[id.index()] + .ops + .iter() + .any(|op| matches!(op, Op::BuiltinStands { .. }))), + "after the call the loop asks again: {function:?}" + ); + } + + #[test] + fn a_loop_that_runs_python_before_every_question_is_left_alone() { + // the copy would leave on every trip before it reached anything it saved + let mut function = counting(false); + let header = function.blocks[0].terminator.clone(); + let Terminator::Goto(header) = header else { + return; + }; + let hook = by_ir::ops::RegisterId(1); + let called = by_ir::ops::RegisterId(function.registers.len()); + function.registers.push(by_ir::function::RegisterDecl { + name: None, + ty: RType::OBJECT, + borrowed: false, + may_be_unassigned: false, + }); + function.blocks[header.index()].ops.insert( + 0, + Op::CallValue { + dest: called, + callee: Value::Register(hook), + args: Vec::new(), + }, + ); + let function = run(function); + assert_eq!(entry_tests(&function).len(), 0, "{function:?}"); + } + + #[test] + fn a_loop_inside_another_is_duplicated_and_the_outer_one_is_not() { + // `for _ in ...: count(s)` with the counting loop written out inside: the inner + // loop asks on every trip, and the outer copy would save only its first question + let mut function = counting(false); + let Terminator::Goto(inner) = function.blocks[0].terminator else { + return; + }; + // a rebound `len` refuses rather than handing back an `int` the next trip round + // the outer loop would have to let go of — the shape a buffer-holding loop has + let Terminator::Branch { else_block, .. } = function.blocks[inner.index()].terminator + else { + return; + }; + function.blocks[else_block.index()].ops = vec![Op::RaiseStandard { + error: by_ir::ops::StandardError::RuntimeError, + message: "rebound".to_string(), + }]; + function.blocks[else_block.index()].terminator = Terminator::Unreachable; + let exit = function + .blocks + .iter() + .position(|block| matches!(block.terminator, Terminator::Return(_))) + .expect("the counting loop returns"); + let restart = by_ir::ops::BlockId(function.blocks.len()); + let mut again = function.blocks[0].clone(); + again.terminator = Terminator::Goto(inner); + function.blocks.push(again); + let done = by_ir::ops::BlockId(function.blocks.len()); + let mut finish = function.blocks[exit].clone(); + finish.ops.clear(); + function.blocks.push(finish); + let outer_more = by_ir::ops::RegisterId(0); + function.blocks[exit].terminator = Terminator::Branch { + cond: Value::Register(outer_more), + then_block: restart, + else_block: done, + }; + function.blocks[0].terminator = Terminator::Goto(restart); + // the condition is the `str` parameter, which the verifier would refuse as a bit, + // so it is swapped for a bit of its own + let bit = by_ir::ops::RegisterId(function.registers.len()); + function.registers.push(by_ir::function::RegisterDecl { + name: None, + ty: RType::BIT, + borrowed: false, + may_be_unassigned: false, + }); + if let Terminator::Branch { cond, .. } = &mut function.blocks[exit].terminator { + *cond = Value::Register(bit); + } + function.blocks[restart.index()].ops.push(Op::Assign { + dest: bit, + src: Value::Bit(true), + }); + + let function = run(function); + assert_eq!(entry_tests(&function).len(), 1, "{function:?}"); + // the test sits in front of the inner loop, inside the outer one + let test = function + .blocks + .iter() + .position(|block| matches!(block.ops.last(), Some(Op::LoopGuardsHold { .. }))) + .expect("the entry test"); + assert!( + function.blocks[restart.index()] + .successors() + .contains(&by_ir::ops::BlockId(test)), + "{function:?}" + ); + } + + /// `def bump(n: int) -> int: return add(n, 1)`, the call to `add` guarded, beside `add` + fn guarded_call() -> ModuleIr { + let mut add = FunctionBuilder::new("add", RType::INT); + let a = add.param("a", RType::INT); + let b = add.param("b", RType::INT); + let sum = add.temp(RType::INT); + add.push(Op::IntBinary { + dest: sum, + op: BinOp::Add, + lhs: Value::Register(a), + rhs: Value::Register(b), + }); + add.terminate(Terminator::Return(Value::Register(sum))); + + let mut builder = FunctionBuilder::new("bump", RType::INT); + let n = builder.param("n", RType::INT); + let resolved = builder.temp(RType::OBJECT); + let stands = builder.temp(RType::BIT); + let answer = builder.temp(RType::INT); + let callee = builder.temp(RType::OBJECT); + let boxed = builder.temp(RType::OBJECT); + let answered = builder.temp(RType::OBJECT); + let native = builder.new_block(); + let rebound = builder.new_block(); + let join = builder.new_block(); + builder.push(Op::ResolveFunction { + dest: resolved, + name: "add".to_string(), + }); + builder.push(Op::FunctionStands { + dest: stands, + src: Value::Register(resolved), + name: "add".to_string(), + }); + builder.terminate(Terminator::Branch { + cond: Value::Register(stands), + then_block: native, + else_block: rebound, + }); + builder.switch_to(native); + builder.push(Op::Release { + value: Value::Register(resolved), + path: Box::new([]), + }); + builder.push(Op::CallNative { + dest: Some(answer), + owner: None, + callee: "add".to_string(), + args: vec![Value::Register(n), Value::Int(1)], + }); + builder.terminate(Terminator::Goto(join)); + builder.switch_to(rebound); + builder.push(Op::FunctionCallee { + dest: callee, + src: Value::Register(resolved), + name: "add".to_string(), + }); + builder.push(Op::Box { + dest: boxed, + src: Value::Register(n), + }); + builder.push(Op::CallThrough { + dest: answered, + callee: Value::Register(callee), + args: vec![Value::Register(boxed), Value::Register(boxed)], + keywords: Vec::new(), + }); + builder.push(Op::Unbox { + dest: answer, + src: Value::Register(answered), + to: RType::INT, + }); + builder.terminate(Terminator::Goto(join)); + builder.switch_to(join); + builder.terminate(Terminator::Return(Value::Register(answer))); + + let mut module = ModuleIr::new("app"); + module.functions.push(add.finish()); + module.functions.push(builder.finish()); + module + } + + #[test] + fn a_function_with_no_loop_asks_on_the_way_in_where_its_arguments_are_exact() { + let mut module = guarded_call(); + super::run(&mut module); + let function = &module.functions[1]; + assert_eq!(verify(function), Ok(())); + let tests = entry_tests(function); + assert_eq!(tests.len(), 1, "{function:?}"); + let Op::LoopGuardsHold { + functions, exact, .. + } = tests[0] + else { + return; + }; + assert_eq!(functions, &["add".to_string()]); + assert_eq!(exact.len(), 1); + // the test is what the function starts with, and the copy it says yes to never asks + // whether the name stands and never reaches the arm that calls whatever it holds + assert!( + matches!(function.blocks[0].terminator, Terminator::Goto(_)), + "{function:?}" + ); + let copy = fast_copy(function); + assert!( + !copy + .iter() + .flat_map(|id| &function.blocks[id.index()].ops) + .any(|op| matches!(op, Op::FunctionStands { .. } | Op::CallThrough { .. })), + "{function:?}" + ); + } + + #[test] + fn duplicating_twice_is_not_attempted() { + let mut module = ModuleIr::new("app"); + module.functions.push(counting(false)); + super::run(&mut module); + let once = module.functions[0].blocks.len(); + super::run(&mut module); + assert_eq!(module.functions[0].blocks.len(), once); + assert_eq!(entry_tests(&module.functions[0]).len(), 1); + } +} diff --git a/crates/by_opt/src/infallible.rs b/crates/by_opt/src/infallible.rs index 04b3537e56..f62cd2ce30 100644 --- a/crates/by_opt/src/infallible.rs +++ b/crates/by_opt/src/infallible.rs @@ -186,6 +186,8 @@ fn op_can_fail(module: &ModuleIr, function: &by_ir::function::Function, op: &Op) | Op::CallMethod { .. } | Op::GetAttr { .. } | Op::SetAttr { .. } + | Op::ReadAttribute { .. } + | Op::WriteAttribute { .. } | Op::BuildList { .. } | Op::BuildSet { .. } | Op::BuildTuple { .. } @@ -269,8 +271,10 @@ fn op_can_fail(module: &ModuleIr, function: &by_ir::function::Function, op: &Op) Op::FieldIsSet { .. } => false, // the name is resolved, and one bound nowhere is a `NameError` Op::BuiltinStands { .. } => true, + // a lookup that fails answers no, and the loop as written asks again + Op::LoopGuardsHold { .. } => false, // the flag is read, and nothing is looked up - Op::FunctionStands { .. } => false, + Op::FunctionStands { .. } | Op::FunctionStood { .. } => false, // a name bound nowhere is a `NameError` Op::ResolveFunction { .. } | Op::FunctionCallee { .. } => true, // an attribute lookup reaches `__getattr__`, and `__match_args__` with it diff --git a/crates/by_opt/src/lib.rs b/crates/by_opt/src/lib.rs index 075e8f7e52..afe89458d5 100644 --- a/crates/by_opt/src/lib.rs +++ b/crates/by_opt/src/lib.rs @@ -13,10 +13,12 @@ pub(crate) mod dead_registers; pub(crate) mod dict_find; mod field_proofs; pub(crate) mod fold; +mod guard_loops; pub(crate) mod infallible; mod liveness; pub(crate) mod refcount; pub(crate) mod release_temporaries; +mod runs_python; mod store_moves; pub(crate) mod str_append; mod str_concat_int; @@ -119,6 +121,12 @@ const PASSES: &[Pass] = &[ }, // after dead-registers, so the body it copies is the final one, and before // infallible/borrow/refcount, which all read the block set + // after dead-registers, so the loop it copies is the final one, and before unswitch, + // which then finds the copy's bound as invariant as the original's + Pass { + name: "guard-loops", + run: guard_loops::run, + }, Pass { name: "unswitch", run: unswitch::run, diff --git a/crates/by_opt/src/runs_python.rs b/crates/by_opt/src/runs_python.rs new file mode 100644 index 0000000000..936e7c63f4 --- /dev/null +++ b/crates/by_opt/src/runs_python.rs @@ -0,0 +1,1582 @@ +//! which operations can run python code +//! +//! a guard answers a question about state only python code can change: whether `len` +//! still names the builtin, whether a published function still stands. a pass that +//! wants to ask such a question once rather than on every trip round a loop has to +//! know that nothing it skips over can have changed the answer — and that is a question +//! about *every* operation on the path, not about the guard. +//! +//! python code runs in more places than a call. a release can reach a `__del__`; an +//! allocation of an object the collector tracks can start a collection, which runs +//! finalizers; a comparison of two objects reaches `__eq__`; a dict lookup hashes and +//! compares keys; an attribute read on a class something can extend reaches whatever +//! the extension put there. so this is written the conservative way round: an operation +//! runs python unless it is one of the few this file has read the emitted C for and +//! found cannot, and a call runs python unless the callee's whole body is proven not to. +//! +//! most of those few depend on what the operands hold at that point. comparing two +//! tagged integers is a machine comparison while both are exact `int`s and a subclass's +//! `__lt__` otherwise, and overwriting a register is free while what it held was an exact +//! `int` and a finalizer otherwise. [`Held`] is what the function knows about that, at +//! each point + +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; +use std::sync::Arc; + +use by_ir::function::{BasicBlock, Function, ModuleIr, qualify}; +use by_ir::ops::{BlockId, Op, RegisterId, Terminator, UnaryOp, Value}; +use by_ir::rtype::{Primitive, RType}; + +/// which of a module's functions a native call can reach python code through +pub(crate) struct Effects { + summaries: Arc, +} + +/// a question a guarded call asks before it goes native: whether a builtin still resolves +/// to itself, or whether a published module function still stands for its native entry +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub(crate) enum Guard { + Builtin(String), + Function(String), +} + +/// what a native call to each function does, as far as it has been proven +#[derive(Debug, Default)] +pub(crate) struct Summaries { + /// the functions a call cannot run python code through, whatever it is handed: every + /// operation in the body cannot, every call the body makes cannot, and nothing the frame + /// lets go of on its way out can + clear: HashSet, + /// the functions a call cannot run python code through where every `int` and `str` it is + /// handed is exact — which is most arithmetic, since a subclass's own operators are the + /// python code the body would otherwise reach + clear_when_exact: HashSet, + /// the functions whose answer is an exact value where every `int` and `str` handed to + /// them is + exact_when_exact: HashSet, + /// the functions a call cannot run python code through where its arguments are exact and + /// every guard in [`Self::assumes`] stands — so each guarded call in the body goes native + /// + /// this is what makes a recursion clear: its own call to itself is guarded, and the arm + /// that guard turns away calls whatever the name holds, which may be anything + clear_when_standing: HashSet, + /// the functions whose answer is exact under the same conditions + exact_when_standing: HashSet, + /// the guards a function's body, and the body of every function it calls natively, asks + assumes: HashMap>, +} + +impl Effects { + /// a fixed point over the native call graph, from the optimistic side: every function + /// starts clear, and one whose own body or whose callee is not loses it. a cycle whose + /// bodies are all clear stays clear, which is right — going round it runs nothing but + /// the bodies, and a depth count running out raises without running anything + pub(crate) fn of(module: &ModuleIr) -> Self { + let names: HashSet = module + .all_functions() + .map(Function::qualified_name) + .collect(); + let assumes = assumed_guards(module); + let mut summaries = Arc::new(Summaries { + clear: names.clone(), + clear_when_exact: names.clone(), + exact_when_exact: names.clone(), + clear_when_standing: names.clone(), + exact_when_standing: names, + assumes: assumes.clone(), + }); + loop { + let mut next = Summaries { + assumes: assumes.clone(), + ..Summaries::default() + }; + for function in module.all_functions() { + let name = function.qualified_name(); + let none = BTreeSet::new(); + let (runs, _) = body_effects(function, &summaries, false, &none); + if !runs { + next.clear.insert(name.clone()); + } + let (runs, exact_answer) = body_effects(function, &summaries, true, &none); + if !runs { + next.clear_when_exact.insert(name.clone()); + } + if exact_answer { + next.exact_when_exact.insert(name.clone()); + } + let standing = assumes.get(&name).cloned().unwrap_or_default(); + let mut answered = function.clone(); + for block in &mut answered.blocks { + answer_yes(block, &standing); + } + let (runs, exact_answer) = body_effects(&answered, &summaries, true, &standing); + if !runs { + next.clear_when_standing.insert(name.clone()); + } + if exact_answer { + next.exact_when_standing.insert(name); + } + } + // the sets only ever shrink from the full ones they start at, so they settle + let settled = next.clear == summaries.clear + && next.clear_when_exact == summaries.clear_when_exact + && next.exact_when_exact == summaries.exact_when_exact + && next.clear_when_standing == summaries.clear_when_standing + && next.exact_when_standing == summaries.exact_when_standing; + summaries = Arc::new(next); + if settled { + return Self { summaries }; + } + } + } + + /// what `function` knows on entry, with what its native calls answer taken from here + pub(crate) fn held_at_entry(&self, function: &Function) -> Held { + Held::at_entry_with(function, Some(Arc::clone(&self.summaries))) + } + + /// whether running `op` in `function` can run python code, where `held` is what the + /// function knows just before it + pub(crate) fn op_runs_python(&self, function: &Function, op: &Op, held: &Held) -> bool { + call_runs_python(&self.summaries, function, op, held) + .unwrap_or_else(|| op_itself_runs_python(function, op, held)) + } + + /// whether a native call to the function named `qualified` can run python code + #[cfg(test)] + fn call_runs_python(&self, qualified: &str) -> bool { + !self.summaries.clear.contains(qualified) + } +} + +/// the guards each function asks, together with those of every function it calls natively +fn assumed_guards(module: &ModuleIr) -> HashMap> { + let mut assumes: HashMap> = HashMap::new(); + let mut calls: BTreeMap> = BTreeMap::new(); + for function in module.all_functions() { + let name = function.qualified_name(); + let own = assumes.entry(name.clone()).or_default(); + let called = calls.entry(name).or_default(); + for op in function.blocks.iter().flat_map(|block| &block.ops) { + match op { + Op::BuiltinStands { name, .. } => { + own.insert(Guard::Builtin(name.clone())); + } + Op::ResolveFunction { name, .. } => { + own.insert(Guard::Function(name.clone())); + } + Op::CallNative { owner, callee, .. } => { + called.push(qualify(owner.as_deref(), callee)); + } + _ => {} + } + } + } + loop { + let mut changed = false; + for (name, called) in &calls { + let gathered: Vec = called + .iter() + .filter_map(|callee| assumes.get(callee)) + .flatten() + .cloned() + .collect(); + if let Some(own) = assumes.get_mut(name) { + for guard in gathered { + changed |= own.insert(guard); + } + } + } + if !changed { + return assumes; + } + } +} + +/// answer each guard of `standing` in `block` with yes, and take the arm a yes takes, +/// handing back where each answered question was +pub(crate) fn answer_yes(block: &mut BasicBlock, standing: &BTreeSet) -> Vec { + let mut answered = Vec::new(); + let mut replaced = Vec::new(); + for (index, op) in block.ops.iter_mut().enumerate() { + match op { + Op::BuiltinStands { dest, name } + if standing.contains(&Guard::Builtin(name.clone())) => + { + replaced.push(index); + answered.push(*dest); + *op = Op::Assign { + dest: *dest, + src: Value::Bit(true), + }; + } + Op::ResolveFunction { dest, name } + if standing.contains(&Guard::Function(name.clone())) => + { + *op = Op::FunctionStood { + dest: *dest, + name: name.clone(), + }; + } + Op::FunctionStands { dest, name, .. } + if standing.contains(&Guard::Function(name.clone())) => + { + replaced.push(index); + answered.push(*dest); + *op = Op::Assign { + dest: *dest, + src: Value::Bit(true), + }; + } + _ => {} + } + } + if let Terminator::Branch { + cond: Value::Register(cond), + then_block, + .. + } = block.terminator + && answered.contains(&cond) + { + block.terminator = Terminator::Goto(then_block); + } + replaced +} + +/// whether a native call can run python code, or `None` for an operation that is not one +fn call_runs_python( + summaries: &Summaries, + function: &Function, + op: &Op, + held: &Held, +) -> Option { + let Op::CallNative { + owner, + callee, + args, + .. + } = op + else { + return None; + }; + let callee = qualify(owner.as_deref(), callee); + let exact = held.all_exact(function, args); + let clear = summaries.clear.contains(&callee) + || (exact && summaries.clear_when_exact.contains(&callee)) + || (exact + && summaries.clear_when_standing.contains(&callee) + && held.stands_for(summaries, &callee)); + Some(!clear || overwrites_something_live(function, op, held)) +} + +/// whether anything in `function`'s body can run python code, and whether every answer it +/// returns is exact, with the parameters taken to be exact where `exact_parameters` +fn body_effects( + function: &Function, + summaries: &Arc, + exact_parameters: bool, + standing: &BTreeSet, +) -> (bool, bool) { + let mut start = Held::at_entry_with(function, Some(Arc::clone(summaries))); + start.assume_standing(standing); + if exact_parameters { + for (index, decl) in function.params().iter().enumerate() { + match &decl.ty { + ty if *ty == RType::INT => start.assume_exact(RegisterId(index), Kind::Int), + ty if *ty == RType::STR => start.assume_exact(RegisterId(index), Kind::Str), + _ => {} + } + } + } + let entries = Held::solve(function, Function::entry(), start, |_| true); + let mut runs = false; + let mut exact_answer = true; + // a failing operation leaves through the function's own exit from wherever it is, and + // that exit lets go of every register the frame owns. what is known at every point of + // the function at once is what holds wherever that happens + let mut everywhere: Option = None; + for (index, block) in function.blocks.iter().enumerate() { + let Some(mut held) = entries.get(&BlockId(index)).cloned() else { + continue; + }; + everywhere = Some(everywhere.map_or_else(|| held.clone(), |known| known.meet(&held))); + for op in &block.ops { + runs |= call_runs_python(summaries, function, op, &held) + .unwrap_or_else(|| op_itself_runs_python(function, op, &held)); + held.step(function, op); + everywhere = everywhere.map(|known| known.meet(&held)); + } + held.step_terminator(&block.terminator); + if let Terminator::Return(value) = &block.terminator { + runs |= !held.lets_go_of_nothing_live(function); + exact_answer &= held.exact_kind(function, value).is_some(); + } + } + if let Some(everywhere) = everywhere { + runs |= !everywhere.lets_go_of_nothing_live(function); + } + (runs, exact_answer) +} + +/// the builtin type an exact value is an instance of, exactly and not through a subclass +/// +/// each has operations the interpreter answers without asking a method a subclass could +/// have written, and all but a list have no finalizer and hold no other object, so letting +/// go of one runs nothing +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Kind { + Int, + Float, + Str, + Bytes, + /// `None`, `True` or `False`, which are never let go of for good + Singleton, + /// a `list`, whose length and elements are read without asking it anything — but + /// whose elements are let go of with it + List, +} + +impl Kind { + /// whether letting go of an exact value of this kind runs nothing + const fn has_no_finalizer(self) -> bool { + !matches!(self, Self::List) + } +} + +/// what one register is known to hold, at one point +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum State { + /// nothing: the frame has not written it, or has let go of what it held + Empty, + /// an exact value of this kind + Exact(Kind), + /// what a parameter the frame never writes holds, which the caller holds a reference + /// to for the whole call + Pinned, + /// an element of a `list` a parameter the frame never writes holds, read since the + /// last operation that could have changed what the list holds + Element, + /// one of the four above, where letting go of the value runs nothing, depending on the + /// path that got here + Inert, +} + +/// what a function knows about the registers it holds, at one point of it +/// +/// a register the frame has not written holds nothing, and letting go of nothing runs +/// nothing. one written from an operation that builds an exact value holds that, until it +/// is written again. and one that copies a parameter the frame never writes holds an object +/// the caller holds a reference to for the whole call, so letting go of the copy cannot be +/// what frees it. a register this knows nothing about may hold anything +#[derive(Clone, Debug)] +pub(crate) struct Held { + states: HashMap, + /// the parameters the frame never writes + unwritten: HashSet, + /// what the native calls the function makes answer with, where it is known + summaries: Option>, + /// the guards known to stand here, because the region was entered through a test of them + /// and nothing since can have run python + standing: Arc>, +} + +impl PartialEq for Held { + fn eq(&self, other: &Self) -> bool { + self.states == other.states && self.unwritten == other.unwritten + } +} + +impl Held { + /// on entry to `function`: every register but the parameters holds nothing + #[cfg(test)] + pub(crate) fn at_entry(function: &Function) -> Self { + Self::at_entry_with(function, None) + } + + fn at_entry_with(function: &Function, summaries: Option>) -> Self { + let mut unwritten: HashSet = + (0..function.param_count).map(RegisterId).collect(); + for block in &function.blocks { + for op in &block.ops { + for register in op + .dest() + .into_iter() + .chain(op.unbinds()) + .chain(op.loop_cursor()) + { + unwritten.remove(®ister); + } + if let Op::Move { + src: Value::Register(register), + .. + } = op + { + unwritten.remove(register); + } + } + if let Some(register) = block.terminator.dest() { + unwritten.remove(®ister); + } + } + Self { + states: (function.param_count..function.registers.len()) + .map(|index| (RegisterId(index), State::Empty)) + .collect(), + unwritten, + summaries, + standing: Arc::default(), + } + } + + /// take every guard of `standing` to stand + pub(crate) fn assume_standing(&mut self, standing: &BTreeSet) { + let mut all = (*self.standing).clone(); + all.extend(standing.iter().cloned()); + self.standing = Arc::new(all); + } + + /// whether every guard a native call to `callee` assumes is known to stand + fn stands_for(&self, summaries: &Summaries, callee: &str) -> bool { + summaries + .assumes + .get(callee) + .is_none_or(|assumed| assumed.is_subset(&self.standing)) + } + + /// whether every `int` and `str` among `values` is exact + fn all_exact(&self, function: &Function, values: &[Value]) -> bool { + values.iter().all(|value| { + let wanted = match function.value_type(value) { + Some(ty) if ty == RType::INT => Kind::Int, + Some(ty) if ty == RType::STR => Kind::Str, + _ => return true, + }; + self.exact_kind(function, value) == Some(wanted) + }) + } + + /// take `register` to hold an exact `kind`, because something tested it and nothing + /// has written it since + pub(crate) fn assume_exact(&mut self, register: RegisterId, kind: Kind) { + self.states.insert(register, State::Exact(kind)); + } + + /// the kind of exact value `value` is, where it is one + /// + /// a literal is built by the module as the builtin it spells + pub(crate) fn exact_kind(&self, function: &Function, value: &Value) -> Option { + match value { + Value::Register(register) => match self.states.get(register) { + Some(State::Exact(kind)) => Some(*kind), + _ => function + .register(*register) + .and_then(|decl| no_reference_kind(&decl.ty)), + }, + Value::Int(_) | Value::Fixed(_) => Some(Kind::Int), + Value::Float(_) => Some(Kind::Float), + Value::Bool(_) | Value::Bit(_) | Value::None => Some(Kind::Singleton), + Value::Str(_) => Some(Kind::Str), + Value::Bytes(_) => Some(Kind::Bytes), + } + } + + /// whether letting go of what `register` holds here can run nothing + pub(crate) fn release_is_inert(&self, function: &Function, register: RegisterId) -> bool { + function + .register(register) + .is_some_and(|decl| holds_no_reference(&decl.ty)) + || self.unwritten.contains(®ister) + || match self.states.get(®ister) { + Some(State::Exact(kind)) => kind.has_no_finalizer(), + Some(State::Empty | State::Pinned | State::Element | State::Inert) => true, + None => false, + } + } + + /// whether an exit here, which lets go of every register the frame owns, runs nothing + fn lets_go_of_nothing_live(&self, function: &Function) -> bool { + (function.param_count..function.registers.len()) + .map(RegisterId) + .all(|register| self.release_is_inert(function, register)) + } + + /// what is known after `op` has run + pub(crate) fn step(&mut self, function: &Function, op: &Op) { + // a list changes only through python code or a call, and an element read before is + // then no longer known to be held by it + let lists_may_change = + matches!(op, Op::CallNative { .. }) || op_itself_runs_python(function, op, self); + let produced = op.dest().map(|dest| { + let state = match self.produced(function, op, dest) { + Some(kind) => Some(State::Exact(kind)), + None => match op { + Op::GetItem { + container: Value::Register(container), + index, + .. + } if self.unwritten.contains(container) + && self.states.get(container) == Some(&State::Exact(Kind::List)) + && self.exact_kind(function, index) == Some(Kind::Int) => + { + Some(State::Element) + } + Op::Assign { + src: Value::Register(src), + .. + } + | Op::Move { + src: Value::Register(src), + .. + } if self.states.get(src) == Some(&State::Pinned) + || self.unwritten.contains(src) => + { + Some(State::Pinned) + } + _ => None, + }, + }; + (dest, state) + }); + if lists_may_change { + self.states.retain(|_, state| *state != State::Element); + } + match op { + Op::Move { + src: Value::Register(src), + path, + .. + } + | Op::Release { + value: Value::Register(src), + path, + } if path.is_empty() => { + self.states.insert(*src, State::Empty); + } + _ => {} + } + if let Some(register) = op.unbinds() { + self.states.insert(register, State::Empty); + } + if let Some((dest, state)) = produced { + match state { + Some(state) => self.states.insert(dest, state), + None => self.states.remove(&dest), + }; + } + // a function known to stand resolves to nothing to call + match op { + Op::ResolveFunction { dest, name } + if self.standing.contains(&Guard::Function(name.clone())) => + { + self.states.insert(*dest, State::Empty); + } + Op::FunctionStood { dest, .. } => { + self.states.insert(*dest, State::Empty); + } + _ => {} + } + } + + /// what is known after `terminator` has run + pub(crate) fn step_terminator(&mut self, terminator: &Terminator) { + if let Some(dest) = terminator.dest() { + self.states.remove(&dest); + } + } + + /// what is known on a path that could have come from either + pub(crate) fn meet(&self, other: &Self) -> Self { + Self { + states: self + .states + .iter() + .filter_map(|(register, state)| { + let theirs = other.states.get(register)?; + let inert = |state: &State| { + !matches!(state, State::Exact(kind) if !kind.has_no_finalizer()) + }; + if theirs == state { + Some((*register, *state)) + } else if inert(state) && inert(theirs) { + Some((*register, State::Inert)) + } else { + None + } + }) + .collect(), + unwritten: self.unwritten.clone(), + summaries: self.summaries.clone(), + standing: Arc::clone(&self.standing), + } + } + + /// what is known at the entry to each block reached from `start`, where `held` is what + /// is known there and only blocks `within` says are part of the region are followed + /// + /// a handler is entered from the middle of the blocks it covers, so it is given only + /// what held at every point of each of them + pub(crate) fn solve( + function: &Function, + start: BlockId, + held: Self, + within: impl Fn(BlockId) -> bool, + ) -> HashMap { + let mut entries: HashMap = HashMap::new(); + entries.insert(start, held); + let mut pending = vec![start]; + while let Some(id) = pending.pop() { + let (Some(block), Some(entry)) = (function.block(id), entries.get(&id).cloned()) else { + continue; + }; + let mut known = entry.clone(); + let mut throughout = entry; + for op in &block.ops { + known.step(function, op); + throughout = throughout.meet(&known); + } + known.step_terminator(&block.terminator); + // the exception edge is the last of the successors, and it carries only what + // held all through the block + let mut normal = block.successors(); + if block.error_target.is_some() { + normal.pop(); + } + let mut flows: Vec<(BlockId, Self)> = normal + .into_iter() + .map(|successor| (successor, known.clone())) + .collect(); + if let Some(handler) = block.error_target { + flows.push((handler, throughout)); + } + for (successor, facts) in flows { + if !within(successor) { + continue; + } + let merged = match entries.get(&successor) { + None => facts, + Some(existing) => existing.meet(&facts), + }; + if entries.get(&successor) != Some(&merged) { + entries.insert(successor, merged); + pending.push(successor); + } + } + } + entries + } + + /// the kind of exact value `op` leaves in `dest`, where it leaves one + fn produced(&self, function: &Function, op: &Op, dest: RegisterId) -> Option { + let ty = function.register(dest).map(|decl| &decl.ty)?; + if let Some(kind) = no_reference_kind(ty) { + return Some(kind); + } + let exact = |value: &Value| self.exact_kind(function, value); + match op { + // an answer a callee is proven to make exact, from arguments that are + Op::CallNative { + owner, + callee, + args, + .. + } => { + let summaries = self.summaries.as_ref()?; + let callee = qualify(owner.as_deref(), callee); + (self.all_exact(function, args) + && (summaries.exact_when_exact.contains(&callee) + || (summaries.exact_when_standing.contains(&callee) + && self.stands_for(summaries, &callee)))) + .then(|| exact_kind_of(ty)) + .flatten() + } + Op::Assign { src, .. } => exact(src), + Op::Move { src, path, .. } if path.is_empty() => exact(src), + // the interpreter's own arithmetic on two exact `int`s answers an exact `int`, on + // the slow path too + Op::IntBinary { lhs, rhs, .. } + if exact(lhs) == Some(Kind::Int) && exact(rhs) == Some(Kind::Int) => + { + Some(Kind::Int) + } + Op::Unary { + op: UnaryOp::Neg | UnaryOp::Invert, + operand, + .. + } if exact(operand) == Some(Kind::Int) && *ty == RType::INT => Some(Kind::Int), + // a machine value boxed is built as the builtin, and an exact value boxed is + // itself + Op::Box { src, .. } => match function.value_type(src)? { + RType::Primitive(Primitive::Fixed(_)) => Some(Kind::Int), + RType::Primitive(Primitive::Float) => Some(Kind::Float), + RType::Primitive(Primitive::Bool | Primitive::Bit | Primitive::None) => { + Some(Kind::Singleton) + } + _ => exact(src), + }, + // a length is a machine integer before it is tagged, whatever `__len__` it came + // from + Op::Len { .. } | Op::ArrayLen { .. } if *ty == RType::INT => Some(Kind::Int), + // a character of an exact `str` is an exact `str` + Op::StrGetItem { + container, index, .. + } if exact(container) == Some(Kind::Str) && exact(index) == Some(Kind::Int) => { + Some(Kind::Str) + } + _ => None, + } + } +} + +/// the kind an exact value of this representation is +fn exact_kind_of(ty: &RType) -> Option { + match ty { + RType::Primitive(Primitive::Int) => Some(Kind::Int), + RType::Primitive(Primitive::Str) => Some(Kind::Str), + other => no_reference_kind(other), + } +} + +/// the kind a value of this representation is when it holds no object at all +fn no_reference_kind(ty: &RType) -> Option { + match ty { + RType::Primitive(Primitive::Fixed(_)) => Some(Kind::Int), + RType::Primitive(Primitive::Float) => Some(Kind::Float), + RType::Primitive(Primitive::Bool | Primitive::Bit | Primitive::None) => { + Some(Kind::Singleton) + } + _ => None, + } +} + +/// whether a value of this representation holds no reference to any object, so that +/// overwriting or letting go of one runs nothing +fn holds_no_reference(ty: &RType) -> bool { + match ty { + RType::Primitive(primitive) => matches!( + primitive, + Primitive::Fixed(_) + | Primitive::Float + | Primitive::Bool + | Primitive::Bit + | Primitive::None + ), + // the buffer is freed with the elements it holds, which are only safe to free when + // they are not objects themselves + RType::Array(element) => holds_no_reference(element), + RType::Tuple(items) => items.iter().all(holds_no_reference), + RType::Instance { .. } => false, + } +} + +/// whether `op` lets go of something that may be the last reference to an object with a +/// finalizer: the value its destination held, or the local it unbinds +fn overwrites_something_live(function: &Function, op: &Op, held: &Held) -> bool { + op.dest() + .into_iter() + .chain(op.unbinds()) + .any(|register| !held.release_is_inert(function, register)) +} + +/// whether `op` can run python code, leaving a native call's callee aside +fn op_itself_runs_python(function: &Function, op: &Op, held: &Held) -> bool { + if overwrites_something_live(function, op, held) { + return true; + } + let kind = |value: &Value| held.exact_kind(function, value); + match op { + // copies and loads of what the frame already holds, comparisons and arithmetic on + // machine values, and questions answered by reading a pointer or a flag + Op::Assign { .. } + | Op::Move { .. } + | Op::FloatBinary { .. } + | Op::FloatCompare { .. } + | Op::IsNull { .. } + | Op::StopIterationValue { .. } + | Op::Line { .. } + | Op::IsMissing { .. } + | Op::Identity { .. } + | Op::IsSequence { .. } + | Op::IsMapping { .. } + | Op::LoadEllipsis { .. } + | Op::ModuleDict { .. } + | Op::LoadClass { .. } + | Op::FunctionStands { .. } + | Op::FunctionStood { .. } + | Op::FunctionCallee { .. } + | Op::TupleGet { .. } + | Op::TupleBuild { .. } + | Op::PushHandled { .. } + | Op::DeleteLocal { .. } => false, + + // a field is a load at an offset, and whether it is set is a load and a compare; + // raising `AttributeError` or `UnboundLocalError` for a missing one builds an + // exception of the interpreter's own class + Op::GetField { .. } + | Op::RequireField { .. } + | Op::FieldIsSet { .. } + | Op::GetCell { .. } => false, + + // `PyLong_AsDouble` reads the digits of any `int`, a subclass's included, and asks + // it nothing + Op::IntToFloat { .. } => false, + + // a buffer of the compiler's own, grown with the allocator and not the collector + Op::ArrayNew { .. } + | Op::ArrayGet { .. } + | Op::ArrayLen { .. } + | Op::ArrayRead { .. } + | Op::ArrayPush { .. } => false, + // a store lets go of the element it replaces + Op::ArraySet { array, .. } => !function + .value_type(array) + .is_some_and(|ty| holds_no_reference(&ty)), + + // a narrowing is a type test, and the failure it raises names the type it found + Op::Unbox { to, .. } => !matches!( + to, + RType::Primitive( + Primitive::Int + | Primitive::Float + | Primitive::Bool + | Primitive::None + | Primitive::Str + | Primitive::List + ) | RType::Instance { .. } + ), + // boxing a machine value builds an `int` or a `float`, neither of which the collector + // tracks, or answers with a singleton; boxing an object is a new reference + Op::Box { src, .. } => !matches!( + function.value_type(src), + Some(RType::Primitive(_) | RType::Instance { .. }) + ), + + // the interpreter's own arithmetic and comparison, while both sides are exact `int`s + Op::IntBinary { lhs, rhs, .. } | Op::IntCompare { lhs, rhs, .. } => { + !(kind(lhs) == Some(Kind::Int) && kind(rhs) == Some(Kind::Int)) + } + Op::StrCompare { lhs, rhs, .. } => { + !(kind(lhs) == Some(Kind::Str) && kind(rhs) == Some(Kind::Str)) + } + Op::Unary { op, operand, .. } => match op { + UnaryOp::Not => false, + UnaryOp::Neg | UnaryOp::Invert => { + !matches!(kind(operand), Some(Kind::Int | Kind::Float)) + } + }, + // an exact `str`, `bytes` or `list` knows its own size + Op::Len { src, .. } => !matches!(kind(src), Some(Kind::Str | Kind::Bytes | Kind::List)), + // an element of an exact `list` at an index that is an exact `int`, which raises + // `IndexError` of the interpreter's own out of range + Op::GetItem { + container, index, .. + } if kind(container) == Some(Kind::List) && kind(index) == Some(Kind::Int) => false, + // a character of an exact `str`, at an index that is an exact `int` + Op::StrGetItem { + container, index, .. + } + | Op::StrItemCompare { + container, index, .. + } => !(kind(container) == Some(Kind::Str) && kind(index) == Some(Kind::Int)), + // a release frees the object only where nothing else is known to hold it and it + // may have a finalizer or hold other objects + Op::Release { value, path } => match value { + Value::Register(register) => { + !path.is_empty() || !held.release_is_inert(function, *register) + } + _ => false, + }, + + // the callee decides, and only [`Effects`] knows the callee + Op::CallNative { .. } => true, + + // a store lets go of the value it replaces, and a published `__dict__` holds + // whatever python last wrote into it + Op::SetField { .. } => true, + + // allocations the collector tracks, which can start a collection + Op::NewInstance { .. } + | Op::MakeClosure { .. } + | Op::BuildList { .. } + | Op::BuildSet { .. } + | Op::BuildTuple { .. } + | Op::BuildDict { .. } + | Op::MakeSlice { .. } + | Op::ToTuple { .. } + | Op::RaiseStandard { .. } + | Op::RaiseWith { .. } => true, + + // a function known to stand resolves to nothing to call, from a flag + Op::ResolveFunction { name, .. } => !held.standing.contains(&Guard::Function(name.clone())), + + // a namespace lookup compares keys, and a store or a delete lets go of a value + Op::LoadGlobal { .. } + | Op::StoreGlobal { .. } + | Op::DeleteGlobal { .. } + | Op::BuiltinStands { .. } + | Op::LoopGuardsHold { .. } => true, + + // a licence arms itself with a lookup through the type's dicts, and an instance + // dict can hold a key of any type + Op::MethodStands { .. } + | Op::AccessorStands { .. } + | Op::FieldStands { .. } + | Op::DictShadows { .. } + | Op::LicenceHolds { .. } => true, + + // the object protocol, which is python code wherever a type says so + Op::ObjectBinary { .. } + | Op::ObjectCompare { .. } + | Op::ObjectRichCompare { .. } + | Op::Truthy { .. } + | Op::FloatObjectBinary { .. } + | Op::FloatObjectCompare { .. } + | Op::IsInstance { .. } + | Op::MatchAttr { .. } + | Op::MatchSlice { .. } + | Op::MatchKey { .. } + | Op::MatchRest { .. } + | Op::Contains { .. } + | Op::GetAttr { .. } + | Op::SetAttr { .. } + | Op::ReadAttribute { .. } + | Op::WriteAttribute { .. } + | Op::DeleteAttr { .. } + | Op::GetItem { .. } + | Op::SetItem { .. } + | Op::DeleteItem { .. } + | Op::DictFind { .. } + | Op::Format { .. } + | Op::StrConcat { .. } + | Op::StrOfInt { .. } + | Op::StrConcatInt { .. } + | Op::Unpack { .. } + | Op::Extend { .. } + | Op::MergeKeywords { .. } + | Op::GetIter { .. } + | Op::IterNext { .. } => true, + + // calls out of the unit, and the protocols built on them + Op::CallPython { .. } + | Op::CallValue { .. } + | Op::CallThrough { .. } + | Op::CallMethod { .. } + | Op::CallUnpacked { .. } + | Op::ImportModule { .. } + | Op::ImportFrom { .. } + | Op::Enter { .. } + | Op::BindExit { .. } + | Op::ExitContext { .. } + | Op::DelegateIter { .. } + | Op::DelegateStep { .. } + | Op::AsyncContext { .. } + | Op::AsyncIter { .. } + | Op::Warn { .. } => true, + + // an exception is normalised by calling its class, handled ones are let go of, and + // leaving a frame releases what it holds + Op::FetchException { .. } + | Op::ExceptionMatches { .. } + | Op::PopHandled { .. } + | Op::RaiseObject { .. } + | Op::Reraise { .. } + | Op::LeaveGenerator { .. } + | Op::FinishFrame { .. } => true, + } +} + +#[cfg(test)] +mod tests { + use by_ir::builder::FunctionBuilder; + use by_ir::function::{ClassIr, FieldDecl, Function, ModuleIr}; + use by_ir::ops::{BinOp, BlockId, CmpOp, Op, RegisterId, Terminator, Value}; + use by_ir::rtype::{IntWidth, RType}; + + use std::collections::BTreeSet; + + use super::{Effects, Guard, Held, Kind}; + + fn module(functions: Vec) -> ModuleIr { + let mut module = ModuleIr::new("app"); + module.functions = functions; + module + } + + /// every operation of the function named `name` that a pass asking about each of them + /// would find can run python code, as its debug rendering + fn running(module: &ModuleIr, name: &str) -> Vec { + let effects = Effects::of(module); + let mut out = Vec::new(); + for function in module + .all_functions() + .filter(|function| function.name == name) + { + let entries = Held::solve( + function, + Function::entry(), + effects.held_at_entry(function), + |_| true, + ); + for (index, block) in function.blocks.iter().enumerate() { + let Some(mut held) = entries.get(&BlockId(index)).cloned() else { + continue; + }; + for op in &block.ops { + if effects.op_runs_python(function, op, &held) { + out.push(format!("{op:?}")); + } + held.step(function, op); + } + } + } + out + } + + /// `def scale(x: float) -> float: return x * 2.0` + fn float_body(name: &str) -> Function { + let mut builder = FunctionBuilder::new(name, RType::FLOAT); + let x = builder.param("x", RType::FLOAT); + let out = builder.temp(RType::FLOAT); + builder.push(Op::FloatBinary { + dest: out, + op: BinOp::Mul, + lhs: Value::Register(x), + rhs: Value::Float(2.0), + }); + builder.terminate(Terminator::Return(Value::Register(out))); + builder.finish() + } + + fn call(name: &str, owner: Option<&str>, callee: &str) -> Function { + let mut builder = FunctionBuilder::new(name, RType::FLOAT); + let x = builder.param("x", RType::FLOAT); + let out = builder.temp(RType::FLOAT); + builder.push(Op::CallNative { + dest: Some(out), + owner: owner.map(str::to_string), + callee: callee.to_string(), + args: vec![Value::Register(x)], + }); + builder.terminate(Terminator::Return(Value::Register(out))); + builder.finish() + } + + fn class(name: &str, methods: Vec) -> ClassIr { + ClassIr { + name: name.to_string(), + immutable: false, + environment: false, + exported: true, + base: None, + inherited_init: false, + fields_are_parameters: false, + dataclass: false, + generic: false, + declares_slots: false, + constants: Vec::new(), + properties: Vec::new(), + slot_aliases: Vec::new(), + fields: vec![FieldDecl { + cell: false, + name: "v".to_string(), + ty: RType::INT, + default: None, + optional: false, + defaulted_by: None, + }], + decorators: Vec::new(), + methods, + resume: None, + keywords: Vec::new(), + } + } + + fn receiver() -> RType { + RType::Instance { + class: "Box".to_string(), + exact: false, + } + } + + #[test] + fn float_arithmetic_and_a_call_to_it_run_nothing() { + let module = module(vec![float_body("scale"), call("outer", None, "scale")]); + let effects = Effects::of(&module); + assert!(!effects.call_runs_python("scale")); + assert!(!effects.call_runs_python("outer")); + assert_eq!(running(&module, "outer"), Vec::::new()); + } + + #[test] + fn a_call_out_of_the_unit_runs_python_and_so_does_its_caller() { + let module = module(vec![call("outer", None, "elsewhere")]); + assert!(Effects::of(&module).call_runs_python("outer")); + assert_eq!(running(&module, "outer").len(), 1); + } + + #[test] + fn a_getter_calling_anything_runs_python_through_every_call_to_it() { + // `@property def v(self) -> float: return helper(1.0)`, where `helper` is + // nothing the unit compiled — so the getter's own body is otherwise clear, and it + // is the call alone that decides + let mut getter = FunctionBuilder::new("v$get", RType::FLOAT); + getter.param("self", receiver()); + let out = getter.temp(RType::FLOAT); + getter.push(Op::CallNative { + dest: Some(out), + owner: None, + callee: "helper".to_string(), + args: vec![Value::Float(1.0)], + }); + getter.terminate(Terminator::Return(Value::Register(out))); + + let mut module = module(vec![call("reader", Some("Box"), "v$get")]); + module.classes.push(class("Box", vec![getter.finish()])); + let effects = Effects::of(&module); + assert!(effects.call_runs_python("Box.v$get")); + assert!(effects.call_runs_python("reader")); + assert_eq!(running(&module, "reader").len(), 1); + } + + /// `def add(a: int, b: int) -> int: return a + b` + fn add() -> Function { + let mut builder = FunctionBuilder::new("add", RType::INT); + let a = builder.param("a", RType::INT); + let b = builder.param("b", RType::INT); + let out = builder.temp(RType::INT); + builder.push(Op::IntBinary { + dest: out, + op: BinOp::Add, + lhs: Value::Register(a), + rhs: Value::Register(b), + }); + builder.terminate(Terminator::Return(Value::Register(out))); + builder.finish() + } + + #[test] + fn a_call_handed_exact_ints_runs_nothing_where_one_handed_anything_may() { + // `add(n, 1)` asks a subclass's `__add__` where `n` is one; `add(add(0, 1), 1)` is + // two machine additions, and what the inner call answers is itself an exact `int` + let mut builder = FunctionBuilder::new("caller", RType::INT); + let n = builder.param("n", RType::INT); + let first = builder.temp(RType::INT); + let second = builder.temp(RType::INT); + let third = builder.temp(RType::INT); + let call = |dest, args| Op::CallNative { + dest: Some(dest), + owner: None, + callee: "add".to_string(), + args, + }; + builder.push(call(first, vec![Value::Int(0), Value::Int(1)])); + builder.push(call(second, vec![Value::Register(first), Value::Int(1)])); + builder.push(call(third, vec![Value::Register(n), Value::Int(1)])); + builder.terminate(Terminator::Return(Value::Register(second))); + let module = module(vec![add(), builder.finish()]); + let effects = Effects::of(&module); + assert!(effects.call_runs_python("add")); + let ops = running(&module, "caller"); + assert_eq!(ops.len(), 1, "{ops:?}"); + assert!(ops[0].contains("Register(RegisterId(0))"), "{ops:?}"); + } + + #[test] + fn a_recursion_runs_nothing_only_where_its_own_name_still_stands() { + // `def depth(n): return 0 if n < 1 else depth(n - 1) + 1`, with the recursive call + // guarded: the arm the guard turns away calls whatever the name holds, so a call + // runs nothing only where the name is known to stand + let mut builder = FunctionBuilder::new("depth", RType::INT); + let n = builder.param("n", RType::INT); + let less = builder.temp(RType::BIT); + let resolved = builder.temp(RType::OBJECT); + let stands = builder.temp(RType::BIT); + let smaller = builder.temp(RType::INT); + let below = builder.temp(RType::INT); + let callee = builder.temp(RType::OBJECT); + let boxed = builder.temp(RType::OBJECT); + let answered = builder.temp(RType::OBJECT); + let sum = builder.temp(RType::INT); + let base = builder.new_block(); + let guard = builder.new_block(); + let native = builder.new_block(); + let rebound = builder.new_block(); + let join = builder.new_block(); + builder.push(Op::IntCompare { + dest: less, + op: CmpOp::Lt, + lhs: Value::Register(n), + rhs: Value::Int(1), + }); + builder.terminate(Terminator::Branch { + cond: Value::Register(less), + then_block: base, + else_block: guard, + }); + builder.switch_to(base); + builder.terminate(Terminator::Return(Value::Int(0))); + builder.switch_to(guard); + builder.push(Op::ResolveFunction { + dest: resolved, + name: "depth".to_string(), + }); + builder.push(Op::IntBinary { + dest: smaller, + op: BinOp::Sub, + lhs: Value::Register(n), + rhs: Value::Int(1), + }); + builder.push(Op::FunctionStands { + dest: stands, + src: Value::Register(resolved), + name: "depth".to_string(), + }); + builder.terminate(Terminator::Branch { + cond: Value::Register(stands), + then_block: native, + else_block: rebound, + }); + builder.switch_to(native); + builder.push(Op::Release { + value: Value::Register(resolved), + path: Box::new([]), + }); + builder.push(Op::CallNative { + dest: Some(below), + owner: None, + callee: "depth".to_string(), + args: vec![Value::Register(smaller)], + }); + builder.terminate(Terminator::Goto(join)); + builder.switch_to(rebound); + builder.push(Op::FunctionCallee { + dest: callee, + src: Value::Register(resolved), + name: "depth".to_string(), + }); + builder.push(Op::Box { + dest: boxed, + src: Value::Register(smaller), + }); + builder.push(Op::CallThrough { + dest: answered, + callee: Value::Register(callee), + args: vec![Value::Register(boxed)], + keywords: Vec::new(), + }); + builder.push(Op::Unbox { + dest: below, + src: Value::Register(answered), + to: RType::INT, + }); + builder.terminate(Terminator::Goto(join)); + builder.switch_to(join); + builder.push(Op::IntBinary { + dest: sum, + op: BinOp::Add, + lhs: Value::Register(below), + rhs: Value::Int(1), + }); + builder.terminate(Terminator::Return(Value::Register(sum))); + + let module = module(vec![builder.finish()]); + let effects = Effects::of(&module); + let function = &module.functions[0]; + let call = Op::CallNative { + dest: None, + owner: None, + callee: "depth".to_string(), + args: vec![Value::Int(5)], + }; + let held = effects.held_at_entry(function); + assert!(effects.op_runs_python(function, &call, &held)); + let mut standing = held; + standing.assume_standing(&BTreeSet::from([Guard::Function("depth".to_string())])); + assert!(!effects.op_runs_python(function, &call, &standing)); + // and not with an argument that may be a subclass + let unknown = Op::CallNative { + dest: None, + owner: None, + callee: "depth".to_string(), + args: vec![Value::Register(n)], + }; + assert!(effects.op_runs_python(function, &unknown, &standing)); + } + + #[test] + fn a_cycle_of_clear_bodies_stays_clear() { + let module = module(vec![call("ping", None, "pong"), call("pong", None, "ping")]); + let effects = Effects::of(&module); + assert!(!effects.call_runs_python("ping")); + assert!(!effects.call_runs_python("pong")); + } + + #[test] + fn letting_go_of_an_object_that_may_have_a_finalizer_runs_python() { + // `o.method()` hands back anything at all, `__del__` included, and the local + // holding it lets go of it + let mut builder = FunctionBuilder::new("hold", RType::FLOAT); + let given = builder.param("o", RType::OBJECT); + let held = builder.local("held", RType::OBJECT); + builder.push(Op::CallMethod { + dest: held, + receiver: Value::Register(given), + name: "method".to_string(), + args: Vec::new(), + }); + builder.push(Op::Release { + value: Value::Register(held), + path: Box::new([]), + }); + builder.terminate(Terminator::Return(Value::Float(0.0))); + let module = module(vec![builder.finish()]); + let ops = running(&module, "hold"); + assert!(ops.iter().any(|op| op.starts_with("Release")), "{ops:?}"); + } + + #[test] + fn letting_go_of_an_exact_int_or_str_or_a_copy_of_a_parameter_runs_nothing() { + let mut builder = FunctionBuilder::new("digits", RType::FLOAT); + let count = builder.param("n", RType::fixed(IntWidth::I64)); + let given = builder.param("o", RType::OBJECT); + let tagged = builder.local("t", RType::INT); + let text = builder.local("s", RType::STR); + let copy = builder.local("c", RType::OBJECT); + builder.push(Op::Box { + dest: tagged, + src: Value::Register(count), + }); + builder.assign(text, Value::Str("x".to_string())); + builder.assign(copy, Value::Register(given)); + for register in [tagged, text, copy] { + builder.push(Op::Release { + value: Value::Register(register), + path: Box::new([]), + }); + } + builder.terminate(Terminator::Return(Value::Float(0.0))); + let module = module(vec![builder.finish()]); + assert_eq!(running(&module, "digits"), Vec::::new()); + assert!(!Effects::of(&module).call_runs_python("digits")); + } + + #[test] + fn an_allocation_the_collector_tracks_runs_python_and_a_boxed_int_does_not() { + // an `int` is not tracked by the collector, so building one cannot start a + // collection; a list is + let build = |with_list: bool| { + let mut builder = FunctionBuilder::new("build", RType::FLOAT); + let n = builder.param("n", RType::fixed(IntWidth::I64)); + let tagged = builder.local("t", RType::INT); + builder.push(Op::Box { + dest: tagged, + src: Value::Register(n), + }); + if with_list { + let list = builder.temp(RType::LIST); + builder.push(Op::BuildList { + dest: list, + items: Vec::new(), + }); + } + builder.terminate(Terminator::Return(Value::Float(0.0))); + module(vec![builder.finish()]) + }; + assert_eq!(running(&build(false), "build"), Vec::::new()); + let ops = running(&build(true), "build"); + assert!( + ops.iter().all(|op| op.starts_with("BuildList")) && ops.len() == 1, + "{ops:?}" + ); + } + + #[test] + fn a_comparison_of_objects_runs_python() { + let mut builder = FunctionBuilder::new("same", RType::BIT); + let a = builder.param("a", RType::OBJECT); + let b = builder.param("b", RType::OBJECT); + let out = builder.temp(RType::BIT); + builder.push(Op::ObjectCompare { + dest: out, + op: CmpOp::Eq, + lhs: Value::Register(a), + rhs: Value::Register(b), + }); + builder.terminate(Terminator::Return(Value::Register(out))); + let module = module(vec![builder.finish()]); + assert_eq!(running(&module, "same").len(), 1); + } + + #[test] + fn a_comparison_of_tagged_ints_runs_python_unless_both_are_exact() { + // a parameter may be an `int` subclass with an `__lt__` of its own; a length is + // an exact `int` whatever it measured + let build = |bound_is_a_length: bool| { + let mut builder = FunctionBuilder::new("below", RType::BIT); + let n = builder.param("n", RType::INT); + let s = builder.param("s", RType::STR); + let bound = builder.temp(RType::INT); + let index = builder.local("i", RType::fixed(IntWidth::I64)); + builder.assign(index, Value::Fixed(0)); + if bound_is_a_length { + builder.push(Op::Len { + dest: bound, + src: Value::Register(s), + }); + } else { + builder.push(Op::IntBinary { + dest: bound, + op: BinOp::Add, + lhs: Value::Register(n), + rhs: Value::Int(1), + }); + } + let out = builder.temp(RType::BIT); + builder.push(Op::IntCompare { + dest: out, + op: CmpOp::Lt, + lhs: Value::Register(index), + rhs: Value::Register(bound), + }); + builder.terminate(Terminator::Return(Value::Register(out))); + module(vec![builder.finish()]) + }; + + // `n + 1` asks the subclass, and so does comparing against what it answered + let ops = running(&build(false), "below"); + assert_eq!(ops.len(), 2, "{ops:?}"); + + // the length of a `str` that may be a subclass is its `__len__`, but what that + // answered is an exact `int` + let ops = running(&build(true), "below"); + assert_eq!(ops.len(), 1, "{ops:?}"); + assert!(ops[0].starts_with("Len"), "{ops:?}"); + + // and once something has tested that the `str` is exact, nothing runs at all + let module = build(true); + let function = &module.functions[0]; + let effects = Effects::of(&module); + let mut held = Held::at_entry(function); + held.assume_exact(RegisterId(1), Kind::Str); + for op in &function.blocks[0].ops { + assert!(!effects.op_runs_python(function, op, &held), "{op:?}"); + held.step(function, op); + } + } + + #[test] + fn overwriting_a_register_that_may_hold_an_int_subclass_runs_python() { + let mut builder = FunctionBuilder::new("keep", RType::FLOAT); + let n = builder.param("n", RType::INT); + let held = builder.local("held", RType::INT); + builder.push(Op::IntBinary { + dest: held, + op: BinOp::Add, + lhs: Value::Register(n), + rhs: Value::Int(1), + }); + builder.assign(held, Value::Int(1)); + builder.terminate(Terminator::Return(Value::Float(0.0))); + let module = module(vec![builder.finish()]); + let ops = running(&module, "keep"); + assert!(ops.iter().any(|op| op.starts_with("Assign")), "{ops:?}"); + } + + #[test] + fn what_one_path_leaves_unknown_is_unknown_where_the_paths_meet() { + // `t = len(s)` on one arm and `t = s.count()` on the other: after the merge a + // comparison against `t` may reach an `int` subclass + let mut builder = FunctionBuilder::new("merge", RType::BIT); + let s = builder.param("s", RType::STR); + let which = builder.param("which", RType::BIT); + let bound = builder.local("t", RType::INT); + let other = builder.temp(RType::OBJECT); + let left = builder.new_block(); + let right = builder.new_block(); + let join = builder.new_block(); + builder.terminate(Terminator::Branch { + cond: Value::Register(which), + then_block: left, + else_block: right, + }); + builder.switch_to(left); + builder.push(Op::Len { + dest: bound, + src: Value::Register(s), + }); + builder.terminate(Terminator::Goto(join)); + builder.switch_to(right); + builder.push(Op::CallMethod { + dest: other, + receiver: Value::Register(s), + name: "count".to_string(), + args: Vec::new(), + }); + builder.push(Op::Unbox { + dest: bound, + src: Value::Register(other), + to: RType::INT, + }); + builder.terminate(Terminator::Goto(join)); + builder.switch_to(join); + let out = builder.temp(RType::BIT); + builder.push(Op::IntCompare { + dest: out, + op: CmpOp::Lt, + lhs: Value::Fixed(0), + rhs: Value::Register(bound), + }); + builder.terminate(Terminator::Return(Value::Register(out))); + let module = module(vec![builder.finish()]); + let ops = running(&module, "merge"); + assert!(ops.iter().any(|op| op.starts_with("IntCompare")), "{ops:?}"); + } + + #[test] + fn an_attribute_read_or_a_field_licence_on_an_extendable_class_runs_python() { + let build = |licence: bool| { + let mut builder = FunctionBuilder::new("read", RType::BIT); + let o = builder.param("o", receiver()); + if licence { + let stands = builder.temp(RType::BIT); + builder.push(Op::FieldStands { + dest: stands, + src: Value::Register(o), + class: "Box".to_string(), + field: "v".to_string(), + }); + } else { + let got = builder.temp(RType::OBJECT); + builder.push(Op::GetAttr { + dest: got, + receiver: Value::Register(o), + name: "v".to_string(), + }); + } + builder.terminate(Terminator::Return(Value::Bit(true))); + let mut module = module(vec![builder.finish()]); + module.classes.push(class("Box", Vec::new())); + module + }; + let ops = running(&build(true), "read"); + assert!( + ops.len() == 1 && ops[0].starts_with("FieldStands"), + "{ops:?}" + ); + let ops = running(&build(false), "read"); + assert!(ops.len() == 1 && ops[0].starts_with("GetAttr"), "{ops:?}"); + } + + #[test] + fn a_rebinding_of_a_namespace_runs_python() { + let mut builder = FunctionBuilder::new("rebind", RType::BIT); + let status = builder.temp(RType::BIT); + builder.push(Op::StoreGlobal { + dest: status, + name: "len".to_string(), + value: Value::Int(0), + }); + builder.terminate(Terminator::Return(Value::Register(status))); + let module = module(vec![builder.finish()]); + assert_eq!(running(&module, "rebind").len(), 1); + } +} diff --git a/crates/by_rt/include/by.h b/crates/by_rt/include/by.h index f00cd401e1..b703f778c8 100644 --- a/crates/by_rt/include/by.h +++ b/crates/by_rt/include/by.h @@ -3827,6 +3827,17 @@ static inline void By_ArmAccessor(ByAccessorLicence *licence, PyObject *type, co licence->version = owner->tp_version_tag; } +/* one member's licence joined into its class's: the class's stands only where every member + * was found as compiled, all under the one version */ +static inline void By_JoinLicence(unsigned int *version, int *first, unsigned int member) { + if (*first) { + *version = member; + *first = 0; + } else if (*version != member) { + *version = 0u; + } +} + /* whether `o` is exactly `type` and `type` still answers as [`By_ArmAccessor`] found */ static inline char By_AccessorStands(PyObject *o, PyObject *type, const ByAccessorLicence *licence) { @@ -7073,6 +7084,55 @@ static inline void By_ArmField(ByAccessorLicence *licence, PyObject *type, const licence->version = owner->tp_version_tag; } +#ifdef BY_LICENCE_RECHECK + +/* re-ask the lookup a field read or written at its offset skipped: that the receiver is + * exactly the class, the class answers reads and writes the generic way, and the name still + * resolves on the class to the descriptor this module published for the field + * + * the descriptor is found without being run, as [`By_ArmField`] finds it */ +static void By_RecheckField(PyObject *o, PyObject *type, const char *class_name, + const char *name, getter get) { + if (o == NULL) { + By_LicenceFailed(class_name, name, "the receiver is NULL", NULL); + return; + } + if ((PyObject *)Py_TYPE(o) != type) { + By_LicenceFailed(class_name, name, "the receiver is not this class", + Py_TYPE(o)->tp_name); + return; + } + PyTypeObject *owner = (PyTypeObject *)type; + if (owner->tp_getattro != PyObject_GenericGetAttr + || owner->tp_setattro != PyObject_GenericSetAttr) { + By_LicenceFailed(class_name, name, + "the class answers reads or writes with a hook of its own", NULL); + return; + } + PyObject *key = PyUnicode_InternFromString(name); + if (key == NULL) { + PyErr_Clear(); + By_LicenceFailed(class_name, name, "the name could not be interned", NULL); + return; + } + PyObject *found = _PyType_Lookup(owner, key); + Py_DECREF(key); + if (found == NULL) { + By_LicenceFailed(class_name, name, "the name no longer resolves on the class", NULL); + return; + } + int published = (Py_IS_TYPE(found, &PyGetSetDescr_Type) + && ((PyGetSetDescrObject *)found)->d_getset->get == get) + || (Py_IS_TYPE(found, &By_FieldDefaultType) + && ((By_FieldDefaultObject *)found)->by_get == get); + if (!published) { + By_LicenceFailed(class_name, name, + "the class no longer publishes the compiled field", Py_TYPE(found)->tp_name); + } +} + +#endif /* BY_LICENCE_RECHECK */ + /* ── an emitted instance's `__dict__` ───────────────────────────────────────── * * an emitted instance keeps its attributes in two places. the ones the class itself @@ -7986,6 +8046,91 @@ static inline char By_SetAttr(PyObject *o, PyObject *name, PyObject *value) { return PyObject_SetAttr(o, name, value) < 0 ? 2 : 0; } +/* the arm a licensed field or property access takes where its licence does not stand: the + * attribute through the object protocol, narrowed to the representation the compiled + * access holds or boxed from it + * + * one call kept out of line, because it is the rare arm of a test the hot path asks every + * time. written as a lookup and a narrowing apiece, the arm gave its function a register, a + * retain, a release and two error tests, and the C compiler stopped writing a two-line + * accessor out in place. the name is interned into `*slot` the first time */ +static inline PyObject *By_UnboxStr(PyObject *o); + +BY_COLD PyObject *By_ReadAttrObject(PyObject *o, PyObject **slot, const char *text, + Py_ssize_t length) { + if (*slot == NULL) *slot = By_InternedStr(text, length); + if (*slot == NULL) return NULL; + return By_GetAttr(o, *slot); +} + +BY_COLD ByTagged By_ReadAttrInt(PyObject *o, PyObject **slot, const char *text, + Py_ssize_t length) { + PyObject *found = By_ReadAttrObject(o, slot, text, length); + if (found == NULL) return BY_INT_ERROR; + ByTagged narrowed = By_UnboxInt(found); + Py_DECREF(found); + return narrowed; +} + +BY_COLD double By_ReadAttrFloat(PyObject *o, PyObject **slot, const char *text, + Py_ssize_t length) { + PyObject *found = By_ReadAttrObject(o, slot, text, length); + if (found == NULL) return BY_FLOAT_ERROR; + double narrowed = By_UnboxFloat(found); + Py_DECREF(found); + return narrowed; +} + +BY_COLD char By_ReadAttrBool(PyObject *o, PyObject **slot, const char *text, + Py_ssize_t length) { + PyObject *found = By_ReadAttrObject(o, slot, text, length); + if (found == NULL) return 2; + char narrowed = By_UnboxBool(found); + Py_DECREF(found); + return narrowed; +} + +BY_COLD PyObject *By_ReadAttrStr(PyObject *o, PyObject **slot, const char *text, + Py_ssize_t length) { + PyObject *found = By_ReadAttrObject(o, slot, text, length); + if (found == NULL) return NULL; + PyObject *narrowed = By_UnboxStr(found); + Py_DECREF(found); + return narrowed; +} + +BY_COLD char By_WriteAttrObject(PyObject *o, PyObject **slot, const char *text, + Py_ssize_t length, PyObject *value) { + if (*slot == NULL) *slot = By_InternedStr(text, length); + if (*slot == NULL) return 2; + return By_SetAttr(o, *slot, value); +} + +/* a value boxed, handed to the protocol and let go of again, as the boxing and the store + * would have been */ +BY_COLD char By_WriteAttrBoxed(PyObject *o, PyObject **slot, const char *text, + Py_ssize_t length, PyObject *boxed) { + if (boxed == NULL) return 2; + char refused = By_WriteAttrObject(o, slot, text, length, boxed); + Py_DECREF(boxed); + return refused; +} + +BY_COLD char By_WriteAttrInt(PyObject *o, PyObject **slot, const char *text, + Py_ssize_t length, ByTagged value) { + return By_WriteAttrBoxed(o, slot, text, length, By_BoxInt(value)); +} + +BY_COLD char By_WriteAttrFloat(PyObject *o, PyObject **slot, const char *text, + Py_ssize_t length, double value) { + return By_WriteAttrBoxed(o, slot, text, length, By_BoxFloat(value)); +} + +BY_COLD char By_WriteAttrBool(PyObject *o, PyObject **slot, const char *text, + Py_ssize_t length, char value) { + return By_WriteAttrBoxed(o, slot, text, length, By_BoxBool(value)); +} + /* build a list from `nargs` owned references, stealing each */ static inline PyObject *By_BuildList(PyObject **items, Py_ssize_t nargs) { PyObject *list = PyList_New(nargs); @@ -9459,6 +9604,36 @@ static inline char By_BuiltinStands(ByBuiltinSite *site, PyObject *dict, const c return By_ArmBuiltinSite(site, dict, builtin); } +/* whether `builtin` resolves to the builtin, asked on the way into a loop that asks + * nothing again until it reaches code that can run python + * + * only python code can write a namespace, so while this thread runs no python the answer + * stands — but only while no other thread can run either. a build without the GIL answers + * no. a failed lookup answers no too, and leaves no error behind: the loop as written asks + * the question again where python would, and raises what python raises */ +static inline char By_BuiltinStandsOnEntry(ByBuiltinSite *site, PyObject *dict, + const char *builtin) { +#ifdef Py_GIL_DISABLED + (void)site; + (void)dict; + (void)builtin; + return 0; +#else + char answer = By_BuiltinStands(site, dict, builtin); + if (BY_UNLIKELY(answer == 2)) { + PyErr_Clear(); + return 0; + } + return answer; +#endif +} + +/* whether a tagged integer holds exactly an `int` rather than a subclass: a short always + * does, and a pointer is asked its type */ +static inline char By_IsExactInt(ByTagged x) { + return (char)(By_IsShort(x) || PyLong_CheckExact(By_LongOf(x))); +} + /* `str(n)` for a tagged integer, given whatever the name `str` resolved to * * the resolution is the caller's and still happens every time, so a module that @@ -11791,8 +11966,20 @@ static inline char By_StrItemCompareChar(PyObject *s, ByTagged index, Py_UCS4 co * everything the fast path does not answer is handed to the tagged form, so a * subclass, an out-of-range index and an index that has to become an object are * all still answered in exactly one place */ -static inline char By_StrItemCompareCharI64(PyObject *s, int64_t index, Py_UCS4 codepoint, - int op) { +/* the rest of [`By_StrItemCompareCharI64`]: a subclass, or an index out of range */ +BY_COLD char By_StrItemCompareCharI64Slow(PyObject *s, int64_t index, Py_UCS4 codepoint, + int op) { + ByTagged tagged = By_IntFromI64(index); + char answer; + if (BY_UNLIKELY(tagged == BY_INT_ERROR)) return 2; + answer = By_StrItemCompareChar(s, tagged, codepoint, op); + By_DecRefTagged(tagged); + return answer; +} + +/* inlined, with the slow half kept out of line: a loop the compiler has duplicated reads a + * character in both copies, and the compiler weighs inlining the whole helper twice */ +BY_HOT char By_StrItemCompareCharI64(PyObject *s, int64_t index, Py_UCS4 codepoint, int op) { if (BY_LIKELY(s != NULL && PyUnicode_CheckExact(s))) { int64_t length = (int64_t) PyUnicode_GET_LENGTH(s); int64_t at = index < 0 ? index + length : index; @@ -11808,14 +11995,7 @@ static inline char By_StrItemCompareCharI64(PyObject *s, int64_t index, Py_UCS4 } } } - { - ByTagged tagged = By_IntFromI64(index); - char answer; - if (BY_UNLIKELY(tagged == BY_INT_ERROR)) return 2; - answer = By_StrItemCompareChar(s, tagged, codepoint, op); - By_DecRefTagged(tagged); - return answer; - } + return By_StrItemCompareCharI64Slow(s, index, codepoint, op); } #endif /* BY_RT_H */ From 422ec1bf53f13a6362b3145820a6c076ebf1cdc7 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 15 Sep 2026 06:22:25 +1000 Subject: [PATCH 05/16] native: fill a class's dunder slots and instance surface as python does fill every slot python gives a dunder's name, carry members the module body rewrites after the class statement onto the emitted type, keep an attribute that shares a method's name in the instance dict, give instances a weak reference list on python's terms, hand interpreted subclasses an allocator that marks int fields unset, and fix delegation, parked loop iterators and generator expression evaluation order in generators --- crates/by_build/tests/differential.rs | 559 +++++++++++++++++- crates/by_codegen_c/src/lib.rs | 348 +++++++++-- crates/by_ir/src/builder.rs | 12 + crates/by_ir/src/function.rs | 81 ++- crates/by_ir/src/print.rs | 2 + crates/by_ir/src/verify.rs | 5 + crates/by_irbuild/src/lib.rs | 328 ++++++++-- crates/by_irbuild/src/tests.rs | 153 +++-- crates/by_rt/include/by.h | 260 +++++++- .../development/compilation/runtime.md | 33 ++ 10 files changed, 1595 insertions(+), 186 deletions(-) diff --git a/crates/by_build/tests/differential.rs b/crates/by_build/tests/differential.rs index edffa3b178..1215f19335 100644 --- a/crates/by_build/tests/differential.rs +++ b/crates/by_build/tests/differential.rs @@ -10214,15 +10214,12 @@ def each(xs): } #[test] -fn a_class_that_takes_a_weak_reference_to_itself_is_declined() { - // an emitted instance is its layout and a type spec adds no `__weakref__`, so - // `ref(self)` raises `TypeError` where python hands back a reference. the class is - // turned down and runs interpreted instead, which is what makes both legs agree. - // - // this is `_weakrefset.WeakSet`'s shape, snapshotting the reference into a nested - // function's default — the two constructions are together because the gate is about - // the weak reference and not about where it stands - agree_python_with_declines( +fn a_class_that_takes_a_weak_reference_to_itself_agrees() { + // an emitted instance keeps a weak-reference list, so `ref(self)` hands back a + // reference as python's does. this is `_weakrefset.WeakSet`'s shape, snapshotting the + // reference into a nested function's default — the two constructions are together + // because the reference is about the instance and not about where it is taken + agree_python( "weakself", "\ from weakref import ref @@ -10242,7 +10239,10 @@ class Snapshots: return prefix + type(selfref()).__name__ self.report = report ", - &["m.Holder().alive()", "m.Snapshots().report('is ')"], + &[ + "m.Holder().alive()", + "(lambda s: s.report('is '))(m.Snapshots())", + ], ); } @@ -10927,8 +10927,10 @@ def guarded(words: list[str], index: int) -> str: fn an_attribute_the_instance_is_given_answers_before_a_method_of_the_same_name() { // python looks an instance's own attribute up before a function its class holds, so // an encoder handed a `default` calls that one and an encoder handed nothing calls the - // method — whether the name is called straight away or read first - agree_python_with_declines( + // method — whether the name is called straight away or read first, inside the class or + // from a module function. the attribute is kept in the instance's dict, which is where + // both the lookup and a compiled call look first, and `del` puts the method back + agree_python( "fieldshadowsmethod", "\ from collections.abc import Callable @@ -10948,6 +10950,19 @@ class Encoder: def read(self) -> object: return self.default + def forget(self) -> object: + del self.default + return self.default(3) + + +def encode(e: Encoder) -> object: + return e.default(4) + + +def given(e: Encoder) -> object: + e.default = repr + return e.default(5) + class Base: def label(self) -> str: @@ -10969,6 +10984,13 @@ class Labelled(Base): "m.Encoder.default(m.Encoder(str), 2)", "m.Labelled(lambda: 'given').shown()", "m.Labelled(lambda: 'given').label()", + "m.encode(m.Encoder())", + "m.encode(m.Encoder(str))", + "m.given(m.Encoder())", + "m.Encoder(str).forget()", + "type(_capture(m.Encoder().forget)).__name__", + "sorted(vars(m.Encoder(str)))", + "type(vars(m.Encoder)['default']).__name__ in ('function', 'method_descriptor')", ], ); } @@ -12390,6 +12412,285 @@ def redelegated(v: object) -> object: /// nothing, a one-tuple collapses, and an exception instance is raised in place of /// the error asked for. a subclass carrying its own `value` is the other half, since /// what a delegation collects is the field rather than the attribute +#[test] +fn a_generator_lets_go_of_a_loop_iterator_when_the_loop_is_left() { + // python holds a `for` loop's iterator on the frame's stack and drops it the moment + // the loop is left, however it is left. a generator's frame parks it in a field, and + // that field used to hold it until the frame itself went — so an inner generator's + // `finally` ran after the code that follows a `break`, or after the handler of an + // exception that left the loop, rather than before + agree( + "loopiteratorlifetime", + "\ +def inner(log: list[str]) -> object: + try: + yield 1 + yield 2 + finally: + log.append('inner finally') + + +def raises(log: list[str]) -> object: + try: + for v in inner(log): + raise KeyError('k') + except KeyError: + log.append('outer caught') + yield 5 + + +def breaks(log: list[str]) -> object: + for v in inner(log): + break + log.append('after break') + yield 5 + + +def returns(log: list[str]) -> object: + try: + for v in inner(log): + return + finally: + log.append('outer finally') + yield 5 + + +def continues(log: list[str]) -> object: + for v in inner(log): + log.append('body') + continue + else: + log.append('else') + yield 5 + + +def suspends(log: list[str]) -> object: + for v in inner(log): + yield v + break + log.append('after break') + yield 5 + + +def failing(log: list[str]) -> object: + try: + yield 1 + raise ValueError('inner') + finally: + log.append('inner finally') + + +def raised_by_the_step(log: list[str]) -> object: + try: + for v in failing(log): + log.append('body') + except ValueError: + log.append('outer caught') + yield 5 +", + &["[(lambda log: (list(f(log)), log))([]) for f in \ + (m.raises, m.breaks, m.returns, m.continues, m.suspends, m.raised_by_the_step)]"], + ); +} + +#[test] +fn a_drained_generator_expression_raises_its_first_iterable_where_it_is_written() { + // python evaluates a generator expression's first iterable, and takes its iterator, + // where the expression is written: the generator's frame does not exist yet. so what + // that raises leaves this frame with no `` entry in its traceback, and a + // `StopIteration` it raises stays one rather than becoming the `RuntimeError` a + // generator's frame makes of it. a builtin that drains the generator at once runs its + // loop in this frame, and it used to evaluate the iterable inside that loop's handling + agree_python( + "drainedfirstiterable", + "\ +def boom() -> list[int]: + raise KeyError('k') + + +def stops() -> list[int]: + raise StopIteration + + +def listed() -> list[int]: + return list(x for x in boom()) + + +def tupled() -> tuple[int, ...]: + return tuple(x for x in boom()) + + +def sorted_() -> list[int]: + return sorted(x for x in boom()) + + +def stopped() -> list[int]: + return list(x for x in stops()) + + +def counted(n: object) -> list[int]: + return list(x for x in range(n)) # type: ignore + + +def inner_raises() -> list[int]: + return list(y for x in [1] for y in boom()) +", + &[ + "[(lambda e: (type(e).__name__, _frames(e)))(_capture(f)) \ + for f in (m.listed, m.tupled, m.sorted_, m.stopped, m.inner_raises)]", + "(lambda e: (type(e).__name__, _frames(e)))(_capture(m.counted, 'x'))", + "m.counted(3)", + ], + ); +} + +#[test] +fn an_emitted_instance_can_be_weakly_referenced() { + // an emitted class keeps a weak-reference list on python's own terms, so every weak + // container reaches its instances the way it reaches the interpreted class's: a + // `WeakSet`, the two weak dictionaries, `finalize`, `proxy` and `ref` itself, with the + // reference dying when the instance does. a class declaring `__slots__` throughout + // asks for none, and python refuses it there too. the frontend used to decline a + // `weakref.ref` it could see and left `WeakSet.add` and the rest to raise + agree_python( + "weakreferences", + "\ +import gc +import weakref + + +class Node: + def __init__(self, v: int) -> None: + self.v = v + + +class Leaf(Node): + pass + + +class Tight: + __slots__ = ('v',) + + def __init__(self, v: int) -> None: + self.v = v + + +def containers(n: Node) -> list[int]: + seen: weakref.WeakSet[Node] = weakref.WeakSet() + seen.add(n) + keyed: weakref.WeakKeyDictionary[Node, int] = weakref.WeakKeyDictionary() + keyed[n] = 1 + valued: weakref.WeakValueDictionary[str, Node] = weakref.WeakValueDictionary() + valued['a'] = n + return [len(seen), len(keyed), len(valued), weakref.proxy(n).v] + + +def dies(log: list[str]) -> list[object]: + n = Leaf(2) + r = weakref.ref(n) + weakref.finalize(n, log.append, 'finalized') + alive = r() is n + del n + gc.collect() + return [alive, r(), log] + + +def refuses(t: Tight) -> object: + return weakref.ref(t) +", + &[ + "m.containers(m.Node(3))", + "m.containers(m.Leaf(4))", + "m.dies([])", + "type(_capture(m.refuses, m.Tight(1))).__name__", + "m.Node(1).__weakref__", + "'__weakref__' in vars(m.Node)", + "'__weakref__' in vars(m.Leaf)", + "hasattr(m.Tight, '__weakref__')", + ], + ); +} + +#[test] +fn a_missing_key_is_the_one_argument_of_its_key_error() { + // python raises `KeyError(key)` with the key as its only argument. `PyErr_SetObject` + // takes a tuple value as the whole argument list and an exception instance as the + // exception to raise, so a missing `(1, 2)` raised `KeyError(1, 2)`, a missing `()` + // raised a bare `KeyError()`, and a missing `KeyError('x')` raised that key itself + agree_python( + "keyerrorargument", + "\ +class Point: + def __init__(self) -> None: + self.x = 1 + + +def get(d: dict[object, int], k: object) -> int: + return d[k] + + +def get_tuple(d: dict[tuple[int, int], int], k: tuple[int, int]) -> int: + return d[k] +", + &[ + "[(lambda e: (type(e).__name__, e.args))(_capture(m.get, {}, k)) \ + for k in ((1, 2), (), (1,), KeyError('x'), ValueError('v'), 3, 'k')]", + "(lambda e: (type(e).__name__, e.args))(_capture(m.get_tuple, {}, (1, 2)))", + "[(lambda e: (type(e).__name__, e.args))(_capture(m.Point().__dict__.pop, k)) \ + for k in ((1, 2), KeyError('x'), 'y')]", + ], + ); +} + +#[test] +fn a_delegation_starts_its_iterator_with_none_whatever_the_frame_was_last_sent() { + // a `yield from` sends `None` into the iterator on its first step, as python's does. + // the frame used to send whatever its last `send` handed it, so a `yield from` + // straight after a `yield` that was sent a value raised `can't send non-None value to + // a just-started generator` — and an iterator that accepts anything was handed a value + // meant for the frame + agree( + "delegationstartsnone", + "\ +def inner(log: list[object]) -> object: + got = yield 1 + log.append(got) + + +def outer(log: list[object]) -> object: + got = yield 0 + log.append(got) + yield from inner(log) + + +class Recording: + def __init__(self, log: list[object]) -> None: + self.log = log + + def __iter__(self) -> object: + return self + + def __next__(self) -> object: + self.log.append('next') + return 'n' + + def send(self, value: object) -> object: + self.log.append(('send', value)) + return 's' + + +def through(log: list[object]) -> object: + got = yield 0 + log.append(got) + yield from Recording(log) +", + &[ + "(lambda log: (_sent(m.outer(log), ('v', 'w')), log))([])", + "(lambda log: (_sent(m.through(log), ('v', 'w', 'x')), log))([])", + ], + ); +} + #[test] fn a_return_agrees_between_the_raise_and_the_send_slot() { agree( @@ -21394,6 +21695,140 @@ def install(cls: type) -> type: assert_eq!(out, "function\ngt"); } +#[test] +fn an_interpreted_subclass_instance_no_init_ran_on_has_no_int_field() { + // an unset `int` field is told apart by a value an allocation does not leave, and the + // class's own allocator writes it. python gives a class made by `type(...)` the generic + // allocator rather than inheriting its base's, so a bare `S.__new__(S)` handed back a + // zeroed block — and a zeroed `int` field reads as `0` where python raises + // `AttributeError` + agree_python( + "subclassunsetint", + "\ +class Cell: + def __init__(self) -> None: + self._v = 0 + self._w = 5 + + @property + def v(self) -> int: + return self._v + + def width(self) -> int: + return self._w + + +class Sized(Cell): + def extra(self) -> int: + return 1 + + +class Made: + def __new__(cls) -> 'Made': + return object.__new__(cls) + + def __init__(self) -> None: + self._v = 1 + + +def read_v(cell: Cell) -> int: + return cell._v + + +def made_bare(cls: type) -> object: + return cls.__new__(cls) +", + &[ + "str(_capture(lambda: m.made_bare(m.Cell)._v))", + "str(_capture(lambda: m.made_bare(type('S', (m.Cell,), {}))._v))", + "str(_capture(lambda: m.made_bare(type('S', (m.Cell,), {})).v))", + "str(_capture(lambda: m.read_v(m.made_bare(type('S', (m.Cell,), {})))))", + "str(_capture(lambda: m.made_bare(type('S', (m.Sized,), {})).width()))", + "str(_capture(lambda: (lambda S: S.__new__(S)._w)(type('S', (m.Cell,), {'__new__': lambda cls: super(cls, cls).__new__(cls)}))))", + "(lambda s: (setattr(s, '_v', 4), s._v)[1])(m.made_bare(type('S', (m.Cell,), {})))", + "type('S', (m.Cell,), {})()._v", + // a written `__new__` allocates through `object.__new__(cls)`, which reaches + // the subclass's allocator without passing the class's `tp_new` + "str(_capture(lambda: m.made_bare(type('S', (m.Made,), {}))._v))", + "type('S', (m.Made,), {})()._v", + ], + ); +} + +#[test] +fn a_member_the_module_body_rewrites_after_the_class_statement_is_what_the_class_answers() { + // the module body runs against the interpreted definition, so a property, a method or a + // class constant replaced after the `class` statement is replaced on that definition + // alone. the emitted type holds every name its own body wrote, and it went on answering + // with the property the statement wrote — 7 where python gives 130 — and a direct read + // of the member trusted the licence it had been armed with. `del` is the same shape: the + // name has to go, so the lookup reaches the base's method + agree_python( + "rewrittenmembers", + "\ +class Replaced: + scale = 1 + + def __init__(self) -> None: + self._v = 3 + self._w = 4 + + @property + def v(self) -> int: + return self._v + + def width(self) -> int: + return self._w + + def total(self) -> int: + return self.v + self.width() * self.scale + + +class ReplacedSub(Replaced): + pass + + +class Base: + def extra(self) -> str: + return 'base' + + +class Trimmed(Base): + def extra(self) -> str: + return 'trimmed' + + +Replaced.v = property(lambda self: self._v * 10) +Replaced.width = lambda self: 100 +Replaced.scale = 2 +del Trimmed.extra + + +def replaced(r: Replaced) -> int: + return r.v + r.width() + + +def total(r: Replaced) -> int: + return r.total() + + +def extra(t: Trimmed) -> str: + return t.extra() +", + &[ + "m.replaced(m.Replaced())", + "m.replaced(m.ReplacedSub())", + "m.total(m.Replaced())", + "m.Replaced().total()", + "m.Replaced().v", + "m.Replaced.scale", + "m.extra(m.Trimmed())", + "m.Trimmed().extra()", + "'extra' in vars(m.Trimmed)", + ], + ); +} + #[test] fn a_class_keeps_what_a_factory_installed_on_it_after_the_class_statement() { // `multiprocessing.managers` is the case this is drawn from. it defines `SyncManager` @@ -28138,6 +28573,106 @@ def mutated(items: list[int], at: int, to: int) -> list[int]: ); } +#[test] +fn a_dunder_fills_every_slot_python_gives_its_name() { + // python fills both `sq_length` and `mp_length` from `__len__`, and `len` asks the + // sequence half first — so a `str` subclass that filled only the mapping half kept + // `str`'s own length. `__getitem__` fills `sq_item` the same way, which is what + // iteration, `in` and `reversed` fall back to on a class with no `__iter__`. and + // `__add__`, `__mul__`, `__rmul__`, `__iadd__` and `__imul__` *empty* the sequence + // slot of their name, so `operator.concat` reaches the method rather than the + // base's concatenation + agree_python( + "dunderslotfamily", + "\ +import operator + + +class Text(str): + def __len__(self) -> int: + return 0 + + +class Seq: + def __len__(self) -> int: + return 3 + + def __getitem__(self, i: int) -> int: + if i >= 3: + raise IndexError(i) + return i * 10 + + +class Keyed: + def __getitem__(self, key: object) -> str: + return repr(key) + + def __setitem__(self, key: object, value: object) -> None: + pass + + def __delitem__(self, key: object) -> None: + pass + + +class Bag(list): + def __len__(self) -> int: + return 42 + + def __add__(self, other: object) -> str: + return 'add' + + def __iadd__(self, other: object) -> str: + return 'iadd' + + def __mul__(self, other: object) -> str: + return 'mul' + + +class Left: + def __radd__(self, other: object) -> str: + return 'radd' + + +class Right(Left): + def __add__(self, other: object) -> str: + return 'add' + + +def walk(s: Seq) -> list[int]: + return [x for x in s] + + +def concat(b: Bag) -> object: + return operator.concat(b, [1]) +", + &[ + "len(m.Text('abc'))", + "bool(m.Text('abc'))", + "list(m.Seq())", + "m.walk(m.Seq())", + "20 in m.Seq()", + "5 in m.Seq()", + "list(reversed(m.Seq()))", + "m.Seq()[-1]", + "m.Keyed()['k']", + "m.Keyed()[1:2]", + "m.Keyed.__getitem__(m.Keyed(), 'k')", + "m.Keyed.__setitem__(m.Keyed(), 'k', 1)", + "m.Keyed.__delitem__(m.Keyed(), 'k')", + "len(m.Bag([1]))", + "m.concat(m.Bag([1]))", + "type(_capture(m.operator.concat, m.Bag([1]), 5)).__name__", + "type(_capture(m.operator.iconcat, m.Bag([1]), 5)).__name__", + "m.Bag([1]) + [2]", + "m.Bag([1]) * 2", + "2 * m.Bag([1])", + "1 + m.Right()", + "m.Right() + 1", + "len(type('Sub', (m.Text,), {})('abc'))", + ], + ); +} + #[test] fn the_assignment_slot_carries_both_of_its_methods() { // `mp_ass_subscript` is one slot for `__setitem__` and `__delitem__` — a NULL diff --git a/crates/by_codegen_c/src/lib.rs b/crates/by_codegen_c/src/lib.rs index 3bb6ca3d10..c4e5274a3b 100644 --- a/crates/by_codegen_c/src/lib.rs +++ b/crates/by_codegen_c/src/lib.rs @@ -881,6 +881,11 @@ fn emit_class_struct(module: &ModuleIr, class: &ClassIr) -> String { if reserves_dict_word(module, class) { let _ = writeln!(out, " PyObject *{BY_DICT_MEMBER};"); } + // the weak references to the instance, reserved down a chain on the same terms as the + // dict word — see [`reserves_weak_list_word`] + if reserves_weak_list_word(module, class) { + out.push_str(" PyObject *by_weakrefs;\n"); + } // where a `return` puts its value. a resumable frame reports finishing by writing // here and handing back nothing, so that the slot python asks with — `am_send` — // can say what the frame returned without an exception ever being built. it is not @@ -938,6 +943,64 @@ fn writes_absent_values(module: &ModuleIr, class: &ClassIr) -> bool { && (external_storage(module, class) || !inherits_layout(module, class)) } +/// the class whose allocation writes the absent values into this class's layout, where +/// one does: the class itself, or the in-module base whose layout it inherits +fn absent_values_owner<'a>(module: &'a ModuleIr, class: &'a ClassIr) -> Option<&'a ClassIr> { + let mut current = class; + // bounded the way [`inherits_layout`] is + for _ in 0..=module.classes.len() { + if writes_absent_values(module, current) { + return Some(current); + } + current = class_named(module, current.base.as_ref()?.in_module()?)?; + } + None +} + +/// the `tp_new` of a class whose layout starts with fields absent by value +/// +/// the allocation writes those values, but only the allocation this module emits. python +/// gives a class made by a `class` statement or `type(...)` the generic allocator rather +/// than inheriting its base's, so an interpreted subclass's instance arrived zeroed — and a +/// zeroed `int` field is the `int` zero. such an instance comes through the `tp_new` it +/// inherits, so this is where its type is handed the allocator of the layout it extends — +/// see `By_AdoptAllocator` +fn emit_absent_values_new(module: &ModuleIr, class: &ClassIr) -> String { + let Some(owner) = absent_values_owner(module, class) else { + return String::new(); + }; + let type_name = class.type_name(module.name.dotted()); + let owner_name = owner.type_name(module.name.dotted()); + format!( + "static PyObject *{owner_name}_alloc(PyTypeObject *by_type, Py_ssize_t by_items);\n\ + static PyObject *{type_name}_new(PyTypeObject *by_type, PyObject *by_args, PyObject *by_kwds) {{\n\ + \x20 By_AdoptAllocator(by_type, {type_name}_OBJ, {owner_name}_alloc);\n\ + \x20 return PyType_GenericNew(by_type, by_args, by_kwds);\n}}\n\n" + ) +} + +/// the entry a written `__new__` is published through, where its class's layout starts +/// with fields absent by value +/// +/// the body's `object.__new__(cls)` allocates through `cls`'s own allocator, which for an +/// interpreted subclass is the generic one — so the class is handed this layout's before +/// the body runs, for the reason [`emit_absent_values_new`] gives +fn emit_absent_values_new_entry(module: &ModuleIr, class: &ClassIr, new: &Function) -> String { + let Some(owner) = absent_values_owner(module, class) else { + return String::new(); + }; + let type_name = class.type_name(module.name.dotted()); + let owner_name = owner.type_name(module.name.dotted()); + let wrapper = new.wrapper_symbol(module.name.dotted()); + format!( + "static PyObject *{owner_name}_alloc(PyTypeObject *by_type, Py_ssize_t by_items);\n\ + static PyObject *{type_name}_new_entry(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) {{\n\ + \x20 if (PyVectorcall_NARGS(nargs) > 0 && PyType_Check(args[0]))\n\ + \x20 By_AdoptAllocator((PyTypeObject *)args[0], {type_name}_OBJ, {owner_name}_alloc);\n\ + \x20 return {wrapper}(self, args, nargs, kwnames);\n}}\n\n" + ) +} + /// the allocation of a class whose fields start absent by value, and what it writes /// /// every instance comes from `tp_alloc`, however it was asked for — a construction, a @@ -1192,6 +1255,13 @@ fn emit_class_type(module: &ModuleIr, class: &ClassIr) -> String { if class.resume.is_some() { out.push_str(" PyObject_GC_UnTrack(self);\n"); } + // after the finalizer and before anything the instance holds is released, as + // python's own dealloc for a class it gave a weak-reference list does + if reserves_weak_list_word(module, class) { + out.push_str( + " if (self->by_weakrefs != NULL) PyObject_ClearWeakRefs((PyObject *)self);\n", + ); + } if keeps_a_dict { let _ = writeln!(out, " By_ReleaseInstanceDict(&self->{BY_DICT_MEMBER});"); } @@ -2032,6 +2102,18 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { ); } } + // python publishes `__weakref__` on the class that gave an instance its list, and a + // subclass reads the base's + if module.keeps_weak_references(class) + && !class + .base + .as_ref() + .and_then(ClassBase::in_module) + .and_then(|name| class_named(module, name)) + .is_some_and(|base| module.keeps_weak_references(base)) + { + out.push_str(" {\"__weakref__\", By_GetWeakrefList, NULL, NULL, NULL},\n"); + } out.push_str(" {NULL, NULL, NULL, NULL, NULL}\n};\n\n"); out.push_str(&emit_dataclass_members(class, &type_name)); @@ -2039,12 +2121,22 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { // `__dictoffset__` is not an attribute the table binds: `PyType_FromSpec` lifts it // out and writes it into `tp_dictoffset`, which is the only way a type built from a // spec can say where its instances keep a dict + // `__weaklistoffset__` is lifted out the same way, into `tp_weaklistoffset` if instance_dict(module, class) { + let weak_list = if module.keeps_weak_references(class) { + format!( + "\x20 {{\"__weaklistoffset__\", BY_DICT_OFFSET_MEMBER,\n\ + \x20 offsetof({struct_name}, by_weakrefs), BY_DICT_OFFSET_FLAGS}},\n" + ) + } else { + String::new() + }; let _ = write!( out, "static PyMemberDef {type_name}_members[] = {{\n\ \x20 {{\"__dictoffset__\", BY_DICT_OFFSET_MEMBER,\n\ \x20 offsetof({struct_name}, {BY_DICT_MEMBER}), BY_DICT_OFFSET_FLAGS}},\n\ + {weak_list}\ \x20 {{NULL, 0, 0, 0}}\n}};\n\n" ); } @@ -2248,11 +2340,17 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { // module init binds onto the finished type — see `By_PublishNew` for why the slot // cannot come from the spec if let Some(new) = publishes_new(class) { + let entry = emit_absent_values_new_entry(module, class, new); + let symbol = if entry.is_empty() { + new.wrapper_symbol(module.name.dotted()) + } else { + format!("{type_name}_new_entry") + }; + out.push_str(&entry); let _ = writeln!( out, "static PyMethodDef {type_name}_new_def =\n\ - \x20 {{\"__new__\", (PyCFunction)(void(*)(void)){}, METH_FASTCALL | METH_KEYWORDS, NULL}};\n", - new.wrapper_symbol(module.name.dotted()) + \x20 {{\"__new__\", (PyCFunction)(void(*)(void)){symbol}, METH_FASTCALL | METH_KEYWORDS, NULL}};\n" ); } @@ -2673,7 +2771,7 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { ), ) }; - let weakrefs = if class.resume.is_some() { + let weakrefs = if class.resume.is_some() || module.keeps_weak_references(class) { format!(" .tp_weaklistoffset = offsetof({struct_name}, by_weakrefs),\n") } else { String::new() @@ -2793,6 +2891,9 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { // same reason it does there: the class overrode the allocator let construction = if init.is_empty() || constructs_through_a_written_new(module, class) { init + } else if absent_values_owner(module, class).is_some() { + out.push_str(&emit_absent_values_new(module, class)); + format!("{init}\x20 {{Py_tp_new, (void *){type_name}_new}},\n") } else { format!("{init}\x20 {{Py_tp_new, (void *)PyType_GenericNew}},\n") }; @@ -3016,6 +3117,18 @@ fn reserves_dict_word(module: &ModuleIr, class: &ClassIr) -> bool { }) } +/// whether this class's struct carries the weak-reference list, whether or not its own +/// instances can be weakly referenced +/// +/// for the reason [`reserves_dict_word`] gives: `__slots__` decides per rung whether a +/// weak reference can be made, and every rung has to agree on where the fields start +fn reserves_weak_list_word(module: &ModuleIr, class: &ClassIr) -> bool { + module.classes.iter().any(|other| { + module.keeps_weak_references(other) + && (other.name == class.name || descends_from(module, other, &class.name)) + }) +} + /// whether `class` reaches `ancestor` by following in-module bases fn descends_from(module: &ModuleIr, class: &ClassIr, ancestor: &str) -> bool { let mut current = class; @@ -4249,6 +4362,40 @@ const DUNDER_SLOTS: &[(&str, &str, &str, SlotShape)] = &[ ), ]; +/// the sequence slots python fills from a dunder that [`DUNDER_SLOTS`] already gives a +/// mapping slot, as `(dunder, slot id, sub-table member, adapter suffix)` +/// +/// python's slot table names `__len__` for both `mp_length` and `sq_length`, and the +/// same for item access and assignment. the two halves are not interchangeable: `len` +/// and truth ask `sq_length` first, and iteration, `in` and `reversed` on a class with +/// no `__iter__` go through `sq_item`. a class that filled only the mapping half kept +/// whatever sequence half its base had, `str`'s own length included +const SEQUENCE_HALVES: &[(&str, &str, &str, &str)] = &[ + ("__len__", "Py_sq_length", "sq_length", "len"), + ("__getitem__", "Py_sq_item", "sq_item", "sq_item"), + ( + "__setitem__", + "Py_sq_ass_item", + "sq_ass_item", + "sq_ass_item", + ), +]; + +/// the sequence slots a dunder *empties*, as `(dunder, sub-table member)` +/// +/// python's slot table gives each of these names a sequence slot with no function of its +/// own, so a class writing the method leaves that slot NULL rather than inheriting its +/// base's. `operator.concat` on a `list` subclass with its own `__add__` then falls +/// through to `nb_add` and reaches the method, where the inherited slot would have +/// concatenated +const EMPTIED_SEQUENCE_SLOTS: &[(&str, &str)] = &[ + ("__add__", "sq_concat"), + ("__mul__", "sq_repeat"), + ("__rmul__", "sq_repeat"), + ("__iadd__", "sq_inplace_concat"), + ("__imul__", "sq_inplace_repeat"), +]; + /// the arithmetic dunders, each with the reflected method that answers when it /// was the *other* operand's type python asked /// @@ -4317,6 +4464,22 @@ fn on_our_operand( ) } +/// the half of a binary adapter for the direction the class did not write, which python +/// answers by looking the name up on the operand's type — see `By_InheritedBinary` +fn inherited_direction( + type_name: &str, + name: &str, + receiver: &str, + other: &str, + adapter: &str, +) -> String { + format!( + " if (PyObject_TypeCheck({receiver}, (PyTypeObject *){type_name}_OBJ))\n\ + \x20 return By_InheritedBinary({receiver}, {other}, {}, (void *){adapter});\n", + c_string(name) + ) +} + /// `nb_power`, which `__pow__` and `__rpow__` share with the three-argument `pow` fn emit_power_adapter(module: &ModuleIr, class: &ClassIr, type_name: &str) -> String { let (name, reflected, _, field) = POWER; @@ -4326,18 +4489,20 @@ fn emit_power_adapter(module: &ModuleIr, class: &ClassIr, type_name: &str) -> St return String::new(); } let mut binary = String::new(); - for (filler, receiver, other) in [ - (forward.as_ref(), "by_a", "by_b"), - (backward.as_ref(), "by_b", "by_a"), + for (filler, written, receiver, other) in [ + (forward.as_ref(), name, "by_a", "by_b"), + (backward.as_ref(), reflected, "by_b", "by_a"), ] { - let Some(filler) = filler else { continue }; - binary.push_str(&on_our_operand( - module, - type_name, - filler, - receiver, - &[other], - )); + binary.push_str(&match filler { + Some(filler) => on_our_operand(module, type_name, filler, receiver, &[other]), + None => inherited_direction( + type_name, + written, + receiver, + other, + &format!("{type_name}_{field}"), + ), + }); } // a modulus reaches only the left operand's `__pow__`, and a class that wrote a // two-parameter one raises there — which is what python raises too @@ -4809,15 +4974,20 @@ fn emit_dunder_adapters(module: &ModuleIr, class: &ClassIr, type_name: &str) -> out, "static PyObject *{type_name}_{field}(PyObject *by_a, PyObject *by_b) {{" ); - for (filler, receiver, other) in [(forward, "by_a", "by_b"), (backward, "by_b", "by_a")] { - let Some(filler) = filler else { continue }; - out.push_str(&on_our_operand( - module, - type_name, - &filler, - receiver, - &[other], - )); + for (filler, written, receiver, other) in [ + (forward, name, "by_a", "by_b"), + (backward, reflected, "by_b", "by_a"), + ] { + out.push_str(&match filler { + Some(filler) => on_our_operand(module, type_name, &filler, receiver, &[other]), + None => inherited_direction( + type_name, + written, + receiver, + other, + &format!("{type_name}_{field}"), + ), + }); } // neither side is ours, or ours has no method for this direction — and // then `NotImplemented` is what lets python try the other operand @@ -4825,6 +4995,31 @@ fn emit_dunder_adapters(module: &ModuleIr, class: &ClassIr, type_name: &str) -> } out.push_str(&emit_power_adapter(module, class, type_name)); + // python's own `sq_item` and `sq_ass_item` box the index and look the method up, so + // these hand the boxed index to the mapping adapter the method already has + if answers_slot(class, "__getitem__") { + let _ = writeln!( + out, + "static PyObject *{type_name}_sq_item(PyObject *self, Py_ssize_t by_index) {{\n\ + \x20 PyObject *by_key = PyLong_FromSsize_t(by_index);\n\ + \x20 if (by_key == NULL) return NULL;\n\ + \x20 PyObject *by_r = {type_name}_getitem(self, by_key);\n\ + \x20 Py_DECREF(by_key);\n\ + \x20 return by_r;\n}}" + ); + } + if fills_slot(class, "__setitem__") { + let _ = writeln!( + out, + "static int {type_name}_sq_ass_item(PyObject *self, Py_ssize_t by_index, PyObject *by_value) {{\n\ + \x20 PyObject *by_key = PyLong_FromSsize_t(by_index);\n\ + \x20 if (by_key == NULL) return -1;\n\ + \x20 int by_r = {type_name}_setitem(self, by_key, by_value);\n\ + \x20 Py_DECREF(by_key);\n\ + \x20 return by_r;\n}}" + ); + } + // a static type reaches `nb_bool` and `mp_length` through a sub-table, which // has to exist before the type that points at it let mut mapping = String::new(); @@ -4843,11 +5038,11 @@ fn emit_dunder_adapters(module: &ModuleIr, class: &ClassIr, type_name: &str) -> "static PyMappingMethods {type_name}_mapping = {{\n{mapping}}};" ); } - if answers_slot(class, "__contains__") { + let sequence = sequence_fields(class, type_name); + if !sequence.is_empty() { let _ = writeln!( out, - "static PySequenceMethods {type_name}_sequence = {{\n\ - \x20 .sq_contains = {type_name}_contains,\n}};" + "static PySequenceMethods {type_name}_sequence = {{\n{sequence}}};" ); } // `am_aiter`, `am_anext` and `am_await` share one async sub-table @@ -4889,6 +5084,36 @@ fn sub_table_fields(class: &ClassIr, type_name: &str, table: &str) -> String { out } +/// the sequence halves [`SEQUENCE_HALVES`] this class fills +fn sequence_halves( + class: &ClassIr, +) -> impl Iterator { + SEQUENCE_HALVES + .iter() + .filter(|(name, _, _, _)| fills_slot(class, name)) + .map(|(_, slot, member, suffix)| (*slot, *member, *suffix)) +} + +/// the initializers for the sequence sub-table a static type points at +fn sequence_fields(class: &ClassIr, type_name: &str) -> String { + let mut out = sub_table_fields(class, type_name, "tp_as_sequence"); + for (_, member, suffix) in sequence_halves(class) { + let _ = writeln!(out, " .{member} = {type_name}_{suffix},"); + } + out +} + +/// the sequence slots this class writes a method emptying, each named once +fn emptied_sequence_slots(class: &ClassIr) -> Vec<&'static str> { + let mut members: Vec<&'static str> = Vec::new(); + for (name, member) in EMPTIED_SEQUENCE_SLOTS { + if answers_slot(class, name) && !members.contains(member) { + members.push(member); + } + } + members +} + /// the initializers for the number sub-table a static type points at fn number_fields(class: &ClassIr, type_name: &str) -> String { let mut out = String::new(); @@ -4945,6 +5170,9 @@ fn dunder_slots(class: &ClassIr, type_name: &str) -> Vec<(String, String)> { ) }) .collect(); + for (slot, _, suffix) in sequence_halves(class) { + slots.push((slot.to_string(), format!("{type_name}_{suffix}"))); + } if COMPARISONS .iter() .any(|(name, _)| answers_slot(class, name)) @@ -4999,6 +5227,11 @@ fn dunder_initializers(class: &ClassIr, type_name: &str) -> String { for (field, value) in dataclass_slots(class, type_name) { let _ = writeln!(out, " .{field} = {value},"); } + // `__contains__` names the sequence table already; the halves of the mapping dunders + // still need it pointed at without one + if !answers_slot(class, "__contains__") && sequence_halves(class).next().is_some() { + let _ = writeln!(out, " .tp_as_sequence = &{type_name}_sequence,"); + } // `__bool__` names the number table already; an arithmetic method without one // still needs it pointed at if !answers_slot(class, "__bool__") && !number_fields(class, type_name).is_empty() { @@ -9738,7 +9971,8 @@ fn emit_install_verification(module: &ModuleIr) -> String { \x20* alive at the one moment this can be asked, and `by_type` is what each class's\n\ \x20* name ends up meaning */\n\ static int by_verify_install(PyObject *dict, PyObject *const *by_twin,\n\ - \x20 PyObject *const *by_type) {\n", + \x20 PyObject *const *by_type, PyObject *const *by_body,\n\ + \x20 PyObject *const *by_statement) {\n", ); for (slot, class) in published_classes(module).iter().enumerate() { let type_name = class.type_name(module.name.dotted()); @@ -9784,7 +10018,7 @@ fn emit_install_verification(module: &ModuleIr) -> String { \x20 if (By_VerifyClass({}, dict, {}, by_type[{slot}], {type_name}_OBJ,\n\ \x20 by_twin[{slot}], {base_argument}, {},\n\ \x20 {type_name}_methods,\n\ - \x20 {settled_argument}, {}, {}) < 0) return -1;\n\ + \x20 {settled_argument}, {}, {}, by_body[{slot}], by_statement[{slot}]) < 0) return -1;\n\ \x20 }}\n", c_string(module.name.dotted()), c_string(&class.name), @@ -9964,12 +10198,9 @@ fn emit_module_init(module: &ModuleIr) -> String { // body too — the body is where the decorator's single application landed — and so is // a `@property`, which is one object the body folded its halves into. all three are // named by the one list, so this asks that rather than repeating it - let captures_bodies = module - .classes - .iter() - .any(|class| class.exported && !carried_off_the_body(class).0.is_empty()); + let captures_bodies = module.classes.iter().any(|class| class.exported); let release_bodies = if captures_bodies { - " Py_XDECREF(by_bodies);\n" + " Py_XDECREF(by_bodies);\n Py_XDECREF(by_statements);\n" } else { "" }; @@ -10126,17 +10357,20 @@ fn emit_module_init(module: &ModuleIr) -> String { // now, and `By_RunModuleBody` says what that costs. borrowed from `by_bodies`, // which is held for the whole of this function // a decorated method and a `@property` take their values from here too, so the - // bodies are asked for through the one list that names all three - if twins - .iter() - .any(|class| !carried_off_the_body(class).0.is_empty()) - { - let _ = writeln!(twin_init, " PyObject *by_body[{classes}];"); + // bodies are asked for through the one list that names all three. every class has + // one, because a body is also what tells a member the module rewrote after the + // statement from the one the statement wrote — see `By_CarryRewrittenMembers` + if !twins.is_empty() { + let _ = writeln!( + twin_init, + " PyObject *by_body[{classes}];\n PyObject *by_statement[{classes}];" + ); for (slot, class) in twins.iter().enumerate() { let _ = writeln!( twin_init, - " by_body[{slot}] = By_ClassBody(by_bodies, {});", - c_string(&class.name) + " by_body[{slot}] = By_ClassBody(by_bodies, {name});\n\ + \x20 by_statement[{slot}] = By_ClassBody(by_statements, {name});", + name = c_string(&class.name) ); } } @@ -10175,6 +10409,14 @@ fn emit_module_init(module: &ModuleIr) -> String { adopt_init, " if (By_AdoptTwinAttributes(&by_twins) < 0) return -1;" ); + for (slot, class) in twins.iter().enumerate() { + if class.decorators.is_empty() { + let _ = writeln!( + adopt_init, + " if (By_CarryRewrittenMembers(by_body[{slot}], by_statement[{slot}], &by_twins, {slot}) < 0) return -1;" + ); + } + } // the dataclass bookkeeping the adoption above leaves behind because it is spelled // as a dunder. it has no slot, so nothing about it can disagree with one — and it // is a `dataclasses.Field` dict rather than code, which is why it is carried @@ -10246,7 +10488,7 @@ fn emit_module_init(module: &ModuleIr) -> String { // is *inside* the block because the interpreted definitions are released on the // next line, and comparing against one is half of what it does let verify = if verifies_install(module) { - "\x20 if (by_remapped >= 0) by_remapped = by_verify_install(dict, by_twin, by_type);\n" + "\x20 if (by_remapped >= 0) by_remapped = by_verify_install(dict, by_twin, by_type, by_body, by_statement);\n" } else { "" }; @@ -10415,6 +10657,21 @@ fn emit_module_init(module: &ModuleIr) -> String { \x20 if (By_UnpublishSlotNames({type_name}_OBJ, by_unpublished) < 0) return -1; }}" ); } + // a spec cannot say a slot is empty, so the ones python empties are taken off the + // finished type before anything inherits them + let emptied = emptied_sequence_slots(class); + if !emptied.is_empty() && heap_type(module, class) { + let mut clears = String::new(); + for member in emptied { + let _ = write!(clears, " by_sequence->{member} = NULL;"); + } + let _ = writeln!( + installed, + " {{ PySequenceMethods *by_sequence = ((PyTypeObject *){type_name}_OBJ)->tp_as_sequence;\n\ + \x20 if (by_sequence != NULL) {{{clears} }}\n\ + \x20 PyType_Modified((PyTypeObject *){type_name}_OBJ); }}" + ); + } // the slot alone leaves `C.__hash__` to whatever `PyType_Ready` inferred from it if says_unhashable(class) { let _ = writeln!( @@ -10666,8 +10923,9 @@ fn emit_module_init(module: &ModuleIr) -> String { // statement wrote *before* its own decorators, which only the capturing run keeps let run_body = if captures_bodies { "\x20 PyObject *by_bodies = NULL;\n\ + \x20 PyObject *by_statements = NULL;\n\ \x20 if (by_fallback_source[0] != '\\0') {\n\ - \x20 by_bodies = By_RunModuleBody(&by_fallback, dict);\n\ + \x20 by_bodies = By_RunModuleBody(&by_fallback, dict, &by_statements);\n\ \x20 if (by_bodies == NULL) return -1;\n\ \x20 }\n" .to_string() @@ -12204,7 +12462,9 @@ mod tests { /// census would read as a build with no check in it — and then init returns having /// installed nothing fn refuses_whole_module(type_name: &str) -> String { - format!("if ({type_name} == NULL) {{\n by_record_interpreted();\n return 0;\n }}") + format!( + "if ({type_name} == NULL) {{\n by_record_interpreted();\n Py_XDECREF(by_bodies);\n Py_XDECREF(by_statements);\n return 0;\n }}" + ) } fn appending_class() -> ClassIr { diff --git a/crates/by_ir/src/builder.rs b/crates/by_ir/src/builder.rs index a09a535fc4..6cb0c4962f 100644 --- a/crates/by_ir/src/builder.rs +++ b/crates/by_ir/src/builder.rs @@ -46,6 +46,7 @@ pub struct FunctionBuilder { block_ranges: Vec>, doc: Option, takes_a_weak_reference: bool, + weak_referents: Vec, /// where the ops pushed from here on were written: the byte offset, and the line the /// frontend worked out for it location: Option<(u32, u32)>, @@ -89,6 +90,7 @@ impl FunctionBuilder { block_ranges: vec![None], doc: None, takes_a_weak_reference: false, + weak_referents: Vec::new(), location: None, generator_expression: false, block_positions: vec![None], @@ -135,6 +137,15 @@ impl FunctionBuilder { self } + /// record that the body takes a weak reference of an instance of `class` — see + /// [`Function::weak_referents`] + pub fn weakly_references(&mut self, class: &str) -> &mut Self { + if !self.weak_referents.iter().any(|held| held == class) { + self.weak_referents.push(class.to_string()); + } + self + } + pub fn convention(&mut self, convention: CallConvention) -> &mut Self { self.convention = convention; self @@ -433,6 +444,7 @@ impl FunctionBuilder { coroutine_body: None, doc: self.doc, takes_a_weak_reference: self.takes_a_weak_reference, + weak_referents: self.weak_referents, nested: None, } } diff --git a/crates/by_ir/src/function.rs b/crates/by_ir/src/function.rs index 524f33ed3c..2d0e0958b6 100644 --- a/crates/by_ir/src/function.rs +++ b/crates/by_ir/src/function.rs @@ -365,17 +365,22 @@ pub struct Function { pub doc: Option, /// whether the body takes a weak reference of a value it cannot see the origin of /// - /// an emitted instance *is* its layout and a type spec adds no `__weakref__`, so a - /// weak reference of one raises `TypeError` where python hands back a reference. the - /// frontend refuses that outright wherever the argument is one this module lays out, - /// and sets this wherever it is not — `logging`'s `_addHandlerRef(handler)` is handed - /// its target by whoever called it, and the `weakref.ref` is a whole function away - /// from the `Handler.__init__` that would raise. + /// an instance of a class [`ModuleIr::keeps_weak_references`] turns away cannot be + /// weakly referenced, where python hands back a reference. `logging`'s + /// `_addHandlerRef(handler)` is handed its target by whoever called it, and the + /// `weakref.ref` is a whole function away from the `Handler.__init__` that would raise. /// /// so what one frame cannot settle is settled across the module's call graph - /// instead: a frame that hands one of these an instance of a class this module lays - /// out is the frame that declines, and declining it takes its class with it + /// instead: a frame that hands one of these an instance of such a class is the frame + /// that declines, and declining it takes its class with it pub takes_a_weak_reference: bool, + /// the classes this module lays out that the body takes a weak reference of an + /// instance of, where it can see the instance's class + /// + /// whether that raises is a question about the finished module — see + /// [`ModuleIr::keeps_weak_references`] — so the frontend records the class rather than + /// answering + pub weak_referents: Vec, /// what python calls a *nested* function, which only a nested function has /// /// its receiver is the environment its captures live in, but python sees neither the @@ -1259,6 +1264,65 @@ impl ModuleIr { classes.chain(functions) } + /// whether an instance of `class` can be the target of a weak reference + /// + /// python gives a class a `__weakref__` unless every class in its layout declared + /// `__slots__` without asking for one, and an emitted class keeps a weak-reference list + /// on the same terms — so `weakref.ref`, `WeakSet`, `WeakKeyDictionary`, `finalize` and + /// every other weak container reach one of its instances as they would the interpreted + /// one's. a `data class`'s twin is `@dataclass(slots=True)`, which asks for none. + /// + /// the layout has to be this module's all the way down to `object`: a class standing on + /// a base from outside takes that base's instance whole, and has no word of its own to + /// keep the list in + pub fn keeps_weak_references(&self, class: &ClassIr) -> bool { + if !class.exported || class.environment || class.resume.is_some() { + return false; + } + let mut current = class; + let mut unslotted = false; + // bounded by the class count: a base chain cannot visit one twice without being a + // cycle, and a cycle here would otherwise hang rather than answer + for _ in 0..=self.classes.len() { + if current.dataclass { + return false; + } + unslotted |= !current.declares_slots; + match ¤t.base { + None => return unslotted, + Some(ClassBase::External(_)) => return false, + Some(ClassBase::InModule(name)) => { + match self.classes.iter().find(|other| other.name == *name) { + Some(next) => current = next, + None => return false, + } + } + } + } + false + } + + /// whether `class` takes its instance layout from a base outside the module, which is + /// where [`Self::keeps_weak_references`] turns away a class python would give a + /// weak-reference list to + pub fn takes_layout_from_outside(&self, class: &ClassIr) -> bool { + let mut current = class; + // bounded by the class count, for the reason `keeps_weak_references` gives + for _ in 0..=self.classes.len() { + match ¤t.base { + None => return false, + Some(ClassBase::External(_)) => return true, + Some(ClassBase::InModule(name)) => { + match self.classes.iter().find(|other| other.name == *name) { + Some(next) => current = next, + None => return true, + } + } + } + } + true + } + /// every compiled function, methods included /// /// a pass that iterates `functions` alone silently skips every method, which @@ -1464,6 +1528,7 @@ mod tests { coroutine_body: None, doc: None, takes_a_weak_reference: false, + weak_referents: Vec::new(), nested: None, } } diff --git a/crates/by_ir/src/print.rs b/crates/by_ir/src/print.rs index 7fbdef85e9..c3204ab072 100644 --- a/crates/by_ir/src/print.rs +++ b/crates/by_ir/src/print.rs @@ -992,6 +992,7 @@ mod tests { coroutine_body: None, doc: None, takes_a_weak_reference: false, + weak_referents: Vec::new(), nested: None, } } @@ -1068,6 +1069,7 @@ b2: coroutine_body: None, doc: None, takes_a_weak_reference: false, + weak_referents: Vec::new(), nested: None, }; assert!(print_function(&function).contains("return 1.0")); diff --git a/crates/by_ir/src/verify.rs b/crates/by_ir/src/verify.rs index 23e371f27d..461c6353cc 100644 --- a/crates/by_ir/src/verify.rs +++ b/crates/by_ir/src/verify.rs @@ -2134,6 +2134,7 @@ mod tests { coroutine_body: None, doc: None, takes_a_weak_reference: false, + weak_referents: Vec::new(), nested: None, } } @@ -2326,6 +2327,7 @@ mod tests { coroutine_body: None, doc: None, takes_a_weak_reference: false, + weak_referents: Vec::new(), nested: None, }; let errors = verify(&f).unwrap_err(); @@ -2379,6 +2381,7 @@ mod tests { coroutine_body: None, doc: None, takes_a_weak_reference: false, + weak_referents: Vec::new(), nested: None, }; assert_eq!(verify(&f), Ok(())); @@ -2433,6 +2436,7 @@ mod tests { coroutine_body: None, doc: None, takes_a_weak_reference: false, + weak_referents: Vec::new(), nested: None, }; let errors = verify(&f).unwrap_err(); @@ -2538,6 +2542,7 @@ mod tests { coroutine_body: None, doc: None, takes_a_weak_reference: false, + weak_referents: Vec::new(), nested: None, }; let errors = verify(&f).unwrap_err(); diff --git a/crates/by_irbuild/src/lib.rs b/crates/by_irbuild/src/lib.rs index b9a258bf2d..0bad488743 100644 --- a/crates/by_irbuild/src/lib.rs +++ b/crates/by_irbuild/src/lib.rs @@ -560,6 +560,17 @@ pub fn build_module( .collect(); let properties = published_properties(db, model, suite, &layouts); + let method_names: HashMap> = suite + .iter() + .filter_map(|statement| match statement { + Stmt::ClassDef(class) + if layouts.contains_key(class.name.as_str()) && declared_slots(class).is_none() => + { + Some((class.name.to_string(), class_method_names(class))) + } + _ => None, + }) + .collect(); // read off the source rather than off the lowered methods: a `__new__` python never // reaches the method table for — one carrying a decorator, one whose class declined — @@ -588,6 +599,7 @@ pub fn build_module( constructs: &constructs, bases: &bases, properties: &properties, + method_names: &method_names, accessors: &accessors, language, recheck_licences, @@ -1718,6 +1730,7 @@ fn lower_resume( constructs: unit.constructs, bases: unit.bases, properties: unit.properties, + method_names: unit.method_names, accessors: unit.accessors, db, model, @@ -1775,6 +1788,7 @@ fn lower_resume( delegations: 0, contexts: 0, cleanups: Vec::new(), + drained_prelude: None, }; lowering.block(&function.body)?; @@ -2067,6 +2081,17 @@ fn prune_unbuildable( }; let (weak_referencing, weak_referencing_methods) = weak_referencing_closure(module, &named_receivers); + // a class that keeps no list where python's keeps none either — `__slots__` + // declared throughout, a `data class` — raises the same `TypeError` on both sides, + // so only a layout taken from outside the module is a difference to decline for + let weakless: HashSet = module + .classes + .iter() + .filter(|class| { + !module.keeps_weak_references(class) && module.takes_layout_from_outside(class) + }) + .map(|class| class.name.clone()) + .collect(); let anchors = storage_anchors(module); let unbuildable = |function: &Function| -> Option { @@ -2074,7 +2099,7 @@ fn prune_unbuildable( // something in this module takes a weak reference at all — which most do not let takes_one = !weak_referencing.is_empty(); let ours = if takes_one { - instance_registers(function) + instance_registers(function, &weakless) } else { HashSet::new() }; @@ -2115,6 +2140,15 @@ fn prune_unbuildable( return Some(format!("`{class}` declined, so it has no layout")); } } + if let Some(class) = function + .weak_referents + .iter() + .find(|class| weakless.contains(class.as_str())) + { + return Some(format!( + "a weak reference is taken of an instance of `{class}`, {WEAKLESS}" + )); + } for op in function.blocks.iter().flat_map(|block| &block.ops) { match op { Op::CallNative { @@ -2314,15 +2348,16 @@ fn prune_unbuildable( /// by qualified name and — for the ones emitted as methods — by the bare name a call /// through the object protocol writes /// -/// an emitted instance cannot be the target of a weak reference, and the frontend refuses -/// `weakref.ref(self)` where it is written. `logging.Handler.__init__` writes none: it +/// an instance of a class [`ModuleIr::keeps_weak_references`] turns away cannot be the +/// target of a weak reference, and a frame writing `weakref.ref(self)` over one records the +/// class it can see. `logging.Handler.__init__` writes none: it /// calls `_addHandlerRef(self)`, whose body takes the reference, so nothing about /// `__init__`'s own body says it raises. the frontend marks the frame that writes the /// call, and this closes those marks over the module's calls — a frame that hands a value /// on to a marked one is one more function standing between an instance and the /// reference, so it is marked too. `prune_unbuildable` then declines whoever hands one of -/// these an instance of ours, which is what keeps that instance's class interpreted, and -/// an interpreted class is one a weak reference *can* be made of. +/// these an instance of such a class, which is what keeps that instance's class +/// interpreted, and an interpreted class is one a weak reference *can* be made of. /// /// what is out of reach stays out of reach: a caller in another module, and one that /// reaches a frame through a value rather than by a name. a call whose arguments are all @@ -2497,10 +2532,13 @@ impl NamedReceivers<'_> { } } +/// what an instance of a class [`ModuleIr::keeps_weak_references`] turns away is missing +const WEAKLESS: &str = "whose emitted type keeps no weak-reference list: its layout is taken from a base outside the module"; + /// why a frame that hands an instance of ours to `target` cannot be built fn weakly_referenced(target: &str) -> String { format!( - "`{target}` takes a weak reference of what it is handed, and an emitted instance is its layout — a type spec adds no `__weakref__`, so no weak reference of one can be made" + "`{target}` takes a weak reference of what it is handed, and it is handed an instance {WEAKLESS}" ) } @@ -2513,12 +2551,18 @@ fn weakly_referenced(target: &str) -> String { /// instance, and that widening stands between `Handler.__init__`'s receiver and the /// `_addHandlerRef(self)` that would raise over it. so the two ops that only move a /// value along carry the answer with them -fn instance_registers(function: &Function) -> HashSet { +fn instance_registers(function: &Function, weakless: &HashSet) -> HashSet { let mut ours: HashSet = function .registers .iter() .enumerate() - .filter(|(_, register)| !register.ty.instance_classes().is_empty()) + .filter(|(_, register)| { + register + .ty + .instance_classes() + .into_iter() + .any(|class| weakless.contains(class)) + }) .map(|(index, _)| RegisterId(index)) .collect(); // a move may stand anywhere among the blocks, including before the one that wrote @@ -3539,6 +3583,10 @@ fn attribute_outgrows( /// every write reaches here, whatever statement made it, because the layout is the only /// place an attribute can go: one it never heard about is lowered as the dynamic form and /// lands nowhere at all +#[expect( + clippy::too_many_arguments, + reason = "one call site, and each set is a separate reason a write is not a field" +)] fn init_fields( db: &dyn ty_python_semantic::Db, env: &ProgramEnvironment<'_>, @@ -3546,6 +3594,7 @@ fn init_fields( class: &ast::StmtClassDef, layouts: &Layouts, defaults: &HashSet, + shadowing: &HashSet, mut fields: Vec, ) -> Lowered> { let methods = instance_methods(class); @@ -3572,7 +3621,7 @@ fn init_fields( for (method, receiver) in &methods { for statement in walk(&method.body) { for (name, target) in receiver_writes(statement, receiver, Some(&class.name))? { - if properties.contains(&name) { + if properties.contains(&name) || shadowing.contains(&name) { continue; } let ty = target @@ -3634,7 +3683,10 @@ fn init_fields( let definite = definitely_assigned_attributes(&init.body, receiver, Some(&class.name)); for statement in &init.body { for (name, _) in certain_writes(statement, receiver, Some(&class.name))? { - if properties.contains(&name) || fields.iter().any(|field| field.name == name) { + if properties.contains(&name) + || shadowing.contains(&name) + || fields.iter().any(|field| field.name == name) + { continue; } let ty = widths @@ -5742,9 +5794,18 @@ fn class_fields( // a plain class *is* its `__init__`: the fields are the attributes it gives // the instance, in the order it gives them, and a `__slots__` declares the // ones no assignment reached + // a class declaring `__slots__` has no dict to keep a shadowing attribute in, and + // python refuses a slot named for a method outright + let shadowing = if declared_slots(class).is_none() { + methods_through_bases(db, env, model, suite, class, layouts)? + } else { + HashSet::new() + }; slot_fields( class, - init_fields(db, env, model, class, layouts, &defaults, inherited)?, + init_fields( + db, env, model, class, layouts, &defaults, &shadowing, inherited, + )?, )? }; // a name the body binds *and* `__init__` assigns is a field whose class-level value @@ -5923,6 +5984,61 @@ fn no_field_shadows_a_method( Ok(()) } +/// the names this class body binds with a `def` that is not part of a `property` +/// +/// each is a non-data descriptor on the class, so an instance's own attribute of the same +/// name answers before it — see [`methods_through_bases`] +fn class_method_names(class: &ast::StmtClassDef) -> HashSet { + class + .body + .iter() + .filter_map(|statement| match statement { + Stmt::FunctionDef(method) + if !method + .decorator_list + .iter() + .any(|decorator| is_property_decorator(&decorator.expression)) => + { + Some(mangled(Some(&class.name), method.name.as_str())) + } + _ => None, + }) + .collect() +} + +/// the method names of `class` and of every base of it this module lays out +/// +/// an instance attribute under one of these names is kept in the instance's dict rather +/// than in a field. python answers an instance's own attribute before a function its class +/// holds, and the dict is where both the lookup and a compiled call's shadow probe look for +/// it first — a field's descriptor would stand in the class's dict under the method's name +fn methods_through_bases( + db: &dyn ty_python_semantic::Db, + env: &ProgramEnvironment<'_>, + model: &SemanticModel<'_>, + suite: &[Stmt], + class: &ast::StmtClassDef, + layouts: &Layouts, +) -> Lowered> { + let mut names = HashSet::new(); + let mut current = Some(class); + // a chain of classes is finite, and a cycle among them is not a program python runs + let mut visited: HashSet<&str> = HashSet::new(); + while let Some(owner) = current + && visited.insert(owner.name.as_str()) + { + names.extend(class_method_names(owner)); + current = match base_class(db, env, model, suite, owner, layouts)? { + Some(ClassBase::InModule(name)) => suite.iter().find_map(|statement| match statement { + Stmt::ClassDef(candidate) if candidate.name.as_str() == name => Some(candidate), + _ => None, + }), + _ => None, + }; + } + Ok(names) +} + /// whether a decorator makes a `property`, or one of the halves added to one fn is_property_decorator(expression: &Expr) -> bool { match expression { @@ -7678,6 +7794,7 @@ fn lower_function_with_receiver( constructs: unit.constructs, bases: unit.bases, properties: unit.properties, + method_names: unit.method_names, accessors: unit.accessors, db, model, @@ -7705,6 +7822,7 @@ fn lower_function_with_receiver( delegations: 0, contexts: 0, cleanups: Vec::new(), + drained_prelude: None, captures: own_captures, owned_cells, environment: environment.as_ref().map(|environment| Closures { @@ -7927,6 +8045,9 @@ struct Unit<'a> { /// the attributes each emitted class publishes as a `property` — see /// [`published_properties`] properties: &'a HashMap>, + /// the method names each emitted class with an instance dict binds, which an attribute + /// written on an instance is kept in that dict for — see [`methods_through_bases`] + method_names: &'a HashMap>, /// the class whose body the frame being lowered is written in, which is what /// decides how python mangles a private name — see [`mangled`]. a nested frame /// inherits it, because the mangling follows the source and not the receiver @@ -8105,6 +8226,8 @@ enum Cleanup { previous: RegisterId, exception: RegisterId, }, + /// a generator's parked loop iterator, let go of on the way out of the loop + Iterator(Place), } /// an assignment target with its location parts already evaluated @@ -9688,6 +9811,9 @@ struct Lowering<'a, 'db> { /// the attributes each emitted class publishes as a `property` — see /// [`published_properties`] properties: &'a HashMap>, + /// the method names an instance attribute is kept in the dict for — see + /// [`Unit::method_names`] + method_names: &'a HashMap>, /// the `@property` halves each emitted class lowers a body for — see [`Accessors`] accessors: &'a Accessors, /// the closure environment this frame allocates, when it makes closures @@ -9706,6 +9832,10 @@ struct Lowering<'a, 'db> { /// before it leaves — python does, and skipping it is a silent wrong answer rather /// than a missing optimization cleanups: Vec, + /// the error target and generator-expression flag the *caller* of a drained generator + /// expression runs under, handed to the first clause of its comprehension — see + /// [`Lowering::drained_generator_expression`] + drained_prelude: Option<(Option, bool)>, /// the captures this frame reads through its receiver, when it *is* a closure captures: Option, /// the shared cells this frame *owns*, in the environment it allocated itself @@ -10067,6 +10197,12 @@ impl Lowering<'_, '_> { .properties .get(candidate) .is_some_and(|published| published.contains(name)) + // a name a method is bound under is kept in the instance dict, where the + // write lands the way python's does + || self + .method_names + .get(candidate) + .is_some_and(|bound| bound.contains(name)) }; let mut current = class; // bounded by the class count, the way every base walk here is: a chain that @@ -11408,6 +11544,7 @@ impl Lowering<'_, '_> { Cleanup::Handled { previous, .. } => self.builder.push(Op::PopHandled { value: Value::Register(previous), }), + Cleanup::Iterator(parked) => self.clear_parked(&parked), Cleanup::Context { manager, is_async, @@ -12586,6 +12723,7 @@ impl Lowering<'_, '_> { let natural_exit = self.builder.new_block(); let after = self.builder.new_block(); self.builder.terminate(Terminator::Goto(header)); + let released = self.release_iterator_on_exit(&parked); self.builder.switch_to(header); let (live, _) = self.read_place(&parked)?; @@ -12626,6 +12764,7 @@ impl Lowering<'_, '_> { self.loops.pop(); result?; self.builder.terminate(Terminator::Goto(header)); + self.end_iterator_release(released, &parked); // the end of the iteration is an exception, and only `StopAsyncIteration` // is one: anything else the step raised goes on out @@ -12651,16 +12790,19 @@ impl Lowering<'_, '_> { }); self.builder.switch_to(propagate); + self.clear_parked(&parked); self.builder.push(Op::Reraise { value: Value::Register(exception), }); self.builder.terminate(Terminator::Unreachable); self.builder.switch_to(natural_exit); + self.clear_parked(&parked); let orelse = self.block(&node.orelse); orelse?; self.builder.terminate(Terminator::Goto(after)); self.builder.switch_to(after); + self.clear_parked(&parked); Ok(()) } @@ -12734,6 +12876,7 @@ impl Lowering<'_, '_> { let natural_exit = self.builder.new_block(); let after = self.builder.new_block(); self.builder.terminate(Terminator::Goto(header)); + let released = self.release_iterator_on_exit(&parked); self.builder.switch_to(header); // read the iterator back from wherever it was parked, every trip @@ -12780,12 +12923,15 @@ impl Lowering<'_, '_> { self.loops.pop(); result?; self.builder.terminate(Terminator::Goto(header)); + self.end_iterator_release(released, &parked); self.builder.switch_to(natural_exit); + self.clear_parked(&parked); self.block(&node.orelse)?; self.builder.terminate(Terminator::Goto(after)); self.builder.switch_to(after); + self.clear_parked(&parked); Ok(()) } @@ -13978,6 +14124,7 @@ impl Lowering<'_, '_> { let parked = self.park_iterator(iterator)?; let header = self.builder.new_block(); + let resumed = self.builder.new_block(); let forward = self.builder.new_block(); let returned = self.builder.new_block(); let finished = self.builder.new_block(); @@ -13985,25 +14132,32 @@ impl Lowering<'_, '_> { let result = self .builder .local(format!("$delegated{}", self.delegations), RType::OBJECT); + let sending = self + .builder + .local(format!("$delegatesend{}", self.delegations), RType::OBJECT); self.delegations += 1; + // the first trip sends `None`, as python's does: what `$sent` holds before the + // delegation has suspended answered an earlier suspension of this frame, and an + // iterator nothing has started yet refuses anything else + let nothing = self.widen_to_object(Value::None, &RType::NONE); + self.builder.assign(sending, nothing); self.builder.terminate(Terminator::Goto(header)); // python drops the inner iterator the moment the delegation ends, whichever way it // ends, so one abandoned by an exception is finalized before any handler of this // frame runs rather than whenever the frame itself goes let enclosing = self.builder.set_error_target(Some(unwind)); - // each trip: read the inner iterator back, send in whatever `send` gave us, - // and either forward a value or take the result + // each trip: read the inner iterator back, send in what is to be sent, and either + // forward a value or take the result self.builder.switch_to(header); let (inner, _) = self.read_place(&parked)?; - let sent = self.read_sent()?; let outcome = self .builder .temp(RType::Tuple(Box::from([RType::OBJECT, RType::BIT]))); self.builder.push(Op::DelegateStep { dest: outcome, inner, - sent, + sent: Value::Register(sending), }); let step = self.builder.temp(RType::OBJECT); self.builder.push(Op::TupleGet { @@ -14033,6 +14187,12 @@ impl Lowering<'_, '_> { iterator: &parked, }), )?; + self.builder.terminate(Terminator::Goto(resumed)); + + // after every later suspension, whatever `send` handed the frame goes on in + self.builder.switch_to(resumed); + let sent = self.read_sent()?; + self.builder.assign(sending, sent); self.builder.terminate(Terminator::Goto(header)); self.builder.switch_to(returned); @@ -14054,6 +14214,50 @@ impl Lowering<'_, '_> { Ok((Value::Register(result), RType::OBJECT)) } + /// let go of a generator's parked loop iterator on every way out of the loop body + /// + /// python holds a loop's iterator on the frame's stack and drops it the moment the loop + /// is left, so an inner generator's `finally` runs before whatever follows a `break`, a + /// `return` or an exception's handler. an exception leaving the loop goes through a + /// block that lets go and raises on, and `return` meets the cleanup this pushes. the + /// two exits into the code after the loop let go where they land. a register-held + /// iterator needs none of it: it is released at its last use + /// + /// hands back the error target to restore and the unwinding block, which + /// [`Self::end_iterator_release`] takes once the body is lowered + fn release_iterator_on_exit(&mut self, parked: &Place) -> Option<(Option, BlockId)> { + if !matches!(parked, Place::Field { .. }) { + return None; + } + let unwind = self.builder.new_block(); + let enclosing = self.builder.set_error_target(Some(unwind)); + self.cleanups.push(Cleanup::Iterator(parked.clone())); + Some((enclosing, unwind)) + } + + /// close what [`Self::release_iterator_on_exit`] opened + fn end_iterator_release( + &mut self, + released: Option<(Option, BlockId)>, + parked: &Place, + ) { + let Some((enclosing, unwind)) = released else { + return; + }; + self.cleanups.pop(); + self.builder.set_error_target(enclosing); + let current = self.builder.current_block(); + self.builder.switch_to(unwind); + let pending = self.builder.temp(RType::OBJECT); + self.builder.push(Op::FetchException { dest: pending }); + self.clear_parked(parked); + self.builder.push(Op::Reraise { + value: Value::Register(pending), + }); + self.builder.terminate(Terminator::Unreachable); + self.builder.switch_to(current); + } + /// let go of what a parked field holds, by writing `None` over it fn clear_parked(&mut self, parked: &Place) { if let Place::Field { @@ -16354,10 +16558,23 @@ impl Lowering<'_, '_> { accumulator: RegisterId, accumulator_ty: &RType, ) -> Lowered<()> { + let prelude = self.drained_prelude.take(); let Some((generator, rest)) = generators.split_first() else { return self.comprehension_emit(kind, accumulator, accumulator_ty); }; + // the caller's context for everything up to the first trip round the loop, and + // the generator expression's own from there — see `drained_prelude` + let restore = prelude.map(|(target, inside)| { + ( + self.builder.set_error_target(target), + self.builder.in_generator_expression(inside), + ) + }); if generator.is_async { + if let Some((target, inside)) = restore { + self.builder.set_error_target(target); + self.builder.in_generator_expression(inside); + } return self.comprehension_loop_async( generator, rest, @@ -16382,6 +16599,7 @@ impl Lowering<'_, '_> { kind, accumulator, accumulator_ty, + restore, ); } @@ -16412,6 +16630,10 @@ impl Lowering<'_, '_> { let body = self.builder.new_block(); let exit = self.builder.new_block(); self.builder.terminate(Terminator::Goto(header)); + if let Some((target, inside)) = restore { + self.builder.set_error_target(target); + self.builder.in_generator_expression(inside); + } self.builder.switch_to(header); self.builder.push(Op::IterNext { @@ -16518,6 +16740,7 @@ impl Lowering<'_, '_> { kind: &Comprehension<'_>, accumulator: RegisterId, accumulator_ty: &RType, + restore: Option<(Option, bool)>, ) -> Lowered<()> { let CountedRange { callee, @@ -16582,6 +16805,10 @@ impl Lowering<'_, '_> { cursor: None, }); self.builder.terminate(Terminator::Goto(next)); + if let Some((target, inside)) = restore { + self.builder.set_error_target(target); + self.builder.in_generator_expression(inside); + } self.builder.switch_to(header); let more = self.builder.temp(RType::BIT); @@ -17409,31 +17636,25 @@ impl Lowering<'_, '_> { self.a_warning(node).map(Some) } - /// turn down a weak reference taken of an instance this module lays out - /// - /// an emitted instance *is* its layout, and a type spec adds no `__weakref__` — the - /// same fact that turns down a `__slots__` asking for one, and a body writing the - /// attribute. so `weakref.ref(self)` raises `TypeError` where python hands back a - /// reference, which is a wrong answer rather than a slower one. `_weakrefset.WeakSet` - /// is the shape: its `__init__` writes + /// record a weak reference taken of what may be an instance this module lays out /// - /// ```python - /// def _remove(item, selfref=ref(self)): - /// ... - /// ``` - /// - /// which snapshots a weak reference to the receiver where the `def` stands. + /// most emitted classes keep a weak-reference list, as python's do, and a weak + /// reference of one of their instances is simply made. the ones that cannot where python + /// can — a layout standing on a base from outside the module, see + /// [`ModuleIr::keeps_weak_references`] — raise `TypeError` where python hands back a + /// reference, and which classes those are is only settled once the module is built. so + /// this answers nothing: it records what [`prune_unbuildable`] asks. /// /// an argument this frame can already see as an emitted instance — an /// `RType::Instance` is exactly that, because a class from outside the module maps - /// to a plain object — is refused here and now. + /// to a plain object — is recorded by its class. /// /// where it cannot see one, the value still reached this frame from somewhere, and /// the frame that sent it is where the answer is. `logging` is the shape: /// `Handler.__init__` calls `_addHandlerRef(self)`, whose body writes the /// `weakref.ref` a whole function away from the receiver it would raise over. so the /// frame is marked instead, and [`prune_unbuildable`] carries the refusal back to - /// every caller in this module that hands it one of ours. + /// every caller in this module that hands it an instance of such a class. /// /// a caller in *another* module is out of that reach, and so is one that reaches /// this frame through a value rather than by its name @@ -17442,16 +17663,16 @@ impl Lowering<'_, '_> { /// it, there is nothing for a caller to be blamed for and the frame is left alone. /// `multiprocessing.queues` is the shape: `Queue._start_thread` writes /// `weakref.ref(self._thread)`, and a `threading.Thread` is not a `Queue` - fn a_weak_reference(&mut self, node: &ast::ExprCall) -> Lowered<()> { + fn a_weak_reference(&mut self, node: &ast::ExprCall) { let written = match node.func.as_ref() { Expr::Attribute(attribute) => attribute.attr.as_str(), Expr::Name(name) => name.id.as_str(), - _ => return Ok(()), + _ => return, }; // a syntactic filter first, for the reason [`Self::a_frame_walk`] gives: the // namespace lookup below parses the module it answers out of if !matches!(written, "ref" | "proxy") { - return Ok(()); + return; } // and identity, not spelling: `ref` is a name a great many modules bind to // something of their own @@ -17459,26 +17680,33 @@ impl Lowering<'_, '_> { if node.func.inferred_type(self.model) != ty_python_semantic::basedpython_weakref_symbol(self.db, env, written) { - return Ok(()); + return; } let Some(referent) = node.arguments.args.first() else { - return Ok(()); + return; }; - let emitted = matches!(referent, Expr::Name(name) if matches!( - self.place(name.id.as_str()), - Some(Place::Register(id)) - if matches!(self.register_type(id), Ok(RType::Instance { .. })) - )) || matches!(self.peek_type(referent), Ok(RType::Instance { .. })); - if emitted { - return Err(Decline::new( - "an emitted instance is its layout and a type spec adds no `__weakref__`, so a weak reference to one cannot be made", - )); + let held = match referent { + Expr::Name(name) => match self.place(name.id.as_str()) { + Some(Place::Register(id)) => self.register_type(id).ok(), + _ => None, + }, + _ => None, + }; + let emitted = match held { + Some(RType::Instance { class, .. }) => Some(class), + _ => match self.peek_type(referent) { + Ok(RType::Instance { class, .. }) => Some(class), + _ => None, + }, + }; + if let Some(class) = emitted { + self.builder.weakly_references(&class); + return; } if !self.could_be_one_of_ours(referent) { - return Ok(()); + return; } self.builder.takes_a_weak_reference(); - Ok(()) } /// whether an instance of a class this module lays out could be standing where @@ -17671,7 +17899,7 @@ impl Lowering<'_, '_> { if let Some(handled) = self.a_frame_walk(node)? { return Ok(handled); } - self.a_weak_reference(node)?; + self.a_weak_reference(node); // a `*` or a `**` in the arguments means the binding happens at runtime, so // the arguments become a tuple and a dict and python does the binding if node.arguments.args.iter().any(Expr::is_starred_expr) @@ -17866,7 +18094,13 @@ impl Lowering<'_, '_> { let drained = self.builder.new_block(); let previous = self.builder.set_error_target(Some(leaving)); let around = self.builder.in_generator_expression(true); + // python evaluates the first iterable and takes its iterator where the generator + // expression is written, before the generator's frame exists, so what that raises + // is raised in this frame: no `` entry in its traceback, and a + // `StopIteration` not made into a `RuntimeError` + self.drained_prelude = Some((previous, around)); let built = self.comprehension(&generator.generators, &Comprehension::List(&generator.elt)); + self.drained_prelude = None; if built.is_ok() { self.builder.terminate(Terminator::Goto(drained)); } diff --git a/crates/by_irbuild/src/tests.rs b/crates/by_irbuild/src/tests.rs index d46a050b91..3f5da4ba39 100644 --- a/crates/by_irbuild/src/tests.rs +++ b/crates/by_irbuild/src/tests.rs @@ -1833,14 +1833,16 @@ class Tagged: } #[test] -fn a_method_that_is_also_an_attribute_the_instance_is_given_declines() { +fn an_attribute_sharing_a_method_name_is_kept_in_the_instance_dict() { // python answers an instance's own attribute before a function its class holds under // the same name, so `json.encoder.JSONEncoder` reads the `default` its constructor was - // handed where there was one and the method where there was not. a layout field and a - // method table cannot both stand under one name, and a method call reaches the method - // by the class alone — so either the call or the read would answer the other one - for source in [ - "\ + // handed where there was one and the method where there was not. a layout field would + // publish a descriptor under the method's name, so the attribute is kept in the + // instance's dict instead — which is where the lookup and a compiled call's shadow + // probe look first + for (source, class) in [ + ( + "\ class Encoder: def __init__(self, default: object = None) -> None: if default is not None: @@ -1849,7 +1851,10 @@ class Encoder: def default(self, o: object) -> object: return o ", - "\ + "Encoder", + ), + ( + "\ class Base: def read(self) -> int: return 1 @@ -1859,6 +1864,35 @@ class Reader(Base): def __init__(self) -> None: self.read = len ", + "Reader", + ), + ] { + assert_eq!(declines(source), Vec::new()); + let fields = with_source(source, |db, env, model, suite| { + let module = + crate::build_module(db, env, model, suite, "app", crate::Language::BasedPython); + module + .classes + .iter() + .find(|lowered| lowered.name == class) + .map(|lowered| { + lowered + .fields + .iter() + .map(|field| field.name.clone()) + .collect::>() + }) + }); + assert_eq!(fields, Some(Vec::new()), "{source}"); + } +} + +#[test] +fn a_method_over_an_attribute_a_base_lays_out_declines() { + // the base lays the attribute out as a field before the subclass's method is known, so + // the field's descriptor would stand in the base's dict behind the subclass's method — + // and python's lookup would reach the method first + let reasons = declines( "\ class Base: def __init__(self) -> None: @@ -1869,15 +1903,13 @@ class Reader(Base): def read(self) -> int: return 1 ", - ] { - let reasons = declines(source); - assert!( - reasons - .iter() - .any(|(_, reason)| reason.contains("is both a method and an attribute")), - "{reasons:?}" - ); - } + ); + assert!( + reasons + .iter() + .any(|(_, reason)| reason.contains("is both a method and an attribute")), + "{reasons:?}" + ); } #[test] @@ -3501,22 +3533,19 @@ class Fine: ); } -/// a weak reference taken a whole function away from the instance it is of +/// a weak reference of an instance of a class this module lays out is simply made /// -/// the same fact the `__slots__` case above turns on — a type spec adds no `__weakref__` -/// — and the frontend refuses `weakref.ref(self)` where it is written. but -/// `logging.Handler.__init__` does not write one: it calls `_addHandlerRef(self)`, whose -/// body takes the reference, and no predicate over `__init__`'s own body can see that. -/// so the refusal is carried back to the caller that hands the instance over, and -/// declining that caller is what keeps its class interpreted — which is a class a weak -/// reference *can* be made of +/// an emitted class keeps a weak-reference list on python's own terms, so `weakref.ref`, +/// `WeakSet.add` and every other weak container reach its instances whether the call is +/// written beside the instance or a whole function away from it #[test] -fn a_caller_that_hands_an_instance_to_a_weak_reference_is_declined() { +fn a_weak_reference_of_an_instance_of_ours_is_made() { let reasons = declines( "\ import weakref registry = [] +seen = weakref.WeakSet() def registers(thing): @@ -3524,6 +3553,39 @@ def registers(thing): class Handler: + def __init__(self): + registers(self) + seen.add(self) + + def own(self) -> None: + registry.append(weakref.ref(self)) +", + ); + assert_eq!(reasons, Vec::new()); +} + +/// a weak reference taken a whole function away from an instance that cannot have one +/// +/// a class whose layout is a base's from outside the module keeps no weak-reference list +/// of its own, where python gives its class one. `logging.Handler.__init__` is the shape +/// of the reach: it calls `_addHandlerRef(self)`, whose body takes the reference, and no +/// predicate over `__init__`'s own body can see that. so the refusal is carried back to +/// the caller that hands the instance over, and declining that caller is what keeps its +/// class interpreted — which is a class a weak reference *can* be made of +#[test] +fn a_caller_that_hands_an_instance_to_a_weak_reference_is_declined() { + let reasons = declines( + "\ +import weakref + +registry = [] + + +def registers(thing): + registry.append(weakref.ref(thing)) + + +class Handler(Exception): def __init__(self): registers(self) ", @@ -3532,7 +3594,7 @@ class Handler: reasons, vec![( "Handler".to_string(), - "`registers` takes a weak reference of what it is handed, and an emitted instance is its layout — a type spec adds no `__weakref__`, so no weak reference of one can be made".to_string() + "`registers` takes a weak reference of what it is handed, and it is handed an instance whose emitted type keeps no weak-reference list: its layout is taken from a base outside the module".to_string() )] ); } @@ -3541,8 +3603,7 @@ class Handler: /// /// a frame between the receiver and the weak reference is one more function the answer /// has to travel through, which the pruner's own loop does. handing over something that -/// is not an instance of ours costs nothing at all: an object from anywhere else carries -/// a `__weakref__` of its own +/// is not such an instance costs nothing at all #[test] fn only_a_caller_handing_over_an_instance_of_ours_is_declined() { let reasons = declines( @@ -3560,7 +3621,12 @@ def forwards(thing): registers(thing) -class Handler: +class Handler(Exception): + def __init__(self): + forwards(self) + + +class Plain: def __init__(self): forwards(self) @@ -3602,7 +3668,7 @@ import weakref registry = [] -class Queue: +class Queue(Exception): def __init__(self, thread: threading.Thread) -> None: self.thread = thread @@ -3616,12 +3682,12 @@ class Queue: assert_eq!(reasons, Vec::new()); } -/// and where an instance of ours *can* be standing there, the refusal stays +/// and where an instance that cannot have one *can* be standing there, the refusal stays /// -/// `Holder.node` is declared as a class this module lays out, so the reference is of an -/// emitted instance and raises. that is turned down where it is written rather than -/// carried back to a caller: the frame that reads the field is the frame that raises, -/// and no caller of it is handing the instance over +/// `Holder.node` is declared as a class this module lays out over a base from outside, so +/// the reference is of an instance with no weak-reference list and raises. that is turned +/// down where it is written rather than carried back to a caller: the frame that reads +/// the field is the frame that raises, and no caller of it is handing the instance over #[test] fn a_weak_reference_of_a_field_declared_as_one_of_ours_is_declined() { let reasons = declines( @@ -3631,7 +3697,7 @@ import weakref registry = [] -class Node: +class Node(Exception): def __init__(self) -> None: self.tag = 1 @@ -3648,17 +3714,17 @@ class Holder: reasons, vec![( "Holder".to_string(), - "an emitted instance is its layout and a type spec adds no `__weakref__`, so a weak reference to one cannot be made".to_string() + "a weak reference is taken of an instance of `Node`, whose emitted type keeps no weak-reference list: its layout is taken from a base outside the module".to_string() )] ); } /// a field declared `object` is a place an instance of ours fits, so the frame stays -/// marked and its callers are still turned down +/// marked and its callers are still turned down where they hand one over /// /// this is what says the question asked is assignability and not a match on a class's -/// name: nothing here writes `Node` anywhere near the weak reference, and `object` is a -/// place every one of ours can stand in +/// name: nothing here names a class anywhere near the weak reference, and `object` is a +/// place every one of ours can stand in — the receiver itself among them #[test] fn a_weak_reference_of_a_field_declared_object_still_reaches_its_callers() { let reasons = declines( @@ -3668,12 +3734,7 @@ import weakref registry = [] -class Node: - def __init__(self) -> None: - self.tag = 1 - - -class Holder: +class Holder(Exception): def __init__(self, held: object) -> None: self.held = held diff --git a/crates/by_rt/include/by.h b/crates/by_rt/include/by.h index b703f778c8..4347c1aa4d 100644 --- a/crates/by_rt/include/by.h +++ b/crates/by_rt/include/by.h @@ -2787,33 +2787,38 @@ static inline int By_SpecTakesBases(PyObject *bases) { * own struct, so the room is there and the offset naming that word is the answer that was * wanted. without this a decorated class silently kept its interpreted definition while * every compiled function went on reading that definition's instances as its own struct */ -static inline Py_ssize_t By_SpecDictOffset(PyType_Spec *spec) { +static inline Py_ssize_t By_SpecMemberOffset(PyType_Spec *spec, const char *name) { PyType_Slot *slot; if (spec == NULL || spec->slots == NULL) return 0; for (slot = spec->slots; slot->slot != 0; slot++) { PyMemberDef *member; if (slot->slot != Py_tp_members || slot->pfunc == NULL) continue; for (member = (PyMemberDef *)slot->pfunc; member->name != NULL; member++) { - if (strcmp(member->name, "__dictoffset__") == 0) return member->offset; + if (strcmp(member->name, name) == 0) return member->offset; } } return 0; } +/* the same exception holds for the weak-reference list, which a spec asks for on the same + * terms as the dict */ +static inline int By_OffsetHoldsUp(Py_ssize_t built, Py_ssize_t inherited, PyType_Spec *spec, + const char *name) { + Py_ssize_t asked; + if (built == inherited) return 1; + asked = By_SpecMemberOffset(spec, name); + return asked != 0 && built == asked; +} + static inline int By_OffsetsHoldUp(PyTypeObject *type, PyType_Spec *spec) { PyTypeObject *base = type->tp_base; - Py_ssize_t asked; if (base == NULL) { return 1; } - if (type->tp_weaklistoffset != base->tp_weaklistoffset) { - return 0; - } - if (type->tp_dictoffset == base->tp_dictoffset) { - return 1; - } - asked = By_SpecDictOffset(spec); - return asked != 0 && type->tp_dictoffset == asked; + return By_OffsetHoldsUp(type->tp_weaklistoffset, base->tp_weaklistoffset, spec, + "__weaklistoffset__") + && By_OffsetHoldsUp(type->tp_dictoffset, base->tp_dictoffset, spec, + "__dictoffset__"); } /* the type for a class whose fields sit past a base's instance, or nothing at all @@ -5386,6 +5391,102 @@ static inline int By_AdoptTwinAttributes(const By_Twins *twins) { return 0; } +/* what the module body wrote over a member *after* the `class` statement, carried onto the + * type that took the class's place + * + * `By_AdoptTwinAttributes` leaves a name the type already holds alone, and a type holds + * every name its own body lowered — so a later `C.v = property(...)`, `C.width = f` or + * `del C.extra` landed on the twin and the emitted type went on answering with what the + * `class` statement wrote. `body` is the class's dict as `type.__new__` left it, before + * anything after the statement could reach it (see `By_CaptureClassBody`), which is what + * tells a rewrite from the definition itself: a name whose value on the twin is no longer + * the object the body put there was rewritten, and the type takes the twin's answer or + * loses the name with it. + * + * the licences a class hands out are armed after this and ask the type's dict, so a + * rewritten member refuses the licence it would have held and every compiled read of it + * takes the full lookup. a decorated class is left out: a decorator the source wrote is + * applied to the type again, and what it rewrote on the twin is what it rewrites there, + * while one the transpiler wrote — `@dataclass(slots=True)` — hands back another class, + * which `statement` tells apart. a dunder is left out for the reason + * `By_AdoptTwinAttributes` gives */ +static inline int By_CarryRewrittenMembers(PyObject *body, PyObject *statement, + const By_Twins *twins, Py_ssize_t index) { + PyObject *twin = twins->twins[index]; + PyObject *type = twins->types[index]; + PyObject *source, *target, *names; + Py_ssize_t at; + int rewrote = 0; + /* a twin that is not the class its statement built was replaced by a decorator, and + * everything it holds differs from the body for that reason alone */ + if (body == NULL || twin == NULL || twin != statement || type == NULL || twin == type) { + return 0; + } + if (!PyDict_Check(body) || !PyType_Check(twin) || !PyType_Check(type)) return 0; + source = ((PyTypeObject *)twin)->tp_dict; + target = ((PyTypeObject *)type)->tp_dict; + if (source == NULL || target == NULL) return 0; + /* the keys first: a value is read back out one at a time, and carrying one can run + * python while a dict would otherwise still be walked */ + names = PyDict_Keys(body); + if (names == NULL) return -1; + for (at = 0; at < PyList_GET_SIZE(names); at++) { + PyObject *key = PyList_GET_ITEM(names, at); + PyObject *written, *now, *held, *carried; + int failed; + if (!PyUnicode_Check(key) || By_IsDunder(key)) continue; + written = PyDict_GetItem(body, key); + now = PyDict_GetItem(source, key); + if (now == written) continue; + held = PyDict_GetItem(target, key); + if (held == NULL || held == now) continue; + rewrote = 1; + if (now == NULL) { + if (PyDict_DelItem(target, key) < 0) { + Py_DECREF(names); + return -1; + } + continue; + } + /* a value that refuses to settle is still the one the body chose, and the member + * the class statement wrote is the one answer known to be wrong */ + carried = By_TwinReplacement(now, twins); + if (carried == NULL) { + if (PyErr_Occurred()) { + Py_DECREF(names); + return -1; + } + carried = Py_NewRef(now); + } + failed = PyDict_SetItem(target, key, carried) < 0; + Py_DECREF(carried); + if (failed) { + Py_DECREF(names); + return -1; + } + } + Py_DECREF(names); + if (rewrote) PyType_Modified((PyTypeObject *)type); + return 0; +} + +/* whether `entry` under `name` is what `By_CarryRewrittenMembers` carried: the twin's own + * answer, NULL where the module body deleted the name, and not the one the body wrote */ +static inline int By_RewrittenMember(PyObject *body, PyObject *statement, PyObject *twin, + const char *name, PyObject *entry) { + PyObject *written, *now; + if (body == NULL || !PyDict_Check(body) || twin == NULL || twin != statement) return 0; + if (!PyType_Check(twin) || ((PyTypeObject *)twin)->tp_dict == NULL) return 0; + written = PyDict_GetItemString(body, name); + if (written == NULL) { + PyErr_Clear(); + return 0; + } + now = PyDict_GetItemString(((PyTypeObject *)twin)->tp_dict, name); + if (now == NULL) PyErr_Clear(); + return entry == now && entry != written; +} + /* ── the members `@dataclass` generates ─────────────────────────────────────── * * a `data class` is `@dataclass(slots=True)` on the twin, so python's own decorator gives @@ -6089,12 +6190,15 @@ static int By_SettledMember(const char *const *settled, Py_ssize_t count, const * `by_type` slots — so a family that stood down together is checked against its own * standing definitions rather than against types nothing can reach. `methods` is the * class's `tp_methods` table, which is exactly the set of names lowered into - * descriptors, and `settled` names the entries a decorator has since replaced */ + * descriptors, and `settled` names the entries a decorator has since replaced. `body` is the + * class's dict as its `class` statement left it and `statement` the class that statement + * built, each NULL where none was captured */ static int By_VerifyClass(const char *module, PyObject *dict, const char *name, PyObject *stands, PyObject *emitted, PyObject *twin, PyObject *const *bases, Py_ssize_t base_count, PyMethodDef *methods, const char *const *settled, - Py_ssize_t settled_count, int decorated) { + Py_ssize_t settled_count, int decorated, PyObject *body, + PyObject *statement) { PyObject *published, *target, *source, *held; Py_ssize_t at; /* `emitted != twin` because a construction that could not be rebuilt hands the @@ -6169,6 +6273,12 @@ static int By_VerifyClass(const char *module, PyObject *dict, const char *name, PyObject *entry; if (By_SettledMember(settled, settled_count, methods[at].ml_name)) continue; entry = target == NULL ? NULL : PyDict_GetItemString(target, methods[at].ml_name); + /* the module body rewrote or deleted it after the `class` statement, and the type + * took that answer on purpose */ + if (By_RewrittenMember(body, statement, twin, methods[at].ml_name, entry) + && (entry == NULL || PyFunction_Check(entry))) { + continue; + } if (entry == NULL) { PyErr_Format(PyExc_ImportError, "%s.%s was compiled but publishes nothing under %s, which it " @@ -6189,14 +6299,17 @@ static int By_VerifyClass(const char *module, PyObject *dict, const char *name, return 0; } -/* record one class body against its name, or -1 with an exception set */ -static int By_RecordClassBody(PyObject *state, PyObject *name, PyObject *body) { +/* record one class body, and the class its statement built, against its name, or -1 with + * an exception set */ +static int By_RecordClassBody(PyObject *state, PyObject *name, PyObject *body, PyObject *cls) { PyObject *bodies = PyDict_GetItemString(state, "bodies"); - if (bodies == NULL) { + PyObject *statements = PyDict_GetItemString(state, "statements"); + if (bodies == NULL || statements == NULL) { PyErr_SetString(PyExc_RuntimeError, "module body capture lost its record"); return -1; } - return PyDict_SetItem(bodies, name, body); + if (PyDict_SetItem(bodies, name, body) < 0) return -1; + return PyDict_SetItem(statements, name, cls); } /* `__build_class__`, recording what each module-level `class` statement wrote @@ -6252,7 +6365,7 @@ static PyObject *By_CaptureClassBody(PyObject *state, PyObject *args, PyObject * /* a body that cannot be recorded is raised out of the `class` statement rather than * passed over: what would follow is a type carrying no constants at all, and for a * decorated class that is the defect this capture exists to remove */ - if (body == NULL || By_RecordClassBody(state, name, body) < 0) { + if (body == NULL || By_RecordClassBody(state, name, body, cls) < 0) { Py_XDECREF(body); Py_DECREF(cls); return NULL; @@ -6443,15 +6556,24 @@ static inline PyObject *By_ExecModuleBody(const By_Fallback *fallback, PyObject * delegation. * * hands back `{name: body}` for the classes the body wrote at module level, as a new - * reference, or NULL with an exception set where the body raised */ -static inline PyObject *By_RunModuleBody(const By_Fallback *fallback, PyObject *dict) { + * reference, or NULL with an exception set where the body raised. `*statements` is given + * `{name: class}` alongside it, the class each of those statements built before any + * decorator was handed it, as a new reference or NULL */ +static inline PyObject *By_RunModuleBody(const By_Fallback *fallback, PyObject *dict, + PyObject **statements) { static PyMethodDef capture = {"__build_class__", (PyCFunction)(void (*)(void))By_CaptureClassBody, METH_VARARGS | METH_KEYWORDS, NULL}; PyObject *bodies, *stood, *mapping, *displaced, *state, *wrapper, *result; int failed; + *statements = NULL; bodies = PyDict_New(); if (bodies == NULL) return NULL; + *statements = PyDict_New(); + if (*statements == NULL) { + Py_DECREF(bodies); + return NULL; + } /* an emitted module's dict has no `__builtins__` of its own, and python would then * give the body's frame the running interpreter's. naming it here is what makes the * functions the body defines take the same mapping rather than that fallback */ @@ -6464,6 +6586,7 @@ static inline PyObject *By_RunModuleBody(const By_Fallback *fallback, PyObject * if (displaced == NULL) { Py_XDECREF(stood); Py_DECREF(bodies); + Py_CLEAR(*statements); if (!PyErr_Occurred()) { PyErr_SetString(PyExc_RuntimeError, "no builtins `__build_class__` to run the module body against"); @@ -6474,6 +6597,7 @@ static inline PyObject *By_RunModuleBody(const By_Fallback *fallback, PyObject * state = PyDict_New(); failed = state == NULL || PyDict_SetItemString(state, "delegate", displaced) < 0 || PyDict_SetItemString(state, "bodies", bodies) < 0 + || PyDict_SetItemString(state, "statements", *statements) < 0 || PyDict_SetItemString(state, "globals", dict) < 0; wrapper = failed ? NULL : PyCFunction_New(&capture, state); failed = failed || wrapper == NULL @@ -6499,6 +6623,7 @@ static inline PyObject *By_RunModuleBody(const By_Fallback *fallback, PyObject * Py_XDECREF(stood); if (failed || result == NULL) { Py_DECREF(bodies); + Py_CLEAR(*statements); return NULL; } return bodies; @@ -7133,6 +7258,19 @@ static void By_RecheckField(PyObject *o, PyObject *type, const char *class_name, #endif /* BY_LICENCE_RECHECK */ +/* raise `KeyError(key)`, with the key as the one argument whatever it is + * + * `PyErr_SetObject` reads a tuple value as the whole argument list and an exception + * instance as the exception itself, so handing it the key raised `KeyError(1, 2)` for a + * missing `(1, 2)` and the key itself for a missing `KeyError('x')`. packing the key into + * a tuple of one is what cpython's own dict does for the same reason */ +static inline void By_RaiseKeyError(PyObject *key) { + PyObject *args = PyTuple_Pack(1, key); + if (args == NULL) return; + PyErr_SetObject(PyExc_KeyError, args); + Py_DECREF(args); +} + /* ── an emitted instance's `__dict__` ───────────────────────────────────────── * * an emitted instance keeps its attributes in two places. the ones the class itself @@ -7380,7 +7518,7 @@ static int By_InstanceDict_assign(PyObject *selfobj, PyObject *key, PyObject *va int held = PyDict_Contains(selfobj, key); if (held < 0) return -1; if (held == 0) { - PyErr_SetObject(PyExc_KeyError, key); + By_RaiseKeyError(key); return -1; } } @@ -7554,7 +7692,7 @@ static PyObject *By_InstanceDict_pop(PyObject *selfobj, PyObject *const *args, if (value == NULL) { if (PyErr_Occurred()) return NULL; if (nargs == 2) return By_NewRef(args[1]); - PyErr_SetObject(PyExc_KeyError, args[0]); + By_RaiseKeyError(args[0]); return NULL; } Py_INCREF(value); @@ -7806,6 +7944,37 @@ static inline int By_PublishNew(PyObject *type, PyMethodDef *def) { return stored; } +/* `obj.__weakref__`: the head of the weak references made of the object, or `None` + * + * what python's own descriptor answers, reached through the offset the type was built + * with rather than a member of one struct, so every emitted class can publish the one + * getter */ +static PyObject *By_GetWeakrefList(PyObject *obj, void *closure) { + (void)closure; + Py_ssize_t offset = Py_TYPE(obj)->tp_weaklistoffset; + PyObject *head = offset > 0 ? *(PyObject **)((char *)obj + offset) : NULL; + return Py_NewRef(head != NULL ? head : Py_None); +} + +/* hand an interpreted subclass the allocator of the emitted layout it extends + * + * an emitted class whose `int` fields start absent writes the absent value from its own + * `tp_alloc`, because a zeroed `int` field is the `int` zero. python does not let a + * subclass inherit that allocator: `type.__new__` gives every class it makes + * `PyType_GenericAlloc`, so `S.__new__(S)` handed back a block whose fields read as `0` + * where python raises `AttributeError`. the emitted allocator asks its base's for the block + * and only then writes the absent values, so for the subclass it is the generic allocation + * plus those writes. + * + * a class that wrote an allocator of its own is left with it, as is one that is not built + * on `owner` at all, which the call it arrived through is about to refuse */ +static inline void By_AdoptAllocator(PyTypeObject *type, PyObject *owner, allocfunc alloc) { + if (type->tp_alloc == PyType_GenericAlloc && type != (PyTypeObject *)owner + && PyType_IsSubtype(type, (PyTypeObject *)owner)) { + type->tp_alloc = alloc; + } +} + /* publish a `@property` as the object python builds out of its halves * * not as a `tp_getset` entry, which this was. a getset is two function pointers, and what @@ -8390,7 +8559,7 @@ static inline PyObject *By_GetItem(PyObject *container, PyObject *index) { if (PyErr_Occurred()) return NULL; #endif /* cpython raises the *key*, not a message */ - PyErr_SetObject(PyExc_KeyError, index); + By_RaiseKeyError(index); return NULL; } return By_GetItemSlow(container, index); @@ -8424,7 +8593,7 @@ BY_COLD PyObject *By_GetItemSlow(PyObject *container, PyObject *index) { if (value != NULL) return By_NewRef(value); if (PyErr_Occurred()) return NULL; /* cpython raises the *key*, not a message */ - PyErr_SetObject(PyExc_KeyError, index); + By_RaiseKeyError(index); return NULL; } return PyObject_GetItem(container, index); @@ -8808,11 +8977,7 @@ static int By_MergeKeyword(PyObject *keywords, PyObject *key, PyObject *value) { int present = PyDict_Contains(keywords, key); if (present < 0) return -1; if (present) { - PyObject *args = PyTuple_Pack(1, key); - if (args != NULL) { - PyErr_SetObject(PyExc_KeyError, args); - Py_DECREF(args); - } + By_RaiseKeyError(key); return -1; } return PyDict_SetItem(keywords, key, value); @@ -11010,6 +11175,43 @@ static inline PyObject *By_LookupSpecial(PyObject *self, PyObject *name, int *pr return bound; } +/* the direction of a binary operator a class did not write, answered the way python's own + * number slot answers it: by looking the name up on the operand's type, which reaches + * whatever a base defines under it + * + * a `list` subclass writing only `__mul__` still answers `2 * bag` through `list.__rmul__` + * this way. `adapter` is the slot asking, and an entry wrapping that same slot is refused + * rather than called, since calling it would only ask this function again */ +static inline PyObject *By_InheritedBinary(PyObject *receiver, PyObject *other, + const char *name, void *adapter) { + PyObject *key = PyUnicode_InternFromString(name); + if (key == NULL) return NULL; + PyObject *found = _PyType_Lookup(Py_TYPE(receiver), key); + if (found == NULL + || (Py_IS_TYPE(found, &PyWrapperDescr_Type) + && ((PyWrapperDescrObject *)found)->d_wrapped == adapter)) { + Py_DECREF(key); + Py_RETURN_NOTIMPLEMENTED; + } + int prepend; + PyObject *method = By_LookupSpecial(receiver, key, &prepend); + Py_DECREF(key); + if (method == NULL) { + if (PyErr_Occurred()) return NULL; + Py_RETURN_NOTIMPLEMENTED; + } + PyObject *result; + if (prepend) { + PyObject *argv[] = { receiver, other }; + result = PyObject_Vectorcall(method, argv, 2, NULL); + } else { + PyObject *argv[] = { other }; + result = PyObject_Vectorcall(method, argv, 1, NULL); + } + Py_DECREF(method); + return result; +} + /* [`By_LookupSpecial`] through a memo of what it last answered */ static inline PyObject *By_ProtocolMethod(ByProtocolSite *site, PyObject *manager, PyObject *name, int *prepend) { diff --git a/docs/basedpython/development/compilation/runtime.md b/docs/basedpython/development/compilation/runtime.md index 3191e3989b..6100ac14cb 100644 --- a/docs/basedpython/development/compilation/runtime.md +++ b/docs/basedpython/development/compilation/runtime.md @@ -950,6 +950,39 @@ except ValueError as e: traceback.print_exception(e) # python underlines the failing call; compiled does not ``` +### an interpreted subclass that allocates its first instance itself + +a compiled class keeps an `int` field unboxed, and a field no `__init__` has set holds a +value no `int` can have. the class's allocator writes that value, and a class made by a +`class` statement or `type(...)` is given python's generic allocator instead of its +base's, so the first instance the subclass builds through the compiled class — its +inherited `__new__`, or a written `__new__` the compiled class publishes — hands the +subclass that allocator. a subclass whose own `__new__` calls `object.__new__(cls)` +directly, over a compiled base that wrote a `__new__` of its own, reaches the generic +allocator without passing the compiled class, and until an instance has come through the +compiled class an unset `int` field on what it built reads as `0`: + +```python +class Made: + def __new__(cls) -> "Made": + return object.__new__(cls) + + def __init__(self) -> None: + self.count = 1 + + +# from another module +class Own(mod.Made): + def __new__(cls): + return object.__new__(cls) + + +Own.__new__(Own).count # python: AttributeError; compiled: 0 +``` + +a subclass is not reached by anything the compiled class can run when it is made, so +closing this would mean a test on every read of such a field + ## debugging and inspection - **`#line` directives** in the generated C point at `.by` source. gdb, lldb, From ecd8cc23aa2c322f8485d0cb35a64bcd21a78b58 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 15 Sep 2026 11:25:03 +1000 Subject: [PATCH 06/16] native: speed up generators, reference lifetimes and numeric loops step a generator directly from a for loop with its state number as a machine integer and its iternext inlined, recycle finished generators' memory, release tagged int temporaries at their last read, borrow list elements across an unbox, hold loop counters as machine integers inside a loop copy, convert counters straight to doubles, unswitch nested loops, and follow python's float floor division, remainder and power case for case --- crates/by_build/tests/differential.rs | 1979 ++++++++++++++++- crates/by_codegen_c/src/lib.rs | 900 +++++++- crates/by_ir/src/function.rs | 23 +- crates/by_ir/src/ops.rs | 103 +- crates/by_ir/src/print.rs | 48 +- crates/by_ir/src/rtype.rs | 29 + crates/by_ir/src/unbound_locals.rs | 7 +- crates/by_ir/src/verify.rs | 142 +- crates/by_irbuild/src/generators.rs | 110 +- crates/by_irbuild/src/lib.rs | 162 +- crates/by_irbuild/src/tests.rs | 86 +- crates/by_opt/src/borrow.rs | 65 +- crates/by_opt/src/copy_propagation.rs | 2 + crates/by_opt/src/dead_allocations.rs | 50 + crates/by_opt/src/fold.rs | 48 +- crates/by_opt/src/guard_loops.rs | 70 +- crates/by_opt/src/infallible.rs | 12 +- crates/by_opt/src/lib.rs | 78 +- crates/by_opt/src/push_lengths.rs | 391 ++++ crates/by_opt/src/release_temporaries.rs | 45 +- crates/by_opt/src/runs_python.rs | 6 +- crates/by_opt/src/store_moves.rs | 243 +- crates/by_opt/src/str_concat_int.rs | 2 + crates/by_opt/src/str_of_int.rs | 42 +- crates/by_opt/src/unbox_counters.rs | 1413 +++++++++--- crates/by_opt/src/unswitch.rs | 817 ++++--- crates/by_rt/include/by.h | 306 ++- .../development/compilation/runtime.md | 18 + 28 files changed, 6269 insertions(+), 928 deletions(-) create mode 100644 crates/by_opt/src/push_lengths.rs diff --git a/crates/by_build/tests/differential.rs b/crates/by_build/tests/differential.rs index 1215f19335..237c964906 100644 --- a/crates/by_build/tests/differential.rs +++ b/crates/by_build/tests/differential.rs @@ -305,6 +305,24 @@ class _Both: self.log.append(('exit' + self.tag, kind)) return False +# an iterable handing out an iterator that logs being let go of, and on the way asks the +# first generator in `box` for its next value +class _LoggedIterator: + def __init__(self, log, name, box): + self.log, self.name, self.box = log, name, box + def __next__(self): + return self.name + def __del__(self): + self.log.append('del ' + self.name) + if self.box: + self.log.append(type(_capture(next, self.box[0])).__name__) + +class _LoggedSource: + def __init__(self, log, name, box=None): + self.log, self.name, self.box = log, name, box if box is not None else [] + def __iter__(self): + return _LoggedIterator(self.log, self.name, self.box) + def _logged(fn): log = [] try: @@ -921,6 +939,13 @@ def _capture(fn, *args): return e return None +# the answer, or the exception raised in its place +def _outcome(fn, *args): + try: + return fn(*args) + except BaseException as e: + return e + # a subclass of `base` whose `name` rebinds `len` in `module` to answer `n`, then answers # as `answer` would def _sub(module, base, name, n, answer): @@ -1614,6 +1639,285 @@ def stepped(n: int) -> list[int]: ); } +#[test] +fn a_loop_counter_counts_past_the_machine_integer_as_python_does() { + // a counter a loop steps is held as a machine integer while the loop runs, but it is + // still an `int`: a step that leaves the machine word carries on as python's does. the + // counter is taken past the word at its top and at its bottom, by a step of one and by + // a step of a quarter of the word, with the trip's other work done before the step + // and not done again, and inside a nest where the inner counter overflows on a later + // trip of the outer loop. one also crosses the top of the tagged short range on the + // way, and one is read by a handler the loop raises to + agree( + "counteredge", + "\ +from typing import Callable + + +def near_top(n: int) -> int: + i = 9223372036854775806 + total = 0 + while i < n: + total = total + 1 + i = i + 1 + return total + + +def stepped_big(n: int) -> int: + i = 0 + k = 0 + while i < n: + i = i + 4611686018427387904 + k = k + 1 + return k + + +def falling(n: int) -> list[int]: + seen: list[int] = [] + i = 0 + while i > n: + seen.append(i) + i = i - 4611686018427387904 + seen.append(i) + return seen + + +def nested(n: int) -> int: + total = 0 + outer = 0 + while outer < 3: + j = 9223372036854775805 + while j < n: + total = total + j - 9223372036854775000 + j = j + 1 + outer = outer + 1 + return total + outer + + +def crossing(n: int) -> int: + i = 4611686018427387900 + total = 0 + while i < n: + total = total + i + i = i + 2305843009213693953 + return total + i + + +def top_bound(n: int) -> int: + i = 9223372036854775800 + k = 0 + while i < n: + i = i + 1 + k = k + 1 + return i - k + + +def caught(n: int) -> int: + i = 0 + try: + while True: + i = i + 4611686018427387904 + if i > n: + raise ValueError(i) + except ValueError: + return i + + +def climbing(n: int) -> int: + i = 0 + k = 0 + while k < n: + i = i + 2305843009213693953 + k = k + 1 + return i + + +def measured(s: str, n: int) -> int: + i = 0 + k = 0 + while i < n: + if len(s) > 0: + i = i + 4611686018427387904 + k = k + 1 + return k + + +def called(f: Callable[[int], None], n: int) -> int: + i = 0 + k = 0 + while i < n: + f(k) + i = i + 4611686018427387904 + k = k + 1 + return k +", + &[ + "m.near_top(2**63 + 3)", + "m.near_top(2**63 - 1)", + "m.near_top(0)", + "m.stepped_big(2**64)", + "m.stepped_big(2**63)", + "m.falling(-(2**64))", + "m.falling(-(2**63) - 1)", + "m.nested(2**63 + 2)", + "m.nested(2**63 - 2)", + "m.crossing(2**62 + 5)", + "m.crossing(2**63 + 10)", + "m.top_bound(2**63 - 1)", + "m.top_bound(2**63 + 1)", + "m.caught(2**64)", + "[m.climbing(n) for n in (0, 1, 2, 3, 4, 5)]", + "m.measured('ab', 2**64)", + "(lambda seen: (m.called(seen.append, 2**64), seen))([])", + ], + ); +} + +#[test] +fn a_loop_counter_stepped_by_other_counters_counts_past_the_machine_integer() { + // a counter whose steps are sums of other counters, or of a bound nothing in the loop + // writes, is as much a machine integer inside the loop as one stepped by a literal, and + // leaves the machine word as python's does: past the top by doubling, past the bottom + // by a negative step, by a step too large to narrow, and from a sum whose two operands + // are both counters that change on every trip + agree( + "countersums", + "\ +def sieve_count(limit: int) -> int: + flags = [] + i = 0 + while i < limit: + flags.append(True) + i = i + 1 + count = 0 + n = 2 + while n < limit: + if flags[n]: + count = count + 1 + m = n + n + while m < limit: + flags[m] = False + m = m + n + n = n + 1 + return count + + +def doubling(n: int) -> int: + x = 1 + k = 0 + while x < n: + x = x + x + k = k + 1 + return k + x + + +def stepped_by(step: int, n: int) -> int: + i = 0 + k = 0 + while i < n: + i = i + step + k = k + 1 + return k + + +def falling_by(step: int, n: int) -> int: + i = 0 + k = 0 + while i > n: + i = i - step + k = k + 1 + return i + + +def fibonacci(n: int) -> list[int]: + a = 0 + b = 1 + k = 0 + while k < n: + a = a + b + b = a - b + k = k + 1 + return [a, b] + + +def offset(start: int, n: int) -> int: + total = 0 + k = 0 + while k < n: + j = start + j = j + k + total = total + j + k = k + 1 + return total +", + &[ + "[m.sieve_count(n) for n in (0, 2, 3, 100, 1000)]", + "[m.doubling(n) for n in (0, 1, 2, 10**6, 2**62, 2**63, 10**30)]", + "[m.stepped_by(s, n) for s, n in ((3, 100), (2**61, 2**64), (2**62, 2**64), (2**64, 2**66), (1, 0))]", + "[m.falling_by(s, n) for s, n in ((3, -100), (2**61, -(2**64)), (2**62, -(2**64)), (2**64, -(2**66)))]", + "[m.fibonacci(n) for n in (0, 1, 10, 91, 92, 93, 200)]", + "[m.offset(s, n) for s, n in ((0, 10), (2**62 - 3, 10), (2**63 - 5, 10), (-(2**63), 10), (10**20, 3))]", + ], + ); +} + +#[test] +fn a_loop_counter_started_from_a_negative_literal_counts_as_python_does() { + // a negative literal is a literal, so a counter that starts at one counts as a machine + // integer where it can, and as python's `int` where it cannot: from `-1`, from `~3`, + // from the bottom of the machine word, and at both edges of the short range + agree( + "counternegative", + "\ +def from_minus_one(n: int) -> int: + i = -1 + total = 0 + while i < n: + total = total + i + i = i + 1 + return total + + +def inverted(n: int) -> int: + i = ~3 + k = 0 + while i < n: + i = i + 1 + k = k + 1 + return k + + +def bottom(n: int) -> int: + i = -9223372036854775807 - 1 + k = 0 + while i > n: + i = i - 1 + k = k + 1 + return k + + +def below_short(n: int) -> int: + i = -4611686018427387905 + k = 0 + while i < n: + i = i + 1 + k = k + 1 + return k + + +def edges() -> list[int]: + return [-4611686018427387904, -4611686018427387905, -9223372036854775807, ~4611686018427387903, ~-4611686018427387905, -0, ~0] +", + &[ + "[m.from_minus_one(n) for n in (-5, -1, 0, 10)]", + "[m.inverted(n) for n in (-10, 0, 10)]", + "[m.bottom(n) for n in (-(2**63) - 3, -(2**63), 0)]", + "[m.below_short(n) for n in (-(2**62) - 2, -(2**62) + 2)]", + "m.edges()", + ], + ); +} + #[test] fn an_operation_that_keeps_leaving_the_short_range_does_not_leak() { // each trip round this loop boxes both operands, calls cpython, and tags the @@ -1761,6 +2065,355 @@ def float_tdiv(a: float, b: float) -> float: ); } +/// the doubles a float operation can meet at its edges: a nan, both infinities, both +/// zeros, and the error value the runtime once used to signal a failure, which is a legal +/// double a program is free to compute +const FLOAT_EDGES: &str = "[float('nan'), float('inf'), float('-inf'), 0.0, -0.0, 1.5, -113.0]"; + +#[test] +fn float_operations_agree_at_every_edge() { + // a division tests its divisor and jumps to its error, an unbox tests its operand's + // type, and an `int` widened to a double tests its shortness — none of them hands an + // answer back that has to be told apart from an error by asking the thread. what + // has to survive that is every double the operations can legitimately answer, + // `-113.0` among them, and every failure raised as python raises it + let divisions = [ + format!( + "[repr(m.div(a, b)) if b != 0.0 else repr(_capture(m.div, a, b)) for a in {FLOAT_EDGES} for b in {FLOAT_EDGES}]" + ), + format!( + "[repr(m.fdiv(a, b)) if b != 0.0 else repr(_capture(m.fdiv, a, b)) for a in {FLOAT_EDGES} for b in {FLOAT_EDGES}]" + ), + format!( + "[repr(m.mod(a, b)) if b != 0.0 else repr(_capture(m.mod, a, b)) for a in {FLOAT_EDGES} for b in {FLOAT_EDGES}]" + ), + format!("[repr(m.walk([a, b])) for a in {FLOAT_EDGES} for b in {FLOAT_EDGES}]"), + format!("[repr(m.dot([a], [b])) for a in {FLOAT_EDGES} for b in {FLOAT_EDGES}]"), + format!("[repr(m.element({FLOAT_EDGES}, i)) for i in range(-7, 7)]"), + ]; + let mut calls: Vec<&str> = divisions.iter().map(String::as_str).collect(); + calls.extend([ + "m.div(-113.0, 1.0)", + "m.chain(-113.0)", + "repr(_capture(m.chain, 0.0))", + "[m.scaled(n, 3) for n in (0, 1, -1, 2**53 + 1, -(2**53) - 1, 2**62 - 1, 2**62, -(2**62), -(2**62) - 1, 2**63 - 1, 2**63, 10**300)]", + "repr(_capture(m.scaled, 10**400, 3))", + "repr(_capture(m.scaled, -(10**400), 3))", + "repr(_capture(m.scaled, 1, 0))", + "m.scaled(True, 2)", + // a `float` parameter handed an `int`, a `bool` or a subclass reaches the + // interpreted definition, whose answer is python's + "repr(m.div(1, 2))", + "repr(m.div(True, 2.0))", + "repr(_capture(m.div, 1, 0))", + "repr(m.div(type('Plain', (float,), {})(3.0), 2.0))", + // a subclass element with nothing of its own is its value + "repr(m.walk([type('Plain', (float,), {})(2.5), -0.0]))", + "repr(m.dot([type('Plain', (float,), {})(2.5)], [-113.0]))", + "repr(m.element([type('Plain', (float,), {})(2.5)], 0))", + "repr(_capture(m.element, [1.0], 1))", + ]); + agree( + "floatedges", + "\ +def div(a: float, b: float) -> float: + return a / b + + +def fdiv(a: float, b: float) -> float: + return a // b + + +def mod(a: float, b: float) -> float: + return a % b + + +def chain(a: float) -> float: + return (a / 1.0) / (a / -113.0) + + +def scaled(n: int, d: int) -> float: + return 3.0 * n / d + + +def walk(xs: list[float]) -> float: + total = 0.0 + for x in xs: + total = total * 0.5 + x + return total + + +def dot(a: list[float], b: list[float]) -> float: + out = 0.0 + i = 0 + while i < len(a): + out = out + a[i] * b[i] + i = i + 1 + return out + + +def element(xs: list[float], i: int) -> float: + return xs[i] +", + &calls, + ); +} + +/// the operands `//`, `%` and `**` treat specially: every sign of zero and infinity, a +/// nan, a subnormal, results that overflow and underflow, and `-1.0`, `1.0` and `2.0`, +/// which `**` answers without asking the platform +const DIVMOD_EDGES: &str = "[float('nan'), float('inf'), float('-inf'), 0.0, -0.0, 1.0, -1.0, 1.5, -1.5, 2.0, -4.0, 113.0, -113.0, 1e300, -1e300, 5e-324]"; +/// exponents, adding the fractional ones that take a negative base out of the reals +const EXPONENT_EDGES: &str = "[float('nan'), float('inf'), float('-inf'), 0.0, -0.0, 1.0, -1.0, 2.0, 3.0, -3.0, 0.5, -0.5, 1.0 / 3.0, 1e20, -1e20, 1e300]"; + +#[test] +fn float_floor_division_remainder_and_power_agree_at_every_edge() { + // python's `//` and `%` are not `floor(a / b)` and `fmod` with the sign fixed up: + // an infinite dividend floors to a nan, an infinite divisor floors a finite dividend + // of the other sign to `-1.0`, and a zero remainder takes the divisor's sign. its + // `**` raises on a zero base to a negative power and on a result too large for a + // double, and gives a complex number for a negative base to a fractional power — + // which a double cannot hold, so that pair is left to the object protocol + let calls = [ + format!("[repr(_outcome(m.fdiv, a, b)) for a in {DIVMOD_EDGES} for b in {DIVMOD_EDGES}]"), + format!("[repr(_outcome(m.mod, a, b)) for a in {DIVMOD_EDGES} for b in {DIVMOD_EDGES}]"), + format!( + "[repr(_outcome(m.fdiv_mod, a, b)) for a in {DIVMOD_EDGES} for b in {DIVMOD_EDGES}]" + ), + format!( + "[repr(_outcome(m.power, a, b)) for a in {DIVMOD_EDGES} for b in {EXPONENT_EDGES}]" + ), + format!("[repr(_outcome(m.held, a, b)) for a in {DIVMOD_EDGES} for b in {EXPONENT_EDGES}]"), + format!( + "[repr(_outcome(m.declared, a, b)) for a in {DIVMOD_EDGES} for b in {EXPONENT_EDGES}]" + ), + format!( + "[repr(_outcome(m.augmented, a, b)) for a in {DIVMOD_EDGES} for b in (0, 1, -1, 2, 3, -3, 2**70, -(2**70), 10**400)]" + ), + format!( + "[repr(_outcome(m.int_exponent, a, b)) for a in {DIVMOD_EDGES} for b in (0, 1, -1, 2, 3, -3, 2**70, -(2**70), 10**400)]" + ), + format!( + "[repr(_outcome(m.int_base, a, b)) for a in (0, 1, -1, 2, -8, 10**400) for b in {EXPONENT_EDGES}]" + ), + format!("[repr(_outcome(m.squared, a)) for a in {DIVMOD_EDGES}]"), + format!("[repr(_outcome(m.rooted, a)) for a in {DIVMOD_EDGES}]"), + format!("[repr(_outcome(m.from_two, b)) for b in {EXPONENT_EDGES}]"), + format!("[repr(_outcome(m.summed, a)) for a in {DIVMOD_EDGES}]"), + format!( + "[repr(_outcome(m.builtin_pow, a, b)) for a in {DIVMOD_EDGES} for b in {EXPONENT_EDGES}]" + ), + format!( + "[repr(_outcome(m.math_pow, a, b)) for a in {DIVMOD_EDGES} for b in {EXPONENT_EDGES}]" + ), + format!( + "[repr(_outcome(m.mixed_fdiv, a, b)) for a in (0, 7, -7, 10**400) for b in {DIVMOD_EDGES}]" + ), + format!("[repr(_outcome(m.mixed_mod, a, b)) for a in {DIVMOD_EDGES} for b in (0, 2, -2)]"), + ]; + let calls: Vec<&str> = calls.iter().map(String::as_str).collect(); + agree( + "floatdivpow", + "\ +import math + + +def fdiv(a: float, b: float) -> float: + return a // b + + +def mod(a: float, b: float) -> float: + return a % b + + +def fdiv_mod(a: float, b: float) -> tuple[float, float]: + return a // b, a % b + + +def power(a: float, b: float) -> object: + return a ** b + + +def held(a: float, b: float) -> object: + x = a ** b + return x + + +def declared(a: float, b: float) -> float: + return a ** b + + +def augmented(a: float, b: int) -> float: + x = a + x **= b + return x + + +def int_exponent(a: float, b: int) -> float: + return a ** b + + +def int_base(a: int, b: float) -> object: + return a ** b + + +def squared(a: float) -> float: + return a ** 2.0 + + +def rooted(a: float) -> object: + return a ** 0.5 + + +def from_two(b: float) -> float: + return 2.0 ** b + + +def summed(a: float) -> float: + total = 0.0 + i = 0 + while i < 3: + total = total + a ** 3.0 + a // 2.0 + a % -2.0 + i = i + 1 + return total + + +def builtin_pow(a: float, b: float) -> object: + return pow(a, b) + + +def math_pow(a: float, b: float) -> float: + return math.pow(a, b) + + +def mixed_fdiv(a: int, b: float) -> float: + return a // b + + +def mixed_mod(a: float, b: int) -> float: + return a % b +", + &calls, + ); +} + +#[test] +fn a_complex_power_is_refused_where_a_float_local_is_declared() { + // `float ** float` is `Any` to the checker, so a local declared `float` may be handed + // a power python answers with a complex number. the power itself is python's; the + // store is the unbox a declared `float` makes, and it refuses the complex loudly + // rather than holding a nan. this asserts on the compiled leg alone: python keeps + // the complex number + let Some(answers) = compiled_answers( + "floatpowrefused", + "\ +def assigned(a: float, b: float) -> float: + x: float = a ** b + return x + + +def augmented(a: float, b: float) -> float: + x = a + x **= b + return x +", + Options::default(), + &[ + "repr(_capture(m.assigned, -8.0, 0.5))", + "repr(_capture(m.augmented, -8.0, 1.0 / 3.0))", + "m.assigned(-8.0, 3.0)", + "m.augmented(8.0, 0.5)", + ], + ) else { + return; + }; + assert_eq!( + answers, + [ + "\"TypeError('expected float, got complex')\"", + "\"TypeError('expected float, got complex')\"", + "-512.0", + "2.8284271247461903", + ] + ); +} + +#[test] +fn a_float_operation_given_python_s_numeric_promotion_agrees_at_every_edge() { + // python's `float` admits an `int` and a `bool`, so the same operations over a `.py` + // module meet them wherever a double was annotated + agree_python( + "floatedgespy", + "\ +def div(a: float, b: float) -> float: + return a / b + + +def mod(a: float, b: float) -> float: + return a % b + + +def scaled(n: int, d: int) -> float: + return 3.0 * n / d + + +def counted(n: int) -> float: + total = 0.0 + i = 0 + while i < n: + total = total + i / 2 + i = i + 1 + return total +", + &[ + "[repr(_capture(m.div, a, b)) if b == 0 else repr(m.div(a, b)) for a in (1, True, -113.0, float('nan')) for b in (0, False, 2, True, -0.0, float('inf'))]", + "[repr(_capture(m.mod, a, b)) if b == 0 else repr(m.mod(a, b)) for a in (7, True, -113.0, -0.0) for b in (0, False, 2, -2.5, float('-inf'))]", + "[m.scaled(n, 3) for n in (True, False, 2**53 + 1, 2**63 - 1)]", + "repr(_capture(m.scaled, 10**400, 3))", + "repr(_capture(m.scaled, 1, 0))", + "[m.counted(n) for n in (0, 1, 1000)]", + ], + ); +} + +#[test] +fn a_float_element_of_the_wrong_type_is_refused_where_it_is_unboxed() { + // a `.by` `list[float]` holds floats, so an `int` element is refused where the loop + // unboxes it — with the unbox's own error, and nothing left half-done + let Some(answers) = compiled_answers( + "floatrefused", + "\ +def walk(xs: list[float]) -> float: + total = 0.0 + for x in xs: + total = total + x + return total + + +def element(xs: list[float], i: int) -> float: + return xs[i] +", + Options::default(), + &[ + "repr(_capture(m.walk, [1.0, 2]))", + "repr(_capture(m.walk, [None]))", + "repr(_capture(m.element, [1.0, True], 1))", + "m.walk([1.0, -113.0])", + ], + ) else { + return; + }; + assert_eq!( + answers, + [ + "\"TypeError('expected float, got int')\"", + "\"TypeError('expected float, got NoneType')\"", + "\"TypeError('expected float, got bool')\"", + "-112.0", + ] + ); +} + #[test] fn control_flow_agrees() { agree( @@ -2899,6 +3552,68 @@ def through_the_inner_tuple(n: int) -> list[str]: ); } +#[test] +fn an_int_nothing_names_is_dropped_as_soon_as_it_is_used() { + // an `int` is held tagged, and a value too wide for the tag is a real object behind + // a pointer — an `int` subclass among them, which may have a finalizer. so when a + // tagged temporary is let go of is observable in exactly the way it is for any other + // temporary: python drops it once the expression that made it is done, before the + // next statement runs, and the loop shape asks the same of a register the next trip + // would otherwise be the first thing to overwrite. + // + // `handed_on` is the one that answers with a number rather than an order: the value + // the call stores has to outlive the register that handed it over + agree_python( + "inttemps", + "\ +from collections.abc import Callable + + +def consume(make: Callable[[], int], log: list[str]) -> int: + total = make() + 1 + log.append('after') + return total + + +def looped(make: Callable[[], int], log: list[str], n: int) -> int: + total = 0 + i = 0 + while i < n: + total = total + make() + log.append('trip') + i = i + 1 + return total + + +def raised(make: Callable[[], int], log: list[str]) -> list[str]: + try: + make() // 0 + except ZeroDivisionError: + log.append('caught') + log.append('after') + return list(log) + + +def handed_on(make: Callable[[], int], sink: list[int], log: list[str]) -> int: + sink.append(make()) + log.append('stored') + sink.clear() + log.append('cleared') + return len(log) +", + &[ + "(lambda log: (m.consume(lambda: type('B', (int,), \ + {'__del__': lambda s: log.append('del')})(1 << 70), log), log))([])", + "(lambda log: (m.looped(lambda: type('B', (int,), \ + {'__del__': lambda s: log.append('del')})(1 << 70), log, 3), log))([])", + "(lambda log: m.raised(lambda: type('B', (int,), \ + {'__del__': lambda s: log.append('del')})(1 << 70), log))([])", + "(lambda log: (m.handed_on(lambda: type('B', (int,), \ + {'__del__': lambda s: log.append('del')})(1 << 70), [], log), log))([])", + ], + ); +} + #[test] fn a_value_stored_into_a_field_lives_as_long_as_the_field() { // a store that is its value's last read hands the register's reference to the field @@ -4387,6 +5102,35 @@ def walked(o) -> object: ); } +#[test] +fn a_loop_that_walks_an_exact_list_on_one_trip_and_anything_else_on_the_next_agrees() { + // whether a `for` walks an exact list is decided each time the loop opens, so one loop + // run again inside another can walk a list by index on one trip, then a subclass, a + // tuple, a generator or a list again on the next + agree( + "iterexactnest", + "\ +def flattened(rows: list[object]) -> object: + seen = [] + for row in rows: + for x in row: + seen.append(x) + return seen + + +def gen(n: int) -> object: + i = 0 + while i < n: + yield i + i = i + 1 +", + &[ + "m.flattened([[1, 2], (3, 4), type('D', (list,), {'__iter__': lambda self: iter([9])})([5, 6]), m.gen(2), [7], type('P', (list,), {})([8]), '', []])", + "m.flattened([m.gen(3), [1], (2,), [3, 4]])", + ], + ); +} + #[test] fn a_list_resized_under_a_for_loop_agrees() { // cpython's list iterator holds a position and re-reads the length every step, so @@ -4499,6 +5243,83 @@ def doubled(xs: list[int]) -> object: ); } +#[test] +fn a_for_loop_takes_a_reference_to_each_element_it_walks() { + // a `for` loop over an exact list walks it by index, reading the length again every + // step and taking a reference to the element it finds without asking whether the + // slot is empty — a slot inside an exact list's length never is. that reference keeps + // the element alive past anything the loop body does to the list, and everything + // that is not an exact list still goes through the protocol + agree( + "iterelement", + "\ +class Probe: + def __init__(self, tag: int) -> None: + self.tag = tag + + +def walked(o) -> object: + seen = [] + for x in o: + seen.append(x) + return seen + + +def cleared_under(xs: list[Probe]) -> object: + seen = [] + for x in xs: + xs.clear() + seen.append(x.tag) + return seen + + +def replaced_under(xs: list[Probe]) -> object: + seen = [] + i = 0 + for x in xs: + if i + 1 < len(xs): + xs[i + 1] = Probe(x.tag * 10) + seen.append(x.tag) + i = i + 1 + return seen + + +def sliced_under(xs: list[int]) -> object: + seen = [] + for x in xs: + if x == 1: + xs[:] = [7, 8, 9, 10] + seen.append(x) + return seen + + +def summed(xs: list[float]) -> float: + total = 0.0 + for x in xs: + total = total + x + if total > 100.0: + xs.append(-1.0) + return total +", + &[ + "m.walked((1, 2, 3))", + "m.walked(iter([1, 2, 3]))", + "m.walked('abc')", + "m.walked({'a': 1, 'b': 2})", + "m.walked(x * 2 for x in [1, 2])", + "m.walked(type('P', (list,), {})([1, 2, 3]))", + "m.walked(type('G', (list,), {'__getitem__': lambda s, i: -1, '__len__': lambda s: 9})([1, 2]))", + "m.cleared_under([m.Probe(1), m.Probe(2)])", + "m.replaced_under([m.Probe(1), m.Probe(2), m.Probe(3)])", + "m.sliced_under([1, 2, 3])", + "m.summed([60.0, 50.0, 1.0])", + "m.summed([])", + // the element a loop took stays alive for as long as the loop's reference does + "[_xs := [object()], __import__('sys').getrefcount(_xs[0]), m.walked(_xs)[0] is _xs[0], __import__('sys').getrefcount(_xs[0])][1::2]", + ], + ); +} + #[test] fn a_for_loop_starts_over_on_every_trip_through_an_enclosing_one() { // one cursor register serves every trip through whatever encloses it, so it is set @@ -4536,9 +5357,9 @@ def restarted(xs: list[int], times: int) -> object: #[test] fn a_for_loop_in_a_generator_resumes_where_it_left_off() { // a generator parks its iterator in a field because no register survives a - // `yield`, and a cursor is a register — so a generator's loop keeps the protocol. - // given one, a resumed frame would start again from whatever an unset register - // holds, which is to say from the top + // `yield`, and the cursor a loop over an exact list keeps is parked beside it. were + // it left in a register, a resumed frame would start again from whatever an unset + // register holds, which is to say from the top agree( "itergen", "\ @@ -4549,6 +5370,36 @@ def each(xs: list[int]) -> object: def taken(xs: list[int]) -> object: return list(each(xs)) + +def growing(xs: list[int]) -> object: + for x in xs: + if x < 3: + xs.append(x + 10) + yield x + +def shrinking(xs: list[int]) -> object: + for x in xs: + xs.pop() + yield x + +def nested(rows: list[list[int]]) -> object: + for row in rows: + for x in row: + yield x + yield -1 + +def echoed(xs: list[int]) -> object: + for x in xs: + got = yield x + yield got + +def left_early(xs: list[int]) -> object: + for x in xs: + if x == 2: + break + yield x + for x in xs: + yield x * 10 ", &[ "m.taken([1, 2, 3])", @@ -4556,6 +5407,17 @@ def taken(xs: list[int]) -> object: "list(m.each([4, 5]))", // stopping part way and resuming is the same question asked once "[next(it) for it in [iter(m.each([7, 8]))] for _ in range(3)]", + // a list changed while the frame is suspended is read again at the next step + "list(m.growing([1, 2, 5]))", + "list(m.shrinking([1, 2, 3, 4]))", + "(xs := [1, 2], g := m.each(xs), next(g), xs.append(3), list(g))[-1]", + "(xs := [1, 2], g := m.each(xs), next(g), next(g), xs.clear(), list(g))[-1]", + "list(m.nested([[1, 2], [], [3]]))", + "(g := m.echoed([1, 2]), next(g), g.send('a'), next(g), g.send('b'))[1:]", + "(g := m.each([1, 2]), next(g), type(_capture(g.throw, KeyError('k'))).__name__, list(g))[2:]", + "list(m.left_early([1, 2, 3]))", + // a subclass keeps its own iteration + "list(m.each(type('L', (list,), {'__iter__': lambda s: iter([9, 8])})([1, 2, 3])))", ], ); } @@ -4991,6 +5853,68 @@ def lookup(d: dict[str, int], k: str) -> int: ); } +#[test] +fn an_element_unboxed_straight_off_a_list_agrees() { + // an element read whose one use is to be unboxed borrows the element from an exact + // list across the unbox rather than retaining it. every other read — a subclass, a + // missed bound, a protocol read — takes the operations as they were. so what is asked + // here is where the borrowed path meets a failure: an element of the wrong type, an + // index out of range, and a `list` subclass whose `__getitem__` hands back a fresh + // object with a finalizer, which python lets go of before the handler runs. only the + // exception's type is compared, since a `.by` unbox words its `TypeError` its own way + agree( + "elementunbox", + "\ +def dot(a: list[float], b: list[float]) -> float: + out = 0.0 + i = 0 + while i < len(a): + out = out + a[i] * b[i] + i = i + 1 + return out + + +def pair_at(a: list[float], b: list[float], i: int) -> float: + return a[i] * b[i] + + +def total(xs: list[int]) -> int: + out = 0 + i = 0 + while i < len(xs): + out = out + xs[i] * 1 + i = i + 1 + return out + + +def guarded(a: list[float], b: list[float], i: int, log: list[str]) -> str: + try: + v = a[i] * b[i] + log.append('ok') + return str(v) + except TypeError: + log.append('type') + return 'type' + except IndexError: + log.append('index') + return 'index' +", + &[ + "m.dot([0.5, 1.5, 2.5], [2.0, 4.0, 6.0])", + "m.pair_at([1.0, 2.0, 3.0], [4.0, 5.0, 6.0], -1)", + "m.total([1, 2, 3])", + "m.total([1 << 70, 1 << 71, 5])", + "(lambda log: (m.guarded([3.0], [2.0], 0, log), log))([])", + "(lambda log: (m.guarded([3.0, 'x'], [2.0, 2.0], 1, log), log))([])", + "(lambda log: (m.guarded([3.0], [2.0], 9, log), log))([])", + "(lambda log: (m.guarded(type('S', (list,), {'__getitem__': lambda s, i: \ + type('L', (), {'__del__': lambda o: log.append('del')})()})([0.0]), \ + [1.0], 0, log), log))([])", + "(lambda e: type(e).__name__)(_capture(lambda: m.pair_at([1.0, 'x'], [2.0, 3.0], 1)))", + ], + ); +} + #[test] fn displays_agree() { agree( @@ -9432,6 +10356,51 @@ def rebinding(h: Holder, n: int) -> int: ); } +#[test] +fn two_fields_compared_on_loan_outlive_a_comparison_that_rebinds_them() { + // `h.low < h.high` reads both fields on loan, and a comparison of two `int`s outside + // the short range asks their class. a subclass's `__lt__` that writes both fields + // over drops the only references the fields held, so the comparison has to hold its + // operands for itself before it runs anything — and the loop has to read the fields + // again on the next trip rather than what it compared last + agree_python( + "comparefields", + "\ +class Holder: + low: int + high: int + + def __init__(self, low: int, high: int) -> None: + self.low = low + self.high = high + + +def below(h: Holder) -> bool: + return h.low < h.high + + +def trips(h: Holder, cap: int) -> int: + n = 0 + while h.low < h.high and n < cap: + n = n + 1 + return n +", + &[ + "[m.below(m.Holder(1, 2)), m.below(m.Holder(2**70, 2**69)), m.trips(m.Holder(0, 5), 10)]", + "(box := [], Big := type('Big', (int,), {'__lt__': lambda s, o: (\ + setattr(box[0], 'low', 7), setattr(box[0], 'high', 3), \ + __import__('gc').collect(), int(s) < int(o), repr(s))[3]}), \ + h := m.Holder(Big(2**70), Big(2**71)), box.append(h), \ + m.below(h), h.low, h.high)[4:]", + "(box := [], Big := type('Big', (int,), {'__lt__': lambda s, o: (\ + setattr(box[0], 'low', 5), setattr(box[0], 'high', 2), \ + __import__('gc').collect(), int(s) < int(o), repr(s))[3]}), \ + h := m.Holder(Big(2**70), Big(2**71)), box.append(h), \ + m.trips(h, 10), h.low, h.high)[4:]", + ], + ); +} + #[test] fn a_borrowed_narrowing_does_not_over_release_what_it_narrowed() { // the narrowing no longer retains, so the register is reading through something it @@ -12821,6 +13790,281 @@ def guarding(log: list[str]) -> object: ); } +/// `tp_iternext` ends a generator that returned `None` without raising, as cpython's own +/// does, and every other way of asking still sees what python shows +/// +/// the slot's quiet end is invisible from python by design — `next` raises +/// `StopIteration` on its behalf and `for` or `list` never see one — so these pin the +/// faces around it: `send` and `throw` still raise for a `None`, a value other than +/// `None` still rides out on the exception, and a frame that finished stays finished +#[test] +fn a_generator_that_returns_none_ends_its_iteration_as_python_s_does() { + agree( + "iterquiet", + "\ +def plain(n: int) -> object: + i = 0 + while i < n: + yield i + i = i + 1 + +def valued(n: int) -> object: + yield n + return n * 2 + +def caught() -> object: + try: + yield 1 + except ValueError: + return + +def raising() -> object: + yield 1 + raise KeyError('k') +", + &[ + "list(m.plain(3))", + "[v for v in m.plain(2)]", + "(g := m.plain(1), next(g), next(g, 'default'), next(g, 'again'))[1:]", + "(g := m.plain(0), (lambda e: (type(e).__name__, e.args, e.value))(_capture(g.__next__)))[1]", + "(g := m.plain(0), (lambda e: (type(e).__name__, e.args, e.value))(_capture(next, g)))[1]", + "(g := m.plain(1), next(g), (lambda e: (type(e).__name__, e.args, e.value))(_capture(g.send, None)))[2]", + "(g := m.caught(), next(g), (lambda e: (type(e).__name__, e.args, e.value))(_capture(g.throw, ValueError())))[2]", + "(g := m.valued(4), next(g), (lambda e: (type(e).__name__, e.args, e.value))(_capture(next, g)))[2]", + "(g := m.valued(4), list(g), type(_capture(next, g)).__name__)[1:]", + "(g := m.raising(), next(g), type(_capture(next, g)).__name__, type(_capture(next, g)).__name__)[2:]", + "_capture(list, m.raising())", + // an interpreted delegation reaches the same frame through the send slot + "(lambda: (yield from m.plain(2)))().__class__.__name__", + "list((lambda: (r := (yield from m.valued(3)), (yield r)))())", + "list((lambda: (r := (yield from m.plain(2)), (yield r)))())", + "list(zip(m.plain(3), m.plain(2)))", + "sorted(m.plain(3), reverse=True)", + ], + ); +} + +/// generators made and stepped by the same compiled frame, which steps them without the +/// iterator protocol +const STEPPED: &str = "\ +from collections.abc import Callable, Iterator + + +def counting(n: int) -> Iterator[int]: + i = 0 + while i < n: + yield i + i = i + 1 + + +def failing(n: int) -> Iterator[int]: + yield n + raise KeyError(n) + + +def valued(n: int) -> Iterator[int]: + yield n + return n * 2 + + +def total(n: int) -> int: + t = 0 + for v in counting(n): + t = t + v + return t + + +def collected(n: int) -> list[int]: + out: list[int] = [] + for v in counting(n): + out.append(v) + return out + + +def relayed(n: int) -> Iterator[int]: + for v in counting(n): + yield v * 10 + + +def over_an_expression(xs: list[int]) -> list[int]: + out: list[int] = [] + for v in (x * 2 for x in xs): + out.append(v) + return out + + +def left_and_resumed(n: int) -> list[object]: + g = counting(n) + out: list[object] = [] + for v in g: + out.append(v) + if v == 1: + break + out.append(next(g)) + for v in g: + out.append(v) + out.append(next(g, 'done')) + for v in g: + out.append(v) + return out + + +def raised(n: int) -> object: + t = 0 + try: + for v in failing(n): + t = t + v + except KeyError as e: + return ('caught', t, e.args, e.__traceback__.tb_next is not None) + return t + + +def returned(n: int) -> list[int]: + out: list[int] = [] + for v in valued(n): + out.append(v) + return out + + +def chosen(n: int, flip: bool) -> list[int]: + out: list[int] = [] + for v in (counting(n) if flip else failing_free(n)): + out.append(v) + return out + + +def failing_free(n: int) -> Iterator[int]: + yield -n + + +def hooked(n: int, hook: Callable[[int], object]) -> Iterator[int]: + i = 0 + while i < n: + hook(i) + yield i + i = i + 1 + + +def watched(n: int, hook: Callable[[int], object]) -> int: + t = 0 + for v in hooked(n, hook): + t = t + v + return t + + +def summed(n: int) -> int: + if n == 0: + return 0 + t = 0 + for v in feeding(n): + t = t + v + return t + + +def feeding(n: int) -> Iterator[int]: + yield summed(n - 1) + 1 +"; + +/// a `for` over a generator the same compiled frame made steps the compiled generator +/// directly, and every answer is still the protocol's +/// +/// the step is chosen by testing the iterator's type at every step, so a name rebound +/// to something else, a subclass-free python generator, or a list all take the protocol +/// instead, and a generator left by `break` is the same object a later `next` or `for` +/// resumes +#[test] +fn a_for_over_a_compiled_generator_steps_it_directly() { + agree_python( + "stepped", + STEPPED, + &[ + "m.total(5)", + "m.collected(4)", + "list(m.relayed(3))", + "m.over_an_expression([1, 2, 3])", + "m.left_and_resumed(5)", + "m.raised(7)", + "m.returned(3)", + "[m.chosen(3, True), m.chosen(3, False)]", + // a rebinding reaches the loop, which then takes the protocol + "(setattr(m, 'counting', lambda n: iter([n, n])), m.total(4), m.collected(3))[1:]", + "(setattr(m, 'counting', lambda n: (x for x in range(n, 0, -1))), m.total(4))[1]", + "(setattr(m, 'counting', m.failing_free), m.collected(4))[1]", + // the body runs python between the steps, and a hook that raises leaves the + // loop through its error edge + "(log := [], m.watched(3, log.append), log)[1:]", + "type(_capture(m.watched, 3, lambda i: 1 // (i - 1))).__name__", + "_deepest(m.summed, 60) > 0", + "type(_capture(m.summed, 10**5)).__name__", + ], + ); +} + +/// the direct step is what the loops in [`STEPPED`] are emitted as: the type test and +/// the type's own step, with the depth the frame already holds +/// +/// `agree` cannot tell the two apart — the protocol answers the same — so this reads +/// the emitted C of each loop's function +#[test] +fn a_for_over_a_compiled_generator_is_emitted_as_its_type_s_step() { + let Some((_python, toolchain)) = environment() else { + return; + }; + let dir = diff_root().join("by_diff_stepped_pin"); + let _ = std::fs::remove_dir_all(&dir); + let options = Options { + require_native: true, + language: by_irbuild::Language::Python, + ..Options::default() + }; + if build_source(STEPPED, "by_diff_stepped_pin", &toolchain, &dir, &options).is_err() { + eprintln!("skipping: no working C toolchain"); + return; + } + let emitted = std::fs::read_to_string(dir.join("by_diff_stepped_pin.c")) + .expect("the generated C is written beside the extension"); + let step = + "By_by_diff_stepped_pin_counting_gen_Type_next((By_by_diff_stepped_pin_counting_gen *)"; + for symbol in [ + "by_by_diff_stepped_pin_total", + "by_by_diff_stepped_pin_collected", + "by_by_diff_stepped_pin_left_and_resumed", + "by_by_diff_stepped_pin_relayed_gen__resume", + ] { + let body = emitted_function(&emitted, symbol); + assert!(body.contains(step), "{symbol}: {body}"); + assert!(body.contains(", by_depth, 1)"), "{symbol}: {body}"); + } + // a frame handed no depth looks it up once, and a generator's body is handed the one + // its step counted + assert!( + emitted_function(&emitted, "by_by_diff_stepped_pin_total") + .contains("ByDepth by_depth = By_DepthHere();"), + "{emitted}" + ); + assert!( + !emitted_function(&emitted, "by_by_diff_stepped_pin_relayed_gen__resume") + .contains("By_DepthHere"), + "{emitted}" + ); + // a generator expression is stepped the same way, and a loop whose iterable is not + // known to be one is left to the protocol with no depth looked up for it + assert!( + emitted_function(&emitted, "by_by_diff_stepped_pin_over_an_expression") + .contains("_genexpr0_gen_Type_next(("), + "{emitted}" + ); + // `tp_iternext` finishes quietly through the same step + assert!( + emitted_function( + &emitted, + "By_by_diff_stepped_pin_counting_gen_Type_iternext" + ) + .contains("By_by_diff_stepped_pin_counting_gen_Type_next("), + "{emitted}" + ); +} + /// the send slot is answered by the compiled state object, and its return arrives /// without an exception /// @@ -13211,16 +14455,22 @@ async def streamed() -> Any: } } // and a step that carries nothing carries `None` in, because the store cannot be - // skipped or the last `send` survives it: a generator's `__next__`, and the step of - // the awaitable an async generator's `__anext__` hands back — which is its `send` - // with nothing sent. a coroutine has no such step of its own — it is not an iterator + // skipped or the last `send` survives it: a generator's `__next__` — the per-type step + // its iternext slot runs — and the step of the awaitable an async generator's + // `__anext__` hands back, which is its `send` with nothing sent. a coroutine has no + // such step of its own — it is not an iterator for symbol in [ - "By_by_diff_framekind_pin_plain_gen_Type_iternext", + "By_by_diff_framekind_pin_plain_gen_Type_next", "By_by_diff_framekind_pin_streamed_gen_Type_asend_send", ] { let step = emitted_function(&emitted, symbol); assert!(step.contains("Py_None"), "{symbol}: {step}"); } + assert!( + emitted_function(&emitted, "By_by_diff_framekind_pin_plain_gen_Type_iternext") + .contains("By_by_diff_framekind_pin_plain_gen_Type_next("), + "{emitted}" + ); assert!( !emitted.contains("By_by_diff_framekind_pin_awaited_gen_Type_iternext"), "{emitted}" @@ -24523,6 +25773,164 @@ def other_array(xs: list[float], ys: list[float]) -> float: ); } +#[test] +fn a_nest_of_loops_entered_through_one_narrowing_agrees() { + // a nest whose bounds nothing inside it writes is duplicated once, and its copy is + // entered only when every one of those bounds is short — so an inner loop compares + // machine integers without a copy of its own, and a bound widened to a double inside + // the copy converts from the narrowed register. whichever bound is too large for the + // register sends the whole nest down the original, and the two have to agree + agree( + "unswitchnest", + "\ +def grid(rows: int, columns: int, depth: int) -> int: + total = 0 + y = 0 + while y < rows: + x = 0 + while x < columns: + k = 0 + while k < depth: + total = total + 1 + k = k + 1 + total = total + x + x = x + 1 + y = y + 1 + return total + + +def scaled(width: int, height: int) -> float: + total = 0.0 + y = 0 + while y < height: + x = 0 + while x < width: + total = total + 3.0 * x / width + 2.4 * y / height + x = x + 1 + y = y + 1 + return total + + +def early(rows: int, columns: int, stop: int) -> int: + total = 0 + y = 0 + while y < rows: + x = 0 + while x < columns: + total = total + 1 + if total >= stop: + return total + x = x + 1 + y = y + 1 + return total + + +def rebound(rows: int, columns: int, stop: int) -> int: + # the inner bound is written in the outer body, so only the outer bound is + # invariant across the nest and the inner loop is narrowed on its own + total = 0 + y = 0 + while y < rows: + width = columns + y + x = 0 + while x < width: + total = total + x + if x >= stop: + return total + x = x + 1 + y = y + 1 + return total +", + &[ + "[m.grid(r, c, d) for r in (0, 1, 3) for c in (0, 2) for d in (0, 1, 4)]", + "m.grid(2, 3, -(10 ** 30))", + "[m.scaled(w, h) for w in (1, 3, 7) for h in (1, 5)]", + "m.scaled(2**62, 0)", + "m.early(2**62, 3, 7)", + "m.early(3, 2**62, 7)", + "m.early(10 ** 30, 10 ** 30, 11)", + "m.early(-(2**62) - 1, 5, 3)", + "m.early(4, 4, 100)", + "[m.rebound(r, c, 100) for r in (0, 1, 4) for c in (0, 3)]", + "[m.rebound(2, c, 5) for c in (2**62 - 2, 2**62 - 1, 2**62)]", + ], + ); +} + +#[test] +fn a_counter_tagged_inside_a_narrowed_loop_agrees_at_the_edges_of_the_short_range() { + // inside a loop duplicated behind a narrowed bound, a counter the guard has just kept + // below the bound, and whose writes keep it from ever falling below the short range, + // is given its tagged value with no range test. a counter written again between the + // guard and the read is proven nothing, and one started outside the short range is + // never proven to stay inside it — each of those still takes the tested conversion, + // and all of them have to give python's answers at the edges of the range + agree( + "tagshort", + "\ +def split(value: int) -> tuple[int, int]: + return value // 7, value % 7 + + +def up(stop: int, n: int) -> list[int]: + out = [0] + i = 4611686018427387898 + while i < n: + whole, part = split(i) + out.append(whole) + out.append(part) + i = i + 1 + if i > stop: + return out + return out + + +def down(n: int) -> list[int]: + out = [0] + i = 4611686018427387903 + while i > n: + whole, part = split(i) + out.append(whole + part) + i = i - 1 + return out + + +def stepped_after_the_guard(n: int) -> list[int]: + out = [0] + i = 4611686018427387898 + while i < n: + i = i + 2 + whole, part = split(i) + out.append(whole) + out.append(part) + return out + + +def started_outside(n: int) -> list[int]: + out = [0] + i = 4611686018427387906 + while i > n: + i = i - 1 + whole, part = split(i) + out.append(whole) + out.append(part) + return out +", + &[ + "m.up(2**62 + 5, 2**62 - 1)", + "m.up(2**62 + 5, 2**62 + 1)", + "m.up(2**62 - 3, 10**30)", + "m.up(0, 2**62 - 1)", + "m.down(2**62 - 4)", + "m.down(10**30)", + "m.stepped_after_the_guard(2**62 - 1)", + "m.stepped_after_the_guard(2**62)", + "m.started_outside(2**62 - 3)", + "m.started_outside(10**30)", + ], + ); +} + #[test] fn a_float_subclass_handed_to_a_float_parameter_reaches_the_interpreted_definition() { // a `float` parameter compiles to a `double`, and a subclass of `float` is more than @@ -25914,6 +27322,215 @@ async def stream(n: int) -> AsyncIterator[int]: ); } +/// `close` finishes a generator no handler of its own encloses without running its frame, +/// and one that some handler encloses still runs it +#[test] +fn a_close_with_nothing_to_run_finishes_the_frame() { + agree_python( + "quietclose", + QUIET_CLOSES, + &[ + "(log := [], g := m.one(_LoggedSource(log, 'x')), next(g), g.close(), log[:], \ + type(_capture(next, g)).__name__, g.close())[3:]", + "(g := m.plain(1), next(g), g.close(), type(_capture(next, g)).__name__, g.close())[2:]", + "(g := m.plain(1), g.close(), type(_capture(next, g)).__name__)[1:]", + // a value sent in is let go of by the close + "(log := [], O := type('O', (), {'__del__': lambda s: log.append('sent gone')}), \ + g := m.plain(1), next(g), g.send(O()), g.close(), log[:])[5:]", + // a frame some handler encloses still runs it + "(log := [], g := m.guarded([1, 2], log), next(g), g.close(), log[:])[3:]", + "(log := [], g := m.caught([1, 2], log), next(g), g.close(), log[:])[3:]", + // abandoned by `any` part way, and collected the same way + "[m.any_of([1, 2, 3], k) for k in (0, 1, 3)]", + "(log := [], [(lambda g: (next(g), None))(m.one(_LoggedSource(log, str(k)))) \ + for k in range(3)], log)[-1]", + ], + ); +} + +/// generators closed in [`a_close_with_nothing_to_run_finishes_the_frame`] +const QUIET_CLOSES: &str = "\ +from collections.abc import Iterable, Iterator + + +def one(src: Iterable[object]) -> Iterator[object]: + for a in src: + yield a + + +def one_handled(src: Iterable[object]) -> Iterator[object]: + for a in src: + try: + yield a + except ValueError: + pass + + +def looped(outer: Iterable[object], inner: Iterable[object]) -> Iterator[object]: + for a in outer: + for b in inner: + yield (a, b) + + +def looped_handled(outer: Iterable[object], inner: Iterable[object]) -> Iterator[object]: + for a in outer: + for b in inner: + try: + yield (a, b) + except ValueError: + pass + + +def plain(n: int) -> Iterator[int]: + yield n + yield n + 1 + + +def guarded(xs: Iterable[object], log: list[str]) -> Iterator[object]: + try: + for x in xs: + yield x + finally: + log.append('finally') + + +def caught(xs: Iterable[object], log: list[str]) -> Iterator[object]: + for x in xs: + try: + yield x + except GeneratorExit: + log.append('exit') + raise + + +def any_of(xs: list[int], k: int) -> bool: + return any(x == k for x in xs) +"; + +/// the close that runs nothing has what the unwinding would have had: the same iterators +/// let go of in the same order, with the frame running while they go +/// +/// each generator here is paired with one that differs only by an `except` its close +/// runs through, so a compiled leg that finished the frame differently from how it unwinds +/// one answers the pair differently. python itself answers the two halves of a pair +/// differently where a finalizer asks the frame for its next value, and lets go of two +/// iterators in the other order, so these are compiled against compiled +#[test] +fn a_close_with_nothing_to_run_has_the_unwinding_s_effects() { + let probe = |one: &str, looped: &str| { + [ + format!( + "(log := [], box := [], g := m.{one}(_LoggedSource(log, 'x', box)), box.append(g), \ + next(g), g.close(), log[:], type(_capture(next, g)).__name__)[6:]" + ), + format!( + "(log := [], g := m.{looped}(_LoggedSource(log, 'outer'), _LoggedSource(log, 'inner')), \ + next(g), g.close(), log[:])[4:]" + ), + ] + }; + let quiet = probe("one", "looped"); + let handled = probe("one_handled", "looped_handled"); + let calls: Vec<&str> = quiet.iter().chain(&handled).map(String::as_str).collect(); + let Some(answers) = compiled_answers( + "quietcloseeffects", + QUIET_CLOSES, + Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + &calls, + ) else { + return; + }; + assert_eq!(answers[..2], answers[2..], "{answers:?}"); +} + +/// a generator's memory is kept back for the next generator of its type, and the next one +/// is a new generator in every way python can ask about +/// +/// what could survive into the next generator from the last is what the block does not +/// reset: the collector's header, whose finalized bit would keep a suspended frame's +/// `finally` from ever running. a generator the collector finalized in a cycle, one that +/// raised, one closed, one that finished, one whose weak reference is still out and one +/// never started are each followed by generators of the same type that have to run +/// their cleanups, answer `gc.is_tracked` and start from the top +#[test] +fn a_recycled_generator_is_a_new_generator() { + agree_python( + "genrecycle", + "\ +from collections.abc import Iterator + + +def guarded(log: list[str], tag: str) -> Iterator[int]: + try: + yield 1 + yield 2 + finally: + log.append('finally ' + tag) + + +def raising(n: int) -> Iterator[int]: + yield n + raise ValueError(n) + + +def counting(n: int) -> Iterator[int]: + i = 0 + while i < n: + yield i + i = i + 1 + + +def rounds(n: int) -> list[object]: + out: list[object] = [] + i = 0 + while i < n: + g = raising(i) + out.append(next(g)) + try: + next(g) + except ValueError as e: + out.append(('raised', e.args)) + h = raising(i + 100) + out.append(next(h)) + i = i + 1 + return out + + +def drained(n: int) -> list[int]: + out: list[int] = [] + i = 0 + while i < n: + for v in counting(i): + out.append(v) + i = i + 1 + return out +", + &[ + // the collector finalizes a generator in a cycle, then frees its block + "(_cycled(lambda box, log: m.guarded(log, 'cycle')), \ + (lambda log: ([(lambda g: (next(g), gc.is_tracked(g))[1])(m.guarded(log, str(k))) \ + for k in range(3)], log))([]))", + "[_cycled(lambda box, log: m.guarded(log, str(k))) for k in range(3)]", + "m.rounds(4)", + "m.drained(5)", + // closed, finished and unstarted generators, each followed by a new one + "(lambda log: ((lambda g: (next(g), g.close()))(m.guarded(log, 'a')), \ + list(m.guarded(log, 'b')), m.guarded(log, 'c') and None, \ + (lambda k: (next(k), next(k)))(m.guarded(log, 'd')), log))([])", + // a weak reference's callback runs on the old generator, and the new one has + // none of its own + "(lambda fired, wr, refs: ( \ + (lambda g: (list(g), refs.append(wr.ref(g, lambda r: fired.append('gone')))))(m.counting(2)), \ + fired[:], \ + (lambda h: (wr.getweakrefcount(h), list(h)))(m.counting(3)), \ + [r() for r in refs]))([], __import__('weakref'), [])", + ], + ); +} + /// a frame that is running cannot be resumed a second time from inside itself /// /// python refuses every entry point while the frame executes, so the resumption in the @@ -26634,6 +28251,94 @@ def grown(n: int) -> int: ); } +#[test] +fn a_buffer_appended_in_a_loop_reports_every_element_wherever_the_loop_is_left() { + // a loop that only appends to a buffer may keep the buffer's length to itself while it + // runs, but whoever reads the buffer next sees every element: after the loop ends, + // after a `break`, from a handler an exception raised part way through a trip reached + // (past several growths, so the buffer has moved), through a second name bound to the + // same buffer before the loop, and from an outer loop reading it between inner loops + agree( + "pushlength", + "\ +from typing import Callable + + +def partial_len(xs: list[float]) -> int: + out = [] + try: + for x in xs: + out.append(1.0 / x) + except ZeroDivisionError: + pass + return len(out) + + +def partial_last(xs: list[float]) -> float: + out = [0.0] + try: + for x in xs: + out.append(1.0 / x) + except ZeroDivisionError: + return out[len(out) - 1] + len(out) + return -1.0 + + +def broken(n: int, stop: int) -> int: + out = [] + i = 0 + while i < n: + if i == stop: + break + out.append(i * 2) + i = i + 1 + return len(out) + out[len(out) - 1] + + +def aliased(n: int) -> int: + out = [0] + other = out + i = 0 + while i < n: + out.append(i) + i = i + 1 + return len(other) + other[len(other) - 1] + + +def rows(n: int) -> list[int]: + out = [0] + seen = [] + r = 0 + while r < 3: + i = 0 + while i < n: + out.append(i) + i = i + 1 + seen.append(len(out)) + r = r + 1 + return [seen[0], seen[1], seen[2], out[len(out) - 1]] + + +def called(f: Callable[[int], None], n: int) -> int: + out = [] + i = 0 + while i < n: + f(i) + out.append(i) + i = i + 1 + return len(out) +", + &[ + "[m.partial_len(xs) for xs in ([1.0, 2.0, 0.0, 4.0], [1.0] * 100 + [0.0], [0.0], [], [2.0] * 50)]", + "[m.partial_last(xs) for xs in ([1.0, 2.0, 0.0, 4.0], [4.0] * 37 + [0.0], [0.0], [8.0])]", + "[m.broken(n, stop) for n, stop in ((10, 3), (100, 99), (100, 1000), (5, 1))]", + "[m.aliased(n) for n in (0, 1, 7, 100)]", + "[m.rows(n) for n in (0, 1, 5, 40)]", + "(lambda seen: (m.called(seen.append, 20), len(seen)))([])", + ], + ); +} + #[test] fn a_conjunction_pattern_agrees() { // basedpython's `case P and Q:` — every one has to match the *same* subject. @@ -32977,6 +34682,65 @@ def escapes(log: list[str]) -> str: ); } +#[test] +fn a_generator_local_held_in_its_own_representation_agrees() { + // a local some path reads before writing — a loop's target, one assigned in a branch — + // starts unset in the state object, and keeps its own representation where that has + // an unset value to spare: a tagged `int`'s error value, a pointer's `NULL`. every + // read still asks whether it was written, the body's own and a generator + // expression's through the state object alike + agree_python( + "gentyped", + "\ +from collections.abc import Iterator + + +def doubled(xs: list[int]) -> Iterator[int]: + for x in xs: + yield x * 2 + + +def shouted(words: list[str]) -> Iterator[str]: + for w in words: + yield w + '!' + + +def last_after(xs: list[int]) -> Iterator[object]: + for x in xs: + yield x + yield x + + +def branch(n: int) -> Iterator[int]: + if n > 0: + y = n + yield 1 + yield y + + +def expression_reads_later(xs: list[int]) -> Iterator[object]: + g = (x + later for x in xs) + yield 'made' + try: + yield next(g) + except NameError as e: + yield type(e).__name__ + later = 10**20 + yield list(g) +", + &[ + "list(m.doubled([1, 2**70, -3]))", + "list(m.shouted(['a', 'bc']))", + "list(m.last_after([4, 5]))", + "type(_capture(list, m.last_after([]))).__name__", + "list(m.branch(3))", + "type(_capture(list, m.branch(0))).__name__", + "list(m.expression_reads_later([1, 2]))", + "list(m.doubled([True, False]))", + ], + ); +} + #[test] fn an_unset_generator_local_agrees() { // a state field is unboxed only where it is provably assigned. each of these would @@ -36262,6 +38026,158 @@ class Widened(list): ); } +#[test] +fn a_machine_counter_reaches_a_double_an_object_and_a_buffer_at_every_edge() { + // a counter held as a machine integer is converted to a double, built into an object + // and used to index a packed buffer straight from that integer, rather than being + // given its tagged value first. what those have to agree on is every value the + // counter can hold: either side of the short range, either side of 2**53 where a + // double stops holding every integer, and the ends of the register itself + agree( + "counterreach", + "\ +def doubles_from(n: int) -> list[float]: + out = [0.0] + i = 9007199254740985 + while i < n: + out.append(0.5 * i) + out.append(1.0 * i / 3.0) + i = i + 1 + return out + + +def doubles_short(n: int) -> list[float]: + out = [0.0] + i = 4611686018427387900 + while i < n: + out.append(1.0 * i) + i = i + 1 + return out + + +def doubles_top(n: int) -> list[float]: + out = [0.0] + i = 9223372036854775800 + while i < n: + out.append(1.0 * i) + i = i + 1 + return out + + +def doubles_bottom(n: int) -> list[float]: + out = [0.0] + i = -9223372036854775803 + while i > n: + i = i - 1 + out.append(1.0 * i) + return out + + +def members(seen: set[int], n: int) -> list[bool]: + out = [False] + i = 4611686018427387900 + while i < n: + out.append(i in seen) + i = i + 1 + return out + + +def members_top(seen: set[int], n: int) -> list[object]: + out: list[object] = [] + i = 9223372036854775800 + while i < n: + out.append(i) + out.append(i in seen) + i = i + 1 + return out + + +def keys(n: int) -> str: + out = '' + i = 4611686018427387900 + while i < n: + out = out + str(i) + ',' + i = i + 1 + return out + + +def keys_bottom(n: int) -> str: + out = '' + i = -9223372036854775803 + while i > n: + i = i - 1 + out = out + 'k' + str(i) + return out + + +def packed(n: int) -> int: + flags = [True] + i = 1 + while i < n: + flags.append(i % 3 == 0) + i = i + 1 + count = 0 + j = 0 + while j < n: + if flags[j]: + flags[j] = False + count = count + 1 + j = j + 1 + k = -1 + while k >= -n: + if not flags[k]: + flags[k] = True + count = count + 10 + k = k - 1 + return count + + +def packed_past_the_short_range(n: int) -> int: + flags = [True] + j = 4611686018427387902 + while j < n: + if flags[j]: + return 1 + j = j + 1 + return 0 + + +def packed_below_the_short_range(n: int) -> int: + flags = [True] + j = -4611686018427387902 + while j > n: + j = j - 1 + if flags[j]: + return 1 + return 0 + + +def packed_at_the_top(n: int) -> int: + flags = [True] + j = 9223372036854775805 + while j < n: + if flags[j]: + return 1 + j = j + 1 + return 0 +", + &[ + "m.doubles_from(2**53 + 6)", + "m.doubles_short(2**62 + 3)", + "m.doubles_top(2**63 - 1)", + "m.doubles_bottom(-(2**63))", + "m.members({2**62 - 1, 2**62, 2**62 + 1}, 2**62 + 3)", + "m.members_top({2**63 - 2, 2**63 - 1}, 2**63 - 1)", + "m.keys(2**62 + 3)", + "m.keys_bottom(-(2**63))", + "[m.packed(n) for n in (1, 2, 7, 30)]", + "repr(_capture(m.packed_past_the_short_range, 2**62 + 2))", + "repr(_capture(m.packed_below_the_short_range, -(2**62) - 2))", + "repr(_capture(m.packed_at_the_top, 2**63 - 1))", + ], + ); +} + #[test] fn an_unboxed_counter_indexes_every_container_and_every_edge() { // a counter a loop steps by one gets a machine representation, and a subscript @@ -37035,6 +38951,55 @@ def call_rebound(f: Callable[[int], object], log: list[str]) -> None: ); } +#[test] +fn a_tuple_of_ints_holds_its_elements_until_python_would_have_dropped_it() { + // the tagged twin of the test above. an `int` slot holding a value too wide for the + // tag holds a real object, which an `int` subclass can give a finalizer, so the slots + // of a returned pair are let go of on python's schedule too: each name as it is + // deleted once the pair is unpacked, and last to first when the pair itself goes. + // the pair takes each slot's reference from the temporary that computed it rather + // than one of its own, so nothing is left holding a slot past the pair + agree_python( + "inttuplelife", + "\ +from collections.abc import Callable + + +def ints(f: Callable[[int], int]) -> tuple[int, int]: + return f(1), f(2) + + +def unpacked(f: Callable[[int], int], log: list[str]) -> None: + a, b = ints(f) + log.append('unpacked') + del a + log.append('a gone') + del b + log.append('b gone') + + +def held(f: Callable[[int], int], log: list[str]) -> None: + t = ints(f) + log.append('held') + del t + log.append('dropped') + + +def discarded(f: Callable[[int], int], log: list[str]) -> None: + ints(f) + log.append('after') +", + &[ + "(lambda log: (m.unpacked(lambda i: type('B', (int,), \ + {'__del__': lambda s: log.append('del ' + str(int(s) >> 70))})(i << 70), log), log)[1])([])", + "(lambda log: (m.held(lambda i: type('B', (int,), \ + {'__del__': lambda s: log.append('del ' + str(int(s) >> 70))})(i << 70), log), log)[1])([])", + "(lambda log: (m.discarded(lambda i: type('B', (int,), \ + {'__del__': lambda s: log.append('del ' + str(int(s) >> 70))})(i << 70), log), log)[1])([])", + ], + ); +} + #[test] fn a_length_past_the_short_range_agrees() { // a length is a `Py_ssize_t`, and a tagged `int` holds one bit fewer, so a length an diff --git a/crates/by_codegen_c/src/lib.rs b/crates/by_codegen_c/src/lib.rs index c4e5274a3b..174b891c52 100644 --- a/crates/by_codegen_c/src/lib.rs +++ b/crates/by_codegen_c/src/lib.rs @@ -124,6 +124,49 @@ fn delegate_accessor(resume: &by_ir::function::Resumption, type_name: &str) -> S } } +/// how the frame a resumable class steps through presents itself, where `function` is +/// that frame's body +fn resumed_body<'a>( + module: &'a ModuleIr, + function: &Function, +) -> Option<&'a by_ir::function::Resumption> { + function + .owner + .as_deref() + .and_then(|owner| class_named(module, owner)) + .and_then(|class| { + class + .resume + .as_ref() + .filter(|_| class.resumes_through(function)) + }) +} + +/// whether any loop in `function` steps an iterator it expects to be a compiled +/// generator, and so needs the thread's depth in hand — see [`stepped_generator`] +fn direct_steps(module: &ModuleIr, function: &Function) -> bool { + function + .blocks + .iter() + .flat_map(|block| &block.ops) + .any(|op| match op { + Op::IterNext { iter, .. } => stepped_generator(module, function, iter).is_some(), + _ => false, + }) +} + +/// the per-type step a generator's `tp_iternext` runs, which a compiled `for` over that +/// type calls directly +fn generator_step(module: &ModuleIr, class: &ClassIr) -> String { + format!("{}_next", class.type_name(module.name.dotted())) +} + +/// the counted step every resumption of a resumable frame goes through, handed the +/// thread's depth +fn counted_resume(module: &ModuleIr, class: &ClassIr) -> String { + format!("{}_step", class.type_name(module.name.dotted())) +} + fn mangle_member(name: &str) -> String { by_ir::function::FieldDecl { cell: false, @@ -1071,8 +1114,17 @@ const RECYCLED_INSTANCES: usize = 16; /// so a class with a finalizer is turned away. the bit is only ever set after a /// non-null `tp_finalize` has been called, and the two things that fill that slot /// are a written `__del__` — [`finalizes`], which asks the whole base chain — and a -/// generator's own. with neither, the header of a parked block is all zeroes, which -/// is exactly what the allocator hands back +/// resumable frame's own. with neither, the header of a parked block is all zeroes, +/// which is exactly what the allocator hands back +/// +/// a *generator's* finalizer is the exception, because it has something to do only for +/// a frame still suspended: its deallocator calls it for that frame alone, and a +/// generator that ran to its end, or never started, is freed with the bit clear. the +/// collector sets the bit on whatever it finalizes whether or not the finalizer did +/// anything, so the block is asked before it is parked, and one the collector +/// finalized goes back to the allocator instead. a coroutine's finalizer warns about a +/// frame that never ran and an async generator's reaches the event loop's hooks, so +/// neither is recycled /// /// **the object header's own trace links**, which only exist in a build tracing /// every object. those are inside the reset, and zeroing them would cut a live list @@ -1089,7 +1141,12 @@ const RECYCLED_INSTANCES: usize = 16; /// plain member inside the struct — and a weakref list, which a spec of ours never /// asks for fn recycles_instances(module: &ModuleIr, class: &ClassIr) -> bool { - !mutable_type(module, class) && class.resume.is_none() && !finalizes(module, class) + !mutable_type(module, class) + && !finalizes(module, class) + && class + .resume + .as_ref() + .is_none_or(|resume| resume.surface == Surface::Generator) } /// a class keeping the memory of a dead instance back for the next one @@ -1119,10 +1176,21 @@ fn emit_instance_recycling(module: &ModuleIr, class: &ClassIr) -> String { // address. `PyType_GenericAlloc` reads the flag to decide, and these two have // to agree with it or an instance is allocated under one discipline and freed // under another - let (release, track) = if instance_dict(module, class) || class.environment { - ("PyObject_GC_Del", " PyObject_GC_Track(by_block);\n") + let (release, track) = + if instance_dict(module, class) || class.environment || class.resume.is_some() { + ("PyObject_GC_Del", " PyObject_GC_Track(by_block);\n") + } else { + ("PyObject_Free", "") + }; + // a generator's block the collector finalized carries the bit saying so, which no + // call clears — see [`recycles_instances`] + let finalized = if class.resume.is_some() { + "\x20 if (BY_UNLIKELY(PyObject_GC_IsFinalized((PyObject *)by_block))) {\n\ + \x20 PyObject_GC_Del(by_block);\n\ + \x20 return;\n\ + \x20 }\n" } else { - ("PyObject_Free", "") + "" }; // a free-threaded build gets none of it: the array is shared mutable state, and // this module tells such an interpreter it keeps none — see the `Py_mod_gil` @@ -1167,6 +1235,7 @@ fn emit_instance_recycling(module: &ModuleIr, class: &ClassIr) -> String { }}\n\n\ static void {type_name}_free(void *by_block) {{\n\ {recycling}\n\ + {finalized}\ \x20 if (BY_LIKELY({type_name}_recycled_at < {RECYCLED_INSTANCES})) {{\n\ \x20 {type_name}_recycled[{type_name}_recycled_at++] = (PyObject *)by_block;\n\ \x20 return;\n\ @@ -1244,7 +1313,21 @@ fn emit_class_type(module: &ModuleIr, class: &ClassIr) -> String { " if (self->by_weakrefs != NULL) PyObject_ClearWeakRefs((PyObject *)self);\n", ); } - if class.resume.is_some() || finalizes(module, class) { + // a generator's finalizer closes a suspended frame and does nothing for any other, + // so it is only called for one — which leaves the block of every other generator + // free of the collector's finalized bit, and so recyclable. a coroutine's warns + // about a frame that never ran, and an async generator's reaches the event loop + if class + .resume + .as_ref() + .is_some_and(|resume| resume.surface == Surface::Generator) + { + let _ = writeln!( + out, + " if (self->{state} > 0 && PyObject_CallFinalizerFromDealloc((PyObject *)self) < 0) return;", + state = mangle_member(GENERATOR_STATE) + ); + } else if class.resume.is_some() || finalizes(module, class) { out.push_str( " if (PyObject_CallFinalizerFromDealloc((PyObject *)self) < 0) return;\n", ); @@ -2166,12 +2249,36 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { // the *native* entry point, not the wrapper: `tp_iternext` already has the // receiver, and the wrapper's argument binding is pure overhead on what is a // generator's hottest path - let symbol = class + let body = class .methods .iter() .find(|method| method.name == resume.method) .map(|method| method.native_symbol(module.name.dotted())) .unwrap_or_default(); + // every way of resuming the frame goes through one counted step, which counts the + // resume as python counts pushing the frame it would have resumed. the body is + // called from here and nowhere else — the runtime helpers `send`, `throw` and + // `close` share are handed the step behind a pointer rather than the body. + // + // a resume is a cycle the call graph cannot see: `yield from f(n - 1)` makes the + // next generator with a native call and then steps it through the iterator + // protocol, and the interpreter counts none of that — so a generator delegating + // to itself ran off the stack. NULL with `RecursionError` set where the frame could + // not be pushed, which the caller finishes the generator on, as python closes one + // whose frame could not be + let symbol = format!("{type_name}_resume"); + let _ = writeln!( + out, + "static PyObject *{step}({struct_name} *self, ByDepth by_depth) {{\n\ + \x20 if (BY_UNLIKELY(By_DepthEnter(by_depth))) return NULL;\n\ + \x20 PyObject *by_result = {body}(self, by_depth);\n\ + \x20 By_DepthLeave(by_depth);\n\ + \x20 return by_result;\n\ + }}\n\ + static PyObject *{symbol}(PyObject *self) {{\n\ + \x20 return {step}(({struct_name} *)self, By_DepthHere());\n}}", + step = counted_resume(module, class), + ); // the iterator a suspension inside `yield from` or `await` waits on, which `throw` // and `close` reach before the frame. a frame that never delegates has none to // name, and says so with a null accessor rather than one that always answers NULL @@ -2197,7 +2304,7 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { out, "static PyObject *{type_name}_delegate(PyObject *selfobj) {{\n\ \x20 {struct_name} *self = ({struct_name} *)selfobj;\n\ - \x20 switch (By_ShortValue(self->{state})) {{\n\ + \x20 switch (self->{state}) {{\n\ {cases}\ \x20 default: return NULL;\n\ \x20 }}\n}}", @@ -2214,21 +2321,62 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { \x20 {struct_name} *self = ({struct_name} *)selfobj;\n\ \x20 return By_StepGenerator(selfobj, &self->{sent}, &self->by_returned,\n\ \x20 &self->{state}, {frame}, args[0],\n\ - \x20 (PyObject *(*)(PyObject *)){symbol});\n}}", + \x20 {symbol});\n}}", sent = mangle_member(crate::GENERATOR_SENT), state = mangle_member(crate::GENERATOR_STATE) ); // `close` throws `GeneratorExit` in, which runs every enclosing `finally`. the // frame finishes itself on the way out, and one that suspends again instead is - // still suspended — python refuses that close and a later step resumes it + // still suspended — python refuses that close and a later step resumes it. + // + // a generator suspended where nothing of its own handles an exception is finished + // here instead, with the effects its unwinding would have had in the order it + // would have had them: nothing sent, the frame running while its loops' iterators + // are let go of, and then finished. only a generator's surface: a coroutine and an + // async generator are closed by what drives them — see `QuietClose` + let mut quiet = String::new(); + if resume.surface == Surface::Generator { + for close in &resume.quiet_closes { + let _ = writeln!(quiet, "\x20 case {}:", close.state); + let _ = writeln!( + quiet, + "\x20 By_ParkSent(&self->{sent}, Py_None);\n\ + \x20 self->{state} = BY_FRAME_RUNNING;", + sent = mangle_member(crate::GENERATOR_SENT), + state = mangle_member(crate::GENERATOR_STATE) + ); + for field in &close.clears { + let _ = writeln!( + quiet, + "\x20 By_ClearField(&self->{});", + mangle_member(field) + ); + } + let _ = writeln!( + quiet, + "\x20 By_FinishGenerator(&self->{});\n\ + \x20 Py_RETURN_NONE;", + mangle_member(crate::GENERATOR_STATE) + ); + } + } + let quiet = if quiet.is_empty() { + quiet + } else { + format!( + "\x20 switch (self->{}) {{\n{quiet}\x20 default: break;\n\x20 }}\n", + mangle_member(crate::GENERATOR_STATE) + ) + }; let _ = writeln!( out, "static PyObject *{type_name}_close(PyObject *selfobj, PyObject *const *args, Py_ssize_t nargs) {{\n\ \x20 (void)args; (void)nargs;\n\ \x20 {struct_name} *self = ({struct_name} *)selfobj;\n\ + {quiet}\ \x20 return By_CloseGenerator(selfobj, &self->{sent}, &self->{thrown},\n\ \x20 &self->by_returned, &self->{state}, {frame},\n\ - \x20 {delegate}, (PyObject *(*)(PyObject *)){symbol});\n}}", + \x20 {delegate}, {symbol});\n}}", sent = mangle_member(crate::GENERATOR_SENT), thrown = mangle_member(crate::GENERATOR_THROWN), state = mangle_member(crate::GENERATOR_STATE) @@ -2244,7 +2392,7 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { \x20 &(({struct_name} *)selfobj)->{thrown},\n\ \x20 &(({struct_name} *)selfobj)->by_returned,\n\ \x20 &(({struct_name} *)selfobj)->{state}, {frame}, args, nargs,\n\ - \x20 {delegate}, (PyObject *(*)(PyObject *)){symbol});\n}}", + \x20 {delegate}, {symbol});\n}}", sent = mangle_member(crate::GENERATOR_SENT), thrown = mangle_member(crate::GENERATOR_THROWN), state = mangle_member(crate::GENERATOR_STATE) @@ -2261,7 +2409,7 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { // dropped before its first step for a missing `await` let unawaited = if resume.surface == Surface::Coroutine { format!( - "\x20 if ((({struct_name} *)selfobj)->{state} == By_ShortFrom(0)) {{\n\ + "\x20 if ((({struct_name} *)selfobj)->{state} == 0) {{\n\ \x20 PyObject *by_type, *by_value, *by_tb;\n\ \x20 PyErr_Fetch(&by_type, &by_value, &by_tb);\n\ \x20 By_WarnUnawaitedCoroutine(selfobj, {qualname});\n\ @@ -2281,7 +2429,7 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { \x20 /* only a *suspended* frame has anything to unwind: state 0 never\n\ \x20 * started and -1 already finished, and resuming either would run\n\ \x20 * the body a second time */\n\ - \x20 if (By_ShortValue((({struct_name} *)selfobj)->{state}) <= 0) return;\n\ + \x20 if ((({struct_name} *)selfobj)->{state} <= 0) return;\n\ \x20 PyObject *by_type, *by_value, *by_tb;\n\ \x20 PyErr_Fetch(&by_type, &by_value, &by_tb);\n\ \x20 PyObject *by_r = {type_name}_close(selfobj, NULL, 0);\n\ @@ -2355,27 +2503,33 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { } // a generator's state object *is* the iterator: `tp_iternext` drives `$resume`, - // which returns the next yielded value or raises `StopIteration` + // which returns the next yielded value or ends the iteration let iterator = match &class.resume { None => String::new(), Some(resume) => { let frame = frame_kind(resume.surface); - let symbol = class - .methods - .iter() - .find(|method| method.name == resume.method) - .map(|method| method.native_symbol(module.name.dotted())) - .unwrap_or_default(); + let symbol = format!("{type_name}_resume"); // only a generator is an iterator: a coroutine is driven through its send // slot, and an async generator through the awaitable `__anext__` hands back + // + // the step is written out per type rather than through a shared helper handed + // the resume as a pointer, so the resume is a direct call the C compiler can + // inline. it takes the thread's depth from its caller, which `tp_iternext` + // looks up if resume.surface == Surface::Generator { let _ = writeln!( out, - "static PyObject *{type_name}_iternext(PyObject *self) {{\n\ - \x20 return By_StepGenerator(self, &(({struct_name} *)self)->{sent},\n\ - \x20 &(({struct_name} *)self)->by_returned,\n\ - \x20 &(({struct_name} *)self)->{state}, {frame}, Py_None,\n\ - \x20 (PyObject *(*)(PyObject *)){symbol});\n}}", + "static inline PyObject *{next}({struct_name} *self, ByDepth by_depth, int by_quiet) {{\n\ + \x20 if (BY_UNLIKELY(By_RefuseRunning(self->{state}, BY_FRAME_GENERATOR) < 0)) return NULL;\n\ + \x20 By_ParkSent(&self->{sent}, Py_None);\n\ + \x20 PyObject *by_step = {counted}(self, by_depth);\n\ + \x20 if (BY_LIKELY(by_step != NULL)) return by_step;\n\ + \x20 return By_IterFinished(&self->{state}, &self->by_returned, by_quiet);\n\ + }}\n\ + static PyObject *{type_name}_iternext(PyObject *self) {{\n\ + \x20 return {next}(({struct_name} *)self, By_DepthHere(), 0);\n}}", + next = generator_step(module, class), + counted = counted_resume(module, class), sent = mangle_member(crate::GENERATOR_SENT), state = mangle_member(crate::GENERATOR_STATE) ); @@ -2395,7 +2549,7 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { \x20 return By_SendGenerator(self, &(({struct_name} *)self)->{sent},\n\ \x20 &(({struct_name} *)self)->by_returned,\n\ \x20 &(({struct_name} *)self)->{state}, {frame},\n\ - \x20 (PyObject *(*)(PyObject *)){symbol},\n\ + \x20 {symbol},\n\ \x20 by_arg, by_result);\n}}", sent = mangle_member(crate::GENERATOR_SENT), state = mangle_member(crate::GENERATOR_STATE) @@ -2479,7 +2633,7 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { \x20 return NULL;\n\ \x20 return By_ThrowInto((PyObject *)by_gen, &by_gen->{sent}, &by_gen->{thrown},\n\ \x20 &by_gen->by_returned, &by_gen->{state}, {frame}, args, nargs,\n\ - \x20 {delegate}, (PyObject *(*)(PyObject *)){symbol});\n\ + \x20 {delegate}, {symbol});\n\ }}\n\ static PyObject *{type_name}_asend_send(PyObject *self, PyObject *by_arg) {{\n\ \x20 {type_name}_asend *by_self = ({type_name}_asend *)self;\n\ @@ -2492,7 +2646,7 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { \x20 {type_name}_refuse_reuse(by_self);\n\ \x20 return NULL;\n\ \x20 }}\n\ - \x20 if (By_ShortValue(by_gen->{state}) == -1) {{\n\ + \x20 if (by_gen->{state} == -1) {{\n\ \x20 by_self->by_state = 2;\n\ \x20 PyErr_SetNone(PyExc_StopIteration);\n\ \x20 return NULL;\n\ @@ -2521,7 +2675,7 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { \x20 by_step = By_ThrowInto((PyObject *)by_gen, &by_gen->{sent}, &by_gen->{thrown},\n\ \x20 &by_gen->by_returned, &by_gen->{state}, {frame},\n\ \x20 &by_exit, 1, {delegate},\n\ - \x20 (PyObject *(*)(PyObject *)){symbol});\n\ + \x20 {symbol});\n\ \x20 }} else {{\n\ \x20 /* what `athrow` was called with, kept whole: python builds the exception\n\ \x20 * out of it at this step rather than at the call */\n\ @@ -2531,7 +2685,7 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { \x20 &by_gen->by_returned, &by_gen->{state}, {frame},\n\ \x20 &PyTuple_GET_ITEM(by_carried, 0),\n\ \x20 PyTuple_GET_SIZE(by_carried), {delegate},\n\ - \x20 (PyObject *(*)(PyObject *)){symbol});\n\ + \x20 {symbol});\n\ \x20 by_step = {type_name}_unwrap(by_gen, by_step);\n\ \x20 if (by_step != NULL) return by_step;\n\ \x20 by_gen->by_running_async = 0;\n\ @@ -2541,7 +2695,7 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { \x20 }} else {{\n\ \x20 by_step = By_StepGenerator((PyObject *)by_gen, &by_gen->{sent}, &by_gen->by_returned,\n\ \x20 &by_gen->{state}, {frame}, by_arg,\n\ - \x20 (PyObject *(*)(PyObject *)){symbol});\n\ + \x20 {symbol});\n\ \x20 if (!by_closing) return {type_name}_unwrap(by_gen, by_step);\n\ \x20 }}\n\ \x20 /* `aclose()` from here on: a yield is the frame refusing to close, and the frame\n\ @@ -2579,7 +2733,7 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { \x20 PyObject *by_step = By_StepGenerator((PyObject *)by_gen, &by_gen->{sent}, &by_gen->by_returned,\n\ \x20 &by_gen->{state}, {frame},\n\ \x20 by_arg != NULL ? by_arg : Py_None,\n\ - \x20 (PyObject *(*)(PyObject *)){symbol});\n\ + \x20 {symbol});\n\ \x20 by_step = {type_name}_unwrap(by_gen, by_step);\n\ \x20 if (by_step == NULL) by_self->by_state = 2;\n\ \x20 return by_step;\n\ @@ -2776,6 +2930,15 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { } else { String::new() }; + // the pair that keeps a dead instance's memory back for the next one, as a spec + // built class takes it — see [`recycles_instances`] + let recycled = if recycles_instances(module, class) { + format!( + " .tp_alloc = {type_name}_alloc,\n .tp_free = {type_name}_free,\n" + ) + } else { + String::new() + }; let flags = if class.environment || class.resume.is_some() { format!( " .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC{refused},\n .tp_traverse = (traverseproc){type_name}_traverse,\n .tp_clear = (inquiry){type_name}_clear,\n" @@ -2785,7 +2948,7 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { }; let _ = write!( out, - "static PyTypeObject {type_name} = {{\n PyVarObject_HEAD_INIT(NULL, 0)\n .tp_name = \"{dotted}.{}\",\n .tp_basicsize = sizeof({struct_name}),\n .tp_itemsize = 0,\n .tp_dealloc = (destructor){type_name}_dealloc,\n{flags}{weakrefs}{iterator}{dunders} .tp_methods = {type_name}_methods,\n .tp_getset = {type_name}_getset,\n{construction} }};\n\ + "static PyTypeObject {type_name} = {{\n PyVarObject_HEAD_INIT(NULL, 0)\n .tp_name = \"{dotted}.{}\",\n .tp_basicsize = sizeof({struct_name}),\n .tp_itemsize = 0,\n .tp_dealloc = (destructor){type_name}_dealloc,\n{flags}{weakrefs}{recycled}{iterator}{dunders} .tp_methods = {type_name}_methods,\n .tp_getset = {type_name}_getset,\n{construction} }};\n\ ", class.name ); @@ -5269,6 +5432,120 @@ fn out_slot(buffer: &mut String) -> &mut String { } /// the function a `CallNative` names, if this module emits it +/// the compiled generator an iterator is expected to be, where the value it steps can be +/// traced back to a native call that builds and returns one +/// +/// this is a guess about which type to *test* for, never a claim about what the value +/// is: a `for` over `steps(n)` holds whatever `steps` names when the loop starts, which a +/// rebinding can make anything at all, so the step asks `Py_TYPE` and goes through the +/// protocol whenever the answer is no. a wrong guess costs one pointer comparison a step. +/// +/// the value is followed through copies and — in a generator, which parks a loop's +/// iterator in its state object across a suspension — through the field it is parked in +fn stepped_generator<'a>( + module: &'a ModuleIr, + function: &Function, + value: &Value, +) -> Option<&'a ClassIr> { + let Value::Register(start) = value else { + return None; + }; + let mut pending = vec![*start]; + let mut seen: HashSet = HashSet::new(); + while let Some(register) = pending.pop() { + if !seen.insert(register) { + continue; + } + for op in function.blocks.iter().flat_map(|block| &block.ops) { + if op.dest() != Some(register) { + continue; + } + match op { + Op::Assign { + src: Value::Register(src), + .. + } + | Op::Move { + src: Value::Register(src), + .. + } + | Op::GetIter { + src: Value::Register(src), + .. + } => pending.push(*src), + Op::GetField { class, field, .. } => { + pending.extend( + function + .blocks + .iter() + .flat_map(|block| &block.ops) + .filter_map(|store| match store { + Op::SetField { + class: stored_class, + field: stored_field, + value: Value::Register(stored), + .. + } if stored_class == class && stored_field == field => { + Some(*stored) + } + _ => None, + }), + ); + } + Op::CallNative { owner, callee, .. } => { + if let Some(class) = native_callee(module, owner.as_deref(), callee) + .and_then(|callee| generator_built_by(module, callee)) + { + return Some(class); + } + } + _ => {} + } + } + } + None +} + +/// the compiled generator `function` builds and hands back, where every return it makes +/// is a new state object of that one type +fn generator_built_by<'a>(module: &'a ModuleIr, function: &Function) -> Option<&'a ClassIr> { + let defining = |register: RegisterId| { + function + .blocks + .iter() + .flat_map(|block| &block.ops) + .find(|op| op.dest() == Some(register)) + }; + let mut built: Option<&'a ClassIr> = None; + for block in &function.blocks { + let Terminator::Return(Value::Register(returned)) = &block.terminator else { + continue; + }; + let mut register = *returned; + if let Some(Op::Box { + src: Value::Register(boxed), + .. + }) = defining(register) + { + register = *boxed; + } + let Some(Op::NewInstance { class, .. }) = defining(register) else { + return None; + }; + let class = class_named(module, class).filter(|class| { + class + .resume + .as_ref() + .is_some_and(|resume| resume.surface == Surface::Generator) + })?; + if built.is_some_and(|other| other.name != class.name) { + return None; + } + built = Some(class); + } + built +} + fn native_callee<'a>( module: &'a ModuleIr, owner: Option<&str>, @@ -5456,10 +5733,16 @@ fn signature(module: &ModuleIr, function: &Function) -> String { .enumerate() .map(|(index, decl)| format!("{} {}", ctype(module, &decl.ty), local(RegisterId(index)))) .collect(); + let resumed = resumed_body(module, function); let symbol = if is_recursive(function) { params.insert(0, "ByDepth by_depth".to_string()); depth_symbol(module, function) } else { + // a resumable frame's body is handed the depth its counted step took the frame + // from, which a `for` in the body over a compiled generator hands on in turn + if resumed.is_some() { + params.push("ByDepth by_depth".to_string()); + } function.native_symbol(module.name.dotted()) }; let params = if params.is_empty() { @@ -5467,10 +5750,23 @@ fn signature(module: &ModuleIr, function: &Function) -> String { } else { params.join(", ") }; - let linkage = match in_place(function) { + // a resumable frame's body has one caller, its type's counted step, which is + // decided here rather than left to the C compiler. a generator's is written out in + // the step: a `for` over the generator enters the body once an element, where the + // call was a measurable part of the step. a coroutine's or an async generator's is + // kept apart: it is entered once a suspending `await`, and written out in the step it + // runs its loops with the depth the step holds across it in a register (`coro_none` + // +12%), which a single caller would otherwise have had clang do + let placement = match resumed { + Some(resume) if resume.surface == Surface::Generator => InPlace::Always, + Some(_) => InPlace::Apart, + None => in_place(function), + }; + let linkage = match placement { InPlace::Always => "__attribute__((always_inline)) static inline", InPlace::Offered => "static inline", InPlace::Left => "static", + InPlace::Apart => "__attribute__((noinline)) static", }; format!( "{linkage} {} {symbol}({params})", @@ -5487,6 +5783,8 @@ enum InPlace { Offered, /// left to the C compiler's own judgement Left, + /// never written out in place + Apart, } /// the most operations a body may have and still be offered to its callers to write out @@ -5721,6 +6019,14 @@ fn emit_function(module: &ModuleIr, function: &Function) -> String { let _ = writeln!(out, " {retain}"); } } + // a frame handed no depth looks it up once, for every step it makes of a compiled + // generator, where `tp_iternext` would have looked it up once a step + if !is_recursive(function) + && resumed_body(module, function).is_none() + && direct_steps(module, function) + { + out.push_str(" ByDepth by_depth = By_DepthHere();\n"); + } if !function.registers.is_empty() { out.push('\n'); } @@ -5754,7 +6060,14 @@ fn emit_function(module: &ModuleIr, function: &Function) -> String { // path — see [`tested_read`] and [`unset_test`] let mut tested_by_read: Option = None; let mut tested_by_arithmetic: Vec = Vec::new(); + // the unbox and release an element read took into its own C, which are not + // emitted again where they stand + let mut taken_by_read: usize = 0; for (at, op) in block.ops.iter().enumerate() { + if taken_by_read > 0 { + taken_by_read -= 1; + continue; + } if let Op::Line { position } = op { line = Some(position.offset); in_generator_expression = position.generator_expression; @@ -5790,6 +6103,36 @@ fn emit_function(module: &ModuleIr, function: &Function) -> String { } else { std::mem::take(&mut tested_by_arithmetic).concat() }; + // the three share one line, so they share the edge worked out for the read + if unset.is_empty() + && let Some(taken) = block.ops.get(at..at + 3) + && let Some(fused) = borrowed_element_unbox(module, function, taken, edge) + { + fragment.push_str(&fused); + sites.note(&fragment, edge); + if let Some(at_error) = at_error.as_mut() + && edge.target.is_none() + { + for each in taken { + note_shared_error_jump( + &fragment, + &error_label(edge), + &written, + each.dest(), + at_error, + ); + } + } + out.push_str(&fragment); + for each in taken { + out.push_str(&mark_assigned(function, each)); + if let Some(dest) = each.dest() { + written.insert(dest); + } + } + taken_by_read = taken.len() - 1; + continue; + } match at .checked_sub(1) .and_then(|before| block.ops.get(before)) @@ -7291,13 +7634,33 @@ fn emit_op(module: &ModuleIr, function: &Function, op: &Op, error_target: ErrorE } Op::FloatBinary { dest, op, lhs, rhs } => { let (lhs, rhs) = (value_expr(lhs), value_expr(rhs)); + // a division fails only on a zero divisor, and every double it can answer is a + // legal one — so the divisor is tested here and the test jumps to the error + // edge, rather than a sentinel answer being told apart from a real one by + // asking the thread for an exception on every division + let divided = match op { + BinOp::FloorDiv => Some(( + "PyNumber_FloorDivide", + format!("By_FloatFloorDivNonzero({lhs}, {rhs})"), + )), + BinOp::Mod => Some(( + "PyNumber_Remainder", + format!("By_FloatModNonzero({lhs}, {rhs})"), + )), + BinOp::TrueDiv => Some(("PyNumber_TrueDivide", format!("({lhs} / {rhs})"))), + _ => None, + }; + if let Some((operation, quotient)) = divided { + return format!( + " if (BY_UNLIKELY({rhs} == 0.0)) {{ By_ZeroDivision({operation}, 1); goto {}; }}\n{}", + error_label(error_target), + assign_owned(module, function, *dest, "ient) + ); + } let expr = match op { BinOp::Add => format!("({lhs} + {rhs})"), BinOp::Sub => format!("({lhs} - {rhs})"), BinOp::Mul => format!("({lhs} * {rhs})"), - BinOp::FloorDiv => format!("By_FloatFloorDiv({lhs}, {rhs})"), - BinOp::Mod => format!("By_FloatMod({lhs}, {rhs})"), - BinOp::TrueDiv => format!("By_FloatTrueDiv({lhs}, {rhs})"), BinOp::Pow => format!("By_FloatPow({lhs}, {rhs})"), // the bitwise operators have no float form; the frontend routes // them through the object protocol instead @@ -7543,9 +7906,41 @@ fn emit_op(module: &ModuleIr, function: &Function, op: &Op, error_target: ErrorE enter + &body } Op::IntToFloat { dest, src } => { - let expr = format!("By_TaggedToDouble({})", value_expr(src)); - assign_checked(module, function, *dest, &expr, error_target) + let value = value_expr(src); + // a machine integer always has a double, rounded to nearest-even exactly as + // `PyLong_AsDouble` rounds, so there is nothing to fail and nothing to box + if let Some(RType::Primitive(Primitive::Fixed(_))) = operand_type(function, src) { + return assign_owned(module, function, *dest, &format!("(double){value}")); + } + // only an `int` outside the short range reaches cpython, and only that half + // can fail — written apart, so a short never pays for the test + let short = assign_owned( + module, + function, + *dest, + &format!("(double)By_ShortValue({value})"), + ); + let Some(decl) = function.register(*dest) else { + return String::new(); + }; + let slow = format!( + " {{ {ctype} by_t = By_TaggedToDoubleSlow({value});\n \ + if (BY_UNLIKELY(by_t == -1.0 && PyErr_Occurred())) goto {label};\n \ + {target} = by_t; }}\n", + ctype = ctype(module, &decl.ty), + label = error_label(error_target), + target = local(*dest), + ); + format!( + " if (BY_LIKELY(By_IsShort({value}))) {{\n{short} }} else {{\n{slow} }}\n" + ) } + Op::TagShort { dest, src } => assign_owned( + module, + function, + *dest, + &format!("By_ShortFrom((Py_ssize_t){})", value_expr(src)), + ), Op::Box { dest, src } => { // the borrow pass lends only a singleton this way, and a singleton needs no // reference of the frame's own to stay alive @@ -7559,8 +7954,15 @@ fn emit_op(module: &ModuleIr, function: &Function, op: &Op, error_target: ErrorE }; let call = match src_ty { RType::Primitive(Primitive::Int) => format!("By_BoxInt({})", value_expr(src)), - // to the *tagged* representation rather than to an object: an `int` is - // what a machine integer widens to, and that is what `RType::INT` is + // a machine integer widens to the tagged representation, unless what is + // wanted is the object itself, which it is built as directly + RType::Primitive(Primitive::Fixed(_)) + if function + .register(*dest) + .is_some_and(|decl| decl.ty == RType::OBJECT) => + { + format!("PyLong_FromLongLong({})", value_expr(src)) + } RType::Primitive(Primitive::Fixed(_)) => { format!("By_IntFromI64({})", value_expr(src)) } @@ -7594,9 +7996,11 @@ fn emit_op(module: &ModuleIr, function: &Function, op: &Op, error_target: ErrorE target = local(*dest), ); } + if *to == RType::FLOAT { + return unbox_float(function, *dest, &value_expr(src), error_target); + } let call = match to { RType::Primitive(Primitive::Int) => format!("By_UnboxInt({})", value_expr(src)), - RType::Primitive(Primitive::Float) => format!("By_UnboxFloat({})", value_expr(src)), RType::Primitive(Primitive::Bool) => format!("By_UnboxBool({})", value_expr(src)), RType::Primitive(Primitive::None) => format!("By_UnboxNone({})", value_expr(src)), RType::Primitive(Primitive::Str) => format!("By_UnboxStr({})", value_expr(src)), @@ -7609,7 +8013,7 @@ fn emit_op(module: &ModuleIr, function: &Function, op: &Op, error_target: ErrorE }; assign_checked(module, function, *dest, &call, error_target) } - Op::TupleBuild { dest, items } => { + Op::TupleBuild { dest, items, moves } => { let Some(decl) = function.register(*dest) else { return String::new(); }; @@ -7625,8 +8029,21 @@ fn emit_op(module: &ModuleIr, function: &Function, op: &Op, error_target: ErrorE format!("({}){{ {fields} }}", ctype(module, &decl.ty)) }; let mut out = assign_owned(module, function, *dest, &expr); - if let Some(retain) = inc_ref(&decl.ty, &local(*dest)) { - let _ = writeln!(out, " {retain}"); + // a slot given a reference of its own is retained; one handed a register's + // takes it, and the register is emptied where it held it. the emptying + // waits for the whole struct to be stored, because the release the store + // makes of what the destination held can run a finalizer + for (index, item) in items.iter().enumerate() { + let Some(slot) = decl.ty.element(&[index]) else { + continue; + }; + if moves.contains(&index) { + if let Value::Register(id) = item { + let _ = writeln!(out, " {} = {};", local(*id), slot.emptied()); + } + } else if let Some(retain) = inc_ref(slot, &format!("{}.f{index}", local(*dest))) { + let _ = writeln!(out, " {retain}"); + } } out } @@ -7716,7 +8133,8 @@ fn emit_op(module: &ModuleIr, function: &Function, op: &Op, error_target: ErrorE let mut out = String::new(); let _ = writeln!( out, - " {{ Py_ssize_t by_i = By_ArrayIndex((ByArrayHeader *){}, {});", + " {{ Py_ssize_t by_i = {}((ByArrayHeader *){}, {});", + array_index_helper(function, index), value_expr(array), value_expr(index) ); @@ -7743,7 +8161,8 @@ fn emit_op(module: &ModuleIr, function: &Function, op: &Op, error_target: ErrorE let mut out = String::new(); let _ = writeln!( out, - " {{ Py_ssize_t by_i = By_ArrayIndex((ByArrayHeader *){}, {});", + " {{ Py_ssize_t by_i = {}((ByArrayHeader *){}, {});", + array_index_helper(function, index), value_expr(array), value_expr(index) ); @@ -7807,7 +8226,44 @@ fn emit_op(module: &ModuleIr, function: &Function, op: &Op, error_target: ErrorE value_expr(index) } ), - Op::ArrayPush { dest, array, value } => { + Op::ArrayPush { + dest, + array, + value, + length: Some(length), + } => { + let width = element_ctype(module, function, array); + let (a, l) = (value_expr(array), value_expr(length)); + let mut out = String::new(); + // the count is the register's rather than the header's, so the header is never + // stored to and loaded back on every append: only the capacity is read, and it + // changes only when the buffer grows + let _ = writeln!(out, " {{ ByArrayHeader *by_a = (ByArrayHeader *){a};"); + let _ = writeln!( + out, + " if (BY_UNLIKELY({l} >= by_a->cap)) {{ by_a = By_ArrayGrowFull(by_a, sizeof({width})); if (by_a == NULL) goto {}; {a} = ({})by_a; }}", + error_label(error_target), + array_ctype(module, function, array) + ); + let _ = writeln!(out, " {} = 0;", local(*dest)); + let _ = writeln!( + out, + " (({width} *)By_ArrayItems(by_a))[{l}++] = {}; }}", + value_expr(value) + ); + out + } + Op::ArrayStoreLength { array, length } => format!( + " ((ByArrayHeader *){})->len = {};\n", + value_expr(array), + value_expr(length) + ), + Op::ArrayPush { + dest, + array, + value, + length: None, + } => { let width = element_ctype(module, function, array); let mut out = String::new(); let _ = writeln!( @@ -8942,7 +9398,7 @@ fn emit_op(module: &ModuleIr, function: &Function, op: &Op, error_target: ErrorE let mut out = format!( " {{ {} by_m = {place}; {place} = {};\n", ctype(module, ty), - ty.undefined() + ty.emptied() ); out.push_str(&assign_owned(module, function, *dest, "by_m")); out.push_str(" }\n"); @@ -8982,7 +9438,7 @@ fn emit_op(module: &ModuleIr, function: &Function, op: &Op, error_target: ErrorE format!( " {{ {} by_old = {target}; {target} = {}; {release} }}\n", ctype(module, ty), - ty.undefined() + ty.emptied() ) }) } @@ -9018,31 +9474,55 @@ fn emit_op(module: &ModuleIr, function: &Function, op: &Op, error_target: ErrorE error_label(error_target) ), Op::GetIter { dest, src, cursor } => { - let mut out = match cursor { - Some(_) => { - let expr = format!("By_CursorIter({})", value_expr(src)); - assign_checked(module, function, *dest, &expr, error_target) - } - None => { - let expr = format!("By_GetIter({})", value_expr(src)); - assign_checked(module, function, *dest, &expr, error_target) - } + let protocol = match cursor { + Some(_) => format!("By_CursorIter({})", value_expr(src)), + None => format!("By_GetIter({})", value_expr(src)), }; - // the cursor starts a loop at the top of whatever it is walking. it is - // set *here* rather than where the register is declared because one - // register serves every trip through an enclosing loop + // a compiled generator is its own iterator, which is all `PyObject_GetIter` + // would have found out about it + let expr = match stepped_generator(module, function, src) { + Some(class) => format!( + "(BY_LIKELY({value} != NULL && Py_TYPE({value}) == (PyTypeObject *){type_name}_OBJ) ? By_NewRef({value}) : {protocol})", + value = value_expr(src), + type_name = class.type_name(module.name.dotted()), + ), + None => protocol, + }; + let mut out = assign_checked(module, function, *dest, &expr, error_target); + // the cursor starts a loop at the top of whatever it is walking, and says + // whether that is an exact list. it is set *here* rather than where the + // register is declared because one register serves every trip through an + // enclosing loop if let Some(cursor) = cursor { - let _ = writeln!(out, " {} = 0;", local(*cursor)); + let _ = writeln!( + out, + " {} = By_CursorStart({});", + local(*cursor), + local(*dest) + ); } out } Op::IterNext { dest, iter, cursor } => { // a null result is exhaustion *or* failure, so the check has to // consult the exception state rather than the value alone - let expr = match cursor { + let protocol = match cursor { Some(cursor) => format!("By_CursorStep({}, &{})", value_expr(iter), local(*cursor)), None => format!("By_IterNext({})", value_expr(iter)), }; + // a step of a compiled generator calls that type's own step, handing it the + // depth this frame already holds rather than having `tp_iternext` look it up + // again every element — see [`direct_steps`] + let expr = match stepped_generator(module, function, iter) { + Some(class) => format!( + "(BY_LIKELY(Py_TYPE({value}) == (PyTypeObject *){type_name}_OBJ) ? {step}(({struct_name} *){value}, by_depth, 1) : {protocol})", + value = value_expr(iter), + type_name = class.type_name(module.name.dotted()), + struct_name = class.struct_name(module.name.dotted()), + step = generator_step(module, class), + ), + None => protocol, + }; let mut out = assign_owned(module, function, *dest, &expr); let _ = writeln!( out, @@ -9293,6 +9773,52 @@ fn emit_terminator( otherwise.0 ) } + Terminator::MachineStep { + dest, + op, + lhs, + rhs, + fits, + overflows, + } => { + let (builtin, symbol) = match op { + BinOp::Sub => ("__builtin_sub_overflow", "-"), + _ => ("__builtin_add_overflow", "+"), + }; + let (l, r, d) = (value_expr(lhs), value_expr(rhs), local(*dest)); + // the overflow edge reads the operands as they were, and one of them is usually + // `dest` itself: `i = i + 1`. the builtin writes the wrapped result either way, + // and wrapping arithmetic takes it back exactly, so the result is written in + // place and undone on the cold edge rather than kept in a second register on + // every trip + let undo = if lhs == rhs { + // `i = i + i` leaves nothing unchanged to take the result back with + None + } else if *lhs == Value::Register(*dest) { + let inverse = if symbol == "+" { "-" } else { "+" }; + Some(format!( + "{d} = (int64_t)((uint64_t){d} {inverse} (uint64_t){r});" + )) + } else if *rhs == Value::Register(*dest) { + Some(if symbol == "+" { + format!("{d} = (int64_t)((uint64_t){d} - (uint64_t){l});") + } else { + format!("{d} = (int64_t)((uint64_t){l} - (uint64_t){d});") + }) + } else { + None + }; + match undo { + Some(undo) => format!( + " if (BY_UNLIKELY({builtin}({l}, {r}, &{d}))) {{ {undo} goto b{}; }}\n goto b{};\n", + overflows.0, fits.0 + ), + None => format!( + " {{ int64_t by_s; if (BY_UNLIKELY({builtin}({l}, {r}, &by_s))) goto b{}; {d} = by_s; }}\n goto b{};\n", + overflows.0, fits.0 + ), + } + } Terminator::Unreachable => { let mut out = emit_cleanup(function, " ", live, None); let _ = writeln!(out, " return {};", undefined(module, &function.ret)); @@ -9368,6 +9894,142 @@ fn split_int_binary_c( )) } +/// a list element read whose one use is to be unboxed, with the element borrowed from the +/// list across the unbox, or `None` where `ops` is not that shape +/// +/// `a[i]` over a `list[float]` retains the element into a register the unbox reads once and +/// the release straight after lets go of: a retain and a release around a value the list +/// goes on holding. where the container is an exact `list` and the index falls inside it, +/// the element is unboxed straight off the list instead. the list holds it for as long as +/// nothing can run python, and neither the read nor an unbox this allows can — each takes +/// any reference its result needs for itself. +/// +/// every other read, the protocol read and a missed bound among them, runs the three +/// operations exactly as they stood, so what an exception finds held, and when that is let +/// go of, is as it was. `ops` is the read, the unbox and the release, in that order +fn borrowed_element_unbox( + module: &ModuleIr, + function: &Function, + ops: &[Op], + edge: ErrorEdge, +) -> Option { + let [ + read @ Op::GetItem { + dest: element, + container, + index, + }, + unbox @ Op::Unbox { + dest: unboxed, + src: Value::Register(source), + to, + }, + release @ Op::Release { + value: Value::Register(released), + path, + }, + ] = ops + else { + return None; + }; + let temporary = |id: RegisterId| { + function + .register(id) + .is_some_and(|decl| decl.name.is_none() && !decl.borrowed && !decl.may_be_unassigned) + }; + let borrowed = match function.value_type(index) { + Some(RType::Primitive(Primitive::Fixed(_))) => format!( + "By_ListItemBorrowed({}, {})", + value_expr(container), + value_expr(index) + ), + Some(RType::INT) if !holds_the_vararg(function, container) => { + let tagged = value_expr(index); + format!( + "(By_IsShort({tagged}) ? By_ListItemBorrowed({}, (int64_t)By_ShortValue({tagged})) : NULL)", + value_expr(container) + ) + } + _ => return None, + }; + if source != element + || released != element + || !path.is_empty() + || unboxed == element + || !temporary(*element) + || function.register(*unboxed).is_none_or(|decl| decl.borrowed) + || [container, index] + .into_iter() + .any(|operand| *operand == Value::Register(*element)) + { + return None; + } + let helper = unbox_owning_nothing_of_its_source(to)?; + let fast = if *to == RType::FLOAT { + unbox_float(function, *unboxed, "by_e", edge) + } else { + assign_checked(module, function, *unboxed, &format!("{helper}(by_e)"), edge) + }; + let slow = [read, unbox, release] + .into_iter() + .map(|op| emit_op(module, function, op, edge)) + .collect::(); + Some(format!( + " {{ PyObject *by_e = {borrowed};\n\ + \x20 if (BY_LIKELY(by_e != NULL)) {{\n{fast}\ + \x20 }} else {{\n{slow}\ + \x20 }} }}\n" + )) +} + +/// the helper that normalizes and bounds-checks an index into a packed buffer, for the +/// representation the index is in +fn array_index_helper(function: &Function, index: &Value) -> &'static str { + match function.value_type(index) { + Some(RType::Primitive(Primitive::Fixed(_))) => "By_ArrayIndexI64", + _ => "By_ArrayIndex", + } +} + +/// narrow `src` to the double `dest` holds, as a type test with an error edge of its own +/// +/// a double has no value to spare for an error, so the unbox helper's failure is one that +/// has to be confirmed by asking the thread whether an exception is set — on every read, +/// though only a wrong type can fail. testing the type here puts the whole failure on the +/// branch that is never taken +fn unbox_float( + function: &Function, + dest: RegisterId, + src: &str, + error_target: ErrorEdge, +) -> String { + if function.register(dest).is_none() { + return String::new(); + } + format!( + " if (BY_UNLIKELY(!By_IsFloat({src}))) {{ By_UnboxFloatFailed({src}); goto {label}; }}\n \ + {target} = PyFloat_AS_DOUBLE({src});\n", + label = error_label(error_target), + target = local(dest), + ) +} + +/// the unbox helper for a representation whose unbox may read an object it does not own +/// +/// each of these answers a machine value, or a tagged `int` that retains a value behind a +/// pointer for itself, and none of them asks the object for anything — so the object only +/// has to outlive the call. an unbox answering an object reference is not here: its answer +/// is the object itself, which would then need a reference of its own +fn unbox_owning_nothing_of_its_source(to: &RType) -> Option<&'static str> { + Some(match to { + RType::Primitive(Primitive::Int) => "By_UnboxInt", + RType::Primitive(Primitive::Float) => "By_UnboxFloat", + RType::Primitive(Primitive::Bool) => "By_UnboxBool", + RType::Primitive(Primitive::None) => "By_UnboxNone", + _ => return None, + }) +} + fn split_int_binary(op: BinOp, lhs: &Value, rhs: &Value) -> Option<(&'static str, String)> { let (l, r) = (value_expr(lhs), value_expr(rhs)); Some(match op { @@ -11214,6 +11876,7 @@ mod tests { use by_ir::function::{CallConvention, Declined}; use by_ir::rtype::IntWidth; use by_ir::verify::verify; + use std::collections::BTreeSet; fn module_with(function: Function) -> ModuleIr { ModuleIr { @@ -11535,6 +12198,7 @@ mod tests { dest: pushed, array: Value::Register(buffer), value: Value::Bool(true), + length: None, }); builder.terminate(Terminator::Return(Value::Register(pushed))); let c = emit_module(&module_with(builder.finish())); @@ -12881,23 +13545,100 @@ mod tests { } #[test] - fn a_float_error_is_confirmed_against_the_exception_state() { - // BY_FLOAT_ERROR is a legal double, so the sentinel alone proves nothing - let mut builder = FunctionBuilder::new("div", RType::FLOAT); - let a = builder.param("a", RType::FLOAT); - let b = builder.param("b", RType::FLOAT); + fn a_float_division_tests_its_divisor_and_jumps_to_the_error() { + // every double a division answers is a legal one, so the zero divisor is what is + // tested — an error value would have to be confirmed against the thread's + // exception state on every division + for (op, operation, quotient) in [ + (BinOp::TrueDiv, "PyNumber_TrueDivide", "(r0 / r1)"), + ( + BinOp::FloorDiv, + "PyNumber_FloorDivide", + "By_FloatFloorDivNonzero(r0, r1)", + ), + ( + BinOp::Mod, + "PyNumber_Remainder", + "By_FloatModNonzero(r0, r1)", + ), + ] { + let mut builder = FunctionBuilder::new("div", RType::FLOAT); + let a = builder.param("a", RType::FLOAT); + let b = builder.param("b", RType::FLOAT); + let out = builder.temp(RType::FLOAT); + builder.push(Op::FloatBinary { + dest: out, + op, + lhs: Value::Register(a), + rhs: Value::Register(b), + }); + builder.terminate(Terminator::Return(Value::Register(out))); + let c = emit_module(&module_with(builder.finish())); + let expected = format!( + " if (BY_UNLIKELY(r1 == 0.0)) {{ By_ZeroDivision({operation}, 1); goto by_error; }}\n r2 = {quotient};\n" + ); + assert!(c.contains(&expected), "{op:?}: {c}"); + assert!(!c.contains("by_t == BY_FLOAT_ERROR"), "{op:?}: {c}"); + } + } + + #[test] + fn a_float_unbox_is_a_type_test_with_its_own_error_edge() { + let mut builder = FunctionBuilder::new("narrow", RType::FLOAT); + let a = builder.param("a", RType::OBJECT); let out = builder.temp(RType::FLOAT); - builder.push(Op::FloatBinary { + builder.push(Op::Unbox { dest: out, - op: BinOp::TrueDiv, - lhs: Value::Register(a), - rhs: Value::Register(b), + src: Value::Register(a), + to: RType::FLOAT, }); builder.terminate(Terminator::Return(Value::Register(out))); let c = emit_module(&module_with(builder.finish())); - assert!(c.contains( - "if (BY_UNLIKELY(by_t == BY_FLOAT_ERROR && PyErr_Occurred())) goto by_error;" - )); + assert!( + c.contains( + " if (BY_UNLIKELY(!By_IsFloat(r0))) { By_UnboxFloatFailed(r0); goto by_error; }\n \ + r1 = PyFloat_AS_DOUBLE(r0);\n" + ), + "{c}" + ); + } + + #[test] + fn an_int_widened_to_a_float_tests_for_an_error_off_the_short_path() { + let mut builder = FunctionBuilder::new("widen", RType::FLOAT); + let a = builder.param("a", RType::INT); + let out = builder.temp(RType::FLOAT); + builder.push(Op::IntToFloat { + dest: out, + src: Value::Register(a), + }); + builder.terminate(Terminator::Return(Value::Register(out))); + let c = emit_module(&module_with(builder.finish())); + assert!( + c.contains( + " if (BY_LIKELY(By_IsShort(r0))) {\n r1 = (double)By_ShortValue(r0);\n } else {\n \ + { double by_t = By_TaggedToDoubleSlow(r0);\n \ + if (BY_UNLIKELY(by_t == -1.0 && PyErr_Occurred())) goto by_error;\n \ + r1 = by_t; }\n }\n" + ), + "{c}" + ); + } + + #[test] + fn a_machine_integer_widened_to_a_float_is_a_plain_conversion() { + let mut builder = FunctionBuilder::new("widen", RType::FLOAT); + let count = builder.local("count", RType::fixed(by_ir::rtype::IntWidth::I64)); + let out = builder.temp(RType::FLOAT); + builder.assign(count, Value::Fixed(7)); + builder.push(Op::IntToFloat { + dest: out, + src: Value::Register(count), + }); + builder.terminate(Terminator::Return(Value::Register(out))); + let c = emit_module(&module_with(builder.finish())); + assert!(c.contains(" r1 = (double)r0;\n"), "{c}"); + assert!(!c.contains("By_TaggedToDouble"), "{c}"); } #[test] @@ -13494,6 +14235,7 @@ mod tests { builder.push(Op::TupleBuild { dest: out, items: vec![Value::Register(a), Value::Float(1.5)], + moves: BTreeSet::new(), }); builder.terminate(Terminator::Return(Value::Register(out))); let c = emit_module(&module_with(builder.finish())); @@ -13518,6 +14260,7 @@ mod tests { builder.push(Op::TupleBuild { dest: out, items: vec![Value::Register(made), Value::Register(n)], + moves: BTreeSet::new(), }); builder.terminate(Terminator::Return(Value::Register(out))); let mut module = module_with(builder.finish()); @@ -13559,6 +14302,7 @@ mod tests { builder.push(Op::TupleBuild { dest: out, items: vec![Value::Register(point), Value::Register(n)], + moves: BTreeSet::new(), }); builder.terminate(Terminator::Return(Value::Register(out))); builder.finish() diff --git a/crates/by_ir/src/function.rs b/crates/by_ir/src/function.rs index 2d0e0958b6..d271ebf67c 100644 --- a/crates/by_ir/src/function.rs +++ b/crates/by_ir/src/function.rs @@ -525,7 +525,9 @@ pub struct FieldDecl { pub ty: crate::rtype::RType, /// whether this is a closure environment's *shared* cell: a name more than one frame /// writes, which every one of them reads and writes here, and which starts unset. a - /// capture that is only copied in is always set, so it is read without the test + /// capture that is only copied in is always set, so it is read without the test. + /// a generator's state object holds a local some path reads before writing the same + /// way, since a generator expression its body makes reads it from there pub cell: bool, /// the constructor's default for it, `None` where it has none /// @@ -667,6 +669,25 @@ pub struct Resumption { /// iterator it delegates to is kept in. python's `throw` and `close` reach that /// iterator before the frame itself pub delegations: Vec, + /// the suspensions no handler of the frame's own encloses, which `close` finishes + /// without resuming the frame — see [`QuietClose`] + pub quiet_closes: Vec, +} + +/// a suspension where `close` has nothing for the frame to run +/// +/// `close` throws `GeneratorExit` in at the suspension, and where nothing between the +/// suspension and the frame's exit handles an exception — no `try`, no `with`, only the +/// loops whose parked iterators are let go of on the way out — the frame can only unwind +/// and finish. cpython's own `close` makes the same judgement and finishes such a frame +/// without raising anything into it. what the unwinding would have done is these fields +/// cleared, innermost first +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct QuietClose { + /// the value `$state` holds while the frame is suspended there + pub state: i64, + /// the fields the unwinding writes `None` over, in the order it writes them + pub clears: Vec, } /// one suspension of a resumable frame that waits on an inner iterator diff --git a/crates/by_ir/src/ops.rs b/crates/by_ir/src/ops.rs index c47202ec82..da4a81cf3b 100644 --- a/crates/by_ir/src/ops.rs +++ b/crates/by_ir/src/ops.rs @@ -5,6 +5,8 @@ //! exactly one [`Terminator`] — keeping the terminator out of the op list means //! "every block is terminated" is true by construction rather than by check. +use std::collections::BTreeSet; + use crate::rtype::RType; /// index of a register within a [`Function`](crate::function::Function) @@ -659,8 +661,24 @@ pub enum Op { /// rather than an approximation of it. an integer too large to be a float /// raises `OverflowError`, exactly as python does IntToFloat { dest: RegisterId, src: Value }, + /// give a machine integer its tagged representation, where it is proven to lie inside + /// the short range + /// + /// a [`Self::Box`] of a machine integer tests the range and builds an object where it + /// is outside, and the merge of those two arms hides from the C compiler everything it + /// knew about the value — that a counter is never negative, say, which is what lets a + /// `//` or `%` of it drop its sign fix-ups. inside a loop duplicated behind a narrowed + /// bound, the guard that has just held is the proof, and this is written instead + TagShort { dest: RegisterId, src: Value }, /// build a fixed-length tuple from its elements - TupleBuild { dest: RegisterId, items: Vec }, + TupleBuild { + dest: RegisterId, + items: Vec, + /// which items hand their register's reference over rather than having one + /// taken for them, by index into `items`. an index missing from it retains, so + /// the empty set is the plain build every lowering makes + moves: BTreeSet, + }, /// unpack a value into a fixed-length tuple, the way an assignment target list /// does. `starred` is the slot that collects the surplus into a list /// @@ -730,11 +748,20 @@ pub enum Op { index: Value, }, /// append one, growing the buffer when it is full + /// + /// the buffer's register is written in place when a growth moves it. with `length`, + /// the number of elements is the machine register it names rather than the buffer's + /// own header, and the append advances that register in place: a loop that only + /// appends keeps the count to itself, and writes it back with + /// [`Self::ArrayStoreLength`] before anything reads the buffer again ArrayPush { dest: RegisterId, array: Value, value: Value, + length: Option, }, + /// write a count kept in a machine register back into the buffer's header + ArrayStoreLength { array: Value, length: Value }, /// `del container[index]` DeleteItem { dest: RegisterId, @@ -1421,6 +1448,7 @@ impl Op { | Self::Contains { .. } | Self::Identity { .. } | Self::IntToFloat { .. } + | Self::TagShort { .. } | Self::TupleBuild { .. } | Self::Unpack { .. } | Self::CallUnpacked { .. } @@ -1432,6 +1460,7 @@ impl Op { | Self::ArrayLen { .. } | Self::ArrayRead { .. } | Self::ArrayPush { .. } + | Self::ArrayStoreLength { .. } | Self::DeleteItem { .. } | Self::DeleteAttr { .. } | Self::ToTuple { .. } @@ -1559,6 +1588,7 @@ impl Op { | Self::Identity { dest, .. } | Self::FloatObjectBinary { dest, .. } | Self::IntToFloat { dest, .. } + | Self::TagShort { dest, .. } | Self::IntBinary { dest, .. } | Self::FloatBinary { dest, .. } | Self::IntCompare { dest, .. } @@ -1643,6 +1673,7 @@ impl Op { | Self::RaiseObject { .. } | Self::PopHandled { .. } | Self::Release { .. } + | Self::ArrayStoreLength { .. } | Self::Reraise { .. } | Self::LeaveGenerator { .. } | Self::LicenceHolds { .. } @@ -1686,6 +1717,7 @@ impl Op { | Self::Identity { dest, .. } | Self::FloatObjectBinary { dest, .. } | Self::IntToFloat { dest, .. } + | Self::TagShort { dest, .. } | Self::IntBinary { dest, .. } | Self::FloatBinary { dest, .. } | Self::IntCompare { dest, .. } @@ -1770,6 +1802,7 @@ impl Op { | Self::RaiseObject { .. } | Self::PopHandled { .. } | Self::Release { .. } + | Self::ArrayStoreLength { .. } | Self::Reraise { .. } | Self::LeaveGenerator { .. } | Self::LicenceHolds { .. } @@ -1821,6 +1854,7 @@ impl Op { | Self::MatchSlice { sequence: src, .. } | Self::IsSequence { src, .. } | Self::IntToFloat { src, .. } + | Self::TagShort { src, .. } | Self::Unbox { src, .. } | Self::Truthy { src, .. } | Self::Len { src, .. } @@ -1979,7 +2013,13 @@ impl Op { container, index, .. } => vec![container, index], Self::DeleteAttr { receiver, .. } => vec![receiver], - Self::ArrayPush { array, value, .. } => vec![array, value], + Self::ArrayPush { + array, + value, + length, + .. + } => [array, value].into_iter().chain(length.as_ref()).collect(), + Self::ArrayStoreLength { array, length } => vec![array, length], Self::Extend { container, source, .. } => vec![container, source], @@ -2059,6 +2099,7 @@ impl Op { | Self::MatchSlice { sequence: src, .. } | Self::IsSequence { src, .. } | Self::IntToFloat { src, .. } + | Self::TagShort { src, .. } | Self::Unbox { src, .. } | Self::Truthy { src, .. } | Self::Len { src, .. } @@ -2219,7 +2260,13 @@ impl Op { } => vec![container, index], Self::DeleteAttr { receiver, .. } => vec![receiver], Self::ArrayLen { array, .. } => vec![array], - Self::ArrayPush { array, value, .. } => vec![array, value], + Self::ArrayPush { + array, + value, + length, + .. + } => [array, value].into_iter().chain(length.as_mut()).collect(), + Self::ArrayStoreLength { array, length } => vec![array, length], Self::Extend { container, source, .. } => vec![container, source], @@ -2290,6 +2337,21 @@ pub enum Terminator { fits: BlockId, otherwise: BlockId, }, + /// add or subtract two machine integers, taking `overflows` when the result does not + /// fit the machine word + /// + /// a loop counter is a machine integer only inside a copy of its loop, and a step that + /// leaves the word leaves the copy for the loop as written, which carries on in python's + /// arithmetic. so the overflow is an edge rather than a failure, and like + /// [`Self::NarrowShort`] `dest` holds a value only on the `fits` edge + MachineStep { + dest: RegisterId, + op: BinOp, + lhs: Value, + rhs: Value, + fits: BlockId, + overflows: BlockId, + }, /// control cannot reach here. emitted after a raise, and after a body the /// checker proved diverges Unreachable, @@ -2297,7 +2359,7 @@ pub enum Terminator { impl Terminator { /// the blocks control can transfer to - pub(crate) fn successors(&self) -> Vec { + pub fn successors(&self) -> Vec { match self { Self::Goto(target) => vec![*target], Self::Branch { @@ -2308,14 +2370,24 @@ impl Terminator { Self::NarrowShort { fits, otherwise, .. } => vec![*fits, *otherwise], + Self::MachineStep { + fits, overflows, .. + } => vec![*fits, *overflows], Self::Return(_) | Self::Unreachable => Vec::new(), } } /// the register this terminator writes, on the edge that writes one pub fn dest(&self) -> Option { + self.written_on_edge().map(|(dest, _)| dest) + } + + /// the register this terminator writes, and the one edge it holds a value on + pub fn written_on_edge(&self) -> Option<(RegisterId, BlockId)> { match self { - Self::NarrowShort { dest, .. } => Some(*dest), + Self::NarrowShort { dest, fits, .. } | Self::MachineStep { dest, fits, .. } => { + Some((*dest, *fits)) + } Self::Goto(_) | Self::Branch { .. } | Self::Return(_) | Self::Unreachable => None, } } @@ -2325,6 +2397,7 @@ impl Terminator { match self { Self::Branch { cond, .. } => vec![cond], Self::Return(value) | Self::NarrowShort { src: value, .. } => vec![value], + Self::MachineStep { lhs, rhs, .. } => vec![lhs, rhs], Self::Goto(_) | Self::Unreachable => Vec::new(), } } @@ -2334,9 +2407,29 @@ impl Terminator { match self { Self::Branch { cond, .. } => vec![cond], Self::Return(value) | Self::NarrowShort { src: value, .. } => vec![value], + Self::MachineStep { lhs, rhs, .. } => vec![lhs, rhs], Self::Goto(_) | Self::Unreachable => Vec::new(), } } + + /// every edge this terminator can take, mutably + pub fn successors_mut(&mut self) -> Vec<&mut BlockId> { + match self { + Self::Goto(target) => vec![target], + Self::Branch { + then_block, + else_block, + .. + } => vec![then_block, else_block], + Self::NarrowShort { + fits, otherwise, .. + } => vec![fits, otherwise], + Self::MachineStep { + fits, overflows, .. + } => vec![fits, overflows], + Self::Return(_) | Self::Unreachable => Vec::new(), + } + } } #[cfg(test)] diff --git a/crates/by_ir/src/print.rs b/crates/by_ir/src/print.rs index c3204ab072..c92f18e4da 100644 --- a/crates/by_ir/src/print.rs +++ b/crates/by_ir/src/print.rs @@ -365,11 +365,24 @@ fn print_op(function: &Function, op: &Op) -> String { } Op::Box { dest, src } => format!("{} = box {}", name(*dest), value(src)), Op::IntToFloat { dest, src } => format!("{} = float {}", name(*dest), value(src)), + Op::TagShort { dest, src } => format!("{} = tag short {}", name(*dest), value(src)), Op::Unbox { dest, src, to } => { format!("{} = unbox {} as {}", name(*dest), value(src), to) } - Op::TupleBuild { dest, items } => { - let items = items.iter().map(value).collect::>().join(", "); + Op::TupleBuild { dest, items, moves } => { + let items = items + .iter() + .enumerate() + .map(|(index, item)| { + let item = value(item); + if moves.contains(&index) { + format!("move {item}") + } else { + item + } + }) + .collect::>() + .join(", "); format!("{} = ({})", name(*dest), items) } Op::CallUnpacked { @@ -440,8 +453,19 @@ fn print_op(function: &Function, op: &Op) -> String { dest, array, value: v, - } => { - format!("{} = {} push {}", name(*dest), value(array), value(v)) + length, + } => match length { + Some(length) => format!( + "{} = {} push {} counting {}", + name(*dest), + value(array), + value(v), + value(length) + ), + None => format!("{} = {} push {}", name(*dest), value(array), value(v)), + }, + Op::ArrayStoreLength { array, length } => { + format!("{} length = {}", value(array), value(length)) } Op::ToTuple { dest, src } => format!("{} = tuple {}", name(*dest), value(src)), Op::Extend { @@ -911,6 +935,22 @@ fn print_terminator(function: &Function, terminator: &Terminator) -> String { fits.0, otherwise.0 ), + Terminator::MachineStep { + dest, + op, + lhs, + rhs, + fits, + overflows, + } => format!( + "{} = {} {} {} ? b{} : overflow b{}", + register_name(function, *dest), + print_value(function, lhs), + op.symbol(), + print_value(function, rhs), + fits.0, + overflows.0 + ), Terminator::Unreachable => "unreachable".to_string(), } } diff --git a/crates/by_ir/src/rtype.rs b/crates/by_ir/src/rtype.rs index af9bd77bb3..07984921c5 100644 --- a/crates/by_ir/src/rtype.rs +++ b/crates/by_ir/src/rtype.rs @@ -254,6 +254,13 @@ impl RType { ) } + /// whether a value of this type is one reference the ownership passes may hand over + /// or let go of on its own: a single object reference, or a tagged `int`, whose + /// short case owns nothing and whose overflow case owns exactly one `PyLongObject` + pub fn owns_one_reference(&self) -> bool { + self.is_object_reference() || *self == Self::INT + } + /// whether a value of this type is, or holds as an element, a reference to an /// object — something that may have a finalizer, or hold one that does pub fn holds_object_reference(&self) -> bool { @@ -310,6 +317,28 @@ impl RType { } } + /// the value a register holds once the reference it held has been let go of + /// + /// this is [`Self::undefined`] everywhere a representation's unset value costs + /// nothing to release again. a tagged `int`'s does not: `BY_INT_ERROR` is odd, so + /// every later release of the emptied register takes the branch meant for a real + /// `PyLongObject` and calls out of line to drop a `NULL`. a short zero is the same + /// "holds nothing" and its release is the test the straight line already passes + pub fn emptied(&self) -> String { + match self { + Self::Primitive(Primitive::Int) => "BY_INT_EMPTY".to_string(), + Self::Tuple(items) if !items.is_empty() => format!( + "{{ {} }}", + items + .iter() + .map(Self::emptied) + .collect::>() + .join(", ") + ), + other => other.undefined(), + } + } + /// where inside a value of this type an error can be read back off the value /// itself: the field path to the member holding the sentinel, and the pattern /// that member reserves diff --git a/crates/by_ir/src/unbound_locals.rs b/crates/by_ir/src/unbound_locals.rs index f39a079d39..1b1d105f1d 100644 --- a/crates/by_ir/src/unbound_locals.rs +++ b/crates/by_ir/src/unbound_locals.rs @@ -18,7 +18,7 @@ use std::collections::VecDeque; use crate::function::Function; -use crate::ops::{BlockId, Terminator, Value}; +use crate::ops::{BlockId, Value}; /// flag every local this function may read before writing /// @@ -85,10 +85,7 @@ fn unassigned_reads(function: &Function) -> Vec { } // a handler is entered from *before* any of this block's writes: the very // first operation can be the one that failed - let narrowed = match &block.terminator { - Terminator::NarrowShort { dest, fits, .. } => Some((*dest, *fits)), - _ => None, - }; + let narrowed = block.terminator.written_on_edge(); let edges = block .terminator .successors() diff --git a/crates/by_ir/src/verify.rs b/crates/by_ir/src/verify.rs index 461c6353cc..4db0376f89 100644 --- a/crates/by_ir/src/verify.rs +++ b/crates/by_ir/src/verify.rs @@ -12,7 +12,7 @@ use std::collections::{HashMap, HashSet, VecDeque}; use std::fmt; use crate::function::{Function, ModuleIr}; -use crate::ops::{BlockId, Op, RegisterId, Terminator, UnaryOp, Value}; +use crate::ops::{BinOp, BlockId, Op, RegisterId, Terminator, UnaryOp, Value}; use crate::rtype::{IntWidth, Primitive, RType}; /// something wrong with a function's BIR @@ -628,6 +628,17 @@ impl Verifier<'_> { /// an attribute's receiver is an object, or an instance of an emitted class, which /// reaches `PyObject *` by a cast + /// a checked index into a packed buffer: a tagged `int`, or a machine integer a + /// counter holds + fn expect_index(&mut self, block: BlockId, index: &Value) { + if !matches!( + self.operand_type(block, index), + Some(RType::Primitive(Primitive::Fixed(_))) + ) { + self.expect(block, index, &RType::INT, "an array index"); + } + } + fn expect_attribute_receiver(&mut self, block: BlockId, receiver: &Value) { if let Some(actual) = self.operand_type(block, receiver) && !matches!(actual, RType::Instance { .. }) @@ -1016,6 +1027,7 @@ impl Verifier<'_> { Op::IntToFloat { dest, src } => { if let Some(src_ty) = self.operand_type(block, src) && src_ty != RType::INT + && !matches!(src_ty, RType::Primitive(Primitive::Fixed(_))) { self.error( Some(block), @@ -1024,6 +1036,15 @@ impl Verifier<'_> { } self.expect_dest(block, *dest, &RType::FLOAT, "int-to-float"); } + Op::TagShort { dest, src } => { + if !matches!( + self.operand_type(block, src), + Some(RType::Primitive(Primitive::Fixed(_))) + ) { + self.error(Some(block), "tag short tags a machine integer"); + } + self.expect_dest(block, *dest, &RType::INT, "tag short"); + } Op::Box { dest, src } => { // box *widens to* `object`, from either an unboxed value or a // known-class object. widening something already `object` is a @@ -1048,11 +1069,20 @@ impl Verifier<'_> { // that is the one an `int` has everywhere else in the IR, and going // straight to a `PyObject` would give the counter a representation no // consumer of an `int` accepts - let widened = match self.operand_type(block, src) { - Some(RType::Primitive(Primitive::Fixed(_))) => RType::INT, - _ => RType::OBJECT, - }; - self.expect_dest(block, *dest, &widened, "box"); + // + // a counter whose boxed value only ever becomes an object is built as one + // directly, so that is the other destination a machine integer may have + match self.operand_type(block, src) { + Some(RType::Primitive(Primitive::Fixed(_))) + if self + .function + .register(*dest) + .is_some_and(|decl| decl.ty == RType::OBJECT) => {} + Some(RType::Primitive(Primitive::Fixed(_))) => { + self.expect_dest(block, *dest, &RType::INT, "box"); + } + _ => self.expect_dest(block, *dest, &RType::OBJECT, "box"), + } } Op::Unbox { dest, src, to } => { // unbox *narrows from* `object` — to an unboxed representation, or @@ -1067,7 +1097,29 @@ impl Verifier<'_> { } self.expect_dest(block, *dest, to, "unbox"); } - Op::TupleBuild { dest, items } => { + Op::TupleBuild { dest, items, moves } => { + for index in moves { + // the same rule `Op::Move` follows: only a temporary the frame owns + // has a reference of its own to hand to the slot + let hands_over = matches!( + items.get(*index), + Some(Value::Register(id)) + if id.index() >= self.function.param_count + && self.function.register(*id).is_some_and(|decl| { + !decl.borrowed + && decl.name.is_none() + && decl.ty.owns_one_reference() + }) + ); + if !hands_over { + self.error( + Some(block), + format!( + "a tuple build moving item {index}, which owns no reference to hand over" + ), + ); + } + } let declared = self .function .register(*dest) @@ -1151,7 +1203,7 @@ impl Verifier<'_> { let Some(element) = self.array_element(block, array) else { return; }; - self.expect(block, index, &RType::INT, "an array index"); + self.expect_index(block, index); self.expect_dest(block, *dest, &element, "an array read"); } Op::ArraySet { @@ -1163,7 +1215,7 @@ impl Verifier<'_> { let Some(element) = self.array_element(block, array) else { return; }; - self.expect(block, index, &RType::INT, "an array index"); + self.expect_index(block, index); self.expect(block, value, &element, "an array write"); self.expect_dest(block, *dest, &RType::BIT, "an array write"); } @@ -1208,13 +1260,35 @@ impl Verifier<'_> { } self.expect_dest(block, *dest, &element, "an unchecked array read"); } - Op::ArrayPush { dest, array, value } => { + Op::ArrayPush { + dest, + array, + value, + length, + } => { + if let Some(length) = length { + self.expect( + block, + length, + &RType::fixed(IntWidth::I64), + "an array append", + ); + } let Some(element) = self.array_element(block, array) else { return; }; self.expect(block, value, &element, "an array append"); self.expect_dest(block, *dest, &RType::BIT, "an array append"); } + Op::ArrayStoreLength { array, length } => { + self.expect( + block, + length, + &RType::fixed(IntWidth::I64), + "an array's length", + ); + self.array_element(block, array); + } Op::ToTuple { dest, src } => { self.expect(block, src, &RType::OBJECT, "tuple"); self.expect_dest(block, *dest, &RType::OBJECT, "tuple"); @@ -1479,7 +1553,7 @@ impl Verifier<'_> { && self.function.register(*id).is_some_and(|decl| { !decl.borrowed && decl.name.is_none() - && (decl.ty == RType::INT || decl.ty.is_object_reference()) + && decl.ty.owns_one_reference() }) } _ => false, @@ -1686,10 +1760,7 @@ impl Verifier<'_> { let owns = match value { Value::Register(id) => self.function.register(*id).is_some_and(|decl| { !decl.borrowed - && decl - .ty - .element(path) - .is_some_and(RType::is_object_reference) + && decl.ty.element(path).is_some_and(RType::owns_one_reference) }), _ => false, }; @@ -1712,7 +1783,7 @@ impl Verifier<'_> { .register(*id) .filter(|decl| !decl.borrowed && decl.name.is_none()) .and_then(|decl| decl.ty.element(path)) - .filter(|ty| **ty == RType::INT || ty.is_object_reference()) + .filter(|ty| ty.owns_one_reference()) .cloned(), _ => None, }; @@ -1920,6 +1991,20 @@ impl Verifier<'_> { ); } } + Terminator::MachineStep { + dest, op, lhs, rhs, .. + } => { + let fixed = RType::fixed(IntWidth::I64); + if !matches!(op, BinOp::Add | BinOp::Sub) { + self.error( + Some(block), + format!("a machine step cannot `{}`", op.symbol()), + ); + } + self.expect(block, lhs, &fixed, "a machine step"); + self.expect(block, rhs, &fixed, "a machine step"); + self.expect_dest(block, *dest, &fixed, "a machine step"); + } Terminator::Goto(_) | Terminator::Unreachable => {} } } @@ -1948,10 +2033,12 @@ impl Verifier<'_> { continue; }; for op in &block.ops { - if let Some(dest) = op.dest() - && dest.index() < state.len() - { - state[dest.index()] = true; + // a loop's cursor is written in place rather than as a destination: taking + // the iterator starts it, and a generator parks it across a suspension + for dest in op.dest().into_iter().chain(op.loop_cursor()) { + if let Some(slot) = state.get_mut(dest.index()) { + *slot = true; + } } } // a handler is entered from *before* any of this block's writes, because @@ -1960,10 +2047,7 @@ impl Verifier<'_> { // a narrowing terminator writes its destination on one edge only: the // other is the path where the value did not fit and there is nothing to // have written - let narrowed = match &block.terminator { - Terminator::NarrowShort { dest, fits, .. } => Some((*dest, *fits)), - _ => None, - }; + let narrowed = block.terminator.written_on_edge(); let edges = block .terminator .successors() @@ -2022,10 +2106,10 @@ impl Verifier<'_> { self.check_read(id, operand, &state); } } - if let Some(dest) = op.dest() - && dest.index() < state.len() - { - state[dest.index()] = true; + for dest in op.dest().into_iter().chain(op.loop_cursor()) { + if let Some(slot) = state.get_mut(dest.index()) { + *slot = true; + } } } for operand in block.terminator.operands() { @@ -2082,6 +2166,7 @@ mod tests { use crate::builder::FunctionBuilder; use crate::function::{BasicBlock, CallConvention, RegisterDecl}; use crate::ops::{BinOp, CmpOp, Concatenation, Mutation}; + use std::collections::BTreeSet; fn reg(name: &str, ty: RType) -> RegisterDecl { RegisterDecl { @@ -2507,6 +2592,7 @@ mod tests { Value::Register(RegisterId(0)), Value::Register(RegisterId(1)), ], + moves: BTreeSet::new(), }); entry.ops.push(Op::TupleGet { dest: RegisterId(3), diff --git a/crates/by_irbuild/src/generators.rs b/crates/by_irbuild/src/generators.rs index 2ff28b594f..7384543b71 100644 --- a/crates/by_irbuild/src/generators.rs +++ b/crates/by_irbuild/src/generators.rs @@ -89,8 +89,15 @@ pub(crate) const THROWN_FIELD: &str = "$thrown"; /// only an *async generator* has two: a `yield` produces an item for the awaitable /// `__anext__` handed back, and an `await` suspends and has to reach the event loop /// instead. one `resume` returns for both, so the driver reads this to tell them -/// apart. `1` is a yield +/// apart. `1` is a yield. no other frame has the field, since nothing would read it — +/// see [`records_suspension_kinds`] pub(crate) const KIND_FIELD: &str = "$kind"; + +/// whether a frame's suspensions say which kind they are, which only an async +/// generator's driver asks +pub(crate) fn records_suspension_kinds(function: &ast::StmtFunctionDef) -> bool { + function.is_async && is_generator(&function.body) +} /// the method the iterator protocol drives pub(crate) const RESUME_METHOD: &str = "$resume"; @@ -206,8 +213,10 @@ pub(crate) fn state_names(function: &ast::StmtFunctionDef, locals: &[String]) -> STATE_FIELD.to_string(), SENT_FIELD.to_string(), THROWN_FIELD.to_string(), - KIND_FIELD.to_string(), ]; + if records_suspension_kinds(function) { + out.push(KIND_FIELD.to_string()); + } // one per `for`, because the *iterator* is the value most likely to be live // across a suspension and it has no source name to hold it for index in 0..for_loops(&function.body) { @@ -337,6 +346,85 @@ pub(crate) struct Resumption { pub(crate) suspend: BlockId, /// the block `$resume` re-enters at pub(crate) resume: BlockId, + /// the block raising what `throw` or `close` handed the frame, where the suspension + /// waits on nothing else + pub(crate) raising: Option, +} + +/// the suspensions `close` can finish without resuming the frame, and what each unwinds +/// +/// the exception raised at a suspension leaves through its block's error target. what is +/// followed from there is the unwinding a loop over a parked iterator lowers to — take +/// the exception, write `None` over the field, raise it again — and nothing else: a +/// handler that tests the exception, calls anything or suspends again is one the frame +/// runs for itself, and the suspension is left to the resumption +pub(crate) fn quiet_closes( + function: &by_ir::function::Function, + resumptions: &[Resumption], +) -> Vec { + resumptions + .iter() + .filter_map(|point| { + let raising = function.block(point.raising?)?; + let mut clears = Vec::new(); + let mut handler = raising.error_target; + let mut visited = 0; + while let Some(at) = handler { + visited += 1; + if visited > function.blocks.len() { + return None; + } + let block = function.block(at)?; + clears.extend(unwound_field(block)?); + handler = block.error_target; + } + Some(by_ir::function::QuietClose { + state: point.state, + clears, + }) + }) + .collect() +} + +/// the field an unwinding block writes `None` over on its way to raising what it caught +/// again, or `None` for a block that is not only that +fn unwound_field(block: &by_ir::function::BasicBlock) -> Option> { + let [ + ops @ .., + Op::Reraise { + value: Value::Register(raised), + }, + ] = block.ops.as_slice() + else { + return None; + }; + if !matches!(block.terminator, Terminator::Unreachable) { + return None; + } + let mut fetched = None; + let mut nothing = BTreeSet::new(); + let mut cleared = Vec::new(); + for op in ops { + match op { + Op::FetchException { dest } if fetched.is_none() => fetched = Some(*dest), + Op::Line { .. } => {} + Op::Release { .. } => {} + Op::Box { + dest, + src: Value::None, + } => { + nothing.insert(*dest); + } + Op::SetField { + receiver: Value::Register(RegisterId(0)), + field, + value: Value::Register(value), + .. + } if nothing.contains(value) => cleared.push(field.clone()), + _ => return None, + } + } + (fetched == Some(*raised)).then_some(cleared) } /// the field a register is parked in while the frame is suspended @@ -558,6 +646,24 @@ fn live_in( if let Some(dest) = op.dest() { live.remove(&dest); } + // a loop's cursor is neither a destination nor an operand: taking the + // iterator starts it at zero, and a step reads it and writes it back, so + // the position a suspended loop has reached is live across the suspension + match op { + Op::GetIter { + cursor: Some(cursor), + .. + } => { + live.remove(cursor); + } + Op::IterNext { + cursor: Some(cursor), + .. + } => { + live.insert(*cursor); + } + _ => {} + } // `del x` reads its destination before leaving it unbound, so a name // deleted after a suspension is live across it — which is what puts it // in the parked set, where the check below refuses it because the byte diff --git a/crates/by_irbuild/src/lib.rs b/crates/by_irbuild/src/lib.rs index 0bad488743..4da1519889 100644 --- a/crates/by_irbuild/src/lib.rs +++ b/crates/by_irbuild/src/lib.rs @@ -1322,17 +1322,28 @@ fn lower_generator( let mut fields: Vec = names .into_iter() .map(|name| by_ir::function::FieldDecl { - cell: false, + // a name some path reads before writing starts unset, and every frame reading + // it — a generator expression the body makes reaches it through `$outer` — + // reads it with the test + cell: name != generators::STATE_FIELD && !assigned.contains(&name), optional: false, defaulted_by: None, - // the state number is never unset — the constructor writes 0 — so it is - // an unboxed int, and the dispatch reads it without a narrowing - ty: if name == generators::STATE_FIELD || name == generators::KIND_FIELD { + // the state number is never unset — the constructor writes 0 — and never a + // python value at all: it is a machine integer the dispatch compares + // directly, and the runtime reads and writes it the same way + ty: if name == generators::STATE_FIELD { + RType::fixed(by_ir::rtype::IntWidth::I64) + } else if name == generators::KIND_FIELD { RType::INT } else if assigned.contains(&name) { representation(&name).unwrap_or(RType::OBJECT) } else { - RType::OBJECT + // a name some path reads before writing — a loop's target, say — is a + // cell, which starts unset. it keeps the local's own representation where + // that has an unset value of its own, as a shared cell does + representation(&name) + .filter(has_an_unset_value) + .unwrap_or(RType::OBJECT) }, name, default: None, @@ -1450,7 +1461,7 @@ fn lower_generator( outer: None, per_closure: None, }); - let (resume, parked, delegations) = lower_resume( + let (resume, parked, delegations, quiet_closes) = lower_resume( Unit { layouts: &layouts_with_state, methods: &methods_with_state, @@ -1524,6 +1535,7 @@ fn lower_generator( }, qualname: qualname.to_string(), delegations, + quiet_closes, }), fields, decorators: Vec::new(), @@ -1642,7 +1654,8 @@ fn lower_generator_constructor( for field in fields { values.push(match field.name.as_str() { // the machine starts at resumption point 0, and nothing has been sent - generators::STATE_FIELD | generators::KIND_FIELD => Some(Value::Int(0)), + generators::STATE_FIELD => Some(Value::Fixed(0)), + generators::KIND_FIELD => Some(Value::Int(0)), generators::SENT_FIELD => Some(Value::None), name => registers.get(name).or(seeded.get(name)).map(|&id| { // the field only needs boxing when it *is* an object — a @@ -1678,6 +1691,16 @@ fn lower_generator_constructor( /// the body, as a method that resumes at `$state` and returns the next yielded value /// /// also reports the fields the parked registers took, which only the lowered body says +/// what lowering a resumable frame's body hands back: the resume function, the fields its +/// suspensions park registers in, the delegations it waits on, and the closes it can finish +/// without resuming +type ResumeLowering = ( + Function, + Vec, + Vec, + Vec, +); + fn lower_resume( unit: Unit<'_>, function: &ast::StmtFunctionDef, @@ -1686,11 +1709,7 @@ fn lower_resume( fields: &[by_ir::function::FieldDecl], assigned: &HashSet, environment: Option, -) -> Lowered<( - Function, - Vec, - Vec, -)> { +) -> Lowered { let Unit { db, model, @@ -1784,6 +1803,7 @@ fn lower_resume( resumptions: Vec::new(), iterators: 0, delegations: Vec::new(), + records_kinds: generators::records_suspension_kinds(function), }), delegations: 0, contexts: 0, @@ -1819,7 +1839,9 @@ fn lower_resume( // the dispatch: `$state` against each resumption point, falling through to // exhausted. a chain of branches *is* a jump table, and the C compiler builds it lowering.builder.switch_to(Function::entry()); - let narrowed = lowering.builder.temp(RType::INT); + let narrowed = lowering + .builder + .temp(RType::fixed(by_ir::rtype::IntWidth::I64)); lowering.builder.push(Op::GetField { dest: narrowed, receiver: Value::Register(receiver), @@ -1832,7 +1854,7 @@ fn lower_resume( receiver: Value::Register(receiver), class: class.to_string(), field: generators::STATE_FIELD.to_string(), - value: Value::Int(generators::RUNNING_STATE), + value: Value::Fixed(generators::RUNNING_STATE), moves: false, present: false, }); @@ -1844,7 +1866,7 @@ fn lower_resume( dest: matched, op: CmpOp::Eq, lhs: Value::Register(narrowed), - rhs: Value::Int(*value), + rhs: Value::Fixed(*value), }); let next = if index + 1 == targets.len() { exhausted @@ -1870,7 +1892,7 @@ fn lower_resume( receiver: Value::Register(receiver), class: class.to_string(), field: generators::STATE_FIELD.to_string(), - value: Value::Int(-1), + value: Value::Fixed(-1), moves: false, present: false, }); @@ -1886,9 +1908,10 @@ fn lower_resume( let mut lowered = lowering.builder.finish(); let parked = generators::park_live_registers(&mut lowered, class, &resumptions)?; + let quiet_closes = generators::quiet_closes(&lowered, &resumptions); lowered.owner = Some(class.to_string()); lowered.exported = false; - Ok((lowered, parked, delegations)) + Ok((lowered, parked, delegations, quiet_closes)) } /// a lowering that does not verify is a *decline*, not a build failure @@ -8290,6 +8313,8 @@ struct Generator { iterators: usize, /// the suspensions lowered so far that wait inside a delegation delegations: Vec, + /// whether each suspension writes its kind — see [`generators::KIND_FIELD`] + records_kinds: bool, } /// the closure environment a frame allocates @@ -10389,7 +10414,7 @@ impl Lowering<'_, '_> { receiver: Value::Register(RegisterId(0)), class, field: generators::STATE_FIELD.to_string(), - value: Value::Int(-1), + value: Value::Fixed(-1), moves: false, present: false, }); @@ -10508,7 +10533,7 @@ impl Lowering<'_, '_> { Location::Place(Place::Register(dest)) => { let dest = *dest; let declared = self.register_type(dest)?; - let result_ty = binary_result(op, &target_ty, &rhs_ty); + let result_ty = binary_result(op, (&target, &target_ty), (&rhs, &rhs_ty)); // writing the result straight back is only sound when the // operation produces what the register holds. `s += "%s" % x` // on a `str` does not: `%` goes through the object protocol, so @@ -10536,7 +10561,7 @@ impl Lowering<'_, '_> { // anything else is read, then written: a temporary keeps the read // from being released by the write other => { - let result_ty = binary_result(op, &target_ty, &rhs_ty); + let result_ty = binary_result(op, (&target, &target_ty), (&rhs, &rhs_ty)); let temp = self.builder.temp(result_ty.clone()); self.emit_binary( temp, @@ -12806,20 +12831,17 @@ impl Lowering<'_, '_> { Ok(()) } - /// the register a `for` loop keeps its position in, where the loop can hold one + /// the register a `for` loop keeps its position in /// /// with a cursor, an exact list stands in for its own iterator and a step is a /// bounds test and a load rather than a call through `tp_iternext`. /// - /// a generator's loop cannot have one. it parks its iterator in a field because - /// no register survives a `yield`, and the cursor is a register: a resumed frame - /// would start again from whatever an unset one holds. `By_CursorStep` would - /// still refuse to read outside the list, but the loop would silently start over - fn loop_cursor(&mut self) -> Option { - if self.generator.is_some() { - return None; - } - Some(self.builder.temp(RType::fixed(by_ir::rtype::IntWidth::I64))) + /// a generator's loop has one too. no register survives a `yield`, so a cursor live + /// across one is parked in a field of its own and read back at the resumption, as + /// every other register crossing a suspension is — see + /// [`generators::park_live_registers`], whose liveness counts the cursor a step reads + fn loop_cursor(&mut self) -> RegisterId { + self.builder.temp(RType::fixed(by_ir::rtype::IntWidth::I64)) } fn for_over_iterable(&mut self, node: &ast::StmtFor) -> Lowered<()> { @@ -12843,7 +12865,7 @@ impl Lowering<'_, '_> { // the two halves are built together and share one decision, because a `GetIter` // that handed back a list to an `IterNext` with no cursor would step it through // the protocol — and a list is not its own iterator, so that never ends - let cursor = self.loop_cursor(); + let cursor = Some(self.loop_cursor()); self.builder.push(Op::GetIter { dest: iterator, src: boxed, @@ -13708,7 +13730,11 @@ impl Lowering<'_, '_> { items.push(self.coerce(value, &ty, slot)?); } let dest = self.builder.temp(RType::Tuple(slots.into())); - self.builder.push(Op::TupleBuild { dest, items }); + self.builder.push(Op::TupleBuild { + dest, + items, + moves: BTreeSet::new(), + }); Ok(Value::Register(dest)) } @@ -14310,20 +14336,22 @@ impl Lowering<'_, '_> { return Err(Decline::new("a suspension outside a generator")); }; let class = generator.class.clone(); - self.builder.push(Op::SetField { - receiver: Value::Register(RegisterId(0)), - class: class.clone(), - field: generators::KIND_FIELD.to_string(), - value: Value::Int(kind as i64), - moves: false, - present: false, - }); + if generator.records_kinds { + self.builder.push(Op::SetField { + receiver: Value::Register(RegisterId(0)), + class: class.clone(), + field: generators::KIND_FIELD.to_string(), + value: Value::Int(kind as i64), + moves: false, + present: false, + }); + } let state = i64::try_from(generator.resumptions.len()).unwrap_or(i64::MAX - 1) + 1; self.builder.push(Op::SetField { receiver: Value::Register(RegisterId(0)), class: class.clone(), field: generators::STATE_FIELD.to_string(), - value: Value::Int(state), + value: Value::Fixed(state), moves: false, present: false, }); @@ -14375,6 +14403,7 @@ impl Lowering<'_, '_> { state, suspend: suspend_at, resume: resume_at, + raising: None, }); if let Some(Place::Field { name, .. }) = delegated.as_ref().map(|d| d.iterator) { generator.delegations.push(by_ir::function::Delegation { @@ -14401,6 +14430,14 @@ impl Lowering<'_, '_> { let settled = self.builder.new_block(); let raising = self.builder.new_block(); let continuing = self.builder.new_block(); + if delegated.is_none() + && let Some(point) = self + .generator + .as_mut() + .and_then(|generator| generator.resumptions.last_mut()) + { + point.raising = Some(raising); + } self.builder.terminate(Terminator::Branch { cond: Value::Register(quiet), then_block: continuing, @@ -15411,7 +15448,7 @@ impl Lowering<'_, '_> { if let Some(call) = self.operator_dunder(&lhs_ty, &rhs_ty, binary_dunder(op)) { return self.direct_operator(call, (lhs, &lhs_ty), (rhs, &rhs_ty)); } - let mut result_ty = binary_result(op, &lhs_ty, &rhs_ty); + let mut result_ty = binary_result(op, (&lhs, &lhs_ty), (&rhs, &rhs_ty)); // a double meeting an object is an `object` result as far as the // representations go, but the pair may still be provably a float — and // then the object can be tested rather than the double boxed to reach it @@ -15456,7 +15493,7 @@ impl Lowering<'_, '_> { if !matches!( op, BinOp::BitAnd | BinOp::BitOr | BinOp::BitXor | BinOp::Shl | BinOp::Shr - ) => + ) && !power_may_be_complex(op, (&lhs, lhs_ty), (&rhs, rhs_ty)) => { self.builder.push(Op::FloatBinary { dest, op, lhs, rhs }); } @@ -15470,7 +15507,7 @@ impl Lowering<'_, '_> { if !matches!( op, BinOp::BitAnd | BinOp::BitOr | BinOp::BitXor | BinOp::Shl | BinOp::Shr - ) => + ) && !power_may_be_complex(op, (&lhs, lhs_ty), (&rhs, rhs_ty)) => { let lhs = self.widen_to_float(lhs, lhs_ty); let rhs = self.widen_to_float(rhs, rhs_ty); @@ -15913,6 +15950,7 @@ impl Lowering<'_, '_> { dest: status, array: receiver, value, + length: None, }); // `list.append` answers `None`, and a statement discards it return Ok((Value::None, RType::NONE)); @@ -16606,7 +16644,7 @@ impl Lowering<'_, '_> { let (iterable, iterable_ty) = self.expression(&generator.iter)?; let boxed = self.widen_to_object(iterable, &iterable_ty); let iterator = self.builder.temp(RType::OBJECT); - let cursor = self.loop_cursor(); + let cursor = Some(self.loop_cursor()); self.builder.push(Op::GetIter { dest: iterator, src: boxed, @@ -16904,6 +16942,7 @@ impl Lowering<'_, '_> { dest: status, array: Value::Register(accumulator), value, + length: None, }); } Comprehension::List(element) | Comprehension::Set(element) => { @@ -18585,7 +18624,11 @@ impl Lowering<'_, '_> { items.push(self.narrow_answer(Value::Register(item), slot)?); } let built = self.builder.temp(to.clone()); - self.builder.push(Op::TupleBuild { dest: built, items }); + self.builder.push(Op::TupleBuild { + dest: built, + items, + moves: BTreeSet::new(), + }); Ok(Value::Register(built)) } @@ -18871,7 +18914,11 @@ fn narrowable(rtype: &RType) -> bool { /// /// `/` between two ints is a float, and anything that is not a matched pair of /// unboxed numbers goes through the object protocol and yields an object -fn binary_result(op: BinOp, lhs: &RType, rhs: &RType) -> RType { +fn binary_result(op: BinOp, lhs: (&Value, &RType), rhs: (&Value, &RType)) -> RType { + if power_may_be_complex(op, lhs, rhs) { + return RType::OBJECT; + } + let ((_, lhs), (_, rhs)) = (lhs, rhs); let bitwise = matches!( op, BinOp::BitAnd | BinOp::BitOr | BinOp::BitXor | BinOp::Shl | BinOp::Shr @@ -18902,6 +18949,27 @@ fn binary_result(op: BinOp, lhs: &RType, rhs: &RType) -> RType { RType::OBJECT } +/// whether `**` over a pair a double operation would take could answer with a complex +/// number, which a double cannot hold +/// +/// python gives one for a negative base to a finite, fractional power. an `int` exponent +/// is integral whatever its value, and so is a literal one that is integral, infinite or a +/// nan; a base that is a literal and not negative never reaches the question +fn power_may_be_complex(op: BinOp, lhs: (&Value, &RType), rhs: (&Value, &RType)) -> bool { + let ((lhs, lhs_ty), (rhs, rhs_ty)) = (lhs, rhs); + if op != BinOp::Pow || !(*lhs_ty == RType::FLOAT || *rhs_ty == RType::FLOAT) { + return false; + } + let integral_exponent = *rhs_ty == RType::INT + || matches!(rhs, Value::Float(exponent) if !exponent.is_finite() || exponent.fract() == 0.0); + let real_base = match lhs { + Value::Float(base) => base.is_nan() || *base >= 0.0, + Value::Int(base) => *base >= 0, + _ => false, + }; + !(integral_exponent || real_base) +} + fn compare_op(op: AstCmpOp) -> Lowered { Ok(match op { AstCmpOp::Eq => CmpOp::Eq, diff --git a/crates/by_irbuild/src/tests.rs b/crates/by_irbuild/src/tests.rs index 3f5da4ba39..88a105930a 100644 --- a/crates/by_irbuild/src/tests.rs +++ b/crates/by_irbuild/src/tests.rs @@ -7461,11 +7461,17 @@ def counted(n: int) -> object: ); // the state number and the two protocol fields have fixed roles; a local - // takes its own representation where it is definitely assigned + // takes its own representation where it is definitely assigned. the state + // number is a machine integer, never a python value let field = |name: &str| state.fields.iter().find(|f| f.name == name); - assert_eq!(field("$state").map(|f| f.ty.clone()), Some(RType::INT)); + assert_eq!( + field("$state").map(|f| f.ty.clone()), + Some(RType::fixed(by_ir::rtype::IntWidth::I64)) + ); assert_eq!(field("$sent").map(|f| f.ty.clone()), Some(RType::OBJECT)); assert_eq!(field("$thrown").map(|f| f.ty.clone()), Some(RType::OBJECT)); + // only an async generator's driver asks which kind of suspension was made + assert_eq!(field("$kind"), None); assert_eq!(field("n").map(|f| f.ty.clone()), Some(RType::INT)); assert_eq!(field("i").map(|f| f.ty.clone()), Some(RType::INT)); @@ -7517,6 +7523,7 @@ fn a_value_that_has_to_survive_a_suspension_takes_a_field() { state: 1, suspend: suspend_at, resume: resume_at, + raising: None, }], ) .expect("a live register takes a field"); @@ -7582,6 +7589,7 @@ fn a_parked_value_that_may_be_unassigned_declines() { state: 1, suspend: suspend_at, resume: resume_at, + raising: None, }], ) .expect_err("a maybe-unassigned local must be declined"); @@ -7934,7 +7942,7 @@ async def chained(awaitable: object) -> object: fn an_async_generator_presents_the_async_iteration_surface() { // `__aiter__`/`__anext__` rather than `__await__`: one resume method drives all // three surfaces, and which one a state class presents is the only difference - let surface = with_source( + let (surface, kinds) = with_source( "\ async def both(n: int) -> object: yield n @@ -7942,15 +7950,19 @@ async def both(n: int) -> object: |db, env, model, suite| { let module = crate::build_module(db, env, model, suite, "app", crate::Language::BasedPython); - module - .classes - .iter() - .find(|class| class.name == "both$gen") - .and_then(|class| class.resume.as_ref()) - .map(|resume| resume.surface) + let class = module.classes.iter().find(|class| class.name == "both$gen"); + ( + class + .and_then(|class| class.resume.as_ref()) + .map(|resume| resume.surface), + // and its driver tells a `yield` from an `await` by the kind each + // suspension writes + class.is_some_and(|class| class.fields.iter().any(|field| field.name == "$kind")), + ) }, ); assert_eq!(surface, Some(by_ir::function::Surface::AsyncGenerator)); + assert!(kinds); } #[test] @@ -8165,6 +8177,56 @@ def counted(n: int) -> object: ); } +#[test] +fn a_suspension_no_handler_encloses_closes_quietly() { + // a `yield` inside nothing but loops is closed by letting the loops' iterators go, and + // one a `try` encloses is closed by running the frame + with_source( + "\ +def quiet(rows: list[list[int]]) -> object: + for row in rows: + for x in row: + yield x + yield 0 + +def handled(rows: list[int]) -> object: + for x in rows: + try: + yield x + except ValueError: + pass +", + |db, env, model, suite| { + let module = + crate::build_module(db, env, model, suite, "app", crate::Language::BasedPython); + assert!(module.declined.is_empty(), "{:?}", module.declined); + let closes = |name: &str| { + module + .classes + .iter() + .find(|class| class.name == name) + .and_then(|class| class.resume.as_ref()) + .map(|resume| resume.quiet_closes.clone()) + .unwrap_or_default() + }; + assert_eq!( + closes("quiet$gen"), + vec![ + by_ir::function::QuietClose { + state: 1, + clears: vec!["$iter1".to_string(), "$iter0".to_string()], + }, + by_ir::function::QuietClose { + state: 2, + clears: Vec::new(), + }, + ] + ); + assert_eq!(closes("handled$gen"), Vec::new()); + }, + ); +} + #[test] fn a_generator_local_that_may_be_unset_stays_a_cell() { // each of these would read a *zero* instead of raising if it were unboxed @@ -8192,7 +8254,11 @@ fn a_generator_local_that_may_be_unset_stays_a_cell() { .iter() .find(|field| field.name == name) .unwrap_or_else(|| panic!("`{name}` is a state field")); - assert_eq!(field.ty, RType::OBJECT, "`{name}` must stay a cell"); + assert!(field.cell, "`{name}` must stay a cell"); + assert!( + crate::has_an_unset_value(&field.ty), + "`{name}` is held where unset is a value of its own" + ); assert!( has_op(&state.methods[0], |op| matches!(op, Op::GetCell { .. })), "`{name}` must be read with the unset check" diff --git a/crates/by_opt/src/borrow.rs b/crates/by_opt/src/borrow.rs index d839d19f3c..6fd64031a2 100644 --- a/crates/by_opt/src/borrow.rs +++ b/crates/by_opt/src/borrow.rs @@ -147,8 +147,25 @@ fn borrow(function: &mut Function) { // the constants settle first, so that a copy — whose borrow rests on its source // still owning — can see that a register holding one owns nothing to lend mark(function, constants(function)); - // and the field reads before the copies, for the same reason - mark(function, field_reads(function)); + // and the field reads before the copies, for the same reason. a field read that + // borrows no longer releases what its register held before, which can make it the + // inert op another read's window needed — `i < n` over two fields reads `n` inside + // the window of `i` — so they go round until a round borrows nothing new. borrowing + // only ever takes a release away, so a window found safe stays safe + loop { + let reads: Vec = field_reads(function) + .into_iter() + .filter(|register| { + function + .register(*register) + .is_some_and(|decl| !decl.borrowed) + }) + .collect(); + if reads.is_empty() { + break; + } + mark(function, reads); + } // the copies, the element reads and the narrowing checks all rest on their source // still holding the value, so they are settled together against one another @@ -576,7 +593,12 @@ fn borrow_is_safe( // it releases again on the way back, though, and that release can // be the last one. so unlike a field read this has to be the value's // final use: a second one would read what the first let go of - Op::IntBinary { lhs, rhs, .. } + // + // a comparison is the same family on the same terms: `By_IntCompareSlow` + // either reads the sign off an exact `int` it asks nothing, or boxes both + // operands — retaining one behind a pointer — before it compares them. that + // is `i < n` over two fields, the loop test of a counting generator + Op::IntBinary { lhs, rhs, .. } | Op::IntCompare { lhs, rhs, .. } if Some(index) == last_read && is_tagged_int(function, lhs) && is_tagged_int(function, rhs) => {} @@ -794,6 +816,43 @@ mod tests { assert_eq!(verify(function), Ok(())); } + /// `self.i < self.n`, the loop test of a counting generator over two of its fields + fn field_compare() -> Function { + let mut builder = FunctionBuilder::new("test", RType::BIT); + let receiver = builder.param("self", nested()); + let i = builder.temp(RType::INT); + let n = builder.temp(RType::INT); + let below = builder.temp(RType::BIT); + for (dest, field) in [(i, "i"), (n, "n")] { + builder.push(Op::GetField { + dest, + receiver: Value::Register(receiver), + class: "Holder".to_string(), + field: field.to_string(), + }); + } + builder.push(Op::IntCompare { + dest: below, + op: by_ir::ops::CmpOp::Lt, + lhs: Value::Register(i), + rhs: Value::Register(n), + }); + builder.terminate(Terminator::Return(Value::Register(below))); + builder.finish() + } + + #[test] + fn two_field_reads_consumed_by_a_tagged_comparison_borrow() { + // `By_IntCompareSlow` boxes, and so retains, both operands before it can run a + // comparison of a subclass — or reads the sign of an exact `int` and asks nothing + let mut m = module(field_compare()); + run(&mut m); + let function = &m.functions[0]; + assert!(function.registers[1].borrowed); + assert!(function.registers[2].borrowed); + assert_eq!(verify(function), Ok(())); + } + #[test] fn a_field_read_read_twice_by_tagged_arithmetic_does_not_borrow() { // the first `By_IntAdd` releases what it was lent on the way out, and that diff --git a/crates/by_opt/src/copy_propagation.rs b/crates/by_opt/src/copy_propagation.rs index 82d92fdcf2..360fd9bab6 100644 --- a/crates/by_opt/src/copy_propagation.rs +++ b/crates/by_opt/src/copy_propagation.rs @@ -353,6 +353,7 @@ fn retarget(op: &mut Op, new_dest: RegisterId) { | Op::Unary { dest, .. } | Op::Box { dest, .. } | Op::IntToFloat { dest, .. } + | Op::TagShort { dest, .. } | Op::Unbox { dest, .. } | Op::TupleBuild { dest, .. } | Op::FieldIsSet { dest, .. } @@ -367,6 +368,7 @@ fn retarget(op: &mut Op, new_dest: RegisterId) { | Op::RaiseObject { .. } | Op::PopHandled { .. } | Op::Release { .. } + | Op::ArrayStoreLength { .. } | Op::Reraise { .. } | Op::LeaveGenerator { .. } | Op::LicenceHolds { .. } diff --git a/crates/by_opt/src/dead_allocations.rs b/crates/by_opt/src/dead_allocations.rs index e9a149b129..b679cd068d 100644 --- a/crates/by_opt/src/dead_allocations.rs +++ b/crates/by_opt/src/dead_allocations.rs @@ -32,6 +32,12 @@ pub(crate) fn run(module: &mut ModuleIr) { /// of them has is memory exhaustion. so one whose result nothing reads is work the /// program cannot tell apart from work that was never done. /// +/// a field read is a load and a retain: the object stays held by the field it was read +/// from, so letting go of the copy runs nothing either. that is the value a `yield` +/// statement evaluates to, which python computes and drops, and so does every resumption +/// of the frame that makes one. whether the field is set is [`Op::RequireField`]'s +/// question, which is not here +/// /// a `list`, `set` or `dict` display is deliberately *not* here even though a list /// looks the same: a set and a dict hash their elements, and hashing runs whatever /// `__hash__` the element's class wrote @@ -43,6 +49,7 @@ fn computes_only(op: &Op) -> bool { | Op::BuildTuple { .. } | Op::TupleBuild { .. } | Op::TupleGet { .. } + | Op::GetField { .. } ) } @@ -187,6 +194,7 @@ mod tests { use by_ir::function::ModuleIr; use by_ir::ops::{Op, Terminator, Value}; use by_ir::rtype::RType; + use std::collections::BTreeSet; fn module(function: by_ir::function::Function) -> ModuleIr { let mut module = ModuleIr::new("app"); @@ -267,6 +275,47 @@ mod tests { assert!(module.functions[0].blocks[0].ops.is_empty()); } + #[test] + fn a_field_read_nothing_reads_goes() { + // `yield x` as a statement: the value `send` passed in is read out of the state + // object and dropped, and the field goes on holding it either way + let mut builder = FunctionBuilder::new("f", RType::OBJECT); + let state = builder.param("state", RType::OBJECT); + let sent = builder.temp(RType::OBJECT); + builder.push(Op::GetField { + dest: sent, + receiver: Value::Register(state), + class: "g$gen".to_string(), + field: "$sent".to_string(), + }); + builder.terminate(Terminator::Return(Value::Register(state))); + + let mut module = module(builder.finish()); + super::run(&mut module); + + assert!(module.functions[0].blocks[0].ops.is_empty()); + } + + #[test] + fn a_field_read_into_a_name_nothing_reads_is_kept() { + // a name holds what it was given until it is rebound, whatever the field does + let mut builder = FunctionBuilder::new("f", RType::OBJECT); + let state = builder.param("state", RType::OBJECT); + let held = builder.local("held".to_string(), RType::OBJECT); + builder.push(Op::GetField { + dest: held, + receiver: Value::Register(state), + class: "Box".to_string(), + field: "item".to_string(), + }); + builder.terminate(Terminator::Return(Value::Register(state))); + + let mut module = module(builder.finish()); + super::run(&mut module); + + assert_eq!(module.functions[0].blocks[0].ops.len(), 1); + } + #[test] fn a_tuple_nothing_reads_holding_objects_stays() { // `(make(p), make(p))` thrown away: the tuple is what keeps the first element @@ -281,6 +330,7 @@ mod tests { builder.push(Op::TupleBuild { dest: pair, items: vec![Value::Register(first), Value::Register(second)], + moves: BTreeSet::new(), }); builder.terminate(Terminator::Return(Value::Register(p))); diff --git a/crates/by_opt/src/fold.rs b/crates/by_opt/src/fold.rs index 4d9f9b3b4c..ffa2857831 100644 --- a/crates/by_opt/src/fold.rs +++ b/crates/by_opt/src/fold.rs @@ -8,7 +8,7 @@ //! nothing here may fold an operation that could raise. `1 // 0` stays a division //! so that it still raises `ZeroDivisionError` at the right point in the program. -use std::collections::{HashMap, HashSet}; +use std::collections::{BTreeSet, HashMap, HashSet}; use by_ir::function::{Function, ModuleIr}; use by_ir::ops::{BinOp, CmpOp, Op, RegisterId, Terminator, UnaryOp, Value}; @@ -191,7 +191,7 @@ fn fold(function: &mut Function, frozen: &HashSet) { read.retain(|(_, owner, name), _| *owner != class || *name != field); } match op { - Op::BuildTuple { dest, items } | Op::TupleBuild { dest, items } => { + Op::BuildTuple { dest, items } | Op::TupleBuild { dest, items, .. } => { built.insert(*dest, items.clone()); } Op::GetField { @@ -267,6 +267,25 @@ fn fold_op(op: &Op) -> Option { dest: *dest, src: Value::Float(-value), }), + // a negative literal is lowered as the negation of a positive one, and a counter + // written with one has to see the literal to be held as a machine integer. + // the one immediate whose negation has no immediate is left to the runtime + Op::Unary { + dest, + op: UnaryOp::Neg, + operand: Value::Int(value), + } => value.checked_neg().map(|negated| Op::Assign { + dest: *dest, + src: Value::Int(negated), + }), + Op::Unary { + dest, + op: UnaryOp::Invert, + operand: Value::Int(value), + } => Some(Op::Assign { + dest: *dest, + src: Value::Int(!value), + }), _ => None, } } @@ -392,6 +411,7 @@ fn fold_unpack(types: &[RType], built: &HashMap>, op: &Op items.iter().all(boxed).then(|| Op::TupleBuild { dest: *dest, items: items.clone(), + moves: BTreeSet::new(), }) } @@ -616,6 +636,29 @@ mod tests { )); } + #[test] + fn negating_an_integer_immediate_folds() { + let fold = |op: UnaryOp, value: i64| { + fold_op(&Op::Unary { + dest: RegisterId(0), + op, + operand: Value::Int(value), + }) + }; + let assigned = |value: i64| { + Some(Op::Assign { + dest: RegisterId(0), + src: Value::Int(value), + }) + }; + assert_eq!(fold(UnaryOp::Neg, 1), assigned(-1)); + assert_eq!(fold(UnaryOp::Neg, i64::MAX), assigned(-i64::MAX)); + assert_eq!(fold(UnaryOp::Invert, 3), assigned(-4)); + assert_eq!(fold(UnaryOp::Invert, i64::MIN), assigned(i64::MAX)); + // its negation is one past the largest immediate, which only the runtime holds + assert_eq!(fold(UnaryOp::Neg, i64::MIN), None); + } + #[test] fn a_division_is_never_folded() { // `1 // 0` must still raise, and at the right point in the program @@ -889,6 +932,7 @@ mod tests { builder.push(Op::TupleBuild { dest: slots, items: vec![Value::Register(b), Value::Register(a)], + moves: BTreeSet::new(), }); // `a = `, which is `b` builder.push(Op::TupleGet { diff --git a/crates/by_opt/src/guard_loops.rs b/crates/by_opt/src/guard_loops.rs index 87bce7396b..1ee0b2fa1d 100644 --- a/crates/by_opt/src/guard_loops.rs +++ b/crates/by_opt/src/guard_loops.rs @@ -34,6 +34,7 @@ use by_ir::ops::{BlockId, Op, RegisterId, Terminator, Value}; use by_ir::rtype::RType; use crate::runs_python::{Effects, Guard, Held, Kind, answer_yes}; +use crate::unbox_counters::{bounds, counters, enter, unbox}; /// how many loops in one function may be duplicated /// @@ -167,6 +168,18 @@ fn version(effects: &Effects, function: &mut Function, candidate: Candidate) -> region, } = candidate; + // the loop's counters are asked about before anything is copied or split, while the + // body is still the whole of the loop + let (counters, bounds) = match region { + Region::Loop => { + let counters = counters(function, &body); + let bounds = bounds(function, &body, &counters); + (counters, bounds) + } + // a function with no loop in it has no counter worth a machine integer + Region::Entry => (Vec::new(), Vec::new()), + }; + // the copies land at the end, so every existing block keeps its id let first_copy = function.blocks.len(); let copies: BTreeMap = body @@ -272,6 +285,21 @@ fn version(effects: &Effects, function: &mut Function, candidate: Candidate) -> } } + let narrowings = unbox(function, header, &copies, &counters, &bounds); + // once the test has said yes, each counter and bound is narrowed, and one that does not + // fit runs the loop as written + let entries: Vec = (0..first_copy) + .map(BlockId) + .filter(|id| { + !body.contains(id) + && function.blocks[id.index()] + .terminator + .successors() + .contains(&header) + }) + .collect(); + let entered = enter(function, header, &entries, &narrowings, fast_header); + let answer = RegisterId(function.registers.len()); function.registers.push(RegisterDecl { name: None, @@ -292,7 +320,7 @@ fn version(effects: &Effects, function: &mut Function, candidate: Candidate) -> }], terminator: Terminator::Branch { cond: Value::Register(answer), - then_block: fast_header, + then_block: entered, else_block: header, }, owned_at_exit: None, @@ -301,17 +329,12 @@ fn version(effects: &Effects, function: &mut Function, candidate: Candidate) -> error_target: None, }); - // every edge that entered the loop from outside now enters through the test. the loop, - // its copy, the blocks split off it and the test itself are all inside - let inside: HashSet = body - .iter() - .copied() - .chain(copy_ids) - .chain(tails.values().copied()) - .chain([preheader]) - .collect(); + // every edge that entered the loop from outside now enters through the test. the loop + // and every block added for it are inside: its copy, the blocks split off it, the + // narrowings, whose `otherwise` edges *are* edges to the header, and the test itself + let inside: HashSet = body.iter().copied().collect(); for id in 0..function.blocks.len() { - if inside.contains(&BlockId(id)) { + if id >= first_copy || inside.contains(&BlockId(id)) { continue; } map_edges(&mut function.blocks[id], false, |target| { @@ -520,7 +543,7 @@ fn unwritten_parameters(function: &Function) -> HashSet { /// move everything from `at` on in `id` into a block of its own, which `id` then falls /// into, and name the new block -fn split(function: &mut Function, id: BlockId, at: usize) -> BlockId { +pub(crate) fn split(function: &mut Function, id: BlockId, at: usize) -> BlockId { let tail = BlockId(function.blocks.len()); let block = &mut function.blocks[id.index()]; let rest = block.ops.split_off(at); @@ -555,24 +578,9 @@ fn enters_by_test(block: &BasicBlock, header: BlockId) -> bool { } /// point every edge of `block` somewhere else, the error edge too where `errors` -fn map_edges(block: &mut BasicBlock, errors: bool, map: impl Fn(BlockId) -> BlockId) { - match &mut block.terminator { - Terminator::Goto(target) => *target = map(*target), - Terminator::Branch { - then_block, - else_block, - .. - } => { - *then_block = map(*then_block); - *else_block = map(*else_block); - } - Terminator::NarrowShort { - fits, otherwise, .. - } => { - *fits = map(*fits); - *otherwise = map(*otherwise); - } - Terminator::Return(_) | Terminator::Unreachable => {} +pub(crate) fn map_edges(block: &mut BasicBlock, errors: bool, map: impl Fn(BlockId) -> BlockId) { + for target in block.terminator.successors_mut() { + *target = map(*target); } if errors && let Some(target) = &mut block.error_target { *target = map(*target); @@ -626,7 +634,7 @@ fn successors_within(function: &Function, id: BlockId, inside: &HashSet /// a back edge is one into `header` from a block every path from the entry reaches through /// `header`. an edge from anywhere else enters the loop, so a loop inside another is its /// own loop, entered from the outer one's blocks, rather than the whole of the outer one -fn natural_loop(function: &Function, header: BlockId) -> Vec { +pub(crate) fn natural_loop(function: &Function, header: BlockId) -> Vec { let everywhere = reachable_avoiding(function, Function::entry(), None); let around = reachable_avoiding(function, Function::entry(), Some(header)); let latches: Vec = everywhere diff --git a/crates/by_opt/src/infallible.rs b/crates/by_opt/src/infallible.rs index f62cd2ce30..df63744e28 100644 --- a/crates/by_opt/src/infallible.rs +++ b/crates/by_opt/src/infallible.rs @@ -213,6 +213,8 @@ fn op_can_fail(module: &ModuleIr, function: &by_ir::function::Function, op: &Op) | Op::PopHandled { .. } | Op::Release { .. } | Op::SetField { .. } => false, + // a header write into a buffer this frame holds + Op::ArrayStoreLength { .. } => false, // the module namespace is a dict this module already holds — the extension's // own init put it there before anything compiled could be called, so taking a @@ -313,11 +315,15 @@ fn op_can_fail(module: &ModuleIr, function: &by_ir::function::Function, op: &Op) // a call's failure is decided by the callee, in the fixed point above Op::CallNative { .. } => false, + Op::TagShort { .. } => false, + // a machine integer always has a float + Op::IntToFloat { src, .. } => !matches!( + function.value_type(src), + Some(RType::Primitive(Primitive::Fixed(_))) + ), // boxing allocates, unboxing is a checked narrowing, a raise raises, and // an integer with no float at all raises `OverflowError` - Op::Box { .. } | Op::IntToFloat { .. } | Op::Unbox { .. } | Op::RaiseStandard { .. } => { - true - } + Op::Box { .. } | Op::Unbox { .. } | Op::RaiseStandard { .. } => true, } } diff --git a/crates/by_opt/src/lib.rs b/crates/by_opt/src/lib.rs index afe89458d5..52167534c5 100644 --- a/crates/by_opt/src/lib.rs +++ b/crates/by_opt/src/lib.rs @@ -16,6 +16,7 @@ pub(crate) mod fold; mod guard_loops; pub(crate) mod infallible; mod liveness; +mod push_lengths; pub(crate) mod refcount; pub(crate) mod release_temporaries; mod runs_python; @@ -89,14 +90,6 @@ const PASSES: &[Pass] = &[ name: "dict-find", run: dict_find::run, }, - // before dead-registers, which is what removes the temporary it frees up - // after folding, which is what turns the step's operands into the immediates - // the analysis looks for, and before coalesce, which would merge the counter - // with a register of the tagged representation - Pass { - name: "unbox-counters", - run: unbox_counters::run, - }, // after every fold, which is what orphans the allocations it removes, and before // coalesce, which would merge an orphaned register with a live one and so make // the write that fills it look read @@ -131,6 +124,12 @@ const PASSES: &[Pass] = &[ name: "unswitch", run: unswitch::run, }, + // after the passes that copy loops, so each copy keeps a length of its own, and before + // the ones that decide what a register owns and where it is released + Pass { + name: "push-lengths", + run: push_lengths::run, + }, Pass { name: "infallible", run: infallible::run, @@ -200,16 +199,21 @@ mod tests { #[test] fn a_buffer_length_is_unboxed_alongside_the_counter_it_bounds() { - // `while i < len(a)` over an unboxed buffer: the length is a `Py_ssize_t` - // already, so tagging it only to compare it against a machine counter is the - // tag going round in a circle. both sides end up unboxed and the guard is a - // register compare + // `while i < len(a): i = i + 1` over an unboxed buffer: the length is a + // `Py_ssize_t` already, so tagging it only to compare it against a machine counter + // is the tag going round in a circle. inside the loop's copy both sides are + // machine integers and the guard is a register compare let mut builder = FunctionBuilder::new("scan", RType::INT); let array = builder.param("a", RType::Array(Box::new(RType::FLOAT))); let index = builder.local("i", RType::INT); let length = builder.temp(RType::INT); let more = builder.temp(RType::BIT); builder.assign(index, Value::Int(0)); + let header = builder.new_block(); + let body = builder.new_block(); + let exit = builder.new_block(); + builder.terminate(Terminator::Goto(header)); + builder.switch_to(header); builder.push(Op::ArrayLen { dest: length, array: Value::Register(array), @@ -220,30 +224,46 @@ mod tests { lhs: Value::Register(index), rhs: Value::Register(length), }); + builder.terminate(Terminator::Branch { + cond: Value::Register(more), + then_block: body, + else_block: exit, + }); + builder.switch_to(body); + builder.push(Op::IntBinary { + dest: index, + op: BinOp::Add, + lhs: Value::Register(index), + rhs: Value::Int(1), + }); + builder.terminate(Terminator::Goto(header)); + builder.switch_to(exit); builder.terminate(Terminator::Return(Value::Register(index))); - let mut module = ModuleIr { - name: by_ir::ModuleName::new("app"), - functions: vec![builder.finish()], - declined: Vec::new(), - classes: Vec::new(), - gradual: Vec::new(), - promoted: Vec::new(), - lines: None, - fallback_source: None, - fallback_code: None, - shims: None, - verify_install: true, - follow_recursion_limit: true, - }; + let mut module = ModuleIr::new("app"); + module.functions.push(builder.finish()); assert!(optimize(&mut module).is_ok()); - let fixed = |id: by_ir::ops::RegisterId| { + let function = &module.functions[0]; + let fixed = |value: &Value| { matches!( - module.functions[0].register(id).map(|decl| &decl.ty), + function.value_type(value), Some(RType::Primitive(by_ir::rtype::Primitive::Fixed(_))) ) }; - assert!(fixed(index) && fixed(length), "{:?}", module.functions[0]); + let guards: Vec<(&Value, &Value)> = function + .blocks + .iter() + .flat_map(|block| &block.ops) + .filter_map(|op| match op { + Op::IntCompare { lhs, rhs, .. } => Some((lhs, rhs)), + _ => None, + }) + .collect(); + assert!( + guards.iter().any(|(lhs, rhs)| fixed(lhs) && fixed(rhs)), + "{}", + by_ir::print::print_function(function) + ); } #[test] diff --git a/crates/by_opt/src/push_lengths.rs b/crates/by_opt/src/push_lengths.rs new file mode 100644 index 0000000000..b83c82edcf --- /dev/null +++ b/crates/by_opt/src/push_lengths.rs @@ -0,0 +1,391 @@ +//! keeping the length of a buffer a loop appends to in a machine register +//! +//! an append reads the buffer's length and capacity out of its header, stores the +//! element, and stores the length back. the next append loads the length it has just +//! stored, and a load of a value still on its way to memory waits for the store to land: +//! measured on `comp`, that wait is two thirds of the loop's cycles, though it barely shows +//! in the instructions it runs +//! +//! ```python +//! out = [] +//! for x in xs: +//! out.append(x + 0.5) # load `len`, compare against `cap`, store `len + 1` +//! ``` +//! +//! so where a loop does nothing with a buffer but append to it, the length lives in a +//! register while the loop runs: loaded once on the way in, advanced by each append, and +//! written back on every way out that reaches something reading the buffer — an exception +//! a handler catches included. nothing inside the loop reads the header's length, and no +//! other name can reach the buffer, so nothing can tell the header was behind + +use std::collections::{BTreeMap, BTreeSet}; + +use by_ir::function::{BasicBlock, Function, ModuleIr, RegisterDecl}; +use by_ir::ops::{BlockId, Op, RegisterId, Terminator, Value}; +use by_ir::rtype::{IntWidth, RType}; + +use crate::guard_loops::{map_edges, natural_loop}; +use crate::liveness::{live_in, read_registers}; + +pub(crate) fn run(module: &mut ModuleIr) { + for function in module.all_functions_mut() { + hold_lengths(function); + } +} + +fn hold_lengths(function: &mut Function) { + let mut held: Vec<(RegisterId, Vec)> = Vec::new(); + // each loop is taken with every register in the state the loops taken before it left + // them, largest first, so a nest keeps one length for the whole of it + while let Some((header, body, array)) = candidate(function, &held) { + hold(function, header, &body, array); + held.push((array, body)); + } +} + +/// the largest loop that only appends to a buffer, with the buffer +fn candidate( + function: &Function, + held: &[(RegisterId, Vec)], +) -> Option<(BlockId, Vec, RegisterId)> { + let mut best: Option<(BlockId, Vec, RegisterId)> = None; + for index in 0..function.blocks.len() { + let header = BlockId(index); + if header == Function::entry() { + continue; + } + let body = natural_loop(function, header); + if body.is_empty() + || best + .as_ref() + .is_some_and(|(_, best, _)| best.len() >= body.len()) + { + continue; + } + let appended: BTreeSet = body + .iter() + .flat_map(|id| &function.blocks[id.index()].ops) + .filter_map(|op| match op { + Op::ArrayPush { + array: Value::Register(array), + length: None, + .. + } => Some(*array), + _ => None, + }) + .collect(); + let found = appended.into_iter().find(|array| { + !held + .iter() + .any(|(done, region)| done == array && region.contains(&header)) + && only_appended(function, &body, *array) + }); + if let Some(array) = found { + best = Some((header, body, array)); + } + } + best +} + +/// whether the loop made of `body` does nothing with `array` but append to it, and nothing +/// but `array` can reach the buffer it holds +/// +/// a buffer is only ever reached through registers, so the question is what the registers +/// can hold. a register written only by building a new buffer holds one no other register +/// was handed, since handing it over would be a write of that other register — and a +/// parameter's buffer is its caller's too. so the loop may append to any number of those, +/// each of which then has a length of its own, and may read no other buffer at all +fn only_appended(function: &Function, body: &[BlockId], array: RegisterId) -> bool { + let is_array = |register: RegisterId| { + function + .register(register) + .is_some_and(|decl| matches!(decl.ty, RType::Array(_))) + }; + let made_here = |register: RegisterId| { + register.index() >= function.param_count + && is_array(register) + && function + .blocks + .iter() + .flat_map(|block| &block.ops) + .filter(|op| op.dest() == Some(register)) + .all(|op| matches!(op, Op::ArrayNew { .. })) + && function + .blocks + .iter() + .all(|block| block.terminator.dest() != Some(register)) + }; + if !made_here(array) { + return false; + } + let reads_a_buffer = + |value: &Value| matches!(value, Value::Register(register) if is_array(*register)); + body.iter().all(|id| { + let block = &function.blocks[id.index()]; + block.ops.iter().all(|op| match op { + Op::ArrayPush { + array: Value::Register(pushed), + value, + .. + } => made_here(*pushed) && !reads_a_buffer(value), + other => { + !other.operands().into_iter().any(reads_a_buffer) + && other.dest().is_none_or(|dest| !is_array(dest)) + } + }) && !block.terminator.operands().into_iter().any(reads_a_buffer) + }) +} + +fn hold(function: &mut Function, header: BlockId, body: &[BlockId], array: RegisterId) { + let live = live_in(function, &read_registers); + let length = RegisterId(function.registers.len()); + function.registers.push(RegisterDecl { + name: None, + ty: RType::fixed(IntWidth::I64), + borrowed: false, + may_be_unassigned: false, + }); + let existing = function.blocks.len(); + let (range, position) = { + let block = &function.blocks[header.index()]; + (block.range, block.position) + }; + let block = |ops: Vec, terminator: Terminator| BasicBlock { + ops, + terminator, + owned_at_exit: None, + range, + position, + error_target: None, + }; + let stored = || Op::ArrayStoreLength { + array: Value::Register(array), + length: Value::Register(length), + }; + + for id in body { + for op in &mut function.blocks[id.index()].ops { + if let Op::ArrayPush { + array: Value::Register(pushed), + length: counted @ None, + .. + } = op + && *pushed == array + { + *counted = Some(Value::Register(length)); + } + } + } + + // on the way out, and on the way to a handler outside the loop, wherever the buffer is + // read again + let read_at = |target: BlockId| live[target.index()].contains(&array); + let mut handlers: BTreeMap = BTreeMap::new(); + for id in body { + let targets = function.blocks[id.index()].terminator.successors(); + for target in targets { + if body.contains(&target) || target.index() >= existing || !read_at(target) { + continue; + } + let landing = BlockId(function.blocks.len()); + function + .blocks + .push(block(vec![stored()], Terminator::Goto(target))); + for edge in function.blocks[id.index()].terminator.successors_mut() { + if *edge == target { + *edge = landing; + } + } + } + if let Some(handler) = function.blocks[id.index()].error_target + && !body.contains(&handler) + && read_at(handler) + { + let landing = *handlers.entry(handler).or_insert_with(|| { + let landing = BlockId(function.blocks.len()); + function + .blocks + .push(block(vec![stored()], Terminator::Goto(handler))); + landing + }); + function.blocks[id.index()].error_target = Some(landing); + } + } + + // and on the way in, from wherever the loop is entered + let preheader = BlockId(function.blocks.len()); + function.blocks.push(block( + vec![Op::ArrayLen { + dest: length, + array: Value::Register(array), + }], + Terminator::Goto(header), + )); + for id in 0..existing { + if body.contains(&BlockId(id)) { + continue; + } + map_edges(&mut function.blocks[id], false, |target| { + if target == header { preheader } else { target } + }); + } +} + +#[cfg(test)] +mod tests { + use by_ir::builder::FunctionBuilder; + use by_ir::ops::CmpOp; + use by_ir::verify::verify; + + use super::*; + + /// `out = [0.0]; i = 0; while i < n: out.append(x); [len(out)]; i = i + 1; return out`, + /// with a handler reading `out` on the body's error edge where `handled` + fn appending(read_inside: bool, handled: bool) -> Function { + let mut builder = FunctionBuilder::new("fill", RType::INT); + let n = builder.param("n", RType::INT); + let x = builder.param("x", RType::FLOAT); + let out = builder.local("out", RType::Array(Box::new(RType::FLOAT))); + let i = builder.local("i", RType::INT); + let more = builder.temp(RType::BIT); + let status = builder.temp(RType::BIT); + let size = builder.temp(RType::INT); + let answer = builder.temp(RType::INT); + builder.push(Op::ArrayNew { + dest: out, + items: vec![Value::Float(0.0)], + }); + builder.assign(i, Value::Int(0)); + let header = builder.new_block(); + let body = builder.new_block(); + let exit = builder.new_block(); + let handler = builder.new_block(); + builder.terminate(Terminator::Goto(header)); + builder.switch_to(header); + builder.push(Op::IntCompare { + dest: more, + op: CmpOp::Lt, + lhs: Value::Register(i), + rhs: Value::Register(n), + }); + builder.terminate(Terminator::Branch { + cond: Value::Register(more), + then_block: body, + else_block: exit, + }); + builder.switch_to(body); + builder.push(Op::ArrayPush { + dest: status, + array: Value::Register(out), + value: Value::Register(x), + length: None, + }); + if read_inside { + builder.push(Op::ArrayLen { + dest: size, + array: Value::Register(out), + }); + } + builder.push(Op::IntBinary { + dest: i, + op: by_ir::ops::BinOp::Add, + lhs: Value::Register(i), + rhs: Value::Int(1), + }); + builder.terminate(Terminator::Goto(header)); + builder.switch_to(exit); + builder.push(Op::ArrayLen { + dest: answer, + array: Value::Register(out), + }); + builder.terminate(Terminator::Return(Value::Register(answer))); + builder.switch_to(handler); + builder.push(Op::ArrayLen { + dest: answer, + array: Value::Register(out), + }); + builder.terminate(Terminator::Return(Value::Register(answer))); + let mut function = builder.finish(); + if handled { + function.blocks[body.index()].error_target = Some(handler); + } + function + } + + fn held(function: Function) -> Function { + let mut module = ModuleIr::new("app"); + module.functions.push(function); + run(&mut module); + let function = module.functions.remove(0); + assert_eq!(verify(&function), Ok(())); + function + } + + fn stores(function: &Function) -> Vec { + (0..function.blocks.len()) + .map(BlockId) + .filter(|id| { + function.blocks[id.index()] + .ops + .iter() + .any(|op| matches!(op, Op::ArrayStoreLength { .. })) + }) + .collect() + } + + #[test] + fn a_loop_that_only_appends_counts_in_a_register_and_writes_it_back_on_the_way_out() { + let function = held(appending(false, false)); + let printed = by_ir::print::print_function(&function); + assert!( + function + .blocks + .iter() + .flat_map(|block| &block.ops) + .any(|op| matches!( + op, + Op::ArrayPush { + length: Some(_), + .. + } + )), + "{printed}" + ); + assert_eq!(stores(&function).len(), 1, "{printed}"); + } + + #[test] + fn a_handler_that_reads_the_buffer_is_reached_through_a_write_back() { + let function = held(appending(false, true)); + let printed = by_ir::print::print_function(&function); + let stored = stores(&function); + assert_eq!(stored.len(), 2, "{printed}"); + assert!( + function.blocks.iter().any(|block| block + .error_target + .is_some_and(|target| stored.contains(&target))), + "{printed}" + ); + } + + #[test] + fn a_loop_that_reads_the_buffer_keeps_the_header_current() { + let function = held(appending(true, false)); + let printed = by_ir::print::print_function(&function); + assert!(stores(&function).is_empty(), "{printed}"); + assert!( + !function + .blocks + .iter() + .flat_map(|block| &block.ops) + .any(|op| matches!( + op, + Op::ArrayPush { + length: Some(_), + .. + } + )), + "{printed}" + ); + } +} diff --git a/crates/by_opt/src/release_temporaries.rs b/crates/by_opt/src/release_temporaries.rs index a2f2574405..22d53c67b1 100644 --- a/crates/by_opt/src/release_temporaries.rs +++ b/crates/by_opt/src/release_temporaries.rs @@ -221,8 +221,11 @@ fn release_temporaries(function: &mut Function) { /// the places this pass may release: those of temporaries that own one reference apiece /// -/// a tagged `int` is refcounted as well, but an `int` has no finalizer to run, so it is -/// left for the frame to let go of, inside a tuple as much as on its own +/// a tagged `int` is one of them. its short case owns nothing at all and its overflow +/// case owns exactly one `PyLongObject`, and an `int` has no finalizer, so nothing about +/// *when* it is let go of is observable — what is observable is that the register is left +/// holding a reference until the next write of it, and the release that write makes then +/// reads a value from whichever trip round the loop wrote it last fn candidates(function: &Function) -> HashSet { // a loop's cursor is read and written by the operation that advances it, outside // the operands this pass follows, so it is never offered @@ -241,7 +244,7 @@ fn candidates(function: &Function) -> HashSet { places_of(function, register).into_iter().filter(|place| { decl.ty .element(&place.path) - .is_some_and(RType::is_object_reference) + .is_some_and(RType::owns_one_reference) }) }) .collect() @@ -506,6 +509,7 @@ mod tests { Op::TupleBuild { dest, items: items.iter().copied().map(Value::Register).collect(), + moves: BTreeSet::new(), } } @@ -665,6 +669,41 @@ mod tests { assert_eq!(verify(&module.functions[0]), Ok(())); } + #[test] + fn a_tagged_int_temporary_is_released_after_its_last_read() { + // a tagged `int` holds a real `PyLongObject` whenever its value is too wide for + // the tag, and that object can be an `int` subclass with a finalizer. left for + // the frame to let go of, it is also left holding a reference across the next + // trip of whatever loop it sits in + let mut builder = FunctionBuilder::new("f", RType::INT); + let p = builder.param("p", RType::OBJECT); + let made = builder.temp(RType::INT); + let answer = builder.temp(RType::INT); + builder.push(Op::Unbox { + dest: made, + src: Value::Register(p), + to: RType::INT, + }); + builder.push(Op::IntBinary { + dest: answer, + op: by_ir::ops::BinOp::Add, + lhs: Value::Register(made), + rhs: Value::Int(1), + }); + builder.terminate(Terminator::Return(Value::Register(answer))); + + let mut module = module(builder.finish()); + run(&mut module); + + assert_eq!( + releases(&module, 0), + vec![(2, made)], + "{:?}", + module.functions[0].blocks + ); + assert_eq!(verify(&module.functions[0]), Ok(())); + } + #[test] fn a_named_local_is_held_to_the_end() { let mut builder = FunctionBuilder::new("f", RType::OBJECT); diff --git a/crates/by_opt/src/runs_python.rs b/crates/by_opt/src/runs_python.rs index 936e7c63f4..cc1b83f272 100644 --- a/crates/by_opt/src/runs_python.rs +++ b/crates/by_opt/src/runs_python.rs @@ -705,6 +705,7 @@ impl Held { } if exact(operand) == Some(Kind::Int) && *ty == RType::INT => Some(Kind::Int), // a machine value boxed is built as the builtin, and an exact value boxed is // itself + Op::TagShort { .. } => Some(Kind::Int), Op::Box { src, .. } => match function.value_type(src)? { RType::Primitive(Primitive::Fixed(_)) => Some(Kind::Int), RType::Primitive(Primitive::Float) => Some(Kind::Float), @@ -819,13 +820,16 @@ fn op_itself_runs_python(function: &Function, op: &Op, held: &Held) -> bool { // `PyLong_AsDouble` reads the digits of any `int`, a subclass's included, and asks // it nothing Op::IntToFloat { .. } => false, + // a short is a word with its tag, and building one allocates nothing + Op::TagShort { .. } => false, // a buffer of the compiler's own, grown with the allocator and not the collector Op::ArrayNew { .. } | Op::ArrayGet { .. } | Op::ArrayLen { .. } | Op::ArrayRead { .. } - | Op::ArrayPush { .. } => false, + | Op::ArrayPush { .. } + | Op::ArrayStoreLength { .. } => false, // a store lets go of the element it replaces Op::ArraySet { array, .. } => !function .value_type(array) diff --git a/crates/by_opt/src/store_moves.rs b/crates/by_opt/src/store_moves.rs index 3e9c200d21..85af9b2d62 100644 --- a/crates/by_opt/src/store_moves.rs +++ b/crates/by_opt/src/store_moves.rs @@ -105,66 +105,116 @@ fn move_dying_stores(function: &mut Function) { move_dying_copies(function); } -/// turn each copy or element read whose source place is released after it, with -/// nothing touching either in between, into a move +/// one operation's read of a place that is released straight after it, so the read may +/// take the reference rather than have one taken for it +struct Handover { + block: usize, + /// where the reading operation sits + at: usize, + /// which slot of a tuple build reads it, where that is the reading operation + item: Option, + /// where the release of the place sits, which the handover replaces + release: usize, +} + +/// turn each read of a dying place into a move: a copy, an element read off a tuple, or +/// one slot of a tuple being built fn move_dying_copies(function: &mut Function) { - let mut moved: Vec<(usize, usize, usize)> = Vec::new(); + let mut moved: Vec = Vec::new(); for (index, block) in function.blocks.iter().enumerate() { for (position, op) in block.ops.iter().enumerate() { - let (dest, source, path) = match op { + let reads: Vec<(Option, RegisterId, Vec)> = match op { Op::Assign { - dest, + dest: _, src: Value::Register(source), - } => (*dest, *source, Vec::new()), + } => vec![(None, *source, Vec::new())], Op::TupleGet { - dest, + dest: _, src: Value::Register(source), index, - } => (*dest, *source, vec![*index]), + } => vec![(None, *source, vec![*index])], + Op::TupleBuild { items, .. } => items + .iter() + .enumerate() + .filter_map(|(slot, item)| match item { + Value::Register(source) => Some((Some(slot), *source, Vec::new())), + _ => None, + }) + .collect(), _ => continue, }; - // a widening copy changes the representation the reference is held in, - // which a move does not - let same_type = function - .register(source) - .and_then(|decl| decl.ty.element(&path)) - .zip(function.register(dest)) - .is_some_and(|(element, decl)| *element == decl.ty); - if dest == source - || !same_type - || function.register(dest).is_none_or(|decl| decl.borrowed) - || !owns_place(function, source, &path) - { - continue; - } - let touches = |later: &Op| { - later.dest() == Some(source) - || later.unbinds() == Some(source) - || later.dest() == Some(dest) - || later.unbinds() == Some(dest) - || reads_place(later, source, &path) - }; - let Some(offset) = block.ops[position + 1..] - .iter() - .position(|later| touches(later) || releases_place(later, source, &path)) - else { + let Some(dest) = op.dest() else { continue; }; - let at = position + 1 + offset; - if releases_place(&block.ops[at], source, &path) { - moved.push((index, position, at)); + for (item, source, path) in reads { + // a widening read changes the representation the reference is held in, + // which a move does not + let slot = match item { + Some(slot) => vec![slot], + None => Vec::new(), + }; + let same_type = function + .register(source) + .and_then(|decl| decl.ty.element(&path)) + .zip( + function + .register(dest) + .and_then(|decl| decl.ty.element(&slot)), + ) + .is_some_and(|(read, written)| read == written); + if dest == source + || !same_type + || function.register(dest).is_none_or(|decl| decl.borrowed) + || !owns_place(function, source, &path) + { + continue; + } + let touches = |later: &Op| { + later.dest() == Some(source) + || later.unbinds() == Some(source) + || later.dest() == Some(dest) + || later.unbinds() == Some(dest) + || reads_place(later, source, &path) + }; + let Some(offset) = block.ops[position + 1..] + .iter() + .position(|later| touches(later) || releases_place(later, source, &path)) + else { + continue; + }; + let at = position + 1 + offset; + // one release lets go of one reference, so it pays for one slot: a + // register named twice in the same build retains for the second + if releases_place(&block.ops[at], source, &path) + && !moved + .iter() + .any(|held| held.block == index && held.release == at) + { + moved.push(Handover { + block: index, + at: position, + item, + release: at, + }); + } } } } - for (index, position, _) in &moved { + for handover in &moved { let Some(op) = function .blocks - .get_mut(*index) - .and_then(|block| block.ops.get_mut(*position)) + .get_mut(handover.block) + .and_then(|block| block.ops.get_mut(handover.at)) else { continue; }; + if let Some(slot) = handover.item { + if let Op::TupleBuild { moves, .. } = op { + moves.insert(slot); + } + continue; + } let replacement = match op { Op::Assign { dest, src } => Op::Move { dest: *dest, @@ -183,7 +233,7 @@ fn move_dying_copies(function: &mut Function) { // the releases go last and back to front, so removing one cannot move another let mut releases: Vec<(usize, usize)> = moved .into_iter() - .map(|(index, _, release)| (index, release)) + .map(|handover| (handover.block, handover.release)) .collect(); releases.sort_unstable_by(|a, b| b.cmp(a)); for (index, release) in releases { @@ -201,10 +251,7 @@ fn owns_place(function: &Function, register: RegisterId, path: &[usize]) -> bool && function.register(register).is_some_and(|decl| { !decl.borrowed && decl.name.is_none() - && decl - .ty - .element(path) - .is_some_and(|ty| *ty == RType::INT || ty.is_object_reference()) + && decl.ty.element(path).is_some_and(RType::owns_one_reference) }) } @@ -242,11 +289,9 @@ fn may_hand_over(function: &Function, register: RegisterId) -> bool { if register.index() < function.param_count { return false; } - function.register(register).is_some_and(|decl| { - !decl.borrowed - && decl.name.is_none() - && (decl.ty == RType::INT || decl.ty.is_object_reference()) - }) + function + .register(register) + .is_some_and(|decl| !decl.borrowed && decl.name.is_none() && decl.ty.owns_one_reference()) } #[cfg(test)] @@ -254,6 +299,7 @@ mod tests { use super::*; use by_ir::builder::FunctionBuilder; use by_ir::ops::{BinOp, Terminator}; + use std::collections::BTreeSet; fn module(function: Function) -> ModuleIr { ModuleIr { @@ -414,6 +460,7 @@ mod tests { builder.push(Op::TupleBuild { dest: pair, items: vec![Value::Register(p), Value::Register(p)], + moves: BTreeSet::new(), }); builder.push(release(pair, &[1])); builder.push(Op::TupleGet { @@ -440,6 +487,104 @@ mod tests { assert_eq!(by_ir::verify::verify(&module.functions[0]), Ok(())); } + /// which slots of the one tuple build in `block` hand their reference over + fn built_moves(module: &ModuleIr) -> Vec { + module.functions[0].blocks[0] + .ops + .iter() + .find_map(|op| match op { + Op::TupleBuild { moves, .. } => Some(moves.iter().copied().collect()), + _ => None, + }) + .unwrap_or_default() + } + + #[test] + fn a_tuple_takes_the_reference_of_a_slot_released_after_the_build() { + // `return a // 7, a % 7`: each part is built into a temporary and put in the + // tuple, which retains it, and the temporary is then let go of. the retain and + // the release are around nothing, and the tuple can have the reference instead + let mut builder = FunctionBuilder::new("f", RType::Tuple(Box::new([RType::INT; 2]))); + let p = builder.param("p", RType::INT); + let first = builder.temp(RType::INT); + let second = builder.temp(RType::INT); + let pair = builder.temp(RType::Tuple(Box::new([RType::INT; 2]))); + for (dest, op) in [(first, BinOp::FloorDiv), (second, BinOp::Mod)] { + builder.push(Op::IntBinary { + dest, + op, + lhs: Value::Register(p), + rhs: Value::Int(7), + }); + } + builder.push(Op::TupleBuild { + dest: pair, + items: vec![Value::Register(first), Value::Register(second)], + moves: BTreeSet::new(), + }); + builder.push(release(first, &[])); + builder.push(release(second, &[])); + builder.terminate(Terminator::Return(Value::Register(pair))); + + let mut module = module(builder.finish()); + run(&mut module); + + assert_eq!(built_moves(&module), vec![0, 1]); + let ops = &module.functions[0].blocks[0].ops; + assert!( + !ops.iter().any(|op| matches!(op, Op::Release { .. })), + "{ops:?}" + ); + assert_eq!(by_ir::verify::verify(&module.functions[0]), Ok(())); + } + + #[test] + fn one_release_pays_for_one_slot_of_a_tuple_built_from_it_twice() { + // `(x, x)` needs two references and the release lets go of one, so the second + // slot takes one of its own + let mut builder = FunctionBuilder::new("f", RType::Tuple(Box::new([RType::OBJECT; 2]))); + let p = builder.param("p", RType::OBJECT); + let made = builder.temp(RType::OBJECT); + let pair = builder.temp(RType::Tuple(Box::new([RType::OBJECT; 2]))); + builder.push(call(made, p)); + builder.push(Op::TupleBuild { + dest: pair, + items: vec![Value::Register(made), Value::Register(made)], + moves: BTreeSet::new(), + }); + builder.push(release(made, &[])); + builder.terminate(Terminator::Return(Value::Register(pair))); + + let mut module = module(builder.finish()); + run(&mut module); + + assert_eq!(built_moves(&module), vec![0]); + assert_eq!(by_ir::verify::verify(&module.functions[0]), Ok(())); + } + + #[test] + fn a_slot_read_again_before_its_release_keeps_a_reference_of_its_own() { + let mut builder = FunctionBuilder::new("f", RType::OBJECT); + let p = builder.param("p", RType::OBJECT); + let made = builder.temp(RType::OBJECT); + let pair = builder.temp(RType::Tuple(Box::new([RType::OBJECT; 2]))); + let again = builder.temp(RType::OBJECT); + builder.push(call(made, p)); + builder.push(Op::TupleBuild { + dest: pair, + items: vec![Value::Register(made), Value::Register(p)], + moves: BTreeSet::new(), + }); + builder.push(call(again, made)); + builder.push(release(made, &[])); + builder.terminate(Terminator::Return(Value::Register(again))); + + let mut module = module(builder.finish()); + run(&mut module); + + assert_eq!(built_moves(&module), Vec::::new()); + } + #[test] fn a_copy_whose_destination_is_written_before_the_release_is_left_alone() { // the second write lets go of what the first one stored, and a move would have diff --git a/crates/by_opt/src/str_concat_int.rs b/crates/by_opt/src/str_concat_int.rs index 17e8be408e..41addf69d2 100644 --- a/crates/by_opt/src/str_concat_int.rs +++ b/crates/by_opt/src/str_concat_int.rs @@ -215,6 +215,7 @@ mod tests { use by_ir::function::{Function, ModuleIr}; use by_ir::ops::{Concatenation, Mutation, Op, RegisterId, Terminator, Value}; use by_ir::rtype::RType; + use std::collections::BTreeSet; /// `prefix + str(i)`, in the shape the passes before this one leave it fn built(prefix: &str, wire: impl FnOnce(&mut FunctionBuilder, RegisterId)) -> Function { @@ -279,6 +280,7 @@ mod tests { builder.push(Op::TupleBuild { dest: pair, items: vec![Value::Register(checked)], + moves: BTreeSet::new(), }); })); assert!( diff --git a/crates/by_opt/src/str_of_int.rs b/crates/by_opt/src/str_of_int.rs index 7d58bc8fad..c00d6e12fc 100644 --- a/crates/by_opt/src/str_of_int.rs +++ b/crates/by_opt/src/str_of_int.rs @@ -40,9 +40,9 @@ use std::collections::{HashMap, HashSet}; -use by_ir::function::{Function, ModuleIr}; +use by_ir::function::{Function, ModuleIr, RegisterDecl}; use by_ir::ops::{BlockId, Op, RegisterId, Value}; -use by_ir::rtype::RType; +use by_ir::rtype::{Primitive, RType}; pub(crate) fn run(module: &mut ModuleIr) { for function in module.all_functions_mut() { @@ -72,6 +72,22 @@ fn fuse(function: &mut Function) { // a later one was found at pairs.sort_by_key(|pair| std::cmp::Reverse((pair.block.index(), pair.index))); for pair in pairs { + // a counter `unbox_counters` gave a machine representation boxes straight to an + // object, and is handed the tagged value the fused form formats + let tagging = matches!( + type_of(function, &pair.integer), + Some(RType::Primitive(Primitive::Fixed(_))) + ) + .then(|| { + let tagged = RegisterId(function.registers.len()); + function.registers.push(RegisterDecl { + name: None, + ty: RType::INT, + borrowed: false, + may_be_unassigned: false, + }); + tagged + }); let Some(block) = function.blocks.get_mut(pair.block.index()) else { continue; }; @@ -80,13 +96,22 @@ fn fuse(function: &mut Function) { }; let fused = Op::StrOfInt { dest: *dest, - value: pair.integer, + value: tagging.map_or(pair.integer.clone(), Value::Register), }; + let mut replacement = Vec::new(); + if let Some(tagged) = tagging { + replacement.push(Op::Box { + dest: tagged, + src: pair.integer, + }); + } + replacement.push(fused); if droppable.contains(&pair.boxed) { - block.ops[pair.index] = fused; - block.ops.remove(pair.index + 1); + block.ops.splice(pair.index..=pair.index + 1, replacement); } else { - block.ops[pair.index + 1] = fused; + block + .ops + .splice(pair.index + 1..=pair.index + 1, replacement); } } } @@ -105,7 +130,10 @@ fn pairs(function: &Function) -> Vec { if callee != "str" || args.as_slice() != [Value::Register(*dest)] { continue; } - if type_of(function, src) != Some(RType::INT) { + if !matches!( + type_of(function, src), + Some(RType::Primitive(Primitive::Int | Primitive::Fixed(_))) + ) { continue; } out.push(Pair { diff --git a/crates/by_opt/src/unbox_counters.rs b/crates/by_opt/src/unbox_counters.rs index b583310afd..886c289cc9 100644 --- a/crates/by_opt/src/unbox_counters.rs +++ b/crates/by_opt/src/unbox_counters.rs @@ -1,4 +1,4 @@ -//! giving a loop counter a machine-integer representation +//! giving a loop counter a machine-integer representation inside a copy of its loop //! //! a tagged `int` pays, on every trip of a counting loop, a shortness test on each //! operand of the comparison, another pair on the step, the step's overflow @@ -7,357 +7,1048 @@ //! largest single cost left in a loop whose arithmetic already matches what a C //! compiler would emit for the same program written in C. //! -//! what makes it sound is not the guard, which compares against an ordinary `int` -//! that may be any size, but the *step*: `__builtin_add_overflow` costs one -//! instruction and a branch that is never taken, and it is emitted unconditionally, -//! so the representation carries no obligation the frontend has to discharge. +//! a counter is still an `int`, though, and python's `int` does not stop at the machine +//! word: +//! +//! ```python +//! i = 9223372036854775806 +//! while i < n: # n = 2**63 + 3 +//! i = i + 1 # the second step leaves the word, and python counts on +//! ``` +//! +//! so the machine representation lives only in a copy of the loop, which the passes that +//! duplicate loops (`guard_loops` and `unswitch`) make anyway. the copy is entered by +//! narrowing each counter it reads from its tagged register, and leaves by writing each +//! counter read afterwards back. a step that would leave the word takes an edge rather +//! than raising: it writes the counters back and continues at that same step in the loop +//! as written, whose tagged arithmetic carries on exactly as python's does. the tagged +//! registers are the counters everywhere outside the copy, so nothing after the loop can +//! tell which of the two ran. -use std::collections::HashMap; +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; -use by_ir::function::{Function, ModuleIr}; -use by_ir::ops::{BinOp, Op, RegisterId, Value}; +use by_ir::function::{BasicBlock, Function, RegisterDecl}; +use by_ir::ops::{BinOp, BlockId, CmpOp, Op, RegisterId, Terminator, Value}; use by_ir::rtype::{IntWidth, Primitive, RType}; -pub(crate) fn run(module: &mut ModuleIr) { - for function in module.all_functions_mut() { - let picks: Vec<(RegisterId, Vec)> = candidates(function) - .into_iter() - // an earlier pick may have claimed this register as one of its steps, and a - // register already unboxed must not be unboxed again - .filter(|(register, _)| function.registers[register.0].ty == RType::INT) +use crate::guard_loops::split; +use crate::liveness::{error_live, live_in, live_out, read_registers}; + +/// the registers the loop made of `body` can hold as machine integers +/// +/// a counter is an `int` register whose every write inside the loop is a literal, a +/// length, a copy of another counter, or a sum or difference of counters, literals and +/// bounds — `int` registers nothing in the loop writes. only the writes inside the loop +/// decide: the copy narrows whatever a counter or a bound holds on the way in, and a value +/// too large to narrow runs the loop as written +/// +/// no *use* can disqualify one, because a reader that wants the tagged value is handed it. +/// a counter that a handler outside the loop reads is left alone: a failing operation +/// leaves the copy by its error edge with nothing on the way to write the counter back +pub(crate) fn counters(function: &Function, body: &[BlockId]) -> Vec { + let live = live_in(function, &read_registers); + counters_given(function, body, &live) +} + +pub(crate) fn counters_given( + function: &Function, + body: &[BlockId], + live: &[HashSet], +) -> Vec { + // every write to each register inside the loop, and the registers it takes its value + // from + let mut writes: BTreeMap> = BTreeMap::new(); + let mut refused: BTreeSet = BTreeSet::new(); + let invariant = |register: RegisterId| is_invariant(function, body, register); + for id in body { + let block = &function.blocks[id.index()]; + for op in &block.ops { + for written in op.unbinds().into_iter().chain(op.loop_cursor()) { + refused.insert(written); + } + if let Some(dest) = op.dest() { + writes + .entry(dest) + .or_default() + .push(machine_write(function, op)); + } + } + if let Some(dest) = block.terminator.dest() { + refused.insert(dest); + } + // a handler outside the loop is reached with nothing to write a counter back + if let Some(handler) = block.error_target + && !body.contains(&handler) + && let Some(read) = live.get(handler.index()) + { + refused.extend(read.iter().copied()); + } + } + let mut members: BTreeSet = writes + .iter() + .filter(|(register, each)| { + function + .register(**register) + .is_some_and(|decl| decl.ty == RType::INT && !decl.may_be_unassigned) + && !refused.contains(register) + && !each.contains(&Write::Tagged) + }) + .map(|(register, _)| *register) + .collect(); + // a write taking its value from a register that is neither a counter nor a bound is not + // a machine write + loop { + let refusing: Vec = members + .iter() + .filter(|register| { + writes[register].iter().any(|write| { + let Write::From(sources) = write else { + return false; + }; + sources.iter().flatten().any(|source| { + !members.contains(source) + && !is_fixed(function, &Value::Register(*source)) + && !invariant(*source) + }) + }) + }) + .copied() .collect(); - // every representation is settled before any boxing is decided: whether a read - // needs its tagged value back depends on what the *other* operand ended up as, - // and doing both in one pass would make that depend on register order - for (register, steps) in &picks { - retype(function, *register, steps); + if refusing.is_empty() { + break; } - for (register, steps) in &picks { - box_remaining_reads(function, *register, steps); + for register in refusing { + members.remove(®ister); } } + members.into_iter().collect() } -/// how a register is written -enum Definition { - /// `r = ` - Literal, - /// `r = t`, where `t` came from stepping `r` by a literal - Step(RegisterId), +/// what a write gives the register it writes +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Write { + /// a value a machine integer holds, whatever else is + Machine, + /// a value a machine integer holds if each of these registers holds one + From([Option; 2]), + /// a value only the tagged representation holds + Tagged, } -/// the registers whose every definition admits the unboxed form, each with the -/// steps that feed it -/// -/// no *use* can disqualify one. the counter was an `int` before the pass, so every -/// consumer of it already accepts an `int` — and boxing hands back exactly that. -/// only the writes decide, because only they bound the value -/// -/// a *parameter* never qualifies: its representation is the calling convention's, -/// and the caller wrote it -fn candidates(function: &Function) -> Vec<(RegisterId, Vec)> { +fn machine_write(function: &Function, op: &Op) -> Write { + // an operand of a machine write: a literal, or a register that has to hold a machine + // integer too + let operand = |value: &Value| match value { + Value::Int(_) => Some(None), + Value::Register(register) => Some(Some(*register)), + _ => None, + }; + match op { + Op::Assign { + src: Value::Int(_), .. + } + | Op::ArrayLen { .. } => Write::Machine, + Op::Assign { + src: Value::Register(source), + .. + } => Write::From([Some(*source), None]), + Op::IntBinary { + op: BinOp::Add | BinOp::Sub, + lhs, + rhs, + .. + } => match (operand(lhs), operand(rhs)) { + (Some(lhs), Some(rhs)) => Write::From([lhs, rhs]), + _ => Write::Tagged, + }, + // a copy nested inside this loop writing its own machine counter back + Op::Box { + src: source @ Value::Register(_), + .. + } if is_fixed(function, source) => Write::Machine, + _ => Write::Tagged, + } +} + +/// whether `register` is an `int` nothing in `body` writes, which a copy can narrow once on +/// the way in +fn is_invariant(function: &Function, body: &[BlockId], register: RegisterId) -> bool { + function + .register(register) + .is_some_and(|decl| decl.ty == RType::INT && !decl.may_be_unassigned) + && !body.iter().any(|id| { + let block = &function.blocks[id.index()]; + block.terminator.dest() == Some(register) + || block.ops.iter().any(|op| { + op.dest() == Some(register) + || op.unbinds() == Some(register) + || op.loop_cursor() == Some(register) + }) + }) +} + +/// the tagged bounds that nothing in the loop writes and that its guards compare a counter +/// against or its counters are written from, each narrowed once on the way in +pub(crate) fn bounds( + function: &Function, + body: &[BlockId], + counters: &[RegisterId], +) -> Vec { + let written: HashSet = body + .iter() + .flat_map(|id| { + let block = &function.blocks[id.index()]; + block + .ops + .iter() + .flat_map(|op| { + op.dest() + .into_iter() + .chain(op.unbinds()) + .chain(op.loop_cursor()) + }) + .chain(block.terminator.dest()) + }) + .collect(); let mut out = Vec::new(); - for index in function.param_count..function.registers.len() { - let register = RegisterId(index); - if function.registers[index].ty != RType::INT { + for op in body.iter().flat_map(|id| &function.blocks[id.index()].ops) { + // a sum a counter is written with reads its bounds as machine integers + if op.dest().is_some_and(|dest| counters.contains(&dest)) + && let Write::From(sources) = machine_write(function, op) + { + for source in sources.into_iter().flatten() { + if !counters.contains(&source) + && !is_fixed(function, &Value::Register(source)) + && !written.contains(&source) + && !out.contains(&source) + { + out.push(source); + } + } continue; } - if let Some(steps) = definitions(function, register) { - out.push((register, steps)); + let Op::IntCompare { + lhs, + rhs: Value::Register(bound), + .. + } = op + else { + continue; + }; + let counted = is_fixed(function, lhs) + || matches!(lhs, Value::Register(counter) if counters.contains(counter)); + if counted + && function + .register(*bound) + .is_some_and(|decl| decl.ty == RType::INT && !decl.may_be_unassigned) + && !written.contains(bound) + && !counters.contains(bound) + && !out.contains(bound) + { + out.push(*bound); } } out } -/// every write to `register`, when every one of them is a literal or a step +/// hold `counters` as machine integers in the copy of the loop `header` heads, and +/// `bounds` narrowed, answering what the copy narrows on the way in: each tagged register +/// it reads before writing, with the machine register that holds its value /// -/// anything else — a call's result, a read of a field, a value from another -/// register — leaves the tagged representation as the only one that fits -fn definitions(function: &Function, register: RegisterId) -> Option> { - let mut out = Vec::new(); - for block in &function.blocks { - for op in &block.ops { - if op.dest() != Some(register) { - continue; +/// `copies` names the copy of each block of `body`. a copied block may have been cut short +/// where the copy leaves for the loop as written, but what it keeps is that block's own +/// operations in their own places, which is what lets a step leave for the same point in +/// the original. the caller enters the copy through the narrowings this answers with, and +/// sends a value that does not narrow to the loop as written +pub(crate) fn unbox( + function: &mut Function, + header: BlockId, + copies: &BTreeMap, + counters: &[RegisterId], + bounds: &[RegisterId], +) -> Vec<(RegisterId, RegisterId)> { + let live = live_in(function, &read_registers); + let width = RType::fixed(IntWidth::I64); + let fixed: BTreeMap = counters + .iter() + .map(|counter| (*counter, fresh(function, width.clone()))) + .collect(); + let narrowed: BTreeMap = bounds + .iter() + .map(|bound| (*bound, fresh(function, width.clone()))) + .collect(); + // every register a machine write reads its operands from + let machine: BTreeMap = fixed + .iter() + .chain(narrowed.iter()) + .map(|(tagged, machine)| (*tagged, *machine)) + .collect(); + // the pieces of the loop as written a step leaves for, and the blocks it leaves through + let mut added: Vec = Vec::new(); + + // every step of a counter, and the point of the loop as written it leaves for + let mut steps: BTreeMap> = BTreeMap::new(); + for (original, copy) in copies { + let at: Vec = function.blocks[copy.index()] + .ops + .iter() + .enumerate() + .filter(|(_, op)| is_step(op, &fixed)) + .map(|(index, _)| index) + .collect(); + if !at.is_empty() { + steps.insert(*original, at); + } + } + // what is live at each step is asked of the loop as written before any of it is split + let mut live_at_step: HashMap<(BlockId, usize), Vec> = HashMap::new(); + for (original, at) in &steps { + for index in at { + let live_here = live_before(function, &live, *original, *index); + live_at_step.insert( + (*original, *index), + counters + .iter() + .filter(|counter| live_here.contains(counter)) + .copied() + .collect(), + ); + } + } + let mut resumes: HashMap<(BlockId, usize), BlockId> = HashMap::new(); + for (original, at) in &steps { + // latest first, so each split cuts what is left in front of the one after it + for index in at.iter().rev() { + let tail = split(function, *original, *index); + resumes.insert((*original, *index), tail); + added.push(tail); + } + } + + let mut region: BTreeSet = copies.values().copied().collect(); + let mut tagged: BTreeMap = BTreeMap::new(); + for (original, copy) in copies { + let block = &mut function.blocks[copy.index()]; + let ops = std::mem::take(&mut block.ops); + let terminator = std::mem::replace(&mut block.terminator, Terminator::Unreachable); + let (error_target, range, mut position) = (block.error_target, block.range, block.position); + let mut current = *copy; + let mut written: Vec = Vec::new(); + for (index, op) in ops.into_iter().enumerate() { + if let Op::Line { position: line } = &op { + position = Some(*line); } - // a buffer's length is a `Py_ssize_t` — it is already a machine integer, and - // tagging it only to compare it against a counter is the tag going round in - // a circle. `while i < len(a)` is where that shows - if matches!(op, Op::ArrayLen { .. }) { - out.push(Definition::Literal); + if is_step(&op, &fixed) + && let Op::IntBinary { + dest, + op: step, + lhs, + rhs, + } = &op + { + let resume = resumes[&(*original, index)]; + let landing = push_block(function, error_target, range, position); + function.blocks[landing.index()].ops = live_at_step[&(*original, index)] + .iter() + .map(|counter| Op::Box { + dest: *counter, + src: Value::Register(fixed[counter]), + }) + .collect(); + function.blocks[landing.index()].terminator = Terminator::Goto(resume); + added.push(landing); + let rest = push_block(function, error_target, range, position); + region.insert(rest); + let block = &mut function.blocks[current.index()]; + block.ops = std::mem::take(&mut written); + block.terminator = Terminator::MachineStep { + dest: fixed[dest], + op: *step, + lhs: machine_operand(lhs, &machine), + rhs: machine_operand(rhs, &machine), + fits: rest, + overflows: landing, + }; + current = rest; continue; } - let Op::Assign { src, .. } = op else { - return None; - }; - match src { - Value::Int(_) => out.push(Definition::Literal), - Value::Register(source) => out.push(Definition::Step(*source)), - _ => return None, + rewrite(function, op, &fixed, &machine, &mut tagged, &mut written); + } + // a terminator reads a counter tagged. that includes the narrowing of a copy nested + // in this one: a machine integer here still has to be a short to take its edge, + // which is what the nested copy's proofs rest on + let mut terminator = terminator; + for value in terminator.operands_mut() { + if let Value::Register(read) = value + && let Some(machine) = fixed.get(read) + { + let boxed = boxed_register(function, &mut tagged, *read); + written.push(Op::Box { + dest: boxed, + src: Value::Register(*machine), + }); + *value = Value::Register(boxed); } } + let block = &mut function.blocks[current.index()]; + block.ops = written; + block.terminator = terminator; } - if out.is_empty() { - return None; + for id in ®ion { + narrow_reads(function, *id, &narrowed); } - // a step's source has to be a step *of this register*, and one nothing else - // reads — otherwise unboxing it would change what that other reader sees - let steps = out - .iter() - .filter_map(|definition| match definition { - Definition::Step(source) => Some(*source), - Definition::Literal => None, - }) - .collect::>(); - let counts = read_counts(function); - for source in &steps { - let stepped = step_operands(function, *source)?; - if stepped != register || counts.get(source).copied().unwrap_or(0) != 1 { - return None; - } - } - Some(steps) -} -/// the register a step reads, when `source` is `lhs + ` and nothing else -fn step_operands(function: &Function, source: RegisterId) -> Option { - let mut found = None; - for block in &function.blocks { - for op in &block.ops { - if op.dest() != Some(source) { + // on the way out, whatever is read next is written back first + let landings: BTreeSet = added.iter().copied().collect(); + let mut leaving: Vec = Vec::new(); + for id in region.iter().copied().collect::>() { + let targets = function.blocks[id.index()].terminator.successors(); + for target in targets { + if region.contains(&target) || landings.contains(&target) { continue; } - let Op::IntBinary { - op: BinOp::Add | BinOp::Sub, - lhs: Value::Register(lhs), - rhs: Value::Int(_), - .. - } = op - else { - return None; + let read: Vec = counters + .iter() + .filter(|counter| { + live.get(target.index()) + .is_some_and(|set| set.contains(counter)) + }) + .copied() + .collect(); + if read.is_empty() { + continue; + } + let (error_target, range, position) = { + let block = &function.blocks[id.index()]; + (block.error_target, block.range, block.position) }; - if found.is_some() { - return None; + let landing = push_block(function, error_target, range, position); + function.blocks[landing.index()].ops = read + .iter() + .map(|counter| Op::Box { + dest: *counter, + src: Value::Register(fixed[counter]), + }) + .collect(); + function.blocks[landing.index()].terminator = Terminator::Goto(target); + leaving.push(landing); + for edge in function.blocks[id.index()].terminator.successors_mut() { + if *edge == target { + *edge = landing; + } } - found = Some(*lhs); } } - found + + // the way out is where a counter is written back, and it is entered from a single edge + // of the copy, with everything that edge establishes. a step leaving the machine word + // is not: a counter's writes only keep it on one side of the short range while every + // step fits + if let Some(copy) = copies.get(&header) { + let proven: Vec = region.into_iter().chain(leaving).collect(); + let short: Vec = narrowed.values().copied().collect(); + tag_proven_shorts(function, &proven, *copy, &short); + } + + let entered = live.get(header.index()).cloned().unwrap_or_default(); + fixed + .iter() + .filter(|(counter, _)| entered.contains(counter)) + .chain(narrowed.iter()) + .map(|(tagged, machine)| (*tagged, *machine)) + .collect() } -fn reads(value: &Value, register: RegisterId) -> bool { - matches!(value, Value::Register(id) if *id == register) +/// a step of a counter by a literal, which becomes a [`Terminator::MachineStep`] +fn is_step(op: &Op, fixed: &BTreeMap) -> bool { + matches!(op, Op::IntBinary { dest, .. } if fixed.contains_key(dest)) } -/// how many ops read each register -fn read_counts(function: &Function) -> HashMap { - let mut counts = HashMap::new(); - for block in &function.blocks { - for op in &block.ops { - for value in op.operands() { - if let Value::Register(id) = value { - *counts.entry(*id).or_insert(0) += 1; +/// an operand of a machine write, read from the machine register that holds it +fn machine_operand(value: &Value, machine: &BTreeMap) -> Value { + match value { + Value::Register(register) => { + Value::Register(machine.get(register).copied().unwrap_or(*register)) + } + Value::Int(literal) => Value::Fixed(*literal), + other => other.clone(), + } +} + +/// write `op` into the copy, reading and writing the machine registers +fn rewrite( + function: &mut Function, + mut op: Op, + fixed: &BTreeMap, + machine: &BTreeMap, + tagged: &mut BTreeMap, + written: &mut Vec, +) { + if let Some(dest) = op.dest() + && let Some(written_into) = fixed.get(&dest) + { + let op = match op { + Op::Assign { src, .. } => Op::Assign { + dest: *written_into, + src: machine_operand(&src, machine), + }, + Op::Box { src, .. } => Op::Assign { + dest: *written_into, + src, + }, + mut other => { + if let Some(dest) = other.dest_mut() { + *dest = *written_into; } + other } + }; + written.push(op); + return; + } + let served = served(function, &op, fixed); + // each counter an operation reads tagged is boxed once, however many operands read it + let mut boxed: BTreeMap = BTreeMap::new(); + for value in op.operands_mut() { + let Value::Register(read) = value else { + continue; + }; + let Some(machine) = fixed.get(read) else { + continue; + }; + if served.contains(read) { + *value = Value::Register(*machine); + continue; } - for value in block.terminator.operands() { - if let Value::Register(id) = value { - *counts.entry(*id).or_insert(0) += 1; + let register = match boxed.get(read) { + Some(register) => *register, + None => { + let register = boxed_register(function, tagged, *read); + written.push(Op::Box { + dest: register, + src: Value::Register(*machine), + }); + boxed.insert(*read, register); + register } - } + }; + *value = Value::Register(register); } - counts + written.push(op); } -/// rewrite `register`, and the steps that feed it, to the machine representation -fn retype(function: &mut Function, register: RegisterId, steps: &[RegisterId]) { - let width = RType::fixed(IntWidth::I64); - function.registers[register.0].ty = width.clone(); - for step in steps { - function.registers[step.0].ty = width.clone(); - } - - // the literals feeding the counter and its steps are immediates, and an - // immediate carries its own representation - for block in &mut function.blocks { - for op in &mut block.ops { - let touches = op - .dest() - .is_some_and(|dest| dest == register || steps.contains(&dest)); - if !touches { - continue; +/// the blocks a copy is entered through, answering the first of them +/// +/// each of `narrowings` is tested and narrowed in turn, and the first that does not fit +/// runs the loop as written from `header`. a counter every one of `entries` has just set to +/// a short literal needs no test, since the literal is what it holds, and is written +/// straight into its machine register: `i = 0` ahead of the loop is by far the usual way a +/// counter starts +pub(crate) fn enter( + function: &mut Function, + header: BlockId, + entries: &[BlockId], + narrowings: &[(RegisterId, RegisterId)], + copy: BlockId, +) -> BlockId { + let known = |register: RegisterId| { + let mut literal = None; + for entry in entries { + let block = &function.blocks[entry.index()]; + if block.terminator.dest() == Some(register) { + return None; } - for value in op.operands_mut() { - if let Value::Int(literal) = value { - *value = Value::Fixed(*literal); + let written = block.ops.iter().rev().find(|op| { + op.dest() == Some(register) + || op.unbinds() == Some(register) + || op.loop_cursor() == Some(register) + }); + match written { + Some(Op::Assign { + src: Value::Int(value), + .. + }) if SHORT.contains(value) && literal.is_none_or(|seen| seen == *value) => { + literal = Some(*value); } + _ => return None, } } + literal + }; + let mut assigned = Vec::new(); + let mut tested = Vec::new(); + for (tagged, machine) in narrowings { + match known(*tagged) { + Some(value) => assigned.push(Op::Assign { + dest: *machine, + src: Value::Fixed(value), + }), + None => tested.push((*tagged, *machine)), + } + } + if assigned.is_empty() && tested.is_empty() { + return copy; } + let (range, position) = { + let block = &function.blocks[header.index()]; + (block.range, block.position) + }; + let first = BlockId(function.blocks.len()); + let links = tested.len().max(1); + for index in 0..links { + let next = if index + 1 < links { + BlockId(first.index() + index + 1) + } else { + copy + }; + let terminator = match tested.get(index) { + Some((tagged, machine)) => Terminator::NarrowShort { + dest: *machine, + src: Value::Register(*tagged), + fits: next, + otherwise: header, + }, + None => Terminator::Goto(next), + }; + function.blocks.push(BasicBlock { + ops: if index == 0 { + std::mem::take(&mut assigned) + } else { + Vec::new() + }, + terminator, + owned_at_exit: None, + range, + position, + error_target: None, + }); + } + first } -/// give every read that is not the guard or the step its tagged value back -fn box_remaining_reads(function: &mut Function, register: RegisterId, steps: &[RegisterId]) { - let mut boxed: Option = None; - for index in 0..function.blocks.len() { - let mut rewritten = Vec::new(); - for (position, op) in function.blocks[index].ops.iter().enumerate() { - // the guard is served: codegen compares an unboxed left operand against - // either a tagged right one or an unboxed one. a *right* operand is served - // only when the left is unboxed too, since that is the pair codegen reads as - // two machine integers. - // - // anything else is served only if its *destination* was unboxed too, which - // is what `i * i` is not — it reads the counter but produces an ordinary - // `int` - let served = match op { - Op::IntCompare { - lhs: Value::Register(lhs), - .. - } if *lhs == register => true, - Op::IntCompare { - lhs: Value::Register(lhs), - rhs: Value::Register(rhs), - .. - } if *rhs == register => function - .register(*lhs) - .is_some_and(|decl| matches!(decl.ty, RType::Primitive(Primitive::Fixed(_)))), - // a scan reads the character at its counter and asks one question of - // it. the fused comparison reads a machine index as readily as a - // tagged one, so handing the counter back its tagged value here would - // be a shift out and a shift straight back in, once per character - Op::StrItemCompare { - index: Value::Register(index), - .. - } if *index == register => true, - // and a subscript names its element by an offset, which is the - // number already in the register. handing the counter back its - // tagged value here would be a shift out and a shift straight - // back in, once per element - Op::GetItem { - index: Value::Register(index), - .. - } if *index == register => true, - _ => false, - } || op - .dest() - .is_some_and(|dest| dest == register || steps.contains(&dest)); - if served { +/// the counters `op` reads that it can read as machine integers +/// +/// codegen compares an unboxed left operand against either a tagged right one or an +/// unboxed one, and a right operand is only read unboxed when the left is too, since that +/// is the pair it reads as two machine integers. a character comparison, a subscript and a +/// packed buffer name their element by an offset, which is the number the register holds, +/// and a double or an object is built from the machine integer directly +fn served( + function: &Function, + op: &Op, + fixed: &BTreeMap, +) -> Vec { + let counter = |value: &Value| match value { + Value::Register(register) if fixed.contains_key(register) => Some(*register), + _ => None, + }; + match op { + Op::IntCompare { lhs, rhs, .. } => { + let left = counter(lhs); + let right = counter(rhs).filter(|_| left.is_some() || is_fixed(function, lhs)); + left.into_iter().chain(right).collect() + } + Op::StrItemCompare { index, .. } + | Op::GetItem { index, .. } + | Op::ArrayGet { index, .. } + | Op::ArraySet { index, .. } => counter(index).into_iter().collect(), + Op::IntToFloat { src, .. } => counter(src).into_iter().collect(), + Op::Box { dest, src } + if function + .register(*dest) + .is_some_and(|decl| decl.ty == RType::OBJECT) => + { + counter(src).into_iter().collect() + } + _ => Vec::new(), + } +} + +/// the one register each counter's tagged value is handed to readers in, inside a copy +fn boxed_register( + function: &mut Function, + tagged: &mut BTreeMap, + counter: RegisterId, +) -> RegisterId { + if let Some(register) = tagged.get(&counter) { + return *register; + } + let register = fresh(function, RType::INT); + tagged.insert(counter, register); + register +} + +fn fresh(function: &mut Function, ty: RType) -> RegisterId { + let id = RegisterId(function.registers.len()); + function.registers.push(RegisterDecl { + name: None, + ty, + borrowed: false, + may_be_unassigned: false, + }); + id +} + +fn push_block( + function: &mut Function, + error_target: Option, + range: Option<(u32, u32)>, + position: Option, +) -> BlockId { + let id = BlockId(function.blocks.len()); + function.blocks.push(BasicBlock { + ops: Vec::new(), + terminator: Terminator::Unreachable, + owned_at_exit: None, + range, + position, + error_target, + }); + id +} + +/// the registers live just before the operation at `at` in `block` +fn live_before( + function: &Function, + live: &[HashSet], + block: BlockId, + at: usize, +) -> HashSet { + let block = &function.blocks[block.index()]; + let mut out = live_out(block, live); + let handler = error_live(block, live); + for op in block.ops[at..].iter().rev() { + out.extend(handler.iter().copied()); + if let Some(dest) = op.dest() { + out.remove(&dest); + } + out.extend(read_registers(op)); + } + out +} + +pub(crate) fn is_fixed(function: &Function, value: &Value) -> bool { + let ty = match value { + Value::Register(id) => function.register(*id).map(|decl| decl.ty.clone()), + other => other.immediate_type(), + }; + matches!(ty, Some(RType::Primitive(Primitive::Fixed(_)))) +} + +/// read each bound out of its narrowed register instead of the tagged one, wherever the +/// narrowed one serves +/// +/// a guard comparing a machine counter against the bound becomes a comparison of two +/// machine integers, and a bound widened to a double converts from the machine integer +/// with nothing to fail +pub(crate) fn narrow_reads( + function: &mut Function, + block: BlockId, + narrowed: &BTreeMap, +) { + let fixed_lhs: Vec = function.blocks[block.index()] + .ops + .iter() + .map(|op| matches!(op, Op::IntCompare { lhs, .. } if is_fixed(function, lhs))) + .collect(); + let narrow = |value: &mut Value| { + if let Value::Register(register) = value + && let Some(machine) = narrowed.get(register) + { + *value = Value::Register(*machine); + } + }; + for (op, fixed_lhs) in function.blocks[block.index()].ops.iter_mut().zip(fixed_lhs) { + match op { + Op::IntCompare { rhs, .. } if fixed_lhs => narrow(rhs), + Op::IntToFloat { src, .. } => narrow(src), + _ => {} + } + } +} + +/// the short range, in the machine integer a counter holds +const SHORT: std::ops::RangeInclusive = -(1 << 62)..=(1 << 62) - 1; + +/// the side of the short range a guard that held keeps a counter on +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +enum Side { + /// no larger than the largest short + Below, + /// no smaller than the smallest short + Above, +} + +type Facts = BTreeSet<(RegisterId, Side)>; + +/// give a counter its tagged value with no range test wherever the copy has proven it short +/// +/// the proof is the copy's own guards. one comparing the counter against a narrowed bound, +/// or against a short immediate, keeps the counter on one side of the short range on the +/// edge it held on, for as long as nothing writes the counter again. the other side comes +/// from a second guard, or from the counter's writes: a counter only ever narrowed from a +/// short, set to one, or stepped towards the side a guard holds is on the other side for +/// its whole life, because a step that would leave the machine integer leaves the copy +/// +/// ```python +/// i = 0 +/// while i < n: # below the largest short, on the edge into the body +/// split(i) # and never below zero, since `i` only ever counts up from it +/// i = i + 1 +/// ``` +pub(crate) fn tag_proven_shorts( + function: &mut Function, + copy: &[BlockId], + header: BlockId, + narrowed: &[RegisterId], +) { + let short = |value: &Value| match value { + Value::Register(id) => narrowed.contains(id), + Value::Fixed(literal) => SHORT.contains(literal), + _ => false, + }; + // a block an error edge reaches is entered part way through another, where nothing + // was established + let handlers: BTreeSet = copy + .iter() + .filter_map(|id| function.blocks[id.index()].error_target) + .collect(); + let mut entry: BTreeMap = BTreeMap::from([(header, Facts::new())]); + let mut queue = vec![header]; + while let Some(id) = queue.pop() { + let Some(facts) = entry.get(&id).cloned() else { + continue; + }; + for (next, facts) in exits(function, &function.blocks[id.index()], facts, &short) { + if !copy.contains(&next) || next == header || handlers.contains(&next) { continue; } - if op.operands().iter().any(|value| reads(value, register)) { - rewritten.push(position); + let met = match entry.get(&next) { + Some(known) => known.intersection(&facts).copied().collect(), + None => facts, + }; + if entry.get(&next) != Some(&met) { + entry.insert(next, met); + queue.push(next); } } - let terminator_reads = function.blocks[index] - .terminator - .operands() - .iter() - .any(|value| reads(value, register)); - if rewritten.is_empty() && !terminator_reads { + } + for handler in handlers { + entry.insert(handler, Facts::new()); + } + + for id in copy { + let Some(mut facts) = entry.get(id).cloned() else { continue; - } - // one boxed copy serves the whole function: the counter's tagged value is - // wanted at an exit, and an exit is reached once - let target = match boxed { - Some(id) => id, - None => { - let id = RegisterId(function.registers.len()); - function.registers.push(by_ir::function::RegisterDecl { - name: None, - ty: RType::INT, - borrowed: false, - may_be_unassigned: false, - }); - boxed = Some(id); - id - } }; - for position in rewritten.iter().rev() { - for value in function.blocks[index].ops[*position].operands_mut() { - if let Value::Register(id) = value - && *id == register - { - *value = Value::Register(target); - } + for index in 0..function.blocks[id.index()].ops.len() { + let op = &function.blocks[id.index()].ops[index]; + if let Op::Box { + dest, + src: Value::Register(counter), + } = op + && function + .register(*dest) + .is_some_and(|decl| decl.ty == RType::INT) + && is_fixed(function, &Value::Register(*counter)) + && [Side::Below, Side::Above].into_iter().all(|side| { + facts.contains(&(*counter, side)) || never_past(function, *counter, side) + }) + { + let tagged = Op::TagShort { + dest: *dest, + src: Value::Register(*counter), + }; + function.blocks[id.index()].ops[index] = tagged; } - function.blocks[index].ops.insert( - *position, - Op::Box { - dest: target, - src: Value::Register(register), - }, - ); - } - if terminator_reads { - for value in function.blocks[index].terminator.operands_mut() { - if let Value::Register(id) = value - && *id == register - { - *value = Value::Register(target); - } + let op = &function.blocks[id.index()].ops[index]; + for written in op.dest().into_iter().chain(op.loop_cursor()) { + facts.retain(|(register, _)| *register != written); } - function.blocks[index].ops.push(Op::Box { - dest: target, - src: Value::Register(register), - }); } } } +/// the facts on each edge out of `block`, given those it was entered with +fn exits( + function: &Function, + block: &BasicBlock, + mut facts: Facts, + short: &impl Fn(&Value) -> bool, +) -> Vec<(BlockId, Facts)> { + // what a comparison's bit says about its counter when it is true and when it is false + let mut compared: BTreeMap, Vec)> = BTreeMap::new(); + for op in &block.ops { + for written in op.dest().into_iter().chain(op.loop_cursor()) { + facts.retain(|(register, _)| *register != written); + compared.retain(|bit, (counter, _, _)| *bit != written && *counter != written); + } + if let Op::IntCompare { + dest, + op: compare, + lhs: lhs @ Value::Register(counter), + rhs, + } = op + && is_fixed(function, lhs) + && short(rhs) + { + let (held, failed) = match compare { + CmpOp::Lt | CmpOp::Le => (vec![Side::Below], vec![Side::Above]), + CmpOp::Gt | CmpOp::Ge => (vec![Side::Above], vec![Side::Below]), + CmpOp::Eq => (vec![Side::Below, Side::Above], Vec::new()), + CmpOp::Ne => (Vec::new(), vec![Side::Below, Side::Above]), + }; + compared.insert(*dest, (*counter, held, failed)); + } + } + if let Some(written) = block.terminator.dest() { + facts.retain(|(register, _)| *register != written); + } + let with = |sides: &[Side], counter: RegisterId| { + let mut out = facts.clone(); + out.extend(sides.iter().map(|side| (counter, *side))); + out + }; + match &block.terminator { + Terminator::Branch { + cond: Value::Register(bit), + then_block, + else_block, + } if let Some((counter, held, failed)) = compared.get(bit) => vec![ + (*then_block, with(held, *counter)), + (*else_block, with(failed, *counter)), + ], + _ => block + .successors() + .into_iter() + .map(|next| (next, facts.clone())) + .collect(), + } +} + +/// whether every write to `counter` leaves it on `side` of the short range for good: +/// narrowed from a short, set to one, or stepped away from that edge by a literal +fn never_past(function: &Function, counter: RegisterId, side: Side) -> bool { + let away = |op: BinOp, lhs: &Value, rhs: &Value| { + let Value::Fixed(step) = rhs else { + return false; + }; + let away = match (op, side) { + (BinOp::Add, Side::Above) | (BinOp::Sub, Side::Below) => *step >= 0, + (BinOp::Add, Side::Below) | (BinOp::Sub, Side::Above) => *step <= 0, + _ => false, + }; + matches!(lhs, Value::Register(lhs) if *lhs == counter) && away + }; + // how a register is written by a terminator: a narrowing is always short, and a step + // has to be one away from the edge + let terminator_writes = |register: RegisterId| { + function + .blocks + .iter() + .filter(|block| block.terminator.dest() == Some(register)) + .map(|block| match &block.terminator { + Terminator::NarrowShort { .. } => true, + Terminator::MachineStep { op, lhs, rhs, .. } => away(*op, lhs, rhs), + _ => false, + }) + .collect::>() + }; + // a step written into a register of its own and then copied back + let stepped = |source: RegisterId| { + let by_terminator = terminator_writes(source); + !by_terminator.is_empty() + && by_terminator.iter().all(|away| *away) + && !function + .blocks + .iter() + .flat_map(|block| &block.ops) + .any(|op| op.dest() == Some(source)) + }; + // a parameter holds whatever the caller wrote + counter.index() >= function.param_count + && terminator_writes(counter).iter().all(|away| *away) + && function + .blocks + .iter() + .flat_map(|block| &block.ops) + .filter(|op| op.dest() == Some(counter)) + .all(|op| match op { + Op::Assign { + src: Value::Fixed(literal), + .. + } => SHORT.contains(literal), + Op::Assign { + src: Value::Register(source), + .. + } => stepped(*source), + // a length is never negative, and never past the short range either — it + // counts objects in memory + Op::ArrayLen { .. } => side == Side::Above, + _ => false, + }) +} + #[cfg(test)] mod tests { use by_ir::builder::FunctionBuilder; - use by_ir::ops::{CmpOp, Terminator}; use by_ir::verify::verify; use super::*; + use crate::unswitch::tests::{fast_ops, module_with}; - fn module(function: Function) -> ModuleIr { - ModuleIr { - name: by_ir::ModuleName::new("app"), - functions: vec![function], - declined: Vec::new(), - classes: Vec::new(), - gradual: Vec::new(), - promoted: Vec::new(), - lines: None, - fallback_source: None, - fallback_code: None, - shims: None, - verify_install: true, - follow_recursion_limit: true, - } - } - - /// a counter set to zero and then read once, by whatever `reader` pushes - fn counted(reader: impl FnOnce(&mut FunctionBuilder, RegisterId, RegisterId)) -> ModuleIr { + /// `i = 0; while i < len(s): ; i = i + 1`, run through the pass that copies it + fn counted(reader: impl FnOnce(&mut FunctionBuilder, RegisterId, RegisterId)) -> Function { let mut builder = FunctionBuilder::new("scan", RType::BOOL); let text = builder.param("s", RType::STR); + let bound = builder.param("n", RType::INT); let counter = builder.local("i", RType::INT); - builder.push(Op::Assign { - dest: counter, - src: Value::Int(0), + let more = builder.temp(RType::BIT); + builder.assign(counter, Value::Int(0)); + let header = builder.new_block(); + let body = builder.new_block(); + let exit = builder.new_block(); + builder.terminate(Terminator::Goto(header)); + builder.switch_to(header); + builder.push(Op::IntCompare { + dest: more, + op: CmpOp::Lt, + lhs: Value::Register(counter), + rhs: Value::Register(bound), }); + builder.terminate(Terminator::Branch { + cond: Value::Register(more), + then_block: body, + else_block: exit, + }); + builder.switch_to(body); reader(&mut builder, text, counter); + builder.push(Op::IntBinary { + dest: counter, + op: BinOp::Add, + lhs: Value::Register(counter), + rhs: Value::Int(1), + }); + builder.terminate(Terminator::Goto(header)); + builder.switch_to(exit); builder.terminate(Terminator::Return(Value::Bool(true))); - let mut module = module(builder.finish()); - run(&mut module); - module + let mut module = module_with(builder.finish()); + crate::unswitch::run(&mut module); + let function = module.functions.remove(0); + assert_eq!(verify(&function), Ok(())); + function } - fn boxes(module: &ModuleIr) -> usize { - module.functions[0] - .blocks + fn boxes(function: &Function) -> usize { + fast_ops(function) .iter() - .flat_map(|block| &block.ops) - .filter(|op| matches!(op, Op::Box { .. })) + .filter(|op| matches!(op, Op::Box { .. } | Op::TagShort { .. })) .count() } #[test] fn a_counter_that_only_indexes_a_character_comparison_is_never_boxed() { - let module = counted(|builder, text, counter| { + let function = counted(|builder, text, counter| { let answer = builder.temp(RType::BIT); builder.push(Op::StrItemCompare { dest: answer, @@ -367,12 +1058,12 @@ mod tests { character: ' ', }); }); - assert_eq!(boxes(&module), 0, "{}", by_ir::print::print_module(&module)); assert_eq!( - module.functions[0].registers[1].ty, - RType::fixed(IntWidth::I64) + boxes(&function), + 0, + "{}", + by_ir::print::print_function(&function) ); - assert_eq!(verify(&module.functions[0]), Ok(())); } /// a subscript names the element at an offset, and the offset is the number the @@ -380,7 +1071,7 @@ mod tests { /// the way into the read, and codegen reaches `By_GetItemI64` rather than boxing #[test] fn a_counter_that_indexes_a_container_is_never_boxed() { - let module = counted(|builder, text, counter| { + let function = counted(|builder, text, counter| { let element = builder.temp(RType::OBJECT); builder.push(Op::GetItem { dest: element, @@ -388,35 +1079,45 @@ mod tests { index: Value::Register(counter), }); }); - assert_eq!(boxes(&module), 0, "{}", by_ir::print::print_module(&module)); assert_eq!( - module.functions[0].registers[1].ty, - RType::fixed(IntWidth::I64) + boxes(&function), + 0, + "{}", + by_ir::print::print_function(&function) ); - assert_eq!(verify(&module.functions[0]), Ok(())); } - /// a *container* that happens to be the counter is not an index, so it is boxed - /// as any other read would be + /// an object is built from the machine integer directly, with no tagged value on the way #[test] - fn a_counter_subscripted_as_the_container_still_gets_its_tagged_value_back() { - let module = counted(|builder, _text, counter| { - let element = builder.temp(RType::OBJECT); - builder.push(Op::GetItem { - dest: element, - container: Value::Register(counter), - index: Value::Int(0), + fn a_counter_built_into_an_object_is_never_tagged_first() { + let function = counted(|builder, _text, counter| { + let object = builder.temp(RType::OBJECT); + builder.push(Op::Box { + dest: object, + src: Value::Register(counter), }); }); - assert_eq!(boxes(&module), 1, "{}", by_ir::print::print_module(&module)); + let ops = fast_ops(&function); + assert!( + ops.iter() + .any(|op| matches!(op, Op::Box { src, .. } if is_fixed(&function, src))), + "{}", + by_ir::print::print_function(&function) + ); + assert_eq!( + boxes(&function), + 1, + "{}", + by_ir::print::print_function(&function) + ); } - /// the fused comparison is the only reader that takes a machine index. a - /// character *read* has to produce the character as an object, and the index it - /// is asked for is the tagged one + /// the fused comparison is the only character reader that takes a machine index. a + /// character *read* has to produce the character as an object, and the index it is + /// asked for is the tagged one #[test] fn a_counter_that_reads_a_character_out_still_gets_its_tagged_value_back() { - let module = counted(|builder, text, counter| { + let function = counted(|builder, text, counter| { let character = builder.temp(RType::STR); builder.push(Op::StrGetItem { dest: character, @@ -424,7 +1125,137 @@ mod tests { index: Value::Register(counter), }); }); - assert_eq!(boxes(&module), 1, "{}", by_ir::print::print_module(&module)); - assert_eq!(verify(&module.functions[0]), Ok(())); + assert_eq!( + boxes(&function), + 1, + "{}", + by_ir::print::print_function(&function) + ); + } + + /// `m = m + n` with `n` a counter, `m = m + step` with `step` a bound nothing in the loop + /// writes, and `m = m + product` with `product` written from a multiplication, whose + /// value has no machine form + fn summing(source: &str) -> (Function, RegisterId, BlockId, BlockId) { + let mut builder = FunctionBuilder::new("sums", RType::INT); + let step = builder.param("step", RType::INT); + let n = builder.local("n", RType::INT); + let m = builder.local("m", RType::INT); + let product = builder.temp(RType::INT); + let more = builder.temp(RType::BIT); + builder.assign(n, Value::Int(1)); + builder.assign(m, Value::Int(0)); + let header = builder.new_block(); + let body = builder.new_block(); + let exit = builder.new_block(); + builder.terminate(Terminator::Goto(header)); + builder.switch_to(header); + builder.push(Op::IntCompare { + dest: more, + op: CmpOp::Lt, + lhs: Value::Register(m), + rhs: Value::Int(100), + }); + builder.terminate(Terminator::Branch { + cond: Value::Register(more), + then_block: body, + else_block: exit, + }); + builder.switch_to(body); + builder.push(Op::IntBinary { + dest: n, + op: BinOp::Add, + lhs: Value::Register(n), + rhs: Value::Int(1), + }); + builder.push(Op::IntBinary { + dest: product, + op: BinOp::Mul, + lhs: Value::Register(n), + rhs: Value::Int(2), + }); + let added = match source { + "counter" => n, + "bound" => step, + _ => product, + }; + builder.push(Op::IntBinary { + dest: m, + op: BinOp::Add, + lhs: Value::Register(m), + rhs: Value::Register(added), + }); + builder.terminate(Terminator::Goto(header)); + builder.switch_to(exit); + builder.terminate(Terminator::Return(Value::Register(m))); + (builder.finish(), m, header, body) + } + + #[test] + fn a_counter_written_from_a_sum_of_counters_and_bounds_is_a_counter() { + for (source, counted) in [("counter", true), ("bound", true), ("product", false)] { + let (function, m, header, body) = summing(source); + assert_eq!( + counters(&function, &[header, body]).contains(&m), + counted, + "{source}" + ); + } + let (function, _, header, body) = summing("bound"); + let found = counters(&function, &[header, body]); + assert_eq!( + bounds(&function, &[header, body], &found), + vec![RegisterId(0)] + ); + } + + /// a handler outside the loop is reached from the middle of a trip with nothing on the + /// way to write a counter back, so a counter it reads stays tagged + #[test] + fn a_counter_a_handler_outside_the_loop_reads_stays_tagged() { + let mut builder = FunctionBuilder::new("scan", RType::INT); + let bound = builder.param("n", RType::INT); + let callee = builder.param("f", RType::OBJECT); + let counter = builder.local("i", RType::INT); + let more = builder.temp(RType::BIT); + let called = builder.temp(RType::OBJECT); + builder.assign(counter, Value::Int(0)); + let header = builder.new_block(); + let body = builder.new_block(); + let exit = builder.new_block(); + let handler = builder.new_block(); + builder.terminate(Terminator::Goto(header)); + builder.switch_to(header); + builder.push(Op::IntCompare { + dest: more, + op: CmpOp::Lt, + lhs: Value::Register(counter), + rhs: Value::Register(bound), + }); + builder.terminate(Terminator::Branch { + cond: Value::Register(more), + then_block: body, + else_block: exit, + }); + builder.switch_to(body); + builder.push(Op::CallValue { + dest: called, + callee: Value::Register(callee), + args: Vec::new(), + }); + builder.push(Op::IntBinary { + dest: counter, + op: BinOp::Add, + lhs: Value::Register(counter), + rhs: Value::Int(1), + }); + builder.terminate(Terminator::Goto(header)); + builder.switch_to(exit); + builder.terminate(Terminator::Return(Value::Int(0))); + builder.switch_to(handler); + builder.terminate(Terminator::Return(Value::Register(counter))); + let mut function = builder.finish(); + function.blocks[body.index()].error_target = Some(handler); + assert_eq!(counters(&function, &[header, body]), Vec::new()); } } diff --git a/crates/by_opt/src/unswitch.rs b/crates/by_opt/src/unswitch.rs index e88a7319ed..3d73d44396 100644 --- a/crates/by_opt/src/unswitch.rs +++ b/crates/by_opt/src/unswitch.rs @@ -1,30 +1,35 @@ -//! duplicating a counting loop whose bound is invariant +//! duplicating a counting loop, to hold its counters as machine integers and narrow its +//! invariant bounds //! -//! an unboxed counter compared against a bound that is still tagged tests the bound's -//! shortness on every trip. the test cannot be hoisted on its own — the bound is an -//! ordinary python `int` and may be arbitrarily large, so the answer has to be known -//! before the comparison — but when nothing in the loop writes the bound, the *answer* -//! is the same on every trip even though the test is not. +//! a counter is a machine integer only inside a copy of its loop — see +//! [`crate::unbox_counters`] for why — so a loop whose counters are worth that is +//! duplicated: one copy that holds them as machine integers, one copy exactly as it stands, +//! and narrowings ahead of both that pick between them. //! -//! so the loop is duplicated: one copy whose guard is a machine comparison against a -//! bound narrowed once on the way in, one copy exactly as it stands today, and a -//! narrowing terminator ahead of both that picks between them. on a scalar float loop -//! that is 6% of the whole running time, and it is what closes the last of the gap to -//! mypyc on `mandel` — the shortness branch, not its computation, was the cost. +//! the same copy narrows the loop's bounds. a machine counter compared against a bound that +//! is still tagged tests the bound's shortness on every trip. the test cannot be hoisted on +//! its own — the bound is an ordinary python `int` and may be arbitrarily large, so the +//! answer has to be known before the comparison — but when nothing in the loop writes the +//! bound, the *answer* is the same on every trip even though the test is not. on a scalar +//! float loop that is 6% of the whole running time, and it is what closes the last of the +//! gap to mypyc on `mandel` — the shortness branch, not its computation, was the cost. //! //! see `Terminator::NarrowShort` for why the test and the narrowing are one thing. -use std::collections::BTreeMap; +use std::collections::{BTreeMap, HashSet}; -use by_ir::function::{BasicBlock, Function, ModuleIr, RegisterDecl}; -use by_ir::ops::{BlockId, Op, RegisterId, Terminator, Value}; -use by_ir::rtype::{IntWidth, Primitive, RType}; +use by_ir::function::{BasicBlock, Function, ModuleIr}; +use by_ir::ops::{BlockId, Op, Terminator}; + +use crate::guard_loops::{map_edges, natural_loop}; +use crate::liveness::{live_in, read_registers}; +use crate::unbox_counters::{bounds, counters_given, enter, unbox}; /// how many loops in one function may be duplicated /// /// every unswitch copies a loop body, so an unbounded pass would grow a deeply nested -/// function geometrically. four covers a triply-nested loop and its own copies, which -/// is past anything the benchmarks contain +/// function geometrically. a nest is one duplicate however deep it is, so four is spent +/// only where a nest's loops have to be taken one at a time const MAX_PER_FUNCTION: usize = 4; /// the largest loop worth duplicating, in blocks @@ -44,151 +49,197 @@ pub(crate) fn run(module: &mut ModuleIr) { } } -/// duplicate the first loop whose bound is invariant, reporting whether one was found +/// duplicate the outermost loop worth it, reporting whether one was found fn unswitch_one(function: &mut Function) -> bool { let Some(candidate) = candidate(function) else { return false; }; let Candidate { header, - bound, + counters, + bounds, body, } = candidate; - let narrowed = RegisterId(function.registers.len()); - function.registers.push(RegisterDecl { - name: None, - ty: RType::fixed(IntWidth::I64), - borrowed: false, - may_be_unassigned: false, - }); - // the copies land at the end, so every existing block keeps its id and only the // edges into the header have to move + let before = function.blocks.len(); let copies: BTreeMap = body .iter() .enumerate() - .map(|(offset, id)| (*id, BlockId(function.blocks.len() + offset))) + .map(|(offset, id)| (*id, BlockId(before + offset))) .collect(); for id in &body { let mut block = function.blocks[id.index()].clone(); - retarget(&mut block, &copies); - if *id == header { - narrow_guard(&mut block, bound, narrowed); - } + map_edges(&mut block, true, |id| { + copies.get(&id).copied().unwrap_or(id) + }); function.blocks.push(block); } + let narrowings = unbox(function, header, &copies, &counters, &bounds); + + // the copy is entered only once everything it reads has narrowed + let entries: Vec = (0..before) + .map(BlockId) + .filter(|id| { + !body.contains(id) + && function.blocks[id.index()] + .terminator + .successors() + .contains(&header) + }) + .collect(); + let preheader = enter(function, header, &entries, &narrowings, copies[&header]); // the preheader replaces the header at every edge that entered the loop from - // outside it, so both copies are entered only through the narrowing - let preheader = BlockId(function.blocks.len()); - function.blocks.push(BasicBlock { - ops: Vec::new(), - terminator: Terminator::NarrowShort { - dest: narrowed, - src: Value::Register(bound), - fits: copies[&header], - otherwise: header, - }, - owned_at_exit: None, - range: function.blocks[header.index()].range, - position: function.blocks[header.index()].position, - error_target: None, - }); - // the preheader is excluded along with the loop: its own `otherwise` edge *is* an - // edge to the header, and redirecting that one would send it to itself - for id in 0..function.blocks.len() { - let block = BlockId(id); - if block == preheader || body.contains(&block) || copies.values().any(|copy| *copy == block) - { + // outside it. every block added here is inside: the narrowings' own `otherwise` edges + // *are* edges to the header, and so are the back edges of the pieces of the loop as + // written that a step leaves for + for id in 0..before { + if body.contains(&BlockId(id)) { continue; } - redirect(&mut function.blocks[id], header, preheader); + map_edges(&mut function.blocks[id], false, |target| { + if target == header { preheader } else { target } + }); } + clear_unreached(function, &body); true } +/// empty the blocks of the loop as written that nothing reaches any more +/// +/// a copy that narrows nothing is entered unconditionally, and one whose steps cannot +/// leave the machine word has nowhere to leave for, so the loop as written would only be +/// dead C +fn clear_unreached(function: &mut Function, body: &[BlockId]) { + let mut seen: HashSet = HashSet::from([Function::entry()]); + let mut queue = vec![Function::entry()]; + while let Some(id) = queue.pop() { + let block = &function.blocks[id.index()]; + for next in block.successors().into_iter().chain(block.error_target) { + if seen.insert(next) { + queue.push(next); + } + } + } + for id in body { + if !seen.contains(id) { + let block = &mut function.blocks[id.index()]; + block.ops.clear(); + block.terminator = Terminator::Unreachable; + block.error_target = None; + } + } +} + /// a loop that can be duplicated struct Candidate { /// the block holding the guard, which is the loop's header header: BlockId, - /// the tagged bound the guard compares against - bound: RegisterId, - /// every block on a path from the header back to itself, the header included + /// the registers the copy holds as machine integers + counters: Vec, + /// every tagged bound a guard inside the loop compares a counter against that nothing + /// inside it writes — each narrowed once on the way in + bounds: Vec, + /// the loop's blocks, the header and every loop nested inside it included body: Vec, } +/// the loop worth duplicating first: of every loop with a counter to hold as a machine +/// integer or a bound to narrow, the one holding the most blocks +/// +/// that is the outermost of a nest, and its copy takes every counter and bound the nest +/// reads that it can — so an inner loop gets its fast form from the one copy rather than +/// from a copy of its own inside each copy of the loops around it +/// +/// only a loop the program reaches without leaving a copy is asked about. the loop as +/// written that a copy leaves for runs only where a value did not narrow or a step left +/// the machine word, and a copy of it would be code for a path that is already the rare one fn candidate(function: &Function) -> Option { - for (index, block) in function.blocks.iter().enumerate() { + let fast = reached_without_leaving(function); + let live = live_in(function, &read_registers); + let mut best: Option = None; + for index in 0..function.blocks.len() { let header = BlockId(index); - let Some(bound) = guard_bound(function, block) else { + // a header that is the function's own entry has no edge into it from outside, and + // the preheader takes the loop's entry edges, so without one it would be + // unreachable and the copy dead + if header == Function::entry() || !fast.contains(&header) { continue; - }; - let body = cycle_through(function, header); + } + let body = loop_blocks(function, header); if body.len() < 2 || body.len() > MAX_BODY { continue; } - // the bound is only invariant if nothing round the loop writes it. a narrowing - // ahead of the loop would otherwise describe a value the body has replaced - if body - .iter() - .any(|id| writes(&function.blocks[id.index()], bound)) + if best + .as_ref() + .is_some_and(|best| best.body.len() >= body.len()) { continue; } - // an already-duplicated loop is entered through its own narrowing, and doing - // it again would narrow a bound that is no longer read here - if function + if !function .blocks .iter() - .any(|block| enters_by_narrowing(block, header)) + .enumerate() + .any(|(id, block)| !body.contains(&BlockId(id)) && block.successors().contains(&header)) { continue; } - // the preheader takes the loop's entry edges, so without one it would be - // unreachable and the copy dead. a header that is the function's own entry has - // no such edge - if header == Function::entry() - || !function.blocks.iter().enumerate().any(|(id, block)| { - !body.contains(&BlockId(id)) && block.successors().contains(&header) - }) - { + let counters = counters_given(function, &body, &live); + let bounds = bounds(function, &body, &counters); + if counters.is_empty() && bounds.is_empty() { continue; } - return Some(Candidate { + best = Some(Candidate { header, - bound, + counters, + bounds, body, }); } - None + best } -/// the tagged register this block's guard compares an unboxed counter against -/// -/// only a guard reading a *register* qualifies: an immediate bound is already folded, -/// and there would be nothing to narrow once -fn guard_bound(function: &Function, block: &BasicBlock) -> Option { - block.ops.iter().find_map(|op| { - let Op::IntCompare { lhs, rhs, .. } = op else { - return None; - }; - let Value::Register(bound) = rhs else { - return None; +/// every block reached from the entry without taking an edge that leaves a copy for the +/// loop it was made of: a narrowing that did not fit, a step that left the machine word, +/// or an entry test that answered no +fn reached_without_leaving(function: &Function) -> HashSet { + let mut seen: HashSet = HashSet::from([Function::entry()]); + let mut queue = vec![Function::entry()]; + while let Some(id) = queue.pop() { + let block = &function.blocks[id.index()]; + let taken = match &block.terminator { + Terminator::NarrowShort { fits, .. } | Terminator::MachineStep { fits, .. } => { + vec![*fits] + } + Terminator::Branch { then_block, .. } + if matches!(block.ops.last(), Some(Op::LoopGuardsHold { .. })) => + { + vec![*then_block] + } + _ => block.successors(), }; - let fixed = matches!( - operand_type(function, lhs), - Some(RType::Primitive(Primitive::Fixed(_))) - ); - let tagged = function.register(*bound).map(|decl| &decl.ty) == Some(&RType::INT); - (fixed && tagged).then_some(*bound) - }) + for next in taken.into_iter().chain(block.error_target) { + if seen.insert(next) { + queue.push(next); + } + } + } + seen } -fn operand_type(function: &Function, value: &Value) -> Option { - match value { - Value::Register(id) => function.register(*id).map(|decl| decl.ty.clone()), - other => other.immediate_type(), +/// the blocks of the loop `header` heads +/// +/// the natural loop, which leaves out the loops around it. a loop entered other than +/// through its header — the original a versioned loop leaves for, entered again from its +/// copy's exits — has no natural loop, and is every block on a cycle through its header +fn loop_blocks(function: &Function, header: BlockId) -> Vec { + let natural = natural_loop(function, header); + if natural.is_empty() { + cycle_through(function, header) + } else { + natural } } @@ -233,107 +284,36 @@ fn reachable(function: &Function, start: BlockId, reverse: bool) -> Vec seen } -fn writes(block: &BasicBlock, register: RegisterId) -> bool { - block.ops.iter().any(|op| op.dest() == Some(register)) - || block.terminator.dest() == Some(register) -} - -fn enters_by_narrowing(block: &BasicBlock, header: BlockId) -> bool { - matches!( - block.terminator, - Terminator::NarrowShort { otherwise, .. } if otherwise == header - ) -} - -/// send this block's edges into the loop to the copies instead -fn retarget(block: &mut BasicBlock, copies: &BTreeMap) { - let map = |id: &mut BlockId| { - if let Some(copy) = copies.get(id) { - *id = *copy; - } - }; - match &mut block.terminator { - Terminator::Goto(target) => map(target), - Terminator::Branch { - then_block, - else_block, - .. - } => { - map(then_block); - map(else_block); - } - Terminator::NarrowShort { - fits, otherwise, .. - } => { - map(fits); - map(otherwise); - } - Terminator::Return(_) | Terminator::Unreachable => {} - } - if let Some(target) = &mut block.error_target { - map(target); - } -} - -/// point every edge to `from` at `to`, leaving the error edge alone -/// -/// an error edge reaches a handler, never a loop header, and redirecting one through a -/// narrowing would run the narrowing with an exception set -fn redirect(block: &mut BasicBlock, from: BlockId, to: BlockId) { - let map = |id: &mut BlockId| { - if *id == from { - *id = to; - } - }; - match &mut block.terminator { - Terminator::Goto(target) => map(target), - Terminator::Branch { - then_block, - else_block, - .. - } => { - map(then_block); - map(else_block); - } - Terminator::NarrowShort { - fits, otherwise, .. - } => { - map(fits); - map(otherwise); - } - Terminator::Return(_) | Terminator::Unreachable => {} - } -} - -/// read the guard's bound out of the narrowed register instead of the tagged one -fn narrow_guard(block: &mut BasicBlock, bound: RegisterId, narrowed: RegisterId) { - for op in &mut block.ops { - if let Op::IntCompare { rhs, .. } = op - && *rhs == Value::Register(bound) - { - *rhs = Value::Register(narrowed); - } - } -} - #[cfg(test)] -mod tests { +pub(crate) mod tests { use super::*; use by_ir::builder::FunctionBuilder; - use by_ir::ops::CmpOp; + use by_ir::ops::{BinOp, CmpOp, RegisterId, Value}; + use by_ir::rtype::RType; use by_ir::verify::verify; - /// `i = 0; while i < bound: i = i + 1`, with `bound` a parameter + use crate::unbox_counters::is_fixed; + + /// `i = 0; while i < bound: i = i + 1; return i`, with `bound` a parameter /// - /// `step_writes_bound` reassigns the bound in the body instead, which is the one - /// thing that makes the loop ineligible - fn counting_loop(step_writes_bound: bool) -> Function { + /// `body_writes_bound` also multiplies the bound in the body, which is a write no + /// machine integer takes over and one that makes the bound no longer invariant. + /// `from_parameter` starts `i` from a second parameter instead of from zero + fn counting_loop(body_writes_bound: bool, from_parameter: bool) -> Function { let mut builder = FunctionBuilder::new("count", RType::INT); let bound = builder.param("bound", RType::INT); + let start = builder.param("start", RType::INT); let index = builder.local("i", RType::INT); let more = builder.temp(RType::BIT); let next = builder.temp(RType::INT); - builder.assign(index, Value::Int(0)); + builder.assign( + index, + if from_parameter { + Value::Register(start) + } else { + Value::Int(0) + }, + ); let header = builder.new_block(); let body = builder.new_block(); @@ -356,13 +336,18 @@ mod tests { builder.switch_to(body); builder.push(Op::IntBinary { dest: next, - op: by_ir::ops::BinOp::Add, + op: BinOp::Add, lhs: Value::Register(index), rhs: Value::Int(1), }); builder.assign(index, Value::Register(next)); - if step_writes_bound { - builder.assign(bound, Value::Register(next)); + if body_writes_bound { + builder.push(Op::IntBinary { + dest: bound, + op: BinOp::Mul, + lhs: Value::Register(bound), + rhs: Value::Int(1), + }); } builder.terminate(Terminator::Goto(header)); @@ -371,96 +356,408 @@ mod tests { builder.finish() } - fn module_with(function: Function) -> ModuleIr { - ModuleIr { - name: by_ir::ModuleName::new("app"), - functions: vec![function], - declined: Vec::new(), - classes: Vec::new(), - gradual: Vec::new(), - promoted: Vec::new(), - lines: None, - fallback_source: None, - fallback_code: None, - shims: None, - verify_install: true, - follow_recursion_limit: true, - } + pub(crate) fn module_with(function: Function) -> ModuleIr { + let mut module = ModuleIr::new("app"); + module.functions.push(function); + module + } + + fn unswitched(function: Function) -> Function { + let mut module = module_with(function); + run(&mut module); + let function = module.functions.remove(0); + assert_eq!( + verify(&function), + Ok(()), + "{}", + by_ir::print::print_function(&function) + ); + function } - fn narrowings(function: &Function) -> usize { + /// the tagged registers the narrowings ahead of the copies read + fn narrowed(function: &Function) -> Vec { function .blocks .iter() - .filter(|block| matches!(block.terminator, Terminator::NarrowShort { .. })) - .count() + .filter_map(|block| match block.terminator { + Terminator::NarrowShort { + src: Value::Register(src), + .. + } => Some(src), + _ => None, + }) + .collect() + } + + /// the operations of every block the function reaches without leaving a copy + pub(crate) fn fast_ops(function: &Function) -> Vec<&Op> { + let fast = reached_without_leaving(function); + let mut ids: Vec = fast.into_iter().collect(); + ids.sort_unstable(); + ids.iter() + .flat_map(|id| &function.blocks[id.index()].ops) + .collect() + } + + fn compares(ops: &[&Op]) -> Vec<(Value, Value)> { + ops.iter() + .filter_map(|op| match op { + Op::IntCompare { lhs, rhs, .. } => Some((lhs.clone(), rhs.clone())), + _ => None, + }) + .collect() } #[test] - fn a_loop_whose_bound_is_a_parameter_is_duplicated() { - let mut module = module_with(counting_loop(false)); - crate::unbox_counters::run(&mut module); - let before = module.functions[0].blocks.len(); - run(&mut module); - let function = &module.functions[0]; + fn a_counting_loop_is_duplicated_with_its_counter_and_its_bound_narrowed() { + let function = unswitched(counting_loop(false, true)); + let printed = by_ir::print::print_function(&function); + let mut sources = narrowed(&function); + sources.sort_unstable(); + assert_eq!(sources, vec![RegisterId(0), RegisterId(2)], "{printed}"); + // the whole point: the copy's guard is a comparison of two machine integers + let guards = compares(&fast_ops(&function)); + assert_eq!(guards.len(), 1, "{printed}"); + for (lhs, rhs) in guards { + assert!( + is_fixed(&function, &lhs) && is_fixed(&function, &rhs), + "{printed}" + ); + } + } - assert_eq!(narrowings(function), 1); - // the body and its header, plus the preheader - assert_eq!(function.blocks.len(), before + 3); - assert!(verify(function).is_ok(), "{:?}", verify(function)); + #[test] + fn a_counter_set_to_a_short_literal_ahead_of_the_loop_is_not_tested_on_the_way_in() { + let function = unswitched(counting_loop(false, false)); + let printed = by_ir::print::print_function(&function); + assert_eq!(narrowed(&function), vec![RegisterId(0)], "{printed}"); + assert!( + fast_ops(&function).iter().any(|op| matches!( + op, + Op::Assign { + src: Value::Fixed(0), + .. + } + )), + "{printed}" + ); } #[test] - fn the_duplicate_compares_two_machine_integers() { - let mut module = module_with(counting_loop(false)); - crate::unbox_counters::run(&mut module); - run(&mut module); - let function = &module.functions[0]; + fn the_loop_as_written_keeps_its_tagged_counter() { + let function = unswitched(counting_loop(false, true)); + let printed = by_ir::print::print_function(&function); + let tagged = compares( + &function + .blocks + .iter() + .flat_map(|block| &block.ops) + .collect::>(), + ) + .into_iter() + .filter(|(lhs, _)| *lhs == Value::Register(RegisterId(2))) + .count(); + assert_eq!(tagged, 1, "{printed}"); + } - let Some(Terminator::NarrowShort { dest, fits, .. }) = function + #[test] + fn a_step_that_leaves_the_machine_word_writes_the_counter_back_and_steps_it_as_written() { + let function = unswitched(counting_loop(false, true)); + let printed = by_ir::print::print_function(&function); + let Some(Terminator::MachineStep { + lhs: Value::Register(machine), + overflows, + .. + }) = function .blocks .iter() .map(|block| &block.terminator) - .find(|terminator| matches!(terminator, Terminator::NarrowShort { .. })) - .cloned() + .find(|terminator| matches!(terminator, Terminator::MachineStep { .. })) else { - panic!("the loop was not duplicated"); + panic!("the copy has no machine step: {printed}"); }; - // the whole point: on the edge the narrowing takes, the guard reads the machine - // register rather than the tagged one, which is what makes it a plain compare - let guard = function.blocks[fits.index()] - .ops - .iter() - .find_map(|op| match op { - Op::IntCompare { rhs, .. } => Some(rhs.clone()), - _ => None, - }) - .expect("the copy keeps its guard"); - assert_eq!(guard, Value::Register(dest)); + let landing = &function.blocks[overflows.index()]; + assert_eq!( + landing.ops, + vec![Op::Box { + dest: RegisterId(2), + src: Value::Register(*machine), + }], + "{printed}" + ); + let Terminator::Goto(resume) = landing.terminator else { + panic!("the landing does not continue in the loop as written: {printed}"); + }; + assert!( + matches!( + function.blocks[resume.index()].ops.first(), + Some(Op::IntBinary { + lhs: Value::Register(RegisterId(2)), + .. + }) + ), + "{printed}" + ); } #[test] - fn a_bound_the_body_writes_is_left_alone() { - let mut module = module_with(counting_loop(true)); - crate::unbox_counters::run(&mut module); - let before = module.functions[0].blocks.len(); - run(&mut module); + fn a_bound_the_body_writes_is_not_narrowed() { + let function = unswitched(counting_loop(true, true)); + let printed = by_ir::print::print_function(&function); + assert_eq!(narrowed(&function), vec![RegisterId(2)], "{printed}"); + } + + /// `total = 0.0; y = 0; while y < rows: x = 0; while x < columns: total = total + + /// float(columns); x = x + 1; y = y + 1`, with `rows` and `columns` parameters + /// + /// `inner_bound_written` reassigns `columns` in the outer body instead, which leaves the + /// inner bound invariant across the inner loop alone + fn nested_loops(inner_bound_written: bool) -> Function { + let mut builder = FunctionBuilder::new("grid", RType::FLOAT); + let rows = builder.param("rows", RType::INT); + let columns = builder.param("columns", RType::INT); + let y = builder.local("y", RType::INT); + let x = builder.local("x", RType::INT); + let total = builder.local("total", RType::FLOAT); + let outer_more = builder.temp(RType::BIT); + let inner_more = builder.temp(RType::BIT); + let widened = builder.temp(RType::FLOAT); + let next_x = builder.temp(RType::INT); + let next_y = builder.temp(RType::INT); + builder.push(Op::Assign { + dest: total, + src: Value::Float(0.0), + }); + builder.assign(y, Value::Int(0)); + + let outer = builder.new_block(); + let outer_body = builder.new_block(); + let inner = builder.new_block(); + let inner_body = builder.new_block(); + let outer_step = builder.new_block(); + let exit = builder.new_block(); + builder.terminate(Terminator::Goto(outer)); + + builder.switch_to(outer); + builder.push(Op::IntCompare { + dest: outer_more, + op: CmpOp::Lt, + lhs: Value::Register(y), + rhs: Value::Register(rows), + }); + builder.terminate(Terminator::Branch { + cond: Value::Register(outer_more), + then_block: outer_body, + else_block: exit, + }); + + builder.switch_to(outer_body); + builder.assign(x, Value::Int(0)); + if inner_bound_written { + builder.push(Op::IntBinary { + dest: columns, + op: BinOp::Mul, + lhs: Value::Register(rows), + rhs: Value::Int(1), + }); + } + builder.terminate(Terminator::Goto(inner)); + + builder.switch_to(inner); + builder.push(Op::IntCompare { + dest: inner_more, + op: CmpOp::Lt, + lhs: Value::Register(x), + rhs: Value::Register(columns), + }); + builder.terminate(Terminator::Branch { + cond: Value::Register(inner_more), + then_block: inner_body, + else_block: outer_step, + }); - assert_eq!(narrowings(&module.functions[0]), 0); - assert_eq!(module.functions[0].blocks.len(), before); + builder.switch_to(inner_body); + builder.push(Op::IntToFloat { + dest: widened, + src: Value::Register(columns), + }); + builder.push(Op::FloatBinary { + dest: total, + op: BinOp::Add, + lhs: Value::Register(total), + rhs: Value::Register(widened), + }); + builder.push(Op::IntBinary { + dest: next_x, + op: BinOp::Add, + lhs: Value::Register(x), + rhs: Value::Int(1), + }); + builder.assign(x, Value::Register(next_x)); + builder.terminate(Terminator::Goto(inner)); + + builder.switch_to(outer_step); + builder.push(Op::IntBinary { + dest: next_y, + op: BinOp::Add, + lhs: Value::Register(y), + rhs: Value::Int(1), + }); + builder.assign(y, Value::Register(next_y)); + builder.terminate(Terminator::Goto(outer)); + + builder.switch_to(exit); + builder.terminate(Terminator::Return(Value::Register(total))); + builder.finish() + } + + #[test] + fn a_nest_whose_bounds_are_all_invariant_is_duplicated_once() { + let function = unswitched(nested_loops(false)); + let printed = by_ir::print::print_function(&function); + // `x` is set before it is read on every trip round the outer loop, and `y` is set to + // zero just ahead of the nest, so only the two bounds are narrowed, in one chain + let mut sources = narrowed(&function); + sources.sort_unstable(); + assert_eq!(sources, vec![RegisterId(0), RegisterId(1)], "{printed}"); + let fast = fast_ops(&function); + let guards = compares(&fast); + assert_eq!(guards.len(), 2, "{printed}"); + for (lhs, rhs) in guards { + assert!( + is_fixed(&function, &lhs) && is_fixed(&function, &rhs), + "{printed}" + ); + } + // the bound widened to a double inside the copy reads the narrowed register too + assert!( + fast.iter() + .any(|op| matches!(op, Op::IntToFloat { src, .. } if is_fixed(&function, src))), + "{printed}" + ); + } + + #[test] + fn an_inner_loop_whose_bound_the_outer_one_writes_is_duplicated_on_its_own() { + let function = unswitched(nested_loops(true)); + let printed = by_ir::print::print_function(&function); + // the nest narrows `rows` on the way in, and the inner copy inside it narrows + // `columns` once per trip round the outer loop. the nest as written, which runs only + // where something did not narrow, is not duplicated again + let mut sources = narrowed(&function); + sources.sort_unstable(); + assert_eq!(sources, vec![RegisterId(0), RegisterId(1)], "{printed}"); + for (lhs, rhs) in compares(&fast_ops(&function)) { + assert!( + is_fixed(&function, &lhs) && is_fixed(&function, &rhs), + "{printed}" + ); + } } #[test] fn duplicating_twice_is_not_attempted() { - // the pass runs to a fixpoint inside one call; running it again must find - // nothing, or a loop would be copied on every pipeline run - let mut module = module_with(counting_loop(false)); - crate::unbox_counters::run(&mut module); - run(&mut module); - let after_once = module.functions[0].blocks.len(); - run(&mut module); + for function in [ + counting_loop(false, true), + nested_loops(false), + nested_loops(true), + ] { + let mut module = module_with(function); + run(&mut module); + let once = module.functions[0].blocks.len(); + run(&mut module); + assert_eq!(module.functions[0].blocks.len(), once); + } + } - assert_eq!(module.functions[0].blocks.len(), after_once); - assert_eq!(narrowings(&module.functions[0]), 1); + /// `i = start; while i < bound: doubled = i * 2; i = i + 1`, with the step moved ahead + /// of the multiplication where `stepped_first` + fn doubling_loop(start: i64, stepped_first: bool) -> Function { + let mut builder = FunctionBuilder::new("count", RType::INT); + let bound = builder.param("bound", RType::INT); + let index = builder.local("i", RType::INT); + let more = builder.temp(RType::BIT); + let next = builder.temp(RType::INT); + let doubled = builder.temp(RType::INT); + builder.assign(index, Value::Int(start)); + + let header = builder.new_block(); + let body = builder.new_block(); + let exit = builder.new_block(); + builder.terminate(Terminator::Goto(header)); + + builder.switch_to(header); + builder.push(Op::IntCompare { + dest: more, + op: CmpOp::Lt, + lhs: Value::Register(index), + rhs: Value::Register(bound), + }); + builder.terminate(Terminator::Branch { + cond: Value::Register(more), + then_block: body, + else_block: exit, + }); + + builder.switch_to(body); + let step = |builder: &mut FunctionBuilder| { + builder.push(Op::IntBinary { + dest: next, + op: BinOp::Add, + lhs: Value::Register(index), + rhs: Value::Int(1), + }); + builder.assign(index, Value::Register(next)); + }; + if stepped_first { + step(&mut builder); + } + // a multiplication has no machine form, so it reads the counter's tagged value + builder.push(Op::IntBinary { + dest: doubled, + op: BinOp::Mul, + lhs: Value::Register(index), + rhs: Value::Int(2), + }); + if !stepped_first { + step(&mut builder); + } + builder.terminate(Terminator::Goto(header)); + + builder.switch_to(exit); + builder.terminate(Terminator::Return(Value::Register(index))); + builder.finish() + } + + fn tags(function: &Function) -> usize { + fast_ops(function) + .iter() + .filter(|op| matches!(op, Op::TagShort { .. })) + .count() + } + + #[test] + fn a_counter_the_copy_s_guard_keeps_short_is_tagged_with_no_range_test() { + let function = unswitched(doubling_loop(0, false)); + let printed = by_ir::print::print_function(&function); + assert_eq!(tags(&function), 1, "{printed}"); + } + + #[test] + fn a_counter_written_after_its_guard_is_not_proven_short() { + let function = unswitched(doubling_loop(0, true)); + let printed = by_ir::print::print_function(&function); + assert_eq!(tags(&function), 0, "{printed}"); + } + + #[test] + fn a_counter_is_proven_short_by_the_narrowing_it_entered_the_copy_through() { + // the counter starts below the short range, where it does not narrow and the loop + // as written runs instead. the copy is only ever entered with a short, and a step + // up from one stays above the smallest + let function = unswitched(doubling_loop(-(1 << 62) - 1, false)); + let printed = by_ir::print::print_function(&function); + assert_eq!(tags(&function), 1, "{printed}"); } } diff --git a/crates/by_rt/include/by.h b/crates/by_rt/include/by.h index 4347c1aa4d..9aae516813 100644 --- a/crates/by_rt/include/by.h +++ b/crates/by_rt/include/by.h @@ -34,6 +34,7 @@ #include #include #include +#include #include #include #include @@ -107,6 +108,11 @@ typedef size_t ByTagged; /* a tagged word of 1 is a pointer of 0 with the tag set, so it can never be a * real object and is free to mean "an exception is set" */ #define BY_INT_ERROR ((ByTagged)1) +/* what a register holds once its reference has been let go of. a short zero rather + * than the error sentinel, because the sentinel is odd: releasing it again takes the + * branch kept for a real PyLongObject, and the emptied register is released again on + * every exit and wherever the next write lets go of what was there */ +#define BY_INT_EMPTY ((ByTagged)0) /* a float error sentinel overlaps a valid value, so an error must be confirmed * with PyErr_Occurred() — see RType::error_overlaps */ #define BY_FLOAT_ERROR (-113.0) @@ -858,14 +864,111 @@ static inline double By_FloatObjDiv(double a, PyObject *b) { return By_FloatObjectSlow(a, b, PyNumber_TrueDivide); } -static inline double By_FloatPow(double a, double b) { return pow(a, b); } +/* `0.0 ** -1.0`, in the running interpreter's own words: 3.13 and 3.14 spell it + * differently, so the power is re-performed on a pair that must fail the same way */ +BY_COLD void By_ZeroToNegativePower(void) { + PyObject *zero = PyFloat_FromDouble(0.0); + PyObject *negative = PyFloat_FromDouble(-1.0); + PyObject *impossible = NULL; + if (zero != NULL && negative != NULL) impossible = PyNumber_Power(zero, negative, Py_None); + Py_XDECREF(zero); + Py_XDECREF(negative); + Py_XDECREF(impossible); + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_ZeroDivisionError, "zero to a negative power"); + } +} + +/* python's `float.__pow__`, case for case (cpython's `float_pow`), for a pair whose + * answer is known to be real: a negative base to a fractional power is a complex number + * in python, and the lowering sends any pair that could be one through the object + * protocol instead. the platform's `pow` is only asked about what cpython asks it, and + * a result it overflows raises `OverflowError` with the errno cpython reports */ +static double By_FloatPow(double iv, double iw) { + int negate = 0; + if (iw == 0.0) return 1.0; + if (isnan(iv)) return iv; + if (isnan(iw)) return iv == 1.0 ? 1.0 : iw; + if (isinf(iw)) { + iv = fabs(iv); + if (iv == 1.0) return 1.0; + if ((iw > 0.0) == (iv > 1.0)) return fabs(iw); + return 0.0; + } + if (isinf(iv)) { + int odd = fmod(fabs(iw), 2.0) == 1.0; + if (iw > 0.0) return odd ? iv : fabs(iv); + return odd ? copysign(0.0, iv) : 0.0; + } + if (iv == 0.0) { + int odd = fmod(fabs(iw), 2.0) == 1.0; + if (iw < 0.0) { + By_ZeroToNegativePower(); + return BY_FLOAT_ERROR; + } + return odd ? iv : 0.0; + } + if (iv < 0.0) { + if (iw != floor(iw)) { + PyErr_SetString(PyExc_SystemError, "a complex power reached the float lowering"); + return BY_FLOAT_ERROR; + } + iv = -iv; + negate = fmod(fabs(iw), 2.0) == 1.0; + } + if (iv == 1.0) return negate ? -1.0 : 1.0; + errno = 0; + double ix = pow(iv, iw); + /* cpython's `_Py_ADJUST_ERANGE1`: a platform that leaves errno alone on overflow + * still has to raise, and an underflow to zero is not an error */ + if (errno == 0) { + if (ix == HUGE_VAL || ix == -HUGE_VAL) errno = ERANGE; + } else if (errno == ERANGE && ix == 0.0) { + errno = 0; + } + if (negate) ix = -ix; + if (errno != 0) { + PyErr_SetFromErrno(errno == ERANGE ? PyExc_OverflowError : PyExc_ValueError); + return BY_FLOAT_ERROR; + } + return ix; +} + +/* the quotient and remainder of a divisor already known not to be zero, as cpython's + * `_float_div_mod` and `float_rem` compute them. emitted code tests the divisor itself + * and jumps straight to its error edge, so the answer never has to be told apart from + * an error value that is also a legal double + * + * `floor(a / b)` is not the same operation: `inf // 1.5` is a nan in python, since the + * remainder it is derived from is one, and `1.5 // -inf` is `-1.0` */ +static inline double By_FloatFloorDivNonzero(double vx, double wx) { + double mod = fmod(vx, wx); + double div = (vx - mod) / wx; + if (mod != 0.0 && ((wx < 0.0) != (mod < 0.0))) div -= 1.0; + if (div != 0.0) { + double floordiv = floor(div); + if (div - floordiv > 0.5) floordiv += 1.0; + return floordiv; + } + return copysign(0.0, vx / wx); +} + +static inline double By_FloatModNonzero(double vx, double wx) { + double mod = fmod(vx, wx); + /* python's % takes the sign of the divisor, a zero remainder included */ + if (mod != 0.0) { + if ((wx < 0.0) != (mod < 0.0)) mod += wx; + return mod; + } + return copysign(0.0, wx); +} static inline double By_FloatFloorDiv(double a, double b) { if (b == 0.0) { By_ZeroDivision(PyNumber_FloorDivide, 1); return BY_FLOAT_ERROR; } - return floor(a / b); + return By_FloatFloorDivNonzero(a, b); } static inline double By_FloatMod(double a, double b) { @@ -873,10 +976,7 @@ static inline double By_FloatMod(double a, double b) { By_ZeroDivision(PyNumber_Remainder, 1); return BY_FLOAT_ERROR; } - double r = fmod(a, b); - /* python's % takes the sign of the divisor */ - if (r != 0.0 && ((r < 0.0) != (b < 0.0))) r += b; - return r; + return By_FloatModNonzero(a, b); } /* the conversion `float.__add__` performs on an `int` operand: correctly rounded, @@ -889,6 +989,11 @@ static inline double By_TaggedToDouble(ByTagged x) { return v; } +/* the half of the conversion that reaches cpython, for a caller that has tested the + * value short itself: `-1.0` with an exception set is the failure, as it is for + * `PyLong_AsDouble` */ +BY_COLD double By_TaggedToDoubleSlow(ByTagged x) { return PyLong_AsDouble(By_LongOf(x)); } + static inline double By_FloatTrueDiv(double a, double b) { if (b == 0.0) { By_ZeroDivision(PyNumber_TrueDivide, 1); @@ -999,6 +1104,13 @@ static inline PyObject *By_CallInterpreted(PyObject *fn, const char *name, return PyObject_Vectorcall(fn, args, (size_t)nargs, kwnames); } +/* `float`'s narrowing as a test, for a caller that branches on it and reads the double + * itself: the double a float holds may be any value at all, so a failure that is only an + * error value would have to ask the thread whether an exception is set on every read */ +static inline int By_IsFloat(PyObject *o) { return o != NULL && PyFloat_Check(o); } + +BY_COLD void By_UnboxFloatFailed(PyObject *o) { By_TypeError("float", o); } + static inline double By_UnboxFloat(PyObject *o) { /* `PyFloat_Check` admits float subclasses but not `int`, which is exactly * what `.by`'s `float` means */ @@ -2079,23 +2191,6 @@ BY_HOT void By_DepthLeave(ByDepth depth) { depth.thread->BY_PY_RECURSION_REMAINI #endif /* BY_RECURSION_STACK_ONLY */ -/* resume a compiled generator or coroutine's frame, counted as python counts pushing - * the frame it would have resumed - * - * this is a cycle the call graph cannot see: `yield from f(n - 1)` makes the next - * generator with a native call and then steps it through the iterator protocol, and the - * interpreter counts none of that — so a generator delegating to itself ran off the - * stack. NULL with `RecursionError` set where the frame could not be pushed, which the - * caller finishes the generator on, as python closes one whose frame could not be */ -static inline PyObject *By_ResumeCounted(PyObject *self, PyObject *(*resume)(PyObject *)) { - ByDepth depth = By_DepthHere(); - PyObject *result; - if (BY_UNLIKELY(By_DepthEnter(depth))) return NULL; - result = resume(self); - By_DepthLeave(depth); - return result; -} - /* a global read that remembers what the name resolved to last time * * `By_LookupGlobal` is two dict probes on every trip, and for a name that resolves to @@ -8436,18 +8531,27 @@ BY_COLD PyObject *By_ItemSlow(PyObject *container, ByTagged index) { * * the index is an `int64_t` rather than a `Py_ssize_t` because that is what an * unboxed counter is. it is narrowed only after `i < n` has proved it is a - * position in this list */ -static inline PyObject *By_ListItemAt(PyObject *container, int64_t i) { + * position in this list + * + * the element is the list's, not the caller's: it stays alive only while nothing + * can run python, since python is what could shrink the list and drop it */ +static inline PyObject *By_ListItemBorrowed(PyObject *container, int64_t i) { if (BY_LIKELY(container != NULL && PyList_CheckExact(container))) { int64_t n = (int64_t)PyList_GET_SIZE(container); if (i < 0) i += n; if (BY_LIKELY(i >= 0 && i < n)) { - return By_NewRef(PyList_GET_ITEM(container, (Py_ssize_t)i)); + return PyList_GET_ITEM(container, (Py_ssize_t)i); } } return NULL; } +/* as [`By_ListItemBorrowed`], with a reference of the caller's own */ +static inline PyObject *By_ListItemAt(PyObject *container, int64_t i) { + PyObject *item = By_ListItemBorrowed(container, i); + return item == NULL ? NULL : By_NewRef(item); +} + /* `container[index]` where the index is already an integer register * * boxing one to look up a list element allocates a `PyLongObject` per iteration @@ -8833,14 +8937,16 @@ static inline void *By_ArrayItems(ByArrayHeader *array) { return (void *)(array /* the index a `list` would use, normalized and bounds-checked the same way — a * negative index counts from the end, and out of range is `IndexError` */ -static inline Py_ssize_t By_ArrayIndex(ByArrayHeader *array, ByTagged tagged) { +/* the same, for an index a counter holds as a machine integer. no index past the short + * range fits a buffer, so this answers exactly as the tagged form does for every value + * the counter can hold */ +static inline Py_ssize_t By_ArrayIndexI64(ByArrayHeader *array, int64_t at) { if (array == NULL) return -1; - if (BY_UNLIKELY(!By_IsShort(tagged))) { - // a big integer cannot be a valid index into a buffer this size + if (BY_UNLIKELY(at < PY_SSIZE_T_MIN || at > PY_SSIZE_T_MAX)) { PyErr_SetString(PyExc_IndexError, "list index out of range"); return -1; } - Py_ssize_t index = By_ShortValue(tagged); + Py_ssize_t index = (Py_ssize_t)at; if (index < 0) index += array->len; if (index < 0 || index >= array->len) { PyErr_SetString(PyExc_IndexError, "list index out of range"); @@ -8849,10 +8955,19 @@ static inline Py_ssize_t By_ArrayIndex(ByArrayHeader *array, ByTagged tagged) { return index; } -/* grow to hold one more, doubling so a run of appends stays amortized constant */ -static inline ByArrayHeader *By_ArrayGrow(ByArrayHeader *array, size_t width) { +static inline Py_ssize_t By_ArrayIndex(ByArrayHeader *array, ByTagged tagged) { + if (array == NULL) return -1; + if (BY_UNLIKELY(!By_IsShort(tagged))) { + // a big integer cannot be a valid index into a buffer this size + PyErr_SetString(PyExc_IndexError, "list index out of range"); + return -1; + } + return By_ArrayIndexI64(array, By_ShortValue(tagged)); +} + +/* grow a full buffer, doubling so a run of appends stays amortized constant */ +BY_COLD ByArrayHeader *By_ArrayGrowFull(ByArrayHeader *array, size_t width) { if (array == NULL) return NULL; - if (array->len < array->cap) return array; Py_ssize_t cap = array->cap < 4 ? 4 : array->cap * 2; ByArrayHeader *grown = (ByArrayHeader *)PyMem_Realloc(array, sizeof(ByArrayHeader) + (size_t)cap * width); @@ -8864,6 +8979,13 @@ static inline ByArrayHeader *By_ArrayGrow(ByArrayHeader *array, size_t width) { return grown; } +/* grow to hold one more */ +static inline ByArrayHeader *By_ArrayGrow(ByArrayHeader *array, size_t width) { + if (array == NULL) return NULL; + if (array->len < array->cap) return array; + return By_ArrayGrowFull(array, width); +} + /* `*x` or `**x` in a display: everything `x` holds, merged into a container this * frame has just built — so the kind test below is exact rather than a guess * @@ -9577,7 +9699,21 @@ static inline PyObject *By_CursorIter(PyObject *o) { return PyObject_GetIter(o); } +/* the position a cursor that steps a real iterator holds instead of one + * + * whether the loop walks an exact list is asked once, of what [`By_CursorIter`] handed + * back, and kept in the cursor's sign: a list's exact type cannot change, and a loop + * never walks anything but what it started with */ +#define BY_CURSOR_PROTOCOL ((int64_t)-1) + +static inline int64_t By_CursorStart(PyObject *it) { + return PyList_CheckExact(it) ? 0 : BY_CURSOR_PROTOCOL; +} + /* one step of such a loop: the element, or NULL at the end + * + * a position that is not negative is one into an exact list — see + * [`By_CursorStart`] — so the list is not asked what it is again on every step. * * the length is read again every step, because that is what cpython's list iterator * does — a list appended to under a `for` keeps feeding it and one popped from ends @@ -9587,6 +9723,9 @@ static inline PyObject *By_CursorIter(PyObject *o) { * and the exactness test is what makes reading `ob_item` at all legitimate: a list * subclass could have overridden `__getitem__`, and it never reaches this arm. * + * a slot inside an exact list's length holds an element and never NULL, which is what + * cpython's own list iterator takes a reference to without asking. + * * the bounds test is unsigned, so it rejects a negative position as well as one past * the end at no extra cost. a cursor should never be negative — the emitted loop sets * it to zero before the first step — but the position is the one number here that @@ -9597,14 +9736,17 @@ static inline PyObject *By_CursorIter(PyObject *o) { * the emitted register has, and the two are distinct types even where they are the * same size */ static inline PyObject *By_CursorStep(PyObject *it, int64_t *at) { - if (BY_LIKELY(PyList_CheckExact(it))) { + if (BY_LIKELY(*at >= 0)) { if (BY_LIKELY((uint64_t) *at < (uint64_t) PyList_GET_SIZE(it))) { PyObject *item = PyList_GET_ITEM(it, (Py_ssize_t) *at); *at += 1; - return By_NewRef(item); + Py_INCREF(item); + return item; } return NULL; } + /* a position no loop started holds nothing to step */ + if (*at != BY_CURSOR_PROTOCOL) return NULL; return PyIter_Next(it); } @@ -10013,9 +10155,8 @@ static inline void By_RaiseWith(PyObject *error, PyObject *value); * that matters: the exception has already unwound the body's `finally` blocks on * its way out, and a machine still calling itself suspended would be resumed by * finalization and run every one of them a second time */ -static inline void By_FinishGenerator(ByTagged *state) { - By_DecRefTagged(*state); - *state = By_ShortFrom(-1); +static inline void By_FinishGenerator(int64_t *state) { + *state = -1; } /* the value a `return` handed back, turned into the exception the iterator protocol @@ -10047,8 +10188,8 @@ static inline PyObject *By_TakeReturn(PyObject **returned) { #define BY_FRAME_RUNNING -2 /* whether a frame's resume is on the stack right now */ -static inline int By_FrameRunning(ByTagged state) { - return state == By_ShortFrom(BY_FRAME_RUNNING); +static inline int By_FrameRunning(int64_t state) { + return state == BY_FRAME_RUNNING; } /* what python calls this surface in a message it writes about one */ @@ -10059,7 +10200,7 @@ static inline const char *By_FrameNoun(int frame) { } /* refuse to resume a frame whose resume is on the stack, which is its own body asking */ -static inline int By_RefuseRunning(ByTagged state, int frame) { +static inline int By_RefuseRunning(int64_t state, int frame) { if (BY_LIKELY(!By_FrameRunning(state))) return 0; PyErr_Format(PyExc_ValueError, "%s already executing", By_FrameNoun(frame)); return -1; @@ -10088,10 +10229,9 @@ static inline int By_RefuseRunning(ByTagged state, int frame) { * `state` is 0 before the frame first runs and -1 once it has left for good, so * anything above 0 is a real suspension point with a `yield` to resume. `arg` is NULL * where the caller carries no sent value at all, as a `throw` does */ -static inline int By_RefuseResumption(ByTagged state, int frame, PyObject *arg) { - if (!By_IsShort(state)) return 0; +static inline int By_RefuseResumption(int64_t state, int frame, PyObject *arg) { if (By_RefuseRunning(state, frame) < 0) return -1; - Py_ssize_t at = (Py_ssize_t)By_ShortValue(state); + int64_t at = state; if (at == 0 && arg != NULL && arg != Py_None) { PyErr_Format(PyExc_TypeError, "can't send non-None value to a just-started %s", By_FrameNoun(frame)); @@ -10173,21 +10313,38 @@ static inline void By_ConvertStopIteration(int frame) { * close — the field would keep whatever the last `send` left in it, and the next * `yield` would read that same value a second time. * - * on 3.12 and later `None` is immortal, so the pair of reference counts a `next()` - * pays here are both branches that do no work */ + * what it can skip is a store of the very object the field already holds, which leaves + * the field as it was — and that is every `next()` after the first, since each one + * parks `None` over the `None` the one before it parked */ static inline void By_ParkSent(PyObject **sent, PyObject *value) { PyObject *old = *sent; + if (old == value) return; *sent = By_NewRef(value); Py_XDECREF(old); } -/* resume a generator's frame, finishing it when the frame leaves for good */ +/* write `None` over a parked field on a frame's way out, letting go of what it held + * + * the field holds `None` before the old value is let go of, so a finalizer that runs on + * that release finds nothing of it left behind */ +static inline void By_ClearField(PyObject **field) { + PyObject *old = *field; + *field = By_NewRef(Py_None); + Py_XDECREF(old); +} + +/* resume a generator's frame, finishing it when the frame leaves for good + * + * `resume` is the type's counted step, which takes a frame from the thread's depth around + * the body as python does on resuming one, and answers NULL with `RecursionError` set + * where the frame could not be pushed. every helper below that takes a `resume` means the + * same function */ static inline PyObject *By_StepGenerator(PyObject *self, PyObject **sent, PyObject **returned, - ByTagged *state, int frame, PyObject *arg, + int64_t *state, int frame, PyObject *arg, PyObject *(*resume)(PyObject *)) { if (By_RefuseResumption(*state, frame, arg) < 0) return NULL; By_ParkSent(sent, arg); - PyObject *result = By_ResumeCounted(self, resume); + PyObject *result = resume(self); if (result != NULL) return result; By_FinishGenerator(state); /* an empty `$returned` is what says the frame left by *raising* rather than by @@ -10196,6 +10353,31 @@ static inline PyObject *By_StepGenerator(PyObject *self, PyObject **sent, PyObje return By_TakeReturn(returned); } +/* what `tp_iternext` answers once a generator's resume handed back nothing, which is + * cpython's `gen_iternext`: a frame that raised passes its error on, a frame that returned + * `None` ends the iteration with no exception at all, and any other value rides out on + * `StopIteration`. the `for`, `list` or `next` asking treats a NULL with nothing raised as + * the end, which is the whole point — a finish builds no exception. + * + * only this slot answers so. `send` and `throw` raise `StopIteration` for a `None` too, + * because `gen_send` and `gen_throw` do. + * + * `quiet` is a compiled `for` stepping the generator itself, which is `PyIter_Next`: the + * value a finish carries is dropped however it would have ridden out, since that is the + * exception `PyIter_Next` would only have cleared */ +static inline PyObject *By_IterFinished(int64_t *state, PyObject **returned, int quiet) { + By_FinishGenerator(state); + PyObject *value = *returned; + if (value == NULL) { + By_ConvertStopIteration(BY_FRAME_GENERATOR); + return NULL; + } + *returned = NULL; + if (!quiet && value != Py_None) By_RaiseWith(PyExc_StopIteration, value); + Py_DECREF(value); + return NULL; +} + /* the argument count `throw` and `athrow` take, in the words python uses for both * * `throw` asks at the call. `athrow` asks when the awaitable it hands back is first @@ -10294,7 +10476,7 @@ static inline PyObject *By_ThrownException(PyObject *const *args, Py_ssize_t nar * is parked as `None` all the same, because leaving the last `send`'s value standing is * what would let a later `yield` read it again */ static inline PyObject *By_ResumeRaising(PyObject *self, PyObject **sent, PyObject **thrown, - PyObject **returned, ByTagged *state, int frame, + PyObject **returned, int64_t *state, int frame, PyObject *instance, PyObject *(*resume)(PyObject *)) { PyObject *old = *thrown; *thrown = instance; @@ -10350,17 +10532,17 @@ static inline int By_CloseDelegate(PyObject *delegate) { * the frame counts as running for as long as the inner iterator has it, so a body that * reaches back into the frame is refused as python refuses it */ static inline PyObject *By_ThrowIntoDelegate(PyObject *self, PyObject **sent, PyObject **thrown, - PyObject **returned, ByTagged *state, int frame, + PyObject **returned, int64_t *state, int frame, PyObject *const *args, Py_ssize_t nargs, PyObject *delegate, PyObject *(*resume)(PyObject *), int *forwarded) { static PyObject *by_throw = NULL; - ByTagged suspended = *state; + int64_t suspended = *state; *forwarded = 1; Py_INCREF(delegate); if (frame != BY_FRAME_ASYNC_GENERATOR && PyErr_GivenExceptionMatches(args[0], PyExc_GeneratorExit)) { - *state = By_ShortFrom(BY_FRAME_RUNNING); + *state = BY_FRAME_RUNNING; int closed = By_CloseDelegate(delegate); *state = suspended; Py_DECREF(delegate); @@ -10390,13 +10572,13 @@ static inline PyObject *By_ThrowIntoDelegate(PyObject *self, PyObject **sent, Py if (instance == NULL) { answer = NULL; } else { - *state = By_ShortFrom(BY_FRAME_RUNNING); + *state = BY_FRAME_RUNNING; answer = PyObject_CallOneArg(method, instance); *state = suspended; Py_DECREF(instance); } } else { - *state = By_ShortFrom(BY_FRAME_RUNNING); + *state = BY_FRAME_RUNNING; answer = PyObject_Vectorcall(method, args, (size_t)nargs, NULL); *state = suspended; } @@ -10423,12 +10605,12 @@ static inline PyObject *By_ThrowIntoDelegate(PyObject *self, PyObject **sent, Py * finish it — see below. otherwise it is the resumption that decides, and a body that * catches what was thrown leaves the machine usable */ static inline PyObject *By_ThrowInto(PyObject *self, PyObject **sent, PyObject **thrown, - PyObject **returned, ByTagged *state, int frame, + PyObject **returned, int64_t *state, int frame, PyObject *const *args, Py_ssize_t nargs, PyObject *(*delegate)(PyObject *), PyObject *(*resume)(PyObject *)) { if (thrown == NULL) return NULL; - PyObject *inner = delegate != NULL && By_IsShort(*state) && By_ShortValue(*state) > 0 + PyObject *inner = delegate != NULL && *state > 0 ? delegate(self) : NULL; if (inner != NULL) { @@ -10461,7 +10643,7 @@ static inline PyObject *By_ThrowInto(PyObject *self, PyObject **sent, PyObject * * resuming instead would run the body from the top for a machine that never * started, and report exhaustion for one that has finished — two different wrong * answers about which exception the caller is holding */ - if (By_ShortValue(*state) <= 0) { + if (*state <= 0) { By_FinishGenerator(state); PyErr_SetObject((PyObject *)Py_TYPE(instance), instance); Py_DECREF(instance); @@ -10540,7 +10722,7 @@ static inline void By_WarnUnawaitedCoroutine(PyObject *coroutine, const char *qu * the throw is python's own `GeneratorExit`, so a frame suspended in a delegation closes * the iterator it delegates to first — see `By_ThrowIntoDelegate` */ static inline PyObject *By_CloseGenerator(PyObject *self, PyObject **sent, PyObject **thrown, - PyObject **returned, ByTagged *state, int frame, + PyObject **returned, int64_t *state, int frame, PyObject *(*delegate)(PyObject *), PyObject *(*resume)(PyObject *)) { if (By_RefuseRunning(*state, frame) < 0) return NULL; @@ -10549,7 +10731,7 @@ static inline PyObject *By_CloseGenerator(PyObject *self, PyObject **sent, PyObj * `By_ThrowInto` would give the right answer for a finished frame and the wrong * one for a frame that never started, which would run the whole body under a * `GeneratorExit` it had no way to see */ - if (By_ShortValue(*state) <= 0) { + if (*state <= 0) { By_FinishGenerator(state); Py_RETURN_NONE; } @@ -11738,13 +11920,13 @@ static ByRaisedIter By_RaisedIter = {PyObject_HEAD_INIT(&By_RaisedIter_Type)}; * `yield` evaluating to the same thing. treating `None` as "carries nothing" and * skipping the store is what used to let a value survive into a later `yield` */ static inline PySendResult By_SendGenerator(PyObject *self, PyObject **sent, - PyObject **returned, ByTagged *state, int frame, + PyObject **returned, int64_t *state, int frame, PyObject *(*resume)(PyObject *), PyObject *arg, PyObject **result) { PyObject *step; if (By_RefuseResumption(*state, frame, arg) < 0) return PYGEN_ERROR; By_ParkSent(sent, arg); - step = By_ResumeCounted(self, resume); + step = resume(self); if (step != NULL) { *result = step; return PYGEN_NEXT; diff --git a/docs/basedpython/development/compilation/runtime.md b/docs/basedpython/development/compilation/runtime.md index 6100ac14cb..011f19fca0 100644 --- a/docs/basedpython/development/compilation/runtime.md +++ b/docs/basedpython/development/compilation/runtime.md @@ -983,6 +983,24 @@ Own.__new__(Own).count # python: AttributeError; compiled: 0 a subclass is not reached by anything the compiled class can run when it is made, so closing this would mean a test on every read of such a field +### a complex power held where a `float` is declared + +python's `**` gives a complex number for a negative base to a fractional power, and the +checker types `float ** float` as `Any` for that reason. a compiled `**` that could answer +with one is python's own, through the object protocol. a place declared `float` holds an +unboxed double, though, so storing a complex answer there raises `TypeError` where python +stores it: + +```python +def root(a: float) -> float: + x = a # `x: float` in a `.by` module + x **= 0.5 + return x + + +root(-4.0) # python: (1.2246467991473532e-16+2j); compiled: TypeError +``` + ## debugging and inspection - **`#line` directives** in the generated C point at `.by` source. gdb, lldb, From 9c19af2f92777faa5b5953bd5072447948d8c9c1 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Thu, 17 Sep 2026 13:15:56 +1000 Subject: [PATCH 07/16] native: publish compiled methods and classes with python's surface and checks methods bind through PyMethod_New and answer their def, module, qualname, defaults, annotations and repr as the interpreted function does, class and static methods are wrapped as a class statement wraps them, tp_new is left to object, class patterns read fields only where the layout is held, and a compiled module makes the interpreted build's soundness checks in the same places and words --- Cargo.lock | 2 + crates/basedpython/Cargo.lock | 3 + crates/by_build/src/lib.rs | 11 + crates/by_build/tests/common/mod.rs | 63 +- crates/by_build/tests/differential.rs | 2118 +++++++++++++---- crates/by_build/tests/end_to_end.rs | 75 +- crates/by_codegen_c/src/lib.rs | 452 +++- crates/by_ir/src/function.rs | 30 +- crates/by_ir/src/ops.rs | 61 + crates/by_ir/src/print.rs | 28 +- crates/by_ir/src/verify.rs | 10 + crates/by_irbuild/Cargo.toml | 2 + crates/by_irbuild/src/annotations.rs | 514 ++++ crates/by_irbuild/src/closures.rs | 146 ++ crates/by_irbuild/src/lib.rs | 629 +++-- crates/by_irbuild/src/tests.rs | 118 +- crates/by_opt/src/borrow.rs | 3 +- crates/by_opt/src/copy_propagation.rs | 2 + crates/by_opt/src/field_proofs.rs | 1 + crates/by_opt/src/fold.rs | 4 + crates/by_opt/src/infallible.rs | 4 + crates/by_opt/src/lib.rs | 1 + crates/by_opt/src/runs_python.rs | 3 + crates/by_rt/include/by.h | 1267 ++++++++-- crates/by_transforms/src/lib.rs | 1 + .../src/transforms/checked_cast.rs | 2 +- .../by_transforms/src/transforms/soundness.rs | 499 ++-- crates/ty/tests/by_e2e.rs | 88 +- crates/ty_python_semantic/src/lib.rs | 3 +- .../ty_python_semantic/src/semantic_model.rs | 15 + .../ty_python_semantic/src/types/soundness.rs | 68 +- .../development/compilation/runtime.md | 312 ++- 32 files changed, 5419 insertions(+), 1116 deletions(-) create mode 100644 crates/by_irbuild/src/annotations.rs diff --git a/Cargo.lock b/Cargo.lock index 7b0ac07ea8..c4e271172d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -383,8 +383,10 @@ name = "by_irbuild" version = "0.0.0" dependencies = [ "by_ir", + "by_transforms", "ruff_db", "ruff_python_ast", + "ruff_python_codegen", "ruff_python_parser", "ruff_python_stdlib", "ruff_source_file", diff --git a/crates/basedpython/Cargo.lock b/crates/basedpython/Cargo.lock index bf75299539..adb0e74ddc 100644 --- a/crates/basedpython/Cargo.lock +++ b/crates/basedpython/Cargo.lock @@ -273,10 +273,13 @@ name = "by_irbuild" version = "0.0.0" dependencies = [ "by_ir", + "by_transforms", "ruff_db", "ruff_python_ast", + "ruff_python_codegen", "ruff_python_parser", "ruff_python_stdlib", + "ruff_source_file", "ruff_text_size", "thin-vec", "ty_project", diff --git a/crates/by_build/src/lib.rs b/crates/by_build/src/lib.rs index 26ef087c88..530978b4b6 100644 --- a/crates/by_build/src/lib.rs +++ b/crates/by_build/src/lib.rs @@ -258,6 +258,14 @@ impl Options { pub fn lowering(&self) -> by_irbuild::LowerOptions { by_irbuild::LowerOptions { language: self.language, + // the checks the interpreted fallback is transpiled with, and a `.py` source + // is its own fallback + soundness: match self.language { + by_irbuild::Language::BasedPython => { + self.fallback.clone().unwrap_or_default().soundness + } + by_irbuild::Language::Python => by_transforms::SoundnessPositions::none(), + }, recheck_licences: self.recheck_licences, bind_functions_early: self.bind_functions_early, } @@ -383,6 +391,9 @@ fn finish( // too, over the twin's — once for each definition rather than once for the name let twin = by_irbuild::without_init_decorators(&twin, &module) .map_err(|error| anyhow::anyhow!("could not prepare the interpreted fallback: {error}"))?; + // a nested function's annotations are the twin's, evaluated by python, so they are + // written from the twin's text + by_irbuild::annotations::write_factories(&mut module, &twin); // and the same program compiled, so that importing the artefact does not have to // parse it all over again. it is asked for after every rewrite above, because what // gets compiled has to be exactly what would otherwise be run diff --git a/crates/by_build/tests/common/mod.rs b/crates/by_build/tests/common/mod.rs index 44302574ff..2025fb3ef4 100644 --- a/crates/by_build/tests/common/mod.rs +++ b/crates/by_build/tests/common/mod.rs @@ -1,4 +1,5 @@ -//! the one place a python child's stdout is read +//! what both test binaries share: the directory a test builds into, and the one place a +//! python child's stdout is read //! //! both test binaries compare what an interpreter printed, so how those bytes //! travel has to be settled once. a pipe that picks its own encoding, or a @@ -6,9 +7,67 @@ //! either build decided — and a comparison that cannot tell the two apart is //! reading the platform rather than the compiler -use std::path::Path; +use std::fmt::Display; +use std::ops::Deref; +use std::path::{Path, PathBuf}; use std::process::Command; +/// a directory a test builds into, which goes when the test passes +/// +/// a failing test keeps it, so the C, the transpiled twin and whatever else the build +/// wrote can still be read, and names it on stderr, which both test runners print with +/// the failure. a test holds the directory for as long as it reads from it, so a helper +/// that builds one and hands back an answer hands the directory back too: otherwise it +/// would be gone before the assertion that fails on the answer +/// +/// the name carries the process id. nextest gives each *test* a process of its own, so +/// one run never collides with itself — but nothing stops two *runs* choosing the same +/// path, and a 3.13 sweep beside a 3.14 one would then overwrite each other's sources +/// between the build and the read. a collision during setup fails fast enough to look +/// like a missing toolchain, and one after the build fails having genuinely compiled and +/// compared, so it reads as a difference the compiler produced — twenty-nine of those were +/// chased as a regression before the shared path was noticed +pub(crate) struct Scratch(PathBuf); + +impl Scratch { + /// an empty place for `name` under the system temp directory + pub(crate) fn new(name: impl Display) -> Self { + let path = std::env::temp_dir().join(format!("{name}_p{}", std::process::id())); + let _ = std::fs::remove_dir_all(&path); + Self(path) + } +} + +impl Deref for Scratch { + type Target = Path; + + fn deref(&self) -> &Path { + &self.0 + } +} + +impl AsRef for Scratch { + fn as_ref(&self) -> &Path { + &self.0 + } +} + +impl Drop for Scratch { + fn drop(&mut self) { + if std::thread::panicking() { + #[expect( + clippy::print_stderr, + reason = "the failure output is where a kept directory has to be named" + )] + { + eprintln!("kept the failing test's directory: {}", self.0.display()); + } + } else { + let _ = std::fs::remove_dir_all(&self.0); + } + } +} + /// run `body` with `dir` on `sys.path` and hand back what it printed pub(crate) fn python_output(python: &str, dir: &Path, body: &str) -> String { let prelude = format!( diff --git a/crates/by_build/tests/differential.rs b/crates/by_build/tests/differential.rs index 237c964906..db1366b1f4 100644 --- a/crates/by_build/tests/differential.rs +++ b/crates/by_build/tests/differential.rs @@ -25,7 +25,7 @@ reason = "skip notices belong on the test harness's stderr" )] -use std::path::{Path, PathBuf}; +use std::path::Path; use std::process::Command; use by_build::{Options, Toolchain, build_source}; @@ -33,6 +33,8 @@ use by_transforms::Config; mod common; +use common::Scratch; + /// a test whose *source* needs a newer interpreter than this one has nothing to /// say: neither leg can run it, so there is nothing to compare /// @@ -51,25 +53,6 @@ fn missing_toolchain(error: &anyhow::Error) -> bool { }) } -/// a temp directory of this process's own -/// -/// every leg below builds into a fixed path under the system temp directory, named after -/// the test. nextest gives each *test* a process of its own, so one run never collides -/// with itself — but nothing stops two *runs* choosing the same directory, and a 3.13 -/// sweep beside a 3.14 one does exactly that. the two then overwrite each other's sources -/// between the build and the read. -/// -/// the failures that produces are the convincing kind. a collision during setup fails -/// fast enough to look like a missing toolchain, and one *after* the build fails having -/// genuinely compiled and compared, so it reads as a difference the compiler produced. -/// twenty-nine of those were chased as a regression before the shared path was noticed. -/// -/// the module's own name is a separate argument from the directory holding it, so putting -/// the process id here changes where a test builds and nothing about what it builds -fn diff_root() -> PathBuf { - std::env::temp_dir().join(format!("by_diff_p{}", std::process::id())) -} - fn environment() -> Option<(String, Toolchain)> { let python = match std::env::var("PYTHON") { Ok(python) => python, @@ -1285,6 +1268,47 @@ def _deleted(h): out.append(hasattr(h, '_Held__hidden')) out.append(repr(_capture(h.drop_hidden))) return out + +# what a function says about its annotations, its signature and its parameters, read the +# ways the standard library reads them. an annotation `annotationlib` could not evaluate is +# a `ForwardRef` whose representation names its owner's address, so only its text is kept +def _annotation_report(f): + import inspect, sys, typing + def attempt(read): + try: + return read() + except Exception as e: + return type(e).__name__ + ': ' + str(e) + out = { + 'annotations': attempt(lambda: f.__annotations__), + 'same': attempt(lambda: f.__annotations__ is f.__annotations__), + 'hints': attempt(lambda: typing.get_type_hints(f)), + 'signature': attempt(lambda: str(inspect.signature(f))), + 'defaults': attempt(lambda: (f.__defaults__, f.__kwdefaults__)), + 'code': attempt(lambda: (f.__code__.co_qualname, f.__code__.co_varnames)), + 'type_params': attempt(lambda: [param.__name__ for param in f.__type_params__]), + } + if sys.version_info >= (3, 14): + import annotationlib + for format in (annotationlib.Format.VALUE, annotationlib.Format.FORWARDREF, + annotationlib.Format.STRING): + out[format.name] = attempt(lambda: { + key: getattr(value, '__forward_arg__', value) + for key, value in annotationlib.get_annotations(f, format=format).items() + }) + out['annotate'] = attempt(lambda: f.__annotate__ and f.__annotate__.__qualname__) + return out + +# whether what `make` built is still alive once the only references left are the ones +# `keep` took from it, handed back beside what `keep` took so a caller can go on to use it +def _outlives(make, keep): + import weakref + made = make() + alive = weakref.ref(made) + kept = keep(made) + del made + gc.collect() + return alive() is not None, kept "; fn run(python: &str, dir: &Path, body: &str) -> String { @@ -1313,6 +1337,11 @@ fn agree_python(tag: &str, source: &str, calls: &[&str]) { agree_in(tag, source, calls, false, by_irbuild::Language::Python); } +/// as [`agree_python`], handing back the directory the compiled leg was built into +fn agree_python_built(tag: &str, source: &str, calls: &[&str]) -> Option { + agree_in(tag, source, calls, false, by_irbuild::Language::Python) +} + /// as [`agree_python`], but the source is expected to contain declined functions fn agree_python_with_declines(tag: &str, source: &str, calls: &[&str]) { agree_in(tag, source, calls, true, by_irbuild::Language::Python); @@ -1328,8 +1357,28 @@ fn agree_inner(tag: &str, source: &str, calls: &[&str], allow_declines: bool) { ); } -/// build `source` under `options` and hand back what each of `calls` printed from the -/// compiled leg alone +/// as [`agree`], with both builds under `soundness` rather than the default positions +fn agree_with_soundness( + tag: &str, + source: &str, + calls: &[&str], + soundness: by_transforms::SoundnessPositions, +) { + agree_built( + tag, + source, + calls, + false, + by_irbuild::Language::BasedPython, + Config { + soundness, + ..Config::default() + }, + ); +} + +/// build `source` under `options` and hand back the directory it was built into and what +/// each of `calls` printed from the compiled leg alone /// /// for the tradeoffs the build makes on purpose, where the interpreted leg is not the /// expected value. `None` where there is no toolchain to build with @@ -1338,10 +1387,9 @@ fn compiled_answers( source: &str, options: Options, calls: &[&str], -) -> Option> { +) -> Option<(Scratch, Vec)> { let (python, toolchain) = environment()?; - let dir = diff_root().join(format!("by_diff_{tag}_c")); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new(format!("by_diff_{tag}_c")); let module = format!("by_diff_{tag}"); let options = Options { recheck_licences: recheck_licences(), @@ -1363,19 +1411,18 @@ fn compiled_answers( "{tag} declined functions the test expects to be compiled: {:?}", built.declined ); - Some( - calls - .iter() - .map(|call| { - let body = format!( - "import {module} as m\n\ - assert m.__file__.endswith(('.so', '.pyd')), m.__file__\n\ - {CAPTURE_HELPER}print(repr({call}))\n" - ); - run(&python, &dir, &body) - }) - .collect(), - ) + let answers = calls + .iter() + .map(|call| { + let body = format!( + "import {module} as m\n\ + assert m.__file__.endswith(('.so', '.pyd')), m.__file__\n\ + {CAPTURE_HELPER}print(repr({call}))\n" + ); + run(&python, &dir, &body) + }) + .collect(); + Some((dir, answers)) } /// whether this run has every licensed call re-ask the lookup it skipped @@ -1393,21 +1440,39 @@ fn recheck_licences() -> bool { !std::env::var("BY_LICENCE_RECHECK").is_ok_and(|value| value == "0") } +/// the directory the compiled leg was built into, for a test that asks that build +/// something more once the two legs agree fn agree_in( tag: &str, source: &str, calls: &[&str], allow_declines: bool, language: by_irbuild::Language, -) { - let Some((python, toolchain)) = environment() else { - return; - }; +) -> Option { + agree_built( + tag, + source, + calls, + allow_declines, + language, + Config::default(), + ) +} + +/// [`agree_in`], with the interpreted leg transpiled under `config` and the compiled leg +/// handed it as its fallback's +fn agree_built( + tag: &str, + source: &str, + calls: &[&str], + allow_declines: bool, + language: by_irbuild::Language, + config: Config, +) -> Option { + let (python, toolchain) = environment()?; - let compiled_dir = diff_root().join(format!("by_diff_{tag}_c")); - let interpreted_dir = diff_root().join(format!("by_diff_{tag}_i")); - let _ = std::fs::remove_dir_all(&compiled_dir); - let _ = std::fs::remove_dir_all(&interpreted_dir); + let compiled_dir = Scratch::new(format!("by_diff_{tag}_c")); + let interpreted_dir = Scratch::new(format!("by_diff_{tag}_i")); let module = format!("by_diff_{tag}"); @@ -1415,9 +1480,10 @@ fn agree_in( // cpython, under the same config `by_build` uses — including the *target // version*, which has to be this interpreter's or the two legs are not the // same program. for python there is nothing to transpile — it already is one + let fallback = config.clone(); let interpreted_source = match language { by_irbuild::Language::BasedPython => { - let mut config = Config::default(); + let mut config = config; if let Some((major, minor)) = toolchain.version && let Ok(parsed) = format!("{major}.{minor}").parse() { @@ -1438,6 +1504,7 @@ fn agree_in( let options = Options { language, recheck_licences: recheck_licences(), + fallback: Some(fallback), ..Options::default() }; let built = match build_source(source, module.as_str(), &toolchain, &compiled_dir, &options) { @@ -1451,7 +1518,7 @@ fn agree_in( "{tag} failed to build: {error:#}" ); eprintln!("skipping {tag}: no working C toolchain ({error})"); - return; + return None; } }; if !allow_declines { @@ -1471,6 +1538,7 @@ fn agree_in( "{tag}: `{call}` differs — compiled {compiled}, interpreted {interpreted}" ); } + Some(compiled_dir) } #[test] @@ -1926,8 +1994,7 @@ fn an_operation_that_keeps_leaving_the_short_range_does_not_leak() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_bigleak"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_bigleak"); let source = "\ def climb(a: int, n: int) -> int: total = 0 @@ -1973,8 +2040,7 @@ fn a_result_that_is_not_an_int_is_refused_rather_than_tagged() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_notanint"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_notanint"); let source = "\ def halve(value: int) -> int: return value // 2 @@ -2003,7 +2069,10 @@ def halve(value: int) -> int: except TypeError as error:\n\ \x20 print(f'TypeError: {error}')\n", ); - assert_eq!(out, "True\nTypeError: expected int, got str"); + assert_eq!( + out, + "True\nTypeError: type soundness violation: expected int, got str" + ); } #[test] @@ -2297,32 +2366,177 @@ def mixed_mod(a: float, b: int) -> float: ); } +#[test] +fn an_int_to_a_negative_int_power_is_a_float() { + // `**` over two ints is a float for a negative exponent, so an int result is only known + // where the exponent is a literal that is not negative. anywhere else the power is + // python's own + agree( + "intpower", + "\ +def power(a: int, b: int) -> object: + return a ** b + + +def held(a: int, b: int) -> object: + x = a ** b + return x + + +def literal(a: int) -> object: + return (a ** 2, a ** 0, a ** -1, a ** -2) + + +def summed(n: int) -> object: + total = 0 + i = -2 + while i < n: + total = total + 2 ** i + i = i + 1 + return total + + +def declared(a: int, b: int) -> int: + return a ** b +", + &[ + "[repr(_outcome(m.power, a, b)) for a in (2, -3, 0, 1, 2**70) for b in (-1, 0, 3, -2, 70)]", + "[repr(_outcome(m.held, a, b)) for a in (2, -3, 0) for b in (-1, 0, 3)]", + "[repr(_outcome(m.literal, a)) for a in (2, -3, 0, 2**70)]", + "m.summed(5)", + "[m.declared(a, b) for a in (2, -3, 2**70) for b in (0, 3, 70)]", + ], + ); +} + +#[test] +fn an_augmented_assignment_holds_whatever_its_operation_answers() { + // the name an augmented assignment rebinds is given the operation's result, and that + // is not always of the right-hand operand's type — nor the left's: `x **= 0.5` on a + // float can be complex, `x /= b` on an int is a float, and `x **= b` on an int is a + // float for a negative `b`. the local has to hold each of them as python does + agree( + "augmentedresult", + "\ +def power(a: float, b: float) -> object: + x: object = a + x **= b + return x + + +def root(a: float, b: float) -> object: + x = a + x **= b + return x + + +def int_div(a: int, b: int) -> object: + x = a + x /= b + return x + + +def int_plus_float(a: int, b: float) -> object: + x = a + x += b + return x + + +def int_pow_neg(a: int, b: int) -> object: + x = a + x **= b + return x + + +def str_times(s: str, n: int) -> object: + x = s + x *= n + return x + + +def bool_plus(a: bool, n: int) -> object: + x = a + x += n + return x + + +def floordiv_float(a: int, b: float) -> object: + x = a + x //= b + return x + + +def mod_float(a: int, b: float) -> object: + x = a + x %= b + return x + + +def list_plus(xs: list[int], ys: list[int]) -> object: + x = xs + x += ys + return x + + +def shift(a: int, b: int) -> object: + x = a + x <<= b + x >>= 1 + x |= 8 + x &= 255 + x ^= 3 + return x + + +def float_minus_int(a: float, b: int) -> object: + x = a + x -= b + x *= b + return x + + +def ann_int_truediv(a: int) -> object: + x: int | float = a + x /= 2 + return x +", + &[ + "[repr(_outcome(m.power, a, b)) for a in (-4.0, 4.0, 0.0) for b in (0.5, 2.0, -1.0)]", + "[repr(_outcome(m.root, a, b)) for a in (-8.0, 8.0) for b in (1.0 / 3.0, 0.5, 3.0)]", + "[repr(_outcome(m.int_div, a, b)) for a in (3, -7, 0) for b in (2, -2, 0)]", + "[repr(_outcome(m.int_plus_float, a, b)) for a in (1, -3) for b in (0.5, float('nan'))]", + "[repr(_outcome(m.int_pow_neg, a, b)) for a in (2, -3, 0) for b in (-1, 0, 3, -2)]", + "m.str_times('ab', 3)", + "[m.bool_plus(a, n) for a in (True, False) for n in (2, -1)]", + "[repr(_outcome(m.floordiv_float, a, b)) for a in (7, -7) for b in (2.0, -2.5, 0.0)]", + "[repr(_outcome(m.mod_float, a, b)) for a in (7, -7) for b in (2.5, -2.5, 0.0)]", + "m.list_plus([1], [2])", + "[m.shift(a, b) for a in (3, -3) for b in (0, 4, 70)]", + "[m.float_minus_int(a, b) for a in (2.5, -1.0) for b in (2, 0)]", + "[m.ann_int_truediv(a) for a in (3, 4, -5)]", + ], + ); +} + #[test] fn a_complex_power_is_refused_where_a_float_local_is_declared() { // `float ** float` is `Any` to the checker, so a local declared `float` may be handed // a power python answers with a complex number. the power itself is python's; the // store is the unbox a declared `float` makes, and it refuses the complex loudly // rather than holding a nan. this asserts on the compiled leg alone: python keeps - // the complex number - let Some(answers) = compiled_answers( + // the complex number. an augmented assignment is not this: what it binds is the + // operation's result, which the local is chosen to hold + let Some((_dir, answers)) = compiled_answers( "floatpowrefused", "\ def assigned(a: float, b: float) -> float: x: float = a ** b return x - - -def augmented(a: float, b: float) -> float: - x = a - x **= b - return x ", Options::default(), &[ "repr(_capture(m.assigned, -8.0, 0.5))", - "repr(_capture(m.augmented, -8.0, 1.0 / 3.0))", "m.assigned(-8.0, 3.0)", - "m.augmented(8.0, 0.5)", ], ) else { return; @@ -2330,10 +2544,8 @@ def augmented(a: float, b: float) -> float: assert_eq!( answers, [ - "\"TypeError('expected float, got complex')\"", - "\"TypeError('expected float, got complex')\"", - "-512.0", - "2.8284271247461903", + "\"TypeError('type soundness violation: expected float, got complex')\"", + "-512.0" ] ); } @@ -2380,7 +2592,7 @@ def counted(n: int) -> float: fn a_float_element_of_the_wrong_type_is_refused_where_it_is_unboxed() { // a `.by` `list[float]` holds floats, so an `int` element is refused where the loop // unboxes it — with the unbox's own error, and nothing left half-done - let Some(answers) = compiled_answers( + let Some((_dir, answers)) = compiled_answers( "floatrefused", "\ def walk(xs: list[float]) -> float: @@ -2406,9 +2618,9 @@ def element(xs: list[float], i: int) -> float: assert_eq!( answers, [ - "\"TypeError('expected float, got int')\"", - "\"TypeError('expected float, got NoneType')\"", - "\"TypeError('expected float, got bool')\"", + "\"TypeError('type soundness violation: expected float, got int')\"", + "\"TypeError('type soundness violation: expected float, got NoneType')\"", + "\"TypeError('type soundness violation: expected float, got bool')\"", "-112.0", ] ); @@ -2643,7 +2855,7 @@ fn a_raised_recursion_limit_still_stops_at_the_stack() { // python frames live on the heap, so python answers these where a native frame // cannot: the limit allows it and the C stack does not. the compiled leg raises // instead, and the same holds on a thread whose stack is smaller than the main one - let Some(answers) = compiled_answers( + let Some((_dir, answers)) = compiled_answers( "recstack", RECURSIONS, Options { @@ -2674,7 +2886,7 @@ fn a_raised_recursion_limit_still_stops_at_the_stack() { fn a_build_not_following_the_recursion_limit_still_stops_at_the_stack() { // the configuration that counts no frames against the limit leaves a lowered limit // unobserved, which is the tradeoff it makes, and still never crashes - let Some(answers) = compiled_answers( + let Some((_dir, answers)) = compiled_answers( "recstackonly", RECURSIONS, Options { @@ -2928,7 +3140,7 @@ fn a_build_binding_functions_early_goes_on_calling_what_it_compiled() { // the configuration that assumes the module's functions are never rebound asks // nothing at the call, which is the tradeoff it makes: a replacement reaches a call // made from python and not one the module's own compiled code makes - let Some(answers) = compiled_answers( + let Some((_dir, answers)) = compiled_answers( "reboundearly", REBOUND, Options { @@ -2957,7 +3169,7 @@ fn a_rebound_function_handed_an_unboxed_buffer_is_refused_loudly() { // a list the caller holds as a buffer has no list object to hand a replacement, and // a copy would be a different list — so the call raises, as a rebound `len` on a // buffer does, rather than calling the replacement with a copy - let Some(answers) = compiled_answers( + let Some((_dir, answers)) = compiled_answers( "reboundbuffer", "\ def total(xs: list[float]) -> float: @@ -3218,8 +3430,7 @@ fn an_error_from_the_object_protocol_propagates() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_objerr"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_objerr"); let source = "def add(a, b) -> object:\n return a + b\n"; if build_source( source, @@ -3251,8 +3462,7 @@ fn boxed_values_do_not_leak() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_objleak"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_objleak"); let source = "\ def chain(a, b) -> object: x = a + b @@ -3292,8 +3502,7 @@ fn an_unboxed_buffer_does_not_leak() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_bufleak"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_bufleak"); let source = "\ def built(n: int) -> float: out = [] @@ -3671,8 +3880,7 @@ def count(n: int) -> int: let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_fieldmove_pin"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_fieldmove_pin"); let built = match build_source( SOURCE, "by_diff_fieldmove_pin", @@ -3705,8 +3913,7 @@ fn a_temporary_object_argument_does_not_crash() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_argtemp"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_argtemp"); let source = "def echo(x) -> object:\n return x\n"; if build_source( source, @@ -3812,8 +4019,7 @@ fn a_borrowed_copy_does_not_over_release_its_source() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_widened_rc"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_widened_rc"); if build_source( WIDENED_COPIES, "by_diff_widened_rc", @@ -3935,8 +4141,7 @@ fn a_borrowed_copy_in_a_duplicated_loop_body_does_not_move_its_source_references let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_borrowed_copies_rc"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_borrowed_copies_rc"); if build_source( BORROWED_COPIES, "by_diff_borrowed_copies_rc", @@ -4000,8 +4205,7 @@ fn short_circuiting_really_short_circuits() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_shortcircuit"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_shortcircuit"); // dividing by zero raises; `a or b` must not evaluate `b` when `a` is truthy let source = "\ def guarded(a: int, b: int) -> object: @@ -4600,8 +4804,7 @@ fn the_compiled_build_is_the_one_that_answers_for_a_nul_literal() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_nulstr_which"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_nulstr_which"); let built = match build_source( NUL_LITERALS, "by_diff_nulstr_which", @@ -4662,8 +4865,7 @@ fn a_call_out_checks_what_comes_back() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_callcheck"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_callcheck"); // `abs` is a builtin the checker knows returns an int, so the call site // narrows with a checked unbox. shadowing it in the module namespace — which // `By_LookupGlobal` consults first, exactly as `LOAD_GLOBAL` does — makes it @@ -4692,7 +4894,10 @@ fn a_call_out_checks_what_comes_back() { except TypeError as e:\n print('TypeError:', e)\n\ else:\n print('no error')\n", ); - assert_eq!(out, "TypeError: expected int, got str"); + assert_eq!( + out, + "TypeError: type soundness violation: expected int, got str" + ); } #[test] @@ -4700,8 +4905,7 @@ fn a_missing_global_raises_a_name_error() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_nameerr"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_nameerr"); let source = "def f() -> object:\n return nowhere()\n"; if build_source( source, @@ -5427,8 +5631,7 @@ fn iterating_a_wrongly_typed_list_raises() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_iterchk"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_iterchk"); // the annotation says int elements; the unbox per element is the // `iterations` soundness position and must catch a lie let source = "\ @@ -5458,7 +5661,211 @@ def total(xs: list[int]) -> int: except TypeError as e:\n print('TypeError:', e)\n\ else:\n print('no error')\n", ); - assert_eq!(out, "TypeError: expected int, got str"); + assert_eq!( + out, + "TypeError: type soundness violation: expected int, got str" + ); +} + +/// the positions a value nothing verified meets a declared type at, each with a +/// representation that cannot hold a wrong value — an `object`, a container, a class +/// neither build lays out, an optional — so nothing but the soundness check itself stands +/// between the value and the caller +const UNVERIFIED_VALUES: &str = "\ +from typing import Any + + +class Box: + def __init__(self, v: int): + self.v = v + + +class Other: + pass + + +def to_int(v: Any) -> int: + return v + + +def to_dict(v: Any) -> dict[str, int]: + return v + + +def to_box(v: Any) -> Box: + return v + + +def to_optional(v: Any) -> int | None: + return v + + +def power(a: int, b: int) -> int: + return a ** b + + +def takes(x: dict[str, int], y: int | None) -> object: + return (x, y) + + +def passes(x: Any, y: Any) -> object: + return takes(x, y) + + +def held(v: Any) -> object: + a: int | None = v + return a + + +def unboxed(v: Any) -> object: + a: int = v + return a + + +def laid_out(v: Any) -> object: + a: Box = v + return a.v + + +def got(d: dict[str, int]) -> object: + x = d.get('k') + return x + + +def first(xs: list[dict[str, int]]) -> object: + y = xs[0] + return y + + +def last(xs: list[int | None]) -> object: + out: object = None + for x in xs: + out = x + return out + + +def kept(xs: list[Box | None]) -> object: + return [x for x in xs] +"; + +#[test] +fn a_value_nothing_verified_is_checked_where_the_interpreted_build_checks_it() { + // the interpreted build checks a value whose type nothing verified — an `Any`, a + // generic call's result, an element read out of a container — where it meets a + // declared type, and raises `TypeError` when it is not what the type says. a compiled + // module makes the same check in the same place, whatever representation the value is + // held in: an `int` return handed back as an object, where `int ** int` can be a + // float, is checked as surely as one unboxed, and an unboxing refuses in the check's + // own words + agree( + "unverified", + UNVERIFIED_VALUES, + &[ + "str(_capture(m.to_int, 0.5))", + "(m.to_int(3), m.to_int(True))", + "str(_capture(m.to_dict, [1]))", + "m.to_dict({'a': 1})", + "str(_capture(m.to_box, m.Other()))", + "m.to_box(m.Box(4)).v", + "str(_capture(m.to_optional, 'x'))", + "(m.to_optional(None), m.to_optional(2))", + "str(_capture(m.power, 2, -1))", + "m.power(2, 3)", + "str(_capture(m.passes, [1], None))", + "str(_capture(m.passes, {}, 'x'))", + "m.passes({}, None)", + "str(_capture(m.held, 'x'))", + "(m.held(None), m.held(5))", + "str(_capture(m.unboxed, 'x'))", + "str(_capture(m.laid_out, 3))", + "m.laid_out(m.Box(6))", + "str(_capture(m.got, {'k': 'x'}))", + "(m.got({}), m.got({'k': 1}))", + "str(_capture(m.first, [[1]]))", + "m.first([{'a': 1}])", + "str(_capture(m.last, [1, 'x']))", + "m.last([1, None])", + "str(_capture(m.kept, [None, 3]))", + "[type(x).__name__ for x in m.kept([m.Box(1), None])]", + ], + ); +} + +#[test] +fn with_no_soundness_checks_neither_build_checks_a_value_nothing_verified() { + // `--soundness none` transpiles no checks, and a compiled module then makes none + // either: a value only a check would have refused reaches its caller in both builds + agree_with_soundness( + "unverifiednone", + UNVERIFIED_VALUES, + &[ + "m.to_int(0.5)", + "m.to_dict([1])", + "type(m.to_box(m.Other())).__name__", + "m.to_optional('x')", + "m.power(2, -1)", + "m.passes([1], 'x')", + "m.held('x')", + "m.got({'k': 'x'})", + "m.first([[1]])", + "m.last([1, 'x'])", + "m.kept([3])", + ], + by_transforms::SoundnessPositions::none(), + ); +} + +#[test] +fn a_parameter_is_checked_where_the_body_begins() { + // the opt-in `parameters` position checks a function's own parameters as its body + // starts: at the call for a plain function, and at the first step for a generator, + // whose body does not run until then + agree_with_soundness( + "unverifiedentry", + "\ +from collections.abc import Iterator + + +def entry(x: dict[str, int], y: int | None) -> object: + return (x, y) + + +def produce(x: dict[str, int]) -> Iterator[object]: + yield x +", + &[ + "str(_capture(m.entry, [1], None))", + "str(_capture(m.entry, {}, 'x'))", + "m.entry({}, None)", + "str(_capture(m.produce, [1]))", + "str(_capture(next, m.produce([1])))", + "next(m.produce({'a': 1}))", + ], + by_transforms::SoundnessPositions::all(), + ); +} + +#[test] +fn an_element_check_reads_the_classes_it_names_once_where_the_loop_begins() { + // `_soundness_iter` is handed the class it tests against before the first element is + // drawn, so a loop that rebinds that name goes on testing against the class it began + // with + agree( + "unverifiedrebind", + "\ +class Box: + pass + + +def rebinding(xs: list[Box | None]) -> int: + seen = 0 + for x in xs: + globals()['Box'] = int + seen = seen + 1 + return seen +", + &["str(_capture(m.rebinding, [m.Box(), m.Box()]))"], + ); } #[test] @@ -5726,8 +6133,7 @@ fn a_missing_attribute_raises() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_attrerr"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_attrerr"); let source = "def f(o) -> object:\n return o.nope\n"; if build_source( source, @@ -5757,8 +6163,7 @@ fn list_displays_agree_and_do_not_leak() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_listbuild"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_listbuild"); let source = "\ def pair(a, b) -> object: return [a, b] @@ -6161,8 +6566,7 @@ fn a_rebound_global_is_observed() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_rebind"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_rebind"); let source = "def size(o) -> object:\n return helper(o)\n"; if build_source( source, @@ -6505,8 +6909,7 @@ fn a_decorated_method_is_the_interpreted_one_and_its_siblings_are_not() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_methoddecoratoronce_t"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_methoddecoratoronce_t"); let built = match build_source( MARKED_METHODS, "by_diff_methoddecoratoronce_t", @@ -6591,8 +6994,7 @@ fn a_decorated_class_is_the_compiled_type() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_classdecoratoronce_t"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_classdecoratoronce_t"); let built = match build_source( MARKED_CLASSES, "by_diff_classdecoratoronce_t", @@ -6668,8 +7070,7 @@ fn a_decorated_class_named_in_a_deferred_annotation_still_compiles() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_classdecoratorannotation"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_classdecoratorannotation"); let source = "\ from __future__ import annotations @@ -6803,8 +7204,7 @@ fn a_path_decorated_definition_is_the_compiled_one() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_pathdecolive"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_pathdecolive"); let source = "\ from __future__ import annotations @@ -6888,8 +7288,7 @@ fn a_decorator_that_is_a_call_declines() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_calldeco"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_calldeco"); let source = "\ MADE = [] @@ -6956,8 +7355,7 @@ fn a_decorator_rooted_in_the_class_body_declines() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_classrooteddeco"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_classrooteddeco"); let source = "\ class Box: def __init__(self, n: int) -> None: @@ -7045,8 +7443,7 @@ fn a_class_decorator_over_a_partly_filled_slot_fills_it_in() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_partialslot"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_partialslot"); let source = "\ import functools @@ -7106,8 +7503,7 @@ fn a_class_decorator_applied_from_another_module_refuses_at_decoration() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_outsideslot"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_outsideslot"); let source = "\ class Ordered: def __init__(self, n: int) -> None: @@ -7167,8 +7563,7 @@ fn a_class_declining_for_its_decorator_leaves_no_layout_behind() { let Some((_, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_decorlayout"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_decorlayout"); // `Holder` names a base, so evaluating its header reaches the module — which is what // leaves `Held` visible, undecorated, in the window a moved decorator opens. the base is a // plain class: an `ABCMeta` one would decline `Holder` for its metaclass and say nothing @@ -7399,15 +7794,14 @@ fn a_static_or_class_method_is_the_compiled_one() { // identically out of its interpreted definition. `type(C.__dict__['m'])` cannot // say either: it is `staticmethod` on both legs. // - // the *descriptor* is what differs. a compiled static or class method is reached - // through a `PyCFunction`, so `type(C.m)` is `builtin_function_or_method` where the - // interpreted leg has a plain `function` or a bound `method` — and the class - // method's dict entry is a `classmethod_descriptor` rather than a `classmethod` + // the *callable* is what differs. a compiled static or class method is published as + // a compiled method inside the `staticmethod` or `classmethod`, so `type(C.m)` and + // `type(C.c.__func__)` are `method_descriptor` where the interpreted leg has a plain + // `function` let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_staticmethod_which"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_staticmethod_which"); let source = "\ class Box: @staticmethod @@ -7441,13 +7835,13 @@ class Box: &dir, "import by_diff_staticmethod_which as m\n\ print(type(m.Box.make).__name__, type(m.Box.__dict__['make']).__name__)\n\ - print(type(m.Box.named).__name__, type(m.Box.__dict__['named']).__name__)\n\ + print(type(m.Box.named.__func__).__name__, type(m.Box.__dict__['named']).__name__)\n\ print(m.Box.make(3), m.Box.named(2))\n", ); assert_eq!( out, - "builtin_function_or_method staticmethod\n\ - builtin_function_or_method classmethod_descriptor\n\ + "method_descriptor staticmethod\n\ + method_descriptor classmethod\n\ 10 Box2" ); } @@ -7551,8 +7945,7 @@ fn a_class_method_the_boundary_would_hand_over_declines() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_classmethod_defer"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_classmethod_defer"); let source = "\ DEFAULT = [1, 2] @@ -8151,8 +8544,7 @@ fn a_compiled_frame_is_what_reaches_the_module_namespace() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_globalidentity"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_globalidentity"); let source = "\ inited = False @@ -8209,8 +8601,7 @@ fn a_declined_function_reads_the_global_a_compiled_one_wrote() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_globaltwin"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_globaltwin"); let source = "\ flag = 0 @@ -8283,8 +8674,7 @@ fn a_second_decorator_over_a_static_method_declines() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_staticmethod_stacked"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_staticmethod_stacked"); let source = "\ def mark(fn): fn.marked = True @@ -8339,8 +8729,7 @@ fn string_literals_do_not_leak() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_strleak"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_strleak"); let source = "\ def classify(n: int) -> str: scratch = \"x\" + \"y\" @@ -8380,8 +8769,7 @@ fn a_concatenated_operand_keeps_the_callers_reference() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_strhold"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_strhold"); let source = "\ def join(a: str, b: str) -> str: return a + b @@ -8583,8 +8971,7 @@ fn a_native_constructor_checks_its_argument_types() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_ctorcheck"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_ctorcheck"); let source = "data class Point:\n x: int\n y: int\n"; if build_source( source, @@ -8606,7 +8993,10 @@ fn a_native_constructor_checks_its_argument_types() { except TypeError as e:\n print('TypeError:', e)\n\ else:\n print('accepted a str')\n", ); - assert_eq!(out, "TypeError: expected int, got str"); + assert_eq!( + out, + "TypeError: type soundness violation: expected int, got str" + ); } #[test] @@ -8622,8 +9012,7 @@ fn importing_a_compiled_module_again_hands_back_the_module_already_imported() { if !supports(&toolchain, (3, 13)) { return; } - let dir = diff_root().join("by_diff_reimport"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_reimport"); let source = "\ _step = 0 _limit = 0 @@ -8702,8 +9091,7 @@ fn importing_a_compiled_module_again_allocates_nothing_that_stays() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_reimportleak"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_reimportleak"); let source = "\ def total(n: int) -> int: return n + 1 @@ -8744,8 +9132,7 @@ fn a_native_class_has_a_fixed_layout() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_layout"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_layout"); let source = "\ data class Point: x: int @@ -8811,8 +9198,7 @@ fn a_class_takes_an_attribute_its_layout_never_had() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_instdict"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_instdict"); let options = Options { language: by_irbuild::Language::Python, ..Options::default() @@ -8914,8 +9300,7 @@ fn writing_into_an_instance_dict_runs_nothing_but_the_write() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_dictwrites_check"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_dictwrites_check"); let options = Options { language: by_irbuild::Language::Python, ..Options::default() @@ -9409,8 +9794,7 @@ fn a_defaulted_field_is_answered_by_a_descriptor_of_ours() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_clsdefaultdesc"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_clsdefaultdesc"); let options = Options { language: by_irbuild::Language::Python, ..Options::default() @@ -9604,8 +9988,7 @@ fn the_class_that_answered_the_dict_is_the_compiled_one() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_instdictlive"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_instdictlive"); if build_source( A_CLASS_WHOSE_DICT_IS_READ, "by_diff_instdictlive", @@ -9701,8 +10084,7 @@ fn a_native_class_instance_does_not_leak() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_classleak"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_classleak"); let source = "\ data class Holder: label: str @@ -9770,8 +10152,7 @@ fn a_float_module_imports_without_any_extra_runtime_module() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_floatimport"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_floatimport"); let source = "def area(r: float) -> float:\n return 3.0 * r * r\n"; if build_source( source, @@ -9802,8 +10183,7 @@ fn a_compiled_function_is_a_c_function_object() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_cfunc"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_cfunc"); let source = "def f(a: int) -> int:\n return a\n"; if build_source( source, @@ -9849,8 +10229,7 @@ fn no_any_turns_a_gradual_decline_into_an_error() { let Some((_, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_noany"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_noany"); let source = "\ def precise(a: int) -> int: @@ -9902,8 +10281,7 @@ fn require_native_rejects_any_decline_at_all() { let Some((_, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_reqnative"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_reqnative"); // precisely typed and still declines: `except*` has no lowering let source = "\ def precise(a: int) -> int: @@ -9957,8 +10335,7 @@ fn no_any_accepts_a_fully_typed_module() { let Some((_, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_noany_ok"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_noany_ok"); let source = "def f(a: int, b: int) -> int:\n return a * b\n"; match build_source( source, @@ -10004,8 +10381,7 @@ fn a_declined_function_is_reported_with_a_reason() { let Some((_, toolchain)) = environment() else { return; }; - let dir: PathBuf = diff_root().join("by_diff_declined"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_declined"); let source = "\ def fast(a: int) -> int: @@ -10093,8 +10469,7 @@ fn a_field_setter_checks_its_value() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_setcheck"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_setcheck"); let source = "\ data class Point: x: int @@ -10140,8 +10515,7 @@ fn a_class_typed_argument_is_checked_at_the_boundary() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_argcheck"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_argcheck"); let source = "\ data class Point: x: int @@ -10173,9 +10547,9 @@ def read(p: Point) -> int: ); assert_eq!( out, - "7\nexpected by_diff_argcheck.Point, got str\n\ - expected by_diff_argcheck.Point, got int\n\ - expected by_diff_argcheck.Point, got NoneType" + "7\ntype soundness violation: expected Point, got str\n\ + type soundness violation: expected Point, got int\n\ + type soundness violation: expected Point, got NoneType" ); } @@ -10184,8 +10558,7 @@ fn a_field_read_does_not_leak() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_fieldleak"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_fieldleak"); let source = "\ data class Holder: label: str @@ -10236,8 +10609,7 @@ fn a_literal_used_in_a_loop_neither_leaks_nor_is_released_twice() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_constborrow"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_constborrow"); let source = "\ def kept(line: str) -> int: total = 0 @@ -10411,8 +10783,7 @@ fn a_borrowed_narrowing_does_not_over_release_what_it_narrowed() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_narrowborrow"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_narrowborrow"); if build_source( BORROWED_NARROWINGS, "by_diff_narrowborrow", @@ -10838,8 +11209,7 @@ fn the_shadowed_calls_are_answered_by_compiled_bodies() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_aloneshadowkind"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_aloneshadowkind"); if build_source( A_CLASS_NOTHING_DERIVES_FROM, "by_diff_aloneshadowkind", @@ -10904,8 +11274,7 @@ fn a_direct_method_call_does_not_leak() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_methleak"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_methleak"); let source = "\ data class Holder: label: str @@ -10953,8 +11322,7 @@ fn a_borrowed_intermediate_does_not_leak_or_lose_a_reference() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_borrow"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_borrow"); let source = "\ data class Holder: label: str @@ -11039,8 +11407,7 @@ fn a_borrow_survives_a_finalizer_that_runs_a_collection() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_finalizer"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_finalizer"); let source = "\ data class Inner: label: str @@ -11215,6 +11582,577 @@ class Snapshots: ); } +#[test] +fn a_method_read_off_an_instance_is_a_bound_method() { + // python binds a class's function to the instance as a `method`, whose `__func__` is + // the function the class holds and whose `__self__` is the instance. `WeakMethod` takes + // one apart into those two and rebuilds it through `type(method)`, so the parts have to + // be there and rebuilding from them has to call the same body + agree_python( + "boundmethod", + "\ +import weakref + + +class Node: + def __init__(self, v: int) -> None: + self.v = v + + def ping(self) -> int: + return self.v + + def add(self, k: int = 1) -> int: + return self.v + k + + +def weakly(n: Node) -> int: + m = weakref.WeakMethod(n.ping) + got = m() + return got() if got is not None else -1 +", + &[ + "m.weakly(m.Node(3))", + "(lambda n: __import__('weakref').WeakMethod(n.ping)()())(m.Node(4))", + // a weak method dies with its instance + "(lambda w: (gc.collect(), w())[1])(__import__('weakref').WeakMethod(m.Node(5).ping))", + "(lambda n: (n.ping.__func__ is m.Node.ping, n.ping.__self__ is n, n.ping == n.ping))(m.Node(6))", + "(lambda n: type(n.ping)(m.Node.add, n)(k=4))(m.Node(7))", + "(lambda n: (m.Node.ping(n), m.Node.add(n, 2), n.add(k=3)))(m.Node(8))", + "(lambda n: (__import__('inspect').ismethod(n.ping), n.ping.__name__, n.ping.__qualname__))(m.Node(9))", + "(lambda c: (c.copy(m.Node.ping) is m.Node.ping, c.deepcopy(m.Node.add) is m.Node.add))(__import__('copy'))", + "__import__('weakref').ref(m.Node.ping)() is m.Node.ping", + ], + ); +} + +#[test] +fn a_class_method_and_a_static_method_bind_as_a_class_statement_s_do() { + // python holds a `classmethod` and a `staticmethod` over a function: reading the class + // method off the class binds the class as a `method` with `__func__` and `__self__`, + // and reading the static method hands back the callable itself. a compiled class holds + // them the same way, over the compiled method, so `WeakMethod` takes a class method + // apart as it does an instance's + agree_python( + "classstatic", + "\ +class Root: + pass + + +class Box(Root): + def __init__(self, v: int) -> None: + self.v = v + + @classmethod + def make(cls, v: int) -> object: + return cls(v) + + @staticmethod + def twice(v: int) -> int: + return v * 2 + + +class Sub(Box): + pass + + +class Sealed: + @classmethod + def name(cls) -> str: + return cls.__name__ + + @staticmethod + def twice(v: int) -> int: + return v * 2 +", + &[ + "(m.Box.make(3).v, type(m.Sub.make(4)).__name__, m.Box(1).make(5).v)", + "(m.Box.twice(4), m.Box(1).twice(5), m.Sub.twice(6), m.Sealed.twice(7), m.Sealed().twice(8))", + "(m.Box.make.__self__ is m.Box, m.Sub.make.__self__ is m.Sub, m.Sealed.name.__self__ is m.Sealed)", + "(m.Box.make.__func__ is m.Box.__dict__['make'].__func__, m.Box.twice is m.Box.__dict__['twice'].__func__)", + "[type(m.Box.__dict__[name]).__name__ for name in ('make', 'twice')]", + "[type(m.Sealed.__dict__[name]).__name__ for name in ('name', 'twice')]", + "(lambda i: (i.ismethod(m.Box.make), i.ismethod(m.Box.twice), m.Box.make.__name__, m.Box.make.__qualname__))(__import__('inspect'))", + "__import__('weakref').WeakMethod(m.Box.make)()(2).v", + "__import__('weakref').WeakMethod(m.Sealed.name)()()", + "type(m.Box.make)(m.Box.__dict__['make'].__func__, m.Sub)(9).__class__.__name__", + ], + ); +} + +#[test] +fn a_method_answers_for_the_annotations_the_class_statement_evaluated() { + // a method of a class answers `__annotations__` with what python evaluated for the + // interpreted definition — on 3.14 when first asked for — whether it is an instance + // method, a class method or a static method + agree_python( + "methodannotations", + "\ +class Root: + pass + + +class Box(Root): + def __init__(self, v: int) -> None: + self.v = v + + def read(self, scale: int) -> int: + return self.v * scale + + @classmethod + def make(cls, v: int) -> 'Box': + return cls(v) + + @staticmethod + def twice(v: int) -> int: + return v * 2 + + def later(self) -> 'Later': + return Later() + + +class Sealed: + def ident[T](self, x: T) -> T: + return x + + +class Later: + pass +", + &[ + "m.Box.read.__annotations__", + "m.Box(1).read.__annotations__", + "(m.Box.make.__annotations__, m.Box.twice.__annotations__)", + "m.Box.later.__annotations__", + "m.Box.read.__annotations__ is m.Box.read.__annotations__", + "[param.__name__ for param in m.Sealed.ident.__type_params__]", + "str(__import__('typing').get_type_hints(m.Box.read))", + ], + ); +} + +#[test] +fn a_method_answers_for_the_def_it_was_made_from() { + // a compiled method stands in for the function a class statement would hold, so what + // that function says about its `def` is the interpreted definition's to answer: where + // it was defined, its defaults and code, which `inspect.signature` reads, and the + // globals `typing.get_type_hints` resolves a forward reference in. `functools.wraps` + // copies `__module__` and skips it silently where it is missing. it prints itself as a + // function does, a static method included, and a class a metaclass built names its + // own class rather than `object` + let Some(compiled) = agree_python_built( + "methoddef", + "\ +import abc + + +class Counter: + def get(self, k: int = 3, *, scale: int = 1) -> int: + \"\"\"get doc\"\"\" + return k * scale + + def fwd(self, other: 'Counter') -> 'Counter': + return other + + @staticmethod + def twice(v: int) -> int: + return v * 2 + + @classmethod + def make(cls, v: int) -> int: + return v + + +class Abstract(abc.ABC): + @staticmethod + def still(v: int) -> int: + return v + + def read(self, v: int = 1) -> int: + return v +", + &[ + "(m.Counter().get.__module__, m.Counter.__dict__['get'].__module__)", + "(m.Counter.twice.__module__, m.Counter.make.__module__)", + "(m.Counter.get.__defaults__, m.Counter.get.__kwdefaults__, m.Counter().get.__defaults__)", + "(m.Counter.get.__code__.co_varnames, m.Counter.get.__globals__ is vars(m))", + "[str(__import__('inspect').signature(f)) for f in (m.Counter.get, m.Counter().get, m.Counter.twice, m.Counter.make, m.Abstract.read)]", + "__import__('typing').get_type_hints(m.Counter().fwd)", + "(lambda wrapped: (wrapped.__module__, wrapped.__qualname__, wrapped.__doc__))(__import__('functools').wraps(m.Counter.get)(lambda: 0))", + "__import__('pickle').loads(__import__('pickle').dumps(m.Counter().get))(2)", + "[repr(f).split(' at ')[0] for f in (m.Counter.get, m.Counter.twice, m.Counter.__dict__['twice'], m.Abstract.still, m.Abstract.read)]", + "[f.__qualname__ for f in (m.Abstract.still, m.Abstract.read)]", + ], + ) else { + return; + }; + // the interpreted leg holds functions, so the answers above only say something where + // the compiled leg holds its own methods + let Some((python, _)) = environment() else { + return; + }; + let out = run( + &python, + &compiled, + "import by_diff_methoddef as m\n\ + print(type(m.Counter.__dict__['get']).__name__, type(m.Counter.twice).__name__,\n\ + \x20 type(m.Abstract.__dict__['read']).__name__)\n", + ); + assert_eq!(out, "method_descriptor method_descriptor method_descriptor"); +} + +#[test] +fn a_method_answers_nothing_its_def_has_no_answer_to() { + // the builtin a compiled method's call goes to answers `__self__`, `__objclass__` and + // `__text_signature__`, and a function raises for all three. a `staticmethod` or a + // `classmethod` copies its function's names, docstring and (on 3.13) annotations when + // it is made, which for a compiled one has to wait for the definition it answers from + let Some(compiled) = agree_python_built( + "methodnoanswer", + "\ +class Counter: + def get(self, k: int = 3) -> int: + \"\"\"get doc\"\"\" + return k + + @staticmethod + def twice(v: int) -> int: + \"\"\"twice doc\"\"\" + return v * 2 + + @classmethod + def make(cls, v: int) -> int: + \"\"\"make doc\"\"\" + return v +", + &[ + "[[hasattr(f, n) for n in ('__self__', '__objclass__', '__text_signature__')] for f in (m.Counter.get, m.Counter.twice, m.Counter().twice, m.Counter.make.__func__, vars(m.Counter)['twice'].__func__)]", + "[(w.__name__, w.__qualname__, w.__module__, w.__doc__) for w in (vars(m.Counter)['twice'], vars(m.Counter)['make'])]", + "[sorted(vars(w)) for w in (vars(m.Counter)['twice'], vars(m.Counter)['make'])]", + "[w.__annotations__ for w in (vars(m.Counter)['twice'], vars(m.Counter)['make'])]", + "(m.Counter.twice(2), m.Counter.make(3), m.Counter().get())", + ], + ) else { + return; + }; + let Some((python, _)) = environment() else { + return; + }; + let out = run( + &python, + &compiled, + "import by_diff_methodnoanswer as m\n\ + print(type(m.Counter.get).__name__, type(m.Counter.twice).__name__,\n\ + \x20 type(m.Counter.make.__func__).__name__)\n", + ); + assert_eq!(out, "method_descriptor method_descriptor method_descriptor"); +} + +#[test] +fn a_nested_function_answers_for_its_annotations() { + // python evaluates a nested function's annotations over the frames it is written in: + // where the `def` stands on 3.13, and when they are first asked for from 3.14. either way + // they, the signature `inspect` reads and the hints `typing` resolves are what the + // interpreted definition answers, and on 3.14 so is every format `annotationlib` asks for + agree_python( + "nestedannotations", + "\ +from collections.abc import Callable + + +def annotated(kind: type) -> Callable[..., object]: + def inner(y: kind, z: int = 1, *rest: str, k: kind | None = None, **kw: 'Later') -> list[kind]: # type: ignore + return [y] + return inner + + +def plain() -> Callable[[], int]: + def inner() -> int: + return 1 + return inner + + +def bare() -> Callable[[object], object]: + def inner(y, z=2, *, w=3): # type: ignore + return y + return inner + + +def rebound(kind: type) -> Callable[[int], object]: + def inner(y: kind) -> kind: # type: ignore + return y + kind = str + return inner + + +def later_bound() -> Callable[[int], object]: + def inner(y: Kind) -> None: # type: ignore + return None + Kind = int + return inner + + +def deep(kind: type) -> Callable[[], Callable[[int], object]]: + def middle() -> Callable[[int], object]: + def inner(y: kind) -> kind: # type: ignore + return y + return inner + return middle + + +def unresolved() -> Callable[[int], object]: + def inner(y: missing_name) -> None: # type: ignore + return None + return inner + + +def generic() -> Callable[[int], object]: + def ident[T: int](x: T) -> T: + return x + return ident + + +def enclosing_generic[T](v: T) -> Callable[[T], T]: + def keep(x: T) -> T: + return x + return keep + + +class Box: + def make(self, scale: int) -> Callable[..., object]: + def inner(y: int, z: int = scale) -> int: + return y * z + return inner + + +class Later: + pass +", + &[ + "_annotation_report(m.annotated(int))", + "_annotation_report(m.plain())", + "_annotation_report(m.bare())", + "_annotation_report(m.rebound(int))", + "isinstance(_capture(m.later_bound), NameError)", + "_annotation_report(m.deep(float)())", + "_annotation_report(m.unresolved()) if __import__('sys').version_info >= (3, 14) else type(_capture(lambda: m.unresolved().__annotations__)).__name__", + "_annotation_report(m.generic())", + "(lambda f: f.__annotations__['x'] is f.__type_params__[0])(m.generic())", + // annotations a build cannot supply are refused where they are asked for, and + // making the closure is not asking + "m.enclosing_generic(1)(5)", + "_annotation_report(m.Box().make(4))", + // a function's annotations are its own to write over, and on 3.14 so is the + // function that evaluates them + "(lambda f: (setattr(f, '__annotations__', {'x': 1}), f.__annotations__)[1])(m.annotated(int))", + "(lambda f: (setattr(f, '__annotations__', None), f.__annotations__)[1])(m.annotated(int))", + "(lambda f: (setattr(f, '__annotate__', lambda format: {'q': format}), f.__annotations__, f.__annotate__(2))[1:] if __import__('sys').version_info >= (3, 14) else None)(m.annotated(int))", + "type(_capture(setattr, m.plain(), '__type_params__', [])).__name__", + ], + ); +} + +#[test] +fn a_nested_function_s_annotations_are_evaluated_when_first_asked_for() { + // python 3.13 evaluates a nested function's annotations where the `def` stands, and a + // compiled one evaluates them when they are first asked for, as 3.14 does — over the + // values the enclosing names held at the `def`, which 3.14 does not keep. so an + // annotation's effect happens at the first read and not before, and not at all once + // something else has been written over the annotations. on 3.14 python answers the + // same + let Some((_dir, answers)) = compiled_answers( + "lazyannotations", + "\ +from collections.abc import Callable + +seen: list[str] = [] + + +def note(label: str) -> type: + seen.append(label) + return int + + +def made() -> Callable[[int], object]: + def inner(y: note('y')) -> None: # type: ignore + return None + return inner + + +def rebound(kind: type) -> Callable[[int], object]: + def inner(y: kind) -> kind: # type: ignore + return y + kind = str + return inner +", + Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + &[ + "(lambda f: (list(m.seen), f.__annotations__, list(m.seen), f.__annotations__ is f.__annotations__, list(m.seen)))(m.made())", + "(lambda f: (setattr(f, '__annotations__', {}), f.__annotations__, list(m.seen)))(m.made())", + // 3.14 reads the names when asked, 3.13 when the `def` stood + "m.rebound(int).__annotations__['y'] is (str if __import__('sys').version_info >= (3, 14) else int)", + ], + ) else { + return; + }; + assert_eq!( + answers, + [ + "([], {'y': , 'return': None}, ['y'], True, ['y'])", + "(None, {}, [])", + "True", + ] + ); +} + +#[test] +fn a_nested_function_in_a_by_module_answers_for_the_annotations_its_twin_evaluates() { + // a `.by` annotation can be written in syntax python does not have, so what python + // evaluates is the transpiler's spelling of it + agree( + "bynestedannotations", + "\ +from __future__ import annotations + + +class Box: + def make(self, scale: int) -> object: + def inner(y: int, z: int = 1, *rest: str, k: str? = None, **kw: (int) -> str) -> list[int]: + return [y * scale + z] + return inner + + +def generic() -> object: + def ident[T](x: T) -> T: + return x + return ident + + +def forward() -> object: + def later(x: Later) -> Later?: + return None + return later + + +class Later: + pass +", + &[ + "_annotation_report(m.Box().make(3))", + "_annotation_report(m.generic())", + "_annotation_report(m.forward())", + ], + ); +} + +#[test] +fn a_closure_holds_only_the_names_it_reads() { + // python's closure holds a cell for each name the body reads from an enclosing frame + // and nothing else. a method's `self` is none of those unless the body reads it, so a + // closure a method made must not keep the instance alive, and must not see the + // instance's attributes as if they were names: `read` reads the module's `label` + agree_python( + "closureholds", + "\ +from weakref import ref + +label = 'module' + + +class Snapshots: + def __init__(self): + def report(prefix, selfref=ref(self)): + return prefix + type(selfref()).__name__ + self.report = report + + +class Labelled: + def __init__(self): + self.label = 'field' + + def reader(self): + def read(): + return label + return read + + def deep(self, n): + def middle(): + def inner(): + return n + len(label) + return inner + return middle + + +class Cycle: + def __init__(self): + def mine(): + return self + self.mine = mine +", + &[ + "(lambda out: (out[0], out[1]('x:')))(_outlives(m.Snapshots, lambda s: s.report))", + "m.Labelled().reader()()", + "(lambda out: (out[0], out[1]()))(_outlives(m.Labelled, lambda l: l.deep(1)()))", + "(lambda out: (out[0], out[1]()()))(_outlives(m.Labelled, lambda l: l.deep(2)))", + // the instance, its function and that function's environment are a cycle + // with nothing outside it, which the collector has to be able to see all of + "_outlives(m.Cycle, lambda c: None)", + ], + ); +} + +#[test] +fn a_class_whose_slots_ask_for_weak_references_agrees() { + // `__slots__` without `__weakref__` gives an instance no weak-reference list, and naming + // it gives one back, down a chain that declares it once + agree_python( + "slotsweakref", + "\ +import weakref + + +class Tight: + __slots__ = ('v', '__weakref__') + + def __init__(self, v: int) -> None: + self.v = v + + def get(self) -> int: + return self.v + + +class Tighter(Tight): + __slots__ = ('w',) + + def __init__(self, v: int) -> None: + super().__init__(v) + self.w = v + 1 + + +class Bare: + __slots__ = ('v',) + + def __init__(self, v: int) -> None: + self.v = v + + +def referenced(t: Tight) -> bool: + return weakref.ref(t)() is t +", + &[ + "m.referenced(m.Tight(3))", + "m.referenced(m.Tighter(3))", + "(lambda t: (t.get(), hasattr(t, '__dict__'), type(_capture(setattr, t, 'x', 1)).__name__))(m.Tight(4))", + "type(_capture(__import__('weakref').ref, m.Bare(1))).__name__", + "(lambda out: (out[0], out[1]() is None))(_outlives(lambda: m.Tight(5), __import__('weakref').ref))", + "(lambda t: len(__import__('weakref').WeakSet([t])))(m.Tighter(6))", + ], + ); +} + #[test] fn an_inherited_default_binds_the_same_value_in_both_halves() { // basedpython carries a method's defaults to its overrides, so `Loud.say` may be @@ -11461,8 +12399,7 @@ fn a_closure_does_not_leak_its_environment() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_closureleak"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_closureleak"); let source = "\ def make(label: str) -> object: def get(times: int) -> str: @@ -11506,8 +12443,7 @@ fn a_closure_environment_is_not_visible_in_the_module() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_envhidden"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_envhidden"); let source = "\ def make(n: int) -> object: def get() -> int: @@ -11817,8 +12753,7 @@ fn a_nested_function_is_the_compiled_one() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_nestedwhich"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_nestedwhich"); let built = match build_source( DECORATING_CLOSURES, "by_diff_nestedwhich", @@ -11854,8 +12789,7 @@ fn a_raise_out_of_a_try_body_does_not_leak_what_it_wrote() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_handlerleak"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_handlerleak"); let source = "\ def guarded(words: list[str], index: int) -> str: held = \"held\" + words[0] @@ -12306,8 +13240,7 @@ fn a_shared_cell_does_not_leak() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_cellleak"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_cellleak"); let source = "\ def holder(label: str) -> ((str) -> str): current = label @@ -12431,8 +13364,7 @@ fn a_generator_is_a_real_iterator_to_python() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_geniter"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_geniter"); let source = "\ def counted(n: int) -> object: i = 0 @@ -12469,10 +13401,9 @@ def counted(n: int) -> object: /// `require_native` is what makes the answers the *compiled* generator's: a /// declined function would run from its interpreted definition and leak nothing, /// so the test would pass without exercising anything -fn leak_module(tag: &'static str) -> Option<(String, std::path::PathBuf)> { +fn leak_module(tag: &'static str) -> Option<(String, Scratch)> { let (python, toolchain) = environment()?; - let dir = diff_root().join(tag); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new(tag); let source = "\ class Boom(Exception): pass @@ -12709,8 +13640,7 @@ fn a_re_raise_does_not_retain_the_exception() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_reraiseleak"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_reraiseleak"); let source = "\ class Boom(Exception): pass @@ -14010,8 +14940,7 @@ fn a_for_over_a_compiled_generator_is_emitted_as_its_type_s_step() { let Some((_python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_stepped_pin"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_stepped_pin"); let options = Options { require_native: true, language: by_irbuild::Language::Python, @@ -14078,8 +15007,7 @@ fn a_compiled_state_object_answers_the_send_slot() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_sendslot_pin"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_sendslot_pin"); let source = "\ def counting(n: int) -> object: i = 0 @@ -14404,8 +15332,7 @@ fn a_state_object_tells_the_runtime_which_surface_it_is() { let Some((_python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_framekind_pin"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_framekind_pin"); let source = "\ from typing import Any @@ -14524,8 +15451,7 @@ fn a_finish_is_emitted_apart_from_a_written_stop_iteration() { let Some((_, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_finish_pin"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_finish_pin"); let source = "\ def bare(n: int) -> object: yield n @@ -14839,8 +15765,7 @@ fn a_parked_value_does_not_leak() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_parkleak"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_parkleak"); // `label + (yield i)` holds the label across the suspension, so the state object // owns it for as long as the frame is parked let source = "\ @@ -14909,8 +15834,7 @@ fn a_coroutine_is_awaitable_and_not_iterable() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_coro"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_coro"); let source = "\ async def plain(n: int) -> int: return n * 2 @@ -14948,8 +15872,7 @@ fn a_coroutine_does_not_leak() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_coroleak"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_coroleak"); let source = "\ async def echo(label: str) -> str: return label @@ -15509,8 +16432,7 @@ fn a_string_default_does_not_leak() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_defaultleak"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_defaultleak"); let source = "\ def padded(a: str, fill: str = \"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\") -> str: return a + fill @@ -15698,8 +16620,7 @@ fn a_variadic_argument_does_not_leak() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_varleak"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_varleak"); let source = "\ def joined(*parts: str) -> int: total = 0 @@ -15999,8 +16920,7 @@ fn a_property_is_published_as_a_property_object() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_propobject"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_propobject"); let source = "\ class Box: def __init__(self, n: int) -> None: @@ -16074,8 +16994,7 @@ fn a_lone_property_getter_is_published_over_the_compiled_body() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_proplone"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_proplone"); let source = "\ class Box: def __init__(self, n: int) -> None: @@ -16136,8 +17055,7 @@ fn a_published_property_carries_the_getters_docstring() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_propdoc"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_propdoc"); let source = "\ class Box: def __init__(self, n: int) -> None: @@ -16219,8 +17137,7 @@ fn a_property_over_an_abstract_base_is_not_left_abstract() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_propabstract"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_propabstract"); let source = "\ import abc @@ -16311,8 +17228,7 @@ fn a_lone_property_getter_over_an_emitted_base_is_published() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_propbase"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_propbase"); let source = "\ class Held: def held(self) -> int: @@ -16447,8 +17363,7 @@ fn a_property_reaches_a_static_type_as_well() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_propstatic"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_propstatic"); let source = "\ class Holder: def __init__(self) -> None: @@ -16623,8 +17538,7 @@ fn a_field_of_a_class_something_extends_is_read_by_name_where_the_name_is_taken_ let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_extfield_check"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_extfield_check"); let options = Options { language: by_irbuild::Language::Python, ..Options::default() @@ -16779,8 +17693,7 @@ fn a_property_the_backend_cannot_fold_declines() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_propdecline"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_propdecline"); let source = "\ def marking(fn: object) -> object: return fn @@ -16939,8 +17852,7 @@ fn an_accessor_blocks_halves_are_the_compiled_bodies() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_accessorhalves"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_accessorhalves"); let source = "\ class Cell: var v: int @@ -17141,8 +18053,7 @@ fn an_initialised_accessor_blocks_halves_are_the_compiled_bodies() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_accessorinithalves"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_accessorinithalves"); let source = "\ class Cell: var v: int = 4 @@ -17420,8 +18331,7 @@ fn an_unboxed_array_does_not_leak_its_buffer() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_arrayleak"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_arrayleak"); // the buffer is `PyMem_Malloc`, not a `PyObject` — so a leak of one is invisible // to `gc.get_objects()` and to a refcount check. the process's own footprint is // the only thing that sees it @@ -17948,10 +18858,8 @@ def counters() -> list[object]: out.append(get) return [f() for f in out] "; - let dir = diff_root().join("by_diff_pyloop"); - let interpreted = diff_root().join("by_diff_pyloop_i"); - let _ = std::fs::remove_dir_all(&dir); - let _ = std::fs::remove_dir_all(&interpreted); + let dir = Scratch::new("by_diff_pyloop"); + let interpreted = Scratch::new("by_diff_pyloop_i"); std::fs::create_dir_all(&interpreted).expect("the directory is created"); std::fs::write(interpreted.join("by_diff_pyloop.py"), source).expect("written"); @@ -18661,8 +19569,7 @@ fn a_subclass_that_appends_nothing_is_the_compiled_type() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_appendnothing_t"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_appendnothing_t"); let built = match build_source( APPENDED_FAMILY, "by_diff_appendnothing_t", @@ -18775,8 +19682,7 @@ fn a_subclass_write_to_a_base_s_field_keeps_the_rest_of_the_module() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_basefieldwrite_t"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_basefieldwrite_t"); let built = match build_source( WRITES_A_BASE_S_FIELD, "by_diff_basefieldwrite_t", @@ -18914,8 +19820,7 @@ fn only_the_class_no_spec_can_build_is_left_interpreted() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_perclassheld_t"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_perclassheld_t"); let built = match build_source( REFUSED_BESIDE_KEPT, "by_diff_perclassheld_t", @@ -18966,8 +19871,7 @@ fn the_classes_beside_a_refused_one_lay_out_and_deallocate() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_perclassheld_d"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_perclassheld_d"); let built = match build_source( REFUSED_BESIDE_KEPT, "by_diff_perclassheld_d", @@ -19092,8 +19996,7 @@ fn a_module_function_stands_where_the_whole_family_stood_down() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_refusedfamily_t"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_refusedfamily_t"); let built = match build_source( REFUSED_FAMILY, "by_diff_refusedfamily_t", @@ -19211,8 +20114,7 @@ fn a_chain_of_appended_storage_deallocates_without_growing() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_appendchain_t"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_appendchain_t"); let built = match build_source( APPENDED_CHAIN, "by_diff_appendchain_t", @@ -19366,8 +20268,7 @@ fn a_chain_over_a_base_that_holds_nothing_deallocates_without_growing() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_hollowchain_t"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_hollowchain_t"); let built = match build_source( HOLLOW_CHAIN, "by_diff_hollowchain_t", @@ -19527,8 +20428,7 @@ fn a_subclass_with_no_storage_stands_on_a_base_that_declined_later() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_lostbase"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_lostbase"); let source = "\ class TokenList(list): @@ -19713,8 +20613,7 @@ fn a_base_beside_an_outside_one_is_built_by_calling_its_metaclass() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_mixedmeta"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_mixedmeta"); let source = "\ import abc import codecs @@ -19939,8 +20838,7 @@ fn a_class_keyed_over_a_base_this_module_emits_is_the_compiled_type() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_keyedbase_t"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_keyedbase_t"); let built = match build_source( KEYED_OVER_AN_EMITTED_BASE, "by_diff_keyedbase_t", @@ -20066,8 +20964,7 @@ fn a_class_carrying_a_decorated_method_is_the_compiled_one() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_metadecolive"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_metadecolive"); let source = "\ from abc import ABCMeta, abstractmethod @@ -20122,7 +21019,7 @@ class Marked(metaclass=ABCMeta): assert_eq!( out, // the plain method is the compiled entry, so this class is not the twin - "method_descriptor classmethod_descriptor\n\ + "method_descriptor classmethod\n\ function function\n\ ['area'] ABCMeta" ); @@ -20151,8 +21048,7 @@ class Point: def total(self) -> int: return self.x + self.y "; - let base = diff_root().join("by_diff_pkgmodule"); - let _ = std::fs::remove_dir_all(&base); + let base = Scratch::new("by_diff_pkgmodule"); let compiled_root = base.join("c"); let interpreted_root = base.join("i"); let compiled = compiled_root.join("by_diff_pkg"); @@ -20281,8 +21177,7 @@ fn which_build_answers_for_a_metaclass_class() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_metafields"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_metafields"); let source = "\ from abc import ABC, ABCMeta @@ -20403,8 +21298,7 @@ fn a_metaclass_that_remakes_a_class_level_constant_is_turned_down_after_the_call let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_metaconstant"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_metaconstant"); let source = "\ from enum import StrEnum, auto @@ -20484,8 +21378,7 @@ fn a_dunder_the_module_body_hangs_on_a_class_keeps_it_off_the_compiled_surface() let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_hungdunder"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_hungdunder"); let source = "\ from abc import ABCMeta @@ -20583,8 +21476,7 @@ fn a_constant_that_reads_back_differently_every_time_is_not_turned_down_for_it() let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_metaunstable"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_metaunstable"); let source = "\ from abc import ABCMeta @@ -20657,8 +21549,7 @@ fn a_metaclass_that_raises_on_the_namespace_it_is_handed_leaves_the_import_stand let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_metaraise"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_metaraise"); let source = "\ from collections import namedtuple from enum import Enum @@ -20742,8 +21633,7 @@ fn a_class_level_constant_beside_a_base_of_ours_reaches_the_metaclass_namespace( let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_metaconstant_base"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_metaconstant_base"); let source = "\ import codecs @@ -20826,8 +21716,7 @@ fn a_conditional_in_a_class_body_carries_across_whichever_leg_ran() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_class_conditional"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_class_conditional"); let source = "\ class Box: n: int = 0 @@ -20923,8 +21812,7 @@ fn the_shapes_a_class_body_block_is_not_lowered_for_decline() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_class_conditional_declines"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_class_conditional_declines"); let source = "\ on = True @@ -21067,8 +21955,7 @@ fn the_outer_class_is_compiled_and_the_class_written_in_it_is_not() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_nested_class_legs"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_nested_class_legs"); let source = "\ def tag(cls: type) -> type: cls.tagged = True @@ -21140,8 +22027,7 @@ fn a_base_a_class_written_in_a_class_body_stands_on_gives_up_its_emission() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_nested_class_emitted_base"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_nested_class_emitted_base"); let source = "\ class Emitted: def __init__(self, v: int) -> None: @@ -21230,8 +22116,7 @@ fn the_shapes_a_class_written_in_a_class_body_is_not_lowered_for_decline() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_nested_class_declines"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_nested_class_declines"); let source = "\ class Dunder: n = 0 @@ -21306,8 +22191,7 @@ fn a_slots_declaration_reaches_the_metaclass_rather_than_the_finished_type() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_metaslots"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_metaslots"); let source = "\ from abc import ABC @@ -21370,8 +22254,7 @@ fn a_class_constant_naming_another_class_reaches_the_metaclass_namespace_remappe let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_metaconstant_remap"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_metaconstant_remap"); let source = "\ from abc import ABCMeta @@ -21435,8 +22318,7 @@ fn a_class_the_module_pops_out_of_its_own_globals_stays_off_the_compiled_surface let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_poppedclass"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_poppedclass"); let source = "\ from abc import ABCMeta @@ -21510,8 +22392,7 @@ fn a_class_the_module_pops_out_under_a_computed_name_takes_the_whole_module_with let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_computedpop"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_computedpop"); let source = "\ class Gone: def label(self) -> str: @@ -21586,8 +22467,7 @@ fn an_annotated_class_attribute_reaches_the_compiled_type() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_annotatedconstant"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_annotatedconstant"); let source = "\ class Tagged: KIND: str = \"tagged\" @@ -21747,8 +22627,7 @@ fn a_late_gift_that_could_hand_the_interpreted_class_back_is_moved_onto_the_type let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_twinshapes"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_twinshapes"); let source = "\ class Other: def tag(self) -> str: @@ -21844,8 +22723,7 @@ fn an_instance_several_names_hold_is_moved_once() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_twinshared"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_twinshared"); let source = "\ class Leaf: def __init__(self, tag: str) -> None: @@ -21923,8 +22801,7 @@ fn an_instance_the_layout_cannot_hold_is_left_where_the_body_built_it() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_twinunmoved"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_twinunmoved"); let source = "\ class Loose: def __init__(self) -> None: @@ -21996,8 +22873,7 @@ fn a_global_holding_an_instance_the_move_left_behind_is_read_as_an_object() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_globaltwin"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_globaltwin"); let source = "\ class Form(Exception): def label(self) -> str: @@ -22103,8 +22979,7 @@ fn an_instance_carrying_a_name_the_layout_never_had_moves_with_it() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_twinextra"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_twinextra"); let built = match build_source( AN_INSTANCE_GIVEN_A_NAME_ITS_CLASS_NEVER_MENTIONED, "by_diff_twinextra", @@ -22239,8 +23114,7 @@ fn a_frozen_instance_moves_only_where_it_has_no_field_to_fill() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_twinfrozen"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_twinfrozen"); let source = "\ frozen data class Fixed: n: int @@ -22313,10 +23187,8 @@ fn a_move_that_fails_partway_does_not_leave_a_half_written_instance_standing() { let Some((python, toolchain)) = environment() else { return; }; - let compiled = diff_root().join("by_diff_twinhalf_c"); - let interpreted = diff_root().join("by_diff_twinhalf_i"); - let _ = std::fs::remove_dir_all(&compiled); - let _ = std::fs::remove_dir_all(&interpreted); + let compiled = Scratch::new("by_diff_twinhalf_c"); + let interpreted = Scratch::new("by_diff_twinhalf_i"); let source = "\ class Load: def __init__(self) -> None: @@ -22505,8 +23377,7 @@ fn a_class_attribute_naming_a_module_function_keeps_the_definition_that_binds() let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_fntwinbind"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_fntwinbind"); let source = "\ def _dump(this: object) -> str: return \"dumped\" @@ -22574,8 +23445,7 @@ fn a_declined_class_keeps_the_dunder_slot_a_module_function_filled() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_fntwindunder"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_fntwindunder"); // the late gift is the decline lever, and a *dunder* one is what turns the class down // rather than having its attributes adopted — see the twin-shapes test above. it is // here to put `Option` on the interpreted leg, which is where `optparse` has it @@ -22652,8 +23522,7 @@ fn a_container_entry_a_decorator_later_installs_on_a_class_keeps_binding() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_fntwinconvert"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_fntwinconvert"); // the late dunder gift turns `Ordered` down, which is the only state `total_ordering` // can act on at all: an emitted type is a *static* type and refuses `setattr` // outright with `cannot set '__gt__' attribute of immutable type`. so this is also @@ -22832,8 +23701,7 @@ fn a_module_level_name_that_only_reaches_a_definition_keeps_what_the_body_put_th let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_fntwinhashed"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_fntwinhashed"); let source = "\ def proxy() -> int: return 7 @@ -22897,8 +23765,7 @@ fn a_module_function_installed_on_a_class_after_import_binds() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_fnnobind"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_fnnobind"); let source = "\ def _gt_from_lt(this: object, other: object) -> str: return \"gt\" @@ -23005,6 +23872,174 @@ def made_bare(cls: type) -> object: ); } +#[test] +fn object_new_allocates_a_compiled_class() { + // `object.__new__(cls)` is how a written `__new__` in a subclass, and a good deal of + // library code, gets a bare instance. python refuses it only for a class whose + // allocation belongs to a base written in C, so a compiled class leaves its allocation to + // `object` the way a class statement does — and the block still starts with its `int` + // fields unset + agree_python( + "objectnew", + "\ +class Node: + def __init__(self, v: int) -> None: + self.v = v + + def ping(self) -> int: + return self.v + + +class Named: + def __init__(self, name: str) -> None: + self.name = name + + +class Leaf: + def __init__(self, n: int) -> None: + self.n = n + + +class Sub(Node): + def twice(self) -> int: + return self.v * 2 + + +class Wrapped: + def __init__(self) -> None: + self.__wrapped__ = 1 + + @property + def value(self) -> int: + return self.__wrapped__ * 2 + + +def bare(cls: type) -> object: + return object.__new__(cls) +", + &[ + "type(object.__new__(m.Node)).__name__", + "str(_capture(lambda: object.__new__(m.Node).v))", + "str(_capture(lambda: object.__new__(m.Node).ping()))", + "(lambda n: (n.__init__(4), n.ping())[1])(object.__new__(m.Node))", + "str(_capture(lambda: object.__new__(m.Named).name))", + "(lambda n: (n.__init__('x'), n.name)[1])(object.__new__(m.Named))", + "str(_capture(lambda: object.__new__(m.Sub).twice()))", + // a class nothing extends is immutable, and publishes its allocation all the same + "str(_capture(lambda: object.__new__(m.Leaf).n))", + "(lambda n: (n.__init__(2), n.n)[1])(object.__new__(m.Leaf))", + "m.Leaf(3).n", + // and so does a class built as a static struct + "str(_capture(lambda: object.__new__(m.Wrapped).value))", + "(lambda w: (w.__init__(), w.value)[1])(object.__new__(m.Wrapped))", + "str(_capture(lambda: m.bare(m.Node).v))", + "type(m.bare(m.Sub)).__name__", + "(lambda n: (n.__init__(5), n.twice())[1])(m.bare(m.Sub))", + // a subclass whose own `__new__` asks `object` for the instance + "str(_capture(lambda: type('S', (m.Node,), {'__new__': lambda cls, v: object.__new__(cls)})(3).v))", + "str(_capture(lambda: type('S', (m.Node,), {'__new__': lambda cls, v: object.__new__(cls)})(3).ping()))", + "m.Node.__new__(m.Node).__class__.__name__", + "str(_capture(lambda: m.Node.__new__(m.Node).v))", + "(lambda c: c.copy(m.Node(6)).v)(__import__('copy'))", + ], + ); +} + +/// run a class statement's worth of python against the module and hand back what it bound +/// to `result` +const SUBCLASS_AGAINST: &str = + "(lambda source: (g := dict(globals(), m=m), exec(source, g), g['result'])[2])"; + +#[test] +fn a_subclass_is_handed_the_allocator_as_it_is_made() { + // an interpreted subclass gets python's generic allocator from `type.__new__`, and a + // compiled class's `int` field reads unset only from an allocation that writes the + // unset marker. the class's published `__new__` hands a subclass that allocator on the + // way to an instance, but `object.__new__(cls)` reaches the subclass's allocator without + // passing it — so a subclass is handed it as it is made, by an `__init_subclass__` that + // goes on up the chain with every argument the class statement passed. a subclass whose + // own `__init_subclass__` chains up with `super()` reaches it too + let hooked = format!( + "{SUBCLASS_AGAINST}(\"\"\" +class Hooking(m.Made): + seen = [] + def __init_subclass__(cls, **kw): + Hooking.seen.append(sorted(kw)) + super().__init_subclass__() +class Deeper(Hooking, flag=1): + pass +result = (Hooking.seen, str(_capture(lambda: object.__new__(Deeper).count))) +\"\"\")" + ); + agree_python( + "subclasshook", + "\ +class Root: + pass + + +class Made(Root): + def __init__(self) -> None: + self.count = 1 + + def read(self) -> int: + return self.count +", + &[ + "str(_capture(lambda: object.__new__(type('Plain', (m.Made,), {})).count))", + "str(_capture(lambda: object.__new__(type('Plain', (m.Made,), {})).read()))", + "(lambda S: (str(_capture(lambda: S.__new__(S).count)), S().count))(type('Own', (m.Made,), {'__new__': lambda cls: object.__new__(cls)}))", + &hooked, + "str(_capture(lambda: type('Keyword', (m.Made,), {}, flag=1)))", + "(m.Made().count, type('Plain', (m.Made,), {})().read())", + ], + ); +} + +#[test] +fn a_subclass_hook_that_does_not_chain_up_keeps_the_allocator_from_what_extends_it() { + // the two things the published `__init_subclass__` leaves telling the builds apart, + // both documented: the entry is in the class's own dict, and a subclass whose own hook + // never calls `super().__init_subclass__()` keeps it from the classes made on top of it + // until one of them is constructed through the compiled `__new__` + let quiet = format!( + "{SUBCLASS_AGAINST}(\"\"\" +class Quiet(m.Made): + def __init_subclass__(cls): + pass +class Below(Quiet): + pass +result = (object.__new__(Below).count, Below().count, str(_capture(lambda: object.__new__(Below).count))) +\"\"\")" + ); + let Some((_dir, answers)) = compiled_answers( + "subclassquiet", + "\ +class Root: + pass + + +class Made(Root): + def __init__(self) -> None: + self.count = 1 +", + Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + &["'__init_subclass__' in vars(m.Made)", &quiet], + ) else { + return; + }; + assert_eq!( + answers, + [ + "True", + "(0, 1, \"'Below' object has no attribute 'count'\")" + ] + ); +} + #[test] fn a_member_the_module_body_rewrites_after_the_class_statement_is_what_the_class_answers() { // the module body runs against the interpreted definition, so a property, a method or a @@ -23113,8 +24148,7 @@ fn a_class_keeps_what_a_factory_installed_on_it_after_the_class_statement() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_late_factory"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_late_factory"); let source = "\ class Other: def tag(self) -> str: @@ -23277,8 +24311,7 @@ fn an_annotation_that_could_hand_the_interpreted_class_back_is_refused() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_annreach"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_annreach"); let source = "\ class Node: def tag(self) -> str: @@ -23366,8 +24399,7 @@ fn an_annotation_that_never_resolves_is_refused_rather_than_emptied() { if !supports(&toolchain, (3, 14)) { return; } - let dir = diff_root().join("by_diff_annlost"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_annlost"); let source = "\ class Node: later: Never @@ -23434,8 +24466,7 @@ fn a_class_that_keeps_a_dunder_of_its_own_stays_a_static_type() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_anndunder"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_anndunder"); let source = "\ class Keeps: held: int @@ -23511,8 +24542,7 @@ fn a_subclass_of_a_class_the_metaclass_gate_turns_down_is_built_here() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_gatedbase"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_gatedbase"); let source = "\ from abc import ABCMeta @@ -23606,8 +24636,7 @@ fn a_pair_the_body_cross_links_agrees_when_the_link_is_made_after_the_class_stat let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_gatedpair"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_gatedpair"); let source = "\ class Root: def kind(self) -> str: @@ -23701,8 +24730,7 @@ fn a_private_name_in_a_class_body_is_mangled() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_privatemangle"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_privatemangle"); let source = "\ class _Printer: __LIMIT = 4 @@ -23766,8 +24794,7 @@ fn an_annotated_class_level_value_beside_a_field_is_the_same_fallback() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_constantfieldclash"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_constantfieldclash"); let source = "\ class Tagged: KIND: str = \"class-level\" @@ -23834,8 +24861,7 @@ fn a_decorated_class_carries_the_body_its_own_decorator_was_handed() { return; }; let one = |tag: &str, source: &str, script: &str| -> Option { - let dir = diff_root().join(format!("by_diff_{tag}")); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new(format!("by_diff_{tag}")); let built = match build_source( source, tag, @@ -23935,8 +24961,7 @@ fn the_class_body_capture_reaches_only_this_module_s_own_body() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_capturescope"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_capturescope"); let source = "\ class Held: kind = \"module\" @@ -24014,8 +25039,7 @@ fn fields_past_a_python_base_leave_that_class_to_its_interpreted_definition() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_pythonbasestorage"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_pythonbasestorage"); let source = "\ import codecs @@ -24099,8 +25123,7 @@ fn a_spec_that_cannot_place_the_dict_leaves_the_module_to_its_interpreted_defini let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_specdictplacement"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_specdictplacement"); let source = "\ import codecs @@ -24167,8 +25190,7 @@ fn a_finalizer_over_fields_answers_for_a_construction_that_raised() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_finalizerfields"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_finalizerfields"); let source = "\ class Held: def __init__(self, path: str) -> None: @@ -24247,8 +25269,7 @@ fn a_dict_offset_from_a_base_that_does_not_own_the_layout_keeps_the_interpreted_ let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_borrowedoffset"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_borrowedoffset"); let source = "\ import codecs @@ -24315,8 +25336,7 @@ fn a_class_whose_base_declined_declines_with_it() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_declinedbase"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_declinedbase"); let source = "\ class Outer: def __new__(cls): @@ -24395,8 +25415,7 @@ fn a_base_an_interpreted_class_extends_declines_with_it() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_interpretedsub"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_interpretedsub"); let source = "\ class Container: def __init__(self, tag: str) -> None: @@ -24502,8 +25521,7 @@ fn a_base_a_class_under_a_module_level_block_extends_declines_with_it() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_blockguard"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_blockguard"); let source = "\ class Base: def __init__(self, name: str) -> None: @@ -24595,8 +25613,7 @@ fn a_zero_argument_super_is_lowered_to_the_two_argument_form() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_zerosuper"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_zerosuper"); let source = "\ class Holder(dict): def __init__(self) -> None: @@ -24669,8 +25686,7 @@ fn a_zero_argument_super_follows_the_mro_of_the_instance() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_supermro"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_supermro"); let source = "\ class A: def go(self) -> str: @@ -24733,8 +25749,7 @@ fn a_zero_argument_super_names_the_class_the_class_statement_made() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_superowner"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_superowner"); let source = "\ class Base: def lbl(self) -> str: @@ -24799,8 +25814,7 @@ fn a_zero_argument_super_declines_where_slot_zero_is_not_the_receiver() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_supernoslot"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_supernoslot"); let source = "\ class A: def go(self) -> str: @@ -24965,8 +25979,7 @@ fn a_shadowed_super_is_called_the_way_python_calls_it() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_supershadow"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_supershadow"); let source = "\ def super() -> str: return \"shadow\" @@ -27432,7 +28445,7 @@ fn a_close_with_nothing_to_run_has_the_unwinding_s_effects() { let quiet = probe("one", "looped"); let handled = probe("one_handled", "looped_handled"); let calls: Vec<&str> = quiet.iter().chain(&handled).map(String::as_str).collect(); - let Some(answers) = compiled_answers( + let Some((_dir, answers)) = compiled_answers( "quietcloseeffects", QUIET_CLOSES, Options { @@ -28047,8 +29060,7 @@ fn python_cannot_make_or_remake_a_closure_environment() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_envforged"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_envforged"); let source = "\ from collections.abc import Callable @@ -28733,6 +29745,96 @@ def nothing(v: object) -> str: ); } +#[test] +fn a_class_pattern_believes_the_class_an_object_claims() { + // a class pattern tests its subject with `isinstance`, which believes an object's + // `__class__`. so an object claiming to be a `Point` matches `case Point(...)` and has + // its attributes looked up, where one whose type really is `Point` has them read at + // their offsets — and an attribute the claimant lacks is no match, as it is for any + // subject + let Some(compiled) = agree_python_built( + "claimedclass", + "\ +class Point: + __match_args__ = ('x', 'y') + + def __init__(self, x: int, y: int) -> None: + self.x = x + self.y = y + + +class Claims: + @property + def __class__(self) -> type: + return Point + + x = 5 + y = 7 + + +class ClaimsHalf: + @property + def __class__(self) -> type: + return Point + + x = 5 + + +def keyword(v: object) -> str: + match v: + case Point(x=x): + return 'point ' + str(x) + case _: + return 'other' + + +def positional(v: object) -> int: + match v: + case Point(a, b) if a > b: + return a - b + case Point(a, b): + return a + b + case _: + return -1 + + +def literal(v: object) -> str: + match v: + case Point(0, y): + return 'zero ' + str(y) + case Point(x, 7): + return 'seven ' + str(x) + case _: + return 'none' +", + &[ + "[f(v) for f in (m.keyword, m.positional, m.literal) for v in (m.Point(9, 2), m.Point(0, 7), m.Claims(), m.ClaimsHalf(), 3)]", + ], + ) else { + return; + }; + // an object that claims the class but holds some other type under a field the class + // keeps unboxed cannot be bound into the name that field's reads bind — the edge + // `runtime.md` documents as a place laid out as `int` needing its representation + let Some((python, _)) = environment() else { + return; + }; + let out = run( + &python, + &compiled, + "import by_diff_claimedclass as m\n\ + class Text:\n\ + \x20 __class__ = property(lambda self: m.Point)\n\ + \x20 x = 'five'\n\ + \x20 y = 7\n\ + try:\n\ + \x20 m.keyword(Text())\n\ + except TypeError as e:\n\ + \x20 print(e)\n", + ); + assert_eq!(out, "type soundness violation: expected int, got str"); +} + #[test] fn a_class_pattern_over_a_base_agrees() { // a class another one in the module extends is emitted as a mutable heap type: @@ -29304,8 +30406,7 @@ fn a_constructor_that_defers_is_still_the_compiled_type() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_computedinitslot"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_computedinitslot"); let source = "\ _sentinel = object() @@ -29739,8 +30840,7 @@ fn the_power_dunder_is_answered_by_the_compiled_type() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_powslot"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_powslot"); let source = "\ class Mod: def __init__(self, n: int) -> None: @@ -30703,8 +31803,7 @@ fn a_complex_conversion_is_answered_by_the_compiled_type() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_complexslot"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_complexslot"); let source = "\ class Cell: def __init__(self, n: int) -> None: @@ -30749,8 +31848,7 @@ fn an_await_method_fills_the_async_slot() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_awaitslot"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_awaitslot"); // the awaited iterator is a class of its own rather than a generator, so what // this exercises is `am_await` alone let source = "\ @@ -30819,8 +31917,7 @@ fn a_del_method_fills_the_finalizer_slot() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_delslot"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_delslot"); let source = "\ class Closer: def __init__(self, log: list[str], tag: str) -> None: @@ -30907,8 +32004,7 @@ fn a_getattr_hook_stands_behind_the_ordinary_lookup() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_getattrhook"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_getattrhook"); let source = "\ class Proxy: def __init__(self, tag: str) -> None: @@ -30978,8 +32074,7 @@ fn a_descriptor_get_fills_its_slot() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_descrget"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_descrget"); let source = "\ class Doubler: def __init__(self, base: int) -> None: @@ -31469,10 +32564,8 @@ def total(xs: list[float]) -> float: def sliced(xs: list[int]) -> str: return str(xs[1:3]) "; - let dir = diff_root().join("by_diff_plainpy"); - let interpreted = diff_root().join("by_diff_plainpy_i"); - let _ = std::fs::remove_dir_all(&dir); - let _ = std::fs::remove_dir_all(&interpreted); + let dir = Scratch::new("by_diff_plainpy"); + let interpreted = Scratch::new("by_diff_plainpy_i"); std::fs::create_dir_all(&interpreted).expect("the directory is created"); std::fs::write(interpreted.join("by_diff_plainpy.py"), source).expect("written"); @@ -31752,8 +32845,7 @@ fn a_compiled_data_class_matches_positionally_from_python() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_dcmatchout"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_dcmatchout"); if build_source( A_PAIR_AND_A_POINT, "by_diff_dcmatchout", @@ -31830,8 +32922,7 @@ fn the_generated_dataclass_members_are_slots_of_the_emitted_type() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_dcslots"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_dcslots"); if build_source( A_PAIR_AND_A_POINT, "by_diff_dcslots", @@ -32232,8 +33323,7 @@ fn a_subscripted_receiver_of_the_wrong_class_still_raises() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_dispatchcheck"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_dispatchcheck"); let source = "\ class Shape: def __init__(self, size: int): @@ -32279,10 +33369,10 @@ def first_area(shapes: list[Shape]) -> int: assert_eq!( out, "5 25\n\ - TypeError: expected by_diff_dispatchcheck.Shape, got object\n\ - TypeError: expected by_diff_dispatchcheck.Shape, got int\n\ - TypeError: expected by_diff_dispatchcheck.Shape, got NoneType\n\ - TypeError: expected by_diff_dispatchcheck.Shape, got Duck" + TypeError: type soundness violation: expected Shape, got object\n\ + TypeError: type soundness violation: expected Shape, got int\n\ + TypeError: type soundness violation: expected Shape, got NoneType\n\ + TypeError: type soundness violation: expected Shape, got Duck" ); } @@ -32296,8 +33386,7 @@ fn a_dispatched_call_does_not_run_an_argument_before_checking_its_receiver() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_dispatchorder"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_dispatchorder"); let source = "\ LOG = [] @@ -32350,7 +33439,7 @@ def scale_first(shapes: list[Shape]) -> int: assert_eq!( out, "10 50\n\ - TypeError: expected by_diff_dispatchorder.Shape, got object\n\ + TypeError: type soundness violation: expected Shape, got object\n\ argument ran: 0" ); } @@ -32425,8 +33514,7 @@ fn a_subclass_that_writes_no_init_inherits_the_slot() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_inheritedinitslot"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_inheritedinitslot"); let source = "\ LOG = [] @@ -32583,8 +33671,7 @@ fn a_slots_declaration_is_storage_the_emitted_type_owns() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_slotsowned"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_slotsowned"); let built = match build_source( SLOTTED, "by_diff_slotsowned", @@ -32790,8 +33877,7 @@ fn a_decorated_class_is_the_compiled_one_and_its_dict_is_collectable() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_decoratedlive"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_decoratedlive"); // `from __future__ import annotations` is what keeps `through`'s parameter from // evaluating `Derived` where the `def` stands, which is inside the window every // decorated definition above it is in until module init closes it. @@ -34821,8 +35907,7 @@ fn a_module_level_name_bound_to_a_class_follows_the_class_it_named() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_classalias"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_classalias"); let source = "\ class C: def hi(self) -> int: @@ -34885,8 +35970,7 @@ fn a_class_constant_naming_another_class_is_the_type_that_replaced_it() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_classconst"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_classconst"); let source = "\ class C: def hi(self) -> int: @@ -34981,8 +36065,7 @@ fn an_alias_reaches_the_compiled_type_rather_than_the_twin() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_aliasbase_type"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_aliasbase_type"); let source = "\ class Root: def root(self) -> str: @@ -35040,8 +36123,7 @@ fn an_alias_does_not_carry_a_base_this_module_lays_out_past_the_gate() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_aliaslaid"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_aliaslaid"); let source = "\ import codecs @@ -35484,8 +36566,7 @@ fn the_compiled_types_are_what_answer_for_every_write_shape() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_attrshapes_t"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_attrshapes_t"); let built = match build_source( ATTRIBUTE_WRITE_SHAPES, "by_diff_attrshapes_t", @@ -35569,8 +36650,7 @@ fn only_the_class_that_reads_its_own_dict_is_left_interpreted() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_attrdict_t"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_attrdict_t"); let built = match build_source( READS_ITS_OWN_DICT, "by_diff_attrdict_t", @@ -35673,8 +36753,7 @@ fn the_class_that_deletes_an_attribute_is_the_compiled_one() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_deleteattr_t"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_deleteattr_t"); let built = match build_source( DELETES_AN_ATTRIBUTE, "by_diff_deleteattr_t", @@ -35716,8 +36795,7 @@ fn deleting_an_attribute_releases_it_exactly_once() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_deleterefs_t"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_deleterefs_t"); let built = match build_source( DELETES_AN_ATTRIBUTE, "by_diff_deleterefs_t", @@ -35850,8 +36928,7 @@ fn the_nested_function_reaching_the_receiver_is_in_the_compiled_init() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_nestedself_t"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_nestedself_t"); let built = match build_source( "\ class Held: @@ -35935,8 +37012,7 @@ fn the_class_a_classmethod_write_declines_is_the_interpreted_one() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_clswrite_t"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_clswrite_t"); let built = match build_source( WRITES_ON_THE_CLASS, "by_diff_clswrite_t", @@ -36108,8 +37184,7 @@ fn the_class_an_assigned_dunder_fills_a_slot_on_is_compiled() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_assigneddunder_t"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_assigneddunder_t"); let built = match build_source( ASSIGNED_DUNDERS, "by_diff_assigneddunder_t", @@ -36299,8 +37374,7 @@ fn a_borrowed_tuple_element_does_not_move_its_source_references() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_elements_rc"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_elements_rc"); if build_source( BORROWED_ELEMENTS, "by_diff_elements_rc", @@ -37323,8 +38397,7 @@ fn a_dispatch_table_holds_the_compiled_methods_the_type_publishes() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_dispatchtablekind"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_dispatchtablekind"); let built = match build_source( A_DISPATCH_TABLE, "by_diff_dispatchtablekind", @@ -37543,8 +38616,7 @@ fn the_str_of_an_int_boxes_nothing_and_still_resolves_the_name() { let Some((_, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_strofint_shape"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_strofint_shape"); // nothing is concatenated onto the digits, so this is the conversion on its own // — a prefix in front of it would be fused further still, into `By_StrConcatInt` let source = "\ @@ -37590,8 +38662,7 @@ fn the_str_of_an_int_in_a_loop_does_not_leak() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_strofintleak"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_strofintleak"); let source = "\ def build(n: int, base: int) -> str: last = '' @@ -37629,8 +38700,7 @@ fn a_prefix_and_the_digits_of_an_int_take_one_allocation() { let Some((_, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_strconcatint_shape"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_strconcatint_shape"); let source = "\ def keys(n: int) -> str: last = 'k' @@ -37703,8 +38773,7 @@ fn an_intermediate_the_program_can_still_see_is_not_fused_away() { let Some((_, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_strconcatint_shared"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_strconcatint_shared"); let source = "\ def twice(i: int) -> str: t = str(i) @@ -37752,8 +38821,7 @@ fn fusing_a_prefix_onto_the_digits_raises_what_python_raises_for_a_bad_str() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_strconcatint_badstr"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_strconcatint_badstr"); let source = "\ from typing import Any @@ -37806,7 +38874,7 @@ def unfused(n: int) -> str: assert_eq!( out, "TypeError: can only concatenate str (not \"int\") to str\n\ - differs: TypeError: expected str, got int" + differs: TypeError: type soundness violation: expected str, got int" ); } @@ -37933,8 +39001,7 @@ fn a_prefix_and_the_digits_in_a_loop_do_not_leak() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_strconcatintleak"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_strconcatintleak"); let source = "\ def build(n: int, base: int) -> str: last = '' @@ -38178,6 +39245,71 @@ def packed_at_the_top(n: int) -> int: ); } +#[test] +fn a_packed_buffer_refuses_an_index_in_the_words_a_list_does() { + // a list whose elements are stored unboxed checks its own bounds, so it owes the + // messages `list` raises: a store says `list assignment index out of range` where a + // read says `list index out of range`, and an index too big for `Py_ssize_t` is + // refused for that before it is ever compared with the length, on both sides. a + // counter the loop holds as a machine integer reaches the same check by another + // helper, so it is asked the same questions + let Some(compiled) = agree_in( + "packedindex", + "\ +def read_at(k: int) -> bool: + flags = [] + i = 0 + while i < 4: + flags.append(True) + i = i + 1 + return flags[k] + + +def write_at(k: int) -> int: + flags = [] + i = 0 + while i < 4: + flags.append(True) + i = i + 1 + flags[k] = False + return len(flags) + + +def write_counted(start: int, n: int) -> int: + flags = [True, True] + j = start + while j < n: + flags[j] = False + j = j + 1 + return len(flags) +", + &[ + "[m.read_at(k) for k in (0, 3, -1, -4, True)]", + "[m.write_at(k) for k in (0, 3, -1, -4, True)]", + "[(type(e).__name__, str(e)) for k in (4, -5, 2**62 - 1, -2**62, 2**62, -2**62 - 1, 2**63 - 1, -2**63, 2**63, -2**63 - 1, 2**70) for e in [_capture(m.read_at, k)]]", + "[(type(e).__name__, str(e)) for k in (4, -5, 2**62 - 1, -2**62, 2**62, -2**62 - 1, 2**63 - 1, -2**63, 2**63, -2**63 - 1, 2**70) for e in [_capture(m.write_at, k)]]", + "m.write_counted(-2, 2)", + "[(type(e).__name__, str(e)) for e in [_capture(m.write_counted, 0, 3)]]", + "[(type(e).__name__, str(e)) for e in [_capture(m.write_counted, -3, 0)]]", + ], + false, + by_irbuild::Language::Python, + ) else { + return; + }; + // the answers only say the buffer's own check was asked if the buffer was built, so + // the helpers each access reaches are asserted too + let c = std::fs::read_to_string(compiled.join("by_diff_packedindex.c")) + .expect("the emitted C is kept"); + for helper in [ + "By_ArrayIndex(", + "By_ArrayStoreIndex(", + "By_ArrayStoreIndexI64(", + ] { + assert!(c.contains(helper), "no `{helper}` in the emitted C"); + } +} + #[test] fn an_unboxed_counter_indexes_every_container_and_every_edge() { // a counter a loop steps by one gets a machine representation, and a subscript @@ -38268,7 +39400,7 @@ fn a_value_a_name_held_is_dropped_when_the_name_is_rebound() { // or not anything reads the name again, whatever the name is annotated as, and // however the value reached the name — a constructor, a call, or an element of a // tuple a call answered - agree_python( + let compiled = agree_python_built( "namerebind", "\ log: list[str] = [] @@ -38448,10 +39580,12 @@ def through_a_tuple(second: Linked) -> int: ], ); // a class that fell back to its interpreted definition would agree as well - if let Some((python, _)) = environment() { + if let Some(compiled) = compiled + && let Some((python, _)) = environment() + { let out = run( &python, - &diff_root().join("by_diff_namerebind_c"), + &compiled, "import by_diff_namerebind as m\n\ print(m.__file__.endswith(('.so', '.pyd')), type(m.Cell.__dict__['__del__']).__name__, \ type(m.Slotted.__dict__['__del__']).__name__)\n", @@ -38574,7 +39708,7 @@ fn a_builtin_the_module_rebinds_is_the_one_called() { // take a native path only while the name still resolves to the interpreter's own // builtin. a module attribute written from outside — which is what // `mock.patch('mod.len')` does — is what python calls instead - agree_python( + let compiled = agree_python_built( "rebuiltin", "\ import sys @@ -38675,10 +39809,12 @@ def patched(name: str, value: object, call: Callable[[], object]) -> object: "[m.Child().who(), m.patched('super', lambda *a: type('S', (), {'who': lambda self: 'fake' + str(len(a))})(), lambda: m.Child().who())]", ], ); - if let Some((python, _)) = environment() { + if let Some(compiled) = compiled + && let Some((python, _)) = environment() + { let out = run( &python, - &diff_root().join("by_diff_rebuiltin_c"), + &compiled, "import by_diff_rebuiltin as m\n\ print(m.__file__.endswith(('.so', '.pyd')), type(m.Child.__dict__['who']).__name__)\n\ # a list held as a buffer has no list object to hand a rebound `len`, and a\n\ @@ -39120,7 +40256,7 @@ fn an_instance_made_without_init_has_no_fields_until_they_are_assigned() { // `AttributeError`, wherever it is read from, and the instance's `__dict__`, // `vars()` and `__getstate__` list only what it has. `copy` and `pickle` build their // copies exactly that way and then fill them in - agree_python( + let compiled = agree_python_built( "unsetfields", "\ class Fields: @@ -39253,10 +40389,12 @@ def by_keyword(value: object) -> int: "[m.by_keyword(m.Point(1, 2)), m.by_keyword(m.Point.__new__(m.Point)), m.by_keyword(0)]", ], ); - if let Some((python, _)) = environment() { + if let Some(compiled) = compiled + && let Some((python, _)) = environment() + { let out = run( &python, - &diff_root().join("by_diff_unsetfields_c"), + &compiled, "import by_diff_unsetfields as m\n\ print(m.__file__.endswith(('.so', '.pyd')), type(m.P.__dict__['size']).__name__, \ type(m.Money.__dict__['__lt__']).__name__)\n", @@ -39544,8 +40682,7 @@ fn a_written_new_is_the_one_the_compiled_type_runs() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_writtennewkind"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_writtennewkind"); let built = match build_source( WRITTEN_NEW, "by_diff_writtennewkind", @@ -39781,8 +40918,7 @@ fn the_docstrings_are_read_off_compiled_definitions() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_docstringkind"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_docstringkind"); let built = match build_source( DOCUMENTED, "by_diff_docstringkind", @@ -39873,8 +41009,7 @@ fn a_published_forwarder_answers_as_the_definition_it_stands_for() { let Some((python, toolchain)) = environment() else { return; }; - let dir = diff_root().join("by_diff_forwardersurface"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_diff_forwardersurface"); let source = "\ import inspect @@ -40527,8 +41662,7 @@ fn a_licensed_call_re_asks_its_lookup_under_the_mode() { let mut written = Vec::new(); for recheck in [false, true] { let tag = format!("by_diff_licencerecheck_{}", u8::from(recheck)); - let dir = diff_root().join(&tag); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new(&tag); let built = match build_source( LICENSED_SHAPES, tag.as_str(), @@ -40610,10 +41744,9 @@ fn a_licensed_call_re_asks_its_lookup_under_the_mode() { /// /// the tests below are not comparisons between two legs — there is only one leg's state /// to ask about — so they build directly rather than through [`agree_python`] -fn built_python(tag: &str, source: &str) -> Option<(String, PathBuf, String)> { +fn built_python(tag: &str, source: &str) -> Option<(String, Scratch, String)> { let (python, toolchain) = environment()?; - let dir = diff_root().join(format!("by_diff_{tag}_v")); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new(format!("by_diff_{tag}_v")); let options = Options { language: by_irbuild::Language::Python, annotate: true, @@ -40754,11 +41887,11 @@ class Derived(Base): ); } -/// build `source` as python, import it, and hand back the message the import refused with -fn refused_import(tag: &str, source: &str) -> Option { +/// build `source` as python, import it, and hand back the directory it was built into and +/// the message the import refused with +fn refused_import(tag: &str, source: &str) -> Option<(Scratch, String)> { let (python, toolchain) = environment()?; - let dir = diff_root().join(format!("by_diff_{tag}_r")); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new(format!("by_diff_{tag}_r")); let module = format!("by_diff_{tag}"); let options = Options { language: by_irbuild::Language::Python, @@ -40795,7 +41928,8 @@ fn refused_import(tag: &str, source: &str) -> Option { "the snippet failed:\n{}", String::from_utf8_lossy(&out.stderr) ); - Some(String::from_utf8_lossy(&out.stdout).trim().to_string()) + let message = String::from_utf8_lossy(&out.stdout).trim().to_string(); + Some((dir, message)) } #[test] @@ -40812,7 +41946,7 @@ fn a_mixture_left_on_a_base_the_module_replaced_refuses_the_import() { // unusable rather than quietly wrong until the compiler stands the whole family down // instead. `logging/config.py` is the shape in the standard library, where // `ConvertingDict(dict, ConvertingMixin)` is the same three lines - let Some(message) = refused_import( + let Some((_dir, message)) = refused_import( "orphanmix", "\ class Ours: @@ -40840,7 +41974,7 @@ fn a_class_on_two_bases_of_this_modules_own_refuses_the_import() { // the same failure without an outside base in sight: two emitted classes each own // their instances, so `type(\"Mixin\", (Root, Extra), ...)` cannot lay one out either, // and `Mixin` is left as the definition the body built on the two interpreted ones - let Some(message) = refused_import( + let Some((_dir, message)) = refused_import( "orphanpair", "\ class Root: diff --git a/crates/by_build/tests/end_to_end.rs b/crates/by_build/tests/end_to_end.rs index 0cdb60189c..f05db96656 100644 --- a/crates/by_build/tests/end_to_end.rs +++ b/crates/by_build/tests/end_to_end.rs @@ -18,17 +18,6 @@ use std::process::Command; use by_build::{Options, Toolchain, build_module, build_source}; -/// a temp directory of this process's own -/// -/// the same reason `differential.rs` has one: every test here builds into a fixed path -/// under the system temp directory, and nextest isolates each *test* but not two *runs*. -/// a 3.13 sweep beside a 3.14 one picks the same directory and the two overwrite each -/// other between the build and the read, which fails at full length and reads exactly -/// like a real difference -fn e2e_root() -> PathBuf { - std::env::temp_dir().join(format!("by_e2e_p{}", std::process::id())) -} - use by_ir::builder::FunctionBuilder; use by_ir::function::{CallConvention, FallbackCode, ModuleIr, ModuleName}; use by_ir::ops::{BinOp, CmpOp, Op, Terminator, Value}; @@ -36,6 +25,8 @@ use by_ir::rtype::RType; mod common; +use common::Scratch; + /// a test whose *source* needs a newer interpreter than this one has nothing to /// say: neither leg can run it, so there is nothing to compare /// @@ -76,9 +67,8 @@ fn environment() -> Option<(String, Toolchain)> { /// build `module` into a fresh directory, or `None` when there is no working C /// compiler on this machine -fn built(module: &ModuleIr, toolchain: &Toolchain, tag: &str) -> Option { - let dir = e2e_root().join(format!("by_e2e_{tag}")); - let _ = std::fs::remove_dir_all(&dir); +fn built(module: &ModuleIr, toolchain: &Toolchain, tag: &str) -> Option { + let dir = Scratch::new(format!("by_e2e_{tag}")); match build_module(module, toolchain, &dir) { Ok(artifact) => { assert!( @@ -321,7 +311,10 @@ fn a_wrong_argument_type_raises_rather_than_misreading_memory() { except TypeError as e:\n print('TypeError:', e)\n\ else:\n print('no error')\n", ); - assert_eq!(out, "TypeError: expected int, got str"); + assert_eq!( + out, + "TypeError: type soundness violation: expected int, got str" + ); } #[test] @@ -458,7 +451,7 @@ fn floats_are_unboxed_and_exclude_int() { except TypeError as e:\n print(e)\n\ else:\n print('accepted an int')\n", ); - assert_eq!(out, "expected float, got int"); + assert_eq!(out, "type soundness violation: expected float, got int"); } #[test] @@ -609,8 +602,7 @@ fn annotate_writes_a_report_beside_the_generated_c() { let Some((_, toolchain)) = environment() else { return; }; - let dir = e2e_root().join("by_e2e_annotate"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_e2e_annotate"); let source = "\ def scale(x: float) -> float: return x * 2.0 @@ -650,8 +642,7 @@ fn an_accumulated_string_is_grown_rather_than_copied() { // `agree` cannot say which build answered, and both builds answer the same // string either way — so the only way to know the accumulator is being grown in // place is to read what was emitted - let dir = e2e_root().join("by_e2e_append"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_e2e_append"); let built = by_build::emit_source( "\ def build(n: int, piece: str) -> str: @@ -689,8 +680,7 @@ fn no_annotation_is_written_unless_it_is_asked_for() { let Some((_, toolchain)) = environment() else { return; }; - let dir = e2e_root().join("by_e2e_noannotate"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_e2e_noannotate"); let Ok(built) = build_source( "def f(a: int) -> int:\n return a\n", "by_e2e_noannotate", @@ -712,8 +702,7 @@ fn the_fallback_honours_a_supplied_transpiler_config() { let Some((_, toolchain)) = environment() else { return; }; - let dir = e2e_root().join("by_e2e_fallbackconfig"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_e2e_fallbackconfig"); // `except*` has no lowering, so this declines and *runs* from the fallback — // which is exactly the code the checks have to reach let source = "\ @@ -749,8 +738,7 @@ def total(a: list[int]) -> int: ); // and the default does not, so the flag is what made the difference - let plain = e2e_root().join("by_e2e_fallbackplain"); - let _ = std::fs::remove_dir_all(&plain); + let plain = Scratch::new("by_e2e_fallbackplain"); let built = build_source( source, "by_e2e_fallbackplain", @@ -792,8 +780,7 @@ fn a_stdlib_module_compiles_without_a_hard_failure(name: &str) { "by_corpus_{}", name.trim_end_matches(".py").replace('/', "_") ); - let dir = e2e_root().join(format!("by_e2e_corpus_{module}")); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new(format!("by_e2e_corpus_{module}")); std::fs::create_dir_all(&dir).unwrap(); let Ok(source) = std::fs::read_to_string(stdlib.join(name)) else { @@ -874,8 +861,7 @@ fn a_caller_supplied_lowering_is_compiled_the_same_way() { let Some((python, toolchain)) = environment() else { return; }; - let dir = e2e_root().join("by_e2e_lowered"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_e2e_lowered"); let source = "\ def double(a: int) -> int: return a * 2 @@ -909,8 +895,7 @@ fn require_native_still_bites_on_a_caller_supplied_lowering() { let Some((_, toolchain)) = environment() else { return; }; - let dir = e2e_root().join("by_e2e_loweredstrict"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_e2e_loweredstrict"); let source = "def slow(a: int) -> None:\n try:\n pass\n except* ValueError:\n pass\n"; let lowered = by_irbuild::module_from_source( source, @@ -938,8 +923,7 @@ fn an_unchanged_module_is_not_recompiled() { let Some((_, toolchain)) = environment() else { return; }; - let dir = e2e_root().join("by_e2e_rebuild"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_e2e_rebuild"); let source = "def double(a: int) -> int:\n return a * 2\n"; let Ok(first) = build_source( source, @@ -996,8 +980,7 @@ fn a_stale_extension_is_rebuilt_even_when_the_c_is_unchanged() { let Some((_, toolchain)) = environment() else { return; }; - let dir = e2e_root().join("by_e2e_rebuild_stale"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_e2e_rebuild_stale"); let source = "def double(a: int) -> int:\n return a * 2\n"; let Ok(built) = build_source( source, @@ -1041,9 +1024,8 @@ fn built_from_source( module: &str, toolchain: &Toolchain, tag: &str, -) -> Option { - let dir = e2e_root().join(format!("by_e2e_{tag}")); - let _ = std::fs::remove_dir_all(&dir); +) -> Option { + let dir = Scratch::new(format!("by_e2e_{tag}")); match build_source(source, module, toolchain, &dir, &Options::default()) { Ok(built) => { assert!( @@ -1196,8 +1178,7 @@ fn the_emitted_c_names_no_pointer_type_it_does_not_mean() { if !supports(&toolchain, (3, 12)) { return; } - let dir = e2e_root().join("by_e2e_pointers"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_e2e_pointers"); let built = by_build::emit_source( POINTER_SOURCE, "by_e2e_pointers", @@ -1253,8 +1234,7 @@ fn a_package_is_built_as_a_tree_and_imports_under_its_dotted_names() { if !supports(&toolchain, (3, 12)) { return; } - let dir = e2e_root().join("by_e2e_package_tree"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_e2e_package_tree"); let options = Options { language: by_irbuild::Language::Python, @@ -1487,8 +1467,7 @@ fn running_the_interpreter_with_o_still_means_o_for_the_twin() { let Some((python, toolchain)) = environment() else { return; }; - let dir = e2e_root().join("by_e2e_twinoptrun"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_e2e_twinoptrun"); let source = "\ try: assert False, \"still here\" @@ -1542,8 +1521,7 @@ fn running_the_interpreter_with_oo_takes_the_docstrings_out_of_the_compiled_defi let Some((python, toolchain)) = environment() else { return; }; - let dir = e2e_root().join("by_e2e_docoptrun"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_e2e_docoptrun"); let source = "\ def plain(): \"the plain one\" @@ -1631,8 +1609,7 @@ fn the_running_version_is_read_off_the_banner() { let Some((python, toolchain)) = environment() else { return; }; - let dir = e2e_root().join("by_e2e_version"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_e2e_version"); std::fs::create_dir_all(&dir).expect("the build directory is made"); std::fs::write(dir.join(by_rt::BY_H_NAME), by_rt::BY_H).expect("the header is written"); diff --git a/crates/by_codegen_c/src/lib.rs b/crates/by_codegen_c/src/lib.rs index 174b891c52..4ec03219b1 100644 --- a/crates/by_codegen_c/src/lib.rs +++ b/crates/by_codegen_c/src/lib.rs @@ -31,8 +31,8 @@ use by_ir::function::{ SlotAlias, Surface, cleaned_doc, }; use by_ir::ops::{ - BinOp, BlockId, CmpOp, Concatenation, LicenceKind, Mutation, Op, RegisterId, Terminator, - UnaryOp, Value, + BinOp, BlockId, CmpOp, Concatenation, LicenceKind, Mutation, Op, RegisterId, SoundTarget, + Terminator, UnaryOp, Value, }; use by_ir::rtype::{Primitive, RType, tuple_mangle}; @@ -386,13 +386,29 @@ pub fn emit_module(module: &ModuleIr) -> String { ); } } + // a nested function's spec names the interpreted module, defined further down + if module + .classes + .iter() + .any(|class| class.methods.iter().any(|method| method.nested.is_some())) + { + out.push_str("static const By_Fallback by_fallback;\n"); + } for class in &module.classes { for method in &class.methods { if let Some(nested) = &method.nested { let wrapper = method.wrapper_symbol(module.name.dotted()); + let annotations = match &nested.annotations { + Some(annotations) => { + emit_annotation_values(&mut out, module, class, method, annotations) + } + None => "NULL, 0, NULL, NULL".to_string(), + }; + let defaults = emit_function_defaults(&mut out, module, method); + let _ = writeln!(out, "static PyObject *{wrapper}_code = NULL;"); let _ = writeln!( out, - "static const ByFunctionSpec {wrapper}_spec = {{ {wrapper}, {}, {}, {}, &by_module_dict }};", + "static const ByFunctionSpec {wrapper}_spec = {{ {wrapper}, {}, {}, {}, &by_module_dict, {annotations}, {defaults}, &by_fallback, &{wrapper}_code }};", c_string(&nested.name), c_string(&nested.qualname), method_doc(method) @@ -1000,14 +1016,71 @@ fn absent_values_owner<'a>(module: &'a ModuleIr, class: &'a ClassIr) -> Option<& None } -/// the `tp_new` of a class whose layout starts with fields absent by value +/// whether a class with no `__new__` of its own publishes one, so that an interpreted +/// subclass is handed the allocator of the layout it extends +/// +/// the allocation writes the values that mark a field absent, but only the allocation this +/// module emits. python gives a class made by a `class` statement or `type(...)` the generic +/// allocator rather than inheriting its base's, so an interpreted subclass's instance +/// arrived zeroed — and a zeroed `int` field is the `int` zero. such an instance comes +/// through the `__new__` it inherits, so that is where its type is handed the allocator — +/// see `By_AdoptAllocator`. +/// +/// it is not a `tp_new` filled from the spec, for the reason [`publishes_new`] gives: python +/// reads a C function there as a base that owns the allocation, and refuses +/// `object.__new__(cls)` on the class outright. see `By_PublishAllocatingNew` +fn publishes_absent_values_new(module: &ModuleIr, class: &ClassIr) -> bool { + heap_type(module, class) + && !external_storage(module, class) + && !inherits_layout(module, class) + && initializes(module, class) + && !constructs_through_a_written_new(module, class) + && absent_values_owner(module, class).is_some() +} + +/// whether a class publishes an `__init_subclass__` that hands every interpreted subclass +/// the allocator of the layout it extends, the moment the subclass is made +/// +/// the `__new__` a class publishes does that for each instance made through it, but +/// `object.__new__(cls)` on a subclass reaches the subclass's own allocator without passing +/// that `__new__` — see `By_PublishInitSubclass`. only the class whose allocation writes the +/// absent values publishes one: a class that inherits that layout reaches its base's through +/// the mro, which is the allocator it needs +fn publishes_init_subclass(module: &ModuleIr, class: &ClassIr) -> bool { + heap_type(module, class) && writes_absent_values(module, class) +} + +/// the `__init_subclass__` [`publishes_init_subclass`] describes, and its method definition +fn emit_init_subclass(module: &ModuleIr, class: &ClassIr) -> String { + let type_name = class.type_name(module.name.dotted()); + // a class that recycles its instances defines its allocator further on + format!( + "static PyObject *{type_name}_alloc(PyTypeObject *by_type, Py_ssize_t by_items);\n\ + static PyObject *{type_name}_init_subclass(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) {{\n\ + \x20 (void)self;\n\ + \x20 return By_InitSubclass({type_name}_OBJ, {type_name}_alloc, args, nargs, kwnames);\n}}\n\n\ + static PyMethodDef {type_name}_init_subclass_def =\n\ + \x20 {{\"__init_subclass__\", (PyCFunction)(void(*)(void)){type_name}_init_subclass, METH_FASTCALL | METH_KEYWORDS, NULL}};\n\n" + ) +} + +/// readying a class built as a static struct /// -/// the allocation writes those values, but only the allocation this module emits. python -/// gives a class made by a `class` statement or `type(...)` the generic allocator rather -/// than inheriting its base's, so an interpreted subclass's instance arrived zeroed — and a -/// zeroed `int` field is the `int` zero. such an instance comes through the `tp_new` it -/// inherits, so this is where its type is handed the allocator of the layout it extends — -/// see `By_AdoptAllocator` +/// one a python name reaches is constructed, and it takes `object`'s `tp_new` as a class +/// statement's class does, so that `object.__new__(cls)` allocates it. a static struct +/// cannot name that function, and one left without a `tp_new` on `object` is made +/// uninstantiable by `PyType_Ready`, so it is written in first. nothing subclasses a static +/// struct, so no allocator has to be handed on — see [`publishes_absent_values_new`] +fn static_type_ready(module: &ModuleIr, class: &ClassIr, type_name: &str) -> String { + let new = if made_only_by_its_frame(module, class) { + String::new() + } else { + format!(" {type_name}.tp_new = PyBaseObject_Type.tp_new;\n") + }; + format!("{new} if (PyType_Ready(&{type_name}) < 0) return -1;") +} + +/// the `__new__` [`publishes_absent_values_new`] describes, and its method definition fn emit_absent_values_new(module: &ModuleIr, class: &ClassIr) -> String { let Some(owner) = absent_values_owner(module, class) else { return String::new(); @@ -1016,9 +1089,14 @@ fn emit_absent_values_new(module: &ModuleIr, class: &ClassIr) -> String { let owner_name = owner.type_name(module.name.dotted()); format!( "static PyObject *{owner_name}_alloc(PyTypeObject *by_type, Py_ssize_t by_items);\n\ - static PyObject *{type_name}_new(PyTypeObject *by_type, PyObject *by_args, PyObject *by_kwds) {{\n\ + static PyObject *{type_name}_new(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) {{\n\ + \x20 (void)self; (void)kwnames;\n\ + \x20 PyTypeObject *by_type = By_NewTarget({type_name}_OBJ, args, PyVectorcall_NARGS(nargs));\n\ + \x20 if (by_type == NULL) return NULL;\n\ \x20 By_AdoptAllocator(by_type, {type_name}_OBJ, {owner_name}_alloc);\n\ - \x20 return PyType_GenericNew(by_type, by_args, by_kwds);\n}}\n\n" + \x20 return PyType_GenericNew(by_type, NULL, NULL);\n}}\n\n\ + static PyMethodDef {type_name}_new_def =\n\ + \x20 {{\"__new__\", (PyCFunction)(void(*)(void)){type_name}_new, METH_FASTCALL | METH_KEYWORDS, NULL}};\n\n" ) } @@ -1272,6 +1350,9 @@ fn emit_class_type(module: &ModuleIr, class: &ClassIr) -> String { let keeps_a_dict = instance_dict(module, class); out.push_str(&emit_absent_values(module, class)); + if publishes_init_subclass(module, class) { + out.push_str(&emit_init_subclass(module, class)); + } if frees_its_instances(module, class) { out.push_str(&emit_appended_storage(module, class)); } else { @@ -2205,7 +2286,15 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { // out and writes it into `tp_dictoffset`, which is the only way a type built from a // spec can say where its instances keep a dict // `__weaklistoffset__` is lifted out the same way, into `tp_weaklistoffset` - if instance_dict(module, class) { + if publishes_offsets(module, class) { + let dict = if instance_dict(module, class) { + format!( + "\x20 {{\"__dictoffset__\", BY_DICT_OFFSET_MEMBER,\n\ + \x20 offsetof({struct_name}, {BY_DICT_MEMBER}), BY_DICT_OFFSET_FLAGS}},\n" + ) + } else { + String::new() + }; let weak_list = if module.keeps_weak_references(class) { format!( "\x20 {{\"__weaklistoffset__\", BY_DICT_OFFSET_MEMBER,\n\ @@ -2217,8 +2306,7 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { let _ = write!( out, "static PyMemberDef {type_name}_members[] = {{\n\ - \x20 {{\"__dictoffset__\", BY_DICT_OFFSET_MEMBER,\n\ - \x20 offsetof({struct_name}, {BY_DICT_MEMBER}), BY_DICT_OFFSET_FLAGS}},\n\ + {dict}\ {weak_list}\ \x20 {{NULL, 0, 0, 0}}\n}};\n\n" ); @@ -2920,9 +3008,8 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { } else { ( "", - format!( - " .tp_init = {type_name}_init,\n .tp_new = PyType_GenericNew,\n" - ), + // `tp_new` is `object`'s, set at module init — see [`static_type_ready`] + format!(" .tp_init = {type_name}_init,\n"), ) }; let weakrefs = if class.resume.is_some() || module.keeps_weak_references(class) { @@ -3052,14 +3139,15 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { // is what installs python's own dispatcher — see [`publishes_new`]. the pair is // then what the *source* wrote, and `object.__init__` lifts its refusal for the // same reason it does there: the class overrode the allocator - let construction = if init.is_empty() || constructs_through_a_written_new(module, class) { - init - } else if absent_values_owner(module, class).is_some() { + // + // a class that initializes leaves `tp_new` to be inherited from `object`, as a class + // statement does, so `object.__new__(cls)` allocates it. where its fields start + // absent by value it publishes a `__new__` as well, for its subclasses — see + // [`publishes_absent_values_new`] + if publishes_absent_values_new(module, class) { out.push_str(&emit_absent_values_new(module, class)); - format!("{init}\x20 {{Py_tp_new, (void *){type_name}_new}},\n") - } else { - format!("{init}\x20 {{Py_tp_new, (void *)PyType_GenericNew}},\n") - }; + } + let construction = init; // a collected type has to hand the collector both halves, or an instance in a // cycle is never reached at all. a class without a dict has nothing extra to // reach, so it is not a collected type and has no pair to hand over. the members @@ -3071,6 +3159,8 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { \x20 {{Py_tp_clear, (void *){type_name}_clear}},\n\ \x20 {{Py_tp_members, (void *){type_name}_members}},\n" ) + } else if publishes_offsets(module, class) { + format!("\x20 {{Py_tp_members, (void *){type_name}_members}},\n") } else { String::new() }; @@ -3178,6 +3268,18 @@ fn instance_dict(module: &ModuleIr, class: &ClassIr) -> bool { && (decorated_chain(module, class) || !slots_declared_throughout(module, class)) } +/// whether this class's spec tells the type where an instance keeps its dict or its +/// weak-reference list, which `PyType_Spec` can only say through the members table +/// +/// a class `__slots__` refuses a dict can still ask for the list, by naming `__weakref__` +fn publishes_offsets(module: &ModuleIr, class: &ClassIr) -> bool { + instance_dict(module, class) + || (class.exported + && heap_type(module, class) + && !inherits_layout(module, class) + && module.keeps_weak_references(class)) +} + /// whether this class answers `__dict__` with a view over its layout as well as its dict /// /// a class keeping a dict and no fields has nothing outside that dict, so python's own @@ -6825,6 +6927,167 @@ fn synthetic_table_entries(class: &by_ir::function::ClassIr, type_name: &str) -> out } +/// what a nested function's spec says about its annotations, written after its first +/// fields, and the function listing the values the annotations read from the environment +/// +/// the listing is the reader's native body, called on the environment the function holds, +/// which is the environment the reader is a method of too +fn emit_annotation_values( + out: &mut String, + module: &ModuleIr, + class: &by_ir::function::ClassIr, + method: &Function, + annotations: &by_ir::function::NestedAnnotations, +) -> String { + let wrapper = method.wrapper_symbol(module.name.dotted()); + let _ = writeln!(out, "static PyObject *{wrapper}_annotation_factory = NULL;"); + let reader = annotations.reader.as_ref().and_then(|name| { + class + .methods + .iter() + .find(|candidate| candidate.name == *name) + }); + let values = match reader { + Some(reader) + if matches!(reader.ret, RType::LIST | RType::OBJECT) && reader.param_count == 1 => + { + let receiver = ctype(module, &reader.params()[0].ty); + let call = if is_recursive(reader) { + format!( + "{}(By_DepthHere(), ({receiver})env)", + depth_symbol(module, reader) + ) + } else { + format!( + "{}(({receiver})env)", + reader.native_symbol(module.name.dotted()) + ) + }; + let _ = writeln!( + out, + "static PyObject *{wrapper}_annotation_values(PyObject *env) {{ return {call}; }}" + ); + format!("{wrapper}_annotation_values") + } + _ => "NULL".to_string(), + }; + // a factory that is missing is one the build never wrote, which the runtime refuses in + // the same words as one the twin could not supply + let source = annotations.factory.clone().unwrap_or_else(|| { + "def _by_annotations(values, type_params):\n raise RuntimeError('the annotations of this compiled function were not written')\n".to_string() + }); + let refused = annotations.refused || annotations.factory.is_none(); + format!( + "{}, {}, {values}, &{wrapper}_annotation_factory", + c_string(&source), + i32::from(refused) + ) +} + +/// the function a nested function's spec hands `__defaults__` and `__kwdefaults__` out +/// through, or `NULL` where the definition has no defaults +/// +/// each value is the one the boundary fills a missing argument with: an immediate as it +/// is written, and a default the frame evaluated where the `def` stood out of the +/// environment it parked it in +fn emit_function_defaults(out: &mut String, module: &ModuleIr, method: &Function) -> String { + let params = method.params(); + // the receiver is the environment, which python never sees + let written = params.len().saturating_sub(1); + let packed = usize::from(method.vararg) + usize::from(method.kwarg); + let named = written.saturating_sub(packed); + let positional_end = 1 + named.saturating_sub(method.kwonly); + let value = |index: usize| -> Option { + let decl = params.get(index)?; + let computed = method.defaults_held_by == by_ir::function::DefaultsHeldBy::Receiver + && method.computed_defaults.contains(&index); + if computed { + let field = method.receiver_default_field(index)?; + let receiver = ctype(module, ¶ms.first()?.ty); + return Some(format!( + "By_ReadCell((({receiver})env)->{}, {}, 0)", + mangle_member(&field), + c_string(decl.name.as_deref().unwrap_or("")) + )); + } + method + .defaults + .get(index) + .and_then(Option::as_ref) + .map(|default| default_expr(&RType::OBJECT, default)) + }; + let positional: Vec = (1..positional_end) + .skip_while(|index| value(*index).is_none()) + // python has no positional parameter without a default after one with a default + .map(|index| { + value(index).unwrap_or_else(|| { + "(PyErr_SetString(PyExc_SystemError, \"a positional default is missing\"), NULL)" + .to_string() + }) + }) + .collect(); + let keyword: Vec<(String, String)> = (positional_end..=named) + .filter_map(|index| { + let name = params.get(index)?.name.clone()?; + Some((name, value(index)?)) + }) + .collect(); + if positional.is_empty() && keyword.is_empty() { + return "NULL".to_string(); + } + let symbol = format!("{}_defaults", method.wrapper_symbol(module.name.dotted())); + let _ = writeln!(out, "static PyObject *{symbol}(PyObject *env) {{"); + let _ = writeln!(out, " (void)env;"); + let _ = writeln!( + out, + " PyObject *by_positional = {};", + if positional.is_empty() { + "By_NewRef(Py_None)".to_string() + } else { + format!("PyTuple_New({})", positional.len()) + } + ); + let _ = writeln!( + out, + " PyObject *by_keyword = {};", + if keyword.is_empty() { + "By_NewRef(Py_None)" + } else { + "PyDict_New()" + } + ); + out.push_str(" if (by_positional == NULL || by_keyword == NULL) goto by_failed;\n"); + for (at, expr) in positional.iter().enumerate() { + let _ = writeln!( + out, + " {{ PyObject *by_value = {expr};\n\ + \x20 if (by_value == NULL) goto by_failed;\n\ + \x20 PyTuple_SET_ITEM(by_positional, {at}, by_value); }}" + ); + } + for (name, expr) in &keyword { + let _ = writeln!( + out, + " {{ PyObject *by_value = {expr};\n\ + \x20 int by_set = by_value == NULL ? -1 : PyDict_SetItemString(by_keyword, {}, by_value);\n\ + \x20 Py_XDECREF(by_value);\n\ + \x20 if (by_set < 0) goto by_failed; }}", + c_string(name) + ); + } + out.push_str( + " PyObject *by_both = PyTuple_Pack(2, by_positional, by_keyword);\n\ + \x20 Py_DECREF(by_positional);\n\ + \x20 Py_DECREF(by_keyword);\n\ + \x20 return by_both;\n\ + by_failed:\n\ + \x20 Py_XDECREF(by_positional);\n\ + \x20 Py_XDECREF(by_keyword);\n\ + \x20 return NULL;\n}\n", + ); + symbol +} + fn c_string(text: &str) -> String { c_byte_string(text.as_bytes()) } @@ -7206,6 +7469,7 @@ fn emit_op(module: &ModuleIr, function: &Function, op: &Op, error_target: ErrorE let expr = format!("{call}({}, {})", value_expr(lhs), value_expr(rhs)); assign_checked(module, function, *dest, &expr, error_target) } + Op::CheckSound { src, target } => emit_check_sound(src, target, error_target), Op::IsInstance { dest, src, class } => { let expr = format!("By_IsInstance({}, {})", value_expr(src), value_expr(class)); assign_checked(module, function, *dest, &expr, error_target) @@ -7257,6 +7521,20 @@ fn emit_op(module: &ModuleIr, function: &Function, op: &Op, error_target: ErrorE local(*dest), value_expr(src) ), + Op::HoldsLayout { dest, src, class } => match module + .classes + .iter() + .find(|candidate| candidate.name == *class) + { + Some(owner) => format!( + " {} = PyObject_TypeCheck({}, (PyTypeObject *){}_OBJ);\n", + local(*dest), + value_expr(src), + owner.type_name(module.name.dotted()) + ), + // a class with no emitted layout has no offsets to read at + None => format!(" {} = 0;\n", local(*dest)), + }, Op::MatchAttr { dest, subject, @@ -8134,7 +8412,7 @@ fn emit_op(module: &ModuleIr, function: &Function, op: &Op, error_target: ErrorE let _ = writeln!( out, " {{ Py_ssize_t by_i = {}((ByArrayHeader *){}, {});", - array_index_helper(function, index), + array_index_helper(function, index, false), value_expr(array), value_expr(index) ); @@ -8162,7 +8440,7 @@ fn emit_op(module: &ModuleIr, function: &Function, op: &Op, error_target: ErrorE let _ = writeln!( out, " {{ Py_ssize_t by_i = {}((ByArrayHeader *){}, {});", - array_index_helper(function, index), + array_index_helper(function, index, true), value_expr(array), value_expr(index) ); @@ -8594,11 +8872,20 @@ fn emit_op(module: &ModuleIr, function: &Function, op: &Op, error_target: ErrorE else { return String::new(); }; - let call = format!( + let made = format!( "By_MakeFunction(&{}_spec, (PyObject *)({}))", nested.wrapper_symbol(module.name.dotted()), value_expr(env) ); + let call = if nested + .nested + .as_ref() + .is_some_and(|name| name.annotations.is_some()) + { + format!("By_AnnotateAtDefinition({made})") + } else { + made + }; assign_checked(module, function, *dest, &call, error_target) } Op::LoadGlobal { dest, name } => { @@ -9706,6 +9993,67 @@ impl ErrorEdge { } } +/// [`Op::CheckSound`]: each class tried in the order `isinstance` tries a tuple's, the +/// first answer that is not no deciding, and the refusal built only once the answer is no +fn emit_check_sound(src: &Value, target: &SoundTarget, error_target: ErrorEdge) -> String { + /// each class in order, as a C expression for the class object + fn leaves(target: &SoundTarget, out: &mut Vec) { + match target { + SoundTarget::Class(class) => out.push(value_expr(class)), + SoundTarget::NoneType => out.push("(PyObject *)Py_TYPE(Py_None)".to_string()), + SoundTarget::AnyOf(parts) => { + for part in parts { + leaves(part, out); + } + } + } + } + /// the `Py_BuildValue` format that rebuilds the tuple the target is spelled as + fn shape(target: &SoundTarget, out: &mut String) { + match target { + SoundTarget::Class(_) | SoundTarget::NoneType => out.push('O'), + SoundTarget::AnyOf(parts) => { + out.push('('); + for part in parts { + shape(part, out); + } + out.push(')'); + } + } + } + let object = value_expr(src); + let mut classes = Vec::new(); + leaves(target, &mut classes); + let mut out = String::from( + " { int by_s = 0; +", + ); + for (index, class) in classes.iter().enumerate() { + let guard = if index == 0 { "" } else { "if (by_s == 0) " }; + let _ = writeln!(out, " {guard}by_s = By_SoundIs({object}, {class});"); + } + let refusal = match target { + SoundTarget::AnyOf(_) => { + let mut format = String::new(); + shape(target, &mut format); + format!( + "By_SoundViolationOf({object}, Py_BuildValue({}, {}))", + c_string(&format), + classes.join(", ") + ) + } + SoundTarget::Class(_) | SoundTarget::NoneType => { + format!("By_SoundViolation({object}, {})", classes.join(", ")) + } + }; + let _ = writeln!( + out, + " if (BY_UNLIKELY(by_s != 1)) {{ if (by_s == 0) {refusal}; goto {}; }} }}", + error_label(error_target) + ); + out +} + fn error_label(error_target: ErrorEdge) -> String { match error_target { ErrorEdge { @@ -9983,11 +10331,14 @@ fn borrowed_element_unbox( } /// the helper that normalizes and bounds-checks an index into a packed buffer, for the -/// representation the index is in -fn array_index_helper(function: &Function, index: &Value) -> &'static str { - match function.value_type(index) { - Some(RType::Primitive(Primitive::Fixed(_))) => "By_ArrayIndexI64", - _ => "By_ArrayIndex", +/// representation the index is in and for whether the access is a store, which `list` +/// refuses in words of its own +fn array_index_helper(function: &Function, index: &Value, store: bool) -> &'static str { + match (function.value_type(index), store) { + (Some(RType::Primitive(Primitive::Fixed(_))), false) => "By_ArrayIndexI64", + (Some(RType::Primitive(Primitive::Fixed(_))), true) => "By_ArrayStoreIndexI64", + (_, false) => "By_ArrayIndex", + (_, true) => "By_ArrayStoreIndex", } } @@ -11213,7 +11564,7 @@ fn emit_module_init(module: &ModuleIr) -> String { // already built, by the one construction open to it String::new() } else if !heap_type(module, class) { - format!(" if (PyType_Ready(&{type_name}) < 0) return -1;") + static_type_ready(module, class, &type_name) } else if let Some(base) = class .base @@ -11303,6 +11654,16 @@ fn emit_module_init(module: &ModuleIr) -> String { if !ready.is_empty() { let _ = writeln!(installed, "{ready}"); } + // a method the class wrote binds as a function does, before anything below can + // read the type's dict or hand the class to a decorator + let written = class.table_methods().count(); + if written > 0 { + let _ = writeln!( + installed, + " if (By_PublishMethods({type_name}_OBJ, &{type_name}_methods[{}], {written}) < 0) return -1;", + synthetic_table_entries(class, &type_name).len() + ); + } // the wrappers python published for the *rest* of a shared slot's group come off // first, so everything below — a decorator above all — is handed a class whose // surface is the body the `class` statement wrote @@ -11344,6 +11705,18 @@ fn emit_module_init(module: &ModuleIr) -> String { // before anything else asks the type to build an instance: the assignment is what // gives the class its `tp_new`, and until it has run the type still allocates the // way `object` does + if publishes_absent_values_new(module, class) { + let _ = writeln!( + installed, + " if (By_PublishAllocatingNew({type_name}_OBJ, &{type_name}_new_def) < 0) return -1;" + ); + } + if publishes_init_subclass(module, class) { + let _ = writeln!( + installed, + " if (By_PublishInitSubclass({type_name}_OBJ, &{type_name}_init_subclass_def) < 0) return -1;" + ); + } if publishes_new(class).is_some() { let _ = writeln!( installed, @@ -11999,6 +12372,7 @@ mod tests { dataclass: false, generic: false, declares_slots: false, + slots_weak_references: false, constants: Vec::new(), properties: Vec::new(), slot_aliases: Vec::new(), @@ -12265,6 +12639,7 @@ mod tests { dataclass: false, generic: false, declares_slots: false, + slots_weak_references: false, constants: Vec::new(), properties: Vec::new(), slot_aliases: Vec::new(), @@ -13143,6 +13518,7 @@ mod tests { dataclass: false, generic: false, declares_slots: false, + slots_weak_references: false, constants: Vec::new(), properties: Vec::new(), slot_aliases: Vec::new(), @@ -13782,6 +14158,7 @@ mod tests { dataclass: false, generic: false, declares_slots: false, + slots_weak_references: false, constants: Vec::new(), properties: Vec::new(), slot_aliases: Vec::new(), @@ -14582,6 +14959,7 @@ mod tests { dataclass: false, generic: false, declares_slots: false, + slots_weak_references: false, constants: Vec::new(), properties: Vec::new(), slot_aliases: Vec::new(), @@ -14674,6 +15052,7 @@ mod tests { dataclass: false, generic: false, declares_slots: false, + slots_weak_references: false, constants: Vec::new(), properties: Vec::new(), slot_aliases: Vec::new(), @@ -14751,6 +15130,7 @@ mod tests { dataclass: false, generic: false, declares_slots: false, + slots_weak_references: false, constants: Vec::new(), properties: Vec::new(), slot_aliases: Vec::new(), @@ -14875,6 +15255,7 @@ mod tests { nested.nested = Some(by_ir::function::NestedName { name: "call".to_string(), qualname: "make..call".to_string(), + annotations: None, }); let mut class = appending_class(); class.name = "Env".to_string(); @@ -14902,7 +15283,7 @@ mod tests { ); assert!( c.contains( - "static const ByFunctionSpec byw_app_Env_call_spec = { byw_app_Env_call, \"call\", \"make..call\", NULL, &by_module_dict };" + "static const ByFunctionSpec byw_app_Env_call_spec = { byw_app_Env_call, \"call\", \"make..call\", NULL, &by_module_dict, NULL, 0, NULL, NULL, NULL, &by_fallback, &byw_app_Env_call_code };" ), "{c}" ); @@ -15107,6 +15488,7 @@ mod tests { dataclass: false, generic: false, declares_slots: false, + slots_weak_references: false, constants: Vec::new(), properties: Vec::new(), slot_aliases: Vec::new(), diff --git a/crates/by_ir/src/function.rs b/crates/by_ir/src/function.rs index d271ebf67c..b4459b01ce 100644 --- a/crates/by_ir/src/function.rs +++ b/crates/by_ir/src/function.rs @@ -398,6 +398,31 @@ pub struct NestedName { pub name: String, /// `__qualname__`: `counter..step` pub qualname: String, + /// where the function's `__annotations__` come from, where it wrote any + pub annotations: Option, +} + +/// how a nested function answers `__annotations__`, and `__annotate__` from 3.14 +/// +/// the annotations python evaluates are the *interpreted* definition's, which for a `.by` +/// module is the transpiler's output: the source's own annotations may be written in syntax +/// that does not run as python at all. so they are evaluated by python, from that text, +/// over the values the enclosing frames hold for the names in them +#[derive(Debug, Clone, PartialEq)] +pub struct NestedAnnotations { + /// the enclosing names the annotations read, in the order the reader lists their values + pub names: Vec, + /// the method of the same environment that lists the values of `names`, where there are + /// any + pub reader: Option, + /// python source defining the function that evaluates the annotations, filled in once + /// the interpreted twin exists — see `by_irbuild::annotations` + pub factory: Option, + /// whether the factory only raises, because the twin could not say what the annotations + /// are. such a function raises when its annotations are asked for, and never where its + /// `def` stands, so a build that cannot answer the question breaks nothing that does not + /// ask it + pub refused: bool, } impl Function { @@ -838,6 +863,9 @@ pub struct ClassIr { /// the layout an emitted class has anyway, and one that does not is asking for a /// name it never mentioned to still have somewhere to go pub declares_slots: bool, + /// whether that `__slots__` names `__weakref__`, which asks for the weak-reference list + /// a class without `__slots__` is given anyway + pub slots_weak_references: bool, /// whether the class declares type parameters. /// /// they are erased in the *layout* — every `T` field is an object, whatever @@ -1308,7 +1336,7 @@ impl ModuleIr { if current.dataclass { return false; } - unslotted |= !current.declares_slots; + unslotted |= !current.declares_slots || current.slots_weak_references; match ¤t.base { None => return unslotted, Some(ClassBase::External(_)) => return false, diff --git a/crates/by_ir/src/ops.rs b/crates/by_ir/src/ops.rs index da4a81cf3b..9ea6d9799c 100644 --- a/crates/by_ir/src/ops.rs +++ b/crates/by_ir/src/ops.rs @@ -395,6 +395,11 @@ pub enum Op { src: Value, class: Value, }, + /// the soundness check the transpiled build makes where a value whose type + /// nothing verified meets a declared one: python's `isinstance(src, target)`, + /// raising `TypeError: type soundness violation: expected …, got …` when the + /// answer is no. `src` is handed on untouched either way + CheckSound { src: Value, target: SoundTarget }, /// the attribute a class pattern names, or the *missing* answer /// /// a missing attribute is no match rather than an error, so this has a third @@ -627,6 +632,16 @@ pub enum Op { }, /// whether `src` has the shape a mapping pattern matches IsMapping { dest: RegisterId, src: Value }, + /// whether `src` is laid out as `class` is, which a read at one of its offsets needs + /// + /// the type itself answers, never `__class__`: an object may claim a class it was not + /// made by, and `isinstance` believes it. so where python's own test has already said + /// yes, this is what says which way the attributes can be read + HoldsLayout { + dest: RegisterId, + src: Value, + class: String, + }, /// whether `src` has the shape a sequence pattern matches /// /// the interpreter's own test: a type flagged as a sequence, which `str`, @@ -1404,6 +1419,7 @@ impl Op { | Self::GetField { class, .. } | Self::RequireField { class, .. } | Self::FieldIsSet { class, .. } + | Self::HoldsLayout { class, .. } | Self::SetField { class, .. } => vec![class.as_str()], // a method reached directly rather than through the type, which only a // class the emitter laid out has @@ -1425,6 +1441,7 @@ impl Op { | Self::Box { .. } | Self::FloatObjectBinary { .. } | Self::IsInstance { .. } + | Self::CheckSound { .. } | Self::MatchAttr { .. } | Self::MethodStands { .. } | Self::BuiltinStands { .. } @@ -1570,6 +1587,7 @@ impl Op { | Self::MatchKey { dest, .. } | Self::MatchRest { dest, .. } | Self::IsMapping { dest, .. } + | Self::HoldsLayout { dest, .. } | Self::MatchAttr { dest, .. } | Self::MethodStands { dest, .. } | Self::BuiltinStands { dest, .. } @@ -1678,6 +1696,7 @@ impl Op { | Self::LeaveGenerator { .. } | Self::LicenceHolds { .. } | Self::RequireField { .. } + | Self::CheckSound { .. } | Self::Line { .. } | Self::SetField { .. } => None, } @@ -1699,6 +1718,7 @@ impl Op { | Self::MatchKey { dest, .. } | Self::MatchRest { dest, .. } | Self::IsMapping { dest, .. } + | Self::HoldsLayout { dest, .. } | Self::MatchAttr { dest, .. } | Self::MethodStands { dest, .. } | Self::BuiltinStands { dest, .. } @@ -1807,6 +1827,7 @@ impl Op { | Self::LeaveGenerator { .. } | Self::LicenceHolds { .. } | Self::RequireField { .. } + | Self::CheckSound { .. } | Self::Line { .. } | Self::SetField { .. } => None, } @@ -1816,6 +1837,9 @@ impl Op { pub fn operands(&self) -> Vec<&Value> { match self { Self::LoopGuardsHold { exact, .. } => exact.iter().collect(), + Self::CheckSound { src, target } => { + std::iter::once(src).chain(target.classes()).collect() + } Self::AsyncContext { manager, exit: Some(exit), @@ -1834,6 +1858,7 @@ impl Op { | Self::DictShadows { src, .. } | Self::IsMissing { src, .. } | Self::IsMapping { src, .. } + | Self::HoldsLayout { src, .. } | Self::AsyncIter { src, .. } | Self::AsyncContext { manager: src, @@ -2061,6 +2086,9 @@ impl Op { pub fn operands_mut(&mut self) -> Vec<&mut Value> { match self { Self::LoopGuardsHold { exact, .. } => exact.iter_mut().collect(), + Self::CheckSound { src, target } => { + std::iter::once(src).chain(target.classes_mut()).collect() + } Self::AsyncContext { manager, exit: Some(exit), @@ -2079,6 +2107,7 @@ impl Op { | Self::DictShadows { src, .. } | Self::IsMissing { src, .. } | Self::IsMapping { src, .. } + | Self::HoldsLayout { src, .. } | Self::AsyncIter { src, .. } | Self::AsyncContext { manager: src, @@ -2301,6 +2330,38 @@ impl Op { } } +/// what an [`Op::CheckSound`] tests its value against: the second argument of the +/// `isinstance` the transpiled build makes +#[derive(Debug, Clone, PartialEq)] +pub enum SoundTarget { + /// a class, already resolved the way the transpiled build's name for it resolves + Class(Value), + /// `type(None)` + NoneType, + /// any one of several, tried in order, as `isinstance` tries a tuple's + AnyOf(Vec), +} + +impl SoundTarget { + /// every class this target reads, in the order `isinstance` tries them + pub fn classes(&self) -> Vec<&Value> { + match self { + Self::Class(class) => vec![class], + Self::NoneType => Vec::new(), + Self::AnyOf(parts) => parts.iter().flat_map(Self::classes).collect(), + } + } + + /// [`Self::classes`], mutably + pub fn classes_mut(&mut self) -> Vec<&mut Value> { + match self { + Self::Class(class) => vec![class], + Self::NoneType => Vec::new(), + Self::AnyOf(parts) => parts.iter_mut().flat_map(Self::classes_mut).collect(), + } + } +} + /// where lowered code was written — see [`Op::Line`] #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct Position { diff --git a/crates/by_ir/src/print.rs b/crates/by_ir/src/print.rs index c92f18e4da..fd21c69ac8 100644 --- a/crates/by_ir/src/print.rs +++ b/crates/by_ir/src/print.rs @@ -8,7 +8,7 @@ use std::fmt::Write; use crate::function::{Function, ModuleIr}; use crate::ops::{ - Concatenation, LicenceKind, Mutation, Op, RegisterId, Terminator, UnaryOp, Value, + Concatenation, LicenceKind, Mutation, Op, RegisterId, SoundTarget, Terminator, UnaryOp, Value, }; /// render a whole module @@ -98,12 +98,35 @@ fn print_value(function: &Function, value: &Value) -> String { } } +/// a soundness check's target, spelled as the tuple `isinstance` is handed +fn sound_target(function: &Function, target: &SoundTarget) -> String { + match target { + SoundTarget::Class(class) => print_value(function, class), + SoundTarget::NoneType => "NoneType".to_string(), + SoundTarget::AnyOf(parts) => format!( + "({})", + parts + .iter() + .map(|part| sound_target(function, part)) + .collect::>() + .join(", ") + ), + } +} + fn print_op(function: &Function, op: &Op) -> String { let value = |v: &Value| print_value(function, v); let name = |id: RegisterId| register_name(function, id); match op { Op::Assign { dest, src } => format!("{} = {}", name(*dest), value(src)), Op::Truthy { dest, src } => format!("{} = truthy {}", name(*dest), value(src)), + Op::CheckSound { src, target } => { + format!( + "check sound {} {}", + value(src), + sound_target(function, target) + ) + } Op::IsInstance { dest, src, class } => format!( "{} = isinstance {} {}", name(*dest), @@ -140,6 +163,9 @@ fn print_op(function: &Function, op: &Op) -> String { Op::IsMapping { dest, src } => { format!("{} = is-mapping {}", name(*dest), value(src)) } + Op::HoldsLayout { dest, src, class } => { + format!("{} = holds-layout {} <{class}>", name(*dest), value(src)) + } Op::MatchAttr { dest, subject, diff --git a/crates/by_ir/src/verify.rs b/crates/by_ir/src/verify.rs index 4db0376f89..b6199f967a 100644 --- a/crates/by_ir/src/verify.rs +++ b/crates/by_ir/src/verify.rs @@ -722,6 +722,12 @@ impl Verifier<'_> { self.expect(block, rhs, &RType::FLOAT, op.symbol()); self.expect_dest(block, *dest, &RType::FLOAT, op.symbol()); } + Op::CheckSound { src, target } => { + self.expect(block, src, &RType::OBJECT, "a soundness check"); + for class in target.classes() { + self.expect(block, class, &RType::OBJECT, "a soundness check"); + } + } Op::IsInstance { dest, src, class } => { self.expect(block, src, &RType::OBJECT, "an isinstance test"); self.expect(block, class, &RType::OBJECT, "an isinstance test"); @@ -766,6 +772,10 @@ impl Verifier<'_> { self.expect(block, src, &RType::OBJECT, "a mapping-shape test"); self.expect_dest(block, *dest, &RType::BIT, "a mapping-shape test"); } + Op::HoldsLayout { dest, src, .. } => { + self.expect(block, src, &RType::OBJECT, "a layout test"); + self.expect_dest(block, *dest, &RType::BIT, "a layout test"); + } Op::MatchAttr { dest, subject, diff --git a/crates/by_irbuild/Cargo.toml b/crates/by_irbuild/Cargo.toml index 4254e1c62a..f5b1970307 100644 --- a/crates/by_irbuild/Cargo.toml +++ b/crates/by_irbuild/Cargo.toml @@ -12,8 +12,10 @@ license = "MIT OR Apache-2.0" [dependencies] by_ir = { workspace = true } +by_transforms = { workspace = true } ruff_db = { workspace = true } ruff_python_ast = { workspace = true } +ruff_python_codegen = { workspace = true } ruff_python_parser = { workspace = true } ruff_python_stdlib = { workspace = true } ruff_source_file = { workspace = true } diff --git a/crates/by_irbuild/src/annotations.rs b/crates/by_irbuild/src/annotations.rs new file mode 100644 index 0000000000..b9f80a425c --- /dev/null +++ b/crates/by_irbuild/src/annotations.rs @@ -0,0 +1,514 @@ +//! the python that evaluates a nested function's annotations +//! +//! python evaluates a `def`'s annotations over the frames it is written in: on 3.13 where +//! the `def` stands, and from 3.14 when they are first asked for, through an +//! `__annotate__` function that closes over those frames' cells. a compiled nested +//! function has neither the interpreted frames nor, for a `.by` module, annotations that +//! run as python at all — the source may write them in syntax only basedpython has. what +//! does run is the interpreted twin, so the annotations are taken from its text. +//! +//! each annotated nested function is given a *factory*: python source defining +//! +//! ```python +//! def _by_annotations(values, type_params): +//! kind = values[0] +//! def inner(y: kind, z: int, /) -> kind: +//! pass +//! return inner +//! ``` +//! +//! which the runtime compiles against the module's namespace and calls with the values the +//! enclosing names hold — listed by a reader nested beside the function, see +//! `closures::ANNOTATION_READER`. the definition it hands back is python's own, so its +//! `__annotations__` are evaluated as python evaluates them, and on 3.14 its +//! `__annotate__` is a real annotation function over real cells, which is what +//! `annotationlib`'s formats need. the stub keeps the definition's parameters and type +//! parameters and drops everything else: a default and a decorator are evaluated where the +//! `def` stands, and neither is anything the annotations read. +//! +//! a name the factory is given must be exactly a name python would close over, or an +//! annotation reads a global python would not have — so the twin's enclosing frames are +//! read for what they bind, and an annotation that reads one of those the compiled frames +//! do not list is refused. so is a definition the twin does not hold exactly once under its +//! qualified name. a refused factory raises `RuntimeError` when the annotations are asked +//! for, naming why, rather than answering with something python would not + +use std::collections::HashSet; +use std::fmt::Write; + +use by_ir::function::{Function, ModuleIr, NestedAnnotations}; +use ruff_python_ast::visitor::{self, Visitor}; +use ruff_python_ast::{self as ast, Expr, Stmt}; +use ruff_python_codegen::{Generator, Indentation}; +use ruff_source_file::LineEnding; + +/// the name each factory is bound to where the runtime compiles it +pub const FACTORY: &str = "_by_annotations"; + +/// write the factory of every annotated nested function in `module`, from its twin +pub fn write_factories(module: &mut ModuleIr, twin: &str) { + let mut wanting: Vec<&mut Function> = module + .functions + .iter_mut() + .chain( + module + .classes + .iter_mut() + .flat_map(|class| class.methods.iter_mut()), + ) + .filter(|function| { + function + .nested + .as_ref() + .is_some_and(|nested| nested.annotations.is_some()) + }) + .collect(); + if wanting.is_empty() { + return; + } + let parsed = ruff_python_parser::parse_module(twin); + for function in &mut wanting { + let Some(nested) = function.nested.as_mut() else { + continue; + }; + let Some(annotations) = nested.annotations.as_mut() else { + continue; + }; + let factory = match &parsed { + Ok(parsed) => factory(parsed.suite(), &nested.qualname, annotations), + Err(error) => Err(format!( + "the interpreted definition does not parse: {error}" + )), + }; + annotations.refused = factory.is_err(); + annotations.factory = Some(match factory { + Ok((source, reads)) => { + // with nothing read from the enclosing frames — a module compiling its + // annotations as strings, say — there are no values to keep for them + if !reads { + annotations.reader = None; + } + source + } + Err(reason) => refusal(&nested.qualname, &reason), + }); + } +} + +/// the factory for the definition `qualname` names and whether it reads any of the values +/// it is handed, or why there cannot be one +fn factory( + module: &[Stmt], + qualname: &str, + annotations: &NestedAnnotations, +) -> Result<(String, bool), String> { + let (def, enclosing) = locate(module, qualname)?; + let bound: HashSet<&str> = enclosing + .iter() + .filter_map(|scope| match scope { + Scope::Function(def) => Some(*def), + Scope::Class => None, + }) + .flat_map(bound_in) + .collect(); + let in_a_class = enclosing.iter().any(|scope| matches!(scope, Scope::Class)); + let declared: HashSet<&str> = def + .type_params + .iter() + .flat_map(|params| params.iter()) + .map(|param| param.name().as_str()) + .collect(); + + // what python evaluates: every annotation, unless the module has them all compiled as + // strings, and the bounds, constraints and defaults of the type parameters either way + let mut evaluated: Vec<&Expr> = Vec::new(); + if !future_annotations(module) { + evaluated.extend( + def.parameters + .iter() + .filter_map(ast::AnyParameterRef::annotation) + .chain(def.returns.as_deref()), + ); + } + if let Some(type_params) = &def.type_params { + struct Collect<'a, 'e> { + evaluated: &'e mut Vec<&'a Expr>, + } + impl<'a> Visitor<'a> for Collect<'a, '_> { + fn visit_expr(&mut self, expr: &'a Expr) { + self.evaluated.push(expr); + } + } + visitor::walk_type_params( + &mut Collect { + evaluated: &mut evaluated, + }, + type_params, + ); + } + let mut read = Vec::new(); + let mut private = false; + for expr in evaluated { + visit(expr, &mut |expr| match expr { + Expr::Name(name) => { + private |= is_private(name.id.as_str()); + read.push(name.id.as_str()); + } + Expr::Attribute(attribute) => private |= is_private(attribute.attr.as_str()), + _ => {} + }); + } + // python mangles a private name written anywhere inside a class body, annotations of a + // nested function included, and a factory outside the class would read it unmangled + if in_a_class && private { + return Err("an annotation names a private name inside a class".to_string()); + } + + let mut closed: Vec<(&str, usize)> = Vec::new(); + for name in read { + if declared.contains(name) || !bound.contains(name) { + continue; + } + if closed.iter().any(|(seen, _)| *seen == name) { + continue; + } + let Some(index) = annotations.names.iter().position(|listed| listed == name) else { + return Err(format!( + "an annotation reads `{name}` from an enclosing frame, which the compiled frames do not hold for it" + )); + }; + closed.push((name, index)); + } + + let mut stub = def.clone(); + stub.decorator_list.clear(); + stub.is_async = false; + stub.body = thin_vec::thin_vec![Stmt::Pass(ast::StmtPass { + node_index: ruff_python_ast::AtomicNodeIndex::NONE, + range: def.range, + })]; + let parameters = &mut *stub.parameters; + for parameter in parameters + .posonlyargs + .iter_mut() + .chain(parameters.args.iter_mut()) + .chain(parameters.kwonlyargs.iter_mut()) + { + parameter.default = None; + } + let render = |stub: ast::StmtFunctionDef| -> Result { + let indentation = Indentation::default(); + let rendered = Generator::new(&indentation, LineEnding::Lf).stmt(&Stmt::FunctionDef(stub)); + // the signature on one line and `pass` on the next. anything else is a string + // spanning lines, which indenting would change + if rendered.lines().count() == 2 { + Ok(rendered) + } else { + Err("an annotation is written over more than one line".to_string()) + } + }; + // python makes a function's type parameters once, with the function, and its + // annotations name those very objects. the factory makes them the first time it is + // called, and is handed them back every time after, when the definition it writes + // declares none of its own and reads them from its frame instead + let declares = stub + .type_params + .as_ref() + .map(|params| { + params + .iter() + .map(|param| param.name().to_string()) + .collect::>() + }) + .unwrap_or_default(); + let mut undeclared = stub.clone(); + undeclared.type_params = None; + let making = render(stub)?; + let made = render(undeclared)?; + + let values = free_name("_by_values", &making); + let type_params = free_name("_by_type_params", &making); + let mut source = String::new(); + if future_annotations(module) { + source.push_str("from __future__ import annotations\n"); + } + let _ = writeln!(source, "def {FACTORY}({values}, {type_params}):"); + for (name, index) in &closed { + let _ = writeln!(source, " {name} = {values}[{index}]"); + } + let stub = |source: &mut String, rendered: &str, indent: &str| { + for line in rendered.lines() { + source.push_str(indent); + source.push_str(line); + source.push('\n'); + } + let _ = writeln!(source, "{indent}return {}", def.name.as_str()); + }; + if declares.is_empty() { + stub(&mut source, &making, " "); + } else { + let _ = writeln!(source, " if {type_params} is None:"); + stub(&mut source, &making, " "); + for (index, name) in declares.iter().enumerate() { + let _ = writeln!(source, " {name} = {type_params}[{index}]"); + } + stub(&mut source, &made, " "); + } + Ok((source, !closed.is_empty())) +} + +/// a factory that refuses, in words that say which definition and why +fn refusal(qualname: &str, reason: &str) -> String { + let message = format!( + "the annotations of `{qualname}` are not evaluated by this compiled build: {reason}" + ); + format!( + "def {FACTORY}(values, type_params):\n raise RuntimeError({})\n", + python_string(&message) + ) +} + +/// a scope enclosing the definition, outermost first +enum Scope<'a> { + Function(&'a ast::StmtFunctionDef), + Class, +} + +/// the one definition `qualname` names in `module`, and the scopes around it +fn locate<'a>( + module: &'a [Stmt], + qualname: &str, +) -> Result<(&'a ast::StmtFunctionDef, Vec>), String> { + let mut components = qualname.split('.').peekable(); + let mut body = module; + let mut enclosing = Vec::new(); + while let Some(component) = components.next() { + let found = defined(body, component); + let [found] = found.as_slice() else { + return Err(format!( + "the interpreted module does not define `{qualname}` exactly once" + )); + }; + match found { + Stmt::FunctionDef(def) => { + if components.peek().is_none() { + return Ok((def, enclosing)); + } + if components.next() != Some("") { + return Err(format!("`{qualname}` names no nested function")); + } + enclosing.push(Scope::Function(def)); + body = &def.body; + } + Stmt::ClassDef(class) => { + enclosing.push(Scope::Class); + body = &class.body; + } + _ => break, + } + } + Err(format!("`{qualname}` names no nested function")) +} + +/// the `def`s and `class`es named `name` that one scope's body binds, at any depth of the +/// statements that are not scopes of their own +fn defined<'a>(body: &'a [Stmt], name: &str) -> Vec<&'a Stmt> { + let mut out = Vec::new(); + for stmt in body { + match stmt { + Stmt::FunctionDef(def) if def.name.as_str() == name => out.push(stmt), + Stmt::ClassDef(class) if class.name.as_str() == name => out.push(stmt), + Stmt::If(node) => { + out.extend(defined(&node.body, name)); + for clause in &node.elif_else_clauses { + out.extend(defined(&clause.body, name)); + } + } + Stmt::For(node) => { + out.extend(defined(&node.body, name)); + out.extend(defined(&node.orelse, name)); + } + Stmt::While(node) => { + out.extend(defined(&node.body, name)); + out.extend(defined(&node.orelse, name)); + } + Stmt::With(node) => out.extend(defined(&node.body, name)), + Stmt::Try(node) => { + out.extend(defined(&node.body, name)); + for handler in &node.handlers { + let ast::ExceptHandler::ExceptHandler(handler) = handler; + out.extend(defined(&handler.body, name)); + } + out.extend(defined(&node.orelse, name)); + out.extend(defined(&node.finalbody, name)); + } + Stmt::Match(node) => { + for case in &node.cases { + out.extend(defined(&case.body, name)); + } + } + _ => {} + } + } + out +} + +/// every name a function's own scope binds, and so every name a function nested in it +/// closes over rather than reading as a global +/// +/// deliberately generous: a name counted here that python would not close over only +/// refuses a factory, where one missed would have an annotation read a global in place of +/// the enclosing frame's value +fn bound_in(def: &ast::StmtFunctionDef) -> HashSet<&str> { + struct Binds<'a> { + names: HashSet<&'a str>, + globals: HashSet<&'a str>, + } + impl<'a> Visitor<'a> for Binds<'a> { + fn visit_stmt(&mut self, stmt: &'a Stmt) { + match stmt { + // a nested scope binds its own name here and nothing else + Stmt::FunctionDef(node) => { + self.names.insert(node.name.as_str()); + for decorator in &node.decorator_list { + self.visit_expr(&decorator.expression); + } + } + Stmt::ClassDef(node) => { + self.names.insert(node.name.as_str()); + } + Stmt::Global(node) => { + self.globals + .extend(node.names.iter().map(ast::Identifier::as_str)); + } + Stmt::Nonlocal(node) => { + self.names + .extend(node.names.iter().map(ast::Identifier::as_str)); + } + Stmt::Import(node) => { + for alias in &node.names { + let bound = alias.asname.as_ref().unwrap_or(&alias.name).as_str(); + self.names.insert(bound.split('.').next().unwrap_or(bound)); + } + } + Stmt::ImportFrom(node) => { + for alias in &node.names { + self.names + .insert(alias.asname.as_ref().unwrap_or(&alias.name).as_str()); + } + } + _ => visitor::walk_stmt(self, stmt), + } + } + + fn visit_expr(&mut self, expr: &'a Expr) { + match expr { + Expr::Name(name) + if matches!(name.ctx, ast::ExprContext::Store | ast::ExprContext::Del) => + { + self.names.insert(name.id.as_str()); + } + Expr::Lambda(_) => {} + _ => visitor::walk_expr(self, expr), + } + } + + fn visit_except_handler(&mut self, handler: &'a ast::ExceptHandler) { + let ast::ExceptHandler::ExceptHandler(node) = handler; + if let Some(name) = &node.name { + self.names.insert(name.as_str()); + } + visitor::walk_except_handler(self, handler); + } + + fn visit_pattern(&mut self, pattern: &'a ast::Pattern) { + match pattern { + ast::Pattern::MatchAs(node) => { + if let Some(name) = &node.name { + self.names.insert(name.as_str()); + } + } + ast::Pattern::MatchStar(node) => { + if let Some(name) = &node.name { + self.names.insert(name.as_str()); + } + } + ast::Pattern::MatchMapping(node) => { + if let Some(name) = &node.rest { + self.names.insert(name.as_str()); + } + } + _ => {} + } + visitor::walk_pattern(self, pattern); + } + } + + let mut binds = Binds { + names: HashSet::new(), + globals: HashSet::new(), + }; + for parameter in &*def.parameters { + binds.names.insert(parameter.name().as_str()); + } + for param in def.type_params.iter().flat_map(|params| params.iter()) { + binds.names.insert(param.name().as_str()); + } + binds.visit_body(&def.body); + let Binds { names, globals } = binds; + names.difference(&globals).copied().collect() +} + +/// call `f` on `expr` and every expression inside it +fn visit<'a>(expr: &'a Expr, f: &mut impl FnMut(&'a Expr)) { + struct Walk<'f, F> { + f: &'f mut F, + } + impl<'a, F: FnMut(&'a Expr)> Visitor<'a> for Walk<'_, F> { + fn visit_expr(&mut self, expr: &'a Expr) { + (self.f)(expr); + visitor::walk_expr(self, expr); + } + } + Walk { f }.visit_expr(expr); +} + +/// whether python mangles `name` inside a class body +fn is_private(name: &str) -> bool { + name.starts_with("__") && !name.ends_with("__") +} + +/// whether the module compiles every annotation as a string +fn future_annotations(module: &[Stmt]) -> bool { + module.iter().any(|stmt| { + matches!(stmt, Stmt::ImportFrom(node) + if node.module.as_ref().is_some_and(|module| module.as_str() == "__future__") + && node.names.iter().any(|alias| alias.name.as_str() == "annotations")) + }) +} + +/// a name that occurs nowhere in `text` +fn free_name(base: &str, text: &str) -> String { + let mut name = base.to_string(); + while text.contains(&name) { + name.push('_'); + } + name +} + +/// `text` as a python string literal +fn python_string(text: &str) -> String { + let mut out = String::from("\""); + for character in text.chars() { + match character { + '"' => out.push_str("\\\""), + '\\' => out.push_str("\\\\"), + ' '..='~' => out.push(character), + _ => { + let _ = write!(out, "\\U{:08x}", u32::from(character)); + } + } + } + out.push('"'); + out +} diff --git a/crates/by_irbuild/src/closures.rs b/crates/by_irbuild/src/closures.rs index 6122500992..bd5390829b 100644 --- a/crates/by_irbuild/src/closures.rs +++ b/crates/by_irbuild/src/closures.rs @@ -61,8 +61,28 @@ pub(crate) struct Nested { pub(crate) captures: Vec, /// the captures that either frame *writes*, so both must see one cell pub(crate) shared: Vec, + /// the enclosing names a written `def`'s annotations read, and the sibling that lists + /// their values — see [`ANNOTATION_READER`] + pub(crate) annotations: Option, } +/// the enclosing names a nested function's annotations read +pub(crate) struct AnnotationNames { + pub(crate) names: Vec, + /// the name of the sibling nested function that lists their values, where there are any + pub(crate) reader: Option, +} + +/// the prefix of the name of the function that lists the values of the enclosing names a +/// nested function's annotations read +/// +/// python evaluates annotations over the enclosing frames, and from 3.14 it does so when +/// they are first asked for rather than where the `def` stands. either way it is the +/// enclosing frame's names they read, so their values are listed by a function nested +/// beside the annotated one: it captures exactly those names, and it shares the environment +/// the annotated function holds. it is not a python identifier, so no source can name it +pub(crate) const ANNOTATION_READER: &str = "$annotations"; + /// the nested functions of a body, with their captures /// /// `bound` is what the enclosing function binds — its parameters and locals. a @@ -103,6 +123,30 @@ pub(crate) fn nested_functions( } } + // a written `def` whose annotations read enclosing names has a reader listing them + let mut annotations: HashMap = HashMap::new(); + let mut readers = Vec::new(); + for (def, synthesized) in &definitions { + if !matches!(synthesized, Synthesized::Written) || !is_annotated(def) { + continue; + } + let names: Vec = annotation_names(def) + .into_iter() + .filter(|name| bound.contains(*name)) + .map(str::to_string) + .collect(); + let reader = (!names.is_empty()).then(|| { + let name = format!("{ANNOTATION_READER}{}", definitions.len() + readers.len()); + readers.push(( + synthesize_annotation_reader(def, &names, &name), + Synthesized::AnnotationReader, + )); + name + }); + annotations.insert(def.name.to_string(), AnnotationNames { names, reader }); + } + definitions.extend(readers); + // two `def`s of one name in a scope — the `try` / `except` pair is the common // shape — bind whichever one *ran*, so a direct call cannot know which function it // is calling. they would also mangle to one C symbol, which is how this was found @@ -169,11 +213,110 @@ pub(crate) fn nested_functions( }, captures, shared, + annotations: match synthesized { + Synthesized::Written => annotations.remove(def.name.as_str()), + _ => None, + }, }); } Ok(out) } +/// whether a `def` wrote any annotation, on a parameter or on its return, or declares type +/// parameters — which python makes with the definition, and whose bounds it evaluates over +/// the enclosing frames the way it does an annotation +fn is_annotated(def: &ast::StmtFunctionDef) -> bool { + def.type_params.is_some() + || def.returns.is_some() + || def + .parameters + .iter() + .any(|parameter| parameter.annotation().is_some()) +} + +/// every name a `def`'s annotations read, once each and in the order they are written +/// +/// a type parameter the `def` declares is bound where its annotations are evaluated, so +/// it is not one of them +fn annotation_names(def: &ast::StmtFunctionDef) -> Vec<&str> { + let declared: HashSet<&str> = def + .type_params + .iter() + .flat_map(|params| params.iter()) + .map(|param| param.name().as_str()) + .collect(); + let mut read = Vec::new(); + for parameter in &*def.parameters { + if let Some(annotation) = parameter.annotation() { + collect_reads(annotation, &mut read); + } + } + if let Some(returns) = &def.returns { + collect_reads(returns, &mut read); + } + if let Some(type_params) = &def.type_params { + struct Reads<'a, 'r> { + read: &'r mut Vec<&'a str>, + } + impl<'a> Visitor<'a> for Reads<'a, '_> { + fn visit_expr(&mut self, expr: &'a Expr) { + collect_reads(expr, self.read); + } + } + visitor::walk_type_params(&mut Reads { read: &mut read }, type_params); + } + let mut seen = HashSet::new(); + read.into_iter() + .filter(|name| !declared.contains(name) && seen.insert(*name)) + .collect() +} + +/// the function listing the values of `names`, nested beside `def` — see +/// [`ANNOTATION_READER`] +fn synthesize_annotation_reader( + def: &ast::StmtFunctionDef, + names: &[String], + name: &str, +) -> ast::StmtFunctionDef { + let range = def.range; + // fresh nodes rather than clones of the annotations: an annotation is a type expression + // to the checker, and a read of the value is what this is + let elts = names + .iter() + .map(|read| { + Expr::Name(ast::ExprName { + node_index: ruff_python_ast::AtomicNodeIndex::NONE, + range, + id: read.as_str().into(), + ctx: ast::ExprContext::Load, + }) + }) + .collect(); + ast::StmtFunctionDef { + node_index: ruff_python_ast::AtomicNodeIndex::NONE, + range, + is_async: false, + decorator_list: thin_vec::ThinVec::new(), + name: ast::Identifier::new(name, range), + type_params: None, + parameters: Box::new(ast::Parameters::default()), + returns: None, + raises: None, + is_asserts_return: false, + body: thin_vec::thin_vec![Stmt::Return(ast::StmtReturn { + node_index: ruff_python_ast::AtomicNodeIndex::NONE, + range, + value: Some(Box::new(Expr::List(ast::ExprList { + node_index: ruff_python_ast::AtomicNodeIndex::NONE, + range, + elts, + ctx: ast::ExprContext::Load, + }))), + })], + is_trailing_lambda: false, + } +} + /// the nested functions in `body` that nothing binds except their own `def` /// /// a frame calls a closure it made itself at that closure's native entry point, which @@ -290,6 +433,7 @@ enum Synthesized { Written, Lambda(ruff_text_size::TextRange), GeneratorExpression(ruff_text_size::TextRange), + AnnotationReader, } /// the prefix of the name a generator expression's function is lowered under @@ -649,6 +793,8 @@ fn read_names(body: &[Stmt]) -> Vec<&str> { for decorator in &nested.decorator_list { collect_reads(&decorator.expression, &mut out); } + // and so are its annotations, which python evaluates over this frame too + out.extend(annotation_names(nested)); let own = own_names(nested); out.extend( read_names(&nested.body) diff --git a/crates/by_irbuild/src/lib.rs b/crates/by_irbuild/src/lib.rs index 4da1519889..881ebbe22f 100644 --- a/crates/by_irbuild/src/lib.rs +++ b/crates/by_irbuild/src/lib.rs @@ -8,6 +8,7 @@ //! interpreted definition. so coverage of the language is total from the first //! milestone and only the speed varies. +pub mod annotations; mod closures; mod comprehension_scopes; mod generators; @@ -35,9 +36,14 @@ pub enum Language { /// /// a [`Language`] converts into one, so every caller with nothing more to say goes on /// passing the language it always passed -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct LowerOptions { pub language: Language, + /// which of the runtime soundness checks the interpreted build makes a compiled + /// module makes too — the positions the fallback is transpiled with, so that both + /// builds check the same values in the same places. a `.py` source is its own + /// fallback and makes none + pub soundness: by_transforms::SoundnessPositions, /// have each licensed direct call re-ask, at runtime, the lookup it skips — and /// abort where the two disagree. see [`by_ir::ops::Op::LicenceHolds`] /// @@ -58,12 +64,22 @@ impl From for LowerOptions { fn from(language: Language) -> Self { Self { language, + soundness: match language { + Language::BasedPython => by_transforms::Config::default().soundness, + Language::Python => by_transforms::SoundnessPositions::none(), + }, recheck_licences: false, bind_functions_early: false, } } } +impl Default for LowerOptions { + fn default() -> Self { + Language::default().into() + } +} + impl Language { /// the extension a source of this language is written in pub fn extension(self) -> &'static str { @@ -115,7 +131,7 @@ use by_ir::function::{ }; use by_ir::ops::{ BinOp, BlockId, CmpOp, Concatenation, Conversion, LicenceKind, Mutation, Op, RegisterId, - StandardError, Terminator, UnaryOp, Value, + SoundTarget, StandardError, Terminator, UnaryOp, Value, }; use by_ir::rtype::{Primitive, RType}; use mapper::{Decline, Layouts, Lowered, map_fixed_tuple, map_type, map_type_with}; @@ -126,6 +142,7 @@ use ruff_python_ast::{ use ruff_python_stdlib::identifiers::is_identifier; use ruff_text_size::{Ranged, TextRange, TextSize}; use ty_python_semantic::ProgramEnvironment; +use ty_python_semantic::types::soundness::{CheckKind, CheckTarget}; use ty_python_semantic::types::{KnownClass, SpecialFormType, Type, TypeDefinition, TypeQualifier}; use ty_python_semantic::{HasType, SemanticModel}; @@ -144,10 +161,14 @@ pub fn build_module( ) -> ModuleIr { let LowerOptions { language, + soundness, recheck_licences, bind_functions_early, } = options.into(); let mut module = ModuleIr::new(module_name); + // where the interpreted build checks a value it cannot trust, decided once for the + // whole module, so every frame lowered below makes the same checks in the same places + let soundness = by_transforms::soundness_sites(model, suite, soundness); // a call is only lowered natively when the callee is a module-level function // in this same unit, so the set has to be known before any body is lowered @@ -602,6 +623,7 @@ pub fn build_module( method_names: &method_names, accessors: &accessors, language, + soundness: &soundness, recheck_licences, bind_functions_early, db, @@ -1523,6 +1545,7 @@ fn lower_generator( vec![by_ir::function::ClassIr { exported: false, declares_slots: false, + slots_weak_references: false, name: class, // the machine is the same; only the surface differs. a coroutine answers // `__await__` and is deliberately *not* iterable @@ -1773,6 +1796,7 @@ fn lower_resume( comprehensions: 0, unnarrowed: None, language: unit.language, + soundness: unit.soundness, recheck_licences: unit.recheck_licences, bind_functions_early: unit.bind_functions_early, environment, @@ -1810,6 +1834,8 @@ fn lower_resume( cleanups: Vec::new(), drained_prelude: None, }; + // a generator's body, and the checks at its head, run at the first resumption + lowering.check_parameters(function)?; lowering.block(&function.body)?; // falling off the end exhausts the generator @@ -3105,6 +3131,7 @@ fn lower_class<'a>( decorators: class_decorators, generic: class.type_params.is_some(), declares_slots: declared_slots(class).is_some(), + slots_weak_references: slots_ask_for_weak_references(class)?, constants, slot_aliases, methods: lowered, @@ -3786,6 +3813,14 @@ fn declared_slots(class: &ast::StmtClassDef) -> Option<&Expr> { }) } +/// whether a class's `__slots__` names `__weakref__` +fn slots_ask_for_weak_references(class: &ast::StmtClassDef) -> Lowered { + let Some(value) = declared_slots(class) else { + return Ok(false); + }; + Ok(slot_names(value)?.contains(&"__weakref__")) +} + /// the attribute names a `__slots__` value declares /// /// a bare string declares one, and an iterable of strings declares each. python accepts @@ -3825,12 +3860,17 @@ fn slot_fields( return Ok(fields); }; for name in slot_names(value)? { - // neither is storage of the instance's own: they ask the *type* for a dict and - // for weakref support, and a spec adds neither - if matches!(name, "__dict__" | "__weakref__") { - return Err(Decline::new(format!( - "`__slots__` asks for `{name}`, which a type spec cannot add" - ))); + // neither is storage of the instance's own: they ask the *type* for a dict and for + // a weak-reference list. the list is the word every emitted instance whose layout + // asks for one already keeps — see `ModuleIr::keeps_weak_references` — and a dict + // is what a `__slots__` is written to refuse + if name == "__weakref__" { + continue; + } + if name == "__dict__" { + return Err(Decline::new( + "`__slots__` asks for `__dict__`, which a type spec cannot add", + )); } if !is_identifier(name) { return Err(Decline::new("a `__slots__` entry is not an identifier")); @@ -7342,6 +7382,7 @@ fn lower_function_with_receiver( constructor.nested = Some(by_ir::function::NestedName { name: python_name, qualname, + annotations: nested_annotations(captures), }); } return Ok((constructor, classes)); @@ -7466,6 +7507,14 @@ fn lower_function_with_receiver( unit.environment_suffix.unwrap_or_default() ), ); + // the environment holds the frame's receiver only for a name some nested function + // reads from further up. python's closure holds a cell for each name it reads and + // nothing more, so a method's `self` is not held unless a body reads it — and the + // receiver of a method is an instance, whose fields are not names at all + let reads_further_up = nested + .iter() + .flat_map(|entry| entry.captures.iter()) + .any(|name| !owned.contains(name)); let frame_owned: HashSet = if split { owned.difference(&bindings_here).cloned().collect() } else { @@ -7498,7 +7547,7 @@ fn lower_function_with_receiver( }; let outer_environment = closures::environment( &frame_name, - enclosing, + enclosing.filter(|_| reads_further_up), &nested, &representation, &cell, @@ -7836,6 +7885,7 @@ fn lower_function_with_receiver( owner: unit.owner.map(str::to_string), zero_super, language: unit.language, + soundness: unit.soundness, recheck_licences: unit.recheck_licences, bind_functions_early: unit.bind_functions_early, unwound: Vec::new(), @@ -7866,6 +7916,7 @@ fn lower_function_with_receiver( per_closure, }), }; + lowering.check_parameters(function)?; lowering.block(&function.body)?; // a body that falls off the end returns `None` — python's implicit return. what it @@ -7888,6 +7939,7 @@ fn lower_function_with_receiver( lowered.nested = Some(by_ir::function::NestedName { name: python_name, qualname: qualname.clone(), + annotations: nested_annotations(captures), }); } // the environment's methods are the nested bodies, lowered with the environment @@ -7936,6 +7988,7 @@ fn lower_function_with_receiver( keywords: Vec::new(), exported: false, declares_slots: false, + slots_weak_references: false, decorators: Vec::new(), constants: Vec::new(), slot_aliases: Vec::new(), @@ -7957,6 +8010,7 @@ fn lower_function_with_receiver( keywords: Vec::new(), exported: false, declares_slots: false, + slots_weak_references: false, decorators: Vec::new(), constants: Vec::new(), slot_aliases: Vec::new(), @@ -8015,6 +8069,9 @@ struct Unit<'a> { /// about: whether a captured loop target is a shared cell or a per-closure copy, /// and whether a computed parameter default is snapshotted at the `def` language: Language, + /// the runtime soundness checks the interpreted build makes — see + /// [`LowerOptions::soundness`] + soundness: &'a by_transforms::SoundnessSites, /// whether each licensed direct call re-asks the lookup it skips — see /// [`LowerOptions::recheck_licences`] recheck_licences: bool, @@ -9069,8 +9126,13 @@ fn return_type( let mut found: Option = None; for stmt in walk(body) { let Stmt::Return(ret) = stmt else { continue }; - let rtype = match &ret.value { + let rtype = match ret.value.as_deref() { None => RType::NONE, + // a display the lowering wrote itself was never seen by the checker, and a list + // display is a `list` whatever it holds — see `closures::ANNOTATION_READER` + Some(Expr::List(display)) if display.node_index.load().as_u32().is_none() => { + RType::LIST + } Some(value) => { let ty = value .inferred_type(model) @@ -9144,18 +9206,22 @@ fn covering(left: &RType, right: &RType) -> RType { /// out is held in the field's own representation — which is what lets /// `case Point(a, b)` bind two machine integers instead of two boxed objects. /// -/// three things have to hold for that, and any one of them missing leaves the position +/// four things have to hold for that, and any one of them missing leaves the position /// reading a boxed attribute like every other pattern: /// /// - the pattern names a class **this module emits**, as the checker sees it. that is -/// the same identity [`Lowering::attribute`] reads a field on, and it is what says the -/// `isinstance` the pattern runs is a test against the emitted layout +/// the same identity [`Lowering::attribute`] reads a field on. the `isinstance` the +/// pattern runs is python's and believes `__class__`, so the offsets are only read +/// where the subject's type holds the layout, and a subject it does not reads every +/// position by lookup /// - the class is not a mutable heap type. python can write to one of those, and /// `__match_args__` is a name it could rebind — so which attribute a *position* names /// is only settled for a class that is sealed /// - the position resolves to a field the layout always has. an optional field may be /// absent, and python answers an absent attribute in a class pattern by moving to the /// next case rather than by raising, which a field read has no way to say +/// - the field is held in a representation a looked-up object narrows to, since the +/// lookup binds the same names the load does fn class_pattern_reads( db: &dyn ty_python_semantic::Db, model: &SemanticModel<'_>, @@ -9188,10 +9254,21 @@ fn class_pattern_reads( // by the written name rather than the mangled one: a private name a class body // writes is stored under `_Owner__name`, and matching the raw name exactly is what // says no mangling stands between the pattern and the field + // + // a subject `isinstance` accepts without the layout is read by lookup instead, into + // whatever the field's reads bind, so a field is only read at its offset where a + // looked-up object can be narrowed to the representation it is held in let read = |name: &str| { fields .iter() - .find(|field| field.name == name && !field.optional) + .find(|field| { + field.name == name + && !field.optional + && match &field.ty { + RType::Instance { class, .. } => layouts.contains_key(class), + other => *other == RType::OBJECT || narrowable(other), + } + }) .map(|field| (class.clone(), field.name.clone(), field.ty.clone())) }; @@ -9476,9 +9553,16 @@ fn local_representations( } } } + // what the name is given is the operation's result, which the right-hand operand + // alone does not say: `x **= 0.5` on a float can be complex, `x /= 2` on an int is + // a float Stmt::AugAssign(node) => { if let Expr::Name(name) = node.target.as_ref() { - record(name.id.as_str(), peek(&node.value), &mut found); + let rtype = + ty_python_semantic::basedpython_augmented_assignment_type(model, node) + .and_then(|ty| map_type_with(db, env, ty, layouts).ok()) + .unwrap_or(RType::OBJECT); + record(name.id.as_str(), rtype, &mut found); } } Stmt::For(node) => { @@ -9674,6 +9758,20 @@ fn buffer_safe(body: &[Stmt], name: &str, arrays: &ArrayEditions) -> bool { mentions.iter().all(|mention| safe.contains(mention)) } +/// how a nested function answers for its annotations, where it wrote any — the factory +/// that evaluates them is written once the interpreted twin exists +fn nested_annotations( + captures: Option<&closures::Nested>, +) -> Option { + let annotations = captures?.annotations.as_ref()?; + Some(by_ir::function::NestedAnnotations { + names: annotations.names.clone(), + reader: annotations.reader.clone(), + factory: None, + refused: false, + }) +} + /// every name any frame in this module declares `global`, at any depth /// /// a `global` declaration is written in order to rebind: the name stops holding what @@ -9870,6 +9968,9 @@ struct Lowering<'a, 'db> { owned_cells: Option, /// the language the body is written in, for the lowerings whose answer it decides language: Language, + /// the runtime soundness checks the interpreted build makes — see + /// [`LowerOptions::soundness`] + soundness: &'a by_transforms::SoundnessSites, /// whether each licensed direct call re-asks the lookup it skips — see /// [`LowerOptions::recheck_licences`] recheck_licences: bool, @@ -11155,139 +11256,55 @@ impl Lowering<'_, '_> { // [`class_pattern_reads`] let reads = class_pattern_reads(self.db, self.model, self.layouts, self.sealed, node); - // the subject as the layout the pattern named. the `isinstance` above has - // already said it is one, so this narrowing never fails — but it is a - // *checked* one all the same, which is what keeps a namespace entry that - // is no longer the emitted class a `TypeError` rather than a struct read - // through the wrong pointer - let held = match reads.iter().flatten().next() { - Some((layout, _, _)) => { - let layout = RType::Instance { - class: layout.clone(), - exact: false, - }; - let narrowed = self.coerce(subject.clone(), &RType::OBJECT, &layout)?; - Some(narrowed) - } - None => None, - }; - - // a positional sub-pattern names its attribute through the - // class's `__match_args__`, which only exists at runtime - // - // basedpython's `case Cls(a, *_, b)` leaves the star out of the - // count — it names no attribute of its own — and reads what - // follows it from the end of `__match_args__` instead, which is - // the negative index `By_MatchPositional` resolves - let star = node - .arguments - .patterns + let Some(layout) = reads .iter() - .position(ast::Pattern::is_match_star); - let positional: Vec<&ast::Pattern> = node - .arguments - .patterns - .iter() - .filter(|pattern| !pattern.is_match_star()) - .collect(); - let keywords = &node.arguments.keywords; - let count = i64::try_from(positional.len()) - .map_err(|_| Decline::new("a class pattern with too many sub-patterns"))?; - let before_star = star.unwrap_or(positional.len()); - let total = positional.len() + keywords.len(); - let mut done = 0; - let branch_to_sub = |lowering: &mut Self, - read: Value, - read_ty: &RType, - sub: &ast::Pattern, - done: &mut usize| { - *done += 1; - let next = if *done == total { - matched - } else { - lowering.builder.new_block() - }; - lowering.pattern_branch(sub, &read, read_ty, next, unmatched)?; - if next != matched { - lowering.builder.switch_to(next); - } - Ok::<(), Decline>(()) - }; - // a field the layout always has is read at a compile-time offset, in the - // representation the field holds it in. an attribute the pattern names - // any other way still goes out through the lookup, where *absent* is an - // answer rather than an error - // an instance without the field does not match, the way python's lookup - // failing with `AttributeError` does not - let field_read = |lowering: &mut Self, index: usize| { - let receiver = held.clone()?; - let (class, field, rtype) = reads.get(index)?.clone()?; - let has = lowering.builder.temp(RType::BIT); - lowering.builder.push(Op::FieldIsSet { - dest: has, - receiver: receiver.clone(), - class: class.clone(), - field: field.clone(), - }); - let present = lowering.builder.new_block(); - lowering.builder.terminate(Terminator::Branch { - cond: Value::Register(has), - then_block: present, - else_block: unmatched, - }); - lowering.builder.switch_to(present); - let dest = lowering.builder.temp(rtype.clone()); - lowering.builder.push(Op::GetField { - dest, - receiver, - class, - field, - }); - Some((Value::Register(dest), rtype)) + .flatten() + .next() + .map(|(layout, _, _)| layout.clone()) + else { + return self.class_pattern_attributes( + node, subject, &class, &reads, None, matched, unmatched, + ); }; - for (index, sub) in positional.iter().enumerate() { - let (read, read_ty) = match field_read(self, index) { - Some(field) => field, - None => { - let position = i64::try_from(index).unwrap_or(0); - let read = self.read_match_attr( - &subject.clone(), - None, - Some(class.clone()), - if index < before_star { - position - } else { - position - count - }, - count, - unmatched, - ); - (Value::Register(read), RType::OBJECT) - } - }; - branch_to_sub(self, read, &read_ty, sub, &mut done)?; - } - for (index, keyword) in keywords.iter().enumerate() { - let (read, read_ty) = match field_read(self, positional.len() + index) { - Some(field) => field, - None => { - let read = self.read_match_attr( - &subject.clone(), - Some(self.attribute_name(&keyword.attr)), - None, - 0, - 0, - unmatched, - ); - (Value::Register(read), RType::OBJECT) - } - }; - branch_to_sub(self, read, &read_ty, &keyword.pattern, &mut done)?; - } - if total == 0 { - self.builder.terminate(Terminator::Goto(matched)); - } - Ok(()) + // `isinstance` believes an object's `__class__`, and an abstract base's + // `register` besides, so what it said yes to need not be laid out as the + // class is. the offsets are read only where the type itself says so, and + // everything else takes the lookups python makes + let holds = self.builder.temp(RType::BIT); + self.builder.push(Op::HoldsLayout { + dest: holds, + src: subject.clone(), + class: layout.clone(), + }); + let native = self.builder.new_block(); + let generic = self.builder.new_block(); + self.builder.terminate(Terminator::Branch { + cond: Value::Register(holds), + then_block: native, + else_block: generic, + }); + self.builder.switch_to(native); + let held = self.coerce( + subject.clone(), + &RType::OBJECT, + &RType::Instance { + class: layout, + exact: false, + }, + )?; + self.class_pattern_attributes( + node, + subject, + &class, + &reads, + Some(&held), + matched, + unmatched, + )?; + self.builder.switch_to(generic); + self.class_pattern_attributes( + node, subject, &class, &reads, None, matched, unmatched, + ) } // basedpython: `case P and Q:` — every one has to match the *same* // subject, which is the mirror of `P | Q` and needs no restriction on @@ -11332,6 +11349,140 @@ impl Lowering<'_, '_> { } } + /// the attributes a class pattern names, each tested against its sub-pattern + /// + /// with `held`, the subject as the layout [`class_pattern_reads`] resolved, and an + /// attribute a field always holds is read at its offset. without it every attribute + /// is looked up, which is what python does for all of them + #[expect(clippy::too_many_arguments)] + fn class_pattern_attributes( + &mut self, + node: &ast::PatternMatchClass, + subject: &Value, + class: &Value, + reads: &[Option<(String, String, RType)>], + held: Option<&Value>, + matched: by_ir::ops::BlockId, + unmatched: by_ir::ops::BlockId, + ) -> Lowered<()> { + // a positional sub-pattern names its attribute through the + // class's `__match_args__`, which only exists at runtime + // + // basedpython's `case Cls(a, *_, b)` leaves the star out of the + // count — it names no attribute of its own — and reads what + // follows it from the end of `__match_args__` instead, which is + // the negative index `By_MatchPositional` resolves + let star = node + .arguments + .patterns + .iter() + .position(ast::Pattern::is_match_star); + let positional: Vec<&ast::Pattern> = node + .arguments + .patterns + .iter() + .filter(|pattern| !pattern.is_match_star()) + .collect(); + let keywords = &node.arguments.keywords; + let count = i64::try_from(positional.len()) + .map_err(|_| Decline::new("a class pattern with too many sub-patterns"))?; + let before_star = star.unwrap_or(positional.len()); + let total = positional.len() + keywords.len(); + let mut done = 0; + let branch_to_sub = |lowering: &mut Self, + read: Value, + read_ty: &RType, + sub: &ast::Pattern, + done: &mut usize| { + *done += 1; + let next = if *done == total { + matched + } else { + lowering.builder.new_block() + }; + lowering.pattern_branch(sub, &read, read_ty, next, unmatched)?; + if next != matched { + lowering.builder.switch_to(next); + } + Ok::<(), Decline>(()) + }; + // a field the layout always has is read at a compile-time offset, in the + // representation the field holds it in. an attribute the pattern names + // any other way still goes out through the lookup, where *absent* is an + // answer rather than an error + // an instance without the field does not match, the way python's lookup + // failing with `AttributeError` does not + let field_read = |lowering: &mut Self, index: usize| { + let receiver = held.cloned()?; + let (class, field, rtype) = reads.get(index)?.clone()?; + let has = lowering.builder.temp(RType::BIT); + lowering.builder.push(Op::FieldIsSet { + dest: has, + receiver: receiver.clone(), + class: class.clone(), + field: field.clone(), + }); + let present = lowering.builder.new_block(); + lowering.builder.terminate(Terminator::Branch { + cond: Value::Register(has), + then_block: present, + else_block: unmatched, + }); + lowering.builder.switch_to(present); + let dest = lowering.builder.temp(rtype.clone()); + lowering.builder.push(Op::GetField { + dest, + receiver, + class, + field, + }); + Some((Value::Register(dest), rtype)) + }; + for (index, sub) in positional.iter().enumerate() { + let (read, read_ty) = match field_read(self, index) { + Some(field) => field, + None => { + let position = i64::try_from(index).unwrap_or(0); + let read = self.read_match_attr( + subject, + None, + Some(class.clone()), + if index < before_star { + position + } else { + position - count + }, + count, + unmatched, + ); + (Value::Register(read), RType::OBJECT) + } + }; + branch_to_sub(self, read, &read_ty, sub, &mut done)?; + } + for (index, keyword) in keywords.iter().enumerate() { + let (read, read_ty) = match field_read(self, positional.len() + index) { + Some(field) => field, + None => { + let read = self.read_match_attr( + subject, + Some(self.attribute_name(&keyword.attr)), + None, + 0, + 0, + unmatched, + ); + (Value::Register(read), RType::OBJECT) + } + }; + branch_to_sub(self, read, &read_ty, &keyword.pattern, &mut done)?; + } + if total == 0 { + self.builder.terminate(Terminator::Goto(matched)); + } + Ok(()) + } + /// read the attribute a class pattern names, jumping to `unmatched` when the /// subject simply does not have it /// @@ -12638,6 +12789,13 @@ impl Lowering<'_, '_> { return Err(Decline::new("a for over an array needs an array")); }; let element = (**element).clone(); + // a buffer's elements were checked on their way in, as the element type they are + // held in — so a check that representation does not prove has nothing to test + if self.element_check(&node.iter, &element)?.is_some() { + return Err(Decline::new( + "a soundness check an unboxed list's element type does not prove", + )); + } let width = RType::fixed(by_ir::rtype::IntWidth::I64); // the array is read again every trip, so it lives in a register of its own @@ -12731,6 +12889,8 @@ impl Lowering<'_, '_> { } let (iterable, iterable_ty) = self.expression(&node.iter)?; let iterable = self.widen_to_object(iterable, &iterable_ty); + let narrowed = self.element_narrowing(&node.target)?; + let checked = self.element_check(&node.iter, &narrowed)?; let iterator = self.builder.temp(RType::OBJECT); self.builder.push(Op::AsyncIter { dest: iterator, @@ -12768,6 +12928,9 @@ impl Lowering<'_, '_> { self.builder.set_error_target(previous); let (stepped, stepped_ty) = stepped?; let stepped = self.widen_to_object(stepped, &stepped_ty); + if let Some(checked) = &checked { + self.check_sound(&stepped, &RType::OBJECT, checked.clone()); + } // the target binds what the await produced, checked against the type the // checker gave it — the same narrowing the synchronous loop does @@ -12861,6 +13024,8 @@ impl Lowering<'_, '_> { ) -> Lowered<()> { let target = node.target.as_ref(); let boxed = self.widen_to_object(iterable, iterable_ty); + let narrowed = self.element_narrowing(target)?; + let checked = self.element_check(&node.iter, &narrowed)?; let iterator = self.builder.temp(RType::OBJECT); // the two halves are built together and share one decision, because a `GetIter` // that handed back a list to an `IterNext` with no cursor would step it through @@ -12920,6 +13085,9 @@ impl Lowering<'_, '_> { }); self.builder.switch_to(body); + if let Some(checked) = &checked { + self.check_sound(&Value::Register(raw), &RType::OBJECT, checked.clone()); + } // the null test has run, so the item is a real object by here. narrowing // to the element type is a *checked* unbox even when the target is itself // boxed — a `str` element still has to be proven a str @@ -15095,11 +15263,150 @@ impl Lowering<'_, '_> { /// raises on the line the call starts on, not the line its last argument was on fn expression(&mut self, expr: &Expr) -> Lowered<(Value, RType)> { let enclosing = self.locate(expr.range()); - let lowered = self.expression_here(expr); + let sites = self.soundness; + let lowered = self.expression_here(expr).and_then(|(value, ty)| { + // the interpreted build checks this value where it is produced, before + // anything evaluated after it runs + if let Some(plan) = sites.value(expr) + && let Some(target) = self.sound_target(&ty, plan)? + { + self.check_sound(&value, &ty, target); + } + Ok((value, ty)) + }); self.builder.relocate(enclosing); lowered } + /// the classes a soundness check names, read where the check stands, or `None` + /// where the representation `ty` already proves the answer + /// + /// a representation proves it where holding it took a check `isinstance` agrees + /// with: an unboxed `int` came through `PyLong_Check`, and an emitted class's + /// instance through a type test. the narrowing that made the value raised the + /// check's own `TypeError` if it failed, so nothing is left to ask + fn sound_target(&mut self, ty: &RType, plan: &CheckKind) -> Lowered> { + let CheckKind::Isinstance(target) = plan else { + return Err(Decline::new( + "a soundness check against a generic class's type arguments is not lowered", + )); + }; + if self.proves(ty, target) { + return Ok(None); + } + self.resolve_sound_target(target).map(Some) + } + + /// whether a value held as `ty` is always an instance of `target` + fn proves(&self, ty: &RType, target: &CheckTarget) -> bool { + match target { + CheckTarget::AnyOf(parts) => parts.iter().any(|part| self.proves(ty, part)), + CheckTarget::NoneType => *ty == RType::NONE, + CheckTarget::Class(name) => match ty { + RType::Primitive(primitive) => { + let builtin = match primitive { + Primitive::Int | Primitive::Fixed(_) => "int", + Primitive::Float => "float", + Primitive::Bool | Primitive::Bit => "bool", + Primitive::Str => "str", + Primitive::List => "list", + Primitive::None + | Primitive::Object + | Primitive::Bytes + | Primitive::Dict + | Primitive::Tuple => return false, + }; + name == builtin + } + RType::Instance { class, .. } => self.extends(class, name), + RType::Tuple(_) => name == "tuple", + RType::Array(_) => false, + }, + } + } + + /// each class `target` names, read as the check's own spelling of it reads: a + /// name this frame binds, or a global + fn resolve_sound_target(&mut self, target: &CheckTarget) -> Lowered { + Ok(match target { + CheckTarget::Class(name) => { + let class = match self.place(name) { + Some(Place::Global { .. }) | None => { + let dest = self.builder.temp(RType::OBJECT); + self.builder.push(Op::LoadGlobal { + dest, + name: name.clone(), + }); + Value::Register(dest) + } + Some(place) => { + let (value, ty) = self.read_place(&place)?; + self.widen_to_object(value, &ty) + } + }; + SoundTarget::Class(class) + } + CheckTarget::NoneType => SoundTarget::NoneType, + CheckTarget::AnyOf(parts) => SoundTarget::AnyOf( + parts + .iter() + .map(|part| self.resolve_sound_target(part)) + .collect::>()?, + ), + }) + } + + /// make a resolved soundness check on `value`, held as `ty`, handing nothing on: + /// the value itself is what the program goes on with + fn check_sound(&mut self, value: &Value, ty: &RType, target: SoundTarget) { + let src = self.widen_to_object(value.clone(), ty); + self.builder.push(Op::CheckSound { src, target }); + } + + /// the check the interpreted build makes on each element drawn from `iterable`, + /// with the classes it names read once, where the loop begins — `_soundness_iter` + /// is handed them before the first element is drawn. `None` where no check is made, + /// or where `narrowed`, what each element goes on to be narrowed to, proves it + fn element_check(&mut self, iterable: &Expr, narrowed: &RType) -> Lowered> { + let sites = self.soundness; + let Some(plan) = sites.elements(iterable) else { + return Ok(None); + }; + self.sound_target(narrowed, plan) + } + + /// what an element drawn as an object is narrowed to on its way to `target`: a + /// checked narrowing where there is one, and the object itself otherwise + fn element_narrowing(&mut self, target: &Expr) -> Lowered { + let Expr::Name(_) = target else { + return Ok(RType::OBJECT); + }; + let element_ty = self.peek_type(target)?; + Ok(if self.narrowable_here(&element_ty) { + element_ty + } else { + RType::OBJECT + }) + } + + /// the entry checks the interpreted build makes on `function`'s own parameters, + /// where its body begins + fn check_parameters(&mut self, function: &ast::StmtFunctionDef) -> Lowered<()> { + let sites = self.soundness; + for (name, plan) in sites.parameters(function) { + let Some(place) = self.place(name) else { + return Err(Decline::new(format!( + "the parameter `{name}` a soundness check names has no place in the frame" + ))); + }; + let (value, ty) = self.read_place(&place)?; + if let Some(target) = self.sound_target(&ty, plan)? { + self.check_sound(&value, &ty, target); + } + } + Ok(()) + } + fn expression_here(&mut self, expr: &Expr) -> Lowered<(Value, RType)> { match expr { Expr::NumberLiteral(node) => match &node.value { @@ -15475,7 +15782,9 @@ impl Lowering<'_, '_> { ) { let ((lhs, lhs_ty), (rhs, rhs_ty)) = (lhs, rhs); match (lhs_ty, rhs_ty) { - (RType::Primitive(Primitive::Int), RType::Primitive(Primitive::Int)) => { + (RType::Primitive(Primitive::Int), RType::Primitive(Primitive::Int)) + if !power_may_be_fractional(op, (&lhs, lhs_ty), (&rhs, rhs_ty)) => + { self.builder.push(Op::IntBinary { dest, op, lhs, rhs }); } (RType::Primitive(Primitive::Str), RType::Primitive(Primitive::Str)) @@ -16483,6 +16792,8 @@ impl Lowering<'_, '_> { let target = &generator.target; let (iterable, iterable_ty) = self.expression(&generator.iter)?; let iterable = self.widen_to_object(iterable, &iterable_ty); + let narrowed = self.element_narrowing(target)?; + let checked = self.element_check(&generator.iter, &narrowed)?; let iterator = self.builder.temp(RType::OBJECT); self.builder.push(Op::AsyncIter { dest: iterator, @@ -16517,6 +16828,9 @@ impl Lowering<'_, '_> { let (stepped, stepped_ty) = stepped?; let stepped = self.widen_to_object(stepped, &stepped_ty); self.restore_accumulator(&parked_accumulator, accumulator, accumulator_ty)?; + if let Some(checked) = &checked { + self.check_sound(&stepped, &RType::OBJECT, checked.clone()); + } match target { Expr::Name(name) => { @@ -16643,6 +16957,8 @@ impl Lowering<'_, '_> { let (iterable, iterable_ty) = self.expression(&generator.iter)?; let boxed = self.widen_to_object(iterable, &iterable_ty); + let narrowed = self.element_narrowing(target)?; + let checked = self.element_check(&generator.iter, &narrowed)?; let iterator = self.builder.temp(RType::OBJECT); let cursor = Some(self.loop_cursor()); self.builder.push(Op::GetIter { @@ -16691,6 +17007,9 @@ impl Lowering<'_, '_> { }); self.builder.switch_to(body); + if let Some(checked) = &checked { + self.check_sound(&Value::Register(raw), &RType::OBJECT, checked.clone()); + } match &item_place { Some(place) => { if self.narrowable_here(&element_ty) { @@ -18915,7 +19234,7 @@ fn narrowable(rtype: &RType) -> bool { /// `/` between two ints is a float, and anything that is not a matched pair of /// unboxed numbers goes through the object protocol and yields an object fn binary_result(op: BinOp, lhs: (&Value, &RType), rhs: (&Value, &RType)) -> RType { - if power_may_be_complex(op, lhs, rhs) { + if power_may_be_complex(op, lhs, rhs) || power_may_be_fractional(op, lhs, rhs) { return RType::OBJECT; } let ((_, lhs), (_, rhs)) = (lhs, rhs); @@ -18970,6 +19289,18 @@ fn power_may_be_complex(op: BinOp, lhs: (&Value, &RType), rhs: (&Value, &RType)) !(integral_exponent || real_base) } +/// whether `**` over two ints could answer with a float, which an `int` cannot hold +/// +/// python gives one for any negative exponent, so only an exponent written as a literal +/// that is not negative is known to leave the integers alone +fn power_may_be_fractional(op: BinOp, lhs: (&Value, &RType), rhs: (&Value, &RType)) -> bool { + let ((_, lhs_ty), (rhs, rhs_ty)) = (lhs, rhs); + op == BinOp::Pow + && *lhs_ty == RType::INT + && *rhs_ty == RType::INT + && !matches!(rhs, Value::Int(exponent) if *exponent >= 0) +} + fn compare_op(op: AstCmpOp) -> Lowered { Ok(match op { AstCmpOp::Eq => CmpOp::Eq, diff --git a/crates/by_irbuild/src/tests.rs b/crates/by_irbuild/src/tests.rs index 88a105930a..77ce4adaae 100644 --- a/crates/by_irbuild/src/tests.rs +++ b/crates/by_irbuild/src/tests.rs @@ -358,6 +358,90 @@ fn comparisons_produce_a_bit_register() { // a decline is a feature, not a failure: the function falls back to its // interpreted definition and the rest of the module still compiles +#[test] +fn annotations_that_read_no_enclosing_name_keep_no_values() { + // a module compiling its annotations as strings evaluates none of the names in them, so + // a factory has no values to be handed and none are kept where the `def` stands + for (future, keeps) in [("", true), ("from __future__ import annotations\n", false)] { + let source = format!( + "{future}def outer(kind: type) -> object:\n def inner(y: kind) -> kind:\n return y\n return inner\n" + ); + with_source(&source, |db, env, model, suite| { + let mut module = + crate::build_module(db, env, model, suite, "app", crate::Language::Python); + assert!(module.declined.is_empty(), "{:?}", module.declined); + crate::annotations::write_factories(&mut module, &source); + let readers: Vec = module + .classes + .iter() + .flat_map(|class| class.methods.iter()) + .filter_map(|method| method.nested.as_ref()?.annotations.as_ref()) + .map(|annotations| annotations.reader.is_some()) + .collect(); + assert_eq!(readers, [keeps], "{future}"); + }); + } +} + +#[test] +fn a_gradual_value_meeting_a_declared_type_is_checked_where_the_interpreted_build_checks_it() { + // `to_int` hands its value back as an object, so nothing but the check stands between + // the `Any` and a caller told `int`; `to_optional` has no unboxed form to narrow to + let ir = ir("\ +from typing import Any + +def to_int(v: Any) -> int: + return v + +def to_optional(v: Any) -> int | None: + return v +"); + let to_int = + &ir[ir.find("def to_int").unwrap_or(0)..ir.find("def to_optional").unwrap_or(ir.len())]; + let to_optional = &ir[ir.find("def to_optional").unwrap_or(0)..]; + assert!(to_int.contains("check sound v r"), "{ir}"); + assert!(to_optional.contains("check sound v (r"), "{ir}"); + assert!(to_optional.contains(", NoneType)"), "{ir}"); +} + +#[test] +fn a_check_the_representation_already_proves_is_not_made() { + // an element read out of a `list[int]` is unboxed on its way out, and the unbox raises + // the check's own `TypeError` — so there is nothing left for the check to ask, and the + // class it names is not even looked up + let ir = ir("\ +def first(xs: list[int]) -> int: + return xs[0] + +def total(xs: list[int]) -> int: + out = 0 + for x in xs: + out = out + x + return out +"); + assert!(ir.contains("unbox"), "{ir}"); + assert!(!ir.contains("check sound"), "{ir}"); +} + +#[test] +fn a_soundness_check_against_type_arguments_declines() { + // `_soundness_parametric` tests a reified instance's type arguments, which a compiled + // module has no lowering for, so the function is left to its interpreted definition + let reason = decline( + "\ +from typing import Any + +class Cell[T]: + def __init__(self, value: T): + self.value = value + +def unwrap(v: Any) -> Cell[int]: + return v +", + ); + assert!(reason.contains("type arguments"), "{reason}"); +} + #[test] fn a_gradual_parameter_is_an_object() { // `object` assumes nothing, so no check is needed and nothing declines @@ -3483,11 +3567,11 @@ class Private: #[test] fn a_slots_declaration_the_layout_cannot_answer_declines() { - // neither `__dict__` nor `__weakref__` is storage of the instance's own: they ask the - // *type* for a dict and for weakref support, which a spec adds to neither. and the - // names are the layout, so a declaration nothing here can read is one nothing here - // can lay out — python takes any iterable, including one built at class definition - // time. `Fine` is the boundary: a literal declaration of ordinary names + // `__dict__` is not storage of the instance's own: it asks the *type* for a dict, which + // a spec does not add. and the names are the layout, so a declaration nothing here can + // read is one nothing here can lay out — python takes any iterable, including one built + // at class definition time. `Fine` is the boundary: a literal declaration of ordinary + // names, and `__weakref__` among them asks only for the list an emitted instance keeps let reasons = declines( "\ class Weak: @@ -3513,10 +3597,6 @@ class Fine: assert_eq!( reasons, [ - ( - "Weak".to_string(), - "`__slots__` asks for `__weakref__`, which a type spec cannot add".to_string() - ), ( "Dicted".to_string(), "`__slots__` asks for `__dict__`, which a type spec cannot add".to_string() @@ -11528,9 +11608,8 @@ fn lowered_with_recheck(source: &str, recheck_licences: bool) -> by_ir::function suite, "app", crate::LowerOptions { - language: crate::Language::BasedPython, recheck_licences, - bind_functions_early: false, + ..crate::Language::BasedPython.into() }, ) }) @@ -11652,18 +11731,20 @@ def classify(v: object) -> int: #[test] fn a_class_pattern_over_an_emitted_layout_reads_fields() { - // the pattern's `isinstance` has already said the subject is a `Point`, and a - // `Point` is a sealed static type — python can neither write to it nor derive from - // it — so what passed that test is exactly this layout. the two positions - // `__match_args__` names are then loads at compile-time offsets, and each carries - // the field's own representation, which is what makes `a - b` integer arithmetic - // rather than a call into the object protocol + // a `Point` is a sealed static type — python can neither write to it nor derive from + // it — so a subject whose type holds that layout has the two fields `__match_args__` + // names at compile-time offsets, and each carries the field's own representation, + // which is what makes `a - b` integer arithmetic rather than a call into the object + // protocol. the pattern's `isinstance` believes `__class__`, so what passed it need + // not be laid out as a `Point`: the layout is asked, and a subject that does not hold + // it reads each position by lookup into the same two names let rendered = ir(&format!("{A_MATCHABLE_POINT}{A_POSITIONAL_CLASSIFIER}")); assert!(rendered.contains("let a: int"), "{rendered}"); assert!(rendered.contains("let b: int"), "{rendered}"); + assert!(rendered.contains("= holds-layout "), "{rendered}"); assert!(rendered.contains(""), "{rendered}"); assert!(rendered.contains(""), "{rendered}"); - assert!(!rendered.contains(""), "{rendered}"); + assert!(rendered.contains(""), "{rendered}"); // nothing runs past a `match` whose last case takes anything and whose every body // returns, so the return is the `int` all four of them agree on rather than the // `object` a body that might also fall out of its end would need @@ -11687,6 +11768,7 @@ class Corner(Point): )); assert!(rendered.contains(""), "{rendered}"); assert!(!rendered.contains(""), "{rendered}"); + assert!(!rendered.contains("holds-layout"), "{rendered}"); } #[test] diff --git a/crates/by_opt/src/borrow.rs b/crates/by_opt/src/borrow.rs index 6fd64031a2..5d3d6aa788 100644 --- a/crates/by_opt/src/borrow.rs +++ b/crates/by_opt/src/borrow.rs @@ -649,7 +649,8 @@ fn is_inert(function: &Function, op: &Op) -> bool { Op::FloatBinary { .. } | Op::FloatCompare { .. } | Op::RequireField { .. } - | Op::FieldIsSet { .. } => true, + | Op::FieldIsSet { .. } + | Op::HoldsLayout { .. } => true, Op::Assign { dest, .. } | Op::TupleGet { dest, .. } | Op::GetField { dest, .. } => { plain(dest) } diff --git a/crates/by_opt/src/copy_propagation.rs b/crates/by_opt/src/copy_propagation.rs index 360fd9bab6..cb19d36b52 100644 --- a/crates/by_opt/src/copy_propagation.rs +++ b/crates/by_opt/src/copy_propagation.rs @@ -268,6 +268,7 @@ fn retarget(op: &mut Op, new_dest: RegisterId) { | Op::MatchKey { dest, .. } | Op::MatchRest { dest, .. } | Op::IsMapping { dest, .. } + | Op::HoldsLayout { dest, .. } | Op::AsyncIter { dest, .. } | Op::AsyncContext { dest, .. } | Op::IsMissing { dest, .. } @@ -373,6 +374,7 @@ fn retarget(op: &mut Op, new_dest: RegisterId) { | Op::LeaveGenerator { .. } | Op::LicenceHolds { .. } | Op::RequireField { .. } + | Op::CheckSound { .. } | Op::Line { .. } | Op::SetField { .. } => {} } diff --git a/crates/by_opt/src/field_proofs.rs b/crates/by_opt/src/field_proofs.rs index 54ced1278f..fd7dba1168 100644 --- a/crates/by_opt/src/field_proofs.rs +++ b/crates/by_opt/src/field_proofs.rs @@ -228,6 +228,7 @@ mod tests { dataclass: false, generic: false, declares_slots: false, + slots_weak_references: false, constants: Vec::new(), properties: Vec::new(), slot_aliases: Vec::new(), diff --git a/crates/by_opt/src/fold.rs b/crates/by_opt/src/fold.rs index ffa2857831..f23e0b4adc 100644 --- a/crates/by_opt/src/fold.rs +++ b/crates/by_opt/src/fold.rs @@ -1020,6 +1020,7 @@ mod tests { slot_aliases: Vec::new(), generic: false, declares_slots: false, + slots_weak_references: false, methods: Vec::new(), }); run(&mut m); @@ -1089,6 +1090,7 @@ mod tests { slot_aliases: Vec::new(), generic: false, declares_slots: false, + slots_weak_references: false, methods: Vec::new(), }); run(&mut m); @@ -1149,6 +1151,7 @@ mod tests { slot_aliases: Vec::new(), generic: false, declares_slots: false, + slots_weak_references: false, methods: Vec::new(), }); run(&mut m); @@ -1211,6 +1214,7 @@ mod tests { slot_aliases: Vec::new(), generic: false, declares_slots: false, + slots_weak_references: false, methods: Vec::new(), }); run(&mut m); diff --git a/crates/by_opt/src/infallible.rs b/crates/by_opt/src/infallible.rs index df63744e28..933dcdb0e6 100644 --- a/crates/by_opt/src/infallible.rs +++ b/crates/by_opt/src/infallible.rs @@ -265,6 +265,8 @@ fn op_can_fail(module: &ModuleIr, function: &by_ir::function::Function, op: &Op) Op::MatchKey { .. } | Op::MatchRest { .. } => true, // a type flag, read straight off the type Op::IsMapping { .. } => false, + // a walk up the type's bases, which asks the object nothing + Op::HoldsLayout { .. } => false, // an object with no `__aiter__` is a `TypeError`, and the call may raise Op::AsyncIter { .. } | Op::AsyncContext { .. } => true, // a field the instance does not have is an `AttributeError` @@ -296,6 +298,8 @@ fn op_can_fail(module: &ModuleIr, function: &by_ir::function::Function, op: &Op) Op::MatchSlice { .. } => true, // `isinstance` reaches `__instancecheck__`, which can raise Op::IsInstance { .. } => true, + // …and a soundness check raises when its answer is no + Op::CheckSound { .. } => true, Op::FloatBinary { op, .. } => op.can_fail(), // the object side may be anything, so any of them may raise Op::FloatObjectBinary { .. } | Op::FloatObjectCompare { .. } => true, diff --git a/crates/by_opt/src/lib.rs b/crates/by_opt/src/lib.rs index 52167534c5..d737e689b6 100644 --- a/crates/by_opt/src/lib.rs +++ b/crates/by_opt/src/lib.rs @@ -378,6 +378,7 @@ mod tests { slot_aliases: Vec::new(), generic: false, declares_slots: false, + slots_weak_references: false, base: None, inherited_init: false, fields_are_parameters: true, diff --git a/crates/by_opt/src/runs_python.rs b/crates/by_opt/src/runs_python.rs index cc1b83f272..79da6f7a7e 100644 --- a/crates/by_opt/src/runs_python.rs +++ b/crates/by_opt/src/runs_python.rs @@ -798,6 +798,7 @@ fn op_itself_runs_python(function: &Function, op: &Op, held: &Held) -> bool { | Op::Identity { .. } | Op::IsSequence { .. } | Op::IsMapping { .. } + | Op::HoldsLayout { .. } | Op::LoadEllipsis { .. } | Op::ModuleDict { .. } | Op::LoadClass { .. } @@ -935,6 +936,7 @@ fn op_itself_runs_python(function: &Function, op: &Op, held: &Held) -> bool { | Op::FloatObjectBinary { .. } | Op::FloatObjectCompare { .. } | Op::IsInstance { .. } + | Op::CheckSound { .. } | Op::MatchAttr { .. } | Op::MatchSlice { .. } | Op::MatchKey { .. } @@ -1076,6 +1078,7 @@ mod tests { dataclass: false, generic: false, declares_slots: false, + slots_weak_references: false, constants: Vec::new(), properties: Vec::new(), slot_aliases: Vec::new(), diff --git a/crates/by_rt/include/by.h b/crates/by_rt/include/by.h index 9aae516813..2ed71c54b5 100644 --- a/crates/by_rt/include/by.h +++ b/crates/by_rt/include/by.h @@ -285,9 +285,154 @@ BY_HOT ByTagged By_TaggedLiteral(PyObject *o) { return ((ByTagged)(void *)o) | BY_INT_TAG; } -static inline void By_TypeError(const char *expected, PyObject *got) { - PyErr_Format(PyExc_TypeError, "expected %s, got %s", expected, - got == NULL ? "NULL" : Py_TYPE(got)->tp_name); +/* `type(o).__name__` as an f-string writes it, which is how the transpiled build's + * soundness check names what it was handed. NULL with the error set where asking raised */ +BY_COLD PyObject *By_TypeNameOf(PyObject *o) { + PyObject *name = PyObject_GetAttrString((PyObject *)Py_TYPE(o), "__name__"); + PyObject *text; + if (name == NULL) return NULL; + text = PyObject_Format(name, NULL); + Py_DECREF(name); + return text; +} + +/* a refusal a hot narrowing reaches only when it fails + * + * `noinline` alone is not enough for these: a call to a static function inside an inline + * narrowing such as `By_UnboxInt` is priced as if it might run, and `globals_` took 5% + * more instructions for it. `cold` says it will not */ +#if defined(__GNUC__) || defined(__clang__) +#define BY_REFUSAL __attribute__((cold)) BY_COLD +#else +#define BY_REFUSAL BY_COLD +#endif + +/* the `TypeError` a narrowing raises for a value that is not the `expected` builtin, + * worded as the transpiled build's soundness check words the same refusal: the checks + * sit in the same places, so the two builds raise one error rather than two */ +BY_REFUSAL void By_TypeError(const char *expected, PyObject *got) { + PyObject *name; + if (got == NULL) { + PyErr_Format(PyExc_TypeError, "type soundness violation: expected %s, got NULL", + expected); + return; + } + name = By_TypeNameOf(got); + if (name == NULL) return; + PyErr_Format(PyExc_TypeError, "type soundness violation: expected %s, got %U", expected, + name); + Py_DECREF(name); +} + +/* the attribute `name` of `o`, or NULL with nothing raised where it has none */ +static inline int By_OptionalAttr(PyObject *o, PyObject *name, PyObject **found) { +#if PY_VERSION_HEX >= 0x030D0000 + return PyObject_GetOptionalAttr(o, name, found); +#else + return _PyObject_LookupAttr(o, name, found); +#endif +} + +/* ── soundness checks ────────────────────────────────────────────────────────── + * + * where a value whose type nothing verified meets a declared one, the transpiled build + * checks it with `_soundness_check(value, target)`, and a compiled module makes the + * same check in the same place. these are that function's two halves */ + +/* `isinstance(o, cls)` for an object that is not exactly of class `cls` + * + * a class whose own class is `type` has no `__instancecheck__` to ask, and python's answer + * for it is whether `o`'s type derives from `cls`, or failing that whether `o.__class__` + * does. while `o`'s type reads attributes the generic way and its `__class__` is + * `object`'s own descriptor, that second question answers `type(o)` again and runs nothing, + * so the answer is the first one alone. this is the refusal an optional takes for `None` + * on every read, and the call it saves looks `__class__` up and calls its getter */ +static int By_SoundIsOther(PyObject *o, PyObject *cls) { + static PyObject *dunder_class = NULL; + static PyObject *object_class = NULL; +#ifndef Py_GIL_DISABLED + /* the last refusal worked out that way: a type, the version its attributes and bases + * had, and the class it does not derive from. the lookup and the walk cost a hundred + * instructions between them, and are asked again on every read of an optional. + * + * a write to the type's dict, to a base's, or to its `__bases__` moves the version, so + * a match is the answer both would give. the class is not held: a class the type does + * not derive from is not in its mro, and one arriving at the same address can only + * enter the mro through `__bases__`, which moves the version too */ + static PyTypeObject *refused_type = NULL; + static unsigned int refused_version = 0; + static PyObject *refused_class = NULL; +#endif + PyTypeObject *type = Py_TYPE(o); + if (!Py_IS_TYPE(cls, &PyType_Type) || type->tp_getattro != PyObject_GenericGetAttr) { + return PyObject_IsInstance(o, cls); + } +#ifndef Py_GIL_DISABLED + if (type == refused_type && cls == refused_class && refused_version != 0 + && type->tp_version_tag == refused_version) { + return 0; + } +#endif + if (PyType_IsSubtype(type, (PyTypeObject *)cls)) return 1; + if (dunder_class == NULL) { + dunder_class = PyUnicode_InternFromString("__class__"); + if (dunder_class == NULL) return -1; + object_class = _PyType_Lookup(&PyBaseObject_Type, dunder_class); + } + if (object_class == NULL || _PyType_Lookup(type, dunder_class) != object_class) { + return PyObject_IsInstance(o, cls); + } +#ifndef Py_GIL_DISABLED + refused_type = type; + refused_version = type->tp_version_tag; + refused_class = cls; +#endif + return 0; +} + +/* python's `isinstance(o, cls)` for one class, taking an object of exactly that class + * without a call. that is the first thing `PyObject_IsInstance` asks itself, so the + * answer and everything the call would run are unchanged. -1 is an error */ +static inline int By_SoundIs(PyObject *o, PyObject *cls) { + if (BY_LIKELY((PyObject *)Py_TYPE(o) == cls)) return 1; + return By_SoundIsOther(o, cls); +} + +/* the `TypeError` `_soundness_check` raises when `o` is not an instance of `target`, + * which is whatever that check's `isinstance` was handed — a class or a tuple of them: + * + * f"type soundness violation: expected {getattr(_t, '__name__', _t)}, " + * f"got {type(_v).__name__}" */ +BY_REFUSAL void By_SoundViolation(PyObject *o, PyObject *target) { + static PyObject *dunder_name = NULL; + PyObject *name; + PyObject *expected; + PyObject *got; + if (dunder_name == NULL) { + dunder_name = PyUnicode_InternFromString("__name__"); + if (dunder_name == NULL) return; + } + if (By_OptionalAttr(target, dunder_name, &name) < 0) return; + expected = PyObject_Format(name == NULL ? target : name, NULL); + Py_XDECREF(name); + if (expected == NULL) return; + got = By_TypeNameOf(o); + if (got == NULL) { + Py_DECREF(expected); + return; + } + PyErr_Format(PyExc_TypeError, "type soundness violation: expected %U, got %U", expected, + got); + Py_DECREF(expected); + Py_DECREF(got); +} + +/* [`By_SoundViolation`] for a target of several classes, handed the tuple it builds, or + * NULL where building it raised */ +BY_REFUSAL void By_SoundViolationOf(PyObject *o, PyObject *target) { + if (target == NULL) return; + By_SoundViolation(o, target); + Py_DECREF(target); } /* how many digits an `int` has to have before no value of that length is a short @@ -1131,7 +1276,7 @@ static inline char By_UnboxBool(PyObject *o) { static inline char By_UnboxNone(PyObject *o) { if (o != Py_None) { - By_TypeError("None", o); + By_TypeError("NoneType", o); return 2; } return 0; @@ -3362,6 +3507,387 @@ static inline int By_ConstantsHeldUp(PyObject *cls, PyObject *carried) { return 1; } +/* ── a compiled method, as its class publishes it ───────────────────────────── + * + * python's class holds a `function` under a method's name, and reading it off an + * instance hands back a bound `method` whose `__func__` is that function and whose + * `__self__` is the instance. a `method_descriptor` binds to a + * `builtin_function_or_method` instead, which has no `__func__` — so anything that + * takes a bound method apart and puts it back together, `weakref.WeakMethod` among + * them, refuses what a compiled class hands out. + * + * so the class holds one of these under the name, over the `method_descriptor` the + * method table built. it binds the way a function does, through `PyMethod_New`, and a + * call goes straight to the descriptor's own vectorcall, which still checks the + * receiver's type before the compiled body is reached. like a function it can be + * referred to weakly and written to: a class body hands a decorator the function it + * defined, and `abc.abstractmethod` writes `__isabstractmethod__` onto the object it is + * given and hands that same object back. what a decorator did not write is read through + * to the descriptor, so apart from binding it answers as the descriptor did. + * + * the type is shared by every module in the interpreter, the way the function watcher + * is: a direct call is licensed by recognising the method standing under a name, and + * a method a class of another module published has to be recognised as well */ +typedef struct { + PyObject_HEAD + vectorcallfunc vectorcall; + /* what a call goes to: the `method_descriptor` the method table built, or whatever a + * decorated method's lookup answered where that was not one */ + PyObject *fn; + PyObject *dict; + PyObject *weakrefs; + /* the interpreted definition's function, once the twin class has been read, which is + * where the annotations python evaluated for the method are — see + * `By_Method_getattro` */ + PyObject *twin; +} ByMethodObject; + +#define BY_METHOD_TYPE_KEY "_by_method_type_v2" + +/* the type every module in this interpreter publishes its methods as, once the first + * of them has published one — see `By_FindMethodType` */ +static PyTypeObject *by_method_type = NULL; + +static void By_Method_dealloc(ByMethodObject *self) { + PyObject_GC_UnTrack(self); + if (self->weakrefs != NULL) PyObject_ClearWeakRefs((PyObject *)self); + Py_CLEAR(self->fn); + Py_CLEAR(self->dict); + Py_CLEAR(self->twin); + PyObject_GC_Del(self); +} + +static int By_Method_traverse(ByMethodObject *self, visitproc visit, void *arg) { + Py_VISIT(self->fn); + Py_VISIT(self->dict); + Py_VISIT(self->twin); + return 0; +} + +/* only the `__dict__`, which is the only side a cycle can run through that the class's + * own dict does not break first. leaving `fn` in place is what keeps a call and an + * attribute read from meeting a cleared field */ +static int By_Method_clear(ByMethodObject *self) { + Py_CLEAR(self->dict); + Py_CLEAR(self->twin); + return 0; +} + +/* a class method's descriptor has no vectorcall of its own: python calls it by packing the + * arguments into a tuple, binding the class and calling what that made. where the class is + * one the method is defined on, that is the table entry called with the class in front, + * under the recursion check the bound call would have made — and anything else takes + * python's own path, which raises in its own words */ +static PyObject *By_ClassMethodCall(PyObject *fn, PyObject *const *args, size_t nargsf, + PyObject *kwnames) { + PyMethodDescrObject *descriptor = (PyMethodDescrObject *)fn; + PyMethodDef *def = descriptor->d_method; + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + PyObject *result; + if ((def->ml_flags & (METH_FASTCALL | METH_KEYWORDS | METH_METHOD)) + != (METH_FASTCALL | METH_KEYWORDS) + || nargs < 1 || !PyType_Check(args[0]) + || !PyType_IsSubtype((PyTypeObject *)args[0], PyDescr_TYPE(descriptor))) { + return PyObject_Vectorcall(fn, args, nargsf, kwnames); + } + if (Py_EnterRecursiveCall(" while calling a Python object")) return NULL; + result = ((PyCFunctionFastWithKeywords)(void (*)(void))def->ml_meth)(args[0], args + 1, + nargs - 1, kwnames); + Py_LeaveRecursiveCall(); + return result; +} + +static PyObject *By_Method_vectorcall(PyObject *callable, PyObject *const *args, size_t nargsf, + PyObject *kwnames) { + PyObject *fn = ((ByMethodObject *)callable)->fn; + if (BY_LIKELY(Py_IS_TYPE(fn, &PyMethodDescr_Type))) { + vectorcallfunc call = ((PyMethodDescrObject *)fn)->vectorcall; + if (BY_LIKELY(call != NULL)) return call(fn, args, nargsf, kwnames); + } + if (Py_IS_TYPE(fn, &PyClassMethodDescr_Type)) { + return By_ClassMethodCall(fn, args, nargsf, kwnames); + } + return PyObject_Vectorcall(fn, args, nargsf, kwnames); +} + +static PyObject *By_Method_descr_get(PyObject *self, PyObject *obj, PyObject *type) { + (void)type; + if (obj == NULL || obj == Py_None) return By_NewRef(self); + return PyMethod_New(self, obj); +} + +/* as a function prints itself, under the qualified name it answers — which is what a + * decorator that wrote one sees too. the builtin the call goes to prints itself as a + * builtin: a static method's as a method of a type object */ +static PyObject *By_Method_repr(ByMethodObject *self) { + PyObject *qualname = PyObject_GetAttrString((PyObject *)self, "__qualname__"); + PyObject *text; + if (qualname == NULL) { + if (!PyErr_ExceptionMatches(PyExc_AttributeError)) return NULL; + PyErr_Clear(); + return PyObject_Repr(self->fn); + } + text = PyUnicode_Check(qualname) ? PyUnicode_FromFormat("", qualname, self) + : PyUnicode_FromFormat("", qualname, self); + Py_DECREF(qualname); + return text; +} + +/* whether `name` is one of what a function answers about the `def` it was made from — where + * it was defined, its defaults, its code and what python evaluated its annotations into — + * none of which the builtin a call goes to has an answer to. `inspect.signature` and + * `typing.get_type_hints` read these, as `functools.wraps` copies `__module__` + * + * `__closure__` is not one: the interpreted definition's cells belong to the class it was + * defined in, which is not the type standing under the class's name */ +static int By_IsDefinitionName(PyObject *name) { + static const char *const names[] = { + "__annotations__", "__annotate__", "__type_params__", "__module__", "__defaults__", + "__kwdefaults__", "__code__", "__globals__", "__builtins__", + }; + size_t at; + if (!PyUnicode_Check(name)) return 0; + for (at = 0; at < sizeof(names) / sizeof(names[0]); at++) { + if (PyUnicode_CompareWithASCIIString(name, names[at]) == 0) return 1; + } + return 0; +} + +/* what a decorator wrote is this object's own; everything else it reads off a function + * belongs to what the call goes to until the method has a definition — and then what + * describes the `def` is that definition's, since python made it for the interpreted + * definition and nothing compiled has a true answer to it. those are read when asked, so on + * 3.14 the annotations are evaluated no earlier than python evaluates them. a static + * method's builtin does answer `__module__`, with `None`, so the definition is asked first + * rather than on a miss */ +static PyObject *By_Method_getattro(PyObject *self, PyObject *name) { + PyObject *value = PyObject_GenericGetAttr(self, name); + PyObject *twin = ((ByMethodObject *)self)->twin; + if (value == NULL && PyErr_ExceptionMatches(PyExc_AttributeError)) { + if (twin != NULL && By_IsDefinitionName(name)) { + PyErr_Clear(); + return PyObject_GetAttr(twin, name); + } + /* one with a definition stands in for that `function`, which answers nothing else + * this object does not: the builtin's `__self__`, `__objclass__` and + * `__text_signature__` are answers the definition would raise for */ + if (twin != NULL) return NULL; + PyErr_Clear(); + value = PyObject_GetAttr(((ByMethodObject *)self)->fn, name); + } + return value; +} + +/* a name this type itself answers — `__doc__` is one on every type — read the same way: + * written onto the object, or else the definition's, or else read through. `closure` is + * the name. the definition comes before the builtin because a class a metaclass built + * holds builtins owned by `object`, whose `__qualname__` names `object` */ +static PyObject *By_Method_read_through(PyObject *self, void *closure) { + PyObject *dict = ((ByMethodObject *)self)->dict; + PyObject *twin = ((ByMethodObject *)self)->twin; + if (dict != NULL) { + PyObject *own = PyDict_GetItemString(dict, (const char *)closure); /* borrowed */ + if (own != NULL) return By_NewRef(own); + } + if (twin != NULL) return PyObject_GetAttrString(twin, (const char *)closure); + return PyObject_GetAttrString(((ByMethodObject *)self)->fn, (const char *)closure); +} + +static int By_Method_write_through(PyObject *self, PyObject *value, void *closure) { + PyObject *dict = PyObject_GenericGetDict(self, NULL); + int result; + if (dict == NULL) return -1; + if (value != NULL) { + result = PyDict_SetItemString(dict, (const char *)closure, value); + } else { + result = PyDict_DelItemString(dict, (const char *)closure); + if (result < 0 && PyErr_ExceptionMatches(PyExc_KeyError)) { + PyErr_Clear(); + PyErr_SetString(PyExc_AttributeError, (const char *)closure); + } + } + Py_DECREF(dict); + return result; +} + +/* `copy` and `pickle` both go through this, and the descriptor's answer is to be looked + * up again by name on its class — which finds this object */ +static PyObject *By_Method_reduce(PyObject *self, PyObject *unused) { + (void)unused; + return PyObject_CallMethod(((ByMethodObject *)self)->fn, "__reduce__", NULL); +} + +static PyMethodDef By_Method_methods[] = { + {"__reduce__", By_Method_reduce, METH_NOARGS, NULL}, + {NULL, NULL, 0, NULL}, +}; + +static PyGetSetDef By_Method_getset[] = { + {"__dict__", PyObject_GenericGetDict, PyObject_GenericSetDict, NULL, NULL}, + {"__name__", By_Method_read_through, By_Method_write_through, NULL, "__name__"}, + {"__qualname__", By_Method_read_through, By_Method_write_through, NULL, "__qualname__"}, + {"__doc__", By_Method_read_through, By_Method_write_through, NULL, "__doc__"}, + {NULL, NULL, NULL, NULL, NULL}, +}; + +/* named as what it stands in for, which is also what says a compiled body answered: an + * interpreted class holds a `function` under the same name */ +static PyTypeObject By_MethodType = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "by.method_descriptor", + .tp_basicsize = sizeof(ByMethodObject), + .tp_itemsize = 0, + .tp_dealloc = (destructor)By_Method_dealloc, + .tp_vectorcall_offset = offsetof(ByMethodObject, vectorcall), + .tp_repr = (reprfunc)By_Method_repr, + .tp_call = PyVectorcall_Call, + .tp_getattro = By_Method_getattro, + .tp_setattro = PyObject_GenericSetAttr, + /* so `obj.method()` from python calls it with `obj` in front rather than building + * the bound method first, which is what binding it would have handed the call */ + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HAVE_VECTORCALL + | Py_TPFLAGS_METHOD_DESCRIPTOR, + .tp_traverse = (traverseproc)By_Method_traverse, + .tp_clear = (inquiry)By_Method_clear, + .tp_weaklistoffset = offsetof(ByMethodObject, weakrefs), + .tp_methods = By_Method_methods, + .tp_getset = By_Method_getset, + .tp_descr_get = By_Method_descr_get, + .tp_dictoffset = offsetof(ByMethodObject, dict), + .tp_free = PyObject_GC_Del, +}; + +/* settle `by_method_type`: the type an earlier module shared, or else this module's own, + * readied and shared. the key carries the layout's version, so two modules only ever + * share a type whose objects they both read the same way */ +static int By_FindMethodType(void) { + PyInterpreterState *interpreter; + PyObject *state, *found; + if (by_method_type != NULL) return 0; + interpreter = PyInterpreterState_Get(); + state = interpreter == NULL ? NULL : PyInterpreterState_GetDict(interpreter); + if (state == NULL) { + PyErr_SetString(PyExc_RuntimeError, "no interpreter state to share compiled methods in"); + return -1; + } + found = PyDict_GetItemString(state, BY_METHOD_TYPE_KEY); /* borrowed */ + if (found != NULL && PyType_Check(found)) { + by_method_type = (PyTypeObject *)By_NewRef(found); + return 0; + } + if (PyType_Ready(&By_MethodType) < 0) return -1; + if (PyDict_SetItemString(state, BY_METHOD_TYPE_KEY, (PyObject *)&By_MethodType) < 0) return -1; + by_method_type = (PyTypeObject *)By_NewRef((PyObject *)&By_MethodType); + return 0; +} + +/* `fn` as a method: itself where it already is one, or else one over it */ +static inline PyObject *By_Method(PyObject *fn) { + ByMethodObject *self; + if (fn == NULL) return NULL; + if (By_FindMethodType() < 0) return NULL; + if (Py_IS_TYPE(fn, by_method_type)) return By_NewRef(fn); + self = PyObject_GC_New(ByMethodObject, by_method_type); + if (self == NULL) return NULL; + self->vectorcall = By_Method_vectorcall; + self->fn = By_NewRef(fn); + self->dict = NULL; + self->weakrefs = NULL; + self->twin = NULL; + PyObject_GC_Track(self); + return (PyObject *)self; +} + +/* the `method_descriptor` standing under a name, whether the class holds it as itself or + * as a compiled method over it; NULL, with nothing raised, for anything else */ +static inline PyObject *By_MethodDescriptorOf(PyObject *found) { + if (found == NULL) return NULL; + if (by_method_type != NULL && Py_IS_TYPE(found, by_method_type)) { + found = ((ByMethodObject *)found)->fn; + } + return Py_IS_TYPE(found, &PyMethodDescr_Type) ? found : NULL; +} + +/* publish each plain method among the `count` entries of `table` that `type` holds as + * the descriptor the table built as a compiled method over that descriptor + * + * the caller names the entries a `class` statement wrote. the rest of a table is what + * the runtime adds — a generator's `send`, a `__getstate__` — and python's own of those + * are builtins that bind as builtins + * + * an entry is replaced only where the class's own dict holds the descriptor of that very + * table entry. a name a slot backs holds the slot's wrapper instead, and a class the + * interpreted definition stands for holds functions, and both are left as they are. + * + * a class method and a static method are published the way a class statement holds them, + * as a `classmethod` or a `staticmethod` over the method, rather than as the descriptor and + * the builtin the table built. the class method binds the class through `PyMethod_New`, + * so what `C.make` hands back has a `__func__` and a `__self__`, and a call still goes to + * the table's own descriptor, which checks the class */ +static PyObject *By_PublishedMethod(PyObject *found, PyMethodDef *def) { + PyObject *method, *published; + if (def->ml_flags & METH_CLASS) { + if (!Py_IS_TYPE(found, &PyClassMethodDescr_Type) + || ((PyMethodDescrObject *)found)->d_method != def) { + return By_NewRef(Py_None); + } + method = By_Method(found); + if (method == NULL) return NULL; + published = PyClassMethod_New(method); + Py_DECREF(method); + return published; + } + if (def->ml_flags & METH_STATIC) { + PyObject *function; + int ours; + if (!Py_IS_TYPE(found, &PyStaticMethod_Type)) return By_NewRef(Py_None); + function = PyObject_GetAttrString(found, "__func__"); + if (function == NULL) return NULL; + ours = PyCFunction_Check(function) && ((PyCFunctionObject *)function)->m_ml == def; + method = ours ? By_Method(function) : NULL; + Py_DECREF(function); + if (!ours) return By_NewRef(Py_None); + if (method == NULL) return NULL; + published = PyStaticMethod_New(method); + Py_DECREF(method); + return published; + } + if (!Py_IS_TYPE(found, &PyMethodDescr_Type) || ((PyMethodDescrObject *)found)->d_method != def) { + return By_NewRef(Py_None); + } + return By_Method(found); +} + +static int By_PublishMethods(PyObject *type, PyMethodDef *table, Py_ssize_t count) { + PyObject *dict; + Py_ssize_t at; + int replaced = 0; + if (type == NULL || !PyType_Check(type)) return 0; + dict = ((PyTypeObject *)type)->tp_dict; + if (dict == NULL) return 0; + if (By_FindMethodType() < 0) return -1; + for (at = 0; at < count; at++) { + PyMethodDef *def = &table[at]; + PyObject *found, *method; + int failed; + found = PyDict_GetItemString(dict, def->ml_name); /* borrowed */ + if (found == NULL) continue; + /* `None` for an entry the dict does not hold as the table built it */ + method = By_PublishedMethod(found, def); + if (method == NULL) return -1; + if (method == Py_None) { + Py_DECREF(method); + continue; + } + failed = PyDict_SetItemString(dict, def->ml_name, method) < 0; + Py_DECREF(method); + if (failed) return -1; + replaced = 1; + } + if (replaced) PyType_Modified((PyTypeObject *)type); + return 0; +} + /* one entry of a method table, as the descriptor a class namespace holds * * the three cases `type_add_methods` distinguishes: a class method and a static method @@ -3809,9 +4335,10 @@ static inline void By_ArmMethod(ByMethodLicence *licence, PyObject *type, const } /* reading the type's own attribute hands back the descriptor rather than a bound * method, so the compiled entry point is reachable through it */ - int compiled = Py_IS_TYPE(found, &PyMethodDescr_Type) - && ((PyMethodDescrObject *)found)->d_method != NULL - && ((PyMethodDescrObject *)found)->d_method->ml_meth == body; + PyObject *descriptor = By_MethodDescriptorOf(found); + int compiled = descriptor != NULL + && ((PyMethodDescrObject *)descriptor)->d_method != NULL + && ((PyMethodDescrObject *)descriptor)->d_method->ml_meth == body; Py_DECREF(found); if (!compiled) return; #if PY_VERSION_HEX >= 0x030C0000 @@ -4030,11 +4557,20 @@ static void By_RecheckMethod(PyObject *o, PyObject *type, const char *class_name } /* the lookup hands back a *bound* method, so both halves are asked about: the * definition it carries has to be the compiled body, and it has to be bound to this - * receiver rather than to something the lookup found on the way */ - int compiled = (Py_IS_TYPE(found, &PyCFunction_Type) || Py_IS_TYPE(found, &PyCMethod_Type)) + * receiver rather than to something the lookup found on the way. a compiled method + * binds as a function does, and a table entry nothing replaced binds as a builtin */ + int compiled; + if (PyMethod_Check(found)) { + PyObject *descriptor = By_MethodDescriptorOf(PyMethod_GET_FUNCTION(found)); + compiled = descriptor != NULL + && ((PyMethodDescrObject *)descriptor)->d_method->ml_meth == body + && PyMethod_GET_SELF(found) == o; + } else { + compiled = (Py_IS_TYPE(found, &PyCFunction_Type) || Py_IS_TYPE(found, &PyCMethod_Type)) && ((PyCFunctionObject *)found)->m_ml != NULL && ((PyCFunctionObject *)found)->m_ml->ml_meth == body && ((PyCFunctionObject *)found)->m_self == o; + } /* the name of a type outlives the reference the lookup took: a bound builtin's type * is one of the interpreter's own statics */ const char *reached = Py_TYPE(found)->tp_name; @@ -4202,8 +4738,8 @@ static void By_ArmMethodSite(ByMethodSite *site, PyTypeObject *tp, PyObject *nam * run to find it. reading the attribute off the class instead runs a `__get__` of * the entry's own, and what that hands back for the class need not be what it hands * back for an instance */ - PyObject *found = _PyType_Lookup(tp, name); - if (found == NULL || !Py_IS_TYPE(found, &PyMethodDescr_Type)) return; + PyObject *found = By_MethodDescriptorOf(_PyType_Lookup(tp, name)); + if (found == NULL) return; /* a method descriptor serves the instances of the type that defined it, and refuses * anything else with a `TypeError`. `pop = list.pop` in a class of its own is that * refusal, and calling the entry point directly would hand `list_pop` a receiver @@ -5204,6 +5740,12 @@ static PyObject *By_SettledValue(PyObject *value, const By_Twins *twins, int dep return By_SettleFunction(value, twins, depth - 1) ? By_NewRef(value) : NULL; } + /* a compiled method is settled as what its calls go to */ + if (by_method_type != NULL && Py_IS_TYPE(value, by_method_type)) { + return By_SettlesInPlace(((ByMethodObject *)value)->fn, twins, depth - 1) + ? By_NewRef(value) + : NULL; + } /* a function written in C. the only thing it can hand back that python chose is * `__self__` — the module it was defined in, or the object a method of a built-in type * is bound to. its body resolves no names through a closure and none through a @@ -5230,7 +5772,7 @@ static PyObject *By_SettledValue(PyObject *value, const By_Twins *twins, int dep * * `pprint` is why this is here: `_dispatch[dict.__repr__] = _pprint_dict` keys the * table on a slot wrapper, and refusing the *key* left all 18 values where they were */ - if (Py_TYPE(value) == &PyMethodDescr_Type || Py_TYPE(value) == &PyWrapperDescr_Type + if (By_MethodDescriptorOf(value) != NULL || Py_TYPE(value) == &PyWrapperDescr_Type || Py_TYPE(value) == &PyClassMethodDescr_Type || Py_TYPE(value) == &PyGetSetDescr_Type || Py_TYPE(value) == &PyMemberDescr_Type) { PyObject *owner = PyObject_GetAttrString(value, "__objclass__"); @@ -5411,6 +5953,63 @@ static inline int By_AdoptAbcRegistry(PyObject *source, PyObject *target) { return PyDict_SetItemString(target, "_abc_impl", held); } +/* `value` as a class holds it, or the callable inside the `classmethod` or `staticmethod` + * it is, as a new reference */ +static PyObject *By_UnwrappedMethod(PyObject *value) { + if (Py_IS_TYPE(value, &PyClassMethod_Type) || Py_IS_TYPE(value, &PyStaticMethod_Type)) { + return PyObject_GetAttrString(value, "__func__"); + } + return By_NewRef(value); +} + +/* hand each compiled method in `target`, a type's dict, the function the twin class holds + * under its name in `source`, which is where the method's annotations are — see + * `By_Method_getattro` + * + * a `classmethod` or `staticmethod` copies its callable's `__name__`, `__qualname__`, + * `__module__`, `__doc__` (and on 3.13 its `__annotations__`) when it is made, which was + * before the method had a definition to answer them from. so each one over a method given + * its definition here is made again, in place, the way python makes it. that is left until + * the dict has been walked, because making one reads attributes */ +static int By_AdoptMethodTwins(PyObject *source, PyObject *target) { + PyObject *key, *value, *remade; + Py_ssize_t position = 0, at; + int failed = 0; + if (by_method_type == NULL) return 0; + remade = PyList_New(0); + if (remade == NULL) return -1; + while (!failed && PyDict_Next(target, &position, &key, &value)) { + PyObject *method = By_UnwrappedMethod(value); + if (method == NULL) { + failed = 1; + break; + } + if (Py_IS_TYPE(method, by_method_type)) { + PyObject *twin = PyDict_GetItemWithError(source, key); /* borrowed */ + PyObject *function = twin == NULL ? NULL : By_UnwrappedMethod(twin); + if (function == NULL && PyErr_Occurred()) { + failed = 1; + } else if (function != NULL && PyFunction_Check(function)) { + Py_XSETREF(((ByMethodObject *)method)->twin, function); + if (value != method && PyList_Append(remade, value) < 0) failed = 1; + } else { + Py_XDECREF(function); + } + } + Py_DECREF(method); + } + for (at = 0; !failed && at < PyList_GET_SIZE(remade); at++) { + PyObject *wrapper = PyList_GET_ITEM(remade, at); /* borrowed */ + PyObject *method = PyObject_GetAttrString(wrapper, "__func__"); + PyObject *arguments = method == NULL ? NULL : PyTuple_Pack(1, method); + failed = arguments == NULL || Py_TYPE(wrapper)->tp_init(wrapper, arguments, NULL) < 0; + Py_XDECREF(arguments); + Py_XDECREF(method); + } + Py_DECREF(remade); + return failed ? -1 : 0; +} + /* what the module body gave a class *after* its `class` statement, carried onto the * type that takes its place * @@ -5448,6 +6047,7 @@ static inline int By_AdoptTwinAttributes(const By_Twins *twins) { target = ((PyTypeObject *)type)->tp_dict; if (source == NULL || target == NULL) continue; if (By_CarryAnnotations(source, target, twins) < 0) return -1; + if (By_AdoptMethodTwins(source, target) < 0) return -1; /* ahead of the walk below, which leaves a name the target already holds alone — * and the target holds an `_abc_impl` of its own, the empty one it was built with */ if (By_AdoptAbcRegistry(source, target) < 0) return -1; @@ -8039,6 +8639,54 @@ static inline int By_PublishNew(PyObject *type, PyMethodDef *def) { return stored; } +/* publish the `__new__` that hands an interpreted subclass the allocator of the layout it + * extends, while the class itself keeps `object`'s `tp_new` + * + * the class's own construction needs nothing from it: `object`'s `tp_new` allocates through + * the class's own `tp_alloc`, which is the one that marks the fields absent — and keeping + * `object`'s is what lets `object.__new__(cls)` allocate the class at all, since python + * refuses that for a class whose `tp_new` is a C function of its own. the entry goes + * straight into the dict rather than through an assignment, which would install the + * dispatcher on the class itself. + * + * a subclass is what the entry is for. the `class` statement that makes one finds a + * `__new__` among its bases and installs the dispatcher that looks the name up, so every + * construction of the subclass comes through here and is handed the allocator first */ +static inline int By_PublishAllocatingNew(PyObject *type, PyMethodDef *def) { + PyTypeObject *owner = (PyTypeObject *)type; + PyObject *function = PyCFunction_NewEx(def, NULL, NULL); + if (function == NULL) return -1; + PyObject *published = PyStaticMethod_New(function); + Py_DECREF(function); + if (published == NULL) return -1; + int stored = PyDict_SetItemString(owner->tp_dict, "__new__", published); + Py_DECREF(published); + if (stored == 0) PyType_Modified(owner); + return stored; +} + +/* the class a published `__new__` is asked to allocate, refused where python's own + * `__new__` refuses it: missing, not a class, or not built on `owner` */ +static inline PyTypeObject *By_NewTarget(PyObject *owner, PyObject *const *args, Py_ssize_t nargs) { + const char *name = ((PyTypeObject *)owner)->tp_name; + if (nargs < 1) { + PyErr_Format(PyExc_TypeError, "%s.__new__(): not enough arguments", name); + return NULL; + } + if (!PyType_Check(args[0])) { + PyErr_Format(PyExc_TypeError, "%s.__new__(X): X is not a type object (%s)", name, + Py_TYPE(args[0])->tp_name); + return NULL; + } + PyTypeObject *subtype = (PyTypeObject *)args[0]; + if (!PyType_IsSubtype(subtype, (PyTypeObject *)owner)) { + PyErr_Format(PyExc_TypeError, "%s.__new__(%s): %s is not a subtype of %s", name, + subtype->tp_name, subtype->tp_name, name); + return NULL; + } + return subtype; +} + /* `obj.__weakref__`: the head of the weak references made of the object, or `None` * * what python's own descriptor answers, reached through the offset the type was built @@ -8070,6 +8718,58 @@ static inline void By_AdoptAllocator(PyTypeObject *type, PyObject *owner, allocf } } +/* publish the `__init_subclass__` that hands every interpreted subclass the allocator of + * the layout it extends, the moment the subclass is made + * + * the published `__new__` does that for an instance made through the class, but + * `object.__new__(cls)` reaches the subclass's own allocator without passing it, and until + * the subclass had been constructed once that allocator was the generic one: an unset `int` + * field read as `0`. `type.__new__` asks the bases for `__init_subclass__` whenever it makes + * a class, and the answer is looked up through the mro, so a subclass that writes one of + * its own and chains up with `super()` still reaches this. + * + * it is wrapped as the `classmethod` a class statement makes of an `__init_subclass__`, and + * fills no slot, so nothing else about the type changes by its being there */ +static inline int By_PublishInitSubclass(PyObject *type, PyMethodDef *def) { + PyTypeObject *owner = (PyTypeObject *)type; + PyObject *function = PyCFunction_NewEx(def, NULL, NULL); + if (function == NULL) return -1; + PyObject *published = PyClassMethod_New(function); + Py_DECREF(function); + if (published == NULL) return -1; + int stored = PyDict_SetItemString(owner->tp_dict, "__init_subclass__", published); + Py_DECREF(published); + if (stored == 0) PyType_Modified(owner); + return stored; +} + +/* the body of a published `__init_subclass__`: hand `cls` the allocator, then go on up the + * chain exactly as a written one calling `super().__init_subclass__(...)` would, with every + * argument it was given — so a keyword no class takes is refused in python's own words */ +static PyObject *By_InitSubclass(PyObject *owner, allocfunc alloc, PyObject *const *args, + Py_ssize_t nargsf, PyObject *kwnames) { + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + PyObject *cls; + PyObject *above; + PyObject *next; + PyObject *result; + if (nargs < 1 || !PyType_Check(args[0])) { + PyErr_Format(PyExc_TypeError, "%s.__init_subclass__() needs the class it is made for", + ((PyTypeObject *)owner)->tp_name); + return NULL; + } + cls = args[0]; + By_AdoptAllocator((PyTypeObject *)cls, owner, alloc); + above = PyObject_CallFunctionObjArgs((PyObject *)&PySuper_Type, owner, cls, NULL); + if (above == NULL) return NULL; + next = PyObject_GetAttrString(above, "__init_subclass__"); + Py_DECREF(above); + if (next == NULL) return NULL; + result = PyObject_Vectorcall(next, args + 1, nargs - 1, kwnames); + Py_DECREF(next); + return result; +} + /* publish a `@property` as the object python builds out of its halves * * not as a `tp_getset` entry, which this was. a getset is two function pointers, and what @@ -8936,33 +9636,67 @@ static inline ByArrayHeader *By_ArrayNew(Py_ssize_t cap, size_t width) { static inline void *By_ArrayItems(ByArrayHeader *array) { return (void *)(array + 1); } /* the index a `list` would use, normalized and bounds-checked the same way — a - * negative index counts from the end, and out of range is `IndexError` */ -/* the same, for an index a counter holds as a machine integer. no index past the short + * negative index counts from the end, and out of range is `IndexError`. a read and a + * store refuse in different words, as `list` does, so `out_of_range` is the one this + * access raises + * + * this form takes an index a counter holds as a machine integer. no index past the short * range fits a buffer, so this answers exactly as the tagged form does for every value * the counter can hold */ -static inline Py_ssize_t By_ArrayIndexI64(ByArrayHeader *array, int64_t at) { +static inline Py_ssize_t By_ArrayIndexIn(ByArrayHeader *array, int64_t at, + const char *out_of_range) { if (array == NULL) return -1; if (BY_UNLIKELY(at < PY_SSIZE_T_MIN || at > PY_SSIZE_T_MAX)) { - PyErr_SetString(PyExc_IndexError, "list index out of range"); + PyErr_SetString(PyExc_IndexError, "cannot fit 'int' into an index-sized integer"); return -1; } Py_ssize_t index = (Py_ssize_t)at; if (index < 0) index += array->len; if (index < 0 || index >= array->len) { - PyErr_SetString(PyExc_IndexError, "list index out of range"); + PyErr_SetString(PyExc_IndexError, out_of_range); return -1; } return index; } -static inline Py_ssize_t By_ArrayIndex(ByArrayHeader *array, ByTagged tagged) { - if (array == NULL) return -1; - if (BY_UNLIKELY(!By_IsShort(tagged))) { - // a big integer cannot be a valid index into a buffer this size - PyErr_SetString(PyExc_IndexError, "list index out of range"); +/* an index held tagged. a big integer cannot be a valid index into a buffer, but which + * error it raises depends on whether it fits `Py_ssize_t`: python converts the index + * before it compares it with the length */ +BY_COLD Py_ssize_t By_ArrayIndexBig(ByTagged tagged, const char *out_of_range) { + PyObject *big = By_LongOf(tagged); + Py_ssize_t value = PyLong_AsSsize_t(big); + if (value == -1 && PyErr_Occurred()) { + if (!PyErr_ExceptionMatches(PyExc_OverflowError)) return -1; + PyErr_Clear(); + PyErr_Format(PyExc_IndexError, "cannot fit '%.200s' into an index-sized integer", + Py_TYPE(big)->tp_name); return -1; } - return By_ArrayIndexI64(array, By_ShortValue(tagged)); + PyErr_SetString(PyExc_IndexError, out_of_range); + return -1; +} + +static inline Py_ssize_t By_ArrayIndexTaggedIn(ByArrayHeader *array, ByTagged tagged, + const char *out_of_range) { + if (array == NULL) return -1; + if (BY_UNLIKELY(!By_IsShort(tagged))) return By_ArrayIndexBig(tagged, out_of_range); + return By_ArrayIndexIn(array, By_ShortValue(tagged), out_of_range); +} + +static inline Py_ssize_t By_ArrayIndexI64(ByArrayHeader *array, int64_t at) { + return By_ArrayIndexIn(array, at, "list index out of range"); +} + +static inline Py_ssize_t By_ArrayIndex(ByArrayHeader *array, ByTagged tagged) { + return By_ArrayIndexTaggedIn(array, tagged, "list index out of range"); +} + +static inline Py_ssize_t By_ArrayStoreIndexI64(ByArrayHeader *array, int64_t at) { + return By_ArrayIndexIn(array, at, "list assignment index out of range"); +} + +static inline Py_ssize_t By_ArrayStoreIndex(ByArrayHeader *array, ByTagged tagged) { + return By_ArrayIndexTaggedIn(array, tagged, "list assignment index out of range"); } /* grow a full buffer, doubling so a run of appends stays amortized constant */ @@ -9338,104 +10072,6 @@ static inline void By_Reraise(PyObject *value) { #endif } -/* ── a compiled method, as a decorator sees it ──────────────────────────────── - * - * a class body hands a decorator the *function* it defined, and a python function - * carries a `__dict__`: `abc.abstractmethod` writes `__isabstractmethod__` onto the - * object it is given and hands that same object back. compiling the method - * substitutes a method descriptor, which takes no attributes at all — so the - * substitution, and not the decorator, is what would turn that into an import - * failure. - * - * this is the descriptor with a `__dict__` on it: callable, binding, and writable, - * which is the whole of what a decorator asks of a function. only a *decorated* - * method is wrapped, so an ordinary one keeps the descriptor and the direct call - */ -typedef struct { - PyObject_HEAD - PyObject *fn; - PyObject *dict; -} ByMethodObject; - -static void By_Method_dealloc(ByMethodObject *self) { - PyObject_GC_UnTrack(self); - Py_CLEAR(self->fn); - Py_CLEAR(self->dict); - Py_TYPE(self)->tp_free((PyObject *)self); -} - -static int By_Method_traverse(ByMethodObject *self, visitproc visit, void *arg) { - Py_VISIT(self->fn); - Py_VISIT(self->dict); - return 0; -} - -/* only the `__dict__`, which is the only side a cycle can run through: the method - * itself is a descriptor owned by the type and has no way back here. leaving it in - * place is what keeps a call and an attribute read from meeting a cleared field */ -static int By_Method_clear(ByMethodObject *self) { - Py_CLEAR(self->dict); - return 0; -} - -static PyObject *By_Method_call(ByMethodObject *self, PyObject *args, PyObject *kwds) { - return PyObject_Call(self->fn, args, kwds); -} - -/* bound the way a plain function is: reached through the type it is itself, reached - * through an instance it is a method of that instance */ -static PyObject *By_Method_descr_get(PyObject *self, PyObject *obj, PyObject *type) { - (void)type; - if (obj == NULL || obj == Py_None) return By_NewRef(self); - return PyMethod_New(self, obj); -} - -/* what a decorator *wrote* is ours; everything else it reads off a function — - * `__name__`, `__qualname__`, `__doc__` — still belongs to the method */ -static PyObject *By_Method_getattro(PyObject *self, PyObject *name) { - PyObject *value = PyObject_GenericGetAttr(self, name); - if (value == NULL && PyErr_ExceptionMatches(PyExc_AttributeError)) { - PyErr_Clear(); - value = PyObject_GetAttr(((ByMethodObject *)self)->fn, name); - } - return value; -} - -static PyGetSetDef By_Method_getset[] = { - {"__dict__", PyObject_GenericGetDict, PyObject_GenericSetDict, NULL, NULL}, - {NULL, NULL, NULL, NULL, NULL}, -}; - -static PyTypeObject By_MethodType = { - PyVarObject_HEAD_INIT(NULL, 0) - .tp_name = "by.method", - .tp_basicsize = sizeof(ByMethodObject), - .tp_itemsize = 0, - .tp_dealloc = (destructor)By_Method_dealloc, - .tp_call = (ternaryfunc)By_Method_call, - .tp_getattro = By_Method_getattro, - .tp_setattro = PyObject_GenericSetAttr, - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, - .tp_traverse = (traverseproc)By_Method_traverse, - .tp_clear = (inquiry)By_Method_clear, - .tp_getset = By_Method_getset, - .tp_descr_get = By_Method_descr_get, - .tp_dictoffset = offsetof(ByMethodObject, dict), - .tp_free = PyObject_GC_Del, -}; - -static inline PyObject *By_Method(PyObject *fn) { - ByMethodObject *self; - if (fn == NULL) return NULL; - if (PyType_Ready(&By_MethodType) < 0) return NULL; - self = PyObject_GC_New(ByMethodObject, &By_MethodType); - if (self == NULL) return NULL; - self->fn = By_NewRef(fn); - self->dict = NULL; - PyObject_GC_Track(self); - return (PyObject *)self; -} - /* apply a method's decorators, innermost first, to the finished type — which is the * only place a type spec leaves for them. * @@ -10484,14 +11120,6 @@ static inline PyObject *By_ResumeRaising(PyObject *self, PyObject **sent, PyObje return By_StepGenerator(self, sent, returned, state, frame, Py_None, resume); } -/* the attribute `name` of `o`, or NULL with nothing raised where it has none */ -static inline int By_OptionalAttr(PyObject *o, PyObject *name, PyObject **found) { -#if PY_VERSION_HEX >= 0x030D0000 - return PyObject_GetOptionalAttr(o, name, found); -#else - return _PyObject_LookupAttr(o, name, found); -#endif -} /* python's `gen_close_iter`: close the iterator a frame is delegating to, before the frame * itself is unwound. an iterator with no `close` has nothing to close, and one whose @@ -11317,7 +11945,7 @@ static void By_ArmProtocolSite(ByProtocolSite *site, PyTypeObject *tp, PyObject /* the two that bind to the manager by taking it as their first argument, so calling * the entry with the manager in front is what binding it would have done. anything * else binds some other way, or not at all, and is looked up in full every time */ - if (!PyFunction_Check(found) && !Py_IS_TYPE(found, &PyMethodDescr_Type)) return; + if (!PyFunction_Check(found) && By_MethodDescriptorOf(found) == NULL) return; #if PY_VERSION_HEX >= 0x030C0000 /* from 3.12 the tag is handed out on request rather than by whoever reads an * attribute, and a request is the only thing that reliably produces one */ @@ -11344,7 +11972,7 @@ static inline PyObject *By_LookupSpecial(PyObject *self, PyObject *name, int *pr PyObject *found = _PyType_Lookup(Py_TYPE(self), name); *prepend = 0; if (found == NULL) return NULL; - if (PyFunction_Check(found) || Py_IS_TYPE(found, &PyMethodDescr_Type)) { + if (PyFunction_Check(found) || By_MethodDescriptorOf(found) != NULL) { *prepend = 1; return Py_NewRef(found); } @@ -12012,6 +12640,24 @@ typedef struct { const char *doc; /* the module's namespace, where `__module__` is read from */ PyObject **globals; + /* where the definition wrote annotations, the python source of the function that + * evaluates them — see `by_irbuild::annotations` — and NULL where it wrote none */ + const char *annotations; + /* whether `annotations` only raises, because the build could not say what they are: + * then it raises where they are asked for, and not where the `def` stands */ + int annotations_refused; + /* the values of the enclosing names the annotations read, as a list, from the + * environment; NULL where they read none */ + PyObject *(*annotation_values)(PyObject *env); + /* the function `annotations` defines, once it has been compiled */ + PyObject **annotation_factory; + /* `(__defaults__, __kwdefaults__)` as the boundary fills them in, from the environment; + * NULL where the definition has no defaults */ + PyObject *(*defaults)(PyObject *env); + /* the interpreted module, whose code objects hold this definition's own */ + const By_Fallback *fallback; + /* that code object, once it has been found */ + PyObject **code; } ByFunctionSpec; typedef struct { @@ -12024,6 +12670,20 @@ typedef struct { PyObject *module; PyObject *doc; PyObject *dict; + /* the evaluated `__annotations__`, once there are some */ + PyObject *annotations; + /* an `__annotate__` written over the definition's, `None` included, and `None` once + * `__annotations__` has been written; NULL where neither was */ + PyObject *annotate; + /* `__type_params__`, once it has been made or written */ + PyObject *type_params; + /* the type parameters the definition made, which its annotations name whatever + * `__type_params__` is written over with */ + PyObject *made_type_params; + /* before 3.14, the values the enclosing names the annotations read held where the + * `def` stood, kept until the annotations are first asked for — see + * `By_AnnotateAtDefinition` */ + PyObject *annotation_values; } ByFunctionObject; static void By_Function_dealloc(ByFunctionObject *self) { @@ -12034,6 +12694,11 @@ static void By_Function_dealloc(ByFunctionObject *self) { Py_CLEAR(self->module); Py_CLEAR(self->doc); Py_CLEAR(self->dict); + Py_CLEAR(self->annotations); + Py_CLEAR(self->annotate); + Py_CLEAR(self->type_params); + Py_CLEAR(self->made_type_params); + Py_CLEAR(self->annotation_values); PyObject_GC_Del(self); } @@ -12042,6 +12707,11 @@ static int By_Function_traverse(ByFunctionObject *self, visitproc visit, void *a Py_VISIT(self->module); Py_VISIT(self->doc); Py_VISIT(self->dict); + Py_VISIT(self->annotations); + Py_VISIT(self->annotate); + Py_VISIT(self->type_params); + Py_VISIT(self->made_type_params); + Py_VISIT(self->annotation_values); return 0; } @@ -12049,6 +12719,11 @@ static int By_Function_clear(ByFunctionObject *self) { Py_CLEAR(self->module); Py_CLEAR(self->doc); Py_CLEAR(self->dict); + Py_CLEAR(self->annotations); + Py_CLEAR(self->annotate); + Py_CLEAR(self->type_params); + Py_CLEAR(self->made_type_params); + Py_CLEAR(self->annotation_values); return 0; } @@ -12148,6 +12823,319 @@ static int By_Function_set_doc(PyObject *self, PyObject *value, void *closure) { return 0; } +/* python's own `__globals__`: the namespace of the module the `def` was written in, which + * is what `typing.get_type_hints` evaluates a string annotation against */ +static PyObject *By_Function_get_globals(PyObject *self, void *closure) { + (void)closure; + PyObject *globals = *((ByFunctionObject *)self)->spec->globals; + if (globals == NULL) { + PyErr_SetString(PyExc_AttributeError, "__globals__"); + return NULL; + } + return By_NewRef(globals); +} + +/* the definition python would have made where the `def` stands, with its annotations and no + * body, over the values the enclosing names held where the `def` stood where those were + * kept, and over the values they hold now otherwise — see `by_irbuild::annotations` */ +static PyObject *By_AnnotatedDefinition(PyObject *self) { + const ByFunctionSpec *spec = ((ByFunctionObject *)self)->spec; + PyObject *factory = *spec->annotation_factory; + if (factory == NULL) { + PyObject *globals = *spec->globals; + PyObject *code, *ran, *locals; + if (globals == NULL) { + PyErr_SetString(PyExc_RuntimeError, "the module's namespace is gone"); + return NULL; + } + code = Py_CompileString(spec->annotations, "", Py_file_input); + if (code == NULL) return NULL; + locals = PyDict_New(); + if (locals == NULL) { + Py_DECREF(code); + return NULL; + } + ran = PyEval_EvalCode(code, globals, locals); + Py_DECREF(code); + if (ran == NULL) { + Py_DECREF(locals); + return NULL; + } + Py_DECREF(ran); + factory = PyDict_GetItemString(locals, "_by_annotations"); /* borrowed */ + if (factory == NULL) { + Py_DECREF(locals); + PyErr_SetString(PyExc_RuntimeError, "the annotation factory defines no function"); + return NULL; + } + *spec->annotation_factory = By_NewRef(factory); + Py_DECREF(locals); + } + PyObject *values = ((ByFunctionObject *)self)->annotation_values; + if (values != NULL) { + values = By_NewRef(values); + } else if (spec->annotation_values == NULL) { + values = By_NewRef(Py_None); + } else { + values = spec->annotation_values(((ByFunctionObject *)self)->env); + } + if (values == NULL) return NULL; + ByFunctionObject *function = (ByFunctionObject *)self; + PyObject *made = function->made_type_params == NULL ? Py_None : function->made_type_params; + PyObject *definition = PyObject_CallFunctionObjArgs(factory, values, made, NULL); + Py_DECREF(values); + if (definition != NULL && function->made_type_params == NULL) { + function->made_type_params = PyObject_GetAttrString(definition, "__type_params__"); + if (function->made_type_params == NULL) Py_CLEAR(definition); + } + return definition; +} + +/* 3.13 evaluates a function's annotations where its `def` stands, and a closure made there + * is handed them. evaluating them is a python call that makes a whole definition, which + * cost a closure four thousand instructions more to make than one without annotations, so + * what is taken where the `def` stands is only the values the enclosing names hold, and + * the annotations are evaluated over those when they are first asked for. the reference is + * taken over, and let go of where reading the values raises */ +static inline PyObject *By_AnnotateAtDefinition(PyObject *self) { +#if PY_VERSION_HEX < 0x030E0000 + if (self == NULL || ((ByFunctionObject *)self)->spec->annotations == NULL + || ((ByFunctionObject *)self)->spec->annotations_refused + || ((ByFunctionObject *)self)->spec->annotation_values == NULL) { + return self; + } + PyObject *values = + ((ByFunctionObject *)self)->spec->annotation_values(((ByFunctionObject *)self)->env); + if (values == NULL) { + Py_DECREF(self); + return NULL; + } + ((ByFunctionObject *)self)->annotation_values = values; +#endif + return self; +} + +/* python makes a function's type parameters with the function, so they are made once for + * each closure and kept */ +static PyObject *By_Function_get_type_params(PyObject *self, void *closure) { + (void)closure; + ByFunctionObject *function = (ByFunctionObject *)self; + if (function->type_params == NULL) { + if (function->spec->annotations == NULL) { + function->type_params = PyTuple_New(0); + } else { + if (function->made_type_params == NULL) { + PyObject *definition = By_AnnotatedDefinition(self); + if (definition == NULL) return NULL; + Py_DECREF(definition); + } + function->type_params = By_NewRef(function->made_type_params); + } + if (function->type_params == NULL) return NULL; + } + return By_NewRef(function->type_params); +} + +static int By_Function_set_type_params(PyObject *self, PyObject *value, void *closure) { + (void)closure; + if (value == NULL || !PyTuple_Check(value)) { + PyErr_SetString(PyExc_TypeError, "__type_params__ must be set to a tuple"); + return -1; + } + Py_XSETREF(((ByFunctionObject *)self)->type_params, By_NewRef(value)); + return 0; +} + +#if PY_VERSION_HEX >= 0x030E0000 +/* from 3.14 python evaluates annotations when they are first asked for, through an + * `__annotate__` over the enclosing frames' cells. this one is python's own, made for the + * definition from the values the enclosing names hold when it is asked for */ +static PyObject *By_Function_get_annotate(PyObject *self, void *closure) { + (void)closure; + ByFunctionObject *function = (ByFunctionObject *)self; + if (function->annotate != NULL) return By_NewRef(function->annotate); + if (function->spec->annotations == NULL) return By_NewRef(Py_None); + PyObject *definition = By_AnnotatedDefinition(self); + if (definition == NULL) return NULL; + PyObject *annotate = PyObject_GetAttrString(definition, "__annotate__"); + Py_DECREF(definition); + if (annotate == NULL || annotate == Py_None) return annotate; + PyObject *qualname = PyUnicode_FromFormat("%s.__annotate__", function->spec->qualname); + int named = qualname == NULL ? -1 : PyObject_SetAttrString(annotate, "__qualname__", qualname); + Py_XDECREF(qualname); + if (named < 0) { + Py_DECREF(annotate); + return NULL; + } + return annotate; +} + +static int By_Function_set_annotate(PyObject *self, PyObject *value, void *closure) { + (void)closure; + ByFunctionObject *function = (ByFunctionObject *)self; + if (value == NULL) { + PyErr_SetString(PyExc_TypeError, "__annotate__ cannot be deleted"); + return -1; + } + if (value != Py_None && !PyCallable_Check(value)) { + PyErr_SetString(PyExc_TypeError, "__annotate__ must be callable or None"); + return -1; + } + Py_XSETREF(function->annotate, By_NewRef(value)); + if (value != Py_None) Py_CLEAR(function->annotations); + return 0; +} +#endif + +static PyObject *By_Function_get_annotations(PyObject *self, void *closure) { + (void)closure; + ByFunctionObject *function = (ByFunctionObject *)self; + if (function->annotations == NULL) { +#if PY_VERSION_HEX >= 0x030E0000 + PyObject *annotate = By_Function_get_annotate(self, NULL); + if (annotate == NULL) return NULL; + if (annotate != Py_None && PyCallable_Check(annotate)) { + PyObject *value_format = PyLong_FromLong(1); + PyObject *annotations = + value_format == NULL ? NULL : PyObject_CallOneArg(annotate, value_format); + Py_XDECREF(value_format); + Py_DECREF(annotate); + if (annotations == NULL) return NULL; + if (!PyDict_Check(annotations)) { + PyErr_Format(PyExc_TypeError, "__annotate__ returned non-dict of type '%.100s'", + Py_TYPE(annotations)->tp_name); + Py_DECREF(annotations); + return NULL; + } + function->annotations = annotations; + } else { + Py_DECREF(annotate); + } +#else + /* evaluated the first time they are asked for, unless something has been written + * over them since, and the values kept for it are let go of once they have been. + * annotations the build could not supply raise here instead */ + if (function->spec->annotations != NULL && function->annotate == NULL) { + PyObject *definition = By_AnnotatedDefinition(self); + if (definition == NULL) return NULL; + PyObject *annotations = PyObject_GetAttrString(definition, "__annotations__"); + Py_DECREF(definition); + if (annotations == NULL) return NULL; + function->annotations = annotations; + Py_CLEAR(function->annotation_values); + } +#endif + if (function->annotations == NULL) { + function->annotations = PyDict_New(); + if (function->annotations == NULL) return NULL; + } + } + return By_NewRef(function->annotations); +} + +static int By_Function_set_annotations(PyObject *self, PyObject *value, void *closure) { + (void)closure; + ByFunctionObject *function = (ByFunctionObject *)self; + if (value == Py_None) value = NULL; + if (value != NULL && !PyDict_Check(value)) { + PyErr_SetString(PyExc_TypeError, "__annotations__ must be set to a dict object"); + return -1; + } + Py_XSETREF(function->annotations, Py_XNewRef(value)); + Py_XSETREF(function->annotate, By_NewRef(Py_None)); + return 0; +} + +/* the code object python compiled for this definition in the interpreted module, found by + * its qualified name among the constants of every code object the module holds + * + * `count` is how many answer to the name: a module whose body defines the same qualified + * name twice — a `def` under each arm of an `if` — does not say which is this one */ +static void By_FindCode(PyObject *code, PyObject *qualname, PyObject **found, int *count) { + PyObject *consts = PyObject_GetAttrString(code, "co_consts"); + if (consts == NULL) { + PyErr_Clear(); + return; + } + if (PyTuple_Check(consts)) { + for (Py_ssize_t at = 0; at < PyTuple_GET_SIZE(consts); at++) { + PyObject *item = PyTuple_GET_ITEM(consts, at); + if (!PyCode_Check(item)) continue; + PyObject *named = PyObject_GetAttrString(item, "co_qualname"); + if (named == NULL) { + PyErr_Clear(); + } else { + int same = PyUnicode_Check(named) && PyUnicode_Compare(named, qualname) == 0; + Py_DECREF(named); + if (same) { + (*count)++; + Py_XSETREF(*found, By_NewRef(item)); + } + } + By_FindCode(item, qualname, found, count); + } + } + Py_DECREF(consts); +} + +/* python's `__code__`: the one the interpreted definition runs, which is what says what + * parameters the function takes — `inspect.signature` reads them off it, as it does off a + * cython function */ +static PyObject *By_Function_get_code(PyObject *self, void *closure) { + (void)closure; + const ByFunctionSpec *spec = ((ByFunctionObject *)self)->spec; + if (spec->fallback == NULL || spec->code == NULL) { + PyErr_SetString(PyExc_AttributeError, "__code__"); + return NULL; + } + if (*spec->code == NULL) { + PyObject *module = By_FallbackCode(spec->fallback); + if (module == NULL) { + if (PyErr_Occurred()) return NULL; + module = Py_CompileString(spec->fallback->source, "", Py_file_input); + if (module == NULL) return NULL; + } + PyObject *qualname = PyUnicode_FromString(spec->qualname); + PyObject *found = NULL; + int count = 0; + if (qualname != NULL) By_FindCode(module, qualname, &found, &count); + Py_XDECREF(qualname); + Py_DECREF(module); + if (qualname == NULL) return NULL; + if (count != 1) { + Py_XDECREF(found); + PyErr_Format(PyExc_AttributeError, + "the interpreted module does not define `%s` exactly once, so this " + "compiled function has no `__code__`", + spec->qualname); + return NULL; + } + *spec->code = found; + } + return By_NewRef(*spec->code); +} + +/* one half of what `spec->defaults` hands back */ +static PyObject *By_Function_default_half(PyObject *self, Py_ssize_t half) { + ByFunctionObject *function = (ByFunctionObject *)self; + if (function->spec->defaults == NULL) return By_NewRef(Py_None); + PyObject *both = function->spec->defaults(function->env); + if (both == NULL) return NULL; + PyObject *answer = By_NewRef(PyTuple_GET_ITEM(both, half)); + Py_DECREF(both); + return answer; +} + +static PyObject *By_Function_get_defaults(PyObject *self, void *closure) { + (void)closure; + return By_Function_default_half(self, 0); +} + +static PyObject *By_Function_get_kwdefaults(PyObject *self, void *closure) { + (void)closure; + return By_Function_default_half(self, 1); +} + static PyObject *By_Function_self(PyObject *self, PyObject *unused) { (void)unused; return By_NewRef(self); @@ -12173,6 +13161,15 @@ static PyGetSetDef By_Function_getset[] = { {"__module__", By_Function_get_module, By_Function_set_module, NULL, NULL}, {"__doc__", By_Function_get_doc, By_Function_set_doc, NULL, NULL}, {"__dict__", PyObject_GenericGetDict, PyObject_GenericSetDict, NULL, NULL}, + {"__globals__", By_Function_get_globals, NULL, NULL, NULL}, + {"__code__", By_Function_get_code, NULL, NULL, NULL}, + {"__type_params__", By_Function_get_type_params, By_Function_set_type_params, NULL, NULL}, + {"__defaults__", By_Function_get_defaults, NULL, NULL, NULL}, + {"__kwdefaults__", By_Function_get_kwdefaults, NULL, NULL, NULL}, + {"__annotations__", By_Function_get_annotations, By_Function_set_annotations, NULL, NULL}, +#if PY_VERSION_HEX >= 0x030E0000 + {"__annotate__", By_Function_get_annotate, By_Function_set_annotate, NULL, NULL}, +#endif {NULL, NULL, NULL, NULL, NULL}, }; @@ -12214,6 +13211,11 @@ static inline PyObject *By_MakeFunction(const ByFunctionSpec *spec, PyObject *en self->module = NULL; self->doc = NULL; self->dict = NULL; + self->annotations = NULL; + self->annotate = NULL; + self->type_params = NULL; + self->made_type_params = NULL; + self->annotation_values = NULL; PyObject_GC_Track(self); return (PyObject *)self; } @@ -12236,8 +13238,7 @@ static inline PyObject *By_FunctionEnvironment(PyObject *callable) { follow a wild pointer */ static inline PyObject *By_CheckInstance(PyObject *o, PyTypeObject *type) { if (!PyObject_TypeCheck(o, type)) { - PyErr_Format(PyExc_TypeError, "expected %s, got %s", type->tp_name, - Py_TYPE(o)->tp_name); + By_SoundViolation(o, (PyObject *)type); return NULL; } return o; diff --git a/crates/by_transforms/src/lib.rs b/crates/by_transforms/src/lib.rs index d2fc27fe9a..78d8e62ae3 100644 --- a/crates/by_transforms/src/lib.rs +++ b/crates/by_transforms/src/lib.rs @@ -6,6 +6,7 @@ mod transforms; pub(crate) mod type_info; pub use config::{Config, FloatLiteralLowering, PythonVersion, SoundnessPositions}; +pub use transforms::soundness::{SoundnessSites, soundness_sites}; /// A module's `main` as the program's command line — the reading the entry-point /// guard and its argument parser are generated from. diff --git a/crates/by_transforms/src/transforms/checked_cast.rs b/crates/by_transforms/src/transforms/checked_cast.rs index 9053fb3b2d..1c5bb41dd8 100644 --- a/crates/by_transforms/src/transforms/checked_cast.rs +++ b/crates/by_transforms/src/transforms/checked_cast.rs @@ -163,7 +163,7 @@ impl<'a> CastLower<'a> { // `isinstance` target, or the written type when ty offers no faithful one match self.types.cast_check_plan(type_arg) { Some(CastCheck::Kind(SoundnessCheck::Isinstance(target))) => { - (helper, vec![Fragment::Lit(target)]) + (helper, vec![Fragment::Lit(target.to_string())]) } // a `Literal` is a set of values, not a class: `isinstance` against // one raises, and testing the values' class would accept every other diff --git a/crates/by_transforms/src/transforms/soundness.rs b/crates/by_transforms/src/transforms/soundness.rs index ab9c101ae7..0646bcdd16 100644 --- a/crates/by_transforms/src/transforms/soundness.rs +++ b/crates/by_transforms/src/transforms/soundness.rs @@ -62,11 +62,15 @@ //! defined later in the module can raise `NameError` if the checked line runs //! at import time before the class body. +use std::collections::HashMap; use std::fmt::Write as _; use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; -use ruff_python_ast::{Comprehension, Expr, ExprCall, Parameter, Stmt, StmtFunctionDef, UnaryOp}; +use ruff_python_ast::{ + Comprehension, Expr, ExprCall, HasNodeIndex, Parameter, Stmt, StmtFunctionDef, UnaryOp, +}; use ruff_text_size::{Ranged, TextRange, TextSize}; +use ty_python_semantic::types::soundness::CheckTarget; use super::ast_driver::{Fragment, PassContext, TypeAwarePass}; use super::parametric_is::variance_tuple; @@ -88,19 +92,32 @@ enum ArgSlot<'a> { Keyword(&'a str), } -#[expect( - clippy::struct_excessive_bools, - reason = "independent runtime-helper usage flags, not a state machine" -)] -struct Soundness<'a> { +/// one check the pass decided on, before anything is written for it +enum Site<'ast> { + /// the value `expr` produces is checked as it is produced + Value { + expr: &'ast Expr, + plan: SoundnessCheck, + }, + /// each element drawn from `iterable` is checked as it is drawn + Elements { + iterable: &'ast Expr, + is_async: bool, + plan: SoundnessCheck, + }, + /// `function`'s own parameters are checked where its body begins + Parameters { + function: &'ast StmtFunctionDef, + guards: Vec<(&'ast str, SoundnessCheck)>, + }, +} + +/// where the checks go, decided once over a syntax tree, and what is not a place +/// for one +struct Soundness<'a, 'ast> { types: &'a dyn TypeInfo, positions: SoundnessPositions, - /// the working source — needed to read a function body's indentation when - /// inserting `parameters` entry guards - source: &'a str, - edits: Vec<(TextRange, Vec)>, - /// the entry-guard suites, anchored at the body statement they precede - guards: Vec<(TextSize, Vec)>, + sites: Vec>, /// gated expressions already covered by a wrap on an enclosing `!` /// force-unwrap — wrapping them directly would splice the check's second /// argument into `_force_unwrap`'s call parens @@ -112,31 +129,17 @@ struct Soundness<'a> { /// declared-return-type check plans of the enclosing functions (innermost /// last); `None` when a function has no annotation or an uncheckable one return_targets: Vec>, - used_iter: bool, - used_aiter: bool, - used_iter_p: bool, - used_aiter_p: bool, - /// any deep parametric check emitted — pulls in `_soundness_parametric` - /// and the `_parametric_is` probe it reuses - used_parametric: bool, } -impl<'a> Soundness<'a> { - fn new(types: &'a dyn TypeInfo, positions: SoundnessPositions, source: &'a str) -> Self { +impl<'a, 'ast> Soundness<'a, 'ast> { + fn new(types: &'a dyn TypeInfo, positions: SoundnessPositions) -> Self { Self { types, positions, - source, - edits: Vec::new(), - guards: Vec::new(), + sites: Vec::new(), consumed: Vec::new(), verbatim: Vec::new(), return_targets: Vec::new(), - used_iter: false, - used_aiter: false, - used_iter_p: false, - used_aiter_p: false, - used_parametric: false, } } @@ -179,7 +182,7 @@ impl<'a> Soundness<'a> { fn check_plan(&self, expr: &Expr) -> Option { self.types .soundness_check_plan(expr) - .filter(|plan| !matches!(plan, SoundnessCheck::Isinstance(t) if t == "type(None)")) + .filter(|plan| !matches!(plan, SoundnessCheck::Isinstance(CheckTarget::NoneType))) } /// [`Self::check_plan`] for a parameter, read off the parameter rather than off @@ -188,97 +191,51 @@ impl<'a> Soundness<'a> { fn parameter_plan(&self, parameter: &Parameter) -> Option { self.types .parameter_check_plan(parameter) - .filter(|plan| !matches!(plan, SoundnessCheck::Isinstance(t) if t == "type(None)")) + .filter(|plan| !matches!(plan, SoundnessCheck::Isinstance(CheckTarget::NoneType))) } - /// wrap `source[range]` in `helper(, )`. `trailing` - /// carries its own leading `, ` (e.g. `", str"` or `", A[int], (0,)"`) - fn wrap_call(&mut self, range: TextRange, helper: &str, trailing: &str) { - self.edits.push(( - range, - vec![ - Fragment::Lit(format!("{helper}(")), - Fragment::Src(range), - Fragment::Lit(format!("{trailing})")), - ], - )); + /// check the value `expr` produces against `plan` + fn check_value(&mut self, expr: &'ast Expr, plan: SoundnessCheck) { + self.sites.push(Site::Value { expr, plan }); } - /// wrap `range` in the scalar check (`_soundness_check` / - /// `_soundness_parametric`) named by `plan` - fn wrap_check(&mut self, range: TextRange, plan: &SoundnessCheck) { - match plan { - SoundnessCheck::Isinstance(target) => { - self.wrap_call(range, "_soundness_check", &format!(", {target}")); - } - SoundnessCheck::Parametric { alias, variances } => { - self.used_parametric = true; - self.wrap_call( - range, - "_soundness_parametric", - &format!(", {alias}, {}", variance_tuple(variances)), - ); - } - } - } - - /// wrap the iterable of a `for` / comprehension clause when the iterable - /// carries a generic specialization and the element (loop target) type is - /// checkable. the iterable is wrapped in a validating generator whose form - /// (shallow vs parametric) follows the element's check plan - fn wrap_iteration(&mut self, iterable: &Expr, target_expr: &Expr, is_async: bool) { + /// check each element drawn from the iterable of a `for` / comprehension + /// clause when the iterable carries a generic specialization and the element + /// (loop target) type is checkable + fn check_iteration(&mut self, iterable: &'ast Expr, target_expr: &Expr, is_async: bool) { if !self.types.is_specialized_generic_instance(iterable) { return; } let Some(plan) = self.check_plan(target_expr) else { return; }; - let (helper, trailing) = match &plan { - SoundnessCheck::Isinstance(target) => { - let helper = if is_async { - self.used_aiter = true; - "_soundness_aiter" - } else { - self.used_iter = true; - "_soundness_iter" - }; - (helper, format!(", {target}")) - } - SoundnessCheck::Parametric { alias, variances } => { - self.used_parametric = true; - let helper = if is_async { - self.used_aiter_p = true; - "_soundness_aiter_p" - } else { - self.used_iter_p = true; - "_soundness_iter_p" - }; - (helper, format!(", {alias}, {}", variance_tuple(variances))) - } - }; - self.wrap_call(iterable.range(), helper, &trailing); + self.sites.push(Site::Elements { + iterable, + is_async, + plan, + }); } - /// wrap each argument of `call` whose own type is unhelpful against its + /// check each argument of `call` whose own type is unhelpful against its /// matched parameter's annotation. positional mapping stops at the first /// starred spread (positions past it are unknown); `**kwargs` spreads are /// skipped - fn wrap_call_arguments(&mut self, call: &ExprCall) { + fn check_call_arguments(&mut self, call: &'ast ExprCall) { let callee = call.func.as_ref(); for (index, arg) in call.arguments.args.iter().enumerate() { if arg.is_starred_expr() { break; } - self.maybe_wrap_argument(callee, arg, &ArgSlot::Positional(index)); + self.maybe_check_argument(callee, arg, &ArgSlot::Positional(index)); } for keyword in &call.arguments.keywords { if let Some(name) = &keyword.arg { - self.maybe_wrap_argument(callee, &keyword.value, &ArgSlot::Keyword(name.as_str())); + self.maybe_check_argument(callee, &keyword.value, &ArgSlot::Keyword(name.as_str())); } } } - fn maybe_wrap_argument(&mut self, callee: &Expr, arg: &Expr, slot: &ArgSlot<'_>) { + fn maybe_check_argument(&mut self, callee: &Expr, arg: &'ast Expr, slot: &ArgSlot<'_>) { if self.consumed.contains(&arg.range()) || !self.value_needs_context_target(arg) { return; } @@ -287,47 +244,33 @@ impl<'a> Soundness<'a> { ArgSlot::Keyword(name) => self.types.call_keyword_param_plan(callee, name), }; if let Some(plan) = plan { - self.wrap_check(arg.range(), &plan); + self.check_value(arg, plan); } } /// the plan to validate a returned `value` against — the enclosing /// function's declared return plan — but only when `value`'s own type /// is unhelpful (else `generic_calls`/`projections` already covers it) - fn return_wrap_plan(&self, value: &Expr) -> Option { + fn return_plan(&self, value: &Expr) -> Option { let plan = self.return_targets.last()?.clone()?; self.value_needs_context_target(value).then_some(plan) } - /// the guard statement that validates parameter `name` against `plan` - fn guard_stmt(&mut self, name: &str, plan: &SoundnessCheck) -> String { - match plan { - SoundnessCheck::Isinstance(target) => format!("_soundness_check({name}, {target})"), - SoundnessCheck::Parametric { alias, variances } => { - self.used_parametric = true; - format!( - "_soundness_parametric({name}, {alias}, {})", - variance_tuple(variances) - ) - } - } - } - - /// insert entry guards validating each checkable parameter of `func` at the - /// top of its body — the `parameters` position, defending the contract - /// against callers the checker never saw. variadic (`*args` / `**kwargs`) - /// parameters are skipped, and so is any parameter whose source states no - /// type: an unannotated one with no default states nothing, and `x=None` - /// says the argument may be left out rather than that `None` belongs there + /// check each checkable parameter of `func` where its body begins — the + /// `parameters` position, defending the contract against callers the checker + /// never saw. variadic (`*args` / `**kwargs`) parameters are skipped, and so is + /// any parameter whose source states no type: an unannotated one with no + /// default states nothing, and `x=None` says the argument may be left out rather + /// than that `None` belongs there /// /// the plan is read off the *parameter*, not off its annotation, because a /// default is a written type too. `def f(safe='/')` says `safe` is a `str` /// — the native backend lays the parameter out at that bound and checks it /// at the boundary, and asking the annotation node left the interpreted twin /// silently more permissive than its own compiled form - fn insert_param_guards(&mut self, func: &StmtFunctionDef) { + fn check_parameters(&mut self, func: &'ast StmtFunctionDef) { let params = &func.parameters; - let mut guards: Vec = Vec::new(); + let mut guards = Vec::new(); for pwd in params .posonlyargs .iter() @@ -336,58 +279,24 @@ impl<'a> Soundness<'a> { { let parameter = &pwd.parameter; if let Some(plan) = self.parameter_plan(parameter) { - let guard = self.guard_stmt(parameter.name.as_str(), &plan); - guards.push(guard); + guards.push((parameter.name.as_str(), plan)); } } - if guards.is_empty() { - return; + if !guards.is_empty() { + self.sites.push(Site::Parameters { + function: func, + guards, + }); } - - // insert after a leading docstring (which must stay the first - // statement); mirrors `mutable_defaults`' body-prologue insertion - let docstring_count = usize::from(matches!( - func.body.first(), - Some(Stmt::Expr(e)) if matches!(e.value.as_ref(), Expr::StringLiteral(_)) - )); - let mut text = String::new(); - let insert_at = if let Some(stmt) = func.body.get(docstring_count) { - let at = stmt.range().start(); - let prefix = &self.source[usize::from(line_start(self.source, at))..usize::from(at)]; - if prefix.trim().is_empty() { - // multi-line body: each guard sits at the body indent and - // re-establishes it for the statement that follows - for guard in &guards { - let _ = write!(text, "{guard}\n{prefix}"); - } - } else { - // single-line body (`def f(a: A[int]): ...`) — break the body - // onto its own indented line after the guards - let base = format!("{} ", line_indent(self.source, func.range().start())); - for guard in &guards { - let _ = write!(text, "\n{base}{guard}"); - } - let _ = write!(text, "\n{base}"); - } - at - } else { - // docstring-only body: append the guards after it - let base = format!("{} ", line_indent(self.source, func.range().start())); - for guard in &guards { - let _ = write!(text, "\n{base}{guard}"); - } - func.body[docstring_count - 1].range().end() - }; - self.guards.push((insert_at, vec![Fragment::Lit(text)])); } } -impl<'ast> Visitor<'ast> for Soundness<'_> { +impl<'ast> Visitor<'ast> for Soundness<'_, 'ast> { fn visit_stmt(&mut self, stmt: &'ast Stmt) { match stmt { Stmt::FunctionDef(func) => { if self.positions.parameters { - self.insert_param_guards(func); + self.check_parameters(func); } // track the declared return plan so nested `return`s validate // against the right function's annotation @@ -402,7 +311,7 @@ impl<'ast> Visitor<'ast> for Soundness<'_> { } Stmt::For(for_stmt) => { if self.positions.iterations { - self.wrap_iteration(&for_stmt.iter, &for_stmt.target, for_stmt.is_async); + self.check_iteration(&for_stmt.iter, &for_stmt.target, for_stmt.is_async); } } Stmt::AnnAssign(ann) => { @@ -421,15 +330,15 @@ impl<'ast> Visitor<'ast> for Soundness<'_> { && !self.types.is_field_specifier(value) && let Some(plan) = self.check_plan(&ann.annotation) { - self.wrap_check(value.range(), &plan); + self.check_value(value, plan); } } Stmt::Return(ret) => { if self.positions.returns && let Some(value) = &ret.value - && let Some(plan) = self.return_wrap_plan(value) + && let Some(plan) = self.return_plan(value) { - self.wrap_check(value.range(), &plan); + self.check_value(value, plan); } } // a type-alias value is a type expression; nothing in it executes @@ -444,7 +353,7 @@ impl<'ast> Visitor<'ast> for Soundness<'_> { fn visit_comprehension(&mut self, comprehension: &'ast Comprehension) { if self.positions.iterations { - self.wrap_iteration( + self.check_iteration( &comprehension.iter, &comprehension.target, comprehension.is_async, @@ -493,24 +402,260 @@ impl<'ast> Visitor<'ast> for Soundness<'_> { if self.gated_enabled(operand) { self.consumed.push(operand.range()); if !already_consumed && let Some(plan) = self.check_plan(expr) { - self.wrap_check(expr.range(), &plan); + self.check_value(expr, plan); } } } else if self.gated_enabled(expr) && !self.consumed.contains(&expr.range()) && let Some(plan) = self.check_plan(expr) { - self.wrap_check(expr.range(), &plan); + self.check_value(expr, plan); } if self.positions.arguments && let Expr::Call(call) = expr { - self.wrap_call_arguments(call); + self.check_call_arguments(call); } walk_expr(self, expr); } } +/// a syntax node, as the checks a module makes are looked up by +/// +/// the range alone is not enough: a node the lowering synthesised carries a range +/// it borrowed and no index, and must never be mistaken for the source node it +/// stands in for +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +struct SiteKey { + range: TextRange, + index: u32, +} + +impl SiteKey { + fn of(node: &(impl Ranged + HasNodeIndex)) -> Option { + let index = node.node_index().load().as_u32()?; + Some(Self { + range: node.range(), + index, + }) + } +} + +/// every runtime soundness check a module's transpiled form makes, keyed by the +/// syntax node it is made at +/// +/// the transpiler writes each one into the program it emits, and a native build of +/// the same module asks this for the same answers — so the two builds check the +/// same values against the same types, under the same `--soundness` positions +#[derive(Debug, Default)] +pub struct SoundnessSites { + values: HashMap, + elements: HashMap, + parameters: HashMap>, +} + +impl SoundnessSites { + /// the check made on the value `expr` produces, as it is produced + pub fn value(&self, expr: &Expr) -> Option<&SoundnessCheck> { + self.values.get(&SiteKey::of(expr)?) + } + + /// the check made on each element drawn from `iterable`, as it is drawn + pub fn elements(&self, iterable: &Expr) -> Option<&SoundnessCheck> { + self.elements.get(&SiteKey::of(iterable)?) + } + + /// the checks made on `function`'s own parameters where its body begins, by + /// parameter name, in the order the parameters are written + pub fn parameters(&self, function: &StmtFunctionDef) -> &[(String, SoundnessCheck)] { + SiteKey::of(function) + .and_then(|key| self.parameters.get(&key)) + .map_or(&[], Vec::as_slice) + } +} + +/// decide every soundness check `suite` makes under `positions` +pub fn soundness_sites( + model: &ty_python_semantic::SemanticModel<'_>, + suite: &[Stmt], + positions: SoundnessPositions, +) -> SoundnessSites { + let mut sites = SoundnessSites::default(); + if !positions.any() { + return sites; + } + let mut walker = Soundness::new(model, positions); + walker.visit_body(suite); + for site in walker.sites { + match site { + Site::Value { expr, plan } => { + if let Some(key) = SiteKey::of(expr) { + sites.values.insert(key, plan); + } + } + Site::Elements { iterable, plan, .. } => { + if let Some(key) = SiteKey::of(iterable) { + sites.elements.insert(key, plan); + } + } + Site::Parameters { function, guards } => { + if let Some(key) = SiteKey::of(function) { + let guards = guards + .into_iter() + .map(|(name, plan)| (name.to_owned(), plan)) + .collect(); + sites.parameters.insert(key, guards); + } + } + } + } + sites +} + +/// what the checks a pass decided on are written as +#[derive(Default)] +#[expect( + clippy::struct_excessive_bools, + reason = "independent runtime-helper usage flags, not a state machine" +)] +struct Rendered { + edits: Vec<(TextRange, Vec)>, + /// the entry-guard suites, anchored at the body statement they precede + guards: Vec<(TextSize, Vec)>, + used_iter: bool, + used_aiter: bool, + used_iter_p: bool, + used_aiter_p: bool, + /// any deep parametric check emitted — pulls in `_soundness_parametric` + /// and the `_parametric_is` probe it reuses + used_parametric: bool, +} + +impl Rendered { + /// wrap `source[range]` in `helper(, )`. `trailing` + /// carries its own leading `, ` (e.g. `", str"` or `", A[int], (0,)"`) + fn wrap_call(&mut self, range: TextRange, helper: &str, trailing: &str) { + self.edits.push(( + range, + vec![ + Fragment::Lit(format!("{helper}(")), + Fragment::Src(range), + Fragment::Lit(format!("{trailing})")), + ], + )); + } + + /// wrap `range` in the scalar check (`_soundness_check` / + /// `_soundness_parametric`) named by `plan` + fn wrap_check(&mut self, range: TextRange, plan: &SoundnessCheck) { + match plan { + SoundnessCheck::Isinstance(target) => { + self.wrap_call(range, "_soundness_check", &format!(", {target}")); + } + SoundnessCheck::Parametric { alias, variances } => { + self.used_parametric = true; + self.wrap_call( + range, + "_soundness_parametric", + &format!(", {alias}, {}", variance_tuple(variances)), + ); + } + } + } + + /// wrap an iterable in a validating generator whose form (shallow vs + /// parametric) follows the element's check plan + fn wrap_iteration(&mut self, iterable: &Expr, is_async: bool, plan: &SoundnessCheck) { + let (helper, trailing) = match plan { + SoundnessCheck::Isinstance(target) => { + let helper = if is_async { + self.used_aiter = true; + "_soundness_aiter" + } else { + self.used_iter = true; + "_soundness_iter" + }; + (helper, format!(", {target}")) + } + SoundnessCheck::Parametric { alias, variances } => { + self.used_parametric = true; + let helper = if is_async { + self.used_aiter_p = true; + "_soundness_aiter_p" + } else { + self.used_iter_p = true; + "_soundness_iter_p" + }; + (helper, format!(", {alias}, {}", variance_tuple(variances))) + } + }; + self.wrap_call(iterable.range(), helper, &trailing); + } + + /// the guard statement that validates parameter `name` against `plan` + fn guard_stmt(&mut self, name: &str, plan: &SoundnessCheck) -> String { + match plan { + SoundnessCheck::Isinstance(target) => format!("_soundness_check({name}, {target})"), + SoundnessCheck::Parametric { alias, variances } => { + self.used_parametric = true; + format!( + "_soundness_parametric({name}, {alias}, {})", + variance_tuple(variances) + ) + } + } + } + + /// insert the entry guards at the top of `func`'s body, after any docstring + fn insert_param_guards( + &mut self, + source: &str, + func: &StmtFunctionDef, + checks: &[(&str, SoundnessCheck)], + ) { + let guards: Vec = checks + .iter() + .map(|(name, plan)| self.guard_stmt(name, plan)) + .collect(); + + // insert after a leading docstring (which must stay the first + // statement); mirrors `mutable_defaults`' body-prologue insertion + let docstring_count = usize::from(matches!( + func.body.first(), + Some(Stmt::Expr(e)) if matches!(e.value.as_ref(), Expr::StringLiteral(_)) + )); + let mut text = String::new(); + let insert_at = if let Some(stmt) = func.body.get(docstring_count) { + let at = stmt.range().start(); + let prefix = &source[usize::from(line_start(source, at))..usize::from(at)]; + if prefix.trim().is_empty() { + // multi-line body: each guard sits at the body indent and + // re-establishes it for the statement that follows + for guard in &guards { + let _ = write!(text, "{guard}\n{prefix}"); + } + } else { + // single-line body (`def f(a: A[int]): ...`) — break the body + // onto its own indented line after the guards + let base = format!("{} ", line_indent(source, func.range().start())); + for guard in &guards { + let _ = write!(text, "\n{base}{guard}"); + } + let _ = write!(text, "\n{base}"); + } + at + } else { + // docstring-only body: append the guards after it + let base = format!("{} ", line_indent(source, func.range().start())); + for guard in &guards { + let _ = write!(text, "\n{base}{guard}"); + } + func.body[docstring_count - 1].range().end() + }; + self.guards.push((insert_at, vec![Fragment::Lit(text)])); + } +} + pub(crate) struct SoundnessPass<'src> { positions: SoundnessPositions, source: &'src str, @@ -535,14 +680,28 @@ impl TypeAwarePass for SoundnessPass<'_> { if !self.positions.any() { return; } - let mut inner = Soundness::new(types, self.positions, self.source); + let mut walker = Soundness::new(types, self.positions); for (idx, stmt) in stmts.iter().enumerate() { // an AST-mutation pass re-rendered this statement; edits into its // original source range would be dropped and flagged as leaks if ctx.changed.contains(&idx) { continue; } - inner.visit_stmt(stmt); + walker.visit_stmt(stmt); + } + let mut inner = Rendered::default(); + for site in &walker.sites { + match site { + Site::Value { expr, plan } => inner.wrap_check(expr.range(), plan), + Site::Elements { + iterable, + is_async, + plan, + } => inner.wrap_iteration(iterable, *is_async, plan), + Site::Parameters { function, guards } => { + inner.insert_param_guards(self.source, function, guards); + } + } } if inner.edits.is_empty() && inner.guards.is_empty() { return; diff --git a/crates/ty/tests/by_e2e.rs b/crates/ty/tests/by_e2e.rs index 99e7c2d198..c42826333e 100644 --- a/crates/ty/tests/by_e2e.rs +++ b/crates/ty/tests/by_e2e.rs @@ -4,13 +4,53 @@ use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; use ty_static::EnvVars; -/// a temp directory of this process's own +/// a directory a test builds a project in, which goes when the test passes /// -/// the same reason the `by_build` suites have one: a fixed path under the system temp -/// directory is shared by two concurrent runs, which then overwrite each other between -/// the build and the read -fn cli_root() -> std::path::PathBuf { - std::env::temp_dir().join(format!("by_cli_p{}", std::process::id())) +/// a failing test keeps it, so the project and whatever the command wrote into it can +/// still be read, and names it on stderr, which the test runner prints with the failure +/// +/// the name carries the process id, for the same reason the `by_build` suites' directories +/// do: a fixed path under the system temp directory is shared by two concurrent runs, +/// which then overwrite each other between the build and the read +struct Scratch(PathBuf); + +impl Scratch { + /// an empty place for `name` under the system temp directory + fn new(name: impl std::fmt::Display) -> Self { + let path = std::env::temp_dir().join(format!("{name}_p{}", std::process::id())); + let _ = fs::remove_dir_all(&path); + Self(path) + } +} + +impl std::ops::Deref for Scratch { + type Target = Path; + + fn deref(&self) -> &Path { + &self.0 + } +} + +impl AsRef for Scratch { + fn as_ref(&self) -> &Path { + &self.0 + } +} + +impl Drop for Scratch { + fn drop(&mut self) { + if std::thread::panicking() { + #[expect( + clippy::print_stderr, + reason = "the failure output is where a kept directory has to be named" + )] + { + eprintln!("kept the failing test's directory: {}", self.0.display()); + } + } else { + let _ = fs::remove_dir_all(&self.0); + } + } } fn transpile(source: &str) -> String { @@ -63,9 +103,8 @@ fn run_transpile(source: &str, extra_args: &[&str]) -> String { /// /// Written by each `compile` staging test, which then differ only in what they do /// to the tree afterwards. -fn resource_project(name: &str) -> PathBuf { - let dir = cli_root().join(name); - let _ = fs::remove_dir_all(&dir); +fn resource_project(name: &str) -> Scratch { + let dir = Scratch::new(name); fs::create_dir_all(dir.join("data")).unwrap(); fs::write( dir.join("pyproject.toml"), @@ -249,8 +288,7 @@ fn a_build_with_nothing_to_write_leaves_no_output_directory() { // `by build` created the output directory before it knew whether it had // anything to put in it, so a project with no `.by` files — one whose sources // are all python, say — was left holding an empty `build/` it never asked for - let dir = cli_root().join("by_cli_build_no_litter"); - let _ = fs::remove_dir_all(&dir); + let dir = Scratch::new("by_cli_build_no_litter"); fs::create_dir_all(&dir).unwrap(); fs::write( dir.join("pyproject.toml"), @@ -283,8 +321,7 @@ fn a_second_output_directory_is_not_carried_into_the_first() { // be an output from its arguments; the other is recognised by the // `.by-manifest` it carries, and without that it is carried over as though it // were source, putting a whole copy of one tree inside the other - let dir = cli_root().join("by_cli_two_outputs"); - let _ = fs::remove_dir_all(&dir); + let dir = Scratch::new("by_cli_two_outputs"); fs::create_dir_all(&dir).unwrap(); fs::write( dir.join("pyproject.toml"), @@ -463,8 +500,7 @@ fn compile_emits_only_the_files_it_was_given_and_still_resolves_the_others() { // the database still holds the whole project, because a type imported from a // sibling has to resolve. that is what `lib.py` is here to prove: it is never // compiled, and `wanted.py` still lowers `Point` rather than declining - let dir = cli_root().join("by_cli_only_named"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_cli_only_named"); std::fs::create_dir_all(&dir).unwrap(); std::fs::write( dir.join("pyproject.toml"), @@ -551,7 +587,7 @@ fn compile_writes_each_package_member_at_its_own_place_in_the_output_tree() { // then wrote the same file and the second silently won — and *no* package // member's artefact was importable under the name it had been compiled as, // because a flat `dup.so` can only ever be imported as `dup` - let dir = cli_root().join("by_cli_package_tree"); + let dir = Scratch::new("by_cli_package_tree"); write_package_project(&dir); let out = dir.join("o"); @@ -602,8 +638,7 @@ fn compile_refuses_two_sources_that_would_write_the_same_artifact() { // stem. one artefact would be written twice and only the second kept, which is // the silent loss the tree was meant to end — so it is refused before anything // is written rather than half-performed - let dir = cli_root().join("by_cli_artifact_clash"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_cli_artifact_clash"); std::fs::create_dir_all(dir.join("a-one")).unwrap(); std::fs::create_dir_all(dir.join("b-two")).unwrap(); std::fs::write( @@ -645,8 +680,7 @@ fn compile_declines_a_package_body_whose_package_has_no_importable_name() { // no package to be relative to and its submodules are bound to nothing. a // sibling that *is* nameable from its own directory still compiles, because // its stem really is the only name it could be imported under - let dir = cli_root().join("by_cli_unnameable_package"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_cli_unnameable_package"); std::fs::create_dir_all(dir.join("a-one")).unwrap(); std::fs::write( dir.join("pyproject.toml"), @@ -690,8 +724,7 @@ fn compile_transpiles_the_fallback_with_the_lowering_options_it_was_given() { async def total(s: str, n: int) -> int: return len(s) + n "; - let dir = cli_root().join("by_cli_soundness"); - let _ = std::fs::remove_dir_all(&dir); + let dir = Scratch::new("by_cli_soundness"); std::fs::create_dir_all(&dir).unwrap(); let file = dir.join("sound.by"); std::fs::write(&file, source).unwrap(); @@ -737,8 +770,7 @@ fn compile_reads_whether_to_follow_the_recursion_limit_from_the_project() { // the option is only worth anything if the command a project actually runs reads // it: a module compiled with the count left out says so to the header it includes let emitted = |name: &str, table: &str| -> Option { - let dir = cli_root().join(name); - let _ = fs::remove_dir_all(&dir); + let dir = Scratch::new(name); fs::create_dir_all(&dir).unwrap(); fs::write( dir.join("pyproject.toml"), @@ -792,8 +824,7 @@ fn compile_reads_whether_to_bind_functions_early_from_the_project() { // a compiled call asks whether the name still holds the function unless the project // says the module's functions are never rebound, and then it asks nothing let emitted = |name: &str, table: &str| -> Option { - let dir = cli_root().join(name); - let _ = fs::remove_dir_all(&dir); + let dir = Scratch::new(name); fs::create_dir_all(&dir).unwrap(); fs::write( dir.join("pyproject.toml"), @@ -6247,9 +6278,8 @@ fn init_refuses_to_write_over_a_project() { /// a project for the runtime tests: a package whose modules import one another /// relatively, a subpackage, a hoisted named tuple and a call to a helper, and a /// script in a folder that is no package at all -fn runtime_project(name: &str) -> PathBuf { - let dir = cli_root().join(name); - let _ = fs::remove_dir_all(&dir); +fn runtime_project(name: &str) -> Scratch { + let dir = Scratch::new(format!("by_cli_{name}")); let app = dir.join("src").join("app"); fs::create_dir_all(app.join("sub")).unwrap(); fs::create_dir_all(dir.join("scripts")).unwrap(); diff --git a/crates/ty_python_semantic/src/lib.rs b/crates/ty_python_semantic/src/lib.rs index fb9dba7387..1215ab58f8 100644 --- a/crates/ty_python_semantic/src/lib.rs +++ b/crates/ty_python_semantic/src/lib.rs @@ -26,7 +26,8 @@ use rustc_hash::FxHasher; pub use semantic_model::{ Completion, DjangoLookupArgument, ExpectedStringLiteralCompletion, ExtensionOperatorRewrite, HasDefinition, HasType, ImplicitReceiverReference, MemberDefinition, NameKind, - PreludeDunderReceiver, SemanticModel, basedpython_class_type, + PreludeDunderReceiver, SemanticModel, basedpython_augmented_assignment_type, + basedpython_class_type, }; use std::hash::BuildHasherDefault; pub use suppression::UNUSED_IGNORE_COMMENT; diff --git a/crates/ty_python_semantic/src/semantic_model.rs b/crates/ty_python_semantic/src/semantic_model.rs index 02e669e1e8..e9ac85e0da 100644 --- a/crates/ty_python_semantic/src/semantic_model.rs +++ b/crates/ty_python_semantic/src/semantic_model.rs @@ -2283,6 +2283,21 @@ pub fn basedpython_class_type<'db>( Some(binding_type(model.db, index.try_definition(class)?)) } +/// basedpython: the type an augmented assignment to a name binds it to — the result of the +/// operation, which may be wider than either operand — or `None` where the index holds no +/// single definition for it. +/// +/// the target expression of `x += y` is typed as the value it held before, so this is the +/// only place the bound value's type can be read. the native compiler needs it to choose a +/// representation that holds every value a local is given +pub fn basedpython_augmented_assignment_type<'db>( + model: &SemanticModel<'db>, + assignment: &ast::StmtAugAssign, +) -> Option> { + let index = semantic_index(model.db, model.program_file()); + Some(binding_type(model.db, index.try_definition(assignment)?)) +} + macro_rules! impl_binding_has_ty_def { ($ty: ty) => { impl HasDefinition for $ty { diff --git a/crates/ty_python_semantic/src/types/soundness.rs b/crates/ty_python_semantic/src/types/soundness.rs index bd391f4083..e549f62eb1 100644 --- a/crates/ty_python_semantic/src/types/soundness.rs +++ b/crates/ty_python_semantic/src/types/soundness.rs @@ -26,12 +26,46 @@ use crate::types::visitor::any_over_type; use crate::types::{ClassLiteral, ClassType, FunctionType, KnownClass, Type}; use ty_module_resolver::{KnownModule, file_to_module}; +/// The second argument of a shallow `isinstance` soundness check. +/// +/// Structured rather than spelled, because the transpiler writes it as python source +/// while the native backend resolves each class and tests against it directly; its +/// [`Display`](std::fmt::Display) is the spelling. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum CheckTarget { + /// a class, by the name module scope reaches it through: a binding of the + /// module's own, or a builtin the module leaves unbound + Class(String), + /// `type(None)` + NoneType, + /// any one of several, which `isinstance` accepts as a tuple + AnyOf(Vec), +} + +impl std::fmt::Display for CheckTarget { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Class(name) => f.write_str(name), + Self::NoneType => f.write_str("type(None)"), + Self::AnyOf(parts) => { + f.write_str("(")?; + for (index, part) in parts.iter().enumerate() { + if index > 0 { + f.write_str(", ")?; + } + write!(f, "{part}")?; + } + f.write_str(")") + } + } + } +} + /// How a runtime soundness check validates a value against a target type. #[derive(Debug, Clone, PartialEq, Eq)] pub enum CheckKind { - /// a shallow `isinstance(value, )` check — the string is the - /// second `isinstance` argument (`str`, `(int, type(None))`) - Isinstance(String), + /// a shallow `isinstance(value, )` check (`str`, `(int, type(None))`) + Isinstance(CheckTarget), /// a deep check that validates the base class *and*, when the value /// carries `__orig_class__`, its type arguments. `alias` is the runtime /// spelling of the specialization (`A[int]`); `variances` is one code per @@ -264,7 +298,7 @@ pub(crate) fn runtime_check_target<'db>( file: File, ty: Type<'db>, ) -> Option { - target(db, env, file, ty, 0) + target(db, env, file, ty, 0).map(|target| target.to_string()) } /// The runtime soundness check for a value whose declared type is `ty`. @@ -286,7 +320,7 @@ pub fn runtime_check_plan<'db>( variances: variances.iter().copied().map(variance_code).collect(), }); } - runtime_check_target(db, env, file, ty).map(CheckKind::Isinstance) + target(db, env, file, ty, 0).map(CheckKind::Isinstance) } /// [`runtime_check_plan`] for a *parameter*, whose type may be a hole rather than @@ -385,7 +419,7 @@ fn target<'db>( file: File, ty: Type<'db>, depth: u8, -) -> Option { +) -> Option { if depth > 8 { return None; } @@ -395,7 +429,7 @@ fn target<'db>( match ty { Type::NominalInstance(instance) => { if ty.is_none(db) { - return Some("type(None)".to_owned()); + return Some(CheckTarget::NoneType); } let class = instance.class(db, env); let literal = class.class_literal(db); @@ -406,7 +440,7 @@ fn target<'db>( class_target(db, file, literal) } Type::Union(union) => { - let mut parts: Vec = Vec::new(); + let mut parts: Vec = Vec::new(); for element in union.elements(db) { let part = target(db, env, file, *element, depth + 1)?; if !parts.contains(&part) { @@ -418,7 +452,7 @@ fn target<'db>( 1 => parts.pop(), // isinstance accepts nested tuples, so union parts that are // themselves rendered unions compose without flattening - _ => Some(format!("({})", parts.join(", "))), + _ => Some(CheckTarget::AnyOf(parts)), } } // a TypedDict inhabitant is a plain dict at runtime @@ -437,13 +471,19 @@ fn target<'db>( /// `file`: either the module binds the class's name to this exact class /// (definition or import), or the name is unbound and the class is a /// builtin so the bare name reaches it -fn class_target<'db>(db: &'db dyn Db, file: File, literal: ClassLiteral<'db>) -> Option { +fn class_target<'db>( + db: &'db dyn Db, + file: File, + literal: ClassLiteral<'db>, +) -> Option { let name = literal.name(db).as_str(); match explicit_global_symbol(db, db.program_file(file), name).place { Place::Defined(defined) if defined.ty == Type::ClassLiteral(literal) => { - Some(name.to_owned()) + Some(CheckTarget::Class(name.to_owned())) + } + Place::Undefined if class_is_builtin(db, literal) => { + Some(CheckTarget::Class(name.to_owned())) } - Place::Undefined if class_is_builtin(db, literal) => Some(name.to_owned()), _ => None, } } @@ -455,9 +495,9 @@ fn class_is_builtin<'db>(db: &'db dyn Db, literal: ClassLiteral<'db>) -> bool { /// a builtin referenced by bare name is only trustworthy when the module /// does not rebind that name -fn builtin_target(db: &dyn Db, file: File, name: &str) -> Option { +fn builtin_target(db: &dyn Db, file: File, name: &str) -> Option { match explicit_global_symbol(db, db.program_file(file), name).place { - Place::Undefined => Some(name.to_owned()), + Place::Undefined => Some(CheckTarget::Class(name.to_owned())), Place::Defined(_) => None, } } diff --git a/docs/basedpython/development/compilation/runtime.md b/docs/basedpython/development/compilation/runtime.md index 011f19fca0..2cf1c77c79 100644 --- a/docs/basedpython/development/compilation/runtime.md +++ b/docs/basedpython/development/compilation/runtime.md @@ -331,8 +331,10 @@ base's emitted type took the base's name is standing on an orphaned copy of it, reported — so a whole inheritance family goes in or out together. a generator method's state object and a nested function's closure environment are -each a class of their own, and each captures the `self` it was made from — so each -names the class exactly as any other reader would. neither is in the namespace +each a class of their own. a state object holds the `self` it was made from, and an +environment holds it where a function nested in the method reads a name from further up +than the method — and holding it names the class exactly as any other reader would. +neither is in the namespace under any name and neither is built by anything but the methods of the class it belongs to, so where that class has no type they are never constructed: they are part of the family rather than a reason to refuse. @@ -827,6 +829,67 @@ edits it. a function compiled from a module is also published as a closure over native entry, so a `__code__` it is given has to close over as many names: `mod.scaled.__code__ = (lambda a: a).__code__` raises `ValueError` where python takes it +what tells the function is a function watcher, and cpython hands a watcher every function +in the process rather than the ones it cares about. so once a compiled module that watches +its functions has been imported, each function python makes and frees anywhere — a lambda +or a nested `def` made on every pass of a loop, in any module — costs about 70 more +instructions on 3.13 and about 140 more on 3.14, whether or not it is ever edited + +### a method is not a `function` + +a class holds each method it compiled as a `by.method_descriptor` rather than a +`function`, so `isinstance(C.read, types.FunctionType)` is `False`, and its type is named +`method_descriptor` where python's is named `function`. it binds as a function does, and +what a function says about its `def` — `__module__`, `__qualname__`, `__doc__`, +`__defaults__`, `__kwdefaults__`, `__code__`, `__globals__` and the annotations — is the +interpreted definition's, so `inspect.signature`, `typing.get_type_hints` and +`functools.wraps` read the answers python gives. what it does not answer is +`__closure__`, whose cells would belong to the interpreted definition's class rather than +the compiled one, and an attribute written onto it is kept without changing what a call +runs: + +```python +class Scale: + def get(self, k: int = 3) -> int: + return k + + +# from another module +mod.Scale.get.__closure__ # python: None; compiled: AttributeError +mod.Scale.get.__defaults__ = (10,) +mod.Scale().get() # python: 10; compiled: 3 +``` + +### a dunder a slot answers + +a dunder python calls through a type slot — `__init__`, `__eq__`, `__len__` and the rest +— is not one of those methods. the class holds the `wrapper_descriptor` cpython makes for +the slot, which is what `object.__init__` is too, so it says nothing about the `def` it was +compiled from: it has no `__annotations__`, `__defaults__` or `__code__`, and what reads +those reads the slot's own description instead. `inspect.signature` of the class itself +finds no signature at all: + +```python +class Point: + def __init__(self, x: int, y: int = 0) -> None: + self.x = x + self.y = y + + +# from another module +mod.Point.__init__.__annotations__ # python: {'x': int, 'y': int, 'return': None}; compiled: AttributeError +typing.get_type_hints(mod.Point.__init__) # python: {'x': int, 'y': int, 'return': NoneType}; compiled: {} +inspect.signature(mod.Point.__init__) # python: (self, x: int, y: int = 0) -> None; compiled: (self, /, *args, **kwargs) +inspect.signature(mod.Point) # python: (x: int, y: int = 0) -> None; compiled: ValueError +``` + +publishing a method in the slot's place would answer the first three, but cpython then +stops giving the slot to an interpreted subclass, which makes every construction through +one slower, and `inspect.signature(mod.Point)` would answer `(*args, **kwargs)` rather +than raise. answering the class's signature through a `__signature__` of its own would ignore +what the caller asked for: `inspect.signature(mod.Point, eval_str=True)` would hand back +the annotations a `from __future__ import annotations` module left as strings + ### one module object for the whole process a compiled module keeps its namespace, and every memo of a name in it, in state @@ -950,38 +1013,139 @@ except ValueError as e: traceback.print_exception(e) # python underlines the failing call; compiled does not ``` -### an interpreted subclass that allocates its first instance itself +### writing onto a class nothing extends + +a class the module neither extends nor decorates, with no base and no written `__new__`, +is emitted as an immutable type, which is what lets a call reach its compiled methods +directly. python writes to a class through its type, and an immutable type refuses a +write, a replacement or a `del` of an attribute with `TypeError` — so a class written to +once the module has been imported, from a function of its own or from another module, +raises where python's class takes the write. what the module body writes after the +`class` statement lands on the interpreted definition while the module is still being +built, and the compiled class carries it: + +```python +class Sealed: + def read(self) -> int: + return 1 + + +Sealed.early = lambda self: 2 # both: carried onto the compiled class + + +def patch() -> None: + Sealed.late = lambda self: 3 + + +patch() # python: None; compiled: TypeError: cannot set 'late' attribute of immutable type + + +# from another module +mod.Sealed.other = 5 # python: stored; compiled: TypeError +del mod.Sealed.read # python: deleted; compiled: TypeError +``` + +a class the module extends, or one the source decorates, is a mutable type and takes the +write as python's does + +### an interpreted subclass whose `__init_subclass__` does not chain up a compiled class keeps an `int` field unboxed, and a field no `__init__` has set holds a value no `int` can have. the class's allocator writes that value, and a class made by a `class` statement or `type(...)` is given python's generic allocator instead of its -base's, so the first instance the subclass builds through the compiled class — its -inherited `__new__`, or a written `__new__` the compiled class publishes — hands the -subclass that allocator. a subclass whose own `__new__` calls `object.__new__(cls)` -directly, over a compiled base that wrote a `__new__` of its own, reaches the generic -allocator without passing the compiled class, and until an instance has come through the -compiled class an unset `int` field on what it built reads as `0`: +base's. so the compiled class publishes an `__init_subclass__` that hands every subclass +its allocator as the subclass is made, and then goes on up the chain as a written one +calling `super().__init_subclass__(**kwargs)` does. that entry is in the class's own +dict, where python's class has none: ```python -class Made: - def __new__(cls) -> "Made": - return object.__new__(cls) +class Root: + pass + +class Made(Root): def __init__(self) -> None: self.count = 1 # from another module -class Own(mod.Made): - def __new__(cls): - return object.__new__(cls) +"__init_subclass__" in vars(mod.Made) # python: False; compiled: True +``` +a subclass that writes an `__init_subclass__` of its own and does not call +`super().__init_subclass__()` keeps the hook from its own subclasses. one of those gets +the allocator from the first instance built through the compiled class's `__new__`, and +until then an unset `int` field on an instance `object.__new__(cls)` built reads as `0`: -Own.__new__(Own).count # python: AttributeError; compiled: 0 +```python +# from another module +class Quiet(mod.Made): + def __init_subclass__(cls) -> None: + pass + + +class Below(Quiet): + pass + + +object.__new__(Below).count # python: AttributeError; compiled: 0 ``` -a subclass is not reached by anything the compiled class can run when it is made, so -closing this would mean a test on every read of such a field +### a value nothing verified, with its soundness check turned off + +the interpreted build checks a value whose type nothing verified where it meets a +declared type — an `Any`, a generic call's result, an element read out of a container — +and a compiled module makes each of those [soundness checks](../../features/soundness.md) +in the same place, against the same classes, raising the same `TypeError`. a check the +value's own representation already proves is not made again: an element unboxed out of a +`list[int]` passed the unbox's own test, which raises that `TypeError` itself. +`--soundness` turns positions off for both builds, and a value held as an `object` — a +return handed back as one, an optional, a container, an instance of a class the module +does not lay out — then goes unchecked in both. + +what no position turns off is the test a compiled module needs to hold a value at all. a +local, a parameter or a field declared `int`, `float`, `bool` or `str`, or as an instance +of a class the module lays out, holds that representation, so a value that is not one +raises `TypeError` there where the interpreted build goes on holding it: + +```python +# built with `--soundness none` +from typing import Any + + +def held(v: Any) -> object: + a: int = v + return a + + +held("x") # python: "x"; compiled: TypeError +``` + +a name a class pattern binds from such a field is held the same way. `isinstance` believes +an object's `__class__`, so an object claiming a class it was not made by matches that +class's pattern and has its attributes looked up, and one holding something else under +a field the class keeps as an `int` raises where python binds it: + +```python +class Point: + def __init__(self, x: int) -> None: + self.x = x + + +class Claims: + __class__ = property(lambda self: Point) + x = "five" + + +def read(v: object) -> object: + match v: + case Point(x=x): + return x + return None + + +read(Claims()) # python: "five"; compiled: TypeError +``` ### a complex power held where a `float` is declared @@ -993,14 +1157,122 @@ stores it: ```python def root(a: float) -> float: - x = a # `x: float` in a `.by` module - x **= 0.5 + x: float = a**0.5 return x root(-4.0) # python: (1.2246467991473532e-16+2j); compiled: TypeError ``` +an augmented assignment is not one of these: `x **= 0.5` binds the power's result, and the +local is chosen to hold whatever that can be + +### what a closure keeps alive + +python gives each name a closure captures a cell of its own, so a closure keeps alive the +values of the names it reads and nothing else. a compiled frame keeps every name its +closures capture in one environment they all share, so a closure keeps alive what its +siblings captured too, and a lambda made inside a generator holds the generator's whole +state. no value changes; what can tell is a finalizer or a weak reference, which sees the +release later: + +```python +class Big: + pass + + +def pair(big: Big): + small = 1 + + def a() -> int: + return small + + def b() -> Big: + return big + + return a + + +def gen(big: Big): + small = 1 + yield lambda: small + + +# from another module +import gc, weakref + +big = Big() +alive = weakref.ref(big) +kept = pair(big) # or `next(gen(big))` +del big +gc.collect() +alive() is not None # python: False; compiled: True, for as long as `kept` lives +``` + +a cell for each captured name would close this, and that is a different shape for every +closure a compiled frame makes + +### a nested function's annotations + +a compiled nested function answers `__annotations__`, `__type_params__` and, from 3.14, +`__annotate__` with what python evaluates from the interpreted definition's annotations, +over the values the enclosing frames hold for the names in them. `inspect.signature` and +`typing.get_type_hints` read those, `__code__`, `__defaults__` and `__kwdefaults__`, and +answer as they do for the interpreted definition. what can still tell the two apart: + +```python +def outer(kind: type): + def inner(y: kind) -> kind: + return y + + return inner + + +f = outer(int) +``` + +- on 3.13 the annotations are evaluated when they are first asked for, as 3.14 does, over + the values the enclosing names held where the `def` stood. an annotation that raises or + has an effect does so at that first read rather than at the `def`, and never where + `__annotations__` is written before it is read — see below +- on 3.13 a name the annotations read from an enclosing frame that nothing has bound yet + raises `NameError` naming a free variable, where python raises `UnboundLocalError` +- the closure holds the enclosing names its annotations read, as a 3.14 `__annotate__` + does, so on 3.13 a value the enclosing frame binds to one of them after the `def` lives + as long as the closure, and the value each held at the `def` lives until the annotations + are first read +- on 3.14 `f.__annotate__` is made from the values the enclosing names hold when it is + read, and is a new function each time: a name rebound between reading it and calling it + is seen with its earlier value, and `f.__annotate__ is f.__annotate__` is `False` +- an annotation that reads a type parameter of an enclosing function + (`def outer[T](): def inner(x: T)`) where the module does not compile annotations as + strings, or that names a private name inside a class, raises `RuntimeError` when the + annotations are asked for +- `__defaults__` and `__kwdefaults__` can be read and not written, and `__code__` is the + interpreted definition's code object, which the compiled function does not run + +evaluating the annotations where the `def` stands is a python call, which made a closure +with annotations cost about 3,400 more instructions to make on 3.13 than one without — +the price of a `def` inside a loop. what the `def` takes now is the values alone, so an +annotation with an effect shows when it runs: + +```python +def note(label: str) -> type: + print(label) + return int + + +def outer(): + def inner(y: note("y")) -> None: + pass + + return inner + + +f = outer() # python 3.13: prints `y`; compiled: prints nothing +f.__annotations__ # compiled: prints `y` +``` + ## debugging and inspection - **`#line` directives** in the generated C point at `.by` source. gdb, lldb, From 847c50bcc3c92f97744313caeab07c5a8424ff14 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Fri, 18 Sep 2026 07:25:33 +1000 Subject: [PATCH 08/16] native: outline cold recursion, prove narrowings and fix reference leaks emit the copy a versioned recursion falls back to as a function of its own, drop a type test an earlier guard already answered, memoize global reads and the soundness check per type and class, answer equality from the right operand's type, build displays from borrowed operands, stop leaking a reference on every compiled int return, and empty a moved-from register with its declared value --- crates/by_build/tests/differential.rs | 1063 +++++++++++++++++ crates/by_codegen_c/src/lib.rs | 649 +++++++++- crates/by_ir/src/builder.rs | 1 + crates/by_ir/src/ops.rs | 33 +- crates/by_ir/src/print.rs | 54 +- crates/by_ir/src/rtype.rs | 35 +- crates/by_ir/src/verify.rs | 68 +- crates/by_irbuild/src/lib.rs | 31 +- crates/by_irbuild/src/tests.rs | 8 + crates/by_opt/src/borrow.rs | 1 + crates/by_opt/src/dead_allocations.rs | 1 + crates/by_opt/src/fold.rs | 9 +- crates/by_opt/src/guard_loops.rs | 2 + crates/by_opt/src/infallible.rs | 9 +- crates/by_opt/src/lib.rs | 8 + crates/by_opt/src/proved_narrowings.rs | 279 +++++ crates/by_opt/src/release_temporaries.rs | 1 + crates/by_opt/src/runs_python.rs | 3 + crates/by_opt/src/store_moves.rs | 76 +- crates/by_opt/src/str_concat_int.rs | 3 + crates/by_rt/include/by.h | 503 +++++++- .../development/compilation/runtime.md | 11 + 22 files changed, 2645 insertions(+), 203 deletions(-) create mode 100644 crates/by_opt/src/proved_narrowings.rs diff --git a/crates/by_build/tests/differential.rs b/crates/by_build/tests/differential.rs index db1366b1f4..b0c4399ed8 100644 --- a/crates/by_build/tests/differential.rs +++ b/crates/by_build/tests/differential.rs @@ -1006,6 +1006,16 @@ def _reimported_under_builtins(name, call, **overrides): exec('import ' + name + ' as m\\nout = ' + call, frame) return frame['out'] +# rebind `name` in `module` from a module of its own, rather than from the test's frame: +# a namespace is written to the same way whoever holds the module writes to it, and a memo +# of what that name means has to hear about this one too +def _rebind_from_another_module(module, name, value): + import types + writer = types.ModuleType('_by_writer') + writer.__dict__['target'] = module + writer.__dict__['value'] = value + exec('setattr(target, ' + repr(name) + ', value)', writer.__dict__) + # raising an exception and catching it again, which is the only way an exception class # is asked for the traceback and the frames a `raise` hangs on the instance def _raised_and_caught(kind, *args): @@ -2850,6 +2860,61 @@ fn a_native_recursion_follows_the_recursion_limit() { ); } +#[test] +fn a_recursive_body_keeps_the_copy_it_is_versioned_against_in_a_function_of_its_own() { + // a function versioned at its entry holds two copies of itself: the one the test says + // yes to, which knows its parameters are exactly `int`s and that the names it calls + // still stand, and the body as written beside it for when the test says no. + // + // a C function's prologue saves every register the whole body needs, and a recursion + // runs that prologue once for every level, so the copy that no recursion runs twice + // was making every level save the registers only its slow paths use: `fib` saved ten + // callee-saved registers and 128 bytes of frame where the fast copy alone needs six + // and 64. it goes in a C function of its own, which the test reaches by a call the + // recursion limit does not count — the frame making that call has been counted + // already, and no new one is pushed + // + // both copies still have to answer: an `int` subclass is one the exactness test says + // no to, and a rebound `fib` is a name that no longer stands + let Some(compiled) = agree_in( + "recsplit", + "\ +class Odd(int): + pass + + +def fib(n: int) -> int: + if n < 2: + return n + return fib(n - 1) + fib(n - 2) + + +def outer(n: int) -> int: + return fib(n) +", + &[ + "m.fib(10)", + "m.fib(m.Odd(10))", + "[m.outer(k) for k in (0, 2, 6)]", + "(setattr(m, 'fib', lambda k: 100 + k), m.outer(4))[1]", + ], + false, + by_irbuild::Language::Python, + ) else { + return; + }; + let c = std::fs::read_to_string(compiled.join("by_diff_recsplit.c")) + .expect("the emitted C is kept"); + assert!( + c.contains("__attribute__((noinline)) static ByTagged byu_by_diff_recsplit_fib(ByDepth by_depth, ByTagged r0) {"), + "the copy the test says no to is not a C function of its own:\n{c}" + ); + assert!( + c.contains("else return byu_by_diff_recsplit_fib(by_depth, r0);"), + "the test does not reach it by an uncounted call:\n{c}" + ); +} + #[test] fn a_raised_recursion_limit_still_stops_at_the_stack() { // python frames live on the heap, so python answers these where a native frame @@ -4964,6 +5029,358 @@ def twice(flip: object) -> object: ); } +/// a global whose only use is to be narrowed answers the narrowed value the *name* +/// means, on every read +/// +/// such a read keeps the narrowed value beside the site rather than the object, so what +/// it remembers is one step further from the namespace than the memo of the object is — +/// and every way the name can come to mean something else has to reach it just the same. +/// each call below moves the binding underneath a site that has already run +#[test] +fn a_global_read_only_to_be_narrowed_follows_every_rebinding() { + agree_python( + "globalnarrowed", + "\ +_step = 1 + +def stepped(flip: object) -> int: + total = 0 + i = 0 + while i < 3: + total = total + _step + flip() + i = i + 1 + return total + +def once() -> int: + return _step +", + &[ + // rebound between two reads of the one site + "m.stepped(lambda: m.__dict__.__setitem__('_step', m._step * 10))", + // rebound from outside the module, which is the same write to the namespace + "(_rebind_from_another_module(m, '_step', 7), m.once())[1]", + // deleted: the site has to stop answering rather than go on holding + "(m.__dict__.__setitem__('_step', 4), m.once(), m.__dict__.pop('_step'), \ + repr(_capture(m.once)))[3]", + // and bound again afterwards + "(m.__dict__.__setitem__('_step', 9), m.once())[1]", + // a big int is the case where what is remembered is a pointer into the + // namespace rather than a value copied out of it + "(m.__dict__.__setitem__('_step', 2 ** 80), [m.once(), m.once()])[1]", + // and back to a short one, so the two representations follow each other + "(m.__dict__.__setitem__('_step', 2 ** 80), m.once(), \ + m.__dict__.__setitem__('_step', 6), [m.once(), m.once()])[3]", + ], + ); +} + +/// a global rebound to a value the narrowing does not accept is *reported*, by the very +/// site that was answering the old one +/// +/// this is the shape a memo of the narrowed value could get silently wrong: the site +/// holds a perfectly good `int` from the trip before, and the name now means something +/// that is not one. the generation moved, so the memo is not consulted and the read is +/// made in full — which is what reports it. +/// +/// there is only one leg's behaviour to ask about here. the interpreted twin *returns* +/// the string, because trusting the annotation is what the compiled leg does and does +/// not depend on this memo at all — the same program refuses the same way with the +/// narrowing memo taken out +#[test] +fn a_narrowed_global_rebound_out_of_its_type_is_reported_and_not_answered_stale() { + let Some((python, dir, _)) = built_python( + "globalnarrowedwrong", + "\ +_step = 1 + +def once() -> int: + return _step +", + ) else { + return; + }; + let out = run( + &python, + &dir, + "import by_diff_globalnarrowedwrong as m\n\ + first = m.once()\n\ + m.__dict__['_step'] = 'no'\n\ + try:\n\ + \x20 after = repr(m.once())\n\ + except TypeError as e:\n\ + \x20 after = type(e).__name__\n\ + m.__dict__['_step'] = 4\n\ + print(first, after, m.once())\n", + ); + assert_eq!(out, "1 TypeError 4"); +} + +/// reading and returning a global holds nothing once the value is no longer bound +/// +/// this is the one thing about it that no comparison of answers can see: a build that +/// keeps a reference it never gives back answers correctly for as long as the process +/// lasts, and only the reference count says otherwise. a heap `int` is the case that +/// has a reference to keep at all — a short is a value copied out of the object, and a +/// `double` and a byte likewise. +/// +/// the numbers are *growth over the count before the loop*. while the name is bound, +/// the one reference that stands is the namespace's own entry: the memo of the narrowed +/// global keeps a borrow, and neither the read nor the boxing that hands the value back +/// to python adds anything to it. unbinding the name gives that last one back too. +/// +/// both numbers were wrong when this was first written — 50 rebindings left 51 — and +/// the leak was one reference per *call* rather than per rebinding, in the boxing that +/// ends every compiled `-> int` entry +#[test] +fn reading_a_global_and_handing_it_back_keeps_no_reference() { + let Some((python, dir, _)) = built_python( + "globalnarrowedrefs", + "\ +_step = 1 + +def once() -> int: + return _step +", + ) else { + return; + }; + let out = run( + &python, + &dir, + "import sys\n\ + import by_diff_globalnarrowedrefs as m\n\ + big = 2 ** 80\n\ + before = sys.getrefcount(big)\n\ + for _ in range(50):\n\ + \x20 m.__dict__['_step'] = 0\n\ + \x20 m.once()\n\ + \x20 m.__dict__['_step'] = big\n\ + \x20 m.once()\n\ + print('bound', sys.getrefcount(big) - before)\n\ + for _ in range(50):\n\ + \x20 m.once()\n\ + print('read again', sys.getrefcount(big) - before)\n\ + del m.__dict__['_step']\n\ + print('unbound', sys.getrefcount(big) - before)\n", + ); + assert_eq!(out, "bound 1\nread again 1\nunbound 0"); +} + +/// every way a compiled `-> int` hands a value to python, asked the same question +/// +/// the boxing that ends a wrapper is shared by all of them, so a retain it makes that +/// nothing releases is a leak on a module function, on a method, through a `@property`, +/// through the descriptors a `staticmethod` and a `classmethod` publish, and in the +/// slots python reaches `__hash__` and `__index__` through. it was: every line here +/// read 50 before the boxing was made to take the caller's reference over. +/// +/// a field read is in the list as the case that is *not* one of these — it borrows out +/// of the instance and boxes a fresh reference, which is correct there and is what the +/// entries above were doing wrongly. +/// +/// the other representations a return can carry are asked the same question at the end, +/// because they share that boxing's one decision about who owns what: a `str`, a `list` +/// and a plain object are handed on as they stand, and a `float` and a `bool` are copied +/// out of the object and carry no reference at all +#[test] +fn handing_an_int_back_to_python_keeps_no_reference_on_any_entry() { + let Some((python, dir, _)) = built_python( + "intreturnrefs", + "\ +_name = 'held' +_items = [1] + + +class Marker: + pass + + +_marker = Marker() +_ratio = 1.5 + + +def as_str() -> str: + return _name + + +def as_list() -> list: + return _items + + +def as_object() -> object: + return _marker + + +def as_float() -> float: + return _ratio + + +class Holder: + count: int + + def __init__(self, count: int) -> None: + self.count = count + + def method(self) -> int: + return self.count + + @property + def doubled(self) -> int: + return self.count + + @staticmethod + def stat(v: int) -> int: + return v + + @classmethod + def klass(cls, v: int) -> int: + return v + + def __hash__(self) -> int: + return self.count + + def __index__(self) -> int: + return self.count + + +def plain(v: int) -> int: + return v +", + ) else { + return; + }; + let out = run( + &python, + &dir, + "import sys\n\ + import by_diff_intreturnrefs as m\n\ + big = 2 ** 80\n\ + h = m.Holder(big)\n\ + def drift(call, watched=big):\n\ + \x20 before = sys.getrefcount(watched)\n\ + \x20 for _ in range(50):\n\ + \x20 call()\n\ + \x20 return sys.getrefcount(watched) - before\n\ + for name, call in [\n\ + \x20 ('function', lambda: m.plain(big)),\n\ + \x20 ('method', h.method),\n\ + \x20 ('property', lambda: h.doubled),\n\ + \x20 ('staticmethod', lambda: m.Holder.stat(big)),\n\ + \x20 ('classmethod', lambda: m.Holder.klass(big)),\n\ + \x20 ('hash slot', lambda: hash(h)),\n\ + \x20 ('index slot', lambda: h.__index__()),\n\ + \x20 ('field read', lambda: h.count),\n\ + ]:\n\ + \x20 print(name, drift(call))\n\ + for name, call, watched in [\n\ + \x20 ('str', m.as_str, m._name),\n\ + \x20 ('list', m.as_list, m._items),\n\ + \x20 ('object', m.as_object, m._marker),\n\ + \x20 ('float', m.as_float, m._ratio),\n\ + ]:\n\ + \x20 print(name, drift(call, watched))\n", + ); + assert_eq!( + out, + "function 0\n\ + method 0\n\ + property 0\n\ + staticmethod 0\n\ + classmethod 0\n\ + hash slot 0\n\ + index slot 0\n\ + field read 0\n\ + str 0\n\ + list 0\n\ + object 0\n\ + float 0" + ); +} + +/// the same, for the representations that are copied out of the object outright +#[test] +fn a_global_narrowed_to_a_float_or_a_bool_follows_every_rebinding() { + agree_python( + "globalnarrowedother", + "\ +_scale = 1.5 +_on = True + +def scaled() -> float: + return _scale * 2.0 + +def switched() -> bool: + return _on +", + &[ + "[m.scaled(), m.scaled()]", + "(m.__dict__.__setitem__('_scale', 4.25), [m.scaled(), m.scaled()])[1]", + // twice over, so a site re-armed after one rebinding is asked again + "(m.__dict__.__setitem__('_scale', 4.25), m.scaled(), \ + m.__dict__.__setitem__('_scale', -0.5), [m.scaled(), m.scaled()])[3]", + "[m.switched(), m.switched()]", + "(m.__dict__.__setitem__('_on', False), [m.switched(), m.switched()])[1]", + ], + ); +} + +/// a local of the same name is the local, and a global read before its assignment is +/// unbound — neither is a namespace read that a site could answer +#[test] +fn a_name_a_function_binds_is_never_the_narrowed_global_of_that_name() { + agree_python( + "globalnarrowedshadow", + "\ +_step = 100 + +def shadowed() -> int: + _step = 1 + return _step + +def read_before_assigned() -> int: + total = _step + _step = 2 + return total + +def declared(flip: object) -> int: + global _step + first = _step + flip() + return first + _step +", + &[ + "m.shadowed()", + "repr(_capture(m.read_before_assigned))", + // a site the module's own write moves, read either side of it + "m.declared(lambda: m.__dict__.__setitem__('_step', 5))", + ], + ); +} + +/// a generator resumed after a rebinding reads the binding it is resumed under +/// +/// the narrowed value lives beside the site rather than in the generator's state, so a +/// suspension cannot carry a stale one across: the compare is made again on the resume +#[test] +fn a_generator_resumed_after_a_rebinding_reads_the_new_narrowed_global() { + agree_python( + "globalnarrowedresume", + "\ +_step = 1 + +def steps() -> object: + yield _step + yield _step + yield _step +", + &[ + "(lambda g: [next(g), m.__dict__.__setitem__('_step', 20), next(g), \ + m.__dict__.__setitem__('_step', 300), next(g)])(m.steps())", + ], + ); +} + /// a builtin is resolved through the module the function was defined in, and never /// through whichever frame called it /// @@ -29835,6 +30252,159 @@ def literal(v: object) -> str: assert_eq!(out, "type soundness violation: expected int, got str"); } +#[test] +fn a_class_pattern_asks_for_the_claimed_class_wherever_the_answer_could_differ() { + // `isinstance` decides *against* a subject only after asking it for its `__class__`, + // and a subject whose answer is the pattern's class matches after all. skipping that + // question is sound exactly while the subject's type reads attributes the generic way + // and inherits `object`'s own `__class__`, which answers `type(subject)` and runs + // nothing — so a class pattern that skipped it for every subject would stop matching + // either of these. `Attribute` puts a plain class attribute where the inherited + // descriptor would be, and `Gate`'s metaclass answers `isinstance` for itself + let Some(compiled) = agree_python_built( + "askedclass", + "\ +class Point: + __match_args__ = ('x', 'y') + + def __init__(self, x: int, y: int) -> None: + self.x = x + self.y = y + + +class Attribute: + __class__ = Point + x = 1 + y = 2 + + +class Meta(type): + def __instancecheck__(cls, instance: object) -> bool: + return True + + +class Gate(metaclass=Meta): + __match_args__ = () + + +def point(v: object) -> str: + match v: + case Point(a, b): + return 'point ' + str(a) + ' ' + str(b) + case _: + return 'other' + + +def gate(v: object) -> str: + match v: + case Gate(): + return 'gate' + case _: + return 'other' +", + &[ + "[m.point(v) for v in (m.Point(8, 9), m.Attribute(), m.Gate(), 5, 'text')]", + "[m.gate(v) for v in (m.Gate(), 5, 'text', m.Point(1, 2))]", + ], + ) else { + return; + }; + // the two halves no compiled subject can show. a type that replaces + // `__getattribute__` answers `__class__` however it likes, and python believes it — + // a class that replaces that slot is not one we compile, so the subject is written + // here. and an entry in the *instance* dict does not displace `object.__class__`, + // because that is a data descriptor and the generic lookup takes one of those before + // it reads the dict: a subject carrying `__class__` there is not a `Point`, for + // python and for the skipped question alike + let Some((python, _)) = environment() else { + return; + }; + let out = run( + &python, + &compiled, + "import by_diff_askedclass as m\n\ + class Overrides:\n\ + \x20 x = 3\n\ + \x20 y = 4\n\ + \x20 def __getattribute__(self, name):\n\ + \x20 if name == '__class__':\n\ + \x20 return m.Point\n\ + \x20 return object.__getattribute__(self, name)\n\ + class Shadows:\n\ + \x20 x = 6\n\ + \x20 y = 7\n\ + s = Shadows()\n\ + s.__dict__['__class__'] = m.Point\n\ + print(m.point(Overrides()))\n\ + print(isinstance(s, m.Point), m.point(s))\n", + ); + assert_eq!(out, "point 3 4\nFalse other"); +} + +#[test] +fn a_class_pattern_tests_the_layout_once() { + // the layout test decides whether the attributes are read at their offsets, and the + // subject is then narrowed to that layout so that they can be — which used to ask the + // very question the test above it had just answered. nothing between the two can run + // python, so the narrowing is the static type and nothing else + let Some(compiled) = agree_python_built( + "onelayouttest", + "\ +class Point: + __match_args__ = ('x', 'y') + + def __init__(self, x: int, y: int) -> None: + self.x = x + self.y = y + + +def classify(v: object) -> int: + match v: + case Point(a, b): + return a + b + case _: + return -1 +", + &["[m.classify(v) for v in (m.Point(4, 5), 3, 'x')]"], + ) else { + return; + }; + let c = std::fs::read_to_string(compiled.join("by_diff_onelayouttest.c")) + .expect("the emitted C is kept"); + let body = one_function(&c, "_classify("); + assert!( + body.contains("PyObject_TypeCheck("), + "the layout test is gone:\n{body}" + ); + for again in ["By_CheckInstance(", "By_UnboxInstance("] { + assert!( + !body.contains(again), + "`{again}` asks the layout test again:\n{body}" + ); + } +} + +/// the C of the one function whose symbol ends `tail`, from its signature to the brace +/// that closes it +/// +/// a whole-module `contains` says nothing about *which* function a text is in, and every +/// emitted module has several +fn one_function<'a>(c: &'a str, tail: &str) -> &'a str { + // the module declares every function before it defines any, so the definition is + // the occurrence whose line opens a body + let at = c + .match_indices(tail) + .map(|(at, _)| at) + .find(|at| { + let line = c[*at..].lines().next().unwrap_or_default(); + line.trim_end().ends_with('{') + }) + .unwrap_or_else(|| panic!("no definition of `{tail}` in:\n{c}")); + let start = c[..at].rfind('\n').map_or(0, |line| line + 1); + let end = c[start..].find("\n}\n").expect("the closing brace") + start; + &c[start..end] +} + #[test] fn a_class_pattern_over_a_base_agrees() { // a class another one in the module extends is emitted as a mutable heap type: @@ -31715,6 +32285,425 @@ def unequal(a: object, b: object) -> bool: ); } +#[test] +fn an_equality_whose_left_operand_has_no_comparison_of_its_own_asks_the_right_one() { + // a class that writes no `__eq__` inherits `object`'s, which answers nothing but + // identity — so for every other pair python's answer comes from the *right* operand's + // type, and is identity when that side declines too. `Eager` answers, `Refuses` hands + // back `NotImplemented`, and `SameInt` is the shape that makes the rule about the + // right operand rather than about the left: it inherits `object`'s comparison and is + // still an `int`, so `int`'s comparison — reached from the right — compares the two + // by value and says they are equal. + // + // the orderings are not part of this: python's answer where neither side has one is a + // `TypeError`, not a verdict + agree_python( + "rightcompares", + "\ +class Plain: + def __init__(self, n: int) -> None: + self.n = n + + +class Eager: + def __eq__(self, other: object) -> bool: + return True + + def __hash__(self) -> int: + return 0 + + +class Refuses: + def __eq__(self, other: object) -> object: + return NotImplemented + + def __hash__(self) -> int: + return 0 + + +class SameInt(int): + __eq__ = object.__eq__ + __hash__ = object.__hash__ + + +class SameStr(str): + __eq__ = object.__eq__ + __hash__ = object.__hash__ + + +class OddInt(int): + def __eq__(self, other: object) -> bool: + return True + + def __hash__(self) -> int: + return 0 + + +def equal(a: object, b: object) -> int: + if a == b: + return 1 + return 0 + + +def unequal(a: object, b: object) -> int: + if a != b: + return 1 + return 0 + + +def below(a: object, b: object) -> int: + if a < b: + return 1 + return 0 + + +def below_says(a: object, b: object) -> str: + try: + return str(below(a, b)) + except TypeError: + # the wording names the type, and a compiled type carries its module in that + # name — what is being asserted here is that the ordering refuses at all + return 'refused' + + +# the same comparisons again as *values*, where what a comparison answered is handed +# on rather than collapsed to a bit + + +def equal_value(a: object, b: object) -> object: + return a == b + + +def unequal_value(a: object, b: object) -> object: + return a != b + + +def below_value(a: object, b: object) -> object: + return a < b + + +def below_value_says(a: object, b: object) -> str: + try: + return repr(below_value(a, b)) + except TypeError: + return 'refused' +", + &[ + // the left operand has only identity to offer and the right refuses it + "[m.equal(m.Plain(1), v) for v in (0, 'a', 1.5, (), m.Plain(1))]", + "[m.unequal(m.Plain(1), v) for v in (0, 'a', 1.5, (), m.Plain(1))]", + // the same instance on both sides, which identity does answer + "(lambda o: [m.equal(o, o), m.unequal(o, o)])(m.Plain(1))", + // the right operand answers, or declines and leaves identity to decide + "[m.equal(m.Plain(1), m.Eager()), m.unequal(m.Plain(1), m.Eager())]", + "[m.equal(m.Plain(1), m.Refuses()), m.unequal(m.Plain(1), m.Refuses())]", + // a subclass that gave its own comparison away and is still an int, or a str + "[m.equal(m.SameInt(3), 3), m.unequal(m.SameInt(3), 3), m.equal(m.SameInt(3), 4)]", + "[m.equal(m.SameStr('a'), 'a'), m.unequal(m.SameStr('a'), 'a')]", + // and a subclass that kept a comparison of its own, on either side: an `int` + // it is not the exact type of has replaced what `int` would have answered + "[m.equal(m.OddInt(3), 4), m.equal(4, m.OddInt(3)), m.unequal(m.OddInt(3), 4)]", + // `bool` is an `int`, and a float compares equal to one + "[m.equal(v, 0) for v in (True, False, 0.0, 1.0, 0)]", + "[m.equal(v, 1) for v in (True, False, 1.0, 1)]", + // ints of exactly that type, at and past the width one is stored inline at + "[m.equal(3, 3), m.unequal(3, 4), m.below(3, 4), m.below(4, 3)]", + "[m.equal(2 ** 70, 2 ** 70), m.equal(2 ** 70 + 1, 2 ** 70), m.below(2 ** 70, 2 ** 71)]", + "[m.equal(2 ** 70, 3), m.below(3, 2 ** 70), m.below(2 ** 70, 3)]", + // a NaN is not equal to itself, however it is reached + "(lambda x: [m.equal(x, x), m.unequal(x, x)])(float('nan'))", + // an ordering with nothing on either side to answer it is python's refusal + "[m.below_says(m.Plain(1), 0), m.below_says(m.Plain(1), m.Plain(2)), m.below_says(1, 2)]", + // every one of those again as a value, where the answer is the object rather + // than the truth of it + "[m.equal_value(m.Plain(1), v) for v in (0, 'a', 1.5, (), m.Plain(1))]", + "[m.unequal_value(m.Plain(1), v) for v in (0, 'a', 1.5, (), m.Plain(1))]", + "(lambda o: [m.equal_value(o, o), m.unequal_value(o, o)])(m.Plain(1))", + "[m.equal_value(m.Plain(1), m.Eager()), m.unequal_value(m.Plain(1), m.Eager())]", + "[m.equal_value(m.Plain(1), m.Refuses()), m.unequal_value(m.Plain(1), m.Refuses())]", + "[m.equal_value(m.SameInt(3), 3), m.unequal_value(m.SameInt(3), 3), m.equal_value(m.SameInt(3), 4)]", + "[m.equal_value(m.SameStr('a'), 'a'), m.unequal_value(m.SameStr('a'), 'a')]", + "[m.equal_value(m.OddInt(3), 4), m.equal_value(4, m.OddInt(3)), m.unequal_value(m.OddInt(3), 4)]", + "[m.equal_value(v, 0) for v in (True, False, 0.0, 1.0, 0)]", + "[m.equal_value(v, 1) for v in (True, False, 1.0, 1)]", + "[m.equal_value(3, 3), m.unequal_value(3, 4), m.below_value(3, 4), m.below_value(4, 3)]", + "[m.equal_value(2 ** 70, 2 ** 70), m.equal_value(2 ** 70 + 1, 2 ** 70), m.below_value(2 ** 70, 2 ** 71)]", + "[m.equal_value(2 ** 70, 3), m.below_value(3, 2 ** 70), m.below_value(2 ** 70, 3)]", + "(lambda x: [m.equal_value(x, x), m.unequal_value(x, x)])(float('nan'))", + "[m.below_value_says(m.Plain(1), 0), m.below_value_says(m.Plain(1), m.Plain(2)), m.below_value_says(1, 2)]", + // `is` against the singletons python answers with, so an answer built rather + // than handed on would be caught: a pair of ints, the identity python falls + // through to when the right operand declines, and the very object the right + // operand answered with when it does not + "[m.equal_value(3, 3) is True, m.equal_value(3, 4) is False, m.unequal_value(3, 4) is True]", + "[m.equal_value(m.Plain(1), m.Refuses()) is False, m.unequal_value(m.Plain(1), m.Refuses()) is True]", + "m.equal_value(m.Plain(1), m.Eager()) is True", + ], + ); +} + +#[test] +fn a_recursion_through_equality_stops_where_python_stops_it() { + // asking the right operand's type directly is one call python makes that a compiled + // module does not, and `PyObject_RichCompare` enters a recursion of its own before it + // makes that call. so a recursion that goes round through `==` is counted a level + // differently from the interpreter's — and a comparison that reaches python again is + // exactly the shape where that could show. + // + // it does not, because the entry that is skipped is against the C stack and what stops + // a recursion at the limit is the frame count, which a compiled module takes and gives + // back for the same frames python does. both legs therefore stop at the same depth + // with the same message. what the skipped entry costs is one level of headroom, once, + // in a program that has raised the limit far enough for the stack to run out first — + // and that is written down in + // docs/basedpython/development/compilation/runtime.md#how-deep-a-recursion-goes + agree_python( + "eqrecursion", + "\ +import sys + +_depth = 0 + + +class Plain: + pass + + +class Chain: + # `Plain` has no comparison of its own, so each round is answered by asking this + # type — which compares again + def __eq__(self, other: object) -> bool: + global _depth + _depth += 1 + return Plain() == self + + def __hash__(self) -> int: + return 0 + + +def as_condition(limit: int) -> object: + global _depth + sys.setrecursionlimit(limit) + _depth = 0 + try: + if Plain() == Chain(): + return ('true', _depth) + return ('false', _depth) + except RecursionError as e: + return ('RecursionError', _depth, str(e)) + + +def as_value(limit: int) -> object: + global _depth + sys.setrecursionlimit(limit) + _depth = 0 + try: + return ('answer', _depth, repr(Plain() == Chain())) + except RecursionError as e: + return ('RecursionError', _depth, str(e)) +", + &[ + // the depth and the wording, at the limit a program that sets none has + "m.as_condition(1000)", + "m.as_value(1000)", + // and at a lower one, where the count runs out sooner and the stack is + // nowhere near + "m.as_condition(200)", + "m.as_value(200)", + ], + ); +} + +#[test] +fn a_display_holds_exactly_one_reference_to_each_thing_it_was_given() { + // a `list` and a `tuple` write their elements with a store that takes a reference + // over, and a `set` and a `dict` insert through a call that takes a reference of its + // own — two ways of arriving at one reference held per element. how many the display + // takes is not otherwise observable, so it is asked directly: the count before the + // display is built against the count after. + // + // an element that is a *temporary*, built into the display and read nowhere else, + // hands its own reference to the slot rather than having a second one taken. that is + // the same one reference, reached without making and dropping one — and getting it + // wrong either leaks the element or frees it while the display still names it + agree_python( + "displayrefs", + "\ +import sys + + +class Tag: + def __init__(self, n: int) -> None: + self.n = n + + def __hash__(self) -> int: + return self.n + + def __eq__(self, other: object) -> bool: + return isinstance(other, Tag) and other.n == self.n + + def __repr__(self) -> str: + return 'Tag(' + str(self.n) + ')' + + +# each answer is the references the display took, then the length it was built to, so +# the display is read rather than left for a pass to decide nobody wanted it + + +def dict_takes(value: object) -> str: + before = sys.getrefcount(value) + held = {'key': value, value: 'value'} + return str(sys.getrefcount(value) - before) + '/' + str(len(held)) + + +def set_takes(value: object) -> str: + before = sys.getrefcount(value) + held = {value} + return str(sys.getrefcount(value) - before) + '/' + str(len(held)) + + +def tuple_takes(value: object) -> str: + before = sys.getrefcount(value) + held = (value, value) + return str(sys.getrefcount(value) - before) + '/' + str(len(held)) + + +def list_takes(value: object) -> str: + before = sys.getrefcount(value) + held = [value] + return str(sys.getrefcount(value) - before) + '/' + str(len(held)) + + +def fresh_dict(n: int) -> object: + return {Tag(n): Tag(n + 1)} + + +def fresh_set(n: int) -> object: + return {Tag(n), Tag(n + 1)} + + +def fresh_tuple(n: int) -> object: + return (Tag(n), Tag(n + 1)) + + +def fresh_list(n: int) -> object: + return [Tag(n), Tag(n + 1)] + + +def pair(value: object) -> object: + return (value, value) + + +def keyed(value: object) -> object: + return {'key': value} + + +_gone: list[int] = [] + + +class Watched: + def __init__(self, n: int) -> None: + self.n = n + + def __del__(self) -> None: + _gone.append(self.n) + + +def build_and_drop(n: int) -> int: + held = (Watched(n), [Watched(n + 1)], {Watched(n + 2): 1}, {Watched(n + 3)}) + return len(held) +", + &[ + // an object named twice in one display is held twice, once once + "[m.dict_takes(m.Tag(1)), m.set_takes(m.Tag(1)), m.tuple_takes(m.Tag(1)), m.list_takes(m.Tag(1))]", + // each display built entirely from temporaries, read back after the frame + // that made them has gone + "sorted((k.n, v.n) for k, v in m.fresh_dict(3).items())", + "sorted(t.n for t in m.fresh_set(3))", + "[t.n for t in m.fresh_tuple(3)]", + "[t.n for t in m.fresh_list(3)]", + // and a display outliving every other reference to what it holds + "(lambda out: (out[0], out[1][0].n))(_outlives(lambda: m.Tag(9), m.pair))", + "(lambda out: (out[0], out[1]['key'].n))(_outlives(lambda: m.Tag(9), m.keyed))", + // the other direction, which counting the references a *parameter* gains + // cannot show: a display built from temporaries and then let go of releases + // each of them exactly once, so each element's `__del__` runs, and runs once + "(m.build_and_drop(1), sorted(m._gone))", + ], + ); +} + +#[test] +fn a_display_that_took_a_temporary_over_leaves_nothing_for_the_error_exit() { + // a display that takes an element's reference over leaves the register that held it + // empty, and the reason is the *failure* path rather than the ordinary one. a frame + // that fails releases every register some path to its error exit could have written + // — not only the ones still live — so a register the display already handed on is + // released a second time there, and the element is freed while the display goes on + // naming it. + // + // the success path cannot show this: a temporary's last use is the display, so + // nothing releases it again and a register left naming the element is harmless. the + // difference only appears once the frame goes on to fail with the display still + // reachable from somewhere the caller can read — which is what `box` is for + agree_python( + "displayerrorexit", + "\ +import sys + + +def held(value: object) -> object: + return value + + +# each of these leaves the display in `box` and then fails, by the two ways a frame +# reaches its error exit: a `raise` it wrote, and an operation that failed on its own + + +def list_then_raises(box: list, value: object) -> None: + box.append([held(value)]) + raise ValueError('boom') + + +def tuple_then_raises(box: list, value: object) -> None: + box.append((held(value),)) + raise ValueError('boom') + + +def list_then_divides(box: list, value: object, d: int) -> int: + box.append([held(value)]) + return 1 // d + + +def nested_then_raises(box: list, value: object) -> None: + box.append([[held(value)], (held(value),)]) + raise ValueError('boom') + + +# the references `value` gained once the call has failed and its traceback has gone: +# one for every slot of a surviving display that names it +def kept_through(fn, value: object, *rest: object) -> int: + box: list = [] + before = sys.getrefcount(value) + try: + fn(box, value, *rest) + except (ValueError, ZeroDivisionError): + pass + return sys.getrefcount(value) - before +", + &[ + "m.kept_through(m.list_then_raises, ['seed'])", + "m.kept_through(m.tuple_then_raises, ['seed'])", + "m.kept_through(m.list_then_divides, ['seed'], 0)", + "m.kept_through(m.nested_then_raises, ['seed'])", + // and the element is still the object the display was built from, rather + // than whatever came to stand at a freed address + "(lambda box, v: (_capture(m.list_then_raises, box, v), box[0][0] is v))([], ['seed'])", + "(lambda box, v: (_capture(m.nested_then_raises, box, v), box[0][0][0] is v, box[0][1][0] is v))([], ['seed'])", + ], + ); +} + #[test] fn membership_and_index_keep_the_identity_shortcut() { // `x in xs` and `xs.index(x)` are not comparisons. python answers the first with @@ -33253,6 +34242,80 @@ def label_of(shape: Shape) -> str: ); } +#[test] +fn a_dispatch_that_matched_tests_the_receiver_once() { + // `By_MethodStands` compares the receiver's type with the class's, which is the + // exact question the narrowing behind it used to ask again before handing the + // receiver to the body. the licence test stays; the second one is the static type. + // + // under the licence re-check mode there is an attribute lookup in between, and a + // lookup compares keys with whatever `__eq__` they have — so the answer from a + // moment before is no longer an answer, and the narrowing checks for itself again. + // the mode is not what a build ships, and this says what it costs where it is on + let Some((python, toolchain)) = environment() else { + return; + }; + for recheck in [false, true] { + let tag = format!("by_diff_onedispatch_{}", u8::from(recheck)); + let dir = Scratch::new(&tag); + let built = match build_source( + "\ +class Shape: + def __init__(self, size: int): + self.size = size + + def area(self) -> int: + return self.size + + +class Square(Shape): + def area(self) -> int: + return self.size * self.size + + +def area_of(shape: Shape) -> int: + return shape.area() +", + tag.as_str(), + &toolchain, + &dir, + &Options { + language: by_irbuild::Language::Python, + recheck_licences: recheck, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + assert!(built.declined.is_empty(), "declined: {:?}", built.declined); + let c = std::fs::read_to_string(&built.artifact.source).expect("the generated C is read"); + let body = one_function(&c, "_area_of("); + assert!( + body.contains("By_MethodStands("), + "the licence test is gone:\n{body}" + ); + assert_eq!( + body.contains("By_CheckInstance("), + recheck, + "the receiver behind the licence, re-check {recheck}:\n{body}" + ); + // and the answers are the same either way + let out = run( + &python, + &dir, + &format!( + "import {tag} as m\nprint([m.area_of(s) for s in (m.Shape(5), m.Square(5))])\n" + ), + ); + assert_eq!(out, "[5, 25]"); + } +} + #[test] fn an_override_reached_through_a_subscript_agrees() { // the receiver is an element of a `list[Shape]` rather than a name, so it is diff --git a/crates/by_codegen_c/src/lib.rs b/crates/by_codegen_c/src/lib.rs index 4ec03219b1..d7618751e4 100644 --- a/crates/by_codegen_c/src/lib.rs +++ b/crates/by_codegen_c/src/lib.rs @@ -5787,7 +5787,7 @@ fn unbox_expr(ty: &RType, expr: &str) -> String { /// wrapper and one that leaks on every call fn box_owned(ty: &RType, expr: &str) -> String { match ty { - RType::Primitive(Primitive::Int) => format!("By_BoxInt({expr})"), + RType::Primitive(Primitive::Int) => format!("By_BoxIntOwned({expr})"), RType::Primitive(Primitive::Float) => format!("By_BoxFloat({expr})"), RType::Primitive(Primitive::Bool | Primitive::Bit) => format!("By_BoxBool({expr})"), RType::Primitive(Primitive::None) => "By_BoxNone()".to_string(), @@ -6082,9 +6082,156 @@ fn owned_registers(function: &Function) -> Vec { owned } +/// the copy of a function versioned at its entry that the test says no to, when it is +/// worth a C function of its own +/// +/// a function on a cycle of native calls runs its prologue once for every level of the +/// recursion, and a C function's prologue saves every register the *whole* body needs. +/// the copy the test says no to is the original body — every slow path the function has +/// — and no recursion ever runs it twice, so leaving it beside the fast copy makes every +/// level pay for the registers only it uses. `fib` saved ten callee-saved registers and +/// 128 bytes of frame where the fast copy alone needs six and 64 +struct Outlined { + /// the block holding the version test + test: BlockId, + /// the block the test says no to, which the outlined function starts at + entry: BlockId, + /// every block reachable from it + blocks: HashSet, +} + +/// whether the copy the version test says no to can be lifted out, and which blocks it is +/// +/// the two copies have to be able to stand apart: nothing the fast copy runs may reach +/// the slow one, and the slow one may read nothing the test block wrote. both hold for +/// the shape [`by_opt`]'s entry versioning builds — the slow copy *is* the function as it +/// was written, entered where it was entered — and both are asked rather than assumed, +/// because a later pass is free to join the two up again +fn outlined(module: &ModuleIr, function: &Function) -> Option { + // only a body that pays its prologue once a level wins more than the call costs + if !is_recursive(function) || resumed_body(module, function).is_some() { + return None; + } + // the parameters are handed on as the caller's own. a body that rebinds one holds a + // reference of its own to it, which the half that made the call would have to release + // around the call, and it is left as it stands instead + let owned = owned_registers(function); + if owned.iter().take(function.param_count).any(|owned| *owned) { + return None; + } + let entry = function.blocks.get(Function::entry().index())?; + if entry.ops.iter().any(|op| !matches!(op, Op::Line { .. })) || entry.error_target.is_some() { + return None; + } + let Terminator::Goto(test) = entry.terminator else { + return None; + }; + let block = function.blocks.get(test.index())?; + let mut ops = block.ops.iter().filter(|op| !matches!(op, Op::Line { .. })); + let Some(Op::LoopGuardsHold { dest: answer, .. }) = ops.next() else { + return None; + }; + if ops.next().is_some() || block.error_target.is_some() { + return None; + } + let Terminator::Branch { + cond: Value::Register(cond), + then_block, + else_block, + } = block.terminator + else { + return None; + }; + if cond != *answer { + return None; + } + let cold = reachable_blocks(function, else_block); + let mut hot = reachable_blocks(function, then_block); + hot.insert(Function::entry()); + hot.insert(test); + if cold.is_empty() || cold.intersection(&hot).next().is_some() { + return None; + } + Some(Outlined { + test, + entry: else_block, + blocks: cold, + }) +} + +/// every block control can reach from `from`, the exception edges included +fn reachable_blocks(function: &Function, from: BlockId) -> HashSet { + let mut seen = HashSet::from([from]); + let mut queue = vec![from]; + while let Some(id) = queue.pop() { + let Some(block) = function.blocks.get(id.index()) else { + continue; + }; + for next in block.successors() { + if seen.insert(next) { + queue.push(next); + } + } + } + seen +} + +/// the symbol the copy the version test says no to is emitted under +fn unversioned_symbol(module: &ModuleIr, function: &Function) -> String { + format!( + "byu{}", + function + .native_symbol(module.name.dotted()) + .trim_start_matches("by") + ) +} + +/// which of a split function's two halves is being written +#[derive(Clone, Copy, PartialEq, Eq)] +enum Half { + /// the whole function, or the copy the version test says yes to + Hot, + /// the copy the version test says no to, in a C function of its own + Cold, +} + fn emit_function(module: &ModuleIr, function: &Function) -> String { + match outlined(module, function) { + Some(split) => { + let mut out = emit_half(module, function, Some(&split), Half::Cold); + out.push_str(&emit_half(module, function, Some(&split), Half::Hot)); + out + } + None => emit_half(module, function, None, Half::Hot), + } +} + +fn emit_half( + module: &ModuleIr, + function: &Function, + split: Option<&Outlined>, + half: Half, +) -> String { let mut out = line_directive(module, function.range); - let _ = writeln!(out, "{} {{", signature(module, function)); + match half { + Half::Cold => { + let params = std::iter::once("ByDepth by_depth".to_string()) + .chain(function.params().iter().enumerate().map(|(index, decl)| { + format!("{} {}", ctype(module, &decl.ty), local(RegisterId(index))) + })) + .collect::>() + .join(", "); + let _ = writeln!( + out, + "__attribute__((noinline)) static {} {}({params}) {{", + ctype(module, &function.ret), + unversioned_symbol(module, function) + ); + } + Half::Hot => { + let _ = writeln!(out, "{} {{", signature(module, function)); + } + } let owned = owned_registers(function); for (index, decl) in function.registers.iter().enumerate() { @@ -6132,6 +6279,11 @@ fn emit_function(module: &ModuleIr, function: &Function) -> String { if !function.registers.is_empty() { out.push('\n'); } + // the blocks keep the numbers they have in the function, so the copy the test says no + // to is rarely the first of them written out, and the half starts by jumping to it + if let (Half::Cold, Some(split)) = (half, split) { + let _ = writeln!(out, " goto b{};", split.entry.0); + } let entered = written_on_entry(function); // what the shared error label has to release, accumulated as the body is @@ -6141,6 +6293,11 @@ fn emit_function(module: &ModuleIr, function: &Function) -> String { let mut sites = TracebackSites::default(); for (index, block) in function.blocks.iter().enumerate() { + if let Some(split) = split + && split.blocks.contains(&BlockId(index)) != (half == Half::Cold) + { + continue; + } // a label immediately before a declaration is invalid in C89 and merely // ugly later; the empty statement keeps it valid everywhere let _ = writeln!(out, "b{index}: ;"); @@ -6209,6 +6366,7 @@ fn emit_function(module: &ModuleIr, function: &Function) -> String { if unset.is_empty() && let Some(taken) = block.ops.get(at..at + 3) && let Some(fused) = borrowed_element_unbox(module, function, taken, edge) + .or_else(|| unboxed_global(function, taken, edge)) { fragment.push_str(&fused); sites.note(&fragment, edge); @@ -6281,12 +6439,35 @@ fn emit_function(module: &ModuleIr, function: &Function) -> String { } let edge = sites.edge(module, line, in_generator_expression, block.error_target); let mut fragment = guard_unassigned(function, &block.terminator.operands(), edge); - fragment.push_str(&emit_terminator( - module, - function, - &block.terminator, - block.owned_at_exit.as_deref(), - )); + // the half the test says yes to keeps the test, and reaches the other half by + // calling it. the call is not counted against the recursion limit: the frame this + // body is running in has already been counted, and the call makes no new one + match (split, &block.terminator) { + ( + Some(split), + Terminator::Branch { + cond, then_block, .. + }, + ) if half == Half::Hot && split.test == BlockId(index) => { + let arguments = std::iter::once("by_depth".to_string()) + .chain((0..function.param_count).map(|index| local(RegisterId(index)))) + .collect::>() + .join(", "); + let _ = writeln!( + fragment, + " if ({}) goto b{}; else return {}({arguments});", + value_expr(cond), + then_block.0, + unversioned_symbol(module, function), + ); + } + _ => fragment.push_str(&emit_terminator( + module, + function, + &block.terminator, + block.owned_at_exit.as_deref(), + )), + } sites.note(&fragment, edge); if let Some(at_error) = at_error.as_mut() && edge.target.is_none() @@ -6322,7 +6503,7 @@ fn emit_function(module: &ModuleIr, function: &Function) -> String { } out.push_str("}\n"); - if is_recursive(function) { + if is_recursive(function) && half == Half::Hot { out.push_str(&emit_counted_entry(module, function)); } out @@ -7997,7 +8178,7 @@ fn emit_op(module: &ModuleIr, function: &Function, op: &Op, error_target: ErrorE } Op::ObjectRichCompare { dest, op, lhs, rhs } => { let expr = format!( - "PyObject_RichCompare({}, {}, {})", + "By_ObjRichCompare({}, {}, {})", value_expr(lhs), value_expr(rhs), rich_compare_op(*op) @@ -8255,7 +8436,24 @@ fn emit_op(module: &ModuleIr, function: &Function, op: &Op, error_target: ErrorE }; assign_checked(module, function, *dest, &call, error_target) } - Op::Unbox { dest, src, to } => { + Op::Unbox { + dest, + src, + to, + proved, + } => { + // a test standing over the operation has already asked what the check would + // ask, so what is left of the narrowing is the static type: the value keeps + // its representation, and a borrowed destination keeps the reference too + if *proved { + let ctype = ctype(module, to); + return if function.register(*dest).is_some_and(|decl| decl.borrowed) { + format!(" {} = ({ctype}){};\n", local(*dest), value_expr(src)) + } else { + let taken = format!("({ctype})Py_NewRef({})", value_expr(src)); + assign_owned(module, function, *dest, &taken) + }; + } if function.register(*dest).is_some_and(|decl| decl.borrowed) && let Some(check) = check_expr(module, to, &value_expr(src)) { @@ -8317,7 +8515,7 @@ fn emit_op(module: &ModuleIr, function: &Function, op: &Op, error_target: ErrorE }; if moves.contains(&index) { if let Value::Register(id) = item { - let _ = writeln!(out, " {} = {};", local(*id), slot.emptied()); + let _ = writeln!(out, " {} = {};", local(*id), slot.undefined()); } } else if let Some(retain) = inc_ref(slot, &format!("{}.f{index}", local(*dest))) { let _ = writeln!(out, " {retain}"); @@ -8889,27 +9087,12 @@ fn emit_op(module: &ModuleIr, function: &Function, op: &Op, error_target: ErrorE assign_checked(module, function, *dest, &call, error_target) } Op::LoadGlobal { dest, name } => { - let mut out = String::new(); - let slot = format!("by_g_{}", mangle(name)); - let _ = writeln!(out, " {{ static PyObject *{slot} = NULL;"); + let mut out = global_site_preamble(name, error_target); let _ = writeln!( out, - " if ({slot} == NULL) {slot} = By_InternedStr({});", - c_string_sized(name) - ); - let _ = writeln!( - out, - " if ({slot} == NULL) goto {};", - error_label(error_target) - ); - let site = format!("by_gs_{}", mangle(name)); - let _ = writeln!( - out, - " static ByGlobalSite {site} = BY_GLOBAL_SITE_INIT;" - ); - let _ = writeln!( - out, - " PyObject *by_t = By_LookupGlobalSite(&{site}, by_module_dict, {slot});" + " PyObject *by_t = By_LookupGlobalSite(&{}, by_module_dict, {});", + global_site(name), + global_name_slot(name), ); out.push_str(&commit_checked(function, *dest, error_target)); out @@ -9525,28 +9708,40 @@ fn emit_op(module: &ModuleIr, function: &Function, op: &Op, error_target: ErrorE out.push_str(" }\n"); out } - Op::BuildList { dest, items } => emit_container( + Op::BuildList { dest, items, moves } => emit_container( function, *dest, items, "By_BuildList", items.len(), + Handover::Taken(moves), + error_target, + ), + Op::BuildSet { dest, items } => emit_container( + function, + *dest, + items, + "By_BuildSet", + items.len(), + Handover::Borrowed, + error_target, + ), + Op::BuildTuple { dest, items, moves } => emit_container( + function, + *dest, + items, + "By_BuildTuple", + items.len(), + Handover::Taken(moves), error_target, ), - Op::BuildSet { dest, items } | Op::BuildTuple { dest, items } => { - let builder = if matches!(op, Op::BuildSet { .. }) { - "By_BuildSet" - } else { - "By_BuildTuple" - }; - emit_container(function, *dest, items, builder, items.len(), error_target) - } Op::BuildDict { dest, pairs } => emit_container( function, *dest, pairs, "By_BuildDict", pairs.len() / 2, + Handover::Borrowed, error_target, ), Op::GetItem { @@ -9685,7 +9880,7 @@ fn emit_op(module: &ModuleIr, function: &Function, op: &Op, error_target: ErrorE let mut out = format!( " {{ {} by_m = {place}; {place} = {};\n", ctype(module, ty), - ty.emptied() + ty.undefined() ); out.push_str(&assign_owned(module, function, *dest, "by_m")); out.push_str(" }\n"); @@ -9725,7 +9920,7 @@ fn emit_op(module: &ModuleIr, function: &Function, op: &Op, error_target: ErrorE format!( " {{ {} by_old = {target}; {target} = {}; {release} }}\n", ctype(module, ty), - ty.emptied() + ty.undefined() ) }) } @@ -9945,12 +10140,25 @@ fn emit_op(module: &ModuleIr, function: &Function, op: &Op, error_target: ErrorE /// /// each element is retained before the call, because the builder takes ownership /// of what it is handed and the operand registers keep theirs +/// what a container builder does with the references it is handed +#[derive(Clone, Copy)] +enum Handover<'a> { + /// it takes each one over, so the caller hands one across per item — a tuple and a + /// list, whose element writes steal. the items named here hand the register's own + /// reference over; every other item has one taken for it + Taken(&'a BTreeSet), + /// it borrows, and takes references of its own where it needs them — a dict and a + /// set, whose insertions retain for themselves + Borrowed, +} + fn emit_container( function: &Function, dest: RegisterId, items: &[Value], builder: &str, count: usize, + handover: Handover<'_>, error_target: ErrorEdge, ) -> String { let mut out = String::new(); @@ -9964,13 +10172,28 @@ fn emit_container( .join(", ") }; let _ = writeln!(out, " {{ PyObject *by_items[] = {{ {argv} }};"); - for item in items { - if let Some(retain) = inc_ref(&RType::OBJECT, &value_expr(item)) { - let _ = writeln!(out, " {retain}"); + if let Handover::Taken(moves) = handover { + for (index, item) in items.iter().enumerate() { + if moves.contains(&index) { + continue; + } + if let Some(retain) = inc_ref(&RType::OBJECT, &value_expr(item)) { + let _ = writeln!(out, " {retain}"); + } } } let _ = writeln!(out, " PyObject *by_t = {builder}(by_items, {count});"); out.push_str(&commit_checked(function, dest, error_target)); + // a register that handed its reference to a slot holds nothing afterwards, and + // every later release of it finds that. the emptying waits for the build, whose + // failure path releases what it was given + if let Handover::Taken(moves) = handover { + for index in moves { + if let Some(Value::Register(id)) = items.get(*index) { + let _ = writeln!(out, " {} = NULL;", local(*id)); + } + } + } out } @@ -10271,6 +10494,7 @@ fn borrowed_element_unbox( dest: unboxed, src: Value::Register(source), to, + .. }, release @ Op::Release { value: Value::Register(released), @@ -10330,6 +10554,102 @@ fn borrowed_element_unbox( )) } +/// the static holding one global name, interned so the key carries its hash +fn global_name_slot(name: &str) -> String { + format!("by_g_{}", mangle(name)) +} + +/// the static holding what the namespace last answered for one global name +fn global_site(name: &str) -> String { + format!("by_gs_{}", mangle(name)) +} + +/// the opening of a global read: the interned name and the memo the answer is kept in, +/// both in statics of this site's own. the caller computes into `by_t` and closes the +/// block, which is what [`commit_checked`] writes +fn global_site_preamble(name: &str, error_target: ErrorEdge) -> String { + let slot = global_name_slot(name); + format!( + " {{ static PyObject *{slot} = NULL;\n \ + if ({slot} == NULL) {slot} = By_InternedStr({interned});\n \ + if ({slot} == NULL) goto {label};\n \ + static ByGlobalSite {site} = BY_GLOBAL_SITE_INIT;\n", + interned = c_string_sized(name), + label = error_label(error_target), + site = global_site(name), + ) +} + +/// a global read whose only use is the unbox straight after it, as one memoised read +/// +/// the memo already spares the lookup, but the caller is still handed a *reference* to +/// an object it wants only in order to take apart: `while i < _limit` retains the object +/// the namespace holds under the name, narrows it to an `int`, and lets it go again — on +/// every trip. so the site remembers what the program actually asked for, and the retain, +/// the release and the narrowing all go, leaving the generation compare they stood behind. +/// +/// only a narrowing that takes nothing from the object it reads qualifies — the same set +/// [`unbox_owning_nothing_of_its_source`] names for a borrowed element read, and the +/// reason `ByGlobalUnboxedSite` may keep the narrowed value at all +fn unboxed_global(function: &Function, ops: &[Op], edge: ErrorEdge) -> Option { + let [ + Op::LoadGlobal { dest: boxed, name }, + Op::Unbox { + dest: unboxed, + src: Value::Register(source), + to, + proved: false, + }, + Op::Release { + value: Value::Register(released), + path, + }, + ] = ops + else { + return None; + }; + // the object register has to be a temporary this read wrote and this unbox is the + // whole use of: a name goes on holding what it holds, a borrowed register owns + // nothing to have spared, and either would still have to be written + let temporary = function + .register(*boxed) + .is_some_and(|decl| decl.name.is_none() && !decl.borrowed && !decl.may_be_unassigned); + if source != boxed + || released != boxed + || !path.is_empty() + || unboxed == boxed + || !temporary + || function.register(*unboxed).is_none_or(|decl| decl.borrowed) + || function + .register(*unboxed) + .is_none_or(|decl| decl.ty != *to) + { + return None; + } + // `None` and `bool` narrow to the same byte and differ only in what they accept + let (ctype, helper, also) = match to { + RType::Primitive(Primitive::Int) => ("ByTagged", "By_LookupGlobalTaggedSite", ""), + RType::Primitive(Primitive::Float) => ("double", "By_LookupGlobalFloatSite", ""), + RType::Primitive(Primitive::Bool) => ("char", "By_LookupGlobalBitSite", ", 0"), + RType::Primitive(Primitive::None) => ("char", "By_LookupGlobalBitSite", ", 1"), + _ => return None, + }; + let mut out = global_site_preamble(name, edge); + let memo = format!("by_gu_{}", mangle(name)); + let _ = writeln!( + out, + " static ByGlobalUnboxedSite {memo} = BY_GLOBAL_UNBOXED_SITE_INIT;" + ); + let _ = writeln!( + out, + " {ctype} by_t = {helper}(&{memo}, &{site}, by_module_dict, {slot}{also});", + site = global_site(name), + slot = global_name_slot(name), + ); + out.push_str(&commit_checked(function, *unboxed, edge)); + Some(out) +} + /// the helper that normalizes and bounds-checks an index into a packed buffer, for the /// representation the index is in and for whether the access is a store, which `list` /// refuses in words of its own @@ -13967,6 +14287,7 @@ mod tests { dest: out, src: Value::Register(a), to: RType::FLOAT, + proved: false, }); builder.terminate(Terminator::Return(Value::Register(out))); let c = emit_module(&module_with(builder.finish())); @@ -14053,7 +14374,10 @@ mod tests { assert!(c.contains("ByTagged a1 = BY_INT_ERROR;"), "{c}"); assert!(c.contains("a0 = By_UnboxInt(by_bound[0]);"), "{c}"); assert!(c.contains("a1 = By_UnboxInt(by_bound[1]);"), "{c}"); - assert!(c.contains("return By_BoxInt(by_result);")); + // the native entry hands back an owned tagged value, so the boxing that ends + // the wrapper takes that reference over rather than adding one of its own + assert!(c.contains("return By_BoxIntOwned(by_result);")); + assert!(!c.contains("return By_BoxInt(by_result);")); } /// a boundary with something to decide keeps the binding that decides it @@ -14839,6 +15163,239 @@ mod tests { assert!(!text.contains("By_XIncRef(r1.f0)"), "{text}"); } + /// `f` reading the module global `_limit` and narrowing it to `ty`, with the object + /// let go of straight after — the shape a loop condition reading a global takes + fn a_global_read_narrowed_to(ty: RType) -> Function { + let mut builder = FunctionBuilder::new("f", ty.clone()); + let boxed = builder.temp(RType::OBJECT); + let unboxed = builder.temp(ty.clone()); + builder.push(Op::LoadGlobal { + dest: boxed, + name: "_limit".to_string(), + }); + builder.push(Op::Unbox { + dest: unboxed, + src: Value::Register(boxed), + to: ty, + proved: false, + }); + builder.push(Op::Release { + value: Value::Register(boxed), + path: Box::new([]), + }); + builder.terminate(Terminator::Return(Value::Register(unboxed))); + builder.finish() + } + + #[test] + fn a_global_read_whose_only_use_is_an_unbox_remembers_the_unboxed_value() { + // the memo of the object spares the lookup but not the reference, and the caller + // wanted the object only in order to take it apart — so the narrowed value is + // what is remembered, and the retain, the release and the narrowing all go + let function = a_global_read_narrowed_to(RType::INT); + assert_eq!(verify(&function), Ok(())); + + let c = emit_function(&ModuleIr::new("app"), &function); + assert!( + c.contains("static ByGlobalUnboxedSite by_gu__limit = BY_GLOBAL_UNBOXED_SITE_INIT;"), + "{c}" + ); + assert!( + c.contains( + "ByTagged by_t = By_LookupGlobalTaggedSite(&by_gu__limit, &by_gs__limit, \ + by_module_dict, by_g__limit);" + ), + "{c}" + ); + // and neither half of what it replaces is left behind + assert!(!c.contains("By_LookupGlobalSite("), "{c}"); + assert!(!c.contains("By_UnboxInt("), "{c}"); + } + + #[test] + fn a_global_read_narrowed_to_a_double_or_a_byte_remembers_that_instead() { + // the memo is over the representation, not over `int`: a `float` and a `bool` + // are copied out of the object outright, which is if anything the easier case + for (ty, helper) in [ + (RType::FLOAT, "double by_t = By_LookupGlobalFloatSite("), + (RType::BOOL, "char by_t = By_LookupGlobalBitSite("), + ] { + let function = a_global_read_narrowed_to(ty.clone()); + assert_eq!(verify(&function), Ok(()), "{ty}"); + let c = emit_function(&ModuleIr::new("app"), &function); + assert!(c.contains(helper), "{ty}: {c}"); + assert!(!c.contains("By_LookupGlobalSite("), "{ty}: {c}"); + } + } + + #[test] + fn a_global_read_something_else_reads_too_keeps_the_object() { + // the narrowing is not the whole use of the object here, so there is still an + // object to hand back and nothing to memoise in its place + let mut builder = FunctionBuilder::new("f", RType::INT); + let boxed = builder.temp(RType::OBJECT); + let unboxed = builder.temp(RType::INT); + let length = builder.temp(RType::INT); + builder.push(Op::LoadGlobal { + dest: boxed, + name: "_limit".to_string(), + }); + builder.push(Op::Unbox { + dest: unboxed, + src: Value::Register(boxed), + to: RType::INT, + proved: false, + }); + builder.push(Op::Len { + dest: length, + src: Value::Register(boxed), + }); + builder.terminate(Terminator::Return(Value::Register(length))); + let function = builder.finish(); + assert_eq!(verify(&function), Ok(())); + + let c = emit_function(&ModuleIr::new("app"), &function); + assert!(c.contains("By_LookupGlobalSite(&by_gs__limit"), "{c}"); + assert!(c.contains("By_UnboxInt("), "{c}"); + assert!(!c.contains("ByGlobalUnboxedSite"), "{c}"); + } + + #[test] + fn a_display_that_takes_an_element_over_empties_the_register_it_came_from() { + // a slot handed a register's own reference takes it rather than retaining, so + // the register has to be *written* empty and not merely left alone: the release + // the frame makes of every written register on its way out would otherwise find + // that reference a second time. + // + // the value written is the one the register was declared with, and pointedly + // not a second "holds nothing" pattern of its own: one constant through a + // register's whole lifetime is one clang folds the later release of away, and + // a cheaper-looking second constant defeats that — see [`RType::undefined`] + let tuple_ty = RType::Tuple(Box::new([RType::STR, RType::INT])); + let mut builder = FunctionBuilder::new("pair", tuple_ty.clone()); + let seed = builder.param("seed", RType::STR); + let text = builder.temp(RType::STR); + let count = builder.temp(RType::INT); + let out = builder.temp(tuple_ty); + builder.push(Op::CallNative { + owner: None, + dest: Some(text), + callee: "label".to_string(), + args: vec![Value::Register(seed)], + }); + builder.push(Op::CallNative { + owner: None, + dest: Some(count), + callee: "tally".to_string(), + args: vec![Value::Register(seed)], + }); + builder.push(Op::TupleBuild { + dest: out, + items: vec![Value::Register(text), Value::Register(count)], + moves: BTreeSet::from([0, 1]), + }); + builder.terminate(Terminator::Return(Value::Register(out))); + let function = builder.finish(); + assert_eq!(verify(&function), Ok(())); + + let c = emit_function(&ModuleIr::new("app"), &function); + assert!(c.contains(&format!(" {} = NULL;\n", local(text))), "{c}"); + assert!( + c.contains(&format!(" {} = BY_INT_ERROR;\n", local(count))), + "{c}" + ); + // and a slot that took the reference over does not also retain it + assert!( + !c.contains(&format!("By_XIncRef({}.f0)", local(out))), + "{c}" + ); + } + + #[test] + fn a_moving_field_store_empties_the_register_it_took_the_reference_from() { + // the field takes the register's reference rather than retaining, so the + // register has to be written empty — the frame releases every written register + // on its way out, and the field now holds the only reference there was + let mut builder = FunctionBuilder::new("__init__", RType::NONE); + let receiver = builder.param( + "self", + RType::Instance { + class: "Wrapped".to_string(), + exact: true, + }, + ); + let seed = builder.param("seed", RType::OBJECT); + let value = builder.temp(RType::OBJECT); + builder.defaults(vec![None, None]); + builder.push(Op::CallNative { + owner: None, + dest: Some(value), + callee: "label".to_string(), + args: vec![Value::Register(seed)], + }); + builder.push(Op::SetField { + receiver: Value::Register(receiver), + class: "Wrapped".to_string(), + field: "tag".to_string(), + value: Value::Register(value), + moves: true, + present: false, + }); + builder.terminate(Terminator::Return(Value::None)); + let mut init = builder.finish(); + init.owner = Some("Wrapped".to_string()); + assert_eq!(verify(&init), Ok(())); + let mut module = module_with(init); + module.classes.push(appending_class()); + + let c = emit_module(&module); + let register = local(value); + assert!( + c.contains(&format!( + "->by_f_tag = {register};\n {register} = NULL;\n" + )), + "{c}" + ); + // and a store that hands the reference over takes none of its own for the field + assert!(!c.contains(&format!("By_XIncRef({register})")), "{c}"); + } + + #[test] + fn a_function_known_to_stand_resolves_to_an_empty_register() { + // `FunctionStood` is what a resolution becomes once a guard has established that + // the name still stands for the native entry, and its answer is *nothing to + // call*. so the register has to be written, not merely released: every later + // read tests it against NULL to decide whether to go straight to the entry, and + // the release the frame makes of it on the way out would find it a second time. + // + // the guard this replaces a resolution under makes the register empty already, + // which is why no behaviour reaches the difference — the op's contract is + // asserted here instead of through a program + let mut builder = FunctionBuilder::new("f", RType::INT); + let callee = builder.temp(RType::OBJECT); + let answer = builder.temp(RType::INT); + builder.push(Op::FunctionStood { + dest: callee, + name: "helper".to_string(), + }); + builder.push(Op::CallNative { + owner: None, + dest: Some(answer), + callee: "helper".to_string(), + args: Vec::new(), + }); + builder.terminate(Terminator::Return(Value::Register(answer))); + + let text = emit_function(&ModuleIr::new("app"), &builder.finish()); + let register = local(callee); + assert!( + text.contains(&format!( + "{{ PyObject *by_old = {register}; {register} = NULL; Py_XDECREF(by_old); }}" + )), + "{text}" + ); + } + #[test] fn a_line_directive_points_at_the_by_source() { let mut builder = FunctionBuilder::new("f", RType::INT); diff --git a/crates/by_ir/src/builder.rs b/crates/by_ir/src/builder.rs index 6cb0c4962f..74d42091fc 100644 --- a/crates/by_ir/src/builder.rs +++ b/crates/by_ir/src/builder.rs @@ -356,6 +356,7 @@ impl FunctionBuilder { dest: written, src, to: RType::Instance { .. }, + .. }) if *written == dest => src.clone(), _ => return None, }; diff --git a/crates/by_ir/src/ops.rs b/crates/by_ir/src/ops.rs index 9ea6d9799c..04f03bed2d 100644 --- a/crates/by_ir/src/ops.rs +++ b/crates/by_ir/src/ops.rs @@ -366,13 +366,22 @@ pub enum Op { /// give an unboxed value an object representation Box { dest: RegisterId, src: Value }, /// take an object apart into an unboxed representation. this is a *narrowing* - /// and so is always checked — it raises `TypeError` if the object does not - /// have the expected type + /// and so is checked — it raises `TypeError` if the object does not have the + /// expected type Unbox { dest: RegisterId, src: Value, /// the type being unboxed to, which is the destination register's type to: RType, + /// whether a test standing over this operation has already asked the very + /// question the check would ask, so that the narrowing is a change of static + /// type and nothing else + /// + /// nothing that *lowers* an unbox ever claims this: a lowering can see the + /// test it wrote a line ago and not the pass that moved it. it is set by + /// `by_opt`'s `proved_narrowings`, which finds the test for itself and gives + /// the proof up again at the first operation that could run python + proved: bool, }, /// `lhs rhs` where `lhs` is a double and `rhs` is any object, for the /// case where the checker has said the result is a `float` @@ -1142,11 +1151,26 @@ pub enum Op { value: Value, }, /// a list display, from already-boxed elements - BuildList { dest: RegisterId, items: Vec }, + /// + /// `moves` names the items that hand their register's reference over rather than + /// having one taken for them, by index — see [`Self::TupleBuild`], whose `moves` + /// means the same thing. a list and a tuple write their elements with a store that + /// takes a reference over, which is what makes the handover possible; a set and a + /// dict insert through a call that takes a reference of its own, so there is nothing + /// for them to be handed + BuildList { + dest: RegisterId, + items: Vec, + moves: BTreeSet, + }, /// a set display BuildSet { dest: RegisterId, items: Vec }, /// a tuple display - BuildTuple { dest: RegisterId, items: Vec }, + BuildTuple { + dest: RegisterId, + items: Vec, + moves: BTreeSet, + }, /// a dict display, as alternating keys and values BuildDict { dest: RegisterId, pairs: Vec }, /// `container[index]` @@ -2578,6 +2602,7 @@ mod tests { class: "Held".to_string(), exact: true, }, + proved: false, }, ]; for op in reaching { diff --git a/crates/by_ir/src/print.rs b/crates/by_ir/src/print.rs index fd21c69ac8..c9b099592a 100644 --- a/crates/by_ir/src/print.rs +++ b/crates/by_ir/src/print.rs @@ -4,6 +4,7 @@ //! deliberately terse. it is not parsed back — tests build BIR through the //! builder and compare the rendering. +use std::collections::BTreeSet; use std::fmt::Write; use crate::function::{Function, ModuleIr}; @@ -114,6 +115,27 @@ fn sound_target(function: &Function, target: &SoundTarget) -> String { } } +/// a container build's items, with the ones handing their reference over marked +fn built_items( + items: &[Value], + moves: &BTreeSet, + value: impl Fn(&Value) -> String, +) -> String { + items + .iter() + .enumerate() + .map(|(index, item)| { + let item = value(item); + if moves.contains(&index) { + format!("move {item}") + } else { + item + } + }) + .collect::>() + .join(", ") +} + fn print_op(function: &Function, op: &Op) -> String { let value = |v: &Value| print_value(function, v); let name = |id: RegisterId| register_name(function, id); @@ -392,23 +414,17 @@ fn print_op(function: &Function, op: &Op) -> String { Op::Box { dest, src } => format!("{} = box {}", name(*dest), value(src)), Op::IntToFloat { dest, src } => format!("{} = float {}", name(*dest), value(src)), Op::TagShort { dest, src } => format!("{} = tag short {}", name(*dest), value(src)), - Op::Unbox { dest, src, to } => { - format!("{} = unbox {} as {}", name(*dest), value(src), to) + Op::Unbox { + dest, + src, + to, + proved, + } => { + let how = if *proved { "narrow" } else { "unbox" }; + format!("{} = {how} {} as {}", name(*dest), value(src), to) } Op::TupleBuild { dest, items, moves } => { - let items = items - .iter() - .enumerate() - .map(|(index, item)| { - let item = value(item); - if moves.contains(&index) { - format!("move {item}") - } else { - item - } - }) - .collect::>() - .join(", "); + let items = built_items(items, moves, value); format!("{} = ({})", name(*dest), items) } Op::CallUnpacked { @@ -783,16 +799,16 @@ fn print_op(function: &Function, op: &Op) -> String { value(receiver), value(v) ), - Op::BuildList { dest, items } => { - let items = items.iter().map(value).collect::>().join(", "); + Op::BuildList { dest, items, moves } => { + let items = built_items(items, moves, value); format!("{} = [{items}]", name(*dest)) } Op::BuildSet { dest, items } => { let items = items.iter().map(value).collect::>().join(", "); format!("{} = {{{items}}}", name(*dest)) } - Op::BuildTuple { dest, items } => { - let items = items.iter().map(value).collect::>().join(", "); + Op::BuildTuple { dest, items, moves } => { + let items = built_items(items, moves, value); format!("{} = tuple({items})", name(*dest)) } Op::BuildDict { dest, pairs } => { diff --git a/crates/by_ir/src/rtype.rs b/crates/by_ir/src/rtype.rs index 07984921c5..9a7eef754b 100644 --- a/crates/by_ir/src/rtype.rs +++ b/crates/by_ir/src/rtype.rs @@ -283,6 +283,19 @@ impl RType { } } + /// the value a place of this representation holds when it holds nothing: before + /// anything has been written to it, after a move has taken what it held away, and + /// on the error path a fallible function hands back + /// + /// those are deliberately one value rather than three. a tagged `int`'s is odd, so + /// releasing it again costs the branch kept for a real `PyLongObject` — and a short + /// zero, which is the same "holds nothing" and whose release the straight line + /// already passes, looks like the cheaper thing to empty a register with. it is not, + /// because a register whose every value is the *same* constant is one clang folds + /// the release of away outright, where a second constant defeats that and leaves a + /// real branch on every trip. measured 2026-09-21 over a `BY_INT_EMPTY` spelling of + /// the emptied value: `props_ext` +8.6%, `tuples` +7.8%, `coro` +5.5%, `globals_` + /// +4.8%, `inherit` +3.3%, `dot` +2.3%, against `sieve` unmoved pub fn undefined(&self) -> String { match self { Self::Array(_) => "NULL".to_string(), @@ -317,28 +330,6 @@ impl RType { } } - /// the value a register holds once the reference it held has been let go of - /// - /// this is [`Self::undefined`] everywhere a representation's unset value costs - /// nothing to release again. a tagged `int`'s does not: `BY_INT_ERROR` is odd, so - /// every later release of the emptied register takes the branch meant for a real - /// `PyLongObject` and calls out of line to drop a `NULL`. a short zero is the same - /// "holds nothing" and its release is the test the straight line already passes - pub fn emptied(&self) -> String { - match self { - Self::Primitive(Primitive::Int) => "BY_INT_EMPTY".to_string(), - Self::Tuple(items) if !items.is_empty() => format!( - "{{ {} }}", - items - .iter() - .map(Self::emptied) - .collect::>() - .join(", ") - ), - other => other.undefined(), - } - } - /// where inside a value of this type an error can be read back off the value /// itself: the field path to the member holding the sentinel, and the pattern /// that member reserves diff --git a/crates/by_ir/src/verify.rs b/crates/by_ir/src/verify.rs index b6199f967a..68b65ebe0c 100644 --- a/crates/by_ir/src/verify.rs +++ b/crates/by_ir/src/verify.rs @@ -8,7 +8,7 @@ //! this is the guard on the representation invariant. a pass that produces //! ill-typed BIR is a bug that would otherwise surface as miscompiled C. -use std::collections::{HashMap, HashSet, VecDeque}; +use std::collections::{BTreeSet, HashMap, HashSet, VecDeque}; use std::fmt; use crate::function::{Function, ModuleIr}; @@ -661,6 +661,39 @@ impl Verifier<'_> { } } + /// the items a container build hands its own reference to rather than having one + /// taken for them + /// + /// the same rule `Op::Move` follows: only a temporary the frame owns has a reference + /// of its own to give away. a parameter's belongs to the caller, a borrowed register + /// has none, and a name goes on holding what it holds until it is rebound + fn expect_moves( + &mut self, + block: BlockId, + items: &[Value], + moves: &BTreeSet, + what: &str, + ) { + for index in moves { + let hands_over = matches!( + items.get(*index), + Some(Value::Register(id)) + if id.index() >= self.function.param_count + && self.function.register(*id).is_some_and(|decl| { + !decl.borrowed + && decl.name.is_none() + && decl.ty.owns_one_reference() + }) + ); + if !hands_over { + self.error( + Some(block), + format!("{what} moving item {index}, which owns no reference to hand over"), + ); + } + } + } + /// a register an op reads *and* writes, so it is neither an operand nor a dest fn expect_cursor(&mut self, block: BlockId, cursor: RegisterId) { let expected = RType::fixed(IntWidth::I64); @@ -1094,7 +1127,7 @@ impl Verifier<'_> { _ => self.expect_dest(block, *dest, &RType::OBJECT, "box"), } } - Op::Unbox { dest, src, to } => { + Op::Unbox { dest, src, to, .. } => { // unbox *narrows from* `object` — to an unboxed representation, or // to a boxed one whose class is known. narrowing to `object` would // be a no-op the frontend should not have emitted @@ -1108,28 +1141,7 @@ impl Verifier<'_> { self.expect_dest(block, *dest, to, "unbox"); } Op::TupleBuild { dest, items, moves } => { - for index in moves { - // the same rule `Op::Move` follows: only a temporary the frame owns - // has a reference of its own to hand to the slot - let hands_over = matches!( - items.get(*index), - Some(Value::Register(id)) - if id.index() >= self.function.param_count - && self.function.register(*id).is_some_and(|decl| { - !decl.borrowed - && decl.name.is_none() - && decl.ty.owns_one_reference() - }) - ); - if !hands_over { - self.error( - Some(block), - format!( - "a tuple build moving item {index}, which owns no reference to hand over" - ), - ); - } - } + self.expect_moves(block, items, moves, "a tuple build"); let declared = self .function .register(*dest) @@ -1626,7 +1638,8 @@ impl Verifier<'_> { self.expect(block, value, &RType::OBJECT, "attribute assignment"); self.expect_dest(block, *dest, &RType::BIT, "attribute assignment"); } - Op::BuildList { dest, items } => { + Op::BuildList { dest, items, moves } => { + self.expect_moves(block, items, moves, "a list display"); for item in items { self.expect(block, item, &RType::OBJECT, "a list element"); } @@ -1638,7 +1651,8 @@ impl Verifier<'_> { } self.expect_dest(block, *dest, &RType::OBJECT, "a set display"); } - Op::BuildTuple { dest, items } => { + Op::BuildTuple { dest, items, moves } => { + self.expect_moves(block, items, moves, "a tuple display"); for item in items { self.expect(block, item, &RType::OBJECT, "a tuple element"); } @@ -2900,6 +2914,7 @@ mod tests { dest: part, src: Value::Register(whole), to: RType::STR, + proved: false, }); builder.terminate(Terminator::Return(Value::None)); let mut function = builder.finish(); @@ -2912,6 +2927,7 @@ mod tests { dest: part, src: Value::Register(whole), to: RType::INT, + proved: false, }; let errors = verify(&function).unwrap_err(); assert!(errors[0].message.contains("is borrowed"), "{errors:?}"); diff --git a/crates/by_irbuild/src/lib.rs b/crates/by_irbuild/src/lib.rs index 881ebbe22f..3a2708f307 100644 --- a/crates/by_irbuild/src/lib.rs +++ b/crates/by_irbuild/src/lib.rs @@ -8353,9 +8353,17 @@ impl Display { fn build(self, dest: RegisterId, items: Vec) -> Op { match self { - Self::List => Op::BuildList { dest, items }, + Self::List => Op::BuildList { + dest, + items, + moves: BTreeSet::new(), + }, Self::Set => Op::BuildSet { dest, items }, - Self::Tuple => Op::BuildTuple { dest, items }, + Self::Tuple => Op::BuildTuple { + dest, + items, + moves: BTreeSet::new(), + }, Self::Dict => Op::BuildDict { dest, pairs: items }, } } @@ -10219,6 +10227,7 @@ impl Lowering<'_, '_> { dest, src: Value::Register(answer), to: ty.clone(), + proved: false, }); } } @@ -11217,6 +11226,7 @@ impl Lowering<'_, '_> { self.builder.push(Op::BuildTuple { dest: named, items: keys, + moves: BTreeSet::new(), }); let read = self.builder.temp(RType::OBJECT); self.builder.push(Op::MatchRest { @@ -11801,6 +11811,7 @@ impl Lowering<'_, '_> { dest: narrowed, src: manager, to: ty, + proved: false, }); Value::Register(narrowed) } @@ -12744,6 +12755,7 @@ impl Lowering<'_, '_> { dest: item, src: Value::Register(raw), to: RType::INT, + proved: false, }); self.builder.assign(index, Value::Register(item)); self.builder.terminate(Terminator::Goto(body)); @@ -13098,6 +13110,7 @@ impl Lowering<'_, '_> { dest: item, src: Value::Register(raw), to: element_ty.clone(), + proved: false, }); } else { self.builder.assign(item, Value::Register(raw)); @@ -14204,6 +14217,7 @@ impl Lowering<'_, '_> { dest, src: value, to: to.clone(), + proved: false, }); return Ok(Value::Register(dest)); } @@ -15065,7 +15079,11 @@ impl Lowering<'_, '_> { items.push(self.widen_to_object(Value::Register(item), slot)); } let dest = self.builder.temp(RType::OBJECT); - self.builder.push(Op::BuildTuple { dest, items }); + self.builder.push(Op::BuildTuple { + dest, + items, + moves: BTreeSet::new(), + }); return Value::Register(dest); } // a machine integer's *object* representation is the tagged `int`, which is one @@ -16461,6 +16479,7 @@ impl Lowering<'_, '_> { class: candidate.clone(), exact: false, }, + proved: false, }); Value::Register(narrowed) } @@ -16513,6 +16532,7 @@ impl Lowering<'_, '_> { dest, src: Value::Register(answer), to: ret.clone(), + proved: false, }); } self.builder.terminate(Terminator::Goto(join)); @@ -16562,6 +16582,7 @@ impl Lowering<'_, '_> { dest: narrowed, src: Value::Register(boxed), to: declared.clone(), + proved: false, }); return Ok((Value::Register(narrowed), declared)); } @@ -16739,6 +16760,7 @@ impl Lowering<'_, '_> { Comprehension::List(_) => self.builder.push(Op::BuildList { dest: accumulator, items: Vec::new(), + moves: BTreeSet::new(), }), Comprehension::Set(_) => self.builder.push(Op::BuildSet { dest: accumulator, @@ -17017,6 +17039,7 @@ impl Lowering<'_, '_> { dest: item, src: Value::Register(raw), to: element_ty.clone(), + proved: false, }); } else { self.builder.assign(item, Value::Register(raw)); @@ -17205,6 +17228,7 @@ impl Lowering<'_, '_> { dest: item, src: Value::Register(raw), to: RType::INT, + proved: false, }); self.store(counter, Value::Register(item), &RType::INT)?; self.builder.terminate(Terminator::Goto(body)); @@ -19059,6 +19083,7 @@ impl Lowering<'_, '_> { self.builder.push(Op::BuildTuple { dest, items: extra_positional, + moves: BTreeSet::new(), }); args.push(Value::Register(dest)); } diff --git a/crates/by_irbuild/src/tests.rs b/crates/by_irbuild/src/tests.rs index 77ce4adaae..c99412e20a 100644 --- a/crates/by_irbuild/src/tests.rs +++ b/crates/by_irbuild/src/tests.rs @@ -296,6 +296,11 @@ def condition(a: object, b: object) -> int: def ints(a: int, b: int) -> object: return a > b + +def equal(a: object, b: object) -> int: + if a == b: + return 1 + return 0 ", |db, env, model, suite| { let module = crate::build_module(db, env, model, suite, "app", crate::Language::Python); @@ -327,6 +332,9 @@ def ints(a: int, b: int) -> object: named("condition") ); assert!(!rich("ints") && !bit("ints"), "{}", named("ints")); + // `==` as a condition takes the same bit-answering form, which is where the + // runtime decides an equality without asking both operands in turn + assert!(bit("equal") && !rich("equal"), "{}", named("equal")); }, ); } diff --git a/crates/by_opt/src/borrow.rs b/crates/by_opt/src/borrow.rs index 5d3d6aa788..bc386359ff 100644 --- a/crates/by_opt/src/borrow.rs +++ b/crates/by_opt/src/borrow.rs @@ -2081,6 +2081,7 @@ mod tests { dest: part, src: Value::Register(held), to, + proved: false, }); builder.push(Op::Len { dest: length, diff --git a/crates/by_opt/src/dead_allocations.rs b/crates/by_opt/src/dead_allocations.rs index b679cd068d..c9eedd95e2 100644 --- a/crates/by_opt/src/dead_allocations.rs +++ b/crates/by_opt/src/dead_allocations.rs @@ -266,6 +266,7 @@ mod tests { builder.push(Op::BuildTuple { dest: pair, items: vec![Value::Register(first), Value::Register(second)], + moves: BTreeSet::new(), }); builder.terminate(Terminator::Return(Value::Register(whole))); diff --git a/crates/by_opt/src/fold.rs b/crates/by_opt/src/fold.rs index f23e0b4adc..9723dc6dc9 100644 --- a/crates/by_opt/src/fold.rs +++ b/crates/by_opt/src/fold.rs @@ -191,7 +191,7 @@ fn fold(function: &mut Function, frozen: &HashSet) { read.retain(|(_, owner, name), _| *owner != class || *name != field); } match op { - Op::BuildTuple { dest, items } | Op::TupleBuild { dest, items, .. } => { + Op::BuildTuple { dest, items, .. } | Op::TupleBuild { dest, items, .. } => { built.insert(*dest, items.clone()); } Op::GetField { @@ -339,7 +339,7 @@ fn fold_box_round_trip( boxed_from: &HashMap, op: &Op, ) -> Option { - let Op::Unbox { dest, src, to } = op else { + let Op::Unbox { dest, src, to, .. } = op else { return None; }; if matches!(to, RType::Instance { .. }) { @@ -360,7 +360,7 @@ fn fold_box_round_trip( } fn fold_unbox(types: &[RType], op: &Op) -> Option { - let Op::Unbox { dest, src, to } = op else { + let Op::Unbox { dest, src, to, .. } = op else { return None; }; if matches!(to, RType::Instance { .. }) { @@ -837,6 +837,7 @@ mod tests { dest: back, src: Value::Register(boxed), to: RType::STR, + proved: false, }); builder.terminate(Terminator::Return(Value::Register(back))); let mut m = module(builder.finish()); @@ -865,6 +866,7 @@ mod tests { dest: out, src: Value::Register(any), to: RType::STR, + proved: false, }); builder.terminate(Terminator::Return(Value::Register(out))); let mut m = module(builder.finish()); @@ -894,6 +896,7 @@ mod tests { builder.push(Op::BuildTuple { dest: pair, items: vec![Value::Register(b), Value::Register(a)], + moves: BTreeSet::new(), }); builder.push(Op::Unpack { dest: slots, diff --git a/crates/by_opt/src/guard_loops.rs b/crates/by_opt/src/guard_loops.rs index 1ee0b2fa1d..56660ac57b 100644 --- a/crates/by_opt/src/guard_loops.rs +++ b/crates/by_opt/src/guard_loops.rs @@ -768,6 +768,7 @@ mod tests { dest: length, src: Value::Register(answered), to: RType::INT, + proved: false, }); builder.terminate(Terminator::Goto(guard)); @@ -1091,6 +1092,7 @@ mod tests { dest: answer, src: Value::Register(answered), to: RType::INT, + proved: false, }); builder.terminate(Terminator::Goto(join)); builder.switch_to(join); diff --git a/crates/by_opt/src/infallible.rs b/crates/by_opt/src/infallible.rs index 933dcdb0e6..39607a510a 100644 --- a/crates/by_opt/src/infallible.rs +++ b/crates/by_opt/src/infallible.rs @@ -325,9 +325,12 @@ fn op_can_fail(module: &ModuleIr, function: &by_ir::function::Function, op: &Op) function.value_type(src), Some(RType::Primitive(Primitive::Fixed(_))) ), - // boxing allocates, unboxing is a checked narrowing, a raise raises, and - // an integer with no float at all raises `OverflowError` - Op::Box { .. } | Op::Unbox { .. } | Op::RaiseStandard { .. } => true, + // a narrowing a test standing over it already made has nothing left to check — + // see [`crate::proved_narrowings`] + Op::Unbox { proved, .. } => !*proved, + // boxing allocates, a raise raises, and an integer with no float at all raises + // `OverflowError` + Op::Box { .. } | Op::RaiseStandard { .. } => true, } } diff --git a/crates/by_opt/src/lib.rs b/crates/by_opt/src/lib.rs index d737e689b6..06fe1ed831 100644 --- a/crates/by_opt/src/lib.rs +++ b/crates/by_opt/src/lib.rs @@ -16,6 +16,7 @@ pub(crate) mod fold; mod guard_loops; pub(crate) mod infallible; mod liveness; +mod proved_narrowings; mod push_lengths; pub(crate) mod refcount; pub(crate) mod release_temporaries; @@ -130,6 +131,13 @@ const PASSES: &[Pass] = &[ name: "push-lengths", run: push_lengths::run, }, + // after every pass that copies a block, so a copy of a test and a copy of the + // narrowing behind it are matched up in the copy rather than across the two, and + // before infallible, which is what takes the error edge off the narrowings it proves + Pass { + name: "proved-narrowings", + run: proved_narrowings::run, + }, Pass { name: "infallible", run: infallible::run, diff --git a/crates/by_opt/src/proved_narrowings.rs b/crates/by_opt/src/proved_narrowings.rs new file mode 100644 index 0000000000..854b35b6fa --- /dev/null +++ b/crates/by_opt/src/proved_narrowings.rs @@ -0,0 +1,279 @@ +//! narrowings a test standing over them has already made +//! +//! two lowerings ask a question and then, on the arm that got a yes, ask it again. a +//! class pattern tests the subject's layout before reading a field at its offset, and +//! then narrows the subject to that layout — which checks the type a second time. a call +//! through a base-typed name tests whether the method still stands, which compares the +//! receiver's type exactly, and then narrows the receiver to that class — again a second +//! test. +//! +//! neither second test can answer differently from the first, so what is left of the +//! narrowing is the static type. the check goes and the value keeps its representation. +//! +//! the proof is given up at the first operation that could run python. python code can +//! reassign an object's `__class__`, and a release is python code: a finalizer runs +//! wherever the last reference to something goes. so the window the proof covers is the +//! one in which nothing at all can have looked at the object. + +use std::collections::HashMap; + +use by_ir::function::{Function, ModuleIr}; +use by_ir::ops::{BlockId, Op, RegisterId, Terminator, Value}; +use by_ir::rtype::RType; + +use crate::runs_python::{Effects, Held}; + +/// what a test standing over a block proved about the object it asked about +#[derive(Clone, Copy, PartialEq, Eq)] +enum Proof { + /// the object's type *is* the class: what `By_MethodStands` compares + Exact, + /// the object is an instance of the class, a subclass of it included: what + /// `PyObject_TypeCheck` answers, and so what `Op::HoldsLayout` proves + Instance, +} + +impl Proof { + /// whether it licenses a narrowing to this class + fn licenses(self, exact: bool) -> bool { + self == Self::Exact || !exact + } +} + +pub(crate) fn run(module: &mut ModuleIr) { + let effects = Effects::of(module); + for function in module.all_functions_mut() { + prove(&effects, function); + } +} + +fn prove(effects: &Effects, function: &mut Function) { + let entries = Held::solve( + function, + Function::entry(), + effects.held_at_entry(function), + |_| true, + ); + let proofs = proved_on_entry(effects, function, &entries); + let mut rewrites: Vec<(usize, usize)> = Vec::new(); + for (index, block) in function.blocks.iter().enumerate() { + let (Some((object, class, proof)), Some(entry)) = + (proofs.get(&index), entries.get(&BlockId(index))) + else { + continue; + }; + let mut known = entry.clone(); + for (at, op) in block.ops.iter().enumerate() { + if let Op::Unbox { + src: Value::Register(source), + to: RType::Instance { class: to, exact }, + proved: false, + .. + } = op + && source == object + && to == class + && proof.licenses(*exact) + { + rewrites.push((index, at)); + } + // the operation's own use of the object is the one the proof covers; what it + // can do afterwards is what ends the proof for everything behind it + if effects.op_runs_python(function, op, &known) || writes(op, *object) { + break; + } + known.step(function, op); + } + } + for (block, at) in rewrites { + if let Some(Op::Unbox { proved, .. }) = function.blocks[block].ops.get_mut(at) { + *proved = true; + } + } +} + +/// what holds on entry to each block that is reached only by a test saying yes +/// +/// only a block with that one way in: a block some other edge also reaches is entered +/// without the test having been asked +fn proved_on_entry( + effects: &Effects, + function: &Function, + entries: &HashMap, +) -> HashMap { + let mut ways_in: Vec = vec![0; function.blocks.len()]; + for block in &function.blocks { + for successor in block.successors() { + if let Some(count) = ways_in.get_mut(successor.index()) { + *count += 1; + } + } + } + let mut proofs = HashMap::new(); + for (index, block) in function.blocks.iter().enumerate() { + let Terminator::Branch { + cond: Value::Register(cond), + then_block, + .. + } = &block.terminator + else { + continue; + }; + // the entry is entered by the call as well as by any edge, so a count of one + // does not mean one way in + if *then_block == Function::entry() + || then_block.index() == index + || ways_in.get(then_block.index()).copied() != Some(1) + { + continue; + } + let Some((at, proof)) = block.ops.iter().enumerate().rev().find_map(|(at, op)| { + let proof = match op { + Op::HoldsLayout { + dest, + src: Value::Register(src), + class, + } if dest == cond => (*src, class.clone(), Proof::Instance), + Op::MethodStands { + dest, + src: Value::Register(src), + class, + .. + } if dest == cond => (*src, class.clone(), Proof::Exact), + _ => return None, + }; + Some((at, proof)) + }) else { + continue; + }; + // whatever the block does after asking can have run python, and then the answer + // is no longer the answer + let Some(entry) = entries.get(&BlockId(index)) else { + continue; + }; + let mut known = entry.clone(); + let mut stale = false; + for (position, op) in block.ops.iter().enumerate() { + if position > at + && (effects.op_runs_python(function, op, &known) || writes(op, proof.0)) + { + stale = true; + break; + } + known.step(function, op); + } + if !stale { + proofs.insert(then_block.index(), proof); + } + } + proofs +} + +/// whether `op` leaves `register` holding something other than what it held +fn writes(op: &Op, register: RegisterId) -> bool { + op.dest() + .into_iter() + .chain(op.unbinds()) + .chain(op.loop_cursor()) + .any(|written| written == register) + || matches!(op, Op::Move { src: Value::Register(moved), .. } if *moved == register) +} + +#[cfg(test)] +mod tests { + use by_ir::builder::FunctionBuilder; + use by_ir::function::{Function, ModuleIr}; + use by_ir::ops::{Op, Terminator, Value}; + use by_ir::rtype::RType; + + /// `if the subject is laid out as P: read it as a P`, with `extra` in front of the + /// narrowing and `another_way_in` deciding whether some other edge also reaches the + /// block the test says yes to + fn tested(extra: Option, another_way_in: bool) -> Function { + let laid_out = RType::Instance { + class: "P".to_string(), + exact: false, + }; + let mut builder = FunctionBuilder::new("pick", RType::OBJECT); + let subject = builder.param("subject", RType::OBJECT); + let holds = builder.temp(RType::BIT); + let narrowed = builder.temp(laid_out.clone()); + let out = builder.local("out", RType::OBJECT); + let native = builder.new_block(); + let generic = builder.new_block(); + let join = builder.new_block(); + builder.push(Op::HoldsLayout { + dest: holds, + src: Value::Register(subject), + class: "P".to_string(), + }); + builder.terminate(Terminator::Branch { + cond: Value::Register(holds), + then_block: native, + else_block: generic, + }); + + builder.switch_to(native); + if let Some(extra) = extra { + builder.push(extra); + } + builder.push(Op::Unbox { + dest: narrowed, + src: Value::Register(subject), + to: laid_out, + proved: false, + }); + builder.assign(out, Value::Register(narrowed)); + builder.terminate(Terminator::Goto(join)); + + builder.switch_to(generic); + builder.assign(out, Value::Register(subject)); + builder.terminate(if another_way_in { + Terminator::Goto(native) + } else { + Terminator::Goto(join) + }); + + builder.switch_to(join); + builder.terminate(Terminator::Return(Value::Register(out))); + builder.finish() + } + + fn run_on(function: Function) -> Function { + let mut module = ModuleIr::new("app"); + module.functions = vec![function]; + super::run(&mut module); + module.functions.remove(0) + } + + fn narrowings_proved(function: &Function) -> usize { + function + .blocks + .iter() + .flat_map(|block| &block.ops) + .filter(|op| matches!(op, Op::Unbox { proved: true, .. })) + .count() + } + + #[test] + fn a_narrowing_the_test_above_it_already_made_is_proved() { + assert_eq!(narrowings_proved(&run_on(tested(None, false))), 1); + } + + #[test] + fn a_block_some_other_edge_reaches_proves_nothing() { + // the other edge is entered without the test having been asked, so nothing on + // the block holds because the test said yes + assert_eq!(narrowings_proved(&run_on(tested(None, true))), 0); + } + + #[test] + fn an_operation_that_can_run_python_gives_the_proof_up() { + // python code can reassign an object's `__class__`, so a namespace lookup + // between the test and the narrowing leaves the narrowing to check for itself + let lookup = Op::LoadGlobal { + dest: by_ir::ops::RegisterId(0), + name: "anything".to_string(), + }; + assert_eq!(narrowings_proved(&run_on(tested(Some(lookup), false))), 0); + } +} diff --git a/crates/by_opt/src/release_temporaries.rs b/crates/by_opt/src/release_temporaries.rs index 22d53c67b1..ef2fce151f 100644 --- a/crates/by_opt/src/release_temporaries.rs +++ b/crates/by_opt/src/release_temporaries.rs @@ -683,6 +683,7 @@ mod tests { dest: made, src: Value::Register(p), to: RType::INT, + proved: false, }); builder.push(Op::IntBinary { dest: answer, diff --git a/crates/by_opt/src/runs_python.rs b/crates/by_opt/src/runs_python.rs index 79da6f7a7e..974cb9f8f6 100644 --- a/crates/by_opt/src/runs_python.rs +++ b/crates/by_opt/src/runs_python.rs @@ -1274,6 +1274,7 @@ mod tests { dest: below, src: Value::Register(answered), to: RType::INT, + proved: false, }); builder.terminate(Terminator::Goto(join)); builder.switch_to(join); @@ -1383,6 +1384,7 @@ mod tests { builder.push(Op::BuildList { dest: list, items: Vec::new(), + moves: BTreeSet::new(), }); } builder.terminate(Terminator::Return(Value::Float(0.0))); @@ -1522,6 +1524,7 @@ mod tests { dest: bound, src: Value::Register(other), to: RType::INT, + proved: false, }); builder.terminate(Terminator::Goto(join)); builder.switch_to(join); diff --git a/crates/by_opt/src/store_moves.rs b/crates/by_opt/src/store_moves.rs index 85af9b2d62..c99170579e 100644 --- a/crates/by_opt/src/store_moves.rs +++ b/crates/by_opt/src/store_moves.rs @@ -117,12 +117,45 @@ struct Handover { release: usize, } +/// each slot of a container build that reads a register, and the type the slot holds +/// +/// a typed tuple's slots each have their own type, read off the destination; a `list` or +/// a `tuple` object holds objects, whatever the destination register is declared as. a +/// `set` and a `dict` are not here: their insertions take references of their own, so +/// there is nothing for a slot of one to be handed +struct BuiltSlots { + /// each slot that reads a register, by index into the build's items + slots: Vec<(usize, RegisterId)>, + /// the type every slot holds, where the build settles it rather than the + /// destination register's own type doing so + held: Option, +} + +fn built_slots(op: &Op) -> Option { + let (items, held) = match op { + Op::TupleBuild { items, .. } => (items, None), + Op::BuildTuple { items, .. } | Op::BuildList { items, .. } => (items, Some(RType::OBJECT)), + _ => return None, + }; + let slots = items + .iter() + .enumerate() + .filter_map(|(slot, item)| match item { + Value::Register(source) => Some((slot, *source)), + _ => None, + }) + .collect(); + Some(BuiltSlots { slots, held }) +} + /// turn each read of a dying place into a move: a copy, an element read off a tuple, or -/// one slot of a tuple being built +/// one slot of a container being built fn move_dying_copies(function: &mut Function) { let mut moved: Vec = Vec::new(); for (index, block) in function.blocks.iter().enumerate() { for (position, op) in block.ops.iter().enumerate() { + // where a built slot holds its own type rather than the destination's + let mut held: Option = None; let reads: Vec<(Option, RegisterId, Vec)> = match op { Op::Assign { dest: _, @@ -133,15 +166,17 @@ fn move_dying_copies(function: &mut Function) { src: Value::Register(source), index, } => vec![(None, *source, vec![*index])], - Op::TupleBuild { items, .. } => items - .iter() - .enumerate() - .filter_map(|(slot, item)| match item { - Value::Register(source) => Some((Some(slot), *source, Vec::new())), - _ => None, - }) - .collect(), - _ => continue, + built => { + let Some(built) = built_slots(built) else { + continue; + }; + held = built.held; + built + .slots + .into_iter() + .map(|(slot, source)| (Some(slot), source, Vec::new())) + .collect() + } }; let Some(dest) = op.dest() else { continue; @@ -153,14 +188,16 @@ fn move_dying_copies(function: &mut Function) { Some(slot) => vec![slot], None => Vec::new(), }; + let written = match &held { + Some(ty) => Some(ty), + None => function + .register(dest) + .and_then(|decl| decl.ty.element(&slot)), + }; let same_type = function .register(source) .and_then(|decl| decl.ty.element(&path)) - .zip( - function - .register(dest) - .and_then(|decl| decl.ty.element(&slot)), - ) + .zip(written) .is_some_and(|(read, written)| read == written); if dest == source || !same_type @@ -210,8 +247,13 @@ fn move_dying_copies(function: &mut Function) { continue; }; if let Some(slot) = handover.item { - if let Op::TupleBuild { moves, .. } = op { - moves.insert(slot); + match op { + Op::TupleBuild { moves, .. } + | Op::BuildTuple { moves, .. } + | Op::BuildList { moves, .. } => { + moves.insert(slot); + } + _ => {} } continue; } diff --git a/crates/by_opt/src/str_concat_int.rs b/crates/by_opt/src/str_concat_int.rs index 41addf69d2..deb3fe4323 100644 --- a/crates/by_opt/src/str_concat_int.rs +++ b/crates/by_opt/src/str_concat_int.rs @@ -116,6 +116,7 @@ fn chains(function: &Function) -> Vec { dest: checked, src, to, + .. }, Op::StrConcat { rhs, @@ -232,6 +233,7 @@ mod tests { dest: checked, src: Value::Register(digits), to: RType::STR, + proved: false, }); builder.push(Op::StrConcat { dest: out, @@ -308,6 +310,7 @@ mod tests { dest: checked, src: Value::Register(digits), to: RType::STR, + proved: false, }); builder.push(Op::StrConcat { dest: held, diff --git a/crates/by_rt/include/by.h b/crates/by_rt/include/by.h index 2ed71c54b5..6acdb23da2 100644 --- a/crates/by_rt/include/by.h +++ b/crates/by_rt/include/by.h @@ -108,10 +108,16 @@ typedef size_t ByTagged; /* a tagged word of 1 is a pointer of 0 with the tag set, so it can never be a * real object and is free to mean "an exception is set" */ #define BY_INT_ERROR ((ByTagged)1) -/* what a register holds once its reference has been let go of. a short zero rather - * than the error sentinel, because the sentinel is odd: releasing it again takes the - * branch kept for a real PyLongObject, and the emptied register is released again on - * every exit and wherever the next write lets go of what was there */ +/* a tagged word holding no reference, which a short zero is: releasing it takes the + * test the straight line already passes rather than the branch kept for a real + * PyLongObject. + * + * emitted code pointedly does *not* empty a register with this — it writes the error + * sentinel the register was declared with, so that one constant covers the register's + * whole lifetime and clang folds the later release away outright. see + * `RType::undefined` for the measurement. what is left here is the one place a "holds + * nothing" that is never compared against the sentinel is wanted: the initializer of + * an unboxed global's memo, which is cold until its generation is stamped */ #define BY_INT_EMPTY ((ByTagged)0) /* a float error sentinel overlaps a valid value, so an error must be confirmed * with PyErr_Occurred() — see RType::error_overlaps */ @@ -276,6 +282,22 @@ static inline PyObject *By_BoxInt(ByTagged x) { return o; } +/* the same, for a tagged value the caller owns and is done with + * + * a heap `int` is handed straight on: the word held one reference and the `PyObject *` + * that replaces it is that same reference, so there is nothing to take and nothing to + * give back. a short owns nothing, and the object is built exactly as above. + * + * boxing an owned tagged value with [`By_BoxInt`] instead is a retain the caller never + * releases — which is what made every compiled `-> int` entry leak one reference per + * call for a value too wide to be a short */ +static inline PyObject *By_BoxIntOwned(ByTagged x) { + if (By_IsShort(x)) { + return PyLong_FromSsize_t(By_ShortValue(x)); + } + return By_LongOf(x); +} + /* an integer literal too wide to be a short, which the module built once at init * * the tag is added and no reference is taken: like a string literal, the module owns @@ -339,6 +361,64 @@ static inline int By_OptionalAttr(PyObject *o, PyObject *name, PyObject **found) * checks it with `_soundness_check(value, target)`, and a compiled module makes the * same check in the same place. these are that function's two halves */ +#ifndef Py_GIL_DISABLED +/* refusals worked out earlier: a type, the version its attributes and bases had, and a + * class it does not derive from. the lookup and the walk that reach that answer cost a + * hundred instructions between them, and are asked again on every read of an optional and + * at every arm of a `match` a subject falls past. + * + * a write to the type's dict, to a base's, or to its `__bases__` moves the version, so a + * match is the answer both would give. the class is not held: a class the type does not + * derive from is not in its mro, and one arriving at the same address can only enter the + * mro through `__bases__`, which moves the version too. + * + * there are many entries because one is not how the question is asked. a `match` ladder + * walks a subject past every arm before the one that takes it, so a single entry is + * overwritten by the next arm before the next subject reaches the first one again — and + * two subject types alternating do the same to each other. one entry held three pairs in + * the `match_` benchmark and answered two of every five questions; the entry a pair takes + * here is settled by the pair alone, so a pair asked twice with anything in between still + * finds its own answer, and only another pair landing on the same entry displaces it. + * that row goes 12.21M instructions to 10.44M, a seventh of the whole row. + * + * the count is what makes that steady rather than lucky. the table is direct-mapped, so + * two pairs of the moment can land on one entry and take it from each other, and which + * pairs those are is decided by where a run's allocator put two type objects rather than + * by anything the program did. sixteen entries measured *bimodally* on that row — 10.44M + * on a run where its three pairs fell apart and 11.66M on one where two met — and + * sixty-four measured 10.44M every time. the table costs its own size in a module's + * uninitialised data and nothing else, and being wider is the whole of what buys the + * steadiness: a mix that carries the pointers' high bits down instead was measured too, + * and its multiplies cost more than the spreading saved. + * + * this is a cache with a validity test rather than an assumption: a hit is checked + * against the type's current version, and a miss only costs the walk that would have + * happened anyway. + * + * the free-threaded build has none of this. the entries are read and written without + * synchronisation, which is sound only because the thread holding the GIL is the only + * one running */ +#define BY_REFUSED_SLOTS 64 + +typedef struct { + PyTypeObject *type; + PyObject *class_; + unsigned int version; +} ByRefusal; + +static ByRefusal by_refusals[BY_REFUSED_SLOTS]; + +/* where a pair's answer lives + * + * both pointers are shifted past the bits an allocator's alignment holds fixed at zero, + * and by different amounts, so that neither a type asked about itself nor two pairs + * sharing a member folds to one entry */ +static inline size_t By_RefusalSlot(PyTypeObject *type, PyObject *cls) { + uintptr_t mixed = ((uintptr_t)type >> 4) ^ ((uintptr_t)cls >> 9); + return (size_t)(mixed & (BY_REFUSED_SLOTS - 1)); +} +#endif + /* `isinstance(o, cls)` for an object that is not exactly of class `cls` * * a class whose own class is `type` has no `__instancecheck__` to ask, and python's answer @@ -350,26 +430,14 @@ static inline int By_OptionalAttr(PyObject *o, PyObject *name, PyObject **found) static int By_SoundIsOther(PyObject *o, PyObject *cls) { static PyObject *dunder_class = NULL; static PyObject *object_class = NULL; -#ifndef Py_GIL_DISABLED - /* the last refusal worked out that way: a type, the version its attributes and bases - * had, and the class it does not derive from. the lookup and the walk cost a hundred - * instructions between them, and are asked again on every read of an optional. - * - * a write to the type's dict, to a base's, or to its `__bases__` moves the version, so - * a match is the answer both would give. the class is not held: a class the type does - * not derive from is not in its mro, and one arriving at the same address can only - * enter the mro through `__bases__`, which moves the version too */ - static PyTypeObject *refused_type = NULL; - static unsigned int refused_version = 0; - static PyObject *refused_class = NULL; -#endif PyTypeObject *type = Py_TYPE(o); if (!Py_IS_TYPE(cls, &PyType_Type) || type->tp_getattro != PyObject_GenericGetAttr) { return PyObject_IsInstance(o, cls); } #ifndef Py_GIL_DISABLED - if (type == refused_type && cls == refused_class && refused_version != 0 - && type->tp_version_tag == refused_version) { + ByRefusal *refusal = &by_refusals[By_RefusalSlot(type, cls)]; + if (refusal->type == type && refusal->class_ == cls && refusal->version != 0 + && type->tp_version_tag == refusal->version) { return 0; } #endif @@ -383,9 +451,9 @@ static int By_SoundIsOther(PyObject *o, PyObject *cls) { return PyObject_IsInstance(o, cls); } #ifndef Py_GIL_DISABLED - refused_type = type; - refused_version = type->tp_version_tag; - refused_class = cls; + refusal->type = type; + refusal->version = type->tp_version_tag; + refusal->class_ = cls; #endif return 0; } @@ -1357,6 +1425,87 @@ static inline PyObject *By_ObjIShr(PyObject *a, PyObject *b) { static inline PyObject *By_ObjNeg(PyObject *o) { return PyNumber_Negative(o); } static inline PyObject *By_ObjInvert(PyObject *o) { return PyNumber_Invert(o); } +/* ── the two questions python's comparison can be answered without asking ────── + * + * both forms of a compiled comparison ask these, and they are written once here so + * that the answer the two forms give can only be the same answer. what each form does + * with a yes differs — one wants a bit and the other the object python would have + * produced — and that is all that differs */ + +/* whether both operands are `int`s of exactly that type holding a value python stores + * in the object's header + * + * python compares a pair of the same type without asking either side to yield to the + * other, so `int`'s own comparison is what it reaches — and for a value stored that way + * that comparison is a comparison of those values. a subclass is deliberately not here: + * it may have replaced the comparison, and `bool` reaches this from `x == 0` */ +static inline int By_BothInlineInts(PyObject *a, PyObject *b) { +#if PY_VERSION_HEX >= 0x030C0000 + return BY_LIKELY(PyLong_CheckExact(a) && PyLong_CheckExact(b)) + && BY_LIKELY(PyUnstable_Long_IsCompact((PyLongObject *)a) + && PyUnstable_Long_IsCompact((PyLongObject *)b)); +#else + (void)a; + (void)b; + return 0; +#endif +} + +/* `a b` for a pair [`By_BothInlineInts`] answered yes for */ +static inline int By_InlineIntCompare(PyObject *a, PyObject *b, int op) { +#if PY_VERSION_HEX >= 0x030C0000 + Py_ssize_t x = PyUnstable_Long_CompactValue((PyLongObject *)a); + Py_ssize_t y = PyUnstable_Long_CompactValue((PyLongObject *)b); + switch (op) { + case Py_EQ: return x == y; + case Py_NE: return x != y; + case Py_LT: return x < y; + case Py_LE: return x <= y; + case Py_GT: return x > y; + default: return x >= y; + } +#else + (void)a; + (void)b; + (void)op; + return 0; +#endif +} + +/* whether the whole of python's answer is what the *right* operand's type says + * + * a left operand whose type inherits `object`'s comparison has nothing to say about `==` + * or `!=` beyond identity, so python's answer is whatever the right operand's type says, + * and identity if that declines too. asking the right operand first is python's own order + * wherever it reaches an answer at all: it asks the right side first when that side's type + * derives from the left's, and otherwise asks it second, after a `NotImplemented` the + * inherited comparison gives for every pair but an identical one — which is why identity + * is left to the general path, along with the orderings, whose default is a `TypeError` + * rather than an answer. + * + * this is a ladder of `case` arms against literals, where the subject is an instance of + * some class and each literal is an `int` or a `str` that refuses it. it also skips the + * recursion guard `PyObject_RichCompare` enters, which costs one level of headroom once + * rather than a level per round of a recursion that goes through it — the call it makes + * next is counted either way. see runtime.md#how-deep-a-recursion-goes */ +static inline int By_RightOperandDecides(PyObject *a, PyObject *b, int op) { + (void)b; + return (op == Py_EQ || op == Py_NE) && a != b + && Py_TYPE(a)->tp_richcompare == PyBaseObject_Type.tp_richcompare; +} + +/* what the right operand's type answers for a pair [`By_RightOperandDecides`] said yes + * for, or `Py_NotImplemented` where it has nothing to say — a new reference either way, + * or NULL with an exception set + * + * `==` and `!=` are each their own reflection, so the operands swap and the operator + * does not */ +static inline PyObject *By_AskRightOperand(PyObject *a, PyObject *b, int op) { + richcmpfunc other = Py_TYPE(b)->tp_richcompare; + if (other == NULL) return Py_NewRef(Py_NotImplemented); + return other(b, a, op); +} + /* `a b` through the abstract protocol, as a bit — and 2 means an exception is set * * this is `PyObject_RichCompareBool` with its one shortcut removed. that function @@ -1380,6 +1529,20 @@ static inline PyObject *By_ObjInvert(PyObject *o) { return PyNumber_Invert(o); } * would dereference it — [`By_StrCompare`] hands its own such pair straight here */ static inline char By_ObjCompare(PyObject *a, PyObject *b, int op) { if (BY_UNLIKELY(a == NULL || b == NULL)) return 2; + /* the bit is reached without a `bool` ever being made */ + if (By_BothInlineInts(a, b)) return (char)By_InlineIntCompare(a, b, op); + if (By_RightOperandDecides(a, b, op)) { + PyObject *answer = By_AskRightOperand(a, b, op); + if (answer == NULL) return 2; + if (answer != Py_NotImplemented) { + int truth = PyObject_IsTrue(answer); + Py_DECREF(answer); + return truth < 0 ? 2 : (char)truth; + } + Py_DECREF(answer); + /* the right operand declined too, so identity decides */ + return (char)(op == Py_NE); + } PyObject *result = PyObject_RichCompare(a, b, op); if (result == NULL) return 2; int truth = PyObject_IsTrue(result); @@ -1387,6 +1550,32 @@ static inline char By_ObjCompare(PyObject *a, PyObject *b, int op) { return truth < 0 ? 2 : (char)truth; } +/* `a b` through the abstract protocol, as the object python's comparison answered + * — which need not be a `bool`, since a `__eq__` may answer anything at all + * + * this is `PyObject_RichCompare` with the same two questions asked in front of it that + * [`By_ObjCompare`] asks, and each of them produces the very object that function would + * have produced: `int`'s own comparison answers `Py_True` or `Py_False`, and a right + * operand that answers is answering through the slot `do_richcompare` would have called. + * where the right operand declines, python's own fallthrough for `==` and `!=` is + * identity, and this pair is not identical + * + * a NULL operand carries an exception set by whatever produced it */ +static inline PyObject *By_ObjRichCompare(PyObject *a, PyObject *b, int op) { + if (BY_UNLIKELY(a == NULL || b == NULL)) return NULL; + if (By_BothInlineInts(a, b)) { + return Py_NewRef(By_InlineIntCompare(a, b, op) ? Py_True : Py_False); + } + if (By_RightOperandDecides(a, b, op)) { + PyObject *answer = By_AskRightOperand(a, b, op); + if (answer == NULL) return NULL; + if (answer != Py_NotImplemented) return answer; + Py_DECREF(answer); + return Py_NewRef(op == Py_NE ? Py_True : Py_False); + } + return PyObject_RichCompare(a, b, op); +} + /* the type python names in an `AttributeError` * * the instance's *own* type, not the class that declared the field: a subclass @@ -2397,17 +2586,34 @@ typedef struct { int watcher; } ByGlobals; -/* what this module's own sites read. NULL means this module memoises nothing */ -static ByGlobals *by_globals = NULL; +/* what this module's own sites read + * + * it always names a struct, so that a site's test is the one compare it is about rather + * than that compare behind a test for whether there is anything to compare against. a + * module that memoises nothing is pointed at the struct below instead, which carries a + * generation nothing ever stamps: the counter starts at one and is moved by one write + * to a namespace at a time, so reaching this value would take more writes than a + * process can make, and a cold site's zero is no nearer it. so every site misses, on + * every read, which is what "memoises nothing" has to mean. + * + * its watcher is the identifier `PyDict_AddWatcher` never hands out, and nothing reads + * it: every path to the watcher asks [`BY_GLOBALS_FOUND`] first */ +static ByGlobals by_globals_off = { UINT64_MAX, -1 }; + +static ByGlobals *by_globals = &by_globals_off; + +/* whether this module found a counter to share, which is the question the old spelling + * `by_globals != NULL` asked */ +#define BY_GLOBALS_FOUND (by_globals != &by_globals_off) /* what this module's callback bumps, if this is the module that registered the * watcher. it is deliberately *not* the same variable * * only one module in an interpreter registers, and every other module's sites are * invalidated by that one module's callback. so a module that gives up memoising — - * by clearing `by_globals` — must not be able to take the invalidation of everyone - * else's sites with it. this is set once, when the registration succeeds, and never - * cleared */ + * by pointing `by_globals` back at the off struct — must not be able to take the + * invalidation of everyone else's sites with it. this is set once, when the + * registration succeeds, and never cleared */ static ByGlobals *by_globals_watched = NULL; static int By_GlobalsChanged(PyDict_WatchEvent event, PyObject *dict, PyObject *key, @@ -2427,16 +2633,16 @@ static int By_GlobalsChanged(PyDict_WatchEvent event, PyObject *dict, PyObject * * freeing it while any of them could still run would turn a memo into a read of * freed memory. it is two words per interpreter. * - * a failure at any step is answered by leaving `by_globals` NULL, which is what every - * site tests before trusting itself — a build that cannot be told about writes makes - * every lookup in full rather than making one it cannot invalidate */ + * a failure at any step is answered by leaving `by_globals` at the off struct, whose + * generation no site can match — a build that cannot be told about writes makes every + * lookup in full rather than making one it cannot invalidate */ static void By_FindGlobals(void) { static const char *key = "_by_global_memo"; PyInterpreterState *interpreter; PyObject *state; PyObject *capsule; ByGlobals *shared; - if (by_globals != NULL) return; + if (BY_GLOBALS_FOUND) return; interpreter = PyInterpreterState_Get(); if (interpreter == NULL) return; state = PyInterpreterState_GetDict(interpreter); @@ -2444,7 +2650,10 @@ static void By_FindGlobals(void) { capsule = PyDict_GetItemString(state, key); if (capsule != NULL) { by_globals = (ByGlobals *)PyCapsule_GetPointer(capsule, key); - if (by_globals == NULL) PyErr_Clear(); + if (by_globals == NULL) { + by_globals = &by_globals_off; + PyErr_Clear(); + } return; } PyErr_Clear(); @@ -2478,7 +2687,7 @@ static void By_FindGlobals(void) { * every dict a site's answer can come from passes through here first, so a write * that this build would not hear about is a write no site was armed against */ static int By_WatchGlobals(PyObject *dict) { - if (by_globals == NULL || dict == NULL || !PyDict_Check(dict)) return 0; + if (!BY_GLOBALS_FOUND || dict == NULL || !PyDict_Check(dict)) return 0; if (PyDict_Watch(by_globals->watcher, dict) < 0) { PyErr_Clear(); return 0; @@ -2493,10 +2702,14 @@ static int By_WatchGlobals(PyObject *dict) { static void By_WatchModule(PyObject *dict) { By_FindGlobals(); if (!By_WatchGlobals(dict)) { - by_globals = NULL; + by_globals = &by_globals_off; return; } - by_globals->generation += 1; + /* asked again rather than inferred from the line above: the off struct's generation + * is one increment away from the zero a cold site carries, so a stamp that ever + * reached it would make every cold site in the module match at once. `By_WatchGlobals` + * already answers no for it, and this is that answer written where the write is */ + if (BY_GLOBALS_FOUND) by_globals->generation += 1; } /* resolve the name in full and record what answered, or record nothing @@ -2512,7 +2725,7 @@ static PyObject *By_ArmGlobalSite(ByGlobalSite *site, PyObject *dict, PyObject * PyObject *value; PyObject *answered; site->generation = 0u; - if (by_globals == NULL || dict == NULL || name == NULL) { + if (!BY_GLOBALS_FOUND || dict == NULL || name == NULL) { return By_LookupGlobal(dict, name); } generation = by_globals->generation; @@ -2549,7 +2762,7 @@ static void By_WatchModule(PyObject *dict) { (void)dict; } static inline PyObject *By_LookupGlobalSite(ByGlobalSite *site, PyObject *dict, PyObject *name) { #ifdef BY_GLOBAL_SITES - if (BY_LIKELY(by_globals != NULL && site->generation == by_globals->generation)) { + if (BY_LIKELY(site->generation == by_globals->generation)) { return Py_NewRef(site->value); } return By_ArmGlobalSite(site, dict, name); @@ -2559,6 +2772,148 @@ static inline PyObject *By_LookupGlobalSite(ByGlobalSite *site, PyObject *dict, #endif } +/* ── a global read whose only use is to unbox it ────────────────────────────── + * + * `while i < _limit` reads a name, is handed the object the namespace holds under it, + * narrows that object to an `int`, and lets the object go again — every trip. the memo + * above already spares the lookup, but not the reference it takes for an answer the + * caller wants only in order to take it apart. + * + * so the *unboxed* value is what is remembered, beside the same generation. what stays + * is the compare; what goes is a retain, a release and the narrowing. + * + * remembering the narrowed value is exactly as safe as remembering the object, because + * for these representations it either is not a reference at all or is the same borrow: + * + * - a `double` and a `char` are copied out of the object and have no lifetime of their + * own — a later write to the object cannot exist, since `float` and `bool` are + * immutable and a rebinding is a write to the namespace, which is what moves the + * counter + * - a tagged `int` is either a short, which is a copy in the same sense, or a pointer + * to the object the dict itself holds — borrowed on precisely the terms + * [`ByGlobalSite::value`] is borrowed on, and let go of by the very same event + * + * what is remembered is therefore only ever read back under the generation the answer + * was found under, and a site is stamped only where the boxed site was: that is what + * says the namespace that answered is one this build hears about being written to */ +typedef struct { + uint64_t generation; + union { + ByTagged tagged; + double number; + char bit; + } value; +} ByGlobalUnboxedSite; + +#define BY_GLOBAL_UNBOXED_SITE_INIT { 0u, { BY_INT_EMPTY } } + +/* whether the read that has just been made may be remembered in unboxed form, and under + * which generation — the boxed site's answer to both, since it is the one that decides + * what may be kept at all */ +static inline uint64_t By_UnboxedGlobalStamp(const ByGlobalSite *boxed) { +#ifdef BY_GLOBAL_SITES + return boxed->generation; +#else + (void)boxed; + return 0u; +#endif +} + +/* each of the three is the same shape: read the name in full, narrow it, stamp the memo + * where the boxed site was stamped, and hand the narrowed value back. they are apart + * only because each answers a different C type */ + +static ByTagged By_ArmGlobalTaggedSite(ByGlobalUnboxedSite *site, ByGlobalSite *boxed, + PyObject *dict, PyObject *name) { + PyObject *value; + ByTagged unboxed; + site->generation = 0u; + value = By_LookupGlobalSite(boxed, dict, name); + if (value == NULL) return BY_INT_ERROR; + unboxed = By_UnboxInt(value); + if (unboxed == BY_INT_ERROR) { + Py_DECREF(value); + return BY_INT_ERROR; + } + /* the narrowing answers an owned tagged value, and that one reference is the + * caller's: what the site keeps beside it is a *borrow*, on the same terms as + * [`ByGlobalSite::value`]. taking a second here would be a reference the site never + * gives back, since a re-arm writes over what it holds rather than releasing it — + * which showed up as a heap `int` gaining two references per rebinding */ + site->value.tagged = unboxed; + site->generation = By_UnboxedGlobalStamp(boxed); + Py_DECREF(value); + return unboxed; +} + +static inline ByTagged By_LookupGlobalTaggedSite(ByGlobalUnboxedSite *site, + ByGlobalSite *boxed, PyObject *dict, + PyObject *name) { +#ifdef BY_GLOBAL_SITES + if (BY_LIKELY(site->generation == by_globals->generation)) { + ByTagged held = site->value.tagged; + By_IncRefTagged(held); + return held; + } +#endif + return By_ArmGlobalTaggedSite(site, boxed, dict, name); +} + +/* a double has no value to spare for an error, so failure is reported the way every + * other float narrowing reports it: the answer is the sentinel and the caller confirms + * it against the thread's exception */ +static double By_ArmGlobalFloatSite(ByGlobalUnboxedSite *site, ByGlobalSite *boxed, + PyObject *dict, PyObject *name) { + PyObject *value; + double unboxed; + site->generation = 0u; + value = By_LookupGlobalSite(boxed, dict, name); + if (value == NULL) return BY_FLOAT_ERROR; + unboxed = By_UnboxFloat(value); + Py_DECREF(value); + if (unboxed == BY_FLOAT_ERROR && PyErr_Occurred()) return BY_FLOAT_ERROR; + site->value.number = unboxed; + site->generation = By_UnboxedGlobalStamp(boxed); + return unboxed; +} + +static inline double By_LookupGlobalFloatSite(ByGlobalUnboxedSite *site, + ByGlobalSite *boxed, PyObject *dict, + PyObject *name) { +#ifdef BY_GLOBAL_SITES + if (BY_LIKELY(site->generation == by_globals->generation)) { + return site->value.number; + } +#endif + return By_ArmGlobalFloatSite(site, boxed, dict, name); +} + +/* `bool` and `None` share this: both narrow to a byte, and both report failure as 2 */ +static char By_ArmGlobalBitSite(ByGlobalUnboxedSite *site, ByGlobalSite *boxed, + PyObject *dict, PyObject *name, int is_none) { + PyObject *value; + char unboxed; + site->generation = 0u; + value = By_LookupGlobalSite(boxed, dict, name); + if (value == NULL) return 2; + unboxed = is_none ? By_UnboxNone(value) : By_UnboxBool(value); + Py_DECREF(value); + if (unboxed == 2) return 2; + site->value.bit = unboxed; + site->generation = By_UnboxedGlobalStamp(boxed); + return unboxed; +} + +static inline char By_LookupGlobalBitSite(ByGlobalUnboxedSite *site, ByGlobalSite *boxed, + PyObject *dict, PyObject *name, int is_none) { +#ifdef BY_GLOBAL_SITES + if (BY_LIKELY(site->generation == by_globals->generation)) { + return site->value.bit; + } +#endif + return By_ArmGlobalBitSite(site, boxed, dict, name, is_none); +} + /* ── a module function a compiled caller reaches directly ───────────────────── * * a compiled call to a function the module defines goes straight to its native entry, @@ -4356,11 +4711,40 @@ static inline void By_ArmMethod(ByMethodLicence *licence, PyObject *type, const licence->version = owner->tp_version_tag; } +/* a receiver an emitted site could not have had + * + * the two tests below read the receiver's type without asking first whether there is a + * receiver. they can, because an emitted site has no way to reach them without one: + * + * - every read of a register is on a path that has written it, which the IR verifier + * proves, and a register a *name* may not have been bound in yet carries a byte that + * every read of it tests — so the path this is about raises `UnboundLocalError` + * - an operation that can fail takes its error edge before it writes its destination, + * so a written register holds what the operation answered and never the failure + * - a register a store or a display took the reference over from is one liveness has + * already said nothing reads again, and a *parameter* can never be one of those at + * all: handing over is refused for anything below the parameter count + * + * asking anyway cost 5.2 per cent of the `props_ext` benchmark and 1.5 of `inherit`, + * because the `&&` before the load is a sequence point the C compiler may not move the + * type read across: the whole chain stays a ladder of branches rather than folding into + * a run of straight-line loads and bitwise tests. + * + * the claim is not simply dropped, though: the re-check build is where a licence's + * claims are asked out loud, and this is one of them */ +#ifdef BY_LICENCE_RECHECK +#define BY_LICENCE_RECEIVER(o, where) \ + do { if ((o) == NULL) Py_FatalError("by: " where " was given no receiver"); } while (0) +#else +#define BY_LICENCE_RECEIVER(o, where) ((void)0) +#endif + /* whether `o` is exactly `type`, `type` still answers as [`By_ArmMethod`] found, and * `o` itself does not answer the name */ static inline char By_MethodStands(PyObject *o, PyObject *type, const ByMethodLicence *licence) { - return (char)(licence->version != 0u && o != NULL && (PyObject *)Py_TYPE(o) == type + BY_LICENCE_RECEIVER(o, "By_MethodStands"); + return (char)(licence->version != 0u && (PyObject *)Py_TYPE(o) == type && ((PyTypeObject *)type)->tp_version_tag == licence->version && !By_DictShadowsAt(o, licence->dict_offset, licence->name)); } @@ -4468,7 +4852,8 @@ static inline void By_JoinLicence(unsigned int *version, int *first, unsigned in /* whether `o` is exactly `type` and `type` still answers as [`By_ArmAccessor`] found */ static inline char By_AccessorStands(PyObject *o, PyObject *type, const ByAccessorLicence *licence) { - return (char)(licence->version != 0u && o != NULL && (PyObject *)Py_TYPE(o) == type + BY_LICENCE_RECEIVER(o, "By_AccessorStands"); + return (char)(licence->version != 0u && (PyObject *)Py_TYPE(o) == type && ((PyTypeObject *)type)->tp_version_tag == licence->version); } @@ -4920,8 +5305,14 @@ static inline PyObject *By_StrUpper(ByMethodSite *site, PyObject *receiver, PyOb * a type flagged as a sequence, which `str`, `bytes` and `bytearray` are not — so * `case [a, b]:` never takes a two-character string apart */ +/* a class pattern's own test is `isinstance`, and `By_SoundIs` is that function for one + * class. it matters most when the answer is no, which is the common answer in a ladder + * of cases: `PyObject_IsInstance` asks the subject for its `__class__` every time it + * decides against, and that question runs nothing at all for a subject whose type reads + * attributes the generic way and inherits `object`'s own `__class__` — the two facts + * `By_SoundIsOther` establishes before it skips the lookup */ static inline char By_IsInstance(PyObject *o, PyObject *class_) { - int result = PyObject_IsInstance(o, class_); + int result = By_SoundIs(o, class_); return result < 0 ? 2 : (char)result; } @@ -9108,20 +9499,18 @@ static inline PyObject *By_BuildList(PyObject **items, Py_ssize_t nargs) { return list; } -/* build a dict from alternating key/value owned references */ +/* build a dict from alternating key/value *borrowed* references + * + * a tuple or a list takes a reference over, so an item goes into one by being handed + * the caller's. a dict and a set do not: `PyDict_SetItem` and `PySet_Add` take a + * reference of their own, so handing one over would be a reference made at the call + * site only to be dropped again here — two operations per key and two per value, for + * nothing. these two borrow instead, and the caller goes on owning what it passed */ static inline PyObject *By_BuildDict(PyObject **pairs, Py_ssize_t count) { PyObject *dict = PyDict_New(); - if (dict == NULL) { - for (Py_ssize_t i = 0; i < count * 2; i++) Py_XDECREF(pairs[i]); - return NULL; - } + if (dict == NULL) return NULL; for (Py_ssize_t i = 0; i < count; i++) { - int failed = PyDict_SetItem(dict, pairs[i * 2], pairs[i * 2 + 1]) < 0; - Py_XDECREF(pairs[i * 2]); - Py_XDECREF(pairs[i * 2 + 1]); - if (failed) { - /* release whatever is left, then the dict */ - for (Py_ssize_t j = (i + 1) * 2; j < count * 2; j++) Py_XDECREF(pairs[j]); + if (PyDict_SetItem(dict, pairs[i * 2], pairs[i * 2 + 1]) < 0) { Py_DECREF(dict); return NULL; } @@ -9131,15 +9520,9 @@ static inline PyObject *By_BuildDict(PyObject **pairs, Py_ssize_t count) { static inline PyObject *By_BuildSet(PyObject **items, Py_ssize_t count) { PyObject *set = PySet_New(NULL); - if (set == NULL) { - for (Py_ssize_t i = 0; i < count; i++) Py_XDECREF(items[i]); - return NULL; - } + if (set == NULL) return NULL; for (Py_ssize_t i = 0; i < count; i++) { - int failed = PySet_Add(set, items[i]) < 0; - Py_XDECREF(items[i]); - if (failed) { - for (Py_ssize_t j = i + 1; j < count; j++) Py_XDECREF(items[j]); + if (PySet_Add(set, items[i]) < 0) { Py_DECREF(set); return NULL; } @@ -10540,7 +10923,7 @@ static char By_ArmBuiltinSite(ByBuiltinSite *site, PyObject *dict, const char *b * the answer is one comparison and a load: it is asked every trip round a loop */ static inline char By_BuiltinStands(ByBuiltinSite *site, PyObject *dict, const char *builtin) { #ifdef BY_GLOBAL_SITES - if (BY_LIKELY(by_globals != NULL && site->lookup.generation == by_globals->generation)) { + if (BY_LIKELY(site->lookup.generation == by_globals->generation)) { return site->answer; } #endif diff --git a/docs/basedpython/development/compilation/runtime.md b/docs/basedpython/development/compilation/runtime.md index 2cf1c77c79..4818e60b10 100644 --- a/docs/basedpython/development/compilation/runtime.md +++ b/docs/basedpython/development/compilation/runtime.md @@ -788,6 +788,17 @@ sys.setrecursionlimit(100) mod.nested_depth(98) # python: RecursionError; compiled: 98 ``` +`==` and `!=` are one more such call. where the left operand's type has no comparison of +its own, python's answer is whatever the right operand's type says, and a compiled module +asks that type directly instead of going through `PyObject_RichCompare` — which leaves out +the recursion entry that function makes. that entry is against the C stack rather than the +frame count, so it is not what stops a recursion at the limit: a recursion through `__eq__` +stops at the same depth with the same message either way. what it costs is one level of +headroom, once, in a program that has raised the limit far enough for the stack to be the +thing that runs out. measured on 3.13 and 3.14, a recursion through `__eq__` reaches within +one level of the depth the same recursion reaches when the comparison is made to go the +long way round. + counting costs most on a body that does little but recurse — half again the instructions of a plain `fib` — so `follow-recursion-limit = false` in the project's `[tool.ty.compile]` table leaves the count out and watches the stack alone. a compiled From ea90321b657b8f2cf996f7697068855f3de56c41 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Mon, 14 Sep 2026 12:09:26 +1000 Subject: [PATCH 09/16] native-bench: stage only a leg's own source, scope cdiff per definition, and add inherit_ov and loops_aug rows bench.py stages only the source a leg builds from so .by rows build again, the benchmark programs are annotated as their docstrings say, and the sweep renderer prints a compiled method as a callable. cdiff scopes a by.h change definition by definition and writes its summary into its output file, ci runs the scoper's tests, and two benchmark rows measure @override and augmented assignment against their plain siblings --- .github/workflows/ci.yaml | 4 + scripts/native-bench/bench.py | 36 +- scripts/native-bench/programs.toml | 11 + scripts/native-bench/programs/dicthist.py | 4 +- scripts/native-bench/programs/excs.py | 16 +- scripts/native-bench/programs/fields.py | 2 +- scripts/native-bench/programs/gen.py | 4 +- scripts/native-bench/programs/generic.py | 23 +- scripts/native-bench/programs/generic_mono.py | 9 +- scripts/native-bench/programs/inherit_ov.py | 56 ++ scripts/native-bench/programs/keybuild.py | 5 +- scripts/native-bench/programs/loops_aug.py | 31 + scripts/native-bench/programs/props.py | 4 +- scripts/native-bench/programs/props_ext.py | 4 +- scripts/native-bench/programs/props_get.py | 2 +- scripts/native-sweeps/cdiff.sh | 38 +- scripts/native-sweeps/compare.sh | 5 +- scripts/native-sweeps/headerscope.py | 676 ++++++++++++++++++ scripts/native-sweeps/sweeplib.sh | 86 +-- scripts/native-sweeps/test_headerscope.py | 241 +++++++ 20 files changed, 1163 insertions(+), 94 deletions(-) create mode 100644 scripts/native-bench/programs/inherit_ov.py create mode 100644 scripts/native-bench/programs/loops_aug.py create mode 100644 scripts/native-sweeps/headerscope.py create mode 100644 scripts/native-sweeps/test_headerscope.py diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2bc0d2bcfe..694c852bcc 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -883,6 +883,10 @@ jobs: # The build backend decides what the staged `pyproject.toml` says and where # a dynamic version comes from. Neither is reachable from the Rust tests. - run: uv run --no-project --with pytest pytest scripts/test_build_backend.py + # the header scoper decides which modules a `by.h` change can reach, and a + # narrowing that is wrong hides a module from every behavioural sweep over the + # native compiler. nothing in the rust tests runs it + - run: uv run --no-project --with pytest pytest scripts/native-sweeps/test_headerscope.py # Lint/format/type-check py-fuzzer # (dogfooding with ty is done in a separate job) - run: uv run --directory=./python/py-fuzzer mypy diff --git a/scripts/native-bench/bench.py b/scripts/native-bench/bench.py index a8d8cfe49a..e8cf8ab534 100644 --- a/scripts/native-bench/bench.py +++ b/scripts/native-bench/bench.py @@ -448,20 +448,32 @@ def stage( `lowered` is the python this build's module name is given: the program itself for a `.py` benchmark, and the transpiler's output for a `.by` one. - every leg is staged the same way, including the two that go on to compile - the basedpython source instead — staging that differed per leg is how a - build ends up measuring something the others are not + every leg gets exactly one source under its module name — the one it builds + from, see `builds_from_by_source` """ directory = root / program.name / leg (directory / "dist").mkdir(parents=True) module = f"{program.name}_{leg}" - shutil.copy(lowered, directory / f"{module}.py") - if program.language == "by": + if builds_from_by_source(program.language, leg): shutil.copy(program.source, directory / f"{module}.by") + else: + shutil.copy(lowered, directory / f"{module}.py") write_project(directory, python_version) return directory +def builds_from_by_source(language: str, leg: str) -> bool: + """whether a leg compiles the basedpython source rather than the python it lowers to + + the two `by compile` legs of a `.by` row take the source as written, and the + interpreted and mypyc legs run its lowering. a leg is staged with only its own + source: `by compile` refuses a directory holding both `.by` and + `.py`, since both would build to the same module, and that refusal took + every `.by` row out of the suite while each one was staged with both + """ + return language == "by" and leg in ("by", "control") + + def build_by( by: Path, directory: Path, source: str, python: str ) -> tuple[bool, str | None, list[str]]: @@ -1200,6 +1212,20 @@ def lower_canary(where: str, source: str) -> tuple[Path | None, str | None]: else: print(" accepted a basedpython program and kept the python it lowered to") + # and a `.by` program staged for a compiled leg has to build. the lowering + # canaries above never reach `build_by`, which is how a staging that `by + # compile` refused went on taking every `.by` row out of the suite unnoticed + compiled = root / "compilable" + (compiled / "dist").mkdir(parents=True) + write_project(compiled, python_version) + staged = "canary_by.by" if builds_from_by_source("by", "by") else "canary_by.py" + (compiled / staged).write_text("def bench() -> int:\n return 1\n") + built, refusal, _ = build_by(by, compiled, staged, python) + if not built: + failures.append(f"a basedpython program staged for a compiled leg: {refusal}") + else: + print(" compiled a basedpython program staged for a compiled leg") + # one name cannot be two programs. the extension is what says how a # benchmark is built, so a name carrying both is the one thing it cannot # decide, and the harness says so rather than building whichever it globbed diff --git a/scripts/native-bench/programs.toml b/scripts/native-bench/programs.toml index 0dbdd1267d..28ebbd1ef6 100644 --- a/scripts/native-bench/programs.toml +++ b/scripts/native-bench/programs.toml @@ -46,6 +46,11 @@ group = "int" measures = "tagged-integer arithmetic with no container, no call and no float" declines = 0 +[programs.loops_aug] +group = "int" +measures = "the `loops` arithmetic written with augmented assignment" +declines = 0 + [programs.bigint] group = "int" measures = "integer arithmetic that leaves the machine word" @@ -72,6 +77,12 @@ measures = "an override reached through a base-typed local" declines = 0 setup = true +[programs.inherit_ov] +group = "dispatch" +measures = "the `inherit` override marked `@override`" +declines = 0 +setup = true + [programs.alloc] group = "memory" measures = "an object built and dropped every iteration" diff --git a/scripts/native-bench/programs/dicthist.py b/scripts/native-bench/programs/dicthist.py index 54288a680c..2abc9ae02b 100644 --- a/scripts/native-bench/programs/dicthist.py +++ b/scripts/native-bench/programs/dicthist.py @@ -18,7 +18,9 @@ def counted(words: list[str]) -> int: seen[word] = seen[word] + 1 else: seen[word] = 1 - return len(seen) + # the first word's count is read back so that a store dropped on the hit path + # changes the answer, which the number of distinct words alone would not + return len(seen) + seen[words[0]] def repeated(words: list[str], passes: int) -> int: diff --git a/scripts/native-bench/programs/excs.py b/scripts/native-bench/programs/excs.py index 7c38d05181..d28a093d4a 100644 --- a/scripts/native-bench/programs/excs.py +++ b/scripts/native-bench/programs/excs.py @@ -7,7 +7,9 @@ two shapes, because they cost differently. `caught` raises and catches across a call boundary, which is the expensive one. `guarded` never raises at all and measures what merely being inside a `try` costs a loop that succeeds — which -should be nothing, and is worth knowing rather than assuming +should be nothing, and is worth knowing rather than assuming. its callee is +`bounded`, which does almost no work, so that a cost the `try` added would be +most of the row rather than hidden under `parse`'s character scan """ @@ -26,6 +28,12 @@ def parse(text: str) -> int: return total +def bounded(value: int) -> int: + if value < 0: + raise Refused("negative") + return value % 7 + + def caught(rounds: int) -> int: total = 0 i = 0 @@ -39,12 +47,12 @@ def caught(rounds: int) -> int: def guarded(rounds: int) -> int: - """the same loop with a handler that is never reached""" + """a loop inside a `try` whose handler is never reached""" total = 0 i = 0 while i < rounds: try: - total = total + parse("abc") + total = total + bounded(i) except Refused: total = total + 1 i = i + 1 @@ -52,4 +60,4 @@ def guarded(rounds: int) -> int: def bench() -> int: - return caught(20000) + guarded(20000) + return caught(20000) + guarded(200000) diff --git a/scripts/native-bench/programs/fields.py b/scripts/native-bench/programs/fields.py index d908af06a3..37c166e071 100644 --- a/scripts/native-bench/programs/fields.py +++ b/scripts/native-bench/programs/fields.py @@ -7,7 +7,7 @@ class State: - def __init__(self): + def __init__(self) -> None: self.a = 0 self.b = 1 self.c = 2 diff --git a/scripts/native-bench/programs/gen.py b/scripts/native-bench/programs/gen.py index 24ffd768f2..a37bcb57a5 100644 --- a/scripts/native-bench/programs/gen.py +++ b/scripts/native-bench/programs/gen.py @@ -6,8 +6,10 @@ it is also the half real code writes far more often """ +from collections.abc import Iterator -def steps(n: int): + +def steps(n: int) -> Iterator[int]: i = 0 while i < n: yield (i * 7) % 13 diff --git a/scripts/native-bench/programs/generic.py b/scripts/native-bench/programs/generic.py index ff007240ab..380f3a9059 100644 --- a/scripts/native-bench/programs/generic.py +++ b/scripts/native-bench/programs/generic.py @@ -1,19 +1,24 @@ -"""a list built in a hot loop and handed to a generic function +"""a list built in a hot loop and handed to a generic function that reads it -the type parameter is what stops the list being a buffer: the element -representation is not known at the definition, so neither the callee's body nor -the caller's list can be unboxed. `generic_mono` is the same program with the -call monomorphised by hand, and the gap between the two is the whole measurement +the type parameter leaves the element type unknown where `consume` is written, +so the comparison in its body is python's `==` on whatever the lists hold, unless +the compiler specializes the call for the list it is given. `generic_mono` is the +same program with the call monomorphised by hand, so the comparison is between +two floats, and the gap between the two rows is what the type parameter costs + +the body has to read the elements for that to be measured at all: a body that +only counts them never touches `T`, and both rows would compile to the same code """ def consume[T](xs: list[T], ys: list[T]) -> int: - n = 0 + matched = 0 i = 0 while i < len(xs): - n = n + 1 + if xs[i] == ys[i]: + matched = matched + 1 i = i + 1 - return n + return matched def bench() -> int: @@ -25,7 +30,7 @@ def bench() -> int: i = 0 while i < 2000: xs.append(i * 0.5) - ys.append(i * 0.25) + ys.append((i - i % 2) * 0.5) i = i + 1 total = total + consume(xs, ys) r = r + 1 diff --git a/scripts/native-bench/programs/generic_mono.py b/scripts/native-bench/programs/generic_mono.py index e2b1047875..67baca5907 100644 --- a/scripts/native-bench/programs/generic_mono.py +++ b/scripts/native-bench/programs/generic_mono.py @@ -7,12 +7,13 @@ def consume(xs: list[float], ys: list[float]) -> int: - n = 0 + matched = 0 i = 0 while i < len(xs): - n = n + 1 + if xs[i] == ys[i]: + matched = matched + 1 i = i + 1 - return n + return matched def bench() -> int: @@ -24,7 +25,7 @@ def bench() -> int: i = 0 while i < 2000: xs.append(i * 0.5) - ys.append(i * 0.25) + ys.append((i - i % 2) * 0.5) i = i + 1 total = total + consume(xs, ys) r = r + 1 diff --git a/scripts/native-bench/programs/inherit_ov.py b/scripts/native-bench/programs/inherit_ov.py new file mode 100644 index 0000000000..78e55d255c --- /dev/null +++ b/scripts/native-bench/programs/inherit_ov.py @@ -0,0 +1,56 @@ +"""an override marked `@override`, reached through a base-typed local + +the same dispatch as `inherit`, with the one difference that `Square.area` says +it overrides. `typing.override` only sets `__override__` on the function and +hands it back, so the two rows should cost the same: a gap between them is a +marking decorator changing what gets compiled, which is how idiomatic overrides +once ran interpreted while the census counted them compiled + +the mixed list is built by `setup`, as in `inherit`, so only the dispatches are +counted +""" + +from typing import override + + +class Shape: + def __init__(self, size: int): + self.size = size + + def area(self) -> int: + return self.size + + +class Square(Shape): + @override + def area(self) -> int: + return self.size * self.size + + +def total(shapes: list[Shape], passes: int) -> int: + running = 0 + p = 0 + while p < passes: + i = 0 + while i < len(shapes): + running = running + shapes[i].area() + i = i + 1 + p = p + 1 + return running + + +_shapes: list[Shape] = [] + + +def setup() -> None: + i = 0 + while i < 200: + if i % 2 == 0: + _shapes.append(Shape(i)) + else: + _shapes.append(Square(i)) + i = i + 1 + + +def bench() -> int: + return total(_shapes, 300) diff --git a/scripts/native-bench/programs/keybuild.py b/scripts/native-bench/programs/keybuild.py index 1efa72e6ef..4db8fbef0a 100644 --- a/scripts/native-bench/programs/keybuild.py +++ b/scripts/native-bench/programs/keybuild.py @@ -2,7 +2,8 @@ there is no dict here at all. this expression sat in every loop of the dict benchmarks and was most of what they measured, so it is timed on its own and -they hand their keys in already built. `len` is called once, at the end +they hand their keys in already built. `len` and `int` are called once, at the +end, and the digits are read back so that a wrong key changes the answer """ @@ -15,7 +16,7 @@ def keys(n: int, passes: int) -> int: last = "k" + str(i) i = i + 1 p = p + 1 - return len(last) + return len(last) + int(last[1:]) def bench() -> int: diff --git a/scripts/native-bench/programs/loops_aug.py b/scripts/native-bench/programs/loops_aug.py new file mode 100644 index 0000000000..523852c651 --- /dev/null +++ b/scripts/native-bench/programs/loops_aug.py @@ -0,0 +1,31 @@ +"""the arithmetic of `loops`, written with augmented assignment + +`n //= 2` and `running += …` ask an `int` subclass its in-place method before +its plain one, where `n = n // 2` does not, so the two spellings take different +slow paths. on exact ints they must cost the same: a gap between this row and +`loops` is the price of the in-place question leaking onto the fast path +""" + + +def collatz(n: int) -> int: + steps = 0 + while n != 1: + if n % 2 == 0: + n //= 2 + else: + n = 3 * n + 1 + steps += 1 + return steps + + +def total(limit: int) -> int: + running = 0 + i = 1 + while i < limit: + running += collatz(i) + i += 1 + return running + + +def bench() -> int: + return total(6000) diff --git a/scripts/native-bench/programs/props.py b/scripts/native-bench/programs/props.py index 128de4357a..c0a14e3bbf 100644 --- a/scripts/native-bench/programs/props.py +++ b/scripts/native-bench/programs/props.py @@ -9,7 +9,7 @@ class Cell: - def __init__(self): + def __init__(self) -> None: self._v = 0 @property @@ -17,7 +17,7 @@ def v(self) -> int: return self._v @v.setter - def v(self, given: int): + def v(self, given: int) -> None: self._v = given diff --git a/scripts/native-bench/programs/props_ext.py b/scripts/native-bench/programs/props_ext.py index b1ea6d968e..64076ea038 100644 --- a/scripts/native-bench/programs/props_ext.py +++ b/scripts/native-bench/programs/props_ext.py @@ -10,7 +10,7 @@ class Cell: - def __init__(self): + def __init__(self) -> None: self._v = 0 @property @@ -18,7 +18,7 @@ def v(self) -> int: return self._v @v.setter - def v(self, given: int): + def v(self, given: int) -> None: self._v = given diff --git a/scripts/native-bench/programs/props_get.py b/scripts/native-bench/programs/props_get.py index 530c1f1aba..cf55c913e3 100644 --- a/scripts/native-bench/programs/props_get.py +++ b/scripts/native-bench/programs/props_get.py @@ -9,7 +9,7 @@ class Cell: - def __init__(self): + def __init__(self) -> None: self._v = 1 @property diff --git a/scripts/native-sweeps/cdiff.sh b/scripts/native-sweeps/cdiff.sh index 1a17d16565..9948f52fce 100755 --- a/scripts/native-sweeps/cdiff.sh +++ b/scripts/native-sweeps/cdiff.sh @@ -23,14 +23,19 @@ # to walk". a module whose C is unchanged but which calls a helper whose definition # changed is reported as `header`, so the population to pair the behavioural rungs over is # `differs` plus `header` +# +# the summary is written into `OUT` behind a `# ` as well as to the terminal, so a waiter +# watching the file for `walked:` sees the run end — see `sweep_summary` SP="$1"; BY_A="$2"; BY_B="$3"; PY="$4"; OUT="$5"; shift 5 # shellcheck source=scripts/native-sweeps/sweeplib.sh . "$(dirname "$0")/sweeplib.sh" LIB=$(sweep_lib "$PY") # unknown until the first module compiles on both legs; then `same` or `differs`, and in -# the `differs` case `header_scope` is an ERE alternation of the helpers that moved +# the `differs` case `header_scope` is an ERE alternation of the names that moved, or empty +# where the change could not be pinned to names header_state=unknown header_scope="" +header_line="header: never compared" sweep_begin cdiff || exit 1 for b in $(sweep_modules "$LIB" "$@"); do f="$LIB/$b" @@ -52,19 +57,20 @@ for b in $(sweep_modules "$LIB" "$@"); do # module compiles first and not looked at again if [ -f "$out/by.h" ]; then cp "$out/by.h" "$SWEEP_ROOT/$leg.h"; fi done - # which helpers changed between the two runtimes, worked out once and then reused + # which names changed between the two runtimes, worked out once and then reused # - # the answer is `same` (nothing a module calls moved), `unscopable` (it moved in a way - # no line-level reading can attribute, so no module may be scoped out), or an ERE - # alternation of the helpers that changed + # the answer is `same` (nothing a module reaches moved), `unscopable` (it moved in a way + # that cannot be pinned to names, so no module may be scoped out), or the names that + # moved, joined by `|` if [ "$header_state" = unknown ] && [ -f "$SWEEP_ROOT/a.h" ] && [ -f "$SWEEP_ROOT/b.h" ]; then - header_scope=$(sweep_header_scope "$SWEEP_ROOT/a.h" "$SWEEP_ROOT/b.h" "$SWEEP_ROOT") + header_scope=$(sweep_header_scope "$SWEEP_ROOT/a.h" "$SWEEP_ROOT/b.h") case "$header_scope" in - same) header_state=same; header_scope="" ;; - unscopable) header_state=differs; header_scope="" ;; - *) header_state=differs ;; + same) header_state=same; header_scope=""; header_line="header: same" ;; + unscopable) header_state=differs; header_scope=""; header_line="header: unscopable" ;; + *) header_state=differs + header_line="header: scoped to $(printf '%s' "$header_scope" | tr '|' '\n' | grep -c .) names: $header_scope" ;; esac - printf 'header: %s\n' "$header_state${header_scope:+ (scoped)}" >&2 + printf '%s\n' "$header_line" >&2 fi if [ -f "$SWEEP_ROOT/a.c" ] && [ -f "$SWEEP_ROOT/b.c" ]; then if cmp -s "$SWEEP_ROOT/a.c" "$SWEEP_ROOT/b.c"; then @@ -72,7 +78,7 @@ for b in $(sweep_modules "$LIB" "$@"); do calls="" if [ "$header_state" = differs ]; then if [ -n "$header_scope" ]; then - calls=$(grep -oE "$header_scope" "$SWEEP_ROOT/a.c" | sort -u | paste -sd, -) + calls=$(grep -owE "$header_scope" "$SWEEP_ROOT/a.c" | sort -u | paste -sd, -) else calls="unscopable" fi @@ -95,13 +101,11 @@ for b in $(sweep_modules "$LIB" "$@"); do fi done { - printf 'walked: %s\n' "$(cat "$OUT.walked" 2>/dev/null || echo '?')" + printf '%s\n' "$header_line" for kind in same differs header a-only b-only neither; do printf '%s: %s\n' "$kind" "$(grep -c " $kind" "$OUT")" done -} -# the summary goes to stdout, not into `$OUT`: `sweep_end` counts distinct first columns -# there against `$OUT.walked`, so six summary lines read as six extra modules and a -# 550-module walk reported 556 — no `.complete` marker was ever written and the rung -# always exited 1. every other rung echoes its summary for the same reason + # last, so that a waiter seeing it has the whole summary + printf 'walked: %s\n' "$(cat "$OUT.walked" 2>/dev/null || echo '?')" +} | sweep_summary sweep_end || exit 1 diff --git a/scripts/native-sweeps/compare.sh b/scripts/native-sweeps/compare.sh index 0206d4db57..f3a2b5803a 100755 --- a/scripts/native-sweeps/compare.sh +++ b/scripts/native-sweeps/compare.sh @@ -87,9 +87,10 @@ fi # # comparing the sorted rows themselves rather than a digest of each module's block: the # rows a rung writes for one module are already keyed by it in field one, so the lines -# `comm` reports as unique to either side name their own modules +# `comm` reports as unique to either side name their own modules. a `# ` line is a +# summary rather than a row — see `sweep_summary` — and is left out with the rest of it drifted=$(LC_ALL=C comm -3 \ - <(LC_ALL=C sort "$before") <(LC_ALL=C sort "$after") \ + <(grep -v '^#' "$before" | LC_ALL=C sort) <(grep -v '^#' "$after" | LC_ALL=C sort) \ | sed 's/^\t//' | cut -f1 | LC_ALL=C sort -u \ | LC_ALL=C comm -23 - <(printf '%s' "$moved" | cut -f1 | LC_ALL=C sort -u)) diff --git a/scripts/native-sweeps/headerscope.py b/scripts/native-sweeps/headerscope.py new file mode 100644 index 0000000000..cfc5785a09 --- /dev/null +++ b/scripts/native-sweeps/headerscope.py @@ -0,0 +1,676 @@ +# /// script +# requires-python = ">=3.11" +# /// +"""which names two builds of the runtime header disagree about, for `cdiff.sh` + + headerscope.py A_HEADER B_HEADER + +prints one line: `same` where nothing a module can reach differs, `unscopable` where a +difference cannot be pinned to names, and otherwise the names a module's C has to mention +to be affected, joined by `|` for `grep -owE`. why a difference is unscopable goes to +stderr + +the header is included rather than inlined, so a module whose emitted C is byte-identical +under both compilers still moves wherever it reaches a helper whose definition moved. this +answer decides which of those modules the behavioural rungs walk, and a name missing from +it hides a module from every one of them. so everything errs one way: a difference this +cannot attribute exactly is `unscopable`, and a name that might be reached is in the answer + +the header is split into its top-level definitions, each keyed by the name it defines, and +the two sides are compared definition by definition: + +- a function, a variable, a top-level macro invocation, or a macro or a type the other side + does not have at all names what it defines +- so does every definition that mentions a name already in the answer, until nothing new + is added: a module calling an unchanged helper that calls a changed one is affected + without spelling the changed one's name +- a macro that pastes tokens can produce a name nobody spells, so one that can paste a name + in the answer is in the answer itself, and everything that expands it with it +- a macro or a type that changed or went away, a directive that is not a conditional, an + `_Static_assert`, a definition that runs at load, and anything this cannot parse are + unscopable: a macro is expanded in places its name is the only trace of, and a type's + layout is read by everything that holds one, neither of which a name search can follow +- a definition moved under a new `#if` differs from its old self, since the condition is + part of what it is +""" + +from __future__ import annotations + +import re +import sys +from dataclasses import dataclass, field + +KEYWORDS = frozenset( + [ + "auto", + "break", + "case", + "char", + "const", + "continue", + "default", + "do", + "double", + "else", + "enum", + "extern", + "float", + "for", + "goto", + "if", + "inline", + "int", + "long", + "register", + "restrict", + "return", + "short", + "signed", + "sizeof", + "static", + "struct", + "switch", + "typedef", + "union", + "unsigned", + "void", + "volatile", + "while", + "_Bool", + "_Complex", + "_Imaginary", + "_Alignas", + "_Alignof", + "_Atomic", + "_Generic", + "_Noreturn", + "_Static_assert", + "_Thread_local", + ] +) +# spellings that take a parenthesised argument in front of a declaration without being +# what the declaration names +ATTRIBUTES = frozenset( + {"__attribute__", "__declspec", "_Alignas", "__typeof__", "typeof"} +) + +TOKEN = re.compile( + r""" + (?P[ \t\r\f\v]+) + | (?P\n) + | (?P/\*.*?\*/|//[^\n]*) + | (?P[LuU8]*"(?:\\.|[^"\\\n])*") + | (?P[LuU]?'(?:\\.|[^'\\\n])*') + | (?P\.?[0-9](?:[eEpP][+-]|[A-Za-z0-9_.])*) + | (?P[A-Za-z_][A-Za-z0-9_]*) + | (?P\#\#|\.\.\.|<<=|>>=|->|\+\+|--|<<|>>|<=|>=|==|!=|&&|\|\||[-+*/%&|^]=|[^\s]) + """, + re.VERBOSE | re.DOTALL, +) + +OPENING = frozenset("{([") +CLOSING = frozenset("})]") + + +class Unscopable(Exception): + """a difference, or a header, this cannot attribute to names""" + + +@dataclass +class Token: + kind: str + text: str + # whether whitespace stood before it, which is all that survives of the layout: `- -b` + # and `--b` are different programs, and so are `#define F (x)` and `#define F(x)` + spaced: bool + # whether it is the first token on its line, which is what makes a `#` a directive + first: bool + + +def tokenize(source: str) -> list[Token]: + # a backslash-newline is removed before anything else reads the text, as a compiler + # removes it, so a macro spanning several lines is one line here + source = source.replace("\\\n", "") + tokens: list[Token] = [] + spaced = True + first = True + at = 0 + while at < len(source): + match = TOKEN.match(source, at) + kind = None if match is None else match.lastgroup + if match is None or kind is None: + raise Unscopable(f"cannot read the header at offset {at}") + at = match.end() + if kind == "newline": + spaced = True + first = True + elif kind in ("space", "comment"): + spaced = True + first = first or "\n" in match.group() + else: + tokens.append(Token(kind, match.group(), spaced, first)) + spaced = False + first = False + return tokens + + +def spelled(tokens: list[Token]) -> str: + """the tokens as text, with one space wherever the source had any""" + out = [] + for index, token in enumerate(tokens): + if index and token.spaced: + out.append(" ") + out.append(token.text) + return "".join(out) + + +def identifiers(tokens: list[Token]) -> set[str]: + return {token.text for token in tokens if token.kind == "ident"} + + +def closes_at(tokens: list[Token], opening: int) -> int: + """the index of the bracket that closes the one at `opening`, or -1""" + depth = 0 + for index in range(opening, len(tokens)): + if tokens[index].text in OPENING: + depth += 1 + elif tokens[index].text in CLOSING: + depth -= 1 + if depth == 0: + return index + return -1 + + +def top_level(tokens: list[Token]) -> list[tuple[int, Token]]: + """the tokens outside every bracket, with their indices, the brackets included""" + out = [] + depth = 0 + for index, token in enumerate(tokens): + if token.text in CLOSING: + depth -= 1 + if depth == 0: + out.append((index, token)) + if token.text in OPENING: + depth += 1 + return out + + +def split_arguments(tokens: list[Token]) -> list[list[Token]]: + """`(a, f(b, c), d)` as its three arguments""" + arguments: list[list[Token]] = [[]] + depth = 0 + for token in tokens[1:-1]: + if token.text in OPENING: + depth += 1 + elif token.text in CLOSING: + depth -= 1 + if depth == 0 and token.text == ",": + arguments.append([]) + else: + arguments[-1].append(token) + return arguments + + +@dataclass +class Definition: + kind: str + # the key the two sides are compared under + key: str + # every name it lets a module reach + names: set[str] + # every identifier it spells + mentions: set[str] + # the conditional directives it stands inside, outermost first + conditions: tuple[str, ...] + text: str + tokens: list[Token] + + def content(self) -> tuple[tuple[str, ...], str]: + return (self.conditions, self.text) + + +@dataclass +class Macro: + """a function-like macro: its parameters and the tokens it expands to""" + + parameters: list[str] + body: list[Token] + + def pastes(self) -> list[re.Pattern[str]]: + """every name a `##` in the body can make, as a pattern over the finished name""" + patterns = [] + at = 0 + while at < len(self.body): + if at + 1 < len(self.body) and self.body[at + 1].text == "##": + pieces = [self.body[at]] + while at + 2 < len(self.body) and self.body[at + 1].text == "##": + pieces.append(self.body[at + 2]) + at += 2 + patterns.append( + re.compile( + "".join( + r"\w*" + if piece.text in self.parameters + else re.escape(piece.text) + for piece in pieces + ) + ) + ) + at += 1 + return patterns + + def declares(self, arguments: list[list[Token]]) -> set[str]: + """the names an invocation at the top level of the header declares + + a declaration generator declares whatever stands before each parameter list at the + top level of its body: a parameter's argument, a name pasted out of one, or a name + written into the body itself + """ + if len(arguments) != len(self.parameters): + raise Unscopable("a macro is invoked with a different number of arguments") + given = dict(zip(self.parameters, arguments, strict=True)) + + def single(name: str) -> str: + argument = given[name] + if len(argument) != 1 or argument[0].kind != "ident": + raise Unscopable( + f"`{name}` is pasted from an argument that is not one name" + ) + return argument[0].text + + names: set[str] = set() + for index, token in top_level(self.body): + if token.text in ("=", ";"): + raise Unscopable( + "a top-level macro declares something that is not a function" + ) + if token.text != "(" or index == 0: + continue + before = self.body[index - 1] + if before.text in ATTRIBUTES or before.kind != "ident": + continue + # the whole run `a ## b ## c` that ends at the parameter list + run = [before] + back = index - 1 + while back >= 2 and self.body[back - 1].text == "##": + run.insert(0, self.body[back - 2]) + back -= 2 + names.add( + "".join( + single(piece.text) if piece.text in self.parameters else piece.text + for piece in run + ) + ) + if not names: + raise Unscopable( + "a top-level macro invocation declares no name this can see" + ) + return names + + +@dataclass +class Header: + definitions: dict[str, list[Definition]] = field(default_factory=dict) + macros: dict[str, list[Macro]] = field(default_factory=dict) + # every definition in the order the header writes them + order: list[Definition] = field(default_factory=list) + + def add(self, definition: Definition): + self.definitions.setdefault(definition.key, []).append(definition) + self.order.append(definition) + + def expansions(self) -> set[tuple[str, str, str]]: + """which `#define` or `#undef` of each macro every definition spelling it sees + + a macro is expanded only below where it is defined, so a `#define` that moves + changes what the definitions between its two places mean while comparing equal to + itself. `(user, macro, directive)` for each, with the directive empty for a user + that stands above every one of them + """ + macros = { + definition.key for definition in self.order if definition.kind == "macro" + } + seen: dict[str, str] = {} + out = set() + for definition in self.order: + out.update( + (definition.key, name, seen.get(name, "")) + for name in definition.mentions & macros + ) + if definition.kind == "macro": + seen[definition.key] = definition.text + elif ( + definition.kind == "directive" + and len(definition.tokens) > 2 + and definition.tokens[1].text == "undef" + ): + seen[definition.tokens[2].text] = definition.text + return out + + def every(self) -> list[Definition]: + return [each for group in self.definitions.values() for each in group] + + +def parse(source: str) -> Header: + tokens = tokenize(source) + header = Header() + conditions: list[str] = [] + chunk: list[Token] = [] + depth = 0 + at = 0 + while at < len(tokens): + token = tokens[at] + if token.text == "#" and token.first and depth == 0: + if chunk: + raise Unscopable(f"a directive stands inside `{spelled(chunk)[:60]}`") + end = at + 1 + while end < len(tokens) and not tokens[end].first: + end += 1 + directive(header, tokens[at:end], conditions) + at = end + continue + chunk.append(token) + at += 1 + if token.text in OPENING: + depth += 1 + elif token.text in CLOSING: + depth -= 1 + if depth < 0: + raise Unscopable(f"`{token.text}` closes nothing") + if depth != 0: + continue + following = tokens[at] if at < len(tokens) else None + on_its_own = following is None or following.first + if token.text == ";": + ends = True + elif token.text == "}": + # a function's body ends its definition, while a type's or an initialiser's + # goes on to a name or a `;` on the same line + ends = on_its_own + elif token.text == ")": + # `BY_DEFINE_INT_CMP(By_IntEq, ==, Py_EQ)` on a line of its own, which no `;` + # follows, rather than a signature whose body opens on the next line + ends = ( + on_its_own + and invoked(chunk) + and (following is None or following.text != "{") + ) + else: + ends = False + if ends: + header.add(classify(chunk, tuple(conditions))) + chunk = [] + if chunk or depth or conditions: + raise Unscopable("the header ends inside a definition or a conditional") + for definition in header.every(): + if definition.kind == "invocation": + macros = header.macros.get(definition.tokens[0].text) + if macros is None: + raise Unscopable( + f"`{definition.text[:60]}` invokes a macro this header lacks" + ) + arguments = split_arguments(definition.tokens[1:]) + for macro in macros: + definition.names |= macro.declares(arguments) + return header + + +def invoked(chunk: list[Token]) -> bool: + """whether a chunk is `NAME(...)` and nothing else""" + return ( + len(chunk) > 2 + and chunk[0].kind == "ident" + and chunk[1].text == "(" + and closes_at(chunk, 1) == len(chunk) - 1 + ) + + +def directive(header: Header, line: list[Token], conditions: list[str]): + word = line[1].text if len(line) > 1 else "" + if word in ("if", "ifdef", "ifndef"): + conditions.append(spelled(line)) + elif word in ("elif", "else"): + if not conditions: + raise Unscopable(f"`{spelled(line)}` closes nothing") + conditions[-1] = f"{conditions[-1]} {spelled(line)}" + elif word == "endif": + if not conditions: + raise Unscopable("an `#endif` closes nothing") + conditions.pop() + elif word == "define" and len(line) > 2 and line[2].kind == "ident": + name = line[2].text + body = line[3:] + # a parameter list is one only where nothing stands between it and the name + if body and body[0].text == "(" and not body[0].spaced: + close = closes_at(body, 0) + if close < 0: + raise Unscopable(f"`{name}` has a parameter list that never closes") + parameters = [ + spelled(argument) for argument in split_arguments(body[: close + 1]) + ] + header.macros.setdefault(name, []).append( + Macro(parameters, body[close + 1 :]) + ) + header.add( + Definition( + "macro", + name, + {name}, + identifiers(body), + tuple(conditions), + spelled(line), + line, + ) + ) + else: + text = spelled(line) + header.add( + Definition( + "directive", + f"directive {text}", + set(), + identifiers(line), + tuple(conditions), + text, + line, + ) + ) + + +def classify(chunk: list[Token], conditions: tuple[str, ...]) -> Definition: + text = spelled(chunk) + mentions = identifiers(chunk) + + def opaque(kind: str) -> Definition: + return Definition( + kind, f"{kind} {text}", set(), mentions, conditions, text, chunk + ) + + words = [token.text for token in chunk] + if words[0] == "_Static_assert": + return opaque("assert") + if invoked(chunk): + # what it declares is worked out once every macro in the header is known + return Definition("invocation", text, set(), mentions, conditions, text, chunk) + outside = top_level(chunk) + # a type is defined by a `typedef`, or by a tagged body with no declarator after it — + # `static struct PyModuleDef by_module = {...}` is a variable of a type + if words[0] == "typedef" or ( + words[0] in ("struct", "union", "enum") + and not any(token.text == "=" for _, token in outside) + and ([token.text for _, token in outside][-2:] == ["}", ";"] or len(words) == 3) + ): + names = type_names(chunk) + if not names: + return opaque("unknown") + return Definition("type", min(names), names, mentions, conditions, text, chunk) + stops = [ + index + for index, token in outside + if token.text in ("=", "(", "[", ";", "{", ",") + ] + if not stops: + return opaque("unknown") + stop = stops[0] + # an attribute's parenthesised argument in front of a function's name is not its + # parameter list + while chunk[stop].text == "(" and stop > 0 and chunk[stop - 1].text in ATTRIBUTES: + later = [index for index in stops if index > closes_at(chunk, stop)] + if not later: + return opaque("unknown") + stop = later[0] + name = chunk[stop - 1] if stop > 0 else None + if name is None or name.kind != "ident" or name.text in KEYWORDS: + return opaque("unknown") + kind = "function" if chunk[stop].text == "(" else "variable" + # several variables in one declaration is one more shape than this reads + if kind == "variable" and any(token.text == "," for _, token in outside): + return opaque("unknown") + return Definition(kind, name.text, {name.text}, mentions, conditions, text, chunk) + + +def type_names(chunk: list[Token]) -> set[str]: + """the tag, the typedef names and the enumerators a type definition introduces""" + names: set[str] = set() + for index, token in enumerate(chunk[:-1]): + if ( + token.text in ("struct", "union", "enum") + and chunk[index + 1].kind == "ident" + ): + names.add(chunk[index + 1].text) + if token.text == "enum": + body = next( + (at for at in range(index, len(chunk)) if chunk[at].text == "{"), None + ) + if body is not None: + for item in split_arguments(chunk[body : closes_at(chunk, body) + 1]): + if item and item[0].kind == "ident": + names.add(item[0].text) + if chunk[0].text == "typedef": + # the names a typedef introduces stand at its top level before each `,` and the + # closing `;`, or inside `(*name)` for a pointer to a function + outside = [token for _, token in top_level(chunk)] + for index, token in enumerate(outside): + if ( + token.text in (",", ";") + and index > 0 + and outside[index - 1].kind == "ident" + ): + names.add(outside[index - 1].text) + for index in range(len(chunk) - 2): + if ( + chunk[index].text == "(" + and chunk[index + 1].text == "*" + and chunk[index + 2].kind == "ident" + ): + names.add(chunk[index + 2].text) + return names - KEYWORDS + + +# the kinds whose difference is attributed to the names they define +NAMED = frozenset({"function", "variable", "invocation"}) +# the kinds that are attributed only where one side does not have them at all +NAMED_WHERE_NEW = frozenset({"macro", "type"}) + + +def scope(old: str, new: str) -> str: + if old == new: + return "same" + try: + return "|".join(sorted(affected(parse(old), parse(new)))) or "same" + except Unscopable as reason: + # the reason goes where a person reading the rung's log sees it, and the answer + # stays one word for the script reading it + print(f"headerscope: unscopable, {reason}", file=sys.stderr) + return "unscopable" + + +def affected(before: Header, after: Header) -> set[str]: + names: set[str] = set() + unchanged: set[str] = set() + for key in before.definitions.keys() | after.definitions.keys(): + was = before.definitions.get(key, []) + now = after.definitions.get(key, []) + if sorted(each.content() for each in was) == sorted( + each.content() for each in now + ): + unchanged.add(key) + continue + for definition in was + now: + if definition.kind in NAMED or ( + definition.kind in NAMED_WHERE_NEW and not was + ): + names |= definition.names + else: + raise Unscopable(f"a {definition.kind} differs: {definition.text[:80]}") + # a definition that sees a different `#define` of a macro it spells, because the + # `#define` moved, means something else without its own text changing + moved = { + (user, macro) + for user, macro, _ in before.expansions() ^ after.expansions() + if user in unchanged and macro in unchanged + } + if moved: + user, macro = min(moved) + raise Unscopable( + f"`{user}` sees a different definition of `{macro}` than it did" + ) + everything = before.every() + after.every() + pastes = [ + (name, pattern) + for header in (before, after) + for name, macros in header.macros.items() + for macro in macros + for pattern in macro.pastes() + ] + grew = bool(names) + while grew: + grew = False + # a macro that can paste a name in the answer can stand for it wherever it is + # expanded, spelled or not + for macro, pattern in pastes: + if macro not in names and any(pattern.fullmatch(name) for name in names): + names.add(macro) + grew = True + for definition in everything: + if not definition.mentions & names: + continue + if definition.kind not in NAMED | NAMED_WHERE_NEW: + raise Unscopable( + f"a {definition.kind} reaches a name that moved: {definition.text[:80]}" + ) + if definition.names <= names: + continue + names |= definition.names + grew = True + # a definition that runs at load runs in every module, whatever the module mentions + for definition in everything: + if definition.names & names and runs_at_load(definition.tokens): + raise Unscopable(f"`{min(definition.names)}` runs when the module loads") + return names + + +def runs_at_load(tokens: list[Token]) -> bool: + """whether an attribute marks a definition a constructor or a destructor""" + for index, token in enumerate(tokens[:-1]): + if token.text == "__attribute__" and tokens[index + 1].text == "(": + group = tokens[index + 1 : closes_at(tokens, index + 1) + 1] + if identifiers(group) & {"constructor", "destructor"}: + return True + return False + + +def main() -> int: + if len(sys.argv) != 3: + print("usage: headerscope.py A_HEADER B_HEADER", file=sys.stderr) + return 2 + with ( + open(sys.argv[1], encoding="utf-8") as a, + open(sys.argv[2], encoding="utf-8") as b, + ): + print(scope(a.read(), b.read())) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/native-sweeps/sweeplib.sh b/scripts/native-sweeps/sweeplib.sh index a41d637379..d0cdf3935f 100755 --- a/scripts/native-sweeps/sweeplib.sh +++ b/scripts/native-sweeps/sweeplib.sh @@ -162,6 +162,22 @@ sweep_digest() { fi } +# a rung's summary, written into `$OUT` as well as to the terminal +# +# a waiter watching the output file for `walked:` has nothing else to watch: the rows say +# nothing about whether the walk has finished, and a summary that only reached stdout went +# into whichever log the rung was started under. each line lands in `$OUT` behind a `# `, +# which no module name starts with, so `sweep_end` counts it as no module and a reader +# picking rows by their tab-separated verdict never sees it. it has to be written before +# `sweep_end`, whose completion marker vouches for every line the file holds +sweep_summary() { + local line + while IFS= read -r line; do + printf '%s\n' "$line" + printf '# %s\n' "$line" >> "$OUT" + done +} + # refuse to mark a run complete unless it is one, then mark it by content # # these are the same questions of every rung, so they are asked once here and a rung @@ -180,7 +196,8 @@ sweep_digest() { sweep_end() { local walked=0 seen=0 hollow="" [ -f "$OUT.walked" ] && walked=$(cat "$OUT.walked") - seen=$(cut -f1 "$OUT" | LC_ALL=C sort -u | grep -c .) + # a summary line is not a module — see `sweep_summary` + seen=$(grep -v '^#' "$OUT" | cut -f1 | LC_ALL=C sort -u | grep -c .) if [ "$walked" -eq 0 ]; then printf 'sweep: %s walked no modules, so it measured nothing — %s is not marked complete\n' \ "${SWEEP_TAG:-?}" "$OUT" >&2 @@ -474,52 +491,29 @@ sweep_stage() { > "$dir/pyproject.toml" } +# which names two builds of `by.h` disagree about +# +# prints `same` when nothing a module could reach has moved, `unscopable` when the change +# cannot be attributed to particular names, or those names joined by `|` for `grep -owE`. +# takes the two headers, and says on stderr why a change is unscopable +# +# a name is scoped *out* of a walk only when the evidence says so plainly, because getting +# this wrong in the narrowing direction hides a real divergence: a module reported `same` +# is one the behavioural rungs never run. `headerscope.py` compares the two headers +# definition by definition rather than line by line, which is what lets a purely additive +# change — a new helper, whose body is mostly lines naming nothing — scope to the names it +# adds, while a changed macro or a changed struct layout, which every definition expanding +# or holding one depends on, is still unscopable. its docstring has the rules, and +# `test_headerscope.py` a fixture for each +sweep_header_scope() { + "$PY" "$(dirname "${BASH_SOURCE[0]}")/headerscope.py" "$1" "$2" +} + # where the build wrote the staged module's artefacts # # `by compile` lays its output out as the *module* tree, so a package member's `m.c`, # `m.annotated` and `m*.so` sit at the member's own place under `o` rather than at the # top of it — which is where the staged source sits within the project -# which runtime helpers two builds of `by.h` disagree about -# -# prints `same` when nothing a module could call has moved, `unscopable` when the change -# cannot be attributed to particular helpers, or an ERE alternation of the ones that have. -# takes the two headers and a scratch directory -# -# a helper is scoped *out* of a walk only when the evidence says so plainly, because -# getting this wrong in the narrowing direction hides a real divergence: a module reported -# `same` is one the behavioural rungs never run. so comment and blank lines are dropped as -# carrying no behaviour, and of what is left **every** line must name a `By_` symbol of its -# own for the change to be scopable at all. a changed line naming none is something like a -# top-level `#define BY_THRESHOLD 8`, which no line-level reading can attribute and which -# every helper reading it depends on. -# -# `diff -p` is deliberately not trusted to attribute those. it names the function a hunk -# falls in, and for a change *after* the last function in the file it names that function -# regardless — which is how a top-level define came to be blamed on -# `By_StrItemCompareCharI64`. where the change is scopable its names are added to the -# symbols read off the lines, because a change inside a body often does not repeat the -# name of the body it is in. that union over-approximates, which is the safe direction: a -# helper named that did not change costs one module a walk it did not need -sweep_header_scope() { - local a="$1" b="$2" tmp="$3" - cmp -s "$a" "$b" && { printf 'same\n'; return 0; } - diff "$a" "$b" 2>/dev/null | grep '^[<>]' | sed 's/^[<>][[:space:]]*//' > "$tmp/changed" - grep -v '^[[:space:]]*$' "$tmp/changed" | - grep -v '^[[:space:]]*\(\*\|//\|/\*\)' > "$tmp/meaningful" - if [ ! -s "$tmp/meaningful" ]; then - printf 'same\n' - return 0 - fi - if [ "$(grep -c . "$tmp/meaningful")" -ne "$(grep -c 'By_[A-Za-z0-9_]' "$tmp/meaningful")" ]; then - printf 'unscopable\n' - return 0 - fi - { - grep -o 'By_[A-Za-z0-9_]*' "$tmp/meaningful" - diff -p "$a" "$b" 2>/dev/null | grep '^\*\{15\}' | grep -o 'By_[A-Za-z0-9_]*(' | tr -d '(' - } | sort -u | paste -sd'|' - -} - sweep_out_dir() { local rel="${SWEEP_SRC%/*}" if [ "$rel" = "$SWEEP_SRC" ]; then printf '%s/o' "$1"; else printf '%s/o/%s' "$1" "$rel"; fi @@ -897,6 +891,12 @@ _CALLABLE = ( ) +# a compiled class publishes its methods as a `by.method_descriptor`, which binds as a +# function does but is no type python has a name for, so it is known by its own +def _compiled_method(kind): + return getattr(kind, '__module__', None) == 'by' and kind.__qualname__ == 'method_descriptor' + + class Canon: def __init__(self, aliases=()): # longest first: `pkg.m.Outer.Inner` must not be half-replaced by `pkg.m.Outer` @@ -949,7 +949,7 @@ class Canon: getattr(value, '__qualname__', value.__name__))) if isinstance(value, types.ModuleType): return '' % self.scrub(getattr(value, '__name__', '?')) - if isinstance(value, _CALLABLE): + if isinstance(value, _CALLABLE) or _compiled_method(kind): return '' % self.scrub(str(getattr(value, '__name__', '?'))) return self.scrub(repr(value)) diff --git a/scripts/native-sweeps/test_headerscope.py b/scripts/native-sweeps/test_headerscope.py new file mode 100644 index 0000000000..a92b37127e --- /dev/null +++ b/scripts/native-sweeps/test_headerscope.py @@ -0,0 +1,241 @@ +"""tests for the runtime-header scoper `cdiff.sh` narrows its population with + +run with:: + + uv run --no-project --with pytest pytest scripts/native-sweeps/test_headerscope.py + +each fixture is a pair of headers and the answer the scoper has to give. the ones that +matter most are the unscopable ones: a narrowing that is wrong hides a module from every +behavioural rung, so a change the scoper cannot attribute exactly has to say so +""" + +# /// script +# requires-python = ">=3.11" +# dependencies = ["pytest"] +# +# # the module under test is the sibling script, imported the way pytest imports it +# [tool.ty.environment] +# extra-paths = ["."] +# /// + +from __future__ import annotations + +import subprocess +import sys +import tempfile +from pathlib import Path + +from headerscope import parse, scope + +BASE = """\ +#include + +#define BY_LIKELY(x) __builtin_expect(!!(x), 1) +#define BY_TAG 1 + +typedef struct { + PyObject *value; + int flags; +} By_Box; + +/* a helper nothing else calls */ +static inline int By_Leaf(int x) { + return x + BY_TAG; +} + +static inline int By_Caller(int x) { + if (BY_LIKELY(x > 0)) return By_Leaf(x); + return 0; +} + +static inline int By_Unrelated(int x) { return x * 2; } + +static PyObject *By_Cache = NULL; + +#define BY_DEFINE_TWICE(name, op) \\ + static inline int name(int a) { return a op 2; } \\ + static inline int name##Rev(int a) { return 2 op a; } + +BY_DEFINE_TWICE(By_Mul, *) +""" + + +def answer(old: str, new: str) -> str | set[str]: + """the scoper's answer, a set of names where it scoped the change""" + got = scope(old, new) + return got if got in ("same", "unscopable") else set(got.split("|")) + + +def test_the_fixture_parses(): + # every other test leans on this: a header this cannot read is unscopable whatever + # changed in it + assert parse(BASE).every() + + +def test_identical_headers_are_the_same(): + assert answer(BASE, BASE) == "same" + + +def test_a_comment_or_a_blank_line_is_the_same(): + new = BASE.replace("/* a helper nothing else calls */", "/* reworded */\n\n") + assert answer(BASE, new) == "same" + + +def test_an_added_function_is_scoped_to_its_own_name(): + new = BASE + "\nstatic inline int By_New(int x) {\n return By_Leaf(x) - 1;\n}\n" + assert answer(BASE, new) == {"By_New"} + + +def test_an_added_function_under_a_new_conditional_is_scoped_to_its_own_name(): + new = BASE + ( + "\n#if PY_VERSION_HEX >= 0x030E0000\n" + "static inline int By_New(int x) { return x; }\n" + "#else\n" + "static inline int By_New(int x) { return -x; }\n" + "#endif\n" + ) + assert answer(BASE, new) == {"By_New"} + + +def test_an_added_macro_and_type_are_scoped_to_their_names(): + new = BASE + "\n#define BY_NEW 3\ntypedef struct { int a; } By_NewBox;\n" + assert answer(BASE, new) == {"BY_NEW", "By_NewBox"} + + +def test_a_changed_body_reaches_every_caller(): + # a module calling only `By_Caller` spells nothing that changed, and is affected + new = BASE.replace("return x + BY_TAG;", "return x + BY_TAG + 1;") + assert answer(BASE, new) == {"By_Leaf", "By_Caller"} + + +def test_a_removed_function_is_scoped_to_its_name(): + new = BASE.replace("static inline int By_Unrelated(int x) { return x * 2; }\n", "") + assert answer(BASE, new) == {"By_Unrelated"} + + +def test_a_changed_variable_reaches_its_readers(): + new = BASE.replace("By_Cache = NULL;", "By_Cache = Py_None;") + ( + "static PyObject *By_Read(void) { return By_Cache; }\n" + ) + old = BASE + "static PyObject *By_Read(void) { return By_Cache; }\n" + assert answer(old, new) == {"By_Cache", "By_Read"} + + +def test_a_definition_moved_under_a_new_conditional_is_scoped_to_its_name(): + new = BASE.replace( + "static inline int By_Unrelated(int x) { return x * 2; }\n", + "#ifdef BY_EXTRA\nstatic inline int By_Unrelated(int x) { return x * 2; }\n#endif\n", + ) + assert answer(BASE, new) == {"By_Unrelated"} + + +def test_a_changed_macro_is_unscopable(): + # every expansion site is affected and the name is the only trace an expansion leaves + new = BASE.replace("#define BY_TAG 1", "#define BY_TAG 2") + assert answer(BASE, new) == "unscopable" + + +def test_a_changed_function_like_macro_is_unscopable(): + new = BASE.replace("__builtin_expect(!!(x), 1)", "(x)") + assert answer(BASE, new) == "unscopable" + + +def test_a_macro_made_function_like_by_a_removed_space_is_unscopable(): + new = BASE.replace("#define BY_TAG 1", "#define BY_TAG (1)") + assert answer(BASE, new) == "unscopable" + + +def test_a_removed_macro_is_unscopable(): + new = BASE.replace("#define BY_TAG 1\n", "") + assert answer(BASE, new) == "unscopable" + + +def test_a_changed_struct_layout_is_unscopable(): + new = BASE.replace(" int flags;\n", " int flags;\n int more;\n") + assert answer(BASE, new) == "unscopable" + + +def test_an_added_include_is_unscopable(): + new = BASE.replace( + "#include \n", "#include \n#include \n" + ) + assert answer(BASE, new) == "unscopable" + + +def test_an_added_static_assert_is_unscopable(): + new = BASE + '_Static_assert(sizeof(int) == 4, "int");\n' + assert answer(BASE, new) == "unscopable" + + +def test_a_moved_define_is_unscopable(): + # `By_Leaf` reads `BY_TAG` and its text is unchanged, but the `#define` now stands + # below it, so it no longer expands there + moved = BASE.replace("#define BY_TAG 1\n", "") + "#define BY_TAG 1\n" + assert answer(BASE, moved) == "unscopable" + + +def test_an_added_constructor_is_unscopable(): + new = ( + BASE + + "__attribute__((constructor)) static void By_AtLoad(void) { By_Leaf(1); }\n" + ) + assert answer(BASE, new) == "unscopable" + + +def test_a_header_that_does_not_parse_is_unscopable(): + new = BASE + "static inline int By_Broken(int x) {\n" + assert answer(BASE, new) == "unscopable" + + +def test_an_added_invocation_is_scoped_to_the_names_it_declares(): + # the pasted `By_AddRev` is spelled nowhere, and a module can still call it. the + # macro could paste it anywhere it is expanded, so its every expansion comes along + new = BASE + "BY_DEFINE_TWICE(By_Add, +)\n" + assert answer(BASE, new) == { + "By_Add", + "By_AddRev", + "BY_DEFINE_TWICE", + "By_Mul", + "By_MulRev", + } + + +def test_a_pasting_macro_that_can_make_a_changed_name_is_in_the_answer(): + # `By_MulRev` changed, so `BY_DEFINE_TWICE` can stand for it wherever it expands + old = BASE + "static inline int By_Twice(int a) { return By_MulRev(a); }\n" + new = old.replace("BY_DEFINE_TWICE(By_Mul, *)", "BY_DEFINE_TWICE(By_Mul, +)") + assert answer(old, new) == {"By_Mul", "By_MulRev", "BY_DEFINE_TWICE", "By_Twice"} + + +def test_an_unchanged_invocation_of_a_changed_callee_is_scoped(): + old = BASE + ( + "#define BY_CALLS(name, callee) static inline int name(int a) { return callee(a); }\n" + "BY_CALLS(By_Through, By_Unrelated)\n" + ) + new = old.replace("return x * 2;", "return x * 3;") + assert answer(old, new) == {"By_Unrelated", "By_Through"} + + +def test_the_command_line_prints_one_line(): + with tempfile.TemporaryDirectory() as scratch: + a = Path(scratch) / "a.h" + b = Path(scratch) / "b.h" + _ = a.write_text(BASE) + _ = b.write_text(BASE.replace("#define BY_TAG 1", "#define BY_TAG 2")) + script = Path(__file__).with_name("headerscope.py") + result = subprocess.run( + [sys.executable, str(script), str(a), str(b)], + capture_output=True, + text=True, + check=True, + ) + assert result.stdout == "unscopable\n" + assert "BY_TAG" in result.stderr + + +def test_the_real_header_parses_into_definitions_it_can_name(): + # an `unknown` definition is unscopable the moment it differs, so each one the real + # header holds is a change this cannot narrow + header = Path(__file__).resolve().parents[2] / "crates/by_rt/include/by.h" + parsed = parse(header.read_text(encoding="utf-8")) + assert "unknown" not in {definition.kind for definition in parsed.every()} From 70387368b70e1eb69f084d8a8a8a3fa7d62e92d9 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 15 Sep 2026 08:51:14 +1000 Subject: [PATCH 10/16] ty: fix statement expression regions, overload checks and gradual bases an expression region carries the declarations a statement expression makes instead of asserting it has none, an overload set and a valueless Final are checked in a scope whose end is unreachable, and a class with a gradual base answers as having a gradual member --- crates/by_build/tests/differential.rs | 30 +++ crates/ty_python_core/src/builder.rs | 19 +- .../basedpython_statement_expressions.md | 91 +++++++++ .../mdtest/basedpython_trailing_lambda.md | 18 ++ .../resources/mdtest/overloads.md | 93 +++++++++ .../resources/mdtest/type_qualifiers/final.md | 25 +++ crates/ty_python_semantic/src/types.rs | 36 ++++ crates/ty_python_semantic/src/types/infer.rs | 33 ++++ .../src/types/infer/builder.rs | 42 +++-- .../builder/post_inference/final_variable.rs | 29 ++- .../post_inference/overloaded_function.rs | 176 +++++++++++++++--- 11 files changed, 528 insertions(+), 64 deletions(-) diff --git a/crates/by_build/tests/differential.rs b/crates/by_build/tests/differential.rs index b0c4399ed8..1bde73d62d 100644 --- a/crates/by_build/tests/differential.rs +++ b/crates/by_build/tests/differential.rs @@ -14809,6 +14809,36 @@ def raised_by_the_step(log: list[str]) -> object: ); } +#[test] +fn a_class_with_a_gradual_base_proves_no_representation() { + // a class the checker sees a gradual base in is assignable to every class that is not + // final, and its class object to anything its unknown metaclass could be. so neither + // proves a representation: `type(name, bases, namespace)` over bases the checker cannot + // see was taken for `None`, and the function refused its own answer with `expected + // None, got type` — and an instance of that class was taken for an `int` + agree_python( + "gradualbase", + "\ +def made(name: str, bases: tuple[type, ...], namespace: dict[str, object]) -> type: + return type(name, bases, namespace) + + +def made_instance(name: str, bases: tuple[type, ...], namespace: dict[str, object]) -> object: + return type(name, bases, namespace)() + + +def either(name: str, bases: tuple[type, ...], flag: bool) -> object: + return type(name, bases, {})() if flag else None +", + &[ + "m.made('A', (object,), {}).__name__", + "type(m.made_instance('B', (object,), {})).__name__", + "type(m.either('C', (object,), True)).__name__", + "m.either('C', (object,), False)", + ], + ); +} + #[test] fn a_drained_generator_expression_raises_its_first_iterable_where_it_is_written() { // python evaluates a generator expression's first iterable, and takes its iterator, diff --git a/crates/ty_python_core/src/builder.rs b/crates/ty_python_core/src/builder.rs index b37c6768c2..a7bdec45a0 100644 --- a/crates/ty_python_core/src/builder.rs +++ b/crates/ty_python_core/src/builder.rs @@ -5774,11 +5774,7 @@ impl<'db, 'ast> SemanticIndexBuilder<'db, 'ast> { self.visit_expr(&node.annotation); if let Some(value) = &node.value { self.visit_expr(value); - // basedpython: a trailing lambda block defines a function, - // which a standalone expression cannot own - if self.is_method_or_eagerly_executed_in_method().is_some() - && !is_trailing_lambda_value(value) - { + if self.is_method_or_eagerly_executed_in_method().is_some() { // Record the right-hand side of the assignment as a standalone expression // if we're inside a method. This allows type inference to infer the type // of the value for annotated assignments like `self.CONSTANT: Final = 1`, @@ -7616,19 +7612,6 @@ struct CurrentStatement<'ast, 'db> { )>, } -/// basedpython: whether `value` is a [trailing lambda block] standing as a -/// statement's value. Such a value may not be made a standalone expression: the -/// block's suite defines a function, and an expression region cannot own a -/// definition. The parser restricts the block to the assignment shapes whose -/// value is inferred with the target's own definition instead. -/// -/// [trailing lambda block]: ast::ExprStatement::trailing_lambda -fn is_trailing_lambda_value(value: &ast::Expr) -> bool { - value - .as_statement_expr() - .is_some_and(ast::ExprStatement::is_trailing_lambda) -} - /// basedpython-ui: whether `func` is spelled like the callee of a call whose /// argument's scope must learn whom it was handed to from a standalone /// inference of the callee — `.set_root()`, `derived()`, diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_statement_expressions.md b/crates/ty_python_semantic/resources/mdtest/basedpython_statement_expressions.md index 38b65dc90e..d0c98b8915 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_statement_expressions.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_statement_expressions.md @@ -346,6 +346,97 @@ def f(c: bool): reveal_type(b) # revealed: 1 | 2 ``` +## declarations made inside a statement expression are visible after it + +A name a suite declares is declared in the scope the statement expression is written in, the same +way a name it binds is bound there. + +```by +def f(c: bool): + a = if c: + b: int = 1 + b + else: + b: int = 2 + b + reveal_type(a) # revealed: 1 | 2 + reveal_type(b) # revealed: 1 | 2 +``` + +## the checks a scope runs over its declarations reach one made inside a statement expression + +A statement expression is read as an expression, but what its suite declares is declared in the +scope around it. So the checks that scope runs once its types are known — the ones that look at a +declaration rather than at an expression — see those declarations too, and not only the ones written +outside. + +```by +from typing import Final, overload + +def f(c: int): + a = match c: + case _: + total: Final[int] # error: [final-without-value] + @overload + # error: [invalid-overload] "Overloaded function `g` requires at least two overloads" + # error: [invalid-overload] "Overloads for function `g` must be followed by a non-`@overload`-decorated implementation function" + def g(x: int) -> int: ... + 1 + reveal_type(a) # revealed: 1 +``` + +## a returned statement expression may declare a name + +A `def` that writes no return type has one recovered from the expressions its body hands back, and +each of those is read on its own so that what it narrows can be read off it too. A statement +expression read that way is still a statement, so the names its suite declares are declared in the +function around it rather than lost inside the read. + +```by +def f(c: bool): + return if c: + a: int = 1 + a + else: + 0 + +reveal_type(f(True)) # revealed: 1 | 0 +``` + +A nested `def` declares its own name the same way. + +```by +def g(c: bool): + return if c: + def h() -> int: + return 1 + h() + else: + 0 + +reveal_type(g(True)) # revealed: int +``` + +## a statement expression assigned to an attribute may declare a name + +An assignment whose target is not a plain name reads its value on its own as well, so the same holds +there. + +```by +class C: + def m(self, c: bool) -> None: + self.x = match c: + case True: + y: int = 1 + y + case _: + 0 + +c = C() +c.m(True) +reveal_type(c.x) # revealed: int +``` + ## a nested compound statement in tail position supplies the branch's value ```by diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_trailing_lambda.md b/crates/ty_python_semantic/resources/mdtest/basedpython_trailing_lambda.md index a0519bd07f..6b8eff43c3 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_trailing_lambda.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_trailing_lambda.md @@ -834,6 +834,24 @@ def build() -> int: print(it) ``` +## a `return` in a function that writes no return type takes one too + +A `def` that leaves its return type out has one recovered from the expressions its body hands back, +and each of those is read on its own. A block is a `def` written where a value goes, so the +expression read there defines a function, which belongs to the scope the `return` is written in. + +```by +def f(a: (int) -> None) -> str: + a(1) + return "done" + +def build(): + return f: + print(it) + +reveal_type(build()) # revealed: str +``` + ## the call is still checked as a call ```by diff --git a/crates/ty_python_semantic/resources/mdtest/overloads.md b/crates/ty_python_semantic/resources/mdtest/overloads.md index 64a7fbec41..5778b24412 100644 --- a/crates/ty_python_semantic/resources/mdtest/overloads.md +++ b/crates/ty_python_semantic/resources/mdtest/overloads.md @@ -1960,6 +1960,99 @@ def baz(x, y, z=None) -> bytes | list[str]: reveal_type(baz) ``` +### An overload set in a scope that never reaches its end + +An overload set is normally found by reading what its name holds once the scope has finished. A +scope whose every path returns never gets there, so nothing is bound at that point and a set written +inside it would go unchecked — and a function body ending in a `return` is the ordinary shape of +that, not an exotic one. + +A call made inside the scope reaches the `def`s written above it whichever way the scope ends, so +the set the call resolves to is checked on its own. + +```py +from typing import overload + +def outer() -> int: + @overload + # error: [invalid-overload] "Overloaded function `g` requires at least two overloads" + # error: [invalid-overload] "Overloads for function `g` must be followed by a non-`@overload`-decorated implementation function" + def g(x: int) -> int: ... + return g(1) +``` + +A call written part-way down a set resolves to only the `def`s above it, which is a prefix of the +set and not a set of its own. Here `g(1)` sees a lone overload, but the whole set is well formed, so +nothing is reported. + +```py +from typing import overload + +def outer() -> int: + @overload + def g(x: int) -> int: ... + first = g(1) + @overload + def g(x: str) -> str: ... + def g(x: int | str) -> int | str: + return x + return first +``` + +A set the scope never calls is checked too. Whether a `def` is malformed has nothing to do with +whether the function around it happens to end in a `return`, and "attempting to call `g` will raise +`TypeError`" is as true here as anywhere else. This is where we part from upstream, which checks +nothing at all in a scope with no end to read. + +```py +from typing import overload + +def outer() -> int: + @overload + # error: [invalid-overload] "Overloads for function `g` must be followed by a non-`@overload`-decorated implementation function" + def g(x: int) -> int: ... + @overload + def g(x: str) -> str: ... + return 1 +``` + +The question the end-of-scope read was answering — which set the name holds, when the scope writes +the name more than once — is still asked, and answered by the set itself: it is the one the place +holds only if it accounts for every declaration of the name the scope reaches. A second set written +under the same name leaves neither able to say that, so neither is reported against: which of them +an author meant to keep is exactly what the end of the scope would have said. + +```py +from typing import overload + +def outer() -> int: + @overload + def g(x: int) -> int: ... + def g(x: int) -> int: + return x + + @overload + def g(x: str) -> str: ... + return 1 +``` + +Redefining the name in a branch is the same thing: neither set accounts for the other's declaration, +so neither is reported against. + +```py +from typing import overload + +def outer(flag: bool) -> int: + if flag: + @overload + def g(x: int) -> int: ... + + else: + @overload + def g(x: str) -> str: ... + return 1 +``` + ## Generic overloaded protocol members preserve receiver relationships An overloaded method used to satisfy a protocol receiver can relate a method-scoped type variable to diff --git a/crates/ty_python_semantic/resources/mdtest/type_qualifiers/final.md b/crates/ty_python_semantic/resources/mdtest/type_qualifiers/final.md index 601872e764..0dd9d484ad 100644 --- a/crates/ty_python_semantic/resources/mdtest/type_qualifiers/final.md +++ b/crates/ty_python_semantic/resources/mdtest/type_qualifiers/final.md @@ -997,6 +997,31 @@ def f(): x: Final[int] # error: [final-without-value] "read-only symbol `x` is not assigned a value" ``` +### `Final` without value in a scope that never reaches its end + +Which declarations a symbol has is normally read off the end of the scope. A scope whose every path +returns never gets there, so upstream checks nothing at all in one — and a function body ending in a +`return` is the ordinary shape of that, not an exotic one. Whether a `Final` was ever given a value +has nothing to do with whether the function around it happens to end in a `return`, so a scope with +no end to read is asked for every declaration it reaches instead. + +```py +from typing import Final + +def f() -> int: + x: Final[int] # error: [final-without-value] "read-only symbol `x` is not assigned a value" + return 1 + +def g() -> int: + y: Final[int] = 1 + return y + +def loops_forever() -> None: + z: Final[int] # error: [final-without-value] "read-only symbol `z` is not assigned a value" + while True: + pass +``` + ### `typing_extensions.Final` without value ```py diff --git a/crates/ty_python_semantic/src/types.rs b/crates/ty_python_semantic/src/types.rs index c9fa0a1949..976729315d 100644 --- a/crates/ty_python_semantic/src/types.rs +++ b/crates/ty_python_semantic/src/types.rs @@ -2845,6 +2845,10 @@ impl<'db> Type<'db> { /// `socket.getsockname()` is a gradual type wearing a name — and a consumer that /// took the name for a proof gave `self._address = sock.getsockname()` the `None` /// representation, `None` being the first thing a gradual type is assignable to. + /// + /// basedpython: a class with a gradual base in its MRO is answered as gradual too, and so + /// is the class object of one. python's `class C(Any)` is assignable to every class that + /// is not final, so `C()` proves no more about its representation than `Unknown` does. pub fn has_gradual_member(self, db: &'db dyn Db, env: &ProgramEnvironment<'db>) -> bool { self.has_gradual_member_impl(db, env, true) } @@ -2872,6 +2876,28 @@ impl<'db> Type<'db> { bound.is_dynamic() || bound.has_gradual_member_impl(db, env, false) }), Type::TypeAlias(alias) => gradual(alias.value_type(db)), + // basedpython: an instance of a class with a gradual base is assignable to every + // class that is not final, and the class object to whatever its unknown + // metaclass could be. `type(name, bases, namespace)` over bases the checker + // cannot see is the common way to meet one + Type::NominalInstance(instance) => { + inherits_gradually(instance.class(db, env).iter_mro(db)) + } + Type::ClassLiteral(literal) => { + literal.metaclass(db).is_dynamic() || inherits_gradually(literal.iter_mro(db)) + } + Type::GenericAlias(alias) => { + let class = ClassType::Generic(alias); + class.class_literal(db).metaclass(db).is_dynamic() + || inherits_gradually(class.iter_mro(db)) + } + Type::SubclassOf(subclass_of) => match subclass_of.subclass_of() { + SubclassOfInner::Class(class) => { + class.class_literal(db).metaclass(db).is_dynamic() + || inherits_gradually(class.iter_mro(db)) + } + _ => false, + }, _ => false, } } @@ -13398,3 +13424,13 @@ static_assertions::assert_eq_size!(Type, [u8; 16]); #[cfg(not(debug_assertions))] #[cfg(target_pointer_width = "64")] static_assertions::assert_eq_size!(literal::LiteralValueType, [u8; 12]); + +/// whether an MRO passes through a gradual base — see [`Type::has_gradual_member`] +fn inherits_gradually<'db>(mut mro: impl Iterator>) -> bool { + mro.any(|base| { + matches!( + base, + ClassBase::Any | ClassBase::Dynamic(_) | ClassBase::Divergent(_) + ) + }) +} diff --git a/crates/ty_python_semantic/src/types/infer.rs b/crates/ty_python_semantic/src/types/infer.rs index e58145b061..0760fb8e0f 100644 --- a/crates/ty_python_semantic/src/types/infer.rs +++ b/crates/ty_python_semantic/src/types/infer.rs @@ -2056,6 +2056,21 @@ struct ExpressionInferenceExtra<'db> { /// Only very few expression regions have bindings (around 0.1%). bindings: Box<[(Definition<'db>, Type<'db>)]>, + /// The types and type qualifiers of every declaration in this expression region. + /// + /// basedpython: an expression region holds a definition only when it is a + /// [statement expression](ruff_python_ast::ExprStatement) — `return match x: …` + /// stands for a `match` whose branches may declare a name. The definition belongs to + /// the scope the statement is written in, so what the region inferred for it is + /// carried back out to whoever asked for the region, exactly as a binding is. + declarations: Box<[(Definition<'db>, TypeAndQualifiers<'db>)]>, + + /// The definitions in this expression region that have some deferred parts. + /// + /// Populated for the same reason as [`Self::declarations`], and handed back to the + /// scope region, which is where deferred parts are resolved. + deferred: Box<[Definition<'db>]>, + /// The diagnostics for this region. diagnostics: TypeCheckDiagnostics, @@ -2067,6 +2082,24 @@ struct ExpressionInferenceExtra<'db> { } impl<'db> ExpressionInference<'db> { + fn declarations( + &self, + ) -> impl ExactSizeIterator, TypeAndQualifiers<'db>)> { + self.extra + .as_deref() + .map_or(&[][..], |extra| &extra.declarations) + .iter() + .copied() + } + + fn deferred(&self) -> impl ExactSizeIterator> { + self.extra + .as_deref() + .map_or(&[][..], |extra| &extra.deferred) + .iter() + .copied() + } + fn cycle_initial(scope: ScopeId<'db>, cycle_recovery: Type<'db>) -> Self { let _ = scope; Self { diff --git a/crates/ty_python_semantic/src/types/infer/builder.rs b/crates/ty_python_semantic/src/types/infer/builder.rs index 9cd1b53981..58d4f9cdd5 100644 --- a/crates/ty_python_semantic/src/types/infer/builder.rs +++ b/crates/ty_python_semantic/src/types/infer/builder.rs @@ -461,7 +461,9 @@ pub(super) struct TypeInferenceBuilder<'db, 'ast> { /// The scope this region is part of. scope: ScopeId<'db>, - // bindings, declarations, and deferred can only exist in definition, or scope contexts. + // bindings, declarations, and deferred exist in definition and scope contexts, and — for a + // basedpython statement expression, which puts a statement inside an expression — in an + // expression context too, which hands them back to the region that asked for it. /// The types of every binding in this region. /// /// The list should only contain one entry per binding at most. @@ -901,6 +903,13 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { fn extend_expression_without_bindings(&mut self, inference: &ExpressionInference<'db>) { self.extend_expression_types(inference.expressions.iter().copied()); + // basedpython: a statement expression puts a statement inside an expression region, so + // a region can hold definitions written in the scope around it. A declaration is never + // owned by the enclosing statement the way a binding is — the scope runs its + // post-inference checks over every declaration it sees, and resolves the deferred parts + self.declarations.extend(inference.declarations()); + self.deferred.extend(inference.deferred()); + if let Some(extra) = &inference.extra { self.comparison_truthiness .extend(extra.comparison_truthiness.iter().copied()); @@ -937,6 +946,9 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { assert_eq!(self.scope, inference.scope); self.extend_expression_types(inference.expressions.iter().map(|(key, ty)| (*key, *ty))); + self.declarations + .extend(inference.declarations.iter().map(|(key, ty)| (*key, *ty))); + self.deferred.extend(inference.deferred.iter().copied()); self.comparison_truthiness.extend( inference .comparison_truthiness @@ -1495,10 +1507,9 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { } for function in &self.called_functions { - post_inference::overloaded_function::check_overloaded_function( + post_inference::overloaded_function::check_called_overloaded_function( &self.context, - Type::FunctionLiteral(*function), - function.definition(self.db()), + *function, self.scope.scope(self.db()).node(), self.index, &mut seen_overloaded_places, @@ -16712,15 +16723,6 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { let diagnostics = context.finish_uncompacted(); let _ = scope; - assert!( - declarations.is_empty(), - "Expression region can't have declarations" - ); - assert!( - deferred.is_empty(), - "Expression region can't have deferred definitions" - ); - FullExpressionCacheEntry { expressions, comparison_truthiness, @@ -16734,6 +16736,8 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { unsolved_typevar_calls, expected_types, bindings, + declarations, + deferred, diagnostics, called_functions, cycle_recovery, @@ -17475,6 +17479,8 @@ struct FullExpressionCacheEntry<'db> { unsolved_typevar_calls: FxHashSet, expected_types: FxHashMap>, bindings: VecMap, Type<'db>>, + declarations: VecMap, TypeAndQualifiers<'db>>, + deferred: VecSet>, diagnostics: TypeCheckDiagnostics, called_functions: FxIndexSet>, cycle_recovery: Option>, @@ -17505,6 +17511,8 @@ impl<'db> FullExpressionCacheEntry<'db> { && self.unsolved_typevar_calls.is_empty() && self.expected_types.is_empty() && self.bindings.is_empty() + && self.declarations.is_empty() + && self.deferred.is_empty() && self.diagnostics.is_empty() && self.called_functions.is_empty() && self.cycle_recovery.is_none() @@ -17526,6 +17534,8 @@ impl<'db> FullExpressionCacheEntry<'db> { || !self.expected_types.is_empty() || self.cycle_recovery.is_some() || !self.bindings.is_empty() + || !self.declarations.is_empty() + || !self.deferred.is_empty() || !self.called_functions.is_empty() || !self.diagnostics.is_empty()) .then(|| { @@ -17553,6 +17563,8 @@ impl<'db> FullExpressionCacheEntry<'db> { expected_types: FrozenMap::from(self.expected_types), type_expression_flags: FrozenMap::from(self.type_expression_flags), bindings: self.bindings.into_boxed_slice(), + declarations: self.declarations.into_boxed_slice(), + deferred: self.deferred.into_iter().collect(), diagnostics: self.diagnostics, called_functions: self.called_functions.into_iter().collect(), cycle_recovery: self.cycle_recovery, @@ -18053,6 +18065,10 @@ impl VecSet { self.0.is_empty() } + fn iter(&self) -> std::slice::Iter<'_, V> { + self.0.iter() + } + fn into_boxed_slice(self) -> Box<[V]> { self.0.into_boxed_slice() } diff --git a/crates/ty_python_semantic/src/types/infer/builder/post_inference/final_variable.rs b/crates/ty_python_semantic/src/types/infer/builder/post_inference/final_variable.rs index a398c9aab1..9b468b339f 100644 --- a/crates/ty_python_semantic/src/types/infer/builder/post_inference/final_variable.rs +++ b/crates/ty_python_semantic/src/types/infer/builder/post_inference/final_variable.rs @@ -2,7 +2,10 @@ use crate::{ TypeQualifiers, place::place_from_declarations, reachability::binding_reachability, - types::{context::InferContext, diagnostic::FINAL_WITHOUT_VALUE}, + types::{ + context::InferContext, diagnostic::FINAL_WITHOUT_VALUE, + inferred_signature::can_implicitly_return_none, + }, }; use ty_python_core::SemanticIndex; @@ -30,19 +33,19 @@ pub(crate) fn check_final_without_value<'db>( let place_table = index.place_table(file_scope_id); let env = context.program_environment(); - for (symbol_id, declarations) in use_def.all_end_of_scope_symbol_declarations() { + let check = |symbol_id, declarations| { let result = place_from_declarations(db, env, declarations); let first_declaration = result.first_declaration; let (place_and_quals, _) = result.into_place_and_conflicting_declarations(); if !place_and_quals.qualifiers.contains(TypeQualifiers::FINAL) { - continue; + return; } // Imports inherit the `Final` qualifier from the source module, but the // import itself provides the value. if first_declaration.is_some_and(|decl| decl.kind(db).is_import()) { - continue; + return; } // Whether the scope ever gives the symbol a value, which is not the same @@ -62,7 +65,7 @@ pub(crate) fn check_final_without_value<'db>( }); if is_assigned { - continue; + return; } let place = place_table.place(symbol_id); @@ -76,5 +79,21 @@ pub(crate) fn check_final_without_value<'db>( "read-only symbol `{place}` is not assigned a value" )); } + }; + + // Which declarations a symbol has is normally read off the end of the scope, where what + // the scope wrote has settled. A scope whose every path returns never gets there and has + // nothing bound at that point, so nothing at all would be checked — and whether a `Final` + // was ever given a value has nothing to do with whether the function around it happens to + // end in a `return`. So a scope with no end to read is asked for every declaration it + // reaches instead. This is where we part from upstream, which checks nothing there. + if can_implicitly_return_none(db, use_def) { + for (symbol_id, declarations) in use_def.all_end_of_scope_symbol_declarations() { + check(symbol_id, declarations); + } + } else { + for (symbol_id, declarations, _) in use_def.all_reachable_symbols() { + check(symbol_id, declarations); + } } } diff --git a/crates/ty_python_semantic/src/types/infer/builder/post_inference/overloaded_function.rs b/crates/ty_python_semantic/src/types/infer/builder/post_inference/overloaded_function.rs index 563cbd8753..ce9e15fc15 100644 --- a/crates/ty_python_semantic/src/types/infer/builder/post_inference/overloaded_function.rs +++ b/crates/ty_python_semantic/src/types/infer/builder/post_inference/overloaded_function.rs @@ -18,17 +18,27 @@ use crate::{ }, }; use ty_python_core::{ - SemanticIndex, definition::Definition, place::ScopedPlaceId, scope::NodeWithScopeKind, + SemanticIndex, + definition::{Definition, DefinitionState}, + place::ScopedPlaceId, + scope::{FileScopeId, NodeWithScopeKind}, }; -/// Check the overloaded functions in this scope. +/// Check the overloaded function this definition is part of, if it is the one this place holds +/// in this scope. /// -/// This only checks the overloaded functions that are: -/// 1. Visible publicly at the end of this scope -/// 2. Or, defined and called in this scope +/// Which set a place holds is normally read off the end of the scope: the value that survives +/// everything written above it is the one worth reporting against, and a set shadowed by another +/// function of the same name is therefore left alone. /// -/// For (1), this has the consequence of not checking an overloaded function that is being -/// shadowed by another function with the same name in this scope. +/// A scope whose end is unreachable — a function body whose every path returns — has no +/// end-of-scope value at all, and upstream checks nothing there. basedpython does not follow +/// that: whether a nested `def` is malformed has nothing to do with whether the function around +/// it happens to end in `return`, and "attempting to call `g` will raise `TypeError`" is as true +/// in one as in the other. So when the scope has no end to read, the set is taken to be the one +/// this place holds if it accounts for every declaration of the name the scope reaches — which is +/// the same question shadowing asked, answered without needing the scope to finish, and the same +/// rule [`check_called_overloaded_function`] uses. pub(crate) fn check_overloaded_function<'db>( context: &InferContext<'db, '_>, ty: Type<'db>, @@ -38,9 +48,6 @@ pub(crate) fn check_overloaded_function<'db>( seen_overloaded_places: &mut FxHashSet, seen_public_functions: &mut FxHashSet>, ) { - // Collect all the unique overloaded function places in this scope. This requires a set - // because an overloaded function uses the same place for each of the overloads and the - // implementation. let Type::FunctionLiteral(function) = ty else { return; }; @@ -60,29 +67,142 @@ pub(crate) fn check_overloaded_function<'db>( let place = definition.place(db); - let use_def = index.use_def_map(context.scope().file_scope_id(db)); - - let Place::Defined(DefinedPlace { - ty: Type::FunctionLiteral(function), - definedness: Definedness::AlwaysDefined, - .. - }) = place_from_bindings( - db, - env, - use_def.end_of_scope_symbol_bindings(place.as_symbol().unwrap()), - ) - .place - else { + let scope_id = context.scope().file_scope_id(db); + let use_def = index.use_def_map(scope_id); + let symbol = place.as_symbol().unwrap(); + + let function = + match place_from_bindings(db, env, use_def.end_of_scope_symbol_bindings(symbol)).place { + Place::Defined(DefinedPlace { + ty: Type::FunctionLiteral(end_of_scope), + definedness: Definedness::AlwaysDefined, + .. + }) => { + if !end_of_scope.contains_definition(db, definition) { + // The public end-of-scope binding for this place can be a different overloaded + // function value assigned to the same name. In that case, the current local + // overload definition is shadowed, and checking the public function here would + // report against the wrong function. + return; + } + end_of_scope + } + // Nothing survives to the end of the scope: a scope with no end to reach binds nothing + // there, and reads back `Never`. The declarations still ran, and the set is still the one + // the name holds wherever the scope was left — so it is checked, as long as no other + // declaration of the name competes with it. + Place::Undefined + | Place::Defined(DefinedPlace { + ty: Type::Never, .. + }) => { + if !accounts_for_every_declaration(db, index, scope_id, place, function) { + return; + } + function + } + Place::Defined(_) => return, + }; + + check_overload_set( + context, + function, + place, + scope, + index, + seen_overloaded_places, + seen_public_functions, + ); +} + +/// Check an overloaded function that this scope both defines and calls. +/// +/// [`check_overloaded_function`] reads what the name holds once the scope has run, so it sees +/// nothing at all when the end of the scope is unreachable — and a nested `def` inside a +/// function whose every path returns is the ordinary shape of that, not an exotic one. The +/// `def`s written above a call still run when the call does, so an overload set that is +/// malformed there raises `TypeError` just the same. +/// +/// The call may have resolved to only part of an overload set: a `g(1)` written between two +/// `@overload def g`s sees the first of them and not the second. So the called value is checked +/// only when its own overload set already accounts for every declaration written at that place +/// in this scope, which is what makes it the whole set rather than a prefix of one. +pub(crate) fn check_called_overloaded_function<'db>( + context: &InferContext<'db, '_>, + function: FunctionType<'db>, + scope: &NodeWithScopeKind, + index: &SemanticIndex<'db>, + seen_overloaded_places: &mut FxHashSet, + seen_public_functions: &mut FxHashSet>, +) { + let db = context.db(); + + if function.file(db) != context.file() { return; - }; + } - if !function.contains_definition(db, definition) { - // The public end-of-scope binding for this place can be a different overloaded function - // value assigned to the same name. In that case, the current local overload definition is - // shadowed, and checking the public function here would report against the wrong function. + if !function.has_known_decorator(db, FunctionDecorators::OVERLOAD) { + return; + } + + let place = function.definition(db).place(db); + let scope_id = context.scope().file_scope_id(db); + + if !accounts_for_every_declaration(db, index, scope_id, place, function) { return; } + check_overload_set( + context, + function, + place, + scope, + index, + seen_overloaded_places, + seen_public_functions, + ); +} + +/// Whether `function`'s own overload set already covers every declaration of `symbol` this scope +/// reaches. +/// +/// This is what makes a value the whole overload set rather than a part of one. A declaration the +/// set does not contain is either a competing definition of the name — the shadowing case — or a +/// later `@overload` the value was built too early to have seen, which is what a call written +/// part-way down a set resolves to. +fn accounts_for_every_declaration<'db>( + db: &'db dyn crate::Db, + index: &SemanticIndex<'db>, + scope: FileScopeId, + place: ScopedPlaceId, + function: FunctionType<'db>, +) -> bool { + let Some(symbol) = place.as_symbol() else { + return false; + }; + index + .use_def_map(scope) + .reachable_symbol_declarations(symbol) + .all(|declaration| match declaration.declaration { + DefinitionState::Defined(declaration) => function.contains_definition(db, declaration), + DefinitionState::Undefined | DefinitionState::Deleted => true, + }) +} + +/// Report everything that is wrong with one complete overload set. +fn check_overload_set<'db>( + context: &InferContext<'db, '_>, + function: FunctionType<'db>, + place: ScopedPlaceId, + scope: &NodeWithScopeKind, + index: &SemanticIndex<'db>, + seen_overloaded_places: &mut FxHashSet, + seen_public_functions: &mut FxHashSet>, +) { + let db = context.db(); + let env = context.program_environment(); + + // An overloaded function uses the same place for each of the overloads and the + // implementation, so a place is only worth checking once. if !seen_overloaded_places.insert(place) { // We have already checked this overloaded function in this scope, so we can skip it. return; From c965fff7764f02501f88e0459549be5fd5d6ca78 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Thu, 17 Sep 2026 21:30:04 +1000 Subject: [PATCH 11/16] transpiler: keep the author's source, refuse lost edits, and make repeated `_` positional-only re-emit only the nodes an AST pass rewrote and pass the source of the rest through, keeping comments, indentation, clause-line bodies and docstrings, and report the author's own text for f-string and t-string fields. the splice refuses any edit it leaves out unless the pass covering it declares it writes that lowering. a repeated `_` is numbered past every name the module spells, positional-only, and named after an overridden method's parameters, decided once in ty and read by the transpiler, callable types and inline protocols included. `decorator def` gets real overloads, a context parameter is filled through an overload set that agrees on it, a module binding a runtime helper's name is reported, and by run compiles each staged module under its .by path and lines --- crates/by_build/tests/differential.rs | 181 ++ crates/by_irbuild/src/generators.rs | 29 +- crates/by_irbuild/src/lib.rs | 30 +- crates/by_irbuild/src/surface.rs | 50 + crates/by_irbuild/src/tests.rs | 34 + crates/by_stage/src/emit.rs | 19 +- crates/by_transforms/src/lib.rs | 820 ++++++- crates/by_transforms/src/runtime.rs | 755 +++++- .../by_transforms/src/runtime/_by_runtime.py | 40 +- crates/by_transforms/src/source_map.rs | 41 +- .../src/transforms/annotation.rs | 24 +- .../src/transforms/anon_named_tuple.rs | 93 +- .../src/transforms/ast_driver.rs | 1451 +++++++++--- .../by_transforms/src/transforms/callable.rs | 192 +- .../by_transforms/src/transforms/coalesce.rs | 85 +- .../src/transforms/coalesce_chain.rs | 487 ---- .../src/transforms/context_params.rs | 284 ++- .../src/transforms/conversion.rs | 47 +- .../src/transforms/debug_field.rs | 602 +++++ .../src/transforms/decl_site_variance.rs | 4 + .../src/transforms/decorator_keyword.rs | 777 +++++- .../src/transforms/destructure.rs | 102 +- .../src/transforms/dynamic_keyword.rs | 10 +- crates/by_transforms/src/transforms/enums.rs | 57 +- .../by_transforms/src/transforms/extension.rs | 94 +- .../src/transforms/float_const.rs | 4 + .../src/transforms/force_unwrap.rs | 126 +- .../by_transforms/src/transforms/generics.rs | 40 +- crates/by_transforms/src/transforms/if_let.rs | 9 +- .../src/transforms/inferred_annotation.rs | 78 +- .../src/transforms/init_method.rs | 244 +- .../src/transforms/just_float.rs | 4 + .../src/transforms/kw_subscript.rs | 5 + .../src/transforms/lazy_import.rs | 400 +++- .../src/transforms/literal_types.rs | 4 + .../src/transforms/local_once.rs | 15 +- .../src/transforms/main_function.rs | 98 +- .../src/transforms/match_polyfill.rs | 3 +- .../src/transforms/match_type.rs | 10 + crates/by_transforms/src/transforms/mod.rs | 4 +- .../by_transforms/src/transforms/modifiers.rs | 13 +- .../src/transforms/mutable_defaults.rs | 280 ++- .../src/transforms/none_chain.rs | 19 +- .../src/transforms/optional_type.rs | 4 + .../by_transforms/src/transforms/overload.rs | 219 +- .../src/transforms/parametric_is.rs | 9 + .../src/transforms/properties.rs | 10 + .../src/transforms/protocol_type.rs | 114 +- .../src/transforms/raises_clause.rs | 93 +- .../src/transforms/reified_class.rs | 2 +- .../src/transforms/repeated_underscore.rs | 749 +++++- .../by_transforms/src/transforms/rerender.rs | 2080 +++++++++++++++++ .../by_transforms/src/transforms/soundness.rs | 512 +++- .../src/transforms/source_util.rs | 185 +- .../src/transforms/statement_expression.rs | 35 +- .../src/transforms/static_resource.rs | 108 +- .../src/transforms/string_tag.rs | 49 +- .../src/transforms/super_keyword.rs | 29 +- .../src/transforms/symbolic_type_op.rs | 34 +- .../src/transforms/template_expression.rs | 557 +++++ .../src/transforms/trailing_lambda.rs | 30 +- .../by_transforms/src/transforms/type_is.rs | 6 + .../src/transforms/typed_dict_literal.rs | 6 + .../src/transforms/typed_lambda.rs | 15 +- .../src/transforms/typeof_keyword.rs | 3 +- .../src/transforms/unique_loop_bindings.rs | 6 +- crates/by_transforms/src/transforms/unpack.rs | 4 + .../src/transforms/visibility_rename.rs | 10 + crates/by_transforms/src/type_info.rs | 194 +- .../by_transforms/tests/conversion_runtime.rs | 24 + .../tests/fstring_debug_runtime.rs | 115 + .../tests/lazy_import_runtime.rs | 95 + .../tests/tstring_expression_runtime.rs | 138 ++ crates/ruff_db/src/diagnostic/mod.rs | 20 + .../test/fixtures/ruff/force_unwrap.by | 19 + .../snapshots/format@force_unwrap.by.snap | 49 + .../src/parser/expression.rs | 84 +- crates/ruff_python_parser/src/parser/tests.rs | 180 ++ crates/ty/src/by_commands.rs | 140 +- crates/ty/tests/by_e2e.rs | 131 ++ crates/ty/tests/mdtest_divergence.rs | 8 + crates/ty_ide/src/inlay_hints.rs | 31 +- ...n_repeated_underscore_parameter_names.snap | 48 + .../lint_docs/invalid-decorator-def.md | 57 + .../lint_docs/invalid-repeated-underscore.md | 57 + .../mdtest/basedpython_context_parameters.md | 257 +- .../mdtest/basedpython_decorator_keyword.md | 135 ++ .../resources/mdtest/repeated_underscore.md | 360 ++- ...ontex\342\200\246_(7854bce3d4fdb653).snap" | 55 + ...t_lea\342\200\246_(39d2635c95b04ae2).snap" | 56 + ...disag\342\200\246_(420a7d0ca57c17a8).snap" | 57 + .../ty_python_semantic/src/semantic_model.rs | 30 +- crates/ty_python_semantic/src/types.rs | 2 + .../ty_python_semantic/src/types/call/bind.rs | 253 +- .../src/types/context_params.rs | 400 +++- .../src/types/diagnostic.rs | 165 +- .../ty_python_semantic/src/types/function.rs | 162 +- .../src/types/ide_support.rs | 53 +- .../src/types/infer/builder.rs | 61 +- .../src/types/infer/builder/function.rs | 136 +- .../types/infer/builder/type_expression.rs | 51 +- .../src/types/repeated_underscore.rs | 788 +++++++ .../src/types/signatures.rs | 242 +- .../src/types/state_reads.rs | 1 + .../src/server/api/requests/entry_point.rs | 8 +- .../development/how-transpilation-works.md | 51 +- docs/basedpython/development/sourcemaps.md | 10 + .../features/context-parameters.md | 51 +- .../features/decorated-parameters.md | 7 +- .../features/differences-from-python.md | 19 + docs/basedpython/features/exceptions.md | 7 +- docs/basedpython/features/lazy-imports.md | 56 + .../features/repeated-underscore.md | 139 +- docs/basedpython/features/wrapped-results.md | 14 + ty.schema.json | 22 +- 115 files changed, 15651 insertions(+), 1920 deletions(-) delete mode 100644 crates/by_transforms/src/transforms/coalesce_chain.rs create mode 100644 crates/by_transforms/src/transforms/debug_field.rs create mode 100644 crates/by_transforms/src/transforms/rerender.rs create mode 100644 crates/by_transforms/src/transforms/template_expression.rs create mode 100644 crates/by_transforms/tests/fstring_debug_runtime.rs create mode 100644 crates/by_transforms/tests/tstring_expression_runtime.rs create mode 100644 crates/ruff_python_formatter/resources/test/fixtures/ruff/force_unwrap.by create mode 100644 crates/ruff_python_formatter/tests/snapshots/format@force_unwrap.by.snap create mode 100644 crates/ty_ide/src/snapshots/ty_ide__inlay_hints__tests__basedpython_repeated_underscore_parameter_names.snap create mode 100644 crates/ty_python_semantic/resources/lint_docs/invalid-decorator-def.md create mode 100644 crates/ty_python_semantic/resources/lint_docs/invalid-repeated-underscore.md create mode 100644 "crates/ty_python_semantic/resources/mdtest/snapshots/basedpython_context_\342\200\246_-_basedpython___context\342\200\246_-_a_positional_`contex\342\200\246_(7854bce3d4fdb653).snap" create mode 100644 "crates/ty_python_semantic/resources/mdtest/snapshots/basedpython_context_\342\200\246_-_basedpython___context\342\200\246_-_an_overload_that_lea\342\200\246_(39d2635c95b04ae2).snap" create mode 100644 "crates/ty_python_semantic/resources/mdtest/snapshots/basedpython_context_\342\200\246_-_basedpython___context\342\200\246_-_overloads_that_disag\342\200\246_(420a7d0ca57c17a8).snap" create mode 100644 crates/ty_python_semantic/src/types/repeated_underscore.rs diff --git a/crates/by_build/tests/differential.rs b/crates/by_build/tests/differential.rs index 1bde73d62d..6c3a5672b2 100644 --- a/crates/by_build/tests/differential.rs +++ b/crates/by_build/tests/differential.rs @@ -6208,6 +6208,42 @@ fn a_value_nothing_verified_is_checked_where_the_interpreted_build_checks_it() { ); } +#[test] +fn a_value_nothing_verified_is_checked_in_a_statement_the_transpiler_prints_from_its_tree() { + // `typeof` is lowered by rewriting the syntax tree, so the top-level statement holding + // one is printed again rather than edited in place. the interpreted build still makes + // every check in that statement, as the compiled module does + agree( + "unverifiedtree", + "\ +from typing import Any + + +def dyn() -> Any: + return 'x' + + +def typed(n: int) -> object: + a: int = dyn() + b: typeof(n) = n + return a + + +def looped(n: int, xs: list[Any]) -> object: + b: typeof(n) = n + total = 0 + for x in xs: + c: int = x + total = total + c + return total +", + &[ + "str(_capture(m.typed, 1))", + "(str(_capture(m.looped, 1, [2, 'y'])), m.looped(1, [2, 3]))", + ], + ); +} + #[test] fn with_no_soundness_checks_neither_build_checks_a_value_nothing_verified() { // `--soundness none` transpiles no checks, and a compiled module then makes none @@ -6262,6 +6298,45 @@ def produce(x: dict[str, int]) -> Iterator[object]: ); } +#[test] +fn a_repeated_underscore_parameter_is_checked_against_its_own_type() { + // the `parameters` position checks each parameter by the name both builds bind it + // to, and a repeated `_` is numbered there. checked by its source name, the second + // `_` was the first again, and a call passing what was declared was refused + let Some(compiled) = agree_built( + "underscorecheck", + "\ +def pick(_: int, _: str) -> int: + return 1 +", + &[ + "str(_capture(m.pick, 1, 'a'))", + "str(_capture(m.pick, 1, 2))", + ], + false, + by_irbuild::Language::BasedPython, + Config { + soundness: by_transforms::SoundnessPositions::all(), + ..Config::default() + }, + ) else { + return; + }; + let Some((python, _)) = environment() else { + return; + }; + assert_eq!( + run( + &python, + &compiled, + "import by_diff_underscorecheck as m\n\ + assert m.__file__.endswith('.so'), m.__file__\n\ + print(m.pick(1, 'a'))\n", + ), + "1" + ); +} + #[test] fn an_element_check_reads_the_classes_it_names_once_where_the_loop_begins() { // `_soundness_iter` is handed the class it tests against before the first element is @@ -17019,6 +17094,112 @@ def calls_none() -> int: ); } +#[test] +fn repeated_underscore_parameters_agree() { + // basedpython lets a `def` repeat `_` for parameters it ignores, and python refuses a + // repeated name, so the interpreted definition keeps the first `_` and numbers the rest. + // the compiled one answers to the same names — the forwarder spells them, and a keyword + // call reaches them — where it used to spell `_` twice and fail the module's import + let Some(compiled) = agree_in( + "underscores", + "\ +from collections.abc import Iterator + +def pair(_: int, _: int) -> int: + return 7 + +def spaced(a: int, _: int, b: int, _: int) -> int: + return a * 10 + b + +def every_kind(_: int, *_: int, _: int, **_: int) -> int: + return 1 + +def counted(_: int, n: int, _: int) -> Iterator[int]: + for i in range(n): + yield i + +def outer(n: int) -> int: + def inner(_: int, _: int) -> int: + return n + return inner(1, 2) + +def through_lambda(n: int) -> int: + ignore = lambda _, _: n + return ignore(1, 2) + +class Holder: + def method(self, _: int, _: int) -> int: + return 5 +", + &[ + "[str(__import__('inspect').signature(f)) for f in (m.pair, m.spaced, m.every_kind, m.counted, m.Holder().method)]", + "(m.pair(1, 2), m.pair(1, _2=2), m.pair(_=1, _2=2))", + "str(_capture_kw(m.pair, (1,), {'_': 2}))", + "(m.spaced(1, 2, 3, _2=4), m.every_kind(1, 2, 3, _2=4, x=5), m.every_kind(1, _2=4))", + "(list(m.counted(1, 3, 2)), list(m.counted(1, 3, _2=2)))", + "(m.outer(1), m.through_lambda(4))", + "(m.Holder().method(5, _2=6), str(_capture(m.Holder().method, 5, 6, 7)))", + ], + false, + by_irbuild::Language::BasedPython, + ) else { + return; + }; + let Some((python, _)) = environment() else { + return; + }; + assert_eq!( + run( + &python, + &compiled, + "import by_diff_underscores as m\n\ + print([f.__code__.co_filename for f in (m.pair, m.spaced, m.every_kind, m.counted)])\n", + ), + "['', '', '', '']" + ); +} + +#[test] +fn a_numbered_underscore_parameter_does_not_shadow_a_module_name() { + // the numbered name of a repeated `_` is a local of the function, so numbering one + // `_2` in a module that binds `_2` had the default and the body read the second + // argument where python reads the module's list. both builds skip every name the + // module spells, and have to skip the same ones: the signature spells the result + let Some(compiled) = agree_in( + "underscorenames", + "\ +_2: list[int] = [9] + +def defaulted(_: int, _: int, x: list[int] = _2) -> list[int]: + return x + +def reads(_: int, _: int) -> list[int]: + return _2 +", + &[ + "(m.defaulted(1, 2), m.reads(1, 2))", + "[list(__import__('inspect').signature(f).parameters) for f in (m.defaulted, m.reads)]", + ], + false, + by_irbuild::Language::BasedPython, + ) else { + return; + }; + let Some((python, _)) = environment() else { + return; + }; + assert_eq!( + run( + &python, + &compiled, + "import by_diff_underscorenames as m\n\ + assert m.__file__.endswith('.so'), m.__file__\n\ + print(m.defaulted(1, 2), m.reads(1, 2))\n", + ), + "[9] [9]" + ); +} + /// a `*args` is read as the tuple the calling convention built it as, which is a /// guess about the container and not a licence to skip anything the general read /// does. python's own answer to a position off either end of it, to a position diff --git a/crates/by_irbuild/src/generators.rs b/crates/by_irbuild/src/generators.rs index 7384543b71..4eed7ce12c 100644 --- a/crates/by_irbuild/src/generators.rs +++ b/crates/by_irbuild/src/generators.rs @@ -208,7 +208,11 @@ pub(crate) fn check(function: &ast::StmtFunctionDef) -> Lowered<()> { /// conservatively all of them rather than only those live across a yield. a liveness /// analysis would shrink the object; it would not change what compiles, and getting /// it wrong would lose a value across a suspension -pub(crate) fn state_names(function: &ast::StmtFunctionDef, locals: &[String]) -> Vec { +pub(crate) fn state_names( + function: &ast::StmtFunctionDef, + written: by_transforms::WrittenNames, + locals: &[String], +) -> Vec { let mut out = vec![ STATE_FIELD.to_string(), SENT_FIELD.to_string(), @@ -227,7 +231,14 @@ pub(crate) fn state_names(function: &ast::StmtFunctionDef, locals: &[String]) -> for name in function .parameters .iter() - .map(|parameter| parameter.name().to_string()) + .map(|parameter| { + by_transforms::python_parameter_name( + &function.parameters, + parameter.as_parameter(), + written, + ) + .to_string() + }) .chain(locals.iter().cloned()) { if seen.insert(name.clone()) { @@ -697,12 +708,22 @@ fn live_in( /// - an augmented assignment reads before it writes /// - a `for` target, or an assignment nested in an `if` or a loop — the body may not run /// - a name whose first mention is a read, which is `UnboundLocalError` territory -pub(crate) fn definitely_assigned(function: &ast::StmtFunctionDef) -> HashSet { +pub(crate) fn definitely_assigned( + function: &ast::StmtFunctionDef, + written: by_transforms::WrittenNames, +) -> HashSet { // a parameter is assigned on entry: the constructor seeds its field let mut out: HashSet = function .parameters .iter() - .map(|parameter| parameter.name().to_string()) + .map(|parameter| { + by_transforms::python_parameter_name( + &function.parameters, + parameter.as_parameter(), + written, + ) + .to_string() + }) .collect(); let mut mentioned: HashSet = HashSet::new(); diff --git a/crates/by_irbuild/src/lib.rs b/crates/by_irbuild/src/lib.rs index 3a2708f307..a86f47c27b 100644 --- a/crates/by_irbuild/src/lib.rs +++ b/crates/by_irbuild/src/lib.rs @@ -168,7 +168,13 @@ pub fn build_module( let mut module = ModuleIr::new(module_name); // where the interpreted build checks a value it cannot trust, decided once for the // whole module, so every frame lowered below makes the same checks in the same places - let soundness = by_transforms::soundness_sites(model, suite, soundness); + let source = ruff_db::source::source_text(db, model.file()); + let soundness = by_transforms::soundness_sites( + model, + by_transforms::WrittenNames::new(source.as_str()), + suite, + soundness, + ); // a call is only lowered natively when the callee is a module-level function // in this same unit, so the set has to be known before any body is lowered @@ -1315,7 +1321,9 @@ fn lower_generator( .map(|(name, _)| name.clone()) .collect(); // the constructor seeds every one of them, so they are as assigned as a parameter - let mut assigned = generators::definitely_assigned(function); + let source = ruff_db::source::source_text(db, model.file()); + let written = by_transforms::WrittenNames::new(source.as_str()); + let mut assigned = generators::definitely_assigned(function, written); assigned.extend(captured.iter().cloned()); let mut signed = signature(db, env, model, function, layouts, receiver, &[])?; takes_the_first_iterator(captures, &mut signed); @@ -1328,7 +1336,7 @@ fn lower_generator( .map(|(_, rtype)| rtype.clone()) }; let names = { - let mut names = generators::state_names(function, &locals); + let mut names = generators::state_names(function, written, &locals); if captures.is_some_and(|nested| nested.generator_expression.is_some()) { assigned.insert(closures::GENERATOR_ITERATOR.to_string()); names.push(closures::GENERATOR_ITERATOR.to_string()); @@ -8621,6 +8629,8 @@ fn signature( arrays: &[(usize, RType)], ) -> Lowered { let parameters = &function.parameters; + let source = ruff_db::source::source_text(db, model.file()); + let written = by_transforms::WrittenNames::new(source.as_str()); let mut params = Vec::with_capacity(parameters.args.len() + 1); let mut defaults: Vec> = Vec::with_capacity(parameters.args.len() + 1); @@ -8736,8 +8746,8 @@ fn signature( } } }; - let name = parameter.parameter.name.as_str(); - let rtype = match rebound.get(name) { + let name = by_transforms::python_parameter_name(parameters, ¶meter.parameter, written); + let rtype = match rebound.get(name.as_str()) { // an unboxed edition's parameter *is* the caller's buffer, and a buffer is // not a value that widens: handing one out means copying it, and a copy is // a different list. the store in the body declines instead, as it already @@ -8769,11 +8779,17 @@ fn signature( // `*args` and `**kwargs` are ordinary parameters holding an ordinary tuple and // dict — packing them is the boundary's job, not the body's if let Some(vararg) = ¶meters.vararg { - params.push((vararg.name.to_string(), RType::OBJECT)); + params.push(( + by_transforms::python_parameter_name(parameters, vararg, written).to_string(), + RType::OBJECT, + )); defaults.push(None); } if let Some(kwarg) = ¶meters.kwarg { - params.push((kwarg.name.to_string(), RType::OBJECT)); + params.push(( + by_transforms::python_parameter_name(parameters, kwarg, written).to_string(), + RType::OBJECT, + )); defaults.push(None); } diff --git a/crates/by_irbuild/src/surface.rs b/crates/by_irbuild/src/surface.rs index fcac4a247b..a60ec822f6 100644 --- a/crates/by_irbuild/src/surface.rs +++ b/crates/by_irbuild/src/surface.rs @@ -52,11 +52,61 @@ pub(crate) fn gate_function( "`{first}` is a reified type parameter, and a specialization has no emitted function to rebuild" ))); } + if reads_a_repeated_underscore(function) { + return Err(Decline::new( + "a body reading a repeated `_` parameter reads the first of them, where ty reads the last", + )); + } let mut scanner = Scanner { found: None }; scanner.walk_function(function); scanner.into_result() } +/// whether `function` repeats the parameter `_` and its body reads that name +/// +/// python refuses a repeated parameter, so the transpiler keeps the first `_` and +/// numbers the rest — a read of `_` is the first parameter's value. ty binds each +/// parameter in turn and takes the read for the *last* one, so a lowering built on +/// ty's type for the read would treat one parameter's value as the other's type. +/// a body that only ignores its `_` parameters is unaffected, and is lowered under the +/// numbered names +fn reads_a_repeated_underscore(function: &ast::StmtFunctionDef) -> bool { + struct Reads { + found: bool, + } + impl<'a> Visitor<'a> for Reads { + fn visit_stmt(&mut self, stmt: &'a Stmt) { + if let Stmt::AugAssign(node) = stmt + && matches!(node.target.as_ref(), Expr::Name(name) if name.id == "_") + { + self.found = true; + } + walk_stmt(self, stmt); + } + + fn visit_expr(&mut self, expr: &'a Expr) { + if let Expr::Name(name) = expr + && name.id == "_" + && !name.ctx.is_store() + { + self.found = true; + } + walk_expr(self, expr); + } + } + let underscores = function + .parameters + .iter() + .filter(|parameter| parameter.name() == "_") + .count(); + if underscores < 2 { + return false; + } + let mut reads = Reads { found: false }; + reads.visit_body(&function.body); + reads.found +} + /// whether this class may be lowered at all pub(crate) fn gate_class(source_type: PySourceType, class: &ast::StmtClassDef) -> Lowered<()> { // a reified *class* parameter is read off the instance, which means the diff --git a/crates/by_irbuild/src/tests.rs b/crates/by_irbuild/src/tests.rs index c99412e20a..107823767c 100644 --- a/crates/by_irbuild/src/tests.rs +++ b/crates/by_irbuild/src/tests.rs @@ -1131,6 +1131,40 @@ def outer(xs: list[list[int]], y: list[int]) -> object: ); } +#[test] +fn a_body_reading_a_repeated_underscore_parameter_declines() { + // python's `_` is the first of the parameters, the transpiler numbering the rest, + // where ty takes a read of it for the last one. a body that ignores them all is + // lowered, and a nested frame reading the name counts as a read + assert_eq!( + declines( + "\ +def ignores(_: int, _: str) -> int: + return 1 + +def reads(_: int, _: str) -> object: + return _ + +def adds(_: int, _: int) -> int: + _ += 1 + return 0 + +def captures(_: int, _: str) -> int: + def inner() -> object: + return _ + return 0 +" + ), + ["reads", "adds", "captures"] + .map(|name| ( + name.to_string(), + "a body reading a repeated `_` parameter reads the first of them, where ty reads the last" + .to_string() + )) + .to_vec() + ); +} + #[test] fn a_target_list_unpacks_into_a_fixed_tuple() { // one op with one destination, read back element by element — a second diff --git a/crates/by_stage/src/emit.rs b/crates/by_stage/src/emit.rs index 3357ad542e..db3fa14bc4 100644 --- a/crates/by_stage/src/emit.rs +++ b/crates/by_stage/src/emit.rs @@ -203,7 +203,7 @@ pub fn check_and_transpile( })?; } Err(e) => { - all_diagnostics.push(transpile_bug_diagnostic(*file, &e)); + all_diagnostics.push(transpile_failure_diagnostic(*file, &e)); ok = false; if gate == CheckGate::AllErrors { break; @@ -235,15 +235,20 @@ pub fn is_unusable_source(d: &Diagnostic) -> bool { /// source. When the failure maps back to a `.by` range, attach it so the /// diagnostic renders with `--> file:line:col` and a source caret like any /// other; otherwise fall back to a bare message. -pub fn transpile_bug_diagnostic( +/// +/// The id is the failure's own kind, because the three read differently to whoever gets +/// them: a refusal is about the module and says what to rename, while `invalid-syntax` and +/// `internal-error` are about the transpiler and the next step is to report them. +pub fn transpile_failure_diagnostic( file: ruff_db::files::File, err: &by_transforms::TranspileError, ) -> Diagnostic { - let mut diag = Diagnostic::new( - DiagnosticId::InvalidSyntax, - Severity::Error, - err.message.clone(), - ); + let id = match err.kind { + by_transforms::TranspileErrorKind::InvalidSyntax => DiagnosticId::InvalidSyntax, + by_transforms::TranspileErrorKind::Refused => DiagnosticId::TranspileRefused, + by_transforms::TranspileErrorKind::Bug => DiagnosticId::InternalError, + }; + let mut diag = Diagnostic::new(id, Severity::Error, err.message.clone()); if let Some(range) = err.by_range { diag.annotate(Annotation::primary(Span::from(file).with_range(range))); } diff --git a/crates/by_transforms/src/lib.rs b/crates/by_transforms/src/lib.rs index 78d8e62ae3..90bddfd38e 100644 --- a/crates/by_transforms/src/lib.rs +++ b/crates/by_transforms/src/lib.rs @@ -6,6 +6,7 @@ mod transforms; pub(crate) mod type_info; pub use config::{Config, FloatLiteralLowering, PythonVersion, SoundnessPositions}; +pub use transforms::repeated_underscore::{WrittenNames, python_parameter_name}; pub use transforms::soundness::{SoundnessSites, soundness_sites}; /// A module's `main` as the program's command line — the reading the entry-point @@ -149,13 +150,17 @@ fn transpile_with_report( return Ok((source.to_owned(), RuntimeRequirements::default())); } + // the names the module spells as written, which a repeated `_` parameter is + // numbered around. read before any rewrite, as the native compiler reads them + let written_names = WrittenNames::new(source); + // one db over the original source, shared by the qualification phase below // and — as long as nothing rewrites the source — by phase 0's type-aware // passes, which would otherwise build an identical one of their own let (local_db, local_file) = make_in_memory_db(source); // what the author wrote, before any rewrite, so phase 3 can tell a helper // call the transpiler emitted from a name the program reads itself - let written = names_written( + let author = AuthorNames::new( ruff_db::parsed::parsed_module( &local_db, ty_python_semantic::Db::program_file(&local_db, local_file).python_file(&local_db), @@ -199,6 +204,7 @@ fn transpile_with_report( && matches!(enum_lowered.output, std::borrow::Cow::Borrowed(_)); let (source, ast_errors, _phase0_map) = transforms::ast_driver::run_against_source( source, + written_names, config, unchanged.then_some((&local_db as &dyn ty_python_semantic::Db, local_file)), ); @@ -230,7 +236,7 @@ fn transpile_with_report( // --- Phase 2: import-redirect, surface-syntax cleanup, lazy-import marking --- let (final_output, requirements) = run_import_redirect_phase(output, config); - let final_output = run_anon_named_tuple_cleanup(final_output, config)?; + let final_output = run_anon_named_tuple_cleanup(final_output, written_names, config)?; let final_output = run_lazy_import_phase( final_output, config, @@ -240,7 +246,7 @@ fn transpile_with_report( let final_output = run_version_polyfill_phase(final_output, config); // --- Phase 3: syntax verification --- - verify_syntax(&final_output, &written).map_err(|e| e.message)?; + verify_syntax(&final_output, &author).map_err(|e| e.message)?; verify_target_syntax(&final_output, config).map_err(|e| e.message)?; Ok((final_output, requirements)) @@ -343,6 +349,9 @@ pub fn transpile_typed_with_report( }; let source_ref = ruff_db::source::source_text(db, file); let original_source = source_ref.as_str(); + // the names the module spells as written, which a repeated `_` parameter is + // numbered around. read before any rewrite, as the native compiler reads them + let written_names = WrittenNames::new(original_source); if config.is_python { let out = original_source.to_owned(); @@ -353,7 +362,7 @@ pub fn transpile_typed_with_report( )); } - let written = names_written( + let author = AuthorNames::new( ruff_db::parsed::parsed_module( db, ty_python_semantic::Db::program_file(db, file).python_file(db), @@ -425,7 +434,7 @@ pub fn transpile_typed_with_report( let eager_imports = eager_model.eagerly_imported_modules(); let eager_names = eager_model.eagerly_imported_names(); let (spliced, ast_errors, phase0_map) = - transforms::ast_driver::run_against_source(working_source, config, project); + transforms::ast_driver::run_against_source(working_source, written_names, config, project); if let Some(first) = ast_errors.first() { return Err(first.clone().into()); } @@ -466,7 +475,7 @@ pub fn transpile_typed_with_report( } let (final_output, requirements) = run_import_redirect_phase(output, config); - let final_output = run_anon_named_tuple_cleanup(final_output, config)?; + let final_output = run_anon_named_tuple_cleanup(final_output, written_names, config)?; let final_output = run_lazy_import_phase(final_output, config, &eager_imports, &eager_names); let final_output = run_version_polyfill_phase(final_output, config); @@ -507,11 +516,15 @@ pub fn transpile_typed_with_report( line_map.extend(composed[kept..].iter().copied()); // verify last: on failure, map the generated span back to a `.by` range - let verified = verify_syntax(&final_output, &written) + let verified = verify_syntax(&final_output, &author) .and_then(|()| verify_target_syntax(&final_output, config)); if let Err(mut err) = verified { - err.by_range = err.output_range.and_then(|r| { - output_offset_to_by_range(&line_map, &final_output, original_source, r.start()) + // a range already set is one in the `.by` source to begin with — the + // helper-name clash points at where the author bound the name + err.by_range = err.by_range.or_else(|| { + err.output_range.and_then(|r| { + output_offset_to_by_range(&line_map, &final_output, original_source, r.start()) + }) }); return Err(err); } @@ -535,7 +548,11 @@ fn line_count(s: &str) -> usize { /// Re-runs the anon-named-tuple lowering on post-transform output to catch /// expressions that other transforms (e.g. the PEP-695 polyfill) copied /// verbatim from the source after the original pass ran -fn run_anon_named_tuple_cleanup(mut source: String, config: &Config) -> Result { +fn run_anon_named_tuple_cleanup( + mut source: String, + written: WrittenNames, + config: &Config, +) -> Result { use ruff_python_ast::visitor::Visitor; for _ in 0..4 { @@ -553,14 +570,15 @@ fn run_anon_named_tuple_cleanup(mut source: String, config: &Config) -> Result (String, Runtim /// /// `eager_names` names the *bindings* that must be bound to the real object: a /// lazy proxy cannot stand where cpython checks for a real class, which is what -/// `except` does +/// `except` does, nor where a special form is told apart by identity, nor in an +/// annotation something reads back /// /// A stub defers nothing: it is never executed, and deferring its imports would /// hand a checker a call result where the stub declares a module or a class @@ -682,6 +701,22 @@ fn run_lazy_import_phase( ty_python_semantic::Db::program_file(&db, file).python_file(&db), ) .load(&db); + // the lowering writes imports of its own — an enum's `ClassVar` among them — which + // the source the transpile began from never held, so the output is asked too. the + // annotations are read off the output for the same reason: a lowering writes those + let mut eager_names = eager_names.to_vec(); + eager_names.extend( + ty_python_semantic::SemanticModel::new( + &db, + ty_python_semantic::Db::program_file(&db, file), + ) + .eagerly_imported_names(), + ); + eager_names.extend(transforms::lazy_import::annotation_names( + module.suite(), + src, + )); + eager_names.extend(transforms::lazy_import::decorator_names(module.suite())); let deferral = if config.is_stub { transforms::lazy_import::Deferral::Never @@ -694,7 +729,7 @@ fn run_lazy_import_phase( // through a proxy would be a proxy call on every use let mut eager = eager.to_vec(); eager.extend(config.runtime_module.clone()); - let mut lazy = transforms::lazy_import::LazyImport::new(src, deferral, &eager, eager_names); + let mut lazy = transforms::lazy_import::LazyImport::new(src, deferral, &eager, &eager_names); for stmt in module.suite() { lazy.visit_stmt(stmt); } @@ -713,8 +748,21 @@ fn run_lazy_import_phase( return source; } + let (mut body, _) = apply_transforms_once(src, lazy.edits); + // phase 0 already gave the module the helpers *it* needed. where that was an import of the + // runtime written beside it, the helpers this phase needs belong on that same line rather + // than on a second one naming the same module + if let Some(module) = config.runtime_module.as_deref() + && !helpers.is_empty() + { + let line = runtime::import_line(module, helpers.iter().copied()); + let left_over = transforms::ast_driver::merge_into_own_imports(&mut body, vec![line]); + return match left_over.first() { + Some(line) => splice_preamble(&body, &format!("{line}\n")), + None => body, + }; + } let preamble = runtime_preamble(config, &helpers); - let (body, _) = apply_transforms_once(src, lazy.edits); if preamble.is_empty() { body } else { @@ -778,6 +826,7 @@ fn verify_target_syntax(source: &str, config: &Config) -> Result<(), TranspileEr // the parser's own wording already names both versions: "Cannot use // `except*` on Python 3.9 (syntax was added in Python 3.11)" Err(TranspileError { + kind: TranspileErrorKind::InvalidSyntax, message: first.to_string(), output_range: Some(first.range), by_range: None, @@ -828,6 +877,25 @@ pub struct TranspileError { pub message: String, pub output_range: Option, pub by_range: Option, + pub kind: TranspileErrorKind, +} + +/// What a transpile failure is, so that it reaches the user through the channel that fits. +/// +/// The three are not interchangeable. A refusal is about the module and the author can act on +/// it; the other two are about the transpiler, and the reader's next step is to report them. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum TranspileErrorKind { + /// the python the transpiler produced does not parse, or needs syntax the target version + /// does not have + #[default] + InvalidSyntax, + /// the module is well-formed basedpython with no correct lowering, and the message says + /// what to change + Refused, + /// the transpiler broke a contract of its own — emitting a call to a helper it never asked + /// for, or leaving surface syntax unlowered + Bug, } impl std::fmt::Display for TranspileError { @@ -842,11 +910,12 @@ impl From for TranspileError { message, output_range: None, by_range: None, + kind: TranspileErrorKind::default(), } } } -fn verify_syntax(source: &str, written: &HashSet) -> Result<(), TranspileError> { +fn verify_syntax(source: &str, author: &AuthorNames) -> Result<(), TranspileError> { use ruff_python_ast::{PySourceType, visitor::Visitor}; let parsed = ruff_python_parser::parse_unchecked_source(source, PySourceType::Python); @@ -861,6 +930,7 @@ fn verify_syntax(source: &str, written: &HashSet) -> Result<(), Transpil // `first.error` is the clean message; the full `Display` would append // "at byte range …" which is meaningless to the user return Err(TranspileError { + kind: TranspileErrorKind::InvalidSyntax, message: format!("transpiler produced invalid Python: {}", first.error), output_range: Some(first.location), by_range: None, @@ -908,6 +978,7 @@ fn verify_syntax(source: &str, written: &HashSet) -> Result<(), Transpil if let Some(range) = scanner.leftover_range { let snippet = &source[usize::from(range.start())..usize::from(range.end())]; return Err(TranspileError { + kind: TranspileErrorKind::Bug, message: format!("transpiler failed to lower anonymous named tuple syntax `{snippet}`"), output_range: Some(range), by_range: None, @@ -916,6 +987,7 @@ fn verify_syntax(source: &str, written: &HashSet) -> Result<(), Transpil if let Some(range) = scanner.leftover_typeof { let snippet = &source[usize::from(range.start())..usize::from(range.end())]; return Err(TranspileError { + kind: TranspileErrorKind::Bug, message: format!("transpiler failed to lower `typeof` syntax `{snippet}`"), output_range: Some(range), by_range: None, @@ -928,17 +1000,243 @@ fn verify_syntax(source: &str, written: &HashSet) -> Result<(), Transpil // apply again — and a `match` that fails them is python that will not parse if let Some(error) = first_invalid_match_statement(parsed.suite()) { return Err(TranspileError { + kind: TranspileErrorKind::InvalidSyntax, message: format!("transpiler produced invalid Python: {error}"), output_range: Some(error.range), by_range: None, }); } - verify_runtime_helpers(parsed.suite(), written) + verify_runtime_helpers(parsed.suite(), &author.reads)?; + verify_no_helper_name_clash(parsed.suite(), author)?; + verify_no_shadowed_helper(parsed.suite(), author) +} + +/// what the author's own source does with the names the lowering also uses: which it reads, and +/// which it binds at module scope +/// +/// Both halves are read before any rewrite, so a name that reaches phase 3 can be told apart from +/// one the transpiler put there. +pub(crate) struct AuthorNames { + /// every name the source reads + reads: HashSet, + /// every name one of the source's own top-level statements binds, how many of them bind it, + /// and where the first one is. the runtime is only ever provided at module scope, so this is + /// the scope where a name of the author's and a name of the lowering's are the same binding + binds_at_module_scope: std::collections::HashMap, + /// each scope of the source's own that takes a helper name over, and how many reads of that + /// name the source itself makes under it — the reads that are the author's to have written + shadows: HelperShadows, +} + +/// what a parse does with the helper names its own scopes take over: where the taking-over +/// scopes are, and how many reads land on them +/// +/// keyed by the helper name alone, not by the scope that binds it. the author's half of the +/// count is read from the `.by` source and the other half from the lowered output, and the +/// two parses have no scope in common to key on: a lowering renames a scope (a class's +/// `private def helper` reaches the output as `__helper`) and writes scopes of its own, so a +/// per-scope key reads the one scope under two names and reports the module for a read it +/// made itself +#[derive(Default)] +struct HelperShadows { + /// helper name → where in the `.by` source the first scope that takes it over was + /// written, and what to call that scope + scopes: std::collections::HashMap, + /// helper name → how many reads one of those bindings, rather than module scope, answers + reads: std::collections::HashMap, +} + +impl HelperShadows { + fn of(suite: &[Stmt]) -> Self { + let found = runtime::shadowing(suite); + let mut scopes: std::collections::HashMap = + std::collections::HashMap::new(); + // `runtime::shadowing` hands the scopes back in source order, so the first entry for + // a name is the scope the author wrote first + for (name, range, scope) in found.scopes { + scopes.entry(name).or_insert((range, scope)); + } + let mut reads: std::collections::HashMap = std::collections::HashMap::new(); + for (name, _) in found.reads { + *reads.entry(name).or_insert(0) += 1; + } + Self { scopes, reads } + } +} + +impl AuthorNames { + fn new(suite: &[Stmt]) -> Self { + let mut binds_at_module_scope: std::collections::HashMap = + std::collections::HashMap::new(); + for stmt in suite { + // the range is the binding's own, so a refusal points at the `x` of `x = 1` + // rather than at the whole statement + for (name, range) in runtime::bindings(stmt) { + binds_at_module_scope + .entry(name) + .and_modify(|(count, _)| *count += 1) + .or_insert((1, range)); + } + } + Self { + reads: names_written(suite), + binds_at_module_scope, + shadows: HelperShadows::of(suite), + } + } + + #[cfg(test)] + fn none() -> Self { + Self { + reads: HashSet::new(), + binds_at_module_scope: std::collections::HashMap::new(), + shadows: HelperShadows::default(), + } + } +} + +/// reject output whose module binds a name the runtime helpers it was given bind +/// +/// the helpers reach a module under fixed names — pasted in as definitions, or +/// imported from the `_by_runtime` beside it — and both renderings put those names +/// in the module's own globals, ahead of everything the author wrote. a module that +/// binds one of them itself therefore takes the helper's name over, and the lowered +/// lines that run after it call the author's value: `_soundness_check = 3` beside a +/// checked call raises `TypeError: 'int' object is not callable`, and +/// `from typing import Optional` beside a `Some(...)` raises `Cannot instantiate +/// typing.Optional`. The lowering has no way to spell the helper other than its +/// name, so the clash is reported rather than miscompiled +fn verify_no_helper_name_clash(suite: &[Stmt], author: &AuthorNames) -> Result<(), TranspileError> { + let mut provided: std::collections::HashMap = std::collections::HashMap::new(); + for stmt in suite { + for name in runtime::bindings(stmt).into_keys() { + if runtime::defines(&name) { + *provided.entry(name).or_insert(0) += 1; + } + } + } + // the author's own top-level bindings are in `suite` too, so a name bound more often there + // than the author bound it is one the preamble also binds — and the module then has two + // bindings of the one name, of which the author's is the later and therefore the live one + let mut clashes: Vec<(&String, TextRange)> = author + .binds_at_module_scope + .iter() + .filter(|(name, (count, _))| provided.get(*name).is_some_and(|provided| provided > count)) + .map(|(name, (_, range))| (name, *range)) + .collect(); + clashes.sort_by(|a, b| a.1.start().cmp(&b.1.start()).then_with(|| a.0.cmp(b.0))); + match clashes.first() { + Some((name, range)) => Err(TranspileError { + kind: TranspileErrorKind::Refused, + message: format!( + "this module binds `{name}`, which is the name a runtime helper its lowering \ + needs is provided under, so the lowered code would read this module's value \ + instead of the helper. Rename it" + ), + output_range: None, + by_range: Some(*range), + }), + None => Ok(()), + } +} + +/// reject output whose lowered code reads a helper through a name one of the module's own +/// scopes has taken over +/// +/// [`verify_no_helper_name_clash`] answers this at module scope, where the author's binding and +/// the preamble's are the same binding. A `def`, `class`, `lambda` or comprehension that binds a +/// helper name instead *shadows* the preamble's, for that scope alone — so a module is only +/// wrong when the lowering emits a read of the name inside such a scope: +/// +/// ```python +/// def go() -> str: +/// _soundness_check = "mine" # the author's own local +/// return _soundness_check(pick(xs, 0), str) # the lowering's read, which finds it +/// ``` +/// +/// A scope that binds the name and reads it *itself* is not that — `from typing import Optional` +/// beside a `return Optional` is a module using its own import — so the author's own reads are +/// subtracted: the refusal is for a read the module's own source does not account for. +/// +/// The subtraction is per helper name across the whole module, not per scope: see +/// [`HelperShadows`] for why the two parses have no scope to key on. So a read the author makes +/// under one scope pays for a read the lowering emits under another, and a module that both +/// reads a helper name itself and has the lowering emit that name elsewhere is let through. +/// Telling the two apart needs the emitted reads marked where they are emitted. +fn verify_no_shadowed_helper(suite: &[Stmt], author: &AuthorNames) -> Result<(), TranspileError> { + let output = HelperShadows::of(suite); + let own = own_shadowed_reads_in(suite); + let mut emitted: Vec<&String> = output + .reads + .iter() + .filter(|(name, count)| { + let accounted = author.shadows.reads.get(*name).copied().unwrap_or(0) + + own.get(name.as_str()).copied().unwrap_or(0); + **count > accounted + }) + .map(|(name, _)| name) + .collect(); + // the message names one scope, so the choice has to be stable: the one the author wrote + // first, and failing a scope of the author's at all, the name itself + emitted.sort_by_key(|name| { + ( + author + .shadows + .scopes + .get(*name) + .map_or(u32::MAX, |(range, _)| range.start().into()), + (*name).clone(), + ) + }); + let Some(name) = emitted.first() else { + return Ok(()); + }; + let written = author.shadows.scopes.get(*name); + // the span is the binding's own rather than the scope's, which is the only thing to + // point at when the scope has no name — a comprehension or a lambda gives an author + // nothing to search for, and the line the name is bound on does + let range = written.map(|(range, _)| *range); + let binder = match written { + Some((_, scope)) => format!("{scope} binds"), + None => "this module binds".to_owned(), + }; + Err(TranspileError { + kind: TranspileErrorKind::Refused, + message: format!( + "{binder} `{name}`, which is the name a runtime helper its lowering needs is \ + provided under, so the lowered code in that scope would read this binding \ + instead of the helper. Rename it" + ), + output_range: None, + by_range: range, + }) +} + +/// how many shadowed reads of each helper name the runtime definitions this output was *given* +/// make on their own behalf +/// +/// some of the runtime shadows on purpose: `_by_character_class` builds a local `class +/// Character` and hands it back, and those reads say nothing about the module it was written +/// beside. they only reach the output when the definition that makes them was pasted into it, +/// so a build that imports the helper from `_by_runtime` instead gets no allowance for them +fn own_shadowed_reads_in(suite: &[Stmt]) -> std::collections::HashMap<&'static str, usize> { + let bound: HashSet = suite + .iter() + .flat_map(|stmt| runtime::bindings(stmt).into_keys()) + .collect(); + let mut counts: std::collections::HashMap<&'static str, usize> = + std::collections::HashMap::new(); + for ((name, scope), count) in runtime::own_shadowed_reads() { + if bound.contains(scope) { + *counts.entry(name.as_str()).or_insert(0) += count; + } + } + counts } /// every name the author's own source reads -fn names_written(suite: &[Stmt]) -> HashSet { +pub(crate) fn names_written(suite: &[Stmt]) -> HashSet { use ruff_python_ast::visitor::source_order::{SourceOrderVisitor, walk_expr, walk_stmt}; struct Reads(HashSet); @@ -989,7 +1287,10 @@ fn verify_runtime_helpers(suite: &[Stmt], written: &HashSet) -> Result<( } } - let provided: HashSet = suite.iter().flat_map(runtime::bindings).collect(); + let provided: HashSet = suite + .iter() + .flat_map(|stmt| runtime::bindings(stmt).into_keys()) + .collect(); let mut emitted = Emitted { written, reads: Vec::new(), @@ -1003,6 +1304,7 @@ fn verify_runtime_helpers(suite: &[Stmt], written: &HashSet) -> Result<( .find(|(name, _)| !provided.contains(name)) { Some((name, range)) => Err(TranspileError { + kind: TranspileErrorKind::Bug, message: format!( "transpiler emitted a call to the runtime helper `{name}` without asking for \ it, so the module it produced does not define it" @@ -1112,9 +1414,18 @@ fn run_lowering_phase(source: &str, stmts: &[Stmt], config: &Config) -> Lowering let needs_lazy_annotations = config.inject_future_annotations || config.min_version < PythonVersion::PY310; if needs_lazy_annotations && !config.is_stub && !has_future_annotations(stmts) { + // a string is the module's `__doc__` only while it is the first statement, + // so the import goes after it rather than over it — a module whose + // annotations had to be deferred used to read back no docstring at all. + // every later phase splices at the same place, and the line map is built + // for it: see `run_...` phases 2 and 2c, and `kept` in `transpile_inner` + let at = transforms::source_util::docstring_end(source); + output.push_str(&source[..at]); output.push_str("from __future__ import annotations\n"); + output.push_str(&source[at..]); + } else { + output.push_str(source); } - output.push_str(source); LoweringResult { output, @@ -1122,6 +1433,55 @@ fn run_lowering_phase(source: &str, stmts: &[Stmt], config: &Config) -> Lowering } } +#[cfg(test)] +mod lowering_preamble { + use super::{Config, PythonVersion, transpile}; + + /// the transpiled form of `source` on a target whose runtime cannot evaluate + /// `X | Y` annotations, so the deferred-annotations import is emitted + fn with_future_import(source: &str) -> String { + let out = transpile( + source, + &Config { + min_version: PythonVersion::PY39, + ..Config::test_default() + }, + ) + .unwrap_or_else(|error| panic!("{source:?}: {error}")); + assert!( + out.contains("from __future__ import annotations"), + "expected the deferred-annotations import:\n{out}" + ); + out + } + + /// a string is the module's `__doc__` only while it is the first statement, so + /// the deferred-annotations import goes after it. a module built from a `.by` + /// file whose annotations had to be deferred used to read back no docstring + #[test] + fn the_future_import_goes_after_the_module_docstring() { + let out = with_future_import("\"\"\"what this module is for.\"\"\"\n\nx: int = 1\n"); + assert!( + out.starts_with("\"\"\"what this module is for.\"\"\"\n"), + "the docstring is no longer first:\n{out}" + ); + } + + /// a module with no docstring keeps the import at the very top, where a first + /// statement that only looks like one — an expression, an assignment — must not + /// be mistaken for it + #[test] + fn a_module_without_a_docstring_keeps_the_import_first() { + for source in ["x: int = 1\n", "print(\"not a docstring\")\n"] { + let out = with_future_import(source); + assert!( + out.starts_with("from __future__ import annotations\n"), + "{source:?} gave:\n{out}" + ); + } + } +} + /// The `T | None` → `T?` rewrite of [`reverse_transpile`], on its own, as /// `(range, replacement)` text edits over `source`. /// @@ -1314,6 +1674,27 @@ pub fn reverse_transpile(source: &str, config: &Config) -> Result Result<(), String> { - let written: HashSet = written.iter().map(|name| (*name).to_owned()).collect(); - verify_syntax(output, &written).map_err(|error| error.message) + let mut author = AuthorNames::none(); + author.reads = written.iter().map(|name| (*name).to_owned()).collect(); + verify_syntax(output, &author).map_err(|error| error.message) } #[test] @@ -2548,4 +2927,383 @@ mod runtime_helper_check { ) .unwrap(); } + + /// the unit-test config turns the soundness checks off, so a case about them + /// asks for the default set + fn checked_config() -> Config { + Config { + soundness: crate::config::SoundnessPositions::defaults(), + ..Config::test_default() + } + } + + fn clash(source: &str, config: &Config) -> String { + transpile(source, config).expect_err("binding a helper's name should be reported") + } + + /// the helper reaches the module under its own name and the module's own + /// binding takes it over, so every lowered line below reads the module's value + #[test] + fn a_module_that_binds_a_helper_name_is_reported() { + let error = clash( + concat!( + "def pick[T](items: list[T], i: int) -> T:\n", + " return items[i]\n", + "\n", + "xs: list[str] = [\"a\"]\n", + "_soundness_check = 3\n", + "print(pick(xs, 0))\n", + ), + &checked_config(), + ); + assert!(error.contains("`_soundness_check`"), "{error}"); + } + + /// `Optional` is the wrapper `Some(...)` builds, and `from typing import + /// Optional` is how a program most easily takes that name over + #[test] + fn a_typing_import_that_takes_the_wrapper_name_is_reported() { + let error = clash( + "from typing import Optional\n\nx = Some(3)\nprint(x!)\n", + &Config::test_default(), + ); + assert!(error.contains("`Optional`"), "{error}"); + } + + /// the module's own binding need not be the one it wrote last: whatever the module binds at + /// module scope runs after the preamble, so it is the live one from there on + #[test] + fn an_import_that_takes_a_helper_name_is_reported() { + let error = clash( + concat!( + "from string import capwords as _soundness_check\n", + "\n", + "def pick[T](items: list[T], i: int) -> T:\n", + " return items[i]\n", + "\n", + "xs: list[str] = [\"a\"]\n", + "print(pick(xs, 0))\n", + ), + &checked_config(), + ); + assert!(error.contains("`_soundness_check`"), "{error}"); + } + + /// a binding in a nested scope shadows the helper for that scope alone, so it is reported + /// only when the lowering emits a read of the name inside it — here the checked return + /// value, which lands one line below the author's own local + #[test] + fn a_binding_a_lowered_line_reads_in_the_same_function_is_reported() { + let error = clash( + concat!( + "def pick[T](items: list[T], i: int) -> T:\n", + " return items[i]\n", + "\n", + "def go() -> str:\n", + " _soundness_check = \"mine\"\n", + " xs: list[str] = [\"a\"]\n", + " return pick(xs, 0)\n", + ), + &checked_config(), + ); + assert!(error.contains("`_soundness_check`"), "{error}"); + assert!(error.contains("`go` binds"), "{error}"); + } + + /// a comprehension is a scope of its own, so a target of its that spells a helper name + /// takes the name over for the element the lowering emits into. this transpiled to + /// `[_soundness_check(pick(xs, 0), str) for _soundness_check in range(1)]`, which raises + /// `TypeError: 'int' object is not callable` + #[test] + fn a_comprehension_target_a_lowered_element_reads_is_reported() { + let error = clash( + concat!( + "def pick[T](items: list[T], i: int) -> T:\n", + " return items[i]\n", + "\n", + "xs: list[str] = [\"a\"]\n", + "ys = [pick(xs, 0) for _soundness_check in range(1)]\n", + ), + &checked_config(), + ); + assert!(error.contains("`_soundness_check`"), "{error}"); + // python calls that scope ``, which names nothing the author wrote, so + // the refusal describes it instead and points at the binding's own line + assert!(error.contains("a list comprehension binds"), "{error}"); + } + + /// the other scopes an author cannot name: a lambda, a generator expression, a set + /// comprehension and a dict comprehension. each is described rather than called by + /// python's own `` / `` name for it + #[test] + fn a_scope_with_no_name_of_its_own_is_described() { + let preamble = concat!( + "def pick[T](items: list[T], i: int) -> T:\n", + " return items[i]\n", + "\n", + "xs: list[str] = [\"a\"]\n", + ); + for (tail, described) in [ + ( + "f = lambda _soundness_check: pick(xs, 0)\n", + "a lambda binds", + ), + ( + "ys = (pick(xs, 0) for _soundness_check in range(1))\n", + "a generator expression binds", + ), + ( + "ys = {pick(xs, 0) for _soundness_check in range(1)}\n", + "a set comprehension binds", + ), + ( + "ys = {pick(xs, 0): 1 for _soundness_check in range(1)}\n", + "a dict comprehension binds", + ), + ] { + let error = clash(&format!("{preamble}{tail}"), &checked_config()); + assert!(error.contains(described), "{tail}: {error}"); + assert!(error.contains("`_soundness_check`"), "{tail}: {error}"); + } + } + + /// the first iterable of a comprehension is evaluated where the comprehension is + /// written, before its scope exists, so a lowering that emits into that iterable reads + /// the helper and not the target + #[test] + fn a_comprehension_target_does_not_cover_its_first_iterable() { + transpile( + concat!( + "def pick[T](items: list[T], i: int) -> list[T]:\n", + " return items\n", + "\n", + "xs: list[str] = [\"a\"]\n", + "ys = [xs for _soundness_check in pick(xs, 0)]\n", + ), + &checked_config(), + ) + .unwrap(); + } + + /// and a comprehension whose target reads its own name is the module's own business, + /// exactly as a function's local is + #[test] + fn a_comprehension_target_the_module_reads_itself_is_left_alone() { + transpile( + "ys = [Optional for Optional in range(1)]\nprint(Some(3)!)\n", + &Config::test_default(), + ) + .unwrap(); + } + + /// a `:=` binds like any other assignment, wherever the expression around it is + /// evaluated. this transpiled to `_soundness_check(pick(xs, 0), str)` two lines below + /// `if (_soundness_check := 3)`, which raises `TypeError: 'int' object is not callable` + #[test] + fn a_walrus_that_takes_a_helper_name_is_reported() { + for source in [ + concat!( + "def pick[T](items: list[T], i: int) -> T:\n", + " return items[i]\n", + "\n", + "if (_soundness_check := 3):\n", + " pass\n", + "xs: list[str] = [\"a\"]\n", + "print(pick(xs, 0))\n", + ), + // and inside a function, where it shadows for that scope alone + concat!( + "def pick[T](items: list[T], i: int) -> T:\n", + " return items[i]\n", + "\n", + "def go() -> str:\n", + " xs: list[str] = [\"a\"]\n", + " if (_soundness_check := 3):\n", + " pass\n", + " return pick(xs, 0)\n", + ), + ] { + let error = clash(source, &checked_config()); + assert!(error.contains("`_soundness_check`"), "{error}"); + } + } + + /// a `case` binds too, through the names its pattern captures and through its body + #[test] + fn a_match_case_that_takes_a_helper_name_is_reported() { + for source in [ + concat!( + "def pick[T](items: list[T], i: int) -> T:\n", + " return items[i]\n", + "\n", + "match 1:\n", + " case _soundness_check:\n", + " pass\n", + "xs: list[str] = [\"a\"]\n", + "print(pick(xs, 0))\n", + ), + concat!( + "def pick[T](items: list[T], i: int) -> T:\n", + " return items[i]\n", + "\n", + "match 1:\n", + " case 1:\n", + " _soundness_check = 3\n", + "xs: list[str] = [\"a\"]\n", + "print(pick(xs, 0))\n", + ), + ] { + let error = clash(source, &checked_config()); + assert!(error.contains("`_soundness_check`"), "{error}"); + } + } + + /// every basedpython binding form binds too: a destructuring `let`, the pattern of an + /// `if let` clause, the pattern of a destructuring `for`, and the statement a statement + /// expression holds. each of these transpiled to `_soundness_check(pick(xs, 0), str)` + /// beside a module that had taken the name over, which raises + /// `TypeError: 'tuple' object is not callable` the first time the lowered line runs + /// + /// they were each found one at a time, which is why [`runtime::bindings`] now answers for + /// every statement by name rather than defaulting the ones nobody thought of + #[test] + fn a_basedpython_binding_form_that_takes_a_helper_name_is_reported() { + let preamble = concat!( + "def pick[T](items: list[T], i: int) -> T:\n", + " return items[i]\n", + "\n", + "xs: list[str] = [\"a\"]\n", + ); + for tail in [ + // a destructuring `let` + "let (_soundness_check, b) := (1, 2)\nprint(pick(xs, 0))\n", + // the pattern of an `if let` clause + "if let (_soundness_check, b) := (1, 2):\n pass\nprint(pick(xs, 0))\n", + // the pattern of a destructuring `for` + "for (_soundness_check, b) in [(1, 2)]:\n pass\nprint(pick(xs, 0))\n", + // the statement a statement expression holds + "y = match 1:\n case _soundness_check:\n 1\nprint(pick(xs, 0))\n", + ] { + let error = clash(&format!("{preamble}{tail}"), &checked_config()); + assert!(error.contains("`_soundness_check`"), "{tail}: {error}"); + } + } + + /// the same binding in a function the lowering emits nothing into is the module's own + /// business, and is left alone + #[test] + fn a_binding_no_lowered_line_reads_is_left_alone() { + transpile( + concat!( + "def go() -> str:\n", + " _soundness_check = \"mine\"\n", + " return _soundness_check\n", + ), + &checked_config(), + ) + .unwrap(); + } + + /// and an import written inside a function body is that function's own to read: the module + /// accounts for the read itself, so nothing the lowering did lands on it + #[test] + fn a_nested_import_of_a_helper_name_the_module_reads_itself_is_left_alone() { + transpile( + concat!( + "def f() -> object:\n", + " from typing import Optional\n", + " return Optional\n", + "\n", + "x = Some(3)\n", + "print(x!)\n", + ), + &Config::test_default(), + ) + .unwrap(); + } + + /// but a lowering that emits into that same function reads the import, not the wrapper: + /// `Some(...)` beside the author's own `Optional` is one read more than the module made + #[test] + fn a_nested_import_a_lowered_line_reads_is_reported() { + let error = clash( + concat!( + "def f() -> object:\n", + " from typing import Optional\n", + " x = Some(3)\n", + " return Optional\n", + ), + &Config::test_default(), + ); + assert!(error.contains("`Optional`"), "{error}"); + assert!(error.contains("`f` binds"), "{error}"); + } + + /// a scope the lowering renames is the same scope: a class's `private def helper` reaches + /// the output as `__helper`, and the import it reads is still the module's own. counting the + /// author's reads per scope reported this correct program, because the `.by` source and + /// the output spelled the one scope differently + #[test] + fn a_scope_the_lowering_renames_is_still_the_module_reading_its_own() { + transpile( + concat!( + "class C:\n", + " private def helper(self) -> object:\n", + " from typing import Optional\n", + " return Optional\n", + "\n", + "x = Some(3)\n", + "print(x!)\n", + ), + &Config::test_default(), + ) + .unwrap(); + } + + /// the reads are counted per helper name across the module, so a read the lowering emits + /// under one scope is reported even where the author reads the same name under another + #[test] + fn a_lowered_read_under_another_scope_is_still_reported() { + let error = clash( + concat!( + "def a() -> object:\n", + " from typing import Optional\n", + " return Optional\n", + "\n", + "def b() -> object:\n", + " from typing import Optional\n", + " return Some(3)\n", + ), + &Config::test_default(), + ); + assert!(error.contains("`Optional`"), "{error}"); + assert!(error.contains("`a` binds"), "{error}"); + } + + /// nothing is reported where the lowering never asked for the helper: the + /// name is then the module's alone + #[test] + fn a_helper_name_no_lowering_needs_is_left_alone() { + transpile( + "from typing import Optional\n\nx: Optional[int] = 1\nprint(x)\n", + &Config::test_default(), + ) + .unwrap(); + } + + /// and nothing is reported for a module that only reads the name it was given + #[test] + fn reading_a_helper_the_lowering_provided_is_not_a_clash() { + transpile( + concat!( + "def pick[T](items: list[T], i: int) -> T:\n", + " return items[i]\n", + "\n", + "xs: list[str] = [\"a\"]\n", + "print(pick(xs, 0))\n", + ), + &checked_config(), + ) + .unwrap(); + } } diff --git a/crates/by_transforms/src/runtime.rs b/crates/by_transforms/src/runtime.rs index 9679d65eca..962b648467 100644 --- a/crates/by_transforms/src/runtime.rs +++ b/crates/by_transforms/src/runtime.rs @@ -33,7 +33,7 @@ use std::sync::OnceLock; use ruff_python_ast::visitor::source_order::{SourceOrderVisitor, walk_expr}; use ruff_python_ast::{self as ast, ExceptHandler, Expr, Stmt}; use ruff_python_parser::parse_module; -use ruff_text_size::Ranged; +use ruff_text_size::{Ranged, TextRange}; /// the runtime, as the file a build writes out pub const SOURCE: &str = include_str!("runtime/_by_runtime.py"); @@ -87,6 +87,7 @@ helpers! { WITNESS_GET = "_by_witness_get", TEMPLATE = "_Template", INTERPOLATION = "_Interpolation", + TEMPLATE_TEXT = "_by_template", GRAPHEMES = "_by_graphemes", PREFIX = "_by_prefix", SUFFIX = "_by_suffix", @@ -140,7 +141,10 @@ fn build_index() -> BTreeMap { let suite = parsed.suite(); // two passes: the first learns every name the file defines, so the second can // tell a call to a sibling from a call to a builtin - let defined: BTreeSet = suite.iter().flat_map(bindings).collect(); + let defined: BTreeSet = suite + .iter() + .flat_map(|stmt| bindings(stmt).into_keys()) + .collect(); let mut definitions = BTreeMap::new(); let mut order = 0usize; @@ -165,13 +169,13 @@ fn build_index() -> BTreeMap { let mut needs: Vec = suite[at..=last] .iter() .flat_map(reads) - .filter(|read| defined.contains(read) && !names.contains(read)) + .filter(|read| defined.contains(read) && !names.contains_key(read)) .collect(); needs.sort(); needs.dedup(); // `every_name_has_one_definition` keeps a name from being bound twice, // so first-wins here never decides anything - for name in names { + for name in names.into_keys() { definitions.entry(name).or_insert_with(|| Definition { source: source.clone(), order, @@ -184,40 +188,52 @@ fn build_index() -> BTreeMap { definitions } +/// the names a statement binds, each with the range of the binding itself +/// +/// the range is what a refusal points at, so it names the `x` of `x = 1` rather than the +/// whole statement +pub(crate) type Bindings = BTreeMap; + +/// record `name` as bound at `range`, keeping the first of several bindings of one name — +/// which is the one an author reading the refusal will find first +fn record(names: &mut Bindings, name: impl Into, range: TextRange) { + names.entry(name.into()).or_insert(range); +} + /// the names a top-level statement binds at module scope -pub(crate) fn bindings(stmt: &Stmt) -> BTreeSet { - let mut names = BTreeSet::new(); +pub(crate) fn bindings(stmt: &Stmt) -> Bindings { + let mut names = Bindings::new(); bind(stmt, &mut names); names } -/// a block that runs at module scope (an `if`, a `try`, a loop) binds what the +/// a block that runs at module scope (an `if`, a `try`, a loop, a `case`) binds what the /// statements inside it bind. a function or class body is a scope of its own -fn bind(stmt: &Stmt, names: &mut BTreeSet) { - let block = |body: &[Stmt], names: &mut BTreeSet| { +/// +/// every statement is answered for by name, with no `_` arm, because the misses in this +/// file have all been of one kind: a binding form nobody thought of. a `match` case, a +/// walrus, a destructuring `let`, an `if let` clause and the statement a statement +/// expression holds were each found on their own, after a module that spelled a runtime +/// helper's name with one of them was lowered into a call on its own value. with the arms +/// spelled out, a statement added to the syntax tree stops this file compiling until +/// somebody says what it binds +fn bind(stmt: &Stmt, names: &mut Bindings) { + let block = |body: &[Stmt], names: &mut Bindings| { for stmt in body { bind(stmt, names); } }; match stmt { - Stmt::FunctionDef(def) => { - names.insert(def.name.to_string()); - } - Stmt::ClassDef(def) => { - names.insert(def.name.to_string()); - } + Stmt::FunctionDef(def) => record(names, def.name.as_str(), def.name.range()), + Stmt::ClassDef(def) => record(names, def.name.as_str(), def.name.range()), Stmt::Assign(assign) => { for target in &assign.targets { - if let Expr::Name(name) = target { - names.insert(name.id.to_string()); - } - } - } - Stmt::AnnAssign(assign) => { - if let Expr::Name(name) = assign.target.as_ref() { - names.insert(name.id.to_string()); + bind_target(target, names); } } + Stmt::AnnAssign(assign) => bind_target(&assign.target, names), + Stmt::AugAssign(assign) => bind_target(&assign.target, names), + Stmt::TypeAlias(alias) => bind_target(&alias.name, names), Stmt::Import(import) => { for alias in &import.names { let bound = match &alias.asname { @@ -225,33 +241,64 @@ fn bind(stmt: &Stmt, names: &mut BTreeSet) { // `import a.b` binds `a` None => alias.name.split('.').next().unwrap_or_default().to_string(), }; - names.insert(bound); + record(names, bound, alias.range()); } } Stmt::ImportFrom(import) => { for alias in &import.names { - names.insert(match &alias.asname { + let bound = match &alias.asname { Some(asname) => asname.to_string(), None => alias.name.to_string(), - }); + }; + record(names, bound, alias.range()); } } Stmt::If(node) => { + // basedpython `if let := :` binds what the pattern captures, + // for the clause it heads + if let Some(pattern) = &node.pattern { + bind_pattern(pattern, names); + } block(&node.body, names); for clause in &node.elif_else_clauses { + if let Some(pattern) = &clause.pattern { + bind_pattern(pattern, names); + } block(&clause.body, names); } } + // basedpython `let := `, whose `else` block runs where the + // pattern did not match + Stmt::Let(node) => { + bind_pattern(&node.pattern, names); + block(&node.orelse, names); + } Stmt::Try(node) => { block(&node.body, names); for ExceptHandler::ExceptHandler(handler) in &node.handlers { + if let Some(bound) = &handler.name { + record(names, bound.as_str(), bound.range()); + } block(&handler.body, names); } block(&node.orelse, names); block(&node.finalbody, names); } - Stmt::With(node) => block(&node.body, names), + Stmt::With(node) => { + for item in &node.items { + if let Some(target) = &item.optional_vars { + bind_target(target, names); + } + } + block(&node.body, names); + } Stmt::For(node) => { + bind_target(&node.target, names); + // basedpython `for in :`, where `target` above is the binder the + // pattern destructures + if let Some(pattern) = &node.pattern { + bind_pattern(pattern, names); + } block(&node.body, names); block(&node.orelse, names); } @@ -259,10 +306,641 @@ fn bind(stmt: &Stmt, names: &mut BTreeSet) { block(&node.body, names); block(&node.orelse, names); } - _ => {} + Stmt::Match(node) => { + for case in &node.cases { + bind_pattern(&case.pattern, names); + block(&case.body, names); + } + } + // these bind nothing of their own. `global` and `nonlocal` name a binding made + // elsewhere, and `del` removes one rather than making it — a module that deletes a + // helper's name has taken it away from the lowered lines below, which is the + // deleted-author-read hole the shadowing check documents rather than answers + Stmt::Return(_) + | Stmt::Delete(_) + | Stmt::Raise(_) + | Stmt::Assert(_) + | Stmt::Global(_) + | Stmt::Nonlocal(_) + | Stmt::Expr(_) + | Stmt::Pass(_) + | Stmt::Break(_) + | Stmt::Continue(_) + | Stmt::IpyEscapeCommand(_) => {} + } + bind_in_own_expressions(stmt, names); +} + +/// the names a `case` pattern binds: its captures, its `as` names and the `**rest` of a +/// mapping pattern +fn bind_pattern(pattern: &ast::Pattern, names: &mut Bindings) { + struct Captures<'a>(&'a mut Bindings); + impl<'b> SourceOrderVisitor<'b> for Captures<'_> { + fn visit_pattern(&mut self, pattern: &'b ast::Pattern) { + match pattern { + ast::Pattern::MatchAs(node) => { + if let Some(name) = &node.name { + record(self.0, name.as_str(), name.range()); + } + } + ast::Pattern::MatchStar(node) => { + if let Some(name) = &node.name { + record(self.0, name.as_str(), name.range()); + } + } + ast::Pattern::MatchMapping(node) => { + if let Some(rest) = &node.rest { + record(self.0, rest.as_str(), rest.range()); + } + } + // the rest hold sub-patterns and nothing of their own; the walk below + // reaches every capture in them + ast::Pattern::MatchValue(_) + | ast::Pattern::MatchSingleton(_) + | ast::Pattern::MatchSequence(_) + | ast::Pattern::MatchClass(_) + | ast::Pattern::MatchOr(_) + | ast::Pattern::MatchAnd(_) => {} + } + ruff_python_ast::visitor::source_order::walk_pattern(self, pattern); + } + } + Captures(names).visit_pattern(pattern); +} + +/// the names a statement's own expressions bind: a `:=`, and the statement a basedpython +/// statement expression holds +/// +/// a walrus binds where the expression around it is evaluated, which is the scope the +/// statement is written in — a comprehension's own scope is deliberately escaped, so +/// `[y for x in xs if (n := f(x))]` binds `n` beside the comprehension. the one exception is a +/// lambda body, which is a scope of its own +/// +/// a statement expression (`a = match x: ...`) holds a statement that runs where the +/// expression is written, so what it binds is bound here too +fn bind_in_own_expressions(stmt: &Stmt, names: &mut Bindings) { + struct Own<'a>(&'a mut Bindings); + impl<'b> SourceOrderVisitor<'b> for Own<'_> { + fn visit_expr(&mut self, expr: &'b Expr) { + match expr { + Expr::Named(named) => bind_target(&named.target, self.0), + Expr::Statement(held) => { + bind(&held.stmt, self.0); + return; + } + // a lambda body is a scope of its own, so a walrus in it binds there + Expr::Lambda(_) => return, + _ => {} + } + walk_expr(self, expr); + } + // the suites this statement opens are walked as statements by `bind` itself + fn visit_stmt(&mut self, _: &'b Stmt) {} + } + ruff_python_ast::visitor::source_order::walk_stmt(&mut Own(names), stmt); +} + +/// the names an assignment target binds, unpacking a tuple or list target +/// +/// spelled out over every expression, for the reason [`bind`] is +fn bind_target(target: &Expr, names: &mut Bindings) { + match target { + Expr::Name(name) => record(names, name.id.as_str(), name.range()), + Expr::Tuple(tuple) => { + for element in &tuple.elts { + bind_target(element, names); + } + } + Expr::List(list) => { + for element in &list.elts { + bind_target(element, names); + } + } + Expr::Starred(starred) => bind_target(&starred.value, names), + // an attribute or a subscript target writes through a value rather than binding a + // name of its own, and nothing else is a target python accepts — a parenthesised + // target reaches here as what it parenthesises + Expr::Attribute(_) + | Expr::Subscript(_) + | Expr::BoolOp(_) + | Expr::Named(_) + | Expr::BinOp(_) + | Expr::UnaryOp(_) + | Expr::Lambda(_) + | Expr::If(_) + | Expr::Dict(_) + | Expr::Set(_) + | Expr::ListComp(_) + | Expr::SetComp(_) + | Expr::DictComp(_) + | Expr::Generator(_) + | Expr::Await(_) + | Expr::Yield(_) + | Expr::YieldFrom(_) + | Expr::Compare(_) + | Expr::Call(_) + | Expr::FString(_) + | Expr::TString(_) + | Expr::StringLiteral(_) + | Expr::BytesLiteral(_) + | Expr::NumberLiteral(_) + | Expr::BooleanLiteral(_) + | Expr::NoneLiteral(_) + | Expr::EllipsisLiteral(_) + | Expr::Slice(_) + | Expr::IpyEscapeCommand(_) + | Expr::CallableType(_) + | Expr::ProtocolType(_) + | Expr::ProtocolMethod(_) + | Expr::Statement(_) => {} + } +} + +/// what a module does with a helper's name inside a scope of its own +/// +/// the helpers only ever reach a module at module scope, so a `def`, `class`, +/// `lambda` or comprehension that binds one of their names takes it over for +/// everything inside it. this is what such a scope looks like, and what is read +/// under it +pub(crate) struct Shadowing { + /// each scope that binds a helper name, as `(the name, the range of the binding itself, + /// how to describe the scope)`, in source order + /// + /// the range is the binding's rather than the scope's, because a scope with no name of + /// its own — a lambda, a comprehension — gives an author nothing to search for, and the + /// line the name is bound on does + pub(crate) scopes: Vec<(String, TextRange, String)>, + /// each load of a helper name that one of those bindings, rather than module scope, answers, + /// as `(the name, the scope whose binding answers it)` + pub(crate) reads: Vec<(String, String)>, +} + +/// how many reads of each helper name the runtime's own definitions make under a scope of their +/// own, by `(helper name, scope)` +/// +/// the preamble is part of the module phase 3 reads, and some of it shadows on purpose: +/// `_by_character_class` builds a local `class Character` and hands it back. Those reads are +/// the transpiler's own and say nothing about the module it was given +/// +/// the scope is named so a caller can tell whether the definition that makes the read is in +/// the output at all: the runtime reaches a module either pasted in or imported, and only the +/// pasted rendering brings these reads with it +pub(crate) fn own_shadowed_reads() -> &'static BTreeMap<(String, String), usize> { + static OWN: OnceLock> = OnceLock::new(); + OWN.get_or_init(|| { + let Ok(parsed) = parse_module(SOURCE) else { + return BTreeMap::new(); + }; + let mut counts = BTreeMap::new(); + for (name, scope) in shadowing(parsed.suite()).reads { + *counts.entry((name, scope)).or_insert(0) += 1; + } + counts + }) +} + +/// one scope on the walk, holding only the helper names it binds +struct Frame { + is_class: bool, + locals: Bindings, + /// python's own name for the scope — a `def`'s or `class`'s own, or `` / + /// `` / `` / `` / ``. it is only ever a key: the + /// author's parse and the lowered output are asked about the same scope under it + key: String, + /// how a refusal names the scope: ``​`go`​`` for a scope the author named, and "a list + /// comprehension" for one they did not. python's `` names nothing an author + /// wrote, so it is never shown + described: String, +} + +/// find every helper name a scope of the module's own takes over, and every read +/// that lands on one +/// +/// module scope is not a frame here: a name bound there is the one the preamble +/// also binds, which `verify_no_helper_name_clash` answers instead +pub(crate) fn shadowing(suite: &[Stmt]) -> Shadowing { + let mut walk = Walk { + stack: Vec::new(), + found: Shadowing { + scopes: Vec::new(), + reads: Vec::new(), + }, + }; + walk.body(suite); + walk.found + .scopes + .sort_by_key(|(name, range, _)| (range.start(), name.clone())); + walk.found +} + +struct Walk { + stack: Vec, + found: Shadowing, +} + +impl Walk { + /// the innermost scope a load of `name` resolves to, or `None` for module scope. + /// a class body's names are invisible to the scopes nested in it, so a class frame + /// only answers a read written directly in it + fn resolves_to(&self, name: &str) -> Option<&Frame> { + self.stack + .iter() + .rev() + .enumerate() + .filter(|(depth, frame)| !frame.is_class || *depth == 0) + .map(|(_, frame)| frame) + .find(|frame| frame.locals.contains_key(name)) + } + + fn enter(&mut self, frame: Frame, body: impl FnOnce(&mut Self)) { + for (name, range) in &frame.locals { + self.found + .scopes + .push((name.clone(), *range, frame.described.clone())); + } + self.stack.push(frame); + body(self); + self.stack.pop(); + } + + fn body(&mut self, body: &[Stmt]) { + for stmt in body { + self.stmt(stmt); + } + } + + fn stmt(&mut self, stmt: &Stmt) { + match stmt { + Stmt::FunctionDef(def) => { + // a decorator, a default and an annotation are all evaluated where the + // `def` is written, not inside it + for decorator in &def.decorator_list { + self.expr(&decorator.expression); + } + for parameter in &*def.parameters { + if let Some(annotation) = parameter.annotation() { + self.expr(annotation); + } + if let Some(default) = parameter.default() { + self.expr(default); + } + } + if let Some(returns) = def.returns.as_deref() { + self.expr(returns); + } + let mut locals = Bindings::new(); + for parameter in &*def.parameters { + let bound = parameter.name(); + if defines(bound.as_str()) { + record(&mut locals, bound.as_str(), bound.range()); + } + } + collect(&def.body, &mut locals); + self.enter( + Frame { + is_class: false, + locals, + key: def.name.to_string(), + described: format!("`{}`", def.name), + }, + |walk| walk.body(&def.body), + ); + } + Stmt::ClassDef(def) => { + for decorator in &def.decorator_list { + self.expr(&decorator.expression); + } + if let Some(arguments) = def.arguments.as_deref() { + for arg in &*arguments.args { + self.expr(arg); + } + for keyword in &*arguments.keywords { + self.expr(&keyword.value); + } + } + let mut locals = Bindings::new(); + collect(&def.body, &mut locals); + self.enter( + Frame { + is_class: true, + locals, + key: def.name.to_string(), + described: format!("`{}`", def.name), + }, + |walk| walk.body(&def.body), + ); + } + _ => { + let mut nested: Vec<&[Stmt]> = Vec::new(); + push_blocks(stmt, &mut nested); + for expr in statement_expressions(stmt) { + self.expr(expr); + } + for block in nested { + self.body(block); + } + } + } + } + + fn expr(&mut self, expr: &Expr) { + match expr { + Expr::Name(ast::ExprName { id, ctx, .. }) if ctx.is_load() => { + if defines(id.as_str()) + && let Some(frame) = self.resolves_to(id.as_str()) + { + let scope = frame.key.clone(); + self.found.reads.push((id.to_string(), scope)); + } + } + Expr::Lambda(lambda) => { + let mut locals = Bindings::new(); + if let Some(parameters) = lambda.parameters.as_deref() { + for parameter in parameters { + if let Some(default) = parameter.default() { + self.expr(default); + } + let bound = parameter.name(); + if defines(bound.as_str()) { + record(&mut locals, bound.as_str(), bound.range()); + } + } + } + self.enter( + Frame { + is_class: false, + locals, + key: "".to_owned(), + described: "a lambda".to_owned(), + }, + |walk| walk.expr(&lambda.body), + ); + return; + } + Expr::ListComp(comp) => { + self.comprehension( + "", + "a list comprehension", + &comp.generators, + |walk| { + walk.expr(&comp.elt); + }, + ); + return; + } + Expr::SetComp(comp) => { + self.comprehension( + "", + "a set comprehension", + &comp.generators, + |walk| { + walk.expr(&comp.elt); + }, + ); + return; + } + Expr::Generator(comp) => { + self.comprehension( + "", + "a generator expression", + &comp.generators, + |walk| { + walk.expr(&comp.elt); + }, + ); + return; + } + Expr::DictComp(comp) => { + self.comprehension( + "", + "a dict comprehension", + &comp.generators, + |walk| { + if let Some(key) = comp.key.as_deref() { + walk.expr(key); + } + walk.expr(&comp.value); + }, + ); + return; + } + // a basedpython statement expression holds a statement that runs where the + // expression is written, and it may open a scope of its own (a `def` in a + // `match` arm), so it goes back through the statement walk + Expr::Statement(held) => { + self.stmt(&held.stmt); + return; + } + // nothing else opens a scope or reads a name of its own; the walk below reaches + // the sub-expressions. spelled out for the reason [`bind`] is — a scope-opening + // expression added to the syntax tree has to be decided about here + Expr::Name(_) + | Expr::BoolOp(_) + | Expr::Named(_) + | Expr::BinOp(_) + | Expr::UnaryOp(_) + | Expr::If(_) + | Expr::Dict(_) + | Expr::Set(_) + | Expr::Await(_) + | Expr::Yield(_) + | Expr::YieldFrom(_) + | Expr::Compare(_) + | Expr::Call(_) + | Expr::FString(_) + | Expr::TString(_) + | Expr::StringLiteral(_) + | Expr::BytesLiteral(_) + | Expr::NumberLiteral(_) + | Expr::BooleanLiteral(_) + | Expr::NoneLiteral(_) + | Expr::EllipsisLiteral(_) + | Expr::Attribute(_) + | Expr::Subscript(_) + | Expr::Starred(_) + | Expr::List(_) + | Expr::Tuple(_) + | Expr::Slice(_) + | Expr::IpyEscapeCommand(_) + | Expr::CallableType(_) + | Expr::ProtocolType(_) + | Expr::ProtocolMethod(_) => {} + } + // the sub-expressions, without the visitor trait's statement recursion: a + // nested `def` is reached through `stmt`, which knows to open a scope for it + ruff_python_ast::visitor::source_order::walk_expr( + &mut Children(self, std::marker::PhantomData), + expr, + ); + } + + /// a comprehension is a scope of its own, so a target of its that spells a helper name + /// takes that name over for everything the comprehension evaluates — the element, the + /// conditions, and every iterable but the first. that first iterable is evaluated where + /// the comprehension is written, before the scope exists, so it is walked outside it + fn comprehension( + &mut self, + key: &str, + described: &str, + generators: &[ast::Comprehension], + elements: impl FnOnce(&mut Self), + ) { + let Some(first) = generators.first() else { + elements(self); + return; + }; + self.expr(&first.iter); + let mut locals = Bindings::new(); + for generator in generators { + bind_target(&generator.target, &mut locals); + } + locals.retain(|bound, _| defines(bound)); + self.enter( + Frame { + is_class: false, + locals, + key: key.to_owned(), + described: described.to_owned(), + }, + |walk| { + for (index, generator) in generators.iter().enumerate() { + if index > 0 { + walk.expr(&generator.iter); + } + for condition in &generator.ifs { + walk.expr(condition); + } + } + elements(walk); + }, + ); + } +} + +/// hands each of an expression's own sub-expressions back to [`Walk::expr`] +struct Children<'a, 'b>(&'a mut Walk, std::marker::PhantomData<&'b ()>); + +impl<'b> SourceOrderVisitor<'b> for Children<'_, 'b> { + fn visit_expr(&mut self, expr: &'b Expr) { + self.0.expr(expr); } } +/// the helper names `body`'s own statements bind, not descending into the scopes they open +/// +/// a name the body declares `global` or `nonlocal` is bound somewhere else, so an assignment to +/// it is not a local binding and does not take the helper's name over — which is how +/// `_by_match_seq` in the runtime itself assigns `_by_match_seq_types` +fn collect(body: &[Stmt], locals: &mut Bindings) { + for stmt in body { + for (name, range) in bindings(stmt) { + if defines(&name) { + record(locals, name, range); + } + } + } + let mut declared = BTreeSet::new(); + declarations(body, &mut declared); + for name in declared { + locals.remove(&name); + } +} + +/// the names `body` declares `global` or `nonlocal`, through the blocks it opens +fn declarations(body: &[Stmt], out: &mut BTreeSet) { + for stmt in body { + match stmt { + Stmt::Global(node) => out.extend(node.names.iter().map(ToString::to_string)), + Stmt::Nonlocal(node) => out.extend(node.names.iter().map(ToString::to_string)), + _ => { + let mut blocks = Vec::new(); + push_blocks(stmt, &mut blocks); + for block in blocks { + declarations(block, out); + } + } + } + } +} + +/// the suites a statement runs in its own scope +/// +/// spelled out over every statement, for the reason [`bind`] is: this is the other half of +/// the same enumeration, and the two have to agree about which statements hold suites +fn push_blocks<'a>(stmt: &'a Stmt, out: &mut Vec<&'a [Stmt]>) { + match stmt { + Stmt::If(node) => { + out.push(&node.body); + for clause in &node.elif_else_clauses { + out.push(&clause.body); + } + } + Stmt::Try(node) => { + out.push(&node.body); + for ExceptHandler::ExceptHandler(handler) in &node.handlers { + out.push(&handler.body); + } + out.push(&node.orelse); + out.push(&node.finalbody); + } + Stmt::With(node) => out.push(&node.body), + Stmt::For(node) => { + out.push(&node.body); + out.push(&node.orelse); + } + Stmt::While(node) => { + out.push(&node.body); + out.push(&node.orelse); + } + Stmt::Match(node) => { + for case in &node.cases { + out.push(&case.body); + } + } + // basedpython `let := `, whose `else` runs where the pattern + // did not match + Stmt::Let(node) => out.push(&node.orelse), + // a `def` and a `class` hold a suite too, but it is a scope of its own rather than + // one that runs where the statement is written — the callers open a frame for it + Stmt::FunctionDef(_) + | Stmt::ClassDef(_) + | Stmt::Return(_) + | Stmt::Delete(_) + | Stmt::TypeAlias(_) + | Stmt::Assign(_) + | Stmt::AugAssign(_) + | Stmt::AnnAssign(_) + | Stmt::Raise(_) + | Stmt::Assert(_) + | Stmt::Import(_) + | Stmt::ImportFrom(_) + | Stmt::Global(_) + | Stmt::Nonlocal(_) + | Stmt::Expr(_) + | Stmt::Pass(_) + | Stmt::Break(_) + | Stmt::Continue(_) + | Stmt::IpyEscapeCommand(_) => {} + } +} + +/// the expressions a statement evaluates in the scope it is written in, leaving the +/// suites [`push_blocks`] hands back to be walked as statements +fn statement_expressions(stmt: &Stmt) -> Vec<&Expr> { + struct Own<'a> { + exprs: Vec<&'a Expr>, + } + impl<'a> SourceOrderVisitor<'a> for Own<'a> { + fn visit_expr(&mut self, expr: &'a Expr) { + self.exprs.push(expr); + } + fn visit_stmt(&mut self, _: &'a Stmt) {} + } + let mut own = Own { exprs: Vec::new() }; + ruff_python_ast::visitor::source_order::walk_stmt(&mut own, stmt); + own.exprs +} + /// every name a statement loads, at any depth fn reads(stmt: &Stmt) -> BTreeSet { struct Reads(BTreeSet); @@ -363,7 +1041,7 @@ mod tests { let parsed = parse_module(SOURCE).expect("the runtime parses"); let mut seen = BTreeSet::new(); for stmt in parsed.suite() { - for name in bindings(stmt) { + for name in bindings(stmt).into_keys() { assert!(seen.insert(name.clone()), "`{name}` is bound twice"); } } @@ -405,4 +1083,23 @@ mod tests { ); assert!(pasted.contains("def _lazy_module("), "{pasted}"); } + + /// the proxy's three overrides of `object`'s own members are reported by + /// `missing-override-decorator`, and the emitted python is read by this project's own + /// checker. `@override` is `typing.override`, which arrived in 3.12, and the polyfill + /// runs on 3.9 — so each carries the suppression the decorator would have been + #[test] + fn the_proxies_overrides_do_not_report_in_the_emitted_python() { + let pasted = inline([LAZY_ATTR]).concat(); + for member in ["__class__", "__setattr__", "__delattr__"] { + let line = pasted + .lines() + .find(|line| line.trim_start().starts_with(&format!("def {member}("))) + .unwrap_or_else(|| panic!("`{member}` is no longer defined:\n{pasted}")); + assert!( + line.contains("# ty: ignore[missing-override-decorator]"), + "`{member}` lost its suppression: {line}" + ); + } + } } diff --git a/crates/by_transforms/src/runtime/_by_runtime.py b/crates/by_transforms/src/runtime/_by_runtime.py index 1d0a00d024..26d246f200 100644 --- a/crates/by_transforms/src/runtime/_by_runtime.py +++ b/crates/by_transforms/src/runtime/_by_runtime.py @@ -387,6 +387,37 @@ def __iter__(self): yield self.interpolations[index] +# a `t"..."` the author wrote, carrying the text the author wrote for each of its +# fields rather than the text this file was lowered to +# +# `Interpolation.expression` is the source of the field, which python reads off the +# compiled file — `t"{a!}"` would hand a reader `_force_unwrap(a)`. `_exprs` runs +# parallel to the fields: a string is the author's own text for that field, and +# `None` a field nothing lowered, which keeps the interpolation python built and so +# stays byte-identical to what python would have reported +# +# the import is made here rather than at the top of the file because this module is +# shared with runtimes that have no `string.templatelib`. only a t-string can reach +# this helper, and a t-string needs 3.14 +def _by_template(_t, _exprs): + from string.templatelib import Interpolation, Template + + _strings = _t.strings + _parts = [] + for _index, _field in enumerate(_t.interpolations): + _parts.append(_strings[_index]) + _expr = _exprs[_index] + _parts.append( + _field + if _expr is None + else Interpolation( + _field.value, _expr, _field.conversion, _field.format_spec + ) + ) + _parts.append(_strings[-1]) + return Template(*_parts) + + def _by_graphemes(_text): try: import regex as _regex @@ -1230,11 +1261,14 @@ def _by_resolve(self): object.__setattr__(self, "_by_val", v) object.__setattr__(self, "_by_has", True) return self._by_val + # the three below really do override `object`'s, and `@override` is what says + # so — but it is `typing.override`, which arrived in 3.12, and this polyfill + # runs on 3.9. the suppression is what the decorator would have been @property - def __class__(self): return self._by_resolve().__class__ + def __class__(self): return self._by_resolve().__class__ # ty: ignore[missing-override-decorator] def __getattr__(self, k): return getattr(self._by_resolve(), k) - def __setattr__(self, k, v): setattr(self._by_resolve(), k, v) - def __delattr__(self, k): delattr(self._by_resolve(), k) + def __setattr__(self, k, v): setattr(self._by_resolve(), k, v) # ty: ignore[missing-override-decorator] + def __delattr__(self, k): delattr(self._by_resolve(), k) # ty: ignore[missing-override-decorator] def __call__(self, *a, **k): return self._by_resolve()(*a, **k) def __class_getitem__(cls, k): return cls def __instancecheck__(self, o): return isinstance(o, self._by_resolve()) diff --git a/crates/by_transforms/src/source_map.rs b/crates/by_transforms/src/source_map.rs index 0b207ad894..40ef4fb861 100644 --- a/crates/by_transforms/src/source_map.rs +++ b/crates/by_transforms/src/source_map.rs @@ -41,7 +41,7 @@ pub(crate) struct Replacement { impl Replacement { /// Text no source span spells, standing for the construct at `anchor` — a - /// re-rendered statement, a plain-text substitution. + /// hoisted statement, a plain-text substitution. pub(crate) fn generated(text: &str, anchor: usize) -> Self { let mut replacement = Self::default(); replacement.push_generated(text, anchor); @@ -849,14 +849,33 @@ mod tests { ); } - /// a statement an AST pass re-renders is printed from its AST, which keeps - /// no source ranges: nothing says which rendered line came from which source - /// line, so every line of it is charged to the statement's first line (see - /// the re-render edit in `ast_driver::run_against_source`). the repeated - /// `_` parameter is one such pass; the text after the statement resumes on - /// its own lines + /// a name the lowering needs that is folded into an import the module already wrote adds + /// no line, so that line goes on mapping to where the module wrote it. the preamble + /// accounts for the lines it adds, and this is the case where it adds none #[test] - fn a_statement_re_rendered_from_its_ast_is_charged_whole_to_its_first_line() { + fn a_name_folded_into_the_modules_own_import_keeps_its_line() { + assert_mapped( + indoc! {" + from typing import Any + + def apply(f: (int) -> int) -> Any: + return f(1) + "}, + &[ + (0, "from typing import Any, Callable"), + (1, ""), + (2, "def apply(f: Callable[[int], int]) -> Any:"), + (3, " return f(1)"), + ], + ); + } + + /// a statement an AST pass rewrote prints only the nodes the pass changed and + /// passes the rest of its source through (see `rerender`), so the lines it + /// passes through keep their own. the repeated `_` parameter changes only the + /// `def` line, which is charged to itself + #[test] + fn a_statement_an_ast_pass_rewrote_keeps_the_lines_it_passes_through() { assert_mapped( indoc! {" def ignore(_: int, _: int) -> int: @@ -866,9 +885,9 @@ mod tests { print(ignore(1, 2)) "}, &[ - (0, "def ignore(_: int, _2: int) -> int:"), - (0, " a = 1"), - (0, " return a"), + (0, "def ignore(_: int, _2: int, /) -> int:"), + (1, " a = 1"), + (2, " return a"), (3, ""), (4, "print(ignore(1, 2))"), ], diff --git a/crates/by_transforms/src/transforms/annotation.rs b/crates/by_transforms/src/transforms/annotation.rs index fc124c3a07..e1d6c921dc 100644 --- a/crates/by_transforms/src/transforms/annotation.rs +++ b/crates/by_transforms/src/transforms/annotation.rs @@ -9,6 +9,7 @@ use crate::config::FloatLiteralLowering; use crate::transforms::ast_driver::{PassContext, TypeAwarePass}; use crate::transforms::callable::CallableSyntax; use crate::transforms::optional_type; +use crate::transforms::repeated_underscore::WrittenNames; use crate::transforms::type_expr_walker::{Recurse, TypeExprVisitor, TypePos, walk_type_positions}; use crate::type_info::TypeInfo; @@ -61,12 +62,13 @@ pub(crate) struct TupleLiteralType<'src> { impl<'src> TupleLiteralType<'src> { fn new( source: &'src str, + written: WrittenNames<'src>, types: &'src dyn TypeInfo, min_version: PythonVersion, float_literals: FloatLiteralLowering, symbolic_substitutions: &'src [(TextRange, String)], ) -> Self { - let mut leaves = CallableSyntax::new(source, float_literals).with_types(types); + let mut leaves = CallableSyntax::new(source, written, float_literals).with_types(types); for (range, rendered) in symbolic_substitutions { leaves.add_substitution(*range, rendered.clone()); } @@ -376,20 +378,36 @@ impl TypeExprVisitor for TupleLiteralType<'_> { pub(crate) struct TupleLiteralTypePass<'src> { source: &'src str, + written: WrittenNames<'src>, config: Config, } impl<'src> TupleLiteralTypePass<'src> { - pub(crate) fn new(source: &'src str, config: Config) -> Self { - Self { source, config } + pub(crate) fn new(source: &'src str, written: WrittenNames<'src>, config: Config) -> Self { + Self { + source, + written, + config, + } } } impl TypeAwarePass for TupleLiteralTypePass<'_> { + fn lowering(&self) -> Option { + Some(super::ast_driver::Lowering::TupleLiteralType) + } + + /// each element of a tuple type it replaces is printed by the shared type-expression + /// lowerer + fn subsumes(&self) -> &'static [super::ast_driver::Lowering] { + super::callable::TYPE_EXPRESSION + } + fn run(&self, stmts: &[Stmt], types: &dyn TypeInfo, ctx: &mut PassContext) { let symbolic_substitutions = ctx.symbolic_substitutions.clone(); let mut inner = TupleLiteralType::new( self.source, + self.written, types, self.config.min_version, self.config.float_literals, diff --git a/crates/by_transforms/src/transforms/anon_named_tuple.rs b/crates/by_transforms/src/transforms/anon_named_tuple.rs index 580a322cf5..133025cf22 100644 --- a/crates/by_transforms/src/transforms/anon_named_tuple.rs +++ b/crates/by_transforms/src/transforms/anon_named_tuple.rs @@ -39,6 +39,7 @@ use ruff_python_ast::{Expr, Stmt, StmtAnnAssign, StmtFunctionDef, StmtReturn}; use ruff_text_size::Ranged; use crate::transforms::callable::CallableSyntax; +use crate::transforms::repeated_underscore::WrittenNames; use crate::type_info::TypeInfo; /// Synthetic field name for the i-th positional element of a mixed @@ -164,11 +165,16 @@ pub(crate) struct AnonNamedTuple<'src> { } impl<'src> AnonNamedTuple<'src> { - pub(crate) fn new(source: &'src str, types: &'src dyn TypeInfo, config: crate::Config) -> Self { + pub(crate) fn new( + source: &'src str, + written: WrittenNames<'src>, + types: &'src dyn TypeInfo, + config: crate::Config, + ) -> Self { Self { source, types, - callable: CallableSyntax::new(source, config.float_literals).with_types(types), + callable: CallableSyntax::new(source, written, config.float_literals).with_types(types), config, edits: Vec::new(), shapes: indexmap::IndexMap::new(), @@ -377,11 +383,10 @@ impl<'src> AnonNamedTuple<'src> { /// (`(name=v, ...)`). Same conventions as `extract_type_shape`; field /// types come from ty's promoted (literal-stripped) inference. /// - /// Returns `Err(message)` on hard structural failure. Returns - /// `Ok(None)` if any field's value type can't be resolved by ty (e.g. - /// unresolved import) — the caller falls back to leaving the source - /// expression alone, and `verify_syntax` will catch any leftover anon-NT - /// AST in the output. + /// Returns `Err(message)` on hard structural failure, and on a field whose value has no + /// type the emitted python can name: the class it would have to write is the whole + /// lowering, so a field it cannot annotate is a refusal rather than a shape to leave + /// alone. `Ok(None)` means the tuple is not a value-form anonymous named tuple at all. fn extract_value_shape( &self, tuple: &ruff_python_ast::ExprTuple, @@ -414,8 +419,22 @@ impl<'src> AnonNamedTuple<'src> { { class_name } else { - let Some(d) = self.types.promoted_type_display(value_expr) else { - return Ok(None); + // the field needs an annotation the emitted file can read, and + // `promoted_type_display` refuses a spelling that names anything the output + // does not bind — a generator's element type keeps a `Literal` inside it, + // which is not one of the names basedpython imports for a synthesized + // annotation. say so here: written out, the class would read a name that is + // not there, and left alone the tuple reaches the output as a shape python + // has no syntax for + let Some(d) = self + .types + .promoted_type_display(value_expr, self.config.min_version) + else { + return Err(format!( + "anonymous named tuple field `{name}` has no type the emitted python \ + can name; annotate it, or give the tuple a `NamedTuple` class of \ + its own" + )); }; self.type_only_imports.borrow_mut().extend(d.modules); self.typing_names.borrow_mut().extend(d.typing_names); @@ -860,23 +879,50 @@ impl<'ast> Visitor<'ast> for AnonNamedTuple<'_> { pub(crate) struct AnonNamedTuplePass<'src> { source: &'src str, + written: WrittenNames<'src>, config: crate::Config, } impl<'src> AnonNamedTuplePass<'src> { - pub(crate) fn new(source: &'src str, config: crate::Config) -> Self { - Self { source, config } + pub(crate) fn new( + source: &'src str, + written: WrittenNames<'src>, + config: crate::Config, + ) -> Self { + Self { + source, + written, + config, + } } } impl super::ast_driver::TypeAwarePass for AnonNamedTuplePass<'_> { + fn lowering(&self) -> Option { + Some(super::ast_driver::Lowering::AnonNamedTuple) + } + + /// a named tuple type it replaces moves into a class, whose field types the shared + /// type-expression lowerer prints, a `typeof` among them + fn subsumes(&self) -> &'static [super::ast_driver::Lowering] { + &[ + super::ast_driver::Lowering::Callable, + super::ast_driver::Lowering::OptionalType, + super::ast_driver::Lowering::LiteralType, + super::ast_driver::Lowering::JustFloat, + super::ast_driver::Lowering::DynamicKeyword, + super::ast_driver::Lowering::FloatConst, + super::ast_driver::Lowering::Typeof, + ] + } + fn run( &self, stmts: &[ruff_python_ast::Stmt], types: &dyn TypeInfo, ctx: &mut super::ast_driver::PassContext, ) { - let mut inner = AnonNamedTuple::new(self.source, types, self.config.clone()); + let mut inner = AnonNamedTuple::new(self.source, self.written, types, self.config.clone()); for stmt in stmts { inner.visit_stmt(stmt); } @@ -953,9 +999,9 @@ mod tests { ); } - /// `typeof` is lowered by an AST pass that re-renders its whole statement, - /// which drops this pass's edit — the cleanup run then re-lowers the - /// restored surface form, and must not emit the class a second time + /// `typeof` is lowered by an AST pass, inside the construct this pass replaces + /// whole — the cleanup run then lowers what is left in the output, and must + /// not emit the class a second time #[test] fn a_re_rendered_statement_does_not_duplicate_the_class() { let out = transpile("b: int = 1\na: (m: typeof b)\n", &Config::test_default()).unwrap(); @@ -1175,6 +1221,23 @@ mod tests { assert!(err.contains("duplicate field name `arg0`"), "got: {err}"); } + /// the class this lowering writes is the whole of it, so a field it cannot annotate is + /// a refusal. ty spells a generator's type `types.GeneratorType[Literal[1], None, + /// None]`, and `Literal` is not one of the names a synthesized annotation may import — + /// writing the class would give the emitted file a name that is not in it + #[test] + fn a_field_with_no_nameable_type_is_a_hard_error() { + let err = transpile( + "a = (gen=(x for x in [1]), n=20)\n", + &Config::test_default(), + ) + .unwrap_err(); + assert!( + err.contains("field `gen` has no type the emitted python can name"), + "got: {err}" + ); + } + #[test] fn duplicate_named_fields_is_hard_error() { let err = transpile("a = (name=1, name=2)\n", &Config::test_default()).unwrap_err(); diff --git a/crates/by_transforms/src/transforms/ast_driver.rs b/crates/by_transforms/src/transforms/ast_driver.rs index 9a4394321b..4449ab45e4 100644 --- a/crates/by_transforms/src/transforms/ast_driver.rs +++ b/crates/by_transforms/src/transforms/ast_driver.rs @@ -37,7 +37,7 @@ use ruff_text_size::{Ranged, TextRange, TextSize}; use super::source_util::preamble_offset; use super::{ annotation, anon_named_tuple, auto_quote, build_stamps, callable, character_type, checked_cast, - class_pattern_star, coalesce, coalesce_chain, compat, conformance, context_params, conversion, + class_pattern_star, coalesce, compat, conformance, context_params, conversion, debug_field, decl_site_variance, decorated_binding, decorator_keyword, dedent_string, destructure, django_lookup, dynamic_keyword, empty_declarations, export_import, extension, flexible_keyword, float_const, force_unwrap, frameworks, generic_call, generics, grapheme_string, identity_swap, @@ -47,9 +47,9 @@ use super::{ postfix_await, propagate, properties, protocol_type, raises_clause, reified_class, reified_generic, repeated_underscore, return_value_use, runtime_union, sentinel, some_ctor, soundness, statement_expression, static_resource, string_tag, super_keyword, symbolic_type_op, - template_type, top_star, trailing_lambda, tuple_index, type_fn, type_is, type_reification, - typed_dict_literal, typed_lambda, typeof_keyword, unique_loop_bindings, unpack, - use_site_variance, visibility_rename, + template_expression, template_type, top_star, trailing_lambda, tuple_index, type_fn, type_is, + type_reification, typed_dict_literal, typed_lambda, typeof_keyword, unique_loop_bindings, + unpack, use_site_variance, visibility_rename, }; use crate::Config; use crate::source_map::Replacement; @@ -69,7 +69,7 @@ enum SemDb<'p> { /// with any sibling edits inside them applied, so a wide rewrite (e.g. /// `a ?? b` → `a if a is not None else b`) composes with lowerings inside its /// operands instead of clobbering them via first-wins overlap dedup -#[derive(Clone)] +#[derive(Clone, PartialEq, Eq)] pub(crate) enum Fragment { Lit(String), Src(TextRange), @@ -105,9 +105,10 @@ pub(crate) struct PassContext { pub(crate) text_edits: Vec<(TextRange, String)>, /// Structured sub-statement edits whose replacement is a [`Fragment`] list. /// Unlike `text_edits` (whose plain string wins over anything nested inside - /// it), the `Src` passthrough spans of a template are materialized with the - /// sibling edits they contain applied — use this for any rewrite that - /// re-emits operand source + /// it — which is a transpile error unless the pass writes that lowering itself, + /// see [`TypeAwarePass::subsumes`]), the `Src` passthrough spans of a template are + /// materialized with the sibling edits they contain applied — use this for any + /// rewrite that re-emits operand source pub(crate) template_edits: Vec<(TextRange, Vec)>, /// Templates inserted at a *statement* boundary — a guard a pass injects /// ahead of the statement starting at that offset. Identical to a @@ -127,6 +128,12 @@ pub(crate) struct PassContext { /// the substitution by shape alone, which cannot tell two substitutions of /// one span apart pub(crate) relocating_edits: Vec<(TextRange, Vec)>, + /// Soundness checks made in top-level statements an AST pass rewrote, by the + /// statement's index in the original module body. The nodes a pass changed are + /// printed from the syntax tree, where a text edit made at a node the pass + /// replaced does not land, so these are written into that tree before it is + /// printed + pub(crate) rerendered_checks: Vec<(usize, Vec)>, /// Hard transpile errors a pass surfaced — abort the pipeline rather /// than emit partial / invalid output. Each entry is a human-readable /// message suitable for showing the user @@ -146,6 +153,10 @@ pub(crate) struct PassContext { /// [`walk_type_positions_skipping`](super::type_expr_walker::walk_type_positions_skipping) /// so they don't re-process an operation that no longer appears in the output pub(crate) claimed_type_op_ranges: Vec, + /// The `typeof` nodes nested under a structural type form, which the + /// type-expression lowerer rewrites as part of that form. Every other `typeof` + /// is lowered in the syntax tree, and only there + pub(crate) structural_typeof_ranges: Vec, /// The same operations as `(range, rendered)` pairs. A pass that replaces a /// whole statement subsumes any fold inside it — skipping is not enough, the /// rendered text has to be spliced into the replacement or the operation is @@ -153,6 +164,41 @@ pub(crate) struct PassContext { pub(crate) symbolic_substitutions: Vec<(TextRange, String)>, } +/// a lowering whose edits another pass may leave out of its own, because that pass writes +/// the construct the edit lowers itself — see [`TypeAwarePass::subsumes`] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Lowering { + AnonNamedTuple, + Callable, + ContextParams, + Conversion, + DynamicKeyword, + FloatConst, + GenericPolyfill, + InferredAnnotation, + JustFloat, + /// the whitespace the keyword blanking collapses. only spacing is at stake, so any + /// edit it lands inside may leave it out + KeywordPadding, + LiteralType, + LocalOnce, + MatchType, + Modifiers, + NoneChain, + NoneCoalesce, + OptionalType, + ParametricIs, + ProtocolType, + RepeatedUnderscore, + StatementExpression, + SymbolicTypeOp, + TupleLiteralType, + Typeof, + Unpack, + VarianceStrip, + VisibilityRename, +} + /// A single AST-level rewrite pass. pub(crate) trait AstPass { /// Run the pass against the entire parsed module. The pass is free @@ -164,6 +210,21 @@ pub(crate) trait AstPass { fn runtime_only(&self) -> bool { false } + + /// See [`TypeAwarePass::lowering`]. + fn lowering(&self) -> Option { + None + } + + /// See [`TypeAwarePass::subsumes`]. + fn subsumes(&self) -> &'static [Lowering] { + &[] + } + + /// what a transform conflict calls the pass + fn name(&self) -> &'static str { + short_type_name(std::any::type_name::()) + } } /// Type-aware pass that reads semantic info from the salsa-owned parsed @@ -183,6 +244,35 @@ pub(crate) trait TypeAwarePass { fn runtime_only(&self) -> bool { false } + + /// the lowering this pass's edits make, when another pass may write the constructs + /// they lower itself — see [`subsumes`](Self::subsumes) + fn lowering(&self) -> Option { + None + } + + /// the lowerings whose constructs this pass writes itself, wherever one of its edits + /// covers them: an edit such a lowering makes inside one of this pass's edits is left + /// out of the output, and that is not a loss + /// + /// every other edit inside one of this pass's edits has to land in source the edit + /// passes through, or the transpile is refused. so a lowering is listed here only + /// where this pass writes what that lowering would have written, and a lowering added + /// later that reaches inside one of its edits is refused until it is + fn subsumes(&self) -> &'static [Lowering] { + &[] + } + + /// what a transform conflict calls the pass + fn name(&self) -> &'static str { + short_type_name(std::any::type_name::()) + } +} + +/// `path::to::Type` as `Type` +fn short_type_name(name: &'static str) -> &'static str { + let bare = name.split('<').next().unwrap_or(name); + bare.rsplit("::").next().unwrap_or(bare) } /// Adapter: lift a [`Transformer`] (visitor that mutates AST in place) @@ -198,9 +288,18 @@ pub(crate) struct VisitorPass<'a, T: Transformer> { /// computes the new sub-AST, renders it via [`render_expr`], and pushes /// `(original_range, replacement)` here text_edits: RefCell>, + lowering: Option, } impl AstPass for VisitorPass<'_, T> { + fn lowering(&self) -> Option { + self.lowering + } + + fn name(&self) -> &'static str { + short_type_name(std::any::type_name::()) + } + fn run(&self, module: &mut ModModule, ctx: &mut PassContext) { for (idx, stmt) in module.body.iter_mut().enumerate() { self.changed_cell.set(false); @@ -248,6 +347,12 @@ enum SubPatch { /// [`SubPatch::Template`]; the difference is only that it leads every other /// edit at its span Relocating(Vec), + /// a node an AST pass rewrote, re-emitted at its range — see + /// [`super::rerender`]. Materializes exactly like [`SubPatch::Template`], and + /// leads the other rewrites at its span, as printing the whole statement + /// used to. an insertion at its boundary is not absorbed: the text it prints + /// there is the node's own, which no insertion targets + Rerendered(Vec), } /// whether a template spanning `start..end` only wraps it: its one passthrough @@ -271,6 +376,15 @@ fn is_wrapper(frags: &[Fragment], start: usize, end: usize) -> bool { /// default re-evaluation guard re-emits a parameter default at the body start — /// and the lowerings inside that span have to materialize where the source /// lands, not where it was written. +/// +/// A [`SubPatch::Relocating`] edit is the one thing that does *not* travel with +/// that source. It is the stand-in a pass leaves at the span it moved a +/// construct away from — `_MISSING` where the default was written — and a +/// passthrough of exactly that span, from outside the template's own range, is +/// the place the construct moved *to*. Materializing the stand-in there too +/// would put `_MISSING` at both ends and the construct at neither, which is how +/// a `decorator def` whose header template re-emits the whole signature came to +/// emit `if tags is _MISSING: tags = _MISSING`. fn template_claimees( frags: &[Fragment], all: &[(usize, usize, SubPatch)], @@ -289,7 +403,12 @@ fn template_claimees( let in_src = frags.iter().any(|frag| match frag { Fragment::Lit(_) => false, Fragment::Src(span) => { - s >= usize::from(span.start()) && e <= usize::from(span.end()) + let (span_start, span_end) = + (usize::from(span.start()), usize::from(span.end())); + s >= span_start + && e <= span_end + && !(matches!(all[m].2, SubPatch::Relocating(_)) + && (s, e) == (span_start, span_end)) } }); in_own || in_src @@ -297,6 +416,41 @@ fn template_claimees( .collect() } +/// Whether a template over `range` puts text there other than the text already +/// written there. +/// +/// Read fragment by fragment against the covered source, so neither side is +/// built up: a literal has to match the source ahead of it, and a passthrough +/// has to be the very span it stands at. +/// +/// A `Src` passthrough carries any sibling edit that falls inside it, which this +/// reading leaves out — but such a sibling is an entry of the same edit list, and +/// the caller asks the list as a whole. So a template that only re-emits what it +/// covers answers `false` here while a real rewrite nested inside it still +/// answers `true` under its own entry. +fn rewrites_covered_source(range: TextRange, frags: &[Fragment], source: &str) -> bool { + let Some(covered) = source.get(usize::from(range.start())..usize::from(range.end())) else { + return true; + }; + let mut rest = covered; + for frag in frags { + let piece = match frag { + Fragment::Lit(text) => text.as_str(), + Fragment::Src(span) => { + match source.get(usize::from(span.start())..usize::from(span.end())) { + Some(text) => text, + None => return true, + } + } + }; + match rest.strip_prefix(piece) { + Some(tail) => rest = tail, + None => return true, + } + } + !rest.is_empty() +} + /// Materialize a template's fragments into `out`. `Src` passthrough spans are /// emitted from original source with the contained sub-edits (indices into /// `all`) applied. @@ -312,6 +466,7 @@ fn materialize_fragments( all: &[(usize, usize, SubPatch)], contained: &[usize], anchor: usize, + emitted: &mut [bool], ) { for (i, frag) in frags.iter().enumerate() { match frag { @@ -337,6 +492,7 @@ fn materialize_fragments( all, contained, include_end, + emitted, ); } } @@ -351,7 +507,8 @@ fn materialize_fragments( /// templates — one statement expression inside another's suite, and any pass /// whose template lands inside the suite it passes through. `include_end` controls /// whether a zero-width insertion exactly at `e0` is emitted here (see -/// [`materialize_fragments`]). +/// [`materialize_fragments`]). every edit written out is marked in `emitted` +#[expect(clippy::too_many_arguments)] fn apply_within( out: &mut Replacement, source: &str, @@ -360,6 +517,7 @@ fn apply_within( all: &[(usize, usize, SubPatch)], contained: &[usize], include_end: bool, + emitted: &mut [bool], ) { let mut cursor = s0; let mut k = 0; @@ -373,17 +531,19 @@ fn apply_within( continue; } out.push_source(source, cursor, s); + emitted[idx] = true; match &all[idx].2 { SubPatch::Text(t) => out.push_generated(t, s), SubPatch::Template(frags) | SubPatch::Statement(frags) - | SubPatch::Relocating(frags) => { + | SubPatch::Relocating(frags) + | SubPatch::Rerendered(frags) => { let inner: Vec = contained[k + 1..] .iter() .copied() .filter(|&m| all[m].0 >= s && all[m].1 <= e && all[m].0 != e) .collect(); - materialize_fragments(out, frags, source, all, &inner, s); + materialize_fragments(out, frags, source, all, &inner, s, emitted); } } cursor = cursor.max(e); @@ -392,6 +552,542 @@ fn apply_within( out.push_source(source, cursor, e0); } +/// the sub-statement edits spliced into the source, first-wins on overlap, and an error +/// for each edit the splice loses. `origins` names, by index into `authors`, who wrote +/// each edit +/// +/// an edit is left out when a wider edit it sits inside prints text of its own over it: a +/// node an AST pass rewrote that no longer passes the edit's source through, a template +/// that writes the construct itself, or a plain-text replacement. whatever the edit made of +/// the construct is then gone — whether or not the printed text happens to spell the +/// construct again — so that is refused rather than emitted, unless the wider edit +/// accounts for it (see [`lost_edits`]) +fn splice( + source: &str, + sub_edits: Vec<(usize, usize, SubPatch)>, + origins: Vec>, + authors: &[Author], +) -> (Vec<(usize, usize, Replacement)>, Vec) { + let mut edits: Vec<(usize, usize, Replacement)> = Vec::new(); + let mut emitted = vec![false; sub_edits.len()]; + let mut tagged: Vec<_> = sub_edits.into_iter().zip(origins).collect(); + // start asc. tie-break by edit shape: + // 1. zero-width insertions first — they don't consume bytes, so any + // following deletion/replacement at the same start can still apply. + // a statement-anchored insertion leads them: it emits whole statements + // that must precede everything the statement itself lowers to + // 2. then wider replacements before narrower ones — so a wider edit + // wins over (or, for templates, absorbs) a narrow one nested inside + // it + // 3. at one identical span, a *relocating* edit leads: it says the + // construct has moved, and the pass that moved it re-emits the span + // itself, so every other edit there materializes at the new home + // 4. then a *wrapper* — a template whose one passthrough is its whole + // span, adding text around the construct without removing any of it. + // it claims the other edits at that span and materializes them inside + // its passthrough, so whatever the construct becomes ends up inside the + // wrapping (a quoted forward reference around an arrow type the + // callable lowering replaced as text) + // 5. then a node an AST pass rewrote, which the syntax tree says the + // construct became — see [`super::rerender`] + // 6. then a *substitution* — plain text, or a template with no `Src` + // passthrough — ahead of a *rewrite*, a template that re-emits part of + // the span. a substitution says the construct does not appear here at + // all, which a rewrite of it cannot outrank + tagged.sort_by(|(a, _), (b, _)| { + let priority = |e: &(usize, usize, SubPatch)| { + let rewrites = i64::from(match &e.2 { + SubPatch::Text(_) => false, + SubPatch::Template(frags) + | SubPatch::Statement(frags) + | SubPatch::Relocating(frags) + | SubPatch::Rerendered(frags) => { + frags.iter().any(|frag| matches!(frag, Fragment::Src(_))) + } + }); + let statement = i64::from(!matches!(e.2, SubPatch::Statement(_))); + let relocating = i64::from(!matches!(e.2, SubPatch::Relocating(_))); + let wraps = i64::from( + !matches!(&e.2, SubPatch::Template(frags) if is_wrapper(frags, e.0, e.1)), + ); + let rerendered = i64::from(!matches!(e.2, SubPatch::Rerendered(_))); + // (start, is_replacement_not_insertion, statement-insert-first, + // neg_end-for-wider-first, relocating-first, wrapper-first, + // rerendered-first, substitution-before-rewrite) + if e.1 == e.0 { + ( + e.0, 0i64, statement, 0i64, relocating, wraps, rerendered, rewrites, + ) // insertion + } else { + #[allow(clippy::cast_possible_wrap)] + let neg_end = -(e.1 as i64); + ( + e.0, 1i64, statement, neg_end, relocating, wraps, rerendered, rewrites, + ) + } + }; + priority(a).cmp(&priority(b)) + }); + let (sub_edits, origins): (Vec<_>, Vec<_>) = tagged.into_iter().unzip(); + // claim pre-pass: each replacement, outermost-first (the sort guarantees an + // enclosing edit precedes anything inside it), claims the edits nested in + // its span. a template *materializes* its claimees inside its `Src` spans; + // a plain-text replacement drops them (first-wins). same-start zero-width + // insertions are absorbed by a template (they target the construct's first + // token, e.g. `_force_unwrap(` ahead of a coalesce operand) but stay + // independent ahead of a plain-text replacement, preserving the documented + // insertion + deletion compose behaviour. the one exception is a + // statement-anchored insertion sharing a boundary: it emits statements, so + // absorbing it into an expression rewrite of the statement it precedes + // would splice a suite into the middle of an expression + let mut claimed = vec![false; sub_edits.len()]; + for i in 0..sub_edits.len() { + let (s_i, e_i) = (sub_edits[i].0, sub_edits[i].1); + if e_i == s_i || claimed[i] { + continue; + } + // a rerendered node claims what is inside it as a template does, but not an + // insertion at its boundary, which targets the source around it + let is_template = matches!( + sub_edits[i].2, + SubPatch::Template(_) | SubPatch::Statement(_) | SubPatch::Relocating(_) + ); + for (m, edit) in sub_edits.iter().enumerate() { + if m == i || claimed[m] { + continue; + } + let (s_m, e_m) = (edit.0, edit.1); + let inside = s_m >= s_i && e_m <= e_i && s_m != e_i; + let boundary_insertion = s_m == e_m && (s_m == s_i || s_m == e_i); + let anchored = matches!(edit.2, SubPatch::Statement(_)); + if inside && (is_template || !boundary_insertion) && !(boundary_insertion && anchored) { + claimed[m] = true; + } + } + } + + let mut cursor = 0usize; + let mut i = 0; + while i < sub_edits.len() { + if claimed[i] { + i += 1; + continue; + } + let (start, end) = (sub_edits[i].0, sub_edits[i].1); + if start < cursor { + i += 1; + continue; + } + // coalesce all unclaimed zero-width insertions sharing this start into + // a single combined insertion (text concatenated in push order). this + // sidesteps the replace_range-at-same-position ordering issue: each + // pass pushes its slice in left-to-right intent order, and we + // splice them as one contiguous string + if end == start { + let mut combined = Replacement::default(); + let mut j = i; + while j < sub_edits.len() && sub_edits[j].0 == start && sub_edits[j].1 == start { + if !claimed[j] { + emitted[j] = true; + match &sub_edits[j].2 { + SubPatch::Text(t) => combined.push_generated(t, start), + SubPatch::Template(frags) + | SubPatch::Statement(frags) + | SubPatch::Relocating(frags) + | SubPatch::Rerendered(frags) => { + let contained = template_claimees(frags, &sub_edits, &claimed, j, None); + materialize_fragments( + &mut combined, + frags, + source, + &sub_edits, + &contained, + start, + &mut emitted, + ); + } + } + } + j += 1; + } + edits.push((start, start, combined)); + i = j; + continue; + } + emitted[i] = true; + let repl = match &sub_edits[i].2 { + // a plain-text replacement wins over anything inside it + SubPatch::Text(t) => Replacement::generated(t, start), + SubPatch::Template(frags) + | SubPatch::Statement(frags) + | SubPatch::Relocating(frags) + | SubPatch::Rerendered(frags) => { + // the claimees nested in this span materialize inside the + // template's `Src` passthrough fragments + let contained = + template_claimees(frags, &sub_edits, &claimed, i, Some((start, end))); + let mut out = Replacement::default(); + materialize_fragments( + &mut out, + frags, + source, + &sub_edits, + &contained, + start, + &mut emitted, + ); + out + } + }; + edits.push((start, end, repl)); + cursor = end; + i += 1; + } + + let lost = lost_edits(source, &sub_edits, &origins, authors, &emitted); + (edits, lost) +} + +/// the errors for the edits in `sub_edits`, in the order it is sorted in, that the splice +/// left out of its output — those not marked in `emitted` — and that the edit they were +/// left out for does not account for +/// +/// what an edit was left out for is the innermost edit around it, or at exactly its range +/// when that edit sorts ahead of it — the nearest, of several there. that edit accounts +/// for it when: +/// +/// - it deletes what it covers, so nothing inside is printed at all +/// - it is at the same span and writes the same thing, as two lowerings of one construct +/// can +/// - one pass wrote both, and so wrote them to agree +/// - its pass writes the constructs of every pass the lost edit came from itself — see +/// [`TypeAwarePass::subsumes`] +/// +/// a node an AST pass rewrote accounts for nothing: it prints what the syntax tree says, +/// which no edit of the source reaches +fn lost_edits( + source: &str, + sub_edits: &[(usize, usize, SubPatch)], + origins: &[Vec], + authors: &[Author], + emitted: &[bool], +) -> Vec { + let mut errors = Vec::new(); + for (index, (start, end, _)) in sub_edits.iter().enumerate() { + if emitted[index] { + continue; + } + let (start, end) = (*start, *end); + let around = sub_edits + .iter() + .enumerate() + .filter(|&(other, (other_start, other_end, _))| { + let (other_start, other_end) = (*other_start, *other_end); + other != index + && if (other_start, other_end) == (start, end) { + other < index + } else { + other_start <= start && end <= other_end && other_start < other_end + } + }) + // of two around it at one span, the later one sorts nearer to it + .min_by_key(|&(other, (other_start, other_end, _))| { + (other_end - other_start, std::cmp::Reverse(other)) + }) + .map(|(other, (_, _, patch))| (other, patch)); + let lost: Vec<&Author> = origins[index] + .iter() + .filter_map(|&author| authors.get(author)) + .collect(); + let only_spacing = !lost.is_empty() + && lost + .iter() + .all(|author| author.lowering == Some(Lowering::KeywordPadding)); + if only_spacing { + continue; + } + if let Some((other, patch)) = around { + let deletes = matches!(patch, SubPatch::Text(text) if text.is_empty()); + let same = (sub_edits[other].0, sub_edits[other].1) == (start, end) + && writes_the_same(patch, &sub_edits[index].2); + let winner = match origins[other].as_slice() { + [author] if !matches!(patch, SubPatch::Rerendered(_)) => Some(*author), + _ => None, + }; + let accounted = winner.is_some_and(|winner| { + !origins[index].is_empty() + && origins[index].iter().all(|&author| { + author == winner + || authors[author].lowering.is_some_and(|lowering| { + authors[winner].subsumes.contains(&lowering) + }) + }) + }); + if deletes || same || accounted { + continue; + } + } + let what = if start == end { + format!("text inserted at byte {start}") + } else { + let preview: String = source[start..end].chars().take(40).collect(); + format!("an edit of `{preview}`") + }; + let by = match lost.as_slice() { + [] => String::new(), + names => format!( + " by {}", + names + .iter() + .map(|author| format!("`{}`", author.name)) + .collect::>() + .join(" and ") + ), + }; + let lost_to = match around { + Some((_, SubPatch::Rerendered(_))) => { + "lands inside a node an AST pass rewrote and prints again".to_owned() + } + Some((other, _)) => { + let name = match origins[other].as_slice() { + [author] => authors.get(*author).map_or("another pass", |a| a.name), + _ => "another pass", + }; + format!("lands inside a construct `{name}` writes itself") + } + None => "overlaps a construct another edit rewrote".to_owned(), + }; + errors.push(format!( + "transform conflict: {what}{by} {lost_to}, so the lowering would be lost" + )); + } + errors +} + +/// whether `a` and `b`, two edits of one span, put the same thing there +fn writes_the_same(a: &SubPatch, b: &SubPatch) -> bool { + match (a, b) { + (SubPatch::Text(a), SubPatch::Text(b)) => a == b, + ( + SubPatch::Template(a) | SubPatch::Statement(a) | SubPatch::Relocating(a), + SubPatch::Template(b) | SubPatch::Statement(b) | SubPatch::Relocating(b), + ) => a == b, + _ => false, + } +} + +/// a pass, or a part of the driver, that writes edits into a [`PassContext`] +pub(crate) struct Author { + name: &'static str, + lowering: Option, + subsumes: &'static [Lowering], +} + +impl Author { + /// the driver writing edits of its own + fn driver(name: &'static str, lowering: Option) -> Self { + Self { + name, + lowering, + subsumes: &[], + } + } +} + +/// the edit lists of a [`PassContext`], in the order the driver chains them +#[derive(Clone, Copy)] +enum EditList { + Text, + Template, + Statement, + Relocating, +} + +/// who wrote each edit in a [`PassContext`] and rewrote each statement, read off how long +/// each list was as each author finished — the lists only ever grow while passes run +#[derive(Default)] +struct Authorship { + authors: Vec, + /// the length of the four edit lists, then of `changed`, as each author finished + marks: Vec<[usize; 5]>, +} + +impl Authorship { + fn finished(&mut self, author: Author, ctx: &PassContext) { + self.authors.push(author); + self.marks.push([ + ctx.text_edits.len(), + ctx.template_edits.len(), + ctx.statement_inserts.len(), + ctx.relocating_edits.len(), + ctx.changed.len(), + ]); + } + + /// the author of the edit at `index` in `list` + fn author_of(&self, list: EditList, index: usize) -> Option { + self.marks + .iter() + .position(|mark| index < mark[list as usize]) + } + + /// the authors that rewrote each statement, by its index in `changed`, the context's + /// list of rewritten statements before it is sorted + fn rewriters(&self, changed: &[usize]) -> std::collections::HashMap> { + let mut rewriters: std::collections::HashMap> = + std::collections::HashMap::new(); + for (position, &statement) in changed.iter().enumerate() { + if let Some(author) = self.marks.iter().position(|mark| position < mark[4]) { + let authors = rewriters.entry(statement).or_default(); + if !authors.contains(&author) { + authors.push(author); + } + } + } + rewriters + } +} + +/// one `from import …` statement of the module's own that a preamble line could join +struct OwnImport { + /// the module as written, dots included, so `.mod` and `mod` are not the same target + module: String, + /// where to write more names: the end of the statement, which is ahead of any comment + /// trailing it + end: usize, + /// the spellings it already imports, so a name it has is not added twice + spellings: Vec, +} + +/// Fold each `from import …` line of `imports` into a statement of `body`'s own +/// leading import block importing from the same module, and hand back the lines that found no +/// home there. +/// +/// The preamble is written above the module's own imports, so a name the lowering needs and a +/// name the module imports from the same place land on two lines saying `from typing import`. +/// One line is what a reader — and the `.py` the build ships — should have. Folding also keeps +/// the output's line count unchanged, which the line table depends on: the preamble accounts +/// for the lines it adds, and a name written into a line that is already there adds none. +/// +/// Only the leading block, only a statement written on one line, and only a name nothing else +/// in that block binds. Everything between the preamble's position and the statement is then an +/// import too, so moving the binding down to it cannot pass a read of the name — and cannot +/// change which binding wins, either. +/// +/// And only a name the rest of the preamble does not read. The preamble is not imports alone: +/// a hoisted `class _Protocol_…(Protocol)` stands beside them and reads `Protocol` as it is +/// defined, so folding that import into the module's own line would leave the class with no +/// such name at all. +pub(crate) fn merge_into_own_imports(body: &mut String, imports: Vec) -> Vec { + let at = preamble_offset(body); + let parsed = ruff_python_parser::parse_unchecked_source( + &body[at..], + ruff_python_ast::PySourceType::Python, + ); + let mut own: Vec = Vec::new(); + // every name the block binds, which says whether folding a name into one of its statements + // could change which binding the module ends up with + let mut bound: Vec<(usize, String)> = Vec::new(); + for stmt in parsed.suite() { + match stmt { + Stmt::Import(import) => { + for alias in &import.names { + let name = alias.asname.as_ref().map_or_else( + || alias.name.split('.').next().unwrap_or_default().to_owned(), + ToString::to_string, + ); + bound.push((own.len(), name)); + } + } + Stmt::ImportFrom(import) => { + let index = own.len(); + let module = format!( + "{}{}", + ".".repeat(import.level as usize), + import.module.as_ref().map_or("", |module| module.as_str()) + ); + let one_line = !body[at + usize::from(import.range().start()) + ..at + usize::from(import.range().end())] + .contains('\n'); + let mut spellings = Vec::new(); + for alias in &import.names { + bound.push(( + index, + alias.asname.as_ref().unwrap_or(&alias.name).to_string(), + )); + spellings.push(match &alias.asname { + Some(asname) => format!("{} as {asname}", alias.name), + None => alias.name.to_string(), + }); + } + // a star import says nothing about what it binds, so nothing can be folded + // into it or judged against it + if one_line && !import.names.iter().any(|alias| &*alias.name == "*") { + own.push(OwnImport { + module, + end: at + usize::from(import.range().end()), + spellings, + }); + } else { + return imports; + } + } + _ => break, + } + } + + // an import statement reads nothing, so this is what the preamble's *other* entries read + let preamble_reads = crate::names_written( + ruff_python_parser::parse_unchecked_source( + &imports.join("\n"), + ruff_python_ast::PySourceType::Python, + ) + .suite(), + ); + + let mut left_over = Vec::new(); + let mut insertions: Vec<(usize, String)> = Vec::new(); + for line in imports { + let Some((module, names)) = line + .strip_prefix("from ") + .and_then(|rest| rest.split_once(" import ")) + .filter(|(_, names)| !names.contains(['(', '*', '\n'])) + else { + left_over.push(line); + continue; + }; + let Some(index) = own.iter().position(|entry| entry.module == module) else { + left_over.push(line); + continue; + }; + let spellings: Vec<&str> = names.split(", ").collect(); + let elsewhere = spellings.iter().any(|spelling| { + let bind = spelling.rsplit(" as ").next().unwrap_or(spelling); + preamble_reads.contains(bind) + || bound + .iter() + .any(|(owner, name)| *owner != index && name == bind) + }); + if elsewhere { + left_over.push(line); + continue; + } + let added: Vec<&str> = spellings + .into_iter() + .filter(|spelling| !own[index].spellings.iter().any(|had| had == spelling)) + .collect(); + if !added.is_empty() { + insertions.push((own[index].end, format!(", {}", added.join(", ")))); + } + } + // descending, so an earlier statement's offsets are still the ones the parse reported + insertions.sort_by_key(|(end, _)| std::cmp::Reverse(*end)); + for (end, text) in insertions { + body.insert_str(end, &text); + } + left_over +} + /// Coalesce repeated `from import X` lines into a single /// `from import X, Y, ...` line. Preserves any non-matching /// lines (e.g. `import foo`, `_MISSING = object()`) in their original @@ -404,7 +1100,13 @@ fn merge_from_imports(lines: Vec) -> (Vec, Vec) { let mut groups: indexmap::IndexMap> = indexmap::IndexMap::new(); let mut other: Vec = Vec::new(); for line in lines { - if let Some(rest) = line.strip_prefix("from ") + // an entry that spans lines is not an import line but a block — the + // `if TYPE_CHECKING:` one, which opens with a `from typing import`. merged + // as though the whole block were a name list, the block's own lines become + // a "name" and every later `from typing import` lands after them, inside + // the block: `if TYPE_CHECKING:\n import types, overload` + if !line.contains('\n') + && let Some(rest) = line.strip_prefix("from ") && let Some((module, names)) = rest.split_once(" import ") { let entry = groups.entry(module.trim().to_owned()).or_default(); @@ -440,6 +1142,7 @@ fn merge_from_imports(lines: Vec) -> (Vec, Vec) { /// suite must come from one db pub(crate) fn run_against_source<'a>( source: &'a str, + written: repeated_underscore::WrittenNames, config: &Config, project: Option<(&dyn ty_python_semantic::Db, ruff_db::files::File)>, ) -> (Cow<'a, str>, Vec, Vec>) { @@ -490,7 +1193,18 @@ pub(crate) fn run_against_source<'a>( // tokens go when the syntax tree is taken out of the parse let accessor_value_ranges = properties::collect_value_ranges(&parsed.syntax().body, parsed.tokens()); + // a statement an AST pass rewrote is re-emitted in the source's own indentation, + // since the lines it passes through keep theirs + let indentation = ruff_python_codegen::Stylist::from_tokens(parsed.tokens(), source_ref) + .indentation() + .clone(); + // and with the comments between the nodes it passes through, which are found in the + // tokens rather than read off the text, where a `#` may be inside a string + let comments = ruff_python_trivia::CommentRanges::from(parsed.tokens()); let mut module = parsed.into_syntax(); + // what each statement was before any pass rewrote it, which is what says which + // of its nodes a pass rewrote + let parsed_body = module.body.clone(); // capture each top-level statement's original source range before any // pass mutates the AST. AST mutations replace nodes with synthesised // ones whose ranges are zeroed (default `TextRange`), so the splice @@ -502,14 +1216,15 @@ pub(crate) fn run_against_source<'a>( .collect(); let mut ctx = PassContext::default(); - let coalesce_inner = coalesce_chain::CoalesceFold::new(); - let coalesce_pass = VisitorPass { - inner: &coalesce_inner, - changed_cell: coalesce_inner.changed_cell(), - imports: vec![], - hoist: RefCell::new(vec![]), - text_edits: RefCell::new(vec![]), - }; + // how each parameter list that repeats `_` is lowered is ty's answer, which every pass + // that names a parameter reads through `written` — the ones walking a tree of their own + // too, so it is read off the db's parse now, by range + let underscore_lowerings = repeated_underscore::collect( + parsed_handle.suite(), + &semantic_model, + config.min_version >= ruff_python_ast::PythonVersion::PY38, + ); + let written = written.with_lowerings(&underscore_lowerings); // resolve symbolic operations in type positions (`1 + 1` → `Literal[2]`) // up front, from the original parse where `typeof` operands are still @@ -545,6 +1260,7 @@ pub(crate) fn run_against_source<'a>( &semantic_model, &ctx.claimed_type_op_ranges, ); + ctx.structural_typeof_ranges.clone_from(&typeof_skip); let typeof_inner = typeof_keyword::TypeofFold::new(typeof_skip); let typeof_pass = VisitorPass { inner: &typeof_inner, @@ -552,6 +1268,7 @@ pub(crate) fn run_against_source<'a>( imports: vec![], hoist: RefCell::new(vec![]), text_edits: RefCell::new(vec![]), + lowering: Some(Lowering::Typeof), }; let tuple_index_pass = tuple_index::TupleIndexPass::new(); @@ -565,16 +1282,11 @@ pub(crate) fn run_against_source<'a>( imports: vec![], hoist: RefCell::new(vec![]), text_edits: RefCell::new(vec![]), + lowering: None, }; - let repeated_underscore_inner = repeated_underscore::RepeatedUnderscore::new(); - let repeated_underscore_pass = VisitorPass { - inner: &repeated_underscore_inner, - changed_cell: repeated_underscore_inner.changed_cell(), - imports: vec![], - hoist: RefCell::new(vec![]), - text_edits: RefCell::new(vec![]), - }; + let repeated_underscore_pass = + repeated_underscore::RepeatedUnderscore::new(source_ref, written); let typed_lambda_inner = typed_lambda::TypedLambda::new(source_ref); let typed_lambda_pass = VisitorPass { @@ -583,6 +1295,7 @@ pub(crate) fn run_against_source<'a>( imports: vec![], hoist: RefCell::new(vec![]), text_edits: RefCell::new(vec![]), + lowering: None, }; let export_import_pass = export_import::ExportImport::new(source_ref); @@ -595,10 +1308,10 @@ pub(crate) fn run_against_source<'a>( let compat_pass = compat::CompatRewrite::new(source_ref, config.clone()); let string_tag_pass = string_tag::StringTagPass::new(source_ref, config.clone()); let dedent_string_pass = dedent_string::DedentString::new(source_ref); - let super_keyword_pass = super_keyword::SuperKeyword::new(); + let super_keyword_pass = super_keyword::SuperKeyword::new(written); let postfix_await_pass = postfix_await::PostfixAwait::new(source_ref); let mutable_defaults_pass = - mutable_defaults::MutableDefaultsPass::new(source_ref, config.is_stub); + mutable_defaults::MutableDefaultsPass::new(source_ref, written, config.is_stub); let unique_loop_bindings_pass = unique_loop_bindings::UniqueLoopBindingsPass::new(source_ref, config.unique_loop_bindings); let auto_quote_pass = auto_quote::AutoQuote::new( @@ -607,7 +1320,7 @@ pub(crate) fn run_against_source<'a>( config.inject_future_annotations, ); let init_method_pass = - init_method::InitMethod::new(source_ref, config.float_literals, config.is_stub); + init_method::InitMethod::new(source_ref, written, config.float_literals, config.is_stub); let properties_pass = properties::PropertiesPass::new(source_ref, accessor_value_ranges); let local_once_pass = local_once::LocalOncePass::new(source_ref); let raises_strip_pass = raises_clause::RaisesStripPass::new(source_ref); @@ -620,11 +1333,20 @@ pub(crate) fn run_against_source<'a>( let type_fn_pass = type_fn::TypeFnPass::new(source_ref); let match_type_pass = match_type::MatchTypePass::new(source_ref); let modifiers_pass = modifiers::ModifiersPass::new(source_ref, config.is_stub); - let main_function_pass = main_function::MainFunction::new(source_ref); + let main_function_pass = main_function::MainFunction::new(source_ref, written); let build_stamps_pass = build_stamps::BuildStampsPass::new(source_ref, config.stamps.clone()); let empty_declarations_pass = empty_declarations::EmptyDeclarations::new(); let overload_pass = overload::Overload::new(source_ref, config.is_stub); - let decorator_keyword_pass = decorator_keyword::DecoratorKeyword::new(source_ref); + let decorator_keyword_pass = decorator_keyword::DecoratorKeyword::new( + source_ref, + written, + config.is_stub, + decorator_keyword::collect_return_types( + parsed_handle.suite(), + &semantic_model, + config.min_version, + ), + ); let unpack_pass = unpack::UnpackSyntax::new(config.clone()); let typed_dict_literal_pass = typed_dict_literal::TypedDictLiteralPass::new(source_ref); let just_float_pass = just_float::JustFloatPass::new(); @@ -638,12 +1360,16 @@ pub(crate) fn run_against_source<'a>( let visibility_rename_pass = visibility_rename::VisibilityRenamePass; let parametric_is_pass = parametric_is::ParametricIsPass::new(source_ref); let implicit_typing_pass = implicit_typing::ImplicitTypingPass::new(); - let inferred_annotation_pass = inferred_annotation::InferredAnnotationPass::new(); + let inferred_annotation_pass = + inferred_annotation::InferredAnnotationPass::new(config.min_version); let template_type_pass = template_type::TemplateTypePass; - let tuple_types_pass = annotation::TupleLiteralTypePass::new(source_ref, config.clone()); + let tuple_types_pass = + annotation::TupleLiteralTypePass::new(source_ref, written, config.clone()); let literal_types_pass = literal_types::LiteralTypePass::new(source_ref, config.float_literals); - let callable_pass = callable::CallableSyntaxPass::new(source_ref, config.float_literals); - let protocol_type_pass = protocol_type::ProtocolTypePass::new(source_ref, config.clone()); + let callable_pass = + callable::CallableSyntaxPass::new(source_ref, written, config.float_literals); + let protocol_type_pass = + protocol_type::ProtocolTypePass::new(source_ref, written, config.clone()); let coalesce_text_pass = coalesce::NoneCoalescePass::new(source_ref); let force_unwrap_pass = force_unwrap::ForceUnwrapPass::new(source_ref); let flexible_keyword_pass = flexible_keyword::FlexibleKeywordPass; @@ -652,8 +1378,8 @@ pub(crate) fn run_against_source<'a>( let none_chain_pass = none_chain::NoneChainPass::new(source_ref); let optional_type_pass = optional_type::OptionalTypePass::new(source_ref, config.min_version); let runtime_union_pass = runtime_union::RuntimeUnionPass::new(config.min_version); - let generics_pass = generics::GenericPolyfillPass::new(source_ref, config.clone()); - let soundness_pass = soundness::SoundnessPass::new(source_ref, config); + let generics_pass = generics::GenericPolyfillPass::new(source_ref, written, config.clone()); + let soundness_pass = soundness::SoundnessPass::new(source_ref, written, config); let checked_cast_pass = checked_cast::CheckedCastPass; let module_api_pass = module_api::ModuleApiPass::new(source_ref); let trailing_lambda_pass = trailing_lambda::TrailingLambdaPass::new(source_ref); @@ -662,7 +1388,8 @@ pub(crate) fn run_against_source<'a>( let destructure_pass = destructure::DestructurePass::new(source_ref); let statement_expression_pass = statement_expression::StatementExpressionPass::new(source_ref); let context_params_pass = context_params::ContextParamsPass::new(source_ref); - let extension_block_pass = extension::ExtensionBlockPass::new(source_ref, config.is_stub); + let extension_block_pass = + extension::ExtensionBlockPass::new(source_ref, written, config.is_stub); let extension_call_pass = extension::ExtensionCallPass; let witness_dispatch_pass = conformance::WitnessDispatchPass; let conversion_pass = conversion::ConversionPass::new(source_ref); @@ -671,7 +1398,7 @@ pub(crate) fn run_against_source<'a>( let frameworks_pass = frameworks::FrameworksPass::new(source_ref); let variance_pass = decl_site_variance::VarianceStripPass::new(source_ref); let anon_named_tuple_pass = - anon_named_tuple::AnonNamedTuplePass::new(source_ref, config.clone()); + anon_named_tuple::AnonNamedTuplePass::new(source_ref, written, config.clone()); // Order matters: passes that read source ranges via `text_edits` mode // must run BEFORE passes that mutate the AST (which zero source ranges @@ -751,25 +1478,28 @@ pub(crate) fn run_against_source<'a>( // first among the mutation passes — before `typeof` and before any // pass that zeroes ranges &symbolic_pass, - &coalesce_pass, &typeof_pass, &sentinel_pass, &repeated_underscore_pass, &typed_lambda_pass, // a static resource import is replaced whole by the document it names, - // in the source *and* in the AST. last, because the statements it - // splices in are parsed from the rendering and carry that text's - // ranges, which name nothing in this file — no pass after it may read - // them. it declares no change of its own, so the source edit is what - // normally lands and the AST rewrite only matters when another pass - // has already forced the statement to be re-rendered + // as an edit of the source &static_resource_pass, ]; + let mut authorship = Authorship::default(); for pass in passes { if config.is_stub && pass.runtime_only() { continue; } pass.run(&mut module, &mut ctx); + authorship.finished( + Author { + name: pass.name(), + lowering: pass.lowering(), + subsumes: pass.subsumes(), + }, + &ctx, + ); } // type-aware passes: operate on the salsa-owned parsed module (so @@ -960,6 +1690,14 @@ pub(crate) fn run_against_source<'a>( continue; } pass.run(parsed_handle.suite(), &semantic_model, &mut ctx); + authorship.finished( + Author { + name: pass.name(), + lowering: pass.lowering(), + subsumes: pass.subsumes(), + }, + &ctx, + ); } // collect import requests the inner passes raised at the end of their run @@ -982,6 +1720,7 @@ pub(crate) fn run_against_source<'a>( // typed lambdas are removed as source deletions so the statement around // them is never re-rendered (see `typed_lambda`); collect them here ctx.text_edits.extend(typed_lambda_inner.take_edits()); + authorship.finished(Author::driver(typed_lambda_pass.name(), None), &ctx); if sentinel_inner.ever_changed() { ctx.required_imports .push("from typing_extensions import Sentinel".to_owned()); @@ -1002,14 +1741,87 @@ pub(crate) fn run_against_source<'a>( ctx.text_edits .push((*range, use_site_variance::collapsed_to(source_ref, *range))); } + authorship.finished( + Author::driver("keyword blanking", Some(Lowering::KeywordPadding)), + &ctx, + ); for (range, replacement) in literal_string_rewrites.edits() { ctx.text_edits.push((range, replacement)); } + authorship.finished(Author::driver("literal string", None), &ctx); if literal_string_rewrites.needs_import { ctx.required_imports .push("from typing import LiteralString".to_owned()); } + // the last thing written into the tree: a soundness check made in a statement an + // `AstPass` rewrote, which the re-render reads rather than the edit list. it happens + // here, ahead of the runtime block below, because the two rewrites that answer for + // what a program reads back read the finished tree and can ask for a helper of their + // own + for (idx, checks) in std::mem::take(&mut ctx.rerendered_checks) { + if let Some(stmt) = module.body.get_mut(idx) + && let Err(error) = super::soundness::place_in_rerendered(stmt, checks) + { + ctx.errors.push(error); + } + } + + // which t-string fields a pass replaced in the syntax tree, read before the `=` field + // rewrite below moves any of them: the tree still holds one field per field the + // author wrote, which is what matches the two up + let fields_replaced_in_tree = + template_expression::replaced_in_tree(&module, &parsed_body, &ctx.changed); + + // every edit that puts something other than what it covers at the span it covers — + // which is to say, every lowering that actually lowered something. the two rewrites + // below read this to decide which replacement fields report a text that is no longer + // the author's + // + // an edit that writes back what it covers is left out. the keyword blanking makes a + // plain one for every marker it collapses, and a type test against `None` on an + // optional makes a template one — `a is not None` is the python spelling of + // `a is None?`'s negation and of what the author wrote. acting on either would print + // the same text out of a longer f-string, which is how a `.py` file's + // `f"{(a if a is not None else b)=}"` stopped surviving a round trip + let rewriting: Vec = ctx + .text_edits + .iter() + .filter(|(range, replacement)| { + source_ref.get(usize::from(range.start())..usize::from(range.end())) + != Some(replacement.as_str()) + }) + .map(|(range, _)| *range) + .chain( + ctx.template_edits + .iter() + .filter(|(range, frags)| rewrites_covered_source(*range, frags, source_ref)) + .map(|(range, _)| *range), + ) + .chain( + ctx.relocating_edits + .iter() + .filter(|(range, frags)| rewrites_covered_source(*range, frags, source_ref)) + .map(|(range, _)| *range), + ) + .chain( + ctx.statement_inserts + .iter() + .map(|(at, _)| TextRange::empty(*at)), + ) + .collect(); + // a t-string hands a reader the source text of every one of its fields, not only a + // `=` one, so a field any lowering reached has to report what the author wrote. which + // templates those are is known only now, from the finished edit list, for the same + // reason the `=` fields above are + let template_wraps = template_expression::claim( + source_ref, + parsed_handle.suite(), + &rewriting, + &fields_replaced_in_tree, + config.min_version >= ruff_python_ast::PythonVersion::PY314, + ); + // a synthesized annotation may name a class the source never imported. the // import goes under `if TYPE_CHECKING:` as one block: every such annotation // is a string at runtime (the lowering always emits the `__future__` import), @@ -1032,6 +1844,12 @@ pub(crate) fn run_against_source<'a>( // nothing from the module, and a synthesized class may name one of its // helpers where it is evaluated at once — `Optional` in the annotation of a // `NamedTuple` field. never sorted: set-up code follows its definition + if template_wraps + .iter() + .any(template_expression::Wrap::rebuilds) + { + ctx.runtime.insert(crate::runtime::TEMPLATE_TEXT); + } if !ctx.runtime.is_empty() { let helpers = ctx.runtime.iter().copied(); match config.runtime_module.as_deref() { @@ -1042,6 +1860,8 @@ pub(crate) fn run_against_source<'a>( } } ctx.required_imports.extend(definitions); + // who rewrote each statement, read before the list is put in order + let rewritten_by = authorship.rewriters(&ctx.changed); ctx.changed.sort_unstable(); ctx.changed.dedup(); @@ -1064,66 +1884,108 @@ pub(crate) fn run_against_source<'a>( // splice for their target idx let mut hoisted_by_idx: std::collections::BTreeMap> = std::collections::BTreeMap::new(); - for (idx, stmt) in ctx.hoisted { + for (idx, stmt) in std::mem::take(&mut ctx.hoisted) { hoisted_by_idx.entry(idx).or_default().push(stmt); } - let original_body = &module.body; - let mut all_idx: std::collections::BTreeSet = ctx.changed.iter().copied().collect(); - for k in hoisted_by_idx.keys() { - all_idx.insert(*k); + // an f-string `=` field prints the source text between its braces, so a field whose + // expression a pass replaced in the syntax tree would print that replacement rather + // than what the author wrote. the field is taken apart here, once the tree is the one + // the re-render will read, and the fields it claimed are kept so the edit-driven half + // below leaves them alone + let (debug_fields_claimed, debug_field_errors) = + debug_field::rewrite_changed(&mut module, &parsed_body, &ctx.changed); + ctx.errors.extend(debug_field_errors); + + let mut edited: Vec = ctx + .text_edits + .iter() + .map(|(range, _)| *range) + .chain(ctx.template_edits.iter().map(|(range, _)| *range)) + .chain(ctx.relocating_edits.iter().map(|(range, _)| *range)) + .chain( + ctx.statement_inserts + .iter() + .map(|(at, _)| TextRange::empty(*at)), + ) + .collect(); + // the other half of the `=` field rewrite: a field whose expression one of the edits + // above rewrites is taken apart over the whole field, so the author's own text is + // printed beside the value rather than in place of it. it comes last because it is + // the finished edit list that says which fields those are + let (debug_field_edits, debug_field_errors) = + debug_field::claim_lowered(parsed_handle.suite(), &rewriting, &debug_fields_claimed); + ctx.errors.extend(debug_field_errors); + edited.extend(debug_field_edits.iter().map(|(range, _)| *range)); + ctx.template_edits.extend(debug_field_edits); + authorship.finished(Author::driver("`=` field rewrite", None), &ctx); + + if !template_wraps.is_empty() { + // a template in a statement the driver re-renders is rebuilt in the tree, which is + // what that re-render reads. everywhere else the literal stays in the source and + // the rebuild is an edit over it, so a lowering inside a field still applies + let rerendered_spans: Vec = ctx + .changed + .iter() + .filter_map(|&idx| original_ranges.get(idx)) + .map(|&(start, end)| { + TextRange::new( + TextSize::try_from(start).unwrap_or_default(), + TextSize::try_from(end).unwrap_or_default(), + ) + }) + .collect(); + // a string tag's literal stays in the source: its call is written there, as an edit + let (in_tree, in_source): (Vec<_>, Vec<_>) = template_wraps.into_iter().partition(|wrap| { + !wrap.tagged + && rerendered_spans + .iter() + .any(|span| span.contains_range(wrap.range)) + }); + template_expression::rewrite_changed(&mut module, &ctx.changed, &in_tree); + for wrap in in_source { + let fragments = wrap.fragments(); + edited.push(wrap.range); + ctx.template_edits.push((wrap.range, fragments)); + } } + authorship.finished(Author::driver("t-string field rewrite", None), &ctx); - // only statements an AST pass actually re-rendered occupy their range — - // a hoist-only target keeps its original text (the hoists are emitted as a - // zero-width insertion before it), so sub-statement edits inside it still - // apply - let occupied_ranges: Vec<(usize, usize)> = - ctx.changed.iter().map(|&i| original_ranges[i]).collect(); - let overlaps = |start: usize, end: usize| -> bool { - occupied_ranges.iter().any(|(s, e)| start < *e && *s < end) - }; + let rerendering = + super::rerender::Rerendering::new(source_ref, &comments, &edited, &indentation); + let mut rerendered: Vec<(TextRange, Vec)> = Vec::new(); + let mut rerendered_by: Vec> = Vec::new(); + for &idx in &ctx.changed { + let (start, end) = original_ranges[idx]; + let range = TextRange::new( + TextSize::try_from(start).unwrap_or_default(), + TextSize::try_from(end).unwrap_or_default(), + ); + let edits = rerendering.edits(&parsed_body[idx], &module.body[idx], range); + let rewriters = rewritten_by.get(&idx).cloned().unwrap_or_default(); + rerendered_by.extend(std::iter::repeat_n(rewriters, edits.len())); + rerendered.extend(edits); + } let mut edits: Vec<(usize, usize, Replacement)> = Vec::new(); - for idx in all_idx.iter().copied() { - let (start, end) = original_ranges[idx]; + for (idx, hoists) in std::mem::take(&mut hoisted_by_idx) { + let (start, _) = original_ranges[idx]; let line_indent = { let prefix = &source_ref[..start]; let line_start = prefix.rfind('\n').map(|i| i + 1).unwrap_or(0); &source_ref[line_start..start] } .to_owned(); - + // inserted ahead of the statement, whose source bytes (and every edit inside + // them) stay where they are let mut block = String::new(); - if let Some(hoists) = hoisted_by_idx.remove(&idx) { - for h in hoists { - let rendered = render_stmt(&h).trim_end_matches('\n').to_owned(); - block.push_str(&rendered); - block.push('\n'); - block.push_str(&line_indent); - } - } - - if ctx.changed.binary_search(&idx).is_ok() { - let rendered = render_stmt(&original_body[idx]); - // render_stmt emits a trailing newline. drop it when the source - // already has one immediately after the stmt (avoids `\n\n`); keep - // it when the stmt is at end-of-file with no trailing newline so - // we don't lose multi-line structure - let source_has_trailing_newline = source_ref.as_bytes().get(end) == Some(&b'\n'); - if source_has_trailing_newline { - block.push_str(rendered.trim_end_matches('\n')); - } else { - block.push_str(&rendered); - } - // a re-rendered statement is printed from its AST, which keeps no - // source ranges: every line of it is charged to the statement - edits.push((start, end, Replacement::generated(&block, start))); - } else if !block.is_empty() { - // hoist-only: insert the hoisted lines before the statement and - // leave its source bytes (and any edits inside them) in place - edits.push((start, start, Replacement::generated(&block, start))); + for h in hoists { + let rendered = render_stmt(&h).trim_end_matches('\n').to_owned(); + block.push_str(&rendered); + block.push('\n'); + block.push_str(&line_indent); } + edits.push((start, start, Replacement::generated(&block, start))); } // ruff-style first-wins dedup for sub-statement edits. sort by start; skip // any edit whose start is before the running cursor (overlaps a prior @@ -1133,7 +1995,19 @@ pub(crate) fn run_against_source<'a>( // can compose. a plain-text edit wins over anything nested inside it; a // template edit instead *materializes* nested edits within its `Src` // passthrough spans, so wide rewrites compose with inner lowerings - let mut sub_edits: Vec<(usize, usize, SubPatch)> = ctx + // who wrote each edit, in the order they are chained below + let origins: Vec> = [ + (EditList::Text, ctx.text_edits.len()), + (EditList::Template, ctx.template_edits.len()), + (EditList::Statement, ctx.statement_inserts.len()), + (EditList::Relocating, ctx.relocating_edits.len()), + ] + .into_iter() + .flat_map(|(list, len)| (0..len).map(move |index| (list, index))) + .map(|(list, index)| authorship.author_of(list, index).into_iter().collect()) + .chain(rerendered_by) + .collect(); + let sub_edits: Vec<(usize, usize, SubPatch)> = ctx .text_edits .into_iter() .map(|(r, s)| { @@ -1161,180 +2035,17 @@ pub(crate) fn run_against_source<'a>( SubPatch::Relocating(frags), ) })) + .chain(rerendered.iter().map(|(r, frags)| { + ( + usize::from(r.start()), + usize::from(r.end()), + SubPatch::Rerendered(frags.clone()), + ) + })) .collect(); - // start asc. tie-break by edit shape: - // 1. zero-width insertions first — they don't consume bytes, so any - // following deletion/replacement at the same start can still apply. - // a statement-anchored insertion leads them: it emits whole statements - // that must precede everything the statement itself lowers to - // 2. then wider replacements before narrower ones — so a wider edit - // wins over (or, for templates, absorbs) a narrow one nested inside - // it - // 3. at one identical span, a *relocating* edit leads: it says the - // construct has moved, and the pass that moved it re-emits the span - // itself, so every other edit there materializes at the new home - // 4. then a *wrapper* — a template whose one passthrough is its whole - // span, adding text around the construct without removing any of it. - // it claims the other edits at that span and materializes them inside - // its passthrough, so whatever the construct becomes ends up inside the - // wrapping (a quoted forward reference around an arrow type the - // callable lowering replaced as text) - // 5. then a *substitution* — plain text, or a template with no `Src` - // passthrough — ahead of a *rewrite*, a template that re-emits part of - // the span. a substitution says the construct does not appear here at - // all, which a rewrite of it cannot outrank - sub_edits.sort_by(|a, b| { - let priority = |e: &(usize, usize, SubPatch)| { - let rewrites = i64::from(match &e.2 { - SubPatch::Text(_) => false, - SubPatch::Template(frags) - | SubPatch::Statement(frags) - | SubPatch::Relocating(frags) => { - frags.iter().any(|frag| matches!(frag, Fragment::Src(_))) - } - }); - let statement = i64::from(!matches!(e.2, SubPatch::Statement(_))); - let relocating = i64::from(!matches!(e.2, SubPatch::Relocating(_))); - let wraps = i64::from( - !matches!(&e.2, SubPatch::Template(frags) if is_wrapper(frags, e.0, e.1)), - ); - // (start, is_replacement_not_insertion, statement-insert-first, - // neg_end-for-wider-first, relocating-first, wrapper-first, - // substitution-before-rewrite) - if e.1 == e.0 { - (e.0, 0i64, statement, 0i64, relocating, wraps, rewrites) // insertion - } else { - #[allow(clippy::cast_possible_wrap)] - let neg_end = -(e.1 as i64); - (e.0, 1i64, statement, neg_end, relocating, wraps, rewrites) - } - }; - priority(a).cmp(&priority(b)) - }); - // claim pre-pass: each replacement, outermost-first (the sort guarantees an - // enclosing edit precedes anything inside it), claims the edits nested in - // its span. a template *materializes* its claimees inside its `Src` spans; - // a plain-text replacement drops them (first-wins). same-start zero-width - // insertions are absorbed by a template (they target the construct's first - // token, e.g. `_force_unwrap(` ahead of a coalesce operand) but stay - // independent ahead of a plain-text replacement, preserving the documented - // insertion + deletion compose behaviour. the one exception is a - // statement-anchored insertion sharing a boundary: it emits statements, so - // absorbing it into an expression rewrite of the statement it precedes - // would splice a suite into the middle of an expression - let mut claimed = vec![false; sub_edits.len()]; - for i in 0..sub_edits.len() { - let (s_i, e_i) = (sub_edits[i].0, sub_edits[i].1); - if e_i == s_i || claimed[i] { - continue; - } - let is_template = matches!( - sub_edits[i].2, - SubPatch::Template(_) | SubPatch::Statement(_) | SubPatch::Relocating(_) - ); - for (m, edit) in sub_edits.iter().enumerate() { - if m == i || claimed[m] { - continue; - } - let (s_m, e_m) = (edit.0, edit.1); - let inside = s_m >= s_i && e_m <= e_i && s_m != e_i; - let boundary_insertion = s_m == e_m && (s_m == s_i || s_m == e_i); - let anchored = matches!(edit.2, SubPatch::Statement(_)); - if inside && (is_template || !boundary_insertion) && !(boundary_insertion && anchored) { - claimed[m] = true; - } - } - } - - let mut dropped_by_splice: Vec<(usize, usize)> = Vec::new(); - let mut cursor = 0usize; - let mut i = 0; - while i < sub_edits.len() { - if claimed[i] { - i += 1; - continue; - } - let (start, end) = (sub_edits[i].0, sub_edits[i].1); - if start < cursor { - i += 1; - continue; - } - if overlaps(start, end) { - if end > start { - dropped_by_splice.push((start, end)); - } - i += 1; - continue; - } - // coalesce all unclaimed zero-width insertions sharing this start into - // a single combined insertion (text concatenated in push order). this - // sidesteps the replace_range-at-same-position ordering issue: each - // pass pushes its slice in left-to-right intent order, and we - // splice them as one contiguous string - if end == start { - let mut combined = Replacement::default(); - let mut j = i; - while j < sub_edits.len() && sub_edits[j].0 == start && sub_edits[j].1 == start { - if !claimed[j] { - match &sub_edits[j].2 { - SubPatch::Text(t) => combined.push_generated(t, start), - SubPatch::Template(frags) - | SubPatch::Statement(frags) - | SubPatch::Relocating(frags) => { - let contained = template_claimees(frags, &sub_edits, &claimed, j, None); - materialize_fragments( - &mut combined, - frags, - source_ref, - &sub_edits, - &contained, - start, - ); - } - } - } - j += 1; - } - edits.push((start, start, combined)); - i = j; - continue; - } - let repl = match &sub_edits[i].2 { - // a plain-text replacement wins over anything inside it - SubPatch::Text(t) => Replacement::generated(t, start), - SubPatch::Template(frags) - | SubPatch::Statement(frags) - | SubPatch::Relocating(frags) => { - // the claimees nested in this span materialize inside the - // template's `Src` passthrough fragments - let contained = - template_claimees(frags, &sub_edits, &claimed, i, Some((start, end))); - let mut out = Replacement::default(); - materialize_fragments(&mut out, frags, source_ref, &sub_edits, &contained, start); - out - } - }; - edits.push((start, end, repl)); - cursor = end; - i += 1; - } - - // composition invariant: an edit dropped because an AST pass re-rendered - // its enclosing statement is fine when the pass consumed the construct, - // but a leak when the re-render reprinted it. detect the leak precisely - // rather than letting it surface as a confusing syntax error downstream - for (start, end) in dropped_by_splice { - let construct = &source_ref[start..end]; - let leaked = edits.iter().any(|(bs, be, replacement)| { - *be > *bs && *bs <= start && end <= *be && replacement.text().contains(construct) - }); - if leaked { - let preview: String = construct.chars().take(40).collect(); - ctx.errors.push(format!( - "transform conflict: `{preview}` was lowered by a sub-statement edit, but an AST pass re-rendered its enclosing statement and the construct leaked into the output" - )); - } - } + let (spliced, lost) = splice(source_ref, sub_edits, origins, &authorship.authors); + edits.extend(spliced); + ctx.errors.extend(lost); // line table for the spliced body, built from the ascending edit list // before the descending application sort consumes it. generated lines from // the import prefix (top) and epilogue (bottom) have no source origin @@ -1352,6 +2063,13 @@ pub(crate) fn run_against_source<'a>( for (start, end, repl) in edits { out.replace_range(start..end, repl.text()); } + // a name the module already imports from the same place goes on its line rather than on + // one of ours. done here and not against `required_imports` alone, because the statement + // it joins is the module's own + if !ctx.required_imports.is_empty() { + ctx.required_imports = + merge_into_own_imports(&mut out, std::mem::take(&mut ctx.required_imports)); + } // an entry may be multi-line (runtime helper defs), so the table prefix // counts the lines each entry emits, not the entries themselves let prefix_lines: usize = ctx @@ -1418,10 +2136,201 @@ mod driver_tests { use super::*; use crate::Config; + /// what [`splice`] reports lost of `edit`, made against `source`, once its first + /// statement is rewritten by `rewrite` and re-emitted + fn lost(source: &str, rewrite: impl FnOnce(&mut Stmt), edit: (TextRange, &str)) -> Vec { + let parsed = ruff_python_parser::parse_module(source).expect("the source parses"); + let comments = ruff_python_trivia::CommentRanges::from(parsed.tokens()); + let module = parsed.into_syntax(); + let original = &module.body[0]; + let mut rewritten = original.clone(); + rewrite(&mut rewritten); + let indentation = Indentation::default(); + let edited = [edit.0]; + let rerendered = + super::super::rerender::Rerendering::new(source, &comments, &edited, &indentation) + .edits(original, &rewritten, original.range()); + let mut sub_edits: Vec<(usize, usize, SubPatch)> = rerendered + .into_iter() + .map(|(range, frags)| { + ( + usize::from(range.start()), + usize::from(range.end()), + SubPatch::Rerendered(frags), + ) + }) + .collect(); + sub_edits.push(( + usize::from(edit.0.start()), + usize::from(edit.0.end()), + SubPatch::Text(edit.1.to_owned()), + )); + let origins = vec![Vec::new(); sub_edits.len()]; + splice(source, sub_edits, origins, &[]).1 + } + + /// the call of `stmt`, an assignment of one + fn assigned_call(stmt: &mut Stmt) -> &mut ruff_python_ast::ExprCall { + let Stmt::Assign(assign) = stmt else { + panic!("an assignment") + }; + let Expr::Call(call) = assign.value.as_mut() else { + panic!("a call") + }; + call + } + + /// a pass that rebuilds an argument prints it from the syntax tree, where `'s'` is + /// spelled `"s"`. the edit a lowering made of `'s'` is in none of the source the + /// statement passes through, and is refused — looking for its construct again in + /// the output, which does not spell it the same way, would have let it go + #[test] + fn an_edit_in_a_construct_a_rewrite_respells_is_refused() { + let source = "x = f('s')\n"; + let argument = TextRange::new(TextSize::new(6), TextSize::new(9)); + let errors = lost( + source, + |stmt| { + let mut rebuilt = ruff_python_parser::parse_expression("\"s\"") + .expect("an expression") + .into_expr(); + super::super::rerender::forget_expr_ranges(&mut rebuilt); + assigned_call(stmt).arguments.args[0] = rebuilt; + }, + (argument, "S"), + ); + assert_eq!( + errors, + [ + "transform conflict: an edit of `'s'` lands inside a node an AST pass rewrote and prints again, so the lowering would be lost" + ] + ); + } + + /// an insertion inside a node a pass rewrote lands only in source the node passes + /// through. the callee is printed from the syntax tree, and the insertion made + /// after the name it replaced has nowhere to go + #[test] + fn an_insertion_with_no_source_to_land_in_is_refused() { + let source = "x = f(a)\n"; + let errors = lost( + source, + |stmt| { + let mut callee = ruff_python_parser::parse_expression("g") + .expect("an expression") + .into_expr(); + super::super::rerender::forget_expr_ranges(&mut callee); + *assigned_call(stmt).func = callee; + }, + (TextRange::empty(TextSize::new(5)), "_inserted"), + ); + assert_eq!( + errors, + [ + "transform conflict: text inserted at byte 5 lands inside a node an AST pass rewrote and prints again, so the lowering would be lost" + ] + ); + } + + /// an edit inside a node the rewritten statement kept is passed through with it + #[test] + fn an_edit_in_source_a_rewrite_passes_through_is_applied() { + let source = "x = f(a)\n"; + let errors = lost( + source, + |stmt| { + let mut callee = ruff_python_parser::parse_expression("g") + .expect("an expression") + .into_expr(); + super::super::rerender::forget_expr_ranges(&mut callee); + *assigned_call(stmt).func = callee; + }, + (TextRange::new(TextSize::new(6), TextSize::new(7)), "b"), + ); + assert_eq!(errors, Vec::::new()); + } + + /// what [`splice`] reports lost of an edit by `inserter` of `", /"` after `b` in + /// `def f(a, b): ...`, once a template by `writer` spells `a, b)` itself + fn lost_in_a_written_parameter_list(writer: Author, inserter: Author) -> Vec { + let source = "def f(a, b): ...\n"; + let parameters = TextRange::new(TextSize::new(6), TextSize::new(11)); + let sub_edits = vec![ + ( + 6, + 11, + SubPatch::Template(vec![Fragment::Lit("a, b)".to_owned())]), + ), + (10, 10, SubPatch::Text(", /".to_owned())), + ]; + assert_eq!(&source[parameters.to_std_range()], "a, b)"); + splice( + source, + sub_edits, + vec![vec![0], vec![1]], + &[writer, inserter], + ) + .1 + } + + fn author(lowering: Option, subsumes: &'static [Lowering]) -> Author { + Author { + name: "Pass", + lowering, + subsumes, + } + } + + /// a template that writes a construct itself prints none of the source an edit + /// inside it was made against, so the edit is refused — whether or not the text the + /// template writes happens to agree with it + #[test] + fn an_insertion_inside_a_construct_a_template_writes_is_refused() { + let errors = lost_in_a_written_parameter_list( + author(None, &[]), + author(Some(Lowering::RepeatedUnderscore), &[]), + ); + assert_eq!( + errors, + [ + "transform conflict: text inserted at byte 10 by `Pass` lands inside a construct `Pass` writes itself, so the lowering would be lost" + ] + ); + } + + /// a pass that says it writes a lowering's constructs itself takes that lowering's + /// edits inside its own + #[test] + fn a_lowering_the_template_writes_itself_is_accounted_for() { + let errors = lost_in_a_written_parameter_list( + author(None, &[Lowering::RepeatedUnderscore]), + author(Some(Lowering::RepeatedUnderscore), &[]), + ); + assert_eq!(errors, Vec::::new()); + } + + /// what a deletion covers is not printed at all, so nothing inside it is lost + #[test] + fn a_deletion_takes_the_edits_inside_it() { + let source = "x: int? = 1\n"; + let sub_edits = vec![ + (1, 7, SubPatch::Text(String::new())), + (6, 7, SubPatch::Text(" | None".to_owned())), + ]; + let authors = [author(None, &[]), author(Some(Lowering::OptionalType), &[])]; + let errors = splice(source, sub_edits, vec![vec![0], vec![1]], &authors).1; + assert_eq!(errors, Vec::::new()); + } + #[test] fn double_coalesce_spliced() { let src = "x = None\na = x ?? x ?? \"fallback\"\n"; - let (out, _, _) = run_against_source(src, &Config::test_default(), None); + let (out, _, _) = run_against_source( + src, + repeated_underscore::WrittenNames::new(src), + &Config::test_default(), + None, + ); assert!(!out.contains("??"), "still has ??: {out}"); } @@ -1461,7 +2370,7 @@ mod driver_tests { Fragment::Lit("Y".to_owned()), ]; let mut out = Replacement::default(); - materialize_fragments(&mut out, &frags, source, &all, &[0], 0); + materialize_fragments(&mut out, &frags, source, &all, &[0], 0, &mut [false]); assert_eq!(out.text(), "[1])Y"); } @@ -1477,7 +2386,7 @@ mod driver_tests { Fragment::Src(TextRange::new(TextSize::from(3u32), TextSize::from(4u32))), ]; let mut out = Replacement::default(); - materialize_fragments(&mut out, &frags, source, &all, &[0], 0); + materialize_fragments(&mut out, &frags, source, &all, &[0], 0, &mut [false]); assert_eq!(out.text(), "[1])W"); } } diff --git a/crates/by_transforms/src/transforms/callable.rs b/crates/by_transforms/src/transforms/callable.rs index e12e31d789..8a47207ced 100644 --- a/crates/by_transforms/src/transforms/callable.rs +++ b/crates/by_transforms/src/transforms/callable.rs @@ -28,24 +28,41 @@ use ruff_python_ast::helpers::{is_classvar_marker_id, is_final_marker_id, is_let use ruff_python_ast::{Expr, ExprCallableType, Stmt, UnaryOp}; use ruff_text_size::{Ranged, TextRange}; -use super::ast_driver::{PassContext, TypeAwarePass}; +use super::ast_driver::{Lowering, PassContext, TypeAwarePass}; use super::intersection::{collect_intersect, collect_union, is_intersection_node}; use super::just_float::rewrite_type_expr_with_imports; +use super::repeated_underscore::{WrittenNames, lowered_callable_parameters}; use crate::config::FloatLiteralLowering; use crate::type_info::{TypeInfo, UnpackedKwargsLowering}; +/// the lowerings [`CallableSyntax::lower_type_expr`] writes itself, wherever it prints a type +/// expression. a pass that prints one through it in place of the source writes these too +pub(crate) const TYPE_EXPRESSION: &[Lowering] = &[ + Lowering::Callable, + Lowering::OptionalType, + Lowering::LiteralType, + Lowering::JustFloat, + Lowering::DynamicKeyword, + Lowering::FloatConst, +]; + #[expect( clippy::struct_excessive_bools, reason = "transform flags toggled across visit" )] pub(crate) struct CallableSyntax<'src> { source: &'src str, + /// the module as written, whose names a repeated `_` is numbered around + written: WrittenNames<'src>, types: Option<&'src (dyn TypeInfo + 'src)>, /// ranges already folded by `symbolic_type_op` (e.g. `1 + typeof d` → /// `Literal[3]`). a claimed sub-expression is opaque here: descending into /// it would re-render its `typeof`/operator surface and the wider edit /// would clobber the fold, so `rewrite` leaves it for the fold's own edit claimed_ranges: &'src [TextRange], + /// the `typeof` nodes a type position may hold that this lowerer owns, when the + /// `typeof` fold lowers the rest in the syntax tree. `None` owns every one + owned_typeof: Option<&'src [TextRange]>, float_literals: FloatLiteralLowering, edits: Vec, needs_import: bool, @@ -83,11 +100,17 @@ struct ProtocolShape { } impl<'src> CallableSyntax<'src> { - pub(crate) fn new(source: &'src str, float_literals: FloatLiteralLowering) -> Self { + pub(crate) fn new( + source: &'src str, + written: WrittenNames<'src>, + float_literals: FloatLiteralLowering, + ) -> Self { Self { source, + written, types: None, claimed_ranges: &[], + owned_typeof: None, float_literals, edits: Vec::new(), needs_import: false, @@ -130,6 +153,12 @@ impl<'src> CallableSyntax<'src> { self } + /// leave to the `typeof` fold every `typeof` a type position holds outside `owned` + pub(crate) fn with_owned_typeof(mut self, owned: &'src [TextRange]) -> Self { + self.owned_typeof = Some(owned); + self + } + pub(crate) fn class_defs(&self) -> &str { &self.protocol_class_defs } @@ -324,21 +353,30 @@ impl<'src> CallableSyntax<'src> { /// top, ahead of any user class its annotations mention, and an unquoted /// annotation would be evaluated at class-body time and `NameError` /// - /// `args` / `slash` / `star` are passed separately rather than read off a - /// node so a protocol method member can hand over the parameters that - /// follow its receiver, which is a name rather than a type. + /// `method` says `ct` is the signature of a protocol method member, whose + /// first argument is its receiver: a name rather than a type, emitted as + /// `receiver`. a plain callable type is given the synthesized `receiver`. /// /// `implicit_receiver` is the rendered leading parameter of a basedpython /// implicit receiver (`int.() -> str`), which is a type rather than a name and /// so is spelled by a synthesized parameter. + /// + /// a labelled parameter is declared under the name the lowering gives it, + /// and a repeated `_` makes the parameters up to the last of them + /// positional-only, as it does in a `def` — both ty's answer too pub(crate) fn render_protocol_params( &mut self, - args: &[Expr], - explicit_slash: Option, - star: Option, + ct: &ExprCallableType, + method: bool, receiver: &str, implicit_receiver: Option, ) -> String { + let lowered = lowered_callable_parameters(ct, method, self.written); + let offset = ct.args.len() - lowered.names.len(); + let args = &ct.args[offset..]; + let shift = |index: Option| index.map(|i| (i as usize).saturating_sub(offset)); + let explicit_slash = shift(ct.parameter_slash()); + let star = shift(ct.parameter_star()); let mut parts: Vec = vec![receiver.to_owned()]; // implicit `/` after the last bare positional (no label) when followed // by a named/labelled parameter. bare positionals are positional-only @@ -359,7 +397,10 @@ impl<'src> CallableSyntax<'src> { } }) }; - let slash = explicit_slash.or(implicit_slash); + let slash = [explicit_slash.or(implicit_slash), lowered.slash] + .into_iter() + .flatten() + .max(); // an implicit receiver leads the parameter list, positional-only. any `/` // the arguments themselves emit comes after the receiver and so already // closes it off — a second one is a `SyntaxError` @@ -369,6 +410,7 @@ impl<'src> CallableSyntax<'src> { parts.push("/".to_owned()); } } + let names = lowered.names; let mut star_emitted = false; for (i, arg) in args.iter().enumerate() { if Some(i) == slash { @@ -384,31 +426,16 @@ impl<'src> CallableSyntax<'src> { } match arg { Expr::Named(named) => { + let name = &names[i]; let name = match named.target.as_ref() { - Expr::Name(n) => n.id.as_str().to_owned(), Expr::Starred(s) => match s.value.as_ref() { - Expr::Starred(inner_inner) => { - let n = inner_inner - .value - .as_name_expr() - .map(|n| n.id.as_str()) - .unwrap_or("kwargs"); - format!("**{n}") - } + Expr::Starred(_) => format!("**{name}"), _ => { - // the anonymous `*: *Ts` carries the empty name - // marker, and needs a name of its own in python - let n = s - .value - .as_name_expr() - .map(|n| n.id.as_str()) - .filter(|n| !n.is_empty()) - .unwrap_or("args"); star_emitted = true; - format!("*{n}") + format!("*{name}") } }, - _ => "_".to_owned(), + _ => name.to_string(), }; let ty = quote_forward_ref(&self.rewrite_or_leaf(&named.value)); parts.push(format!("{name}: {ty}")); @@ -457,12 +484,8 @@ impl<'src> CallableSyntax<'src> { } }, _ => { - // bare positional type — Protocol's `__call__` needs a - // parameter NAME, so we synthesize one. Use an - // unused-prefixed name so static checkers don't flag - // it as a missing arg let ty = quote_forward_ref(&self.rewrite_or_leaf(arg)); - parts.push(format!("_{i}: {ty}")); + parts.push(format!("{}: {ty}", names[i])); } } } @@ -546,13 +569,7 @@ impl<'src> CallableSyntax<'src> { quote_forward_ref(&rendered) ) }); - let params = self.render_protocol_params( - &ct.args, - ct.parameter_slash().map(|i| i as usize), - ct.parameter_star().map(|i| i as usize), - "self", - implicit_receiver, - ); + let params = self.render_protocol_params(ct, false, "self", implicit_receiver); let returns = quote_forward_ref(&self.rewrite_or_leaf(&ct.returns)); let shape = ProtocolShape { params, returns }; Some(self.class_name_for(shape)) @@ -826,12 +843,13 @@ fn is_named(expr: &Expr, ident: &str) -> bool { /// rendering path, which is what lets one wide edit carry rewrites it subsumes. pub(crate) fn lower_type_expr_full( source: &str, + written: WrittenNames, types: &dyn TypeInfo, expr: &Expr, substitutions: &[(TextRange, String)], float_literals: FloatLiteralLowering, ) -> Option { - let mut inner = CallableSyntax::new(source, float_literals).with_types(types); + let mut inner = CallableSyntax::new(source, written, float_literals).with_types(types); for (range, name) in substitutions { inner.add_substitution(*range, name.clone()); } @@ -904,6 +922,17 @@ impl crate::transforms::type_expr_walker::TypeExprVisitor for CallableSyntax<'_> if matches!(expr, Expr::UnaryOp(u) if u.op == UnaryOp::Optional) { return crate::transforms::type_expr_walker::Recurse::Descend; } + // a `typeof` the fold rewrites in the syntax tree is re-emitted from there, and + // an edit of our own at the same node would be a second lowering of it that + // nothing emits + if let Expr::Subscript(s) = expr + && s.is_typeof + && self + .owned_typeof + .is_some_and(|owned| !owned.contains(&s.range)) + { + return crate::transforms::type_expr_walker::Recurse::Stop; + } // `rewrite` is the single type-expression lowerer — it owns every // structural type-form (callable arrows, `&` / `and`, `or`, `not`, // `typeof`, subscripts) and composes leaves through `lower_leaf`. it @@ -920,13 +949,19 @@ impl crate::transforms::type_expr_walker::TypeExprVisitor for CallableSyntax<'_> pub(crate) struct CallableSyntaxPass<'src> { source: &'src str, + written: WrittenNames<'src>, float_literals: FloatLiteralLowering, } impl<'src> CallableSyntaxPass<'src> { - pub(crate) fn new(source: &'src str, float_literals: FloatLiteralLowering) -> Self { + pub(crate) fn new( + source: &'src str, + written: WrittenNames<'src>, + float_literals: FloatLiteralLowering, + ) -> Self { Self { source, + written, float_literals, } } @@ -963,13 +998,33 @@ impl<'ast> ruff_python_ast::visitor::Visitor<'ast> for ValueCallableWalker<'_, ' } impl TypeAwarePass for CallableSyntaxPass<'_> { + fn lowering(&self) -> Option { + Some(Lowering::Callable) + } + + /// a type expression it replaces is printed by the shared lowerer, which writes every + /// leaf inside it too, an inline protocol as the name of the class it hoists to + fn subsumes(&self) -> &'static [Lowering] { + &[ + Lowering::Callable, + Lowering::OptionalType, + Lowering::LiteralType, + Lowering::JustFloat, + Lowering::DynamicKeyword, + Lowering::FloatConst, + Lowering::ProtocolType, + ] + } + fn run(&self, stmts: &[Stmt], types: &dyn TypeInfo, ctx: &mut PassContext) { // owned copy so `inner`'s borrow doesn't pin `ctx` against the mutable // `required_imports` / `edits` uses below let claimed = ctx.claimed_type_op_ranges.clone(); - let mut inner = CallableSyntax::new(self.source, self.float_literals) + let structural_typeof = ctx.structural_typeof_ranges.clone(); + let mut inner = CallableSyntax::new(self.source, self.written, self.float_literals) .with_types(types) - .with_claimed_ranges(&claimed); + .with_claimed_ranges(&claimed) + .with_owned_typeof(&structural_typeof); crate::transforms::type_expr_walker::walk_type_positions_skipping( stmts, Some(types), @@ -1022,6 +1077,55 @@ mod tests { ); } + /// a callable type naming its parameters is declared as a `__call__` in the + /// synthesized protocol, where a repeated `_` is numbered and made positional-only as + /// it is in any `def` + #[test] + fn a_repeated_underscore_is_numbered_in_the_synthesized_call() { + let out = transpile( + "a: (_: int, _: str, name: bytes) -> None\n", + &Config::test_default(), + ) + .unwrap(); + assert!( + out.contains( + "def __call__(self, _: \"int\", _2: \"str\", /, name: \"bytes\") -> \"None\": ..." + ), + "got:\n{out}" + ); + } + + /// the name a bare positional is given is taken, and numbering skips it + #[test] + fn a_numbered_underscore_skips_a_bare_positional_name() { + let out = transpile( + "a: (int, int, int, _: int, _: str) -> None\n", + &Config::test_default(), + ) + .unwrap(); + assert!( + out.contains( + "def __call__(self, _0: \"int\", _1: \"int\", _2: \"int\", _: \"int\", _3: \"str\", /) -> \"None\": ..." + ), + "got:\n{out}" + ); + } + + /// a numbered `_` after the `*` would be reachable by its number alone, a name the + /// author never wrote, so the callable type is refused as a `def` would be + #[test] + fn a_keyword_only_repeated_underscore_is_refused() { + let error = transpile( + "a: (int, /, _: int, *args: int, _: str) -> None\n", + &Config::test_default(), + ) + .unwrap_err(); + assert!( + error.contains("a repeated `_` parameter cannot be keyword-only in a callable type"), + "got:\n{error}" + ); + } + #[test] fn simple_callable() { check( diff --git a/crates/by_transforms/src/transforms/coalesce.rs b/crates/by_transforms/src/transforms/coalesce.rs index 27dafe4d87..acbcd0ffdd 100644 --- a/crates/by_transforms/src/transforms/coalesce.rs +++ b/crates/by_transforms/src/transforms/coalesce.rs @@ -3,6 +3,7 @@ use ruff_python_ast::{Expr, Operator, Stmt}; use ruff_text_size::{Ranged, TextRange}; use super::ast_driver::{Fragment, PassContext, TypeAwarePass}; +use super::repeated_underscore::WrittenNames; use crate::type_info::TypeInfo; /// rewrites `a ?? b` to `a if a is not None else b`. @@ -31,7 +32,7 @@ fn expand_none_chain(expr: &Expr, source: &str, types: &dyn TypeInfo) -> Option< Some(super::none_chain::build_expansion( &guards, &form, - &super::none_chain::temp_var(0), + &super::none_chain::temp_var(WrittenNames::new(source), 0), base, )) } @@ -82,7 +83,7 @@ impl NoneCoalesce<'_> { } else { "" }; - let temp = super::none_chain::temp_var(0); + let temp = super::none_chain::temp_var(WrittenNames::new(self.source), 0); match expand_none_chain(&b.left, self.source, self.types) { Some(expanded) => { let mut t = vec![Fragment::Lit(format!("{temp}{unwrap} if ({temp} := "))]; @@ -174,6 +175,15 @@ impl<'src> NoneCoalescePass<'src> { } impl TypeAwarePass for NoneCoalescePass<'_> { + fn lowering(&self) -> Option { + Some(super::ast_driver::Lowering::NoneCoalesce) + } + + /// an optional chain on the left of a `??` is written into the coalesce's own test + fn subsumes(&self) -> &'static [super::ast_driver::Lowering] { + &[super::ast_driver::Lowering::NoneChain] + } + fn run(&self, stmts: &[Stmt], types: &dyn TypeInfo, ctx: &mut PassContext) { let mut inner = NoneCoalesce::new(self.source, types); for stmt in stmts { @@ -284,6 +294,77 @@ mod tests { ); } + /// a parameter's default is a value like any other, and the `??` in one lowers + /// here. a second pass used to claim it for an AST rewrite of the whole `def`, + /// which printed the signature again over this edit and left the program refused + #[test] + fn coalesce_in_a_parameter_default() { + check( + indoc::indoc! {" + g: int? = None + + def f(y: int = g ?? 1) -> int: + return y + "}, + indoc::indoc! {" + from typing import Any + _MISSING: Any = object() + g: int | None = None + + def f(y: int = _MISSING) -> int: + if y is _MISSING: + y = g if g is not None else 1 + return y + "}, + ); + } + + /// the same default with the body written on the clause's line + #[test] + fn coalesce_in_a_parameter_default_with_an_inline_body() { + check( + indoc::indoc! {" + g: int? = None + + def f(y: int = g ?? 1) -> int: return y + "}, + indoc::indoc! {" + from typing import Any + _MISSING: Any = object() + g: int | None = None + + def f(y: int = _MISSING) -> int: \n if y is _MISSING: + y = g if g is not None else 1 + return y + "}, + ); + } + + /// and a chain longer than one link, which recurses in the same edit + #[test] + fn a_chained_coalesce_in_a_parameter_default() { + check( + indoc::indoc! {" + g: int? = None + h: int? = None + + def f(y: int = g ?? h ?? 1) -> int: + return y + "}, + indoc::indoc! {" + from typing import Any + _MISSING: Any = object() + g: int | None = None + h: int | None = None + + def f(y: int = _MISSING) -> int: + if y is _MISSING: + y = __by_t_0__ if (__by_t_0__ := g if g is not None else h) is not None else 1 + return y + "}, + ); + } + #[test] fn python_unchanged() { let src = "x = a if a is not None else b\n"; diff --git a/crates/by_transforms/src/transforms/coalesce_chain.rs b/crates/by_transforms/src/transforms/coalesce_chain.rs deleted file mode 100644 index d7c6898b02..0000000000 --- a/crates/by_transforms/src/transforms/coalesce_chain.rs +++ /dev/null @@ -1,487 +0,0 @@ -//! AST rewrite for the `??` (none-coalesce) operator. -//! -//! Replaces the text-edit pass in `crate::transforms::coalesce` for the -//! cases where embedding the LHS source verbatim caused inner basedpython -//! constructs to leak. -//! -//! Rewrites: -//! -//! - `a ?? b` → `a if a is not None else b` (when `a` is side-effect-free) -//! - `a ?? b` → `_t if (_t := a) is not None else b` (otherwise) -//! - non-`None` literal LHS short-circuits to the LHS alone (no comparison) -//! -//! Post-order: child `??` operands are rewritten first, so chained -//! `x ?? x ?? x ?? "fallback"` is fully expanded. - -use std::cell::Cell; - -use ruff_python_ast::name::Name; -use ruff_python_ast::visitor::transformer::{Transformer, walk_expr}; -use ruff_python_ast::{ - AtomicNodeIndex, CmpOp, Expr, ExprCompare, ExprContext, ExprIf, ExprName, ExprNamed, - ExprNoneLiteral, Operator, Stmt, -}; -use ruff_text_size::TextRange; - -pub(crate) struct CoalesceFold { - changed: Cell, - ever_changed: Cell, - engaged: Cell, -} - -impl CoalesceFold { - pub(crate) fn new() -> Self { - Self { - changed: Cell::new(false), - ever_changed: Cell::new(false), - engaged: Cell::new(false), - } - } - - pub(crate) fn changed_cell(&self) -> &Cell { - &self.changed - } - - #[allow(dead_code)] - pub(crate) fn ever_changed(&self) -> bool { - self.ever_changed.get() - } -} - -impl Transformer for CoalesceFold { - fn visit_stmt(&self, stmt: &mut Stmt) { - let prev = self.engaged.get(); - self.engaged.set(statement_needs_ast_rewrite(stmt)); - ruff_python_ast::visitor::transformer::walk_stmt(self, stmt); - self.engaged.set(prev); - } - - fn visit_expr(&self, expr: &mut Expr) { - walk_expr(self, expr); - - if !self.engaged.get() { - return; - } - - let Expr::BinOp(b) = expr else { return }; - if !matches!(b.op, Operator::Coalesce) { - return; - } - if contains_optional_attribute(&b.left) { - return; - } - - let placeholder = || { - Expr::NoneLiteral(ExprNoneLiteral { - node_index: AtomicNodeIndex::NONE, - range: TextRange::default(), - }) - }; - let left = std::mem::replace(b.left.as_mut(), placeholder()); - let right = std::mem::replace(b.right.as_mut(), placeholder()); - - let new_expr = build_coalesce(left, right); - *expr = new_expr; - self.changed.set(true); - self.ever_changed.set(true); - } -} - -/// Return true when this statement contains a `??` configuration that the -/// legacy text-edit `coalesce` transform can't lower correctly. Specifically: -/// - chained `??` (`a ?? b ?? c`) — text edit emits LHS source verbatim, -/// stranding the inner `??` -/// - `??` inside a function parameter default — text-edit visitor doesn't -/// descend into Parameters -/// -/// When this returns false, the AST pass is a no-op for the statement and -/// the text-edit transform owns the rewrite. This preserves the legacy -/// `?.` + `??` interaction (`transforms::coalesce::expand_none_chain`) -/// which the AST pass can't currently reproduce (ruff's [`Generator`] -/// doesn't emit the basedpython `?.` form) -fn statement_needs_ast_rewrite(stmt: &Stmt) -> bool { - let mut state = State { - in_parameter_default: false, - needs: false, - }; - scan_stmt(stmt, &mut state); - state.needs -} - -struct State { - in_parameter_default: bool, - needs: bool, -} - -fn scan_stmt(stmt: &Stmt, state: &mut State) { - if state.needs { - return; - } - match stmt { - Stmt::FunctionDef(f) => { - for p in f - .parameters - .posonlyargs - .iter() - .chain(f.parameters.args.iter()) - .chain(f.parameters.kwonlyargs.iter()) - { - if let Some(d) = p.default.as_deref() { - let prev = state.in_parameter_default; - state.in_parameter_default = true; - scan_expr(d, state); - state.in_parameter_default = prev; - } - if let Some(ann) = p.parameter.annotation.as_deref() { - scan_expr(ann, state); - } - } - if let Some(ret) = f.returns.as_deref() { - scan_expr(ret, state); - } - for s in &f.body { - scan_stmt(s, state); - } - for dec in &f.decorator_list { - scan_expr(&dec.expression, state); - } - } - Stmt::ClassDef(c) => { - if let Some(args) = c.arguments.as_deref() { - for a in &args.args { - scan_expr(a, state); - } - } - for s in &c.body { - scan_stmt(s, state); - } - } - Stmt::Expr(e) => scan_expr(&e.value, state), - Stmt::Assign(a) => { - for t in &a.targets { - scan_expr(t, state); - } - scan_expr(&a.value, state); - } - Stmt::AnnAssign(a) => { - scan_expr(&a.target, state); - scan_expr(&a.annotation, state); - if let Some(v) = a.value.as_deref() { - scan_expr(v, state); - } - } - Stmt::Return(r) => { - if let Some(v) = r.value.as_deref() { - scan_expr(v, state); - } - } - Stmt::If(i) => { - scan_expr(&i.test, state); - for s in &i.body { - scan_stmt(s, state); - } - for e in &i.elif_else_clauses { - if let Some(t) = e.test.as_ref() { - scan_expr(t, state); - } - for s in &e.body { - scan_stmt(s, state); - } - } - } - Stmt::For(f) => { - scan_expr(&f.iter, state); - for s in &f.body { - scan_stmt(s, state); - } - for s in &f.orelse { - scan_stmt(s, state); - } - } - Stmt::While(w) => { - scan_expr(&w.test, state); - for s in &w.body { - scan_stmt(s, state); - } - for s in &w.orelse { - scan_stmt(s, state); - } - } - Stmt::With(w) => { - for item in &w.items { - scan_expr(&item.context_expr, state); - } - for s in &w.body { - scan_stmt(s, state); - } - } - Stmt::Try(t) => { - for s in &t.body { - scan_stmt(s, state); - } - for s in &t.orelse { - scan_stmt(s, state); - } - for s in &t.finalbody { - scan_stmt(s, state); - } - } - _ => {} - } -} - -fn scan_expr(expr: &Expr, state: &mut State) { - if state.needs { - return; - } - // only `??` inside a function parameter default needs the AST rewrite: the - // text-edit `coalesce` transform's visitor doesn't descend into Parameters. - // chained `??` in statement bodies is handled by that text-edit pass - // (recursively), which — unlike a whole-statement AST re-render — composes - // with sibling text-lowered syntax (`int?` annotations, `?.`, …) in the - // same statement - if let Expr::BinOp(b) = expr - && matches!(b.op, Operator::Coalesce) - && !contains_optional_attribute(&b.left) - && state.in_parameter_default - { - state.needs = true; - return; - } - match expr { - Expr::BinOp(b) => { - scan_expr(&b.left, state); - scan_expr(&b.right, state); - } - Expr::UnaryOp(u) => scan_expr(&u.operand, state), - Expr::If(i) => { - scan_expr(&i.test, state); - scan_expr(&i.body, state); - scan_expr(&i.orelse, state); - } - Expr::Call(c) => { - scan_expr(&c.func, state); - for a in &c.arguments.args { - scan_expr(a, state); - } - for kw in &c.arguments.keywords { - scan_expr(&kw.value, state); - } - } - Expr::Subscript(s) => { - scan_expr(&s.value, state); - scan_expr(&s.slice, state); - } - Expr::Attribute(a) => scan_expr(&a.value, state), - Expr::Tuple(t) => { - for e in &t.elts { - scan_expr(e, state); - } - } - Expr::List(l) => { - for e in &l.elts { - scan_expr(e, state); - } - } - Expr::Set(s) => { - for e in &s.elts { - scan_expr(e, state); - } - } - Expr::BoolOp(b) => { - for e in &b.values { - scan_expr(e, state); - } - } - Expr::Lambda(l) => scan_expr(&l.body, state), - Expr::Named(n) => scan_expr(&n.value, state), - Expr::Compare(c) => { - scan_expr(&c.left, state); - for comp in &c.comparators { - scan_expr(comp, state); - } - } - _ => {} - } -} - -fn contains_optional_attribute(expr: &Expr) -> bool { - match expr { - Expr::Attribute(a) => a.optional || contains_optional_attribute(&a.value), - Expr::Subscript(s) => { - contains_optional_attribute(&s.value) || contains_optional_attribute(&s.slice) - } - Expr::BinOp(b) => { - contains_optional_attribute(&b.left) || contains_optional_attribute(&b.right) - } - Expr::UnaryOp(u) => contains_optional_attribute(&u.operand), - Expr::Call(c) => { - contains_optional_attribute(&c.func) - || c.arguments.args.iter().any(contains_optional_attribute) - } - _ => false, - } -} - -/// Construct the `if`/`else` expression that replaces `lhs ?? rhs`. -fn build_coalesce(lhs: Expr, rhs: Expr) -> Expr { - // literal LHS that is statically non-None collapses to the LHS alone — - // matches the existing text-edit behaviour and avoids CPython's - // `1 is not None` SyntaxWarning - if is_non_none_literal(&lhs) { - return lhs; - } - - if is_side_effect_free(&lhs) { - // safe to repeat: `lhs if lhs is not None else rhs` - let cmp = compare_is_not_none(lhs.clone()); - return Expr::If(ExprIf { - node_index: AtomicNodeIndex::NONE, - range: TextRange::default(), - test: Box::new(cmp), - body: Box::new(lhs), - orelse: Box::new(rhs), - }); - } - - // otherwise walrus into a temp: `t if (t := lhs) is not None else rhs`. - // NB: name choice mirrors the legacy text-edit transform; the surrounding - // scope cannot have a pre-existing binding of it because the legacy pass - // already tags those calls. AST-level scope analysis is future work - let temp_name = super::none_chain::temp_var(0); - let temp_name = temp_name.as_str(); - let target = Expr::Name(ExprName { - node_index: AtomicNodeIndex::NONE, - range: TextRange::default(), - id: Name::from(temp_name), - ctx: ExprContext::Store, - }); - let read_target = Expr::Name(ExprName { - node_index: AtomicNodeIndex::NONE, - range: TextRange::default(), - id: Name::from(temp_name), - ctx: ExprContext::Load, - }); - let walrus = Expr::Named(ExprNamed { - node_index: AtomicNodeIndex::NONE, - range: TextRange::default(), - target: Box::new(target), - value: Box::new(lhs), - }); - let cmp = compare_is_not_none(walrus); - Expr::If(ExprIf { - node_index: AtomicNodeIndex::NONE, - range: TextRange::default(), - test: Box::new(cmp), - body: Box::new(read_target), - orelse: Box::new(rhs), - }) -} - -fn compare_is_not_none(left: Expr) -> Expr { - Expr::Compare(ExprCompare { - node_index: AtomicNodeIndex::NONE, - range: TextRange::default(), - left: Box::new(left), - ops: Box::new([CmpOp::IsNot]), - // python identity against `None`, written `is not` — the lowering emits - // python, so it never wants basedpython's `!==` spelling - identity_ops: None, - comparators: Box::new([Expr::NoneLiteral(ExprNoneLiteral { - node_index: AtomicNodeIndex::NONE, - range: TextRange::default(), - })]), - }) -} - -/// Literals whose value cannot be `None`. `??` on these collapses to the LHS. -fn is_non_none_literal(expr: &Expr) -> bool { - match expr { - Expr::NumberLiteral(_) - | Expr::StringLiteral(_) - | Expr::BytesLiteral(_) - | Expr::BooleanLiteral(_) - | Expr::FString(_) => true, - Expr::UnaryOp(u) => is_non_none_literal(&u.operand), - _ => false, - } -} - -/// Conservative side-effect check. A Name read or member access against a -/// Name is safe to evaluate twice; anything that could call user code (Call, -/// Subscript, Yield, ...) goes through the walrus form -fn is_side_effect_free(expr: &Expr) -> bool { - match expr { - Expr::Name(_) | Expr::NoneLiteral(_) | Expr::EllipsisLiteral(_) => true, - Expr::NumberLiteral(_) - | Expr::StringLiteral(_) - | Expr::BytesLiteral(_) - | Expr::BooleanLiteral(_) => true, - Expr::Attribute(a) => is_side_effect_free(&a.value), - _ => false, - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::transforms::ast_driver::render_stmt; - use ruff_python_ast::PySourceType; - use ruff_python_ast::visitor::transformer::Transformer; - use ruff_python_parser::parse_unchecked_source; - - fn rewrite(source: &str) -> String { - let parsed = parse_unchecked_source(source, PySourceType::BasedPython); - let mut module = parsed.into_syntax(); - let pass = CoalesceFold::new(); - let mut any_changed = false; - for stmt in &mut module.body { - pass.changed_cell().set(false); - pass.visit_stmt(stmt); - if pass.changed_cell().get() { - any_changed = true; - } - } - if !any_changed { - return source.to_owned(); - } - module.body.iter().map(render_stmt).collect::() - } - - // simple `??` (non-chained, not in parameter default) deliberately - // falls through to the text-edit transform — gated by - // `statement_needs_ast_rewrite`. The AST pass tests here cover the - // cases where the gate fires (chained, default-arg) - - // chained `??` in a statement body is now owned by the text-edit `coalesce` - // transform (it composes with sibling text-lowered syntax in the same - // statement, which a whole-statement AST re-render cannot). these go through - // the full pipeline; the AST `CoalesceFold` only fires for parameter defaults - #[test] - fn chained_safe_lhs_end_to_end() { - let out = crate::transpile("x = a ?? b ?? c\n", &crate::Config::test_default()).unwrap(); - assert!(!out.contains("??"), "still has ??: {out}"); - } - - #[test] - fn chained_coalesce_fully_expanded_end_to_end() { - let out = crate::transpile( - "x = a ?? b ?? c ?? \"fallback\"\n", - &crate::Config::test_default(), - ) - .unwrap(); - assert!(!out.contains("??"), "leftover ??: {out}"); - } - - #[test] - fn coalesce_in_default_value() { - let out = rewrite("def f(a = x ?? 0):\n return a\n"); - assert!(!out.contains("??"), "leftover ??: {out}"); - } - - #[test] - fn no_coalesce_unchanged() { - let out = rewrite("x = a + b\n"); - assert_eq!(out, "x = a + b\n"); - } -} diff --git a/crates/by_transforms/src/transforms/context_params.rs b/crates/by_transforms/src/transforms/context_params.rs index 19b6447e15..5dc0d58001 100644 --- a/crates/by_transforms/src/transforms/context_params.rs +++ b/crates/by_transforms/src/transforms/context_params.rs @@ -30,6 +30,19 @@ //! a call that fails resolution (missing or ambiguous — both check errors) //! gets no injection: the emitted call raises `TypeError` at runtime, which //! matches the source not having type-checked +//! +//! a decoration is the same bargain. `@deco` is a call, but the source writes no +//! argument list for it, so there is nowhere to append the resolved argument — +//! the decoration is emitted as written, and checking reports the parameter +//! (`missing-context-argument`) rather than letting it quietly take its default. +//! a decoration written in the factory form (`@deco(...)`) is an ordinary call +//! expression and is filled like any other +//! +//! a value that would come from a `_` parameter its function repeats is a +//! transpile error instead. python binds only one of those parameters to `_`, +//! and which one is not decided, so there is no name to write for it. so is a +//! value for a `_` parameter the callee repeats, since which of those a keyword +//! `_` names is not decided either use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; use ruff_python_ast::{self as ast, Expr, Stmt}; @@ -49,16 +62,22 @@ impl<'src> ContextParamsPass<'src> { } impl TypeAwarePass for ContextParamsPass<'_> { + fn lowering(&self) -> Option { + Some(super::ast_driver::Lowering::ContextParams) + } + fn run(&self, stmts: &[Stmt], types: &dyn TypeInfo, ctx: &mut PassContext) { let mut lowerer = ContextLowerer { source: self.source, types, edits: Vec::new(), + errors: Vec::new(), }; for stmt in stmts { lowerer.visit_stmt(stmt); } ctx.text_edits.extend(lowerer.edits); + ctx.errors.extend(lowerer.errors); } } @@ -66,6 +85,7 @@ struct ContextLowerer<'src, 'ti> { source: &'src str, types: &'ti dyn TypeInfo, edits: Vec<(TextRange, String)>, + errors: Vec, } impl ContextLowerer<'_, '_> { @@ -96,10 +116,14 @@ impl ContextLowerer<'_, '_> { { return; } - let implicit = self.types.implicit_context_arguments(call); - if implicit.is_empty() { - return; - } + let implicit = match self.types.implicit_context_arguments(call) { + Ok(implicit) if implicit.is_empty() => return, + Ok(implicit) => implicit, + Err(error) => { + self.errors.push(error); + return; + } + }; let arguments = implicit .iter() .map(|(parameter, variable)| format!("{parameter}={variable}")) @@ -195,6 +219,61 @@ mod tests { ); } + /// a decoration has no argument list to write the implicit argument in, so it is + /// emitted exactly as written and checking reports the parameter. injecting here would + /// have to wrap the decorator expression, which changes what `@` evaluates to + #[test] + fn a_decoration_is_left_as_written() { + check( + indoc! {r#" + def deco(fn: (...) -> object, context b: str = "d") -> object: + return fn + + context s1 = "asdf" + + @deco + def g(): ... + "#}, + indoc! {r#" + from typing import Callable + def deco(fn: Callable[..., object], b: str = "d") -> object: + return fn + + s1 = "asdf" + + @deco + def g(): ... + "#}, + ); + } + + /// and the factory form of the same decoration is an ordinary call expression, which is + /// filled like any other + #[test] + fn a_decoration_written_as_a_call_is_filled() { + check( + indoc! {r#" + def deco(context b: str = "d") -> (((...) -> object)) -> object: + return lambda fn: fn + + context s1 = "asdf" + + @deco() + def g(): ... + "#}, + indoc! {r#" + from typing import Callable + def deco(b: str = "d") -> Callable[[Callable[..., object]], object]: + return lambda fn: fn + + s1 = "asdf" + + @deco(b=s1) + def g(): ... + "#}, + ); + } + #[test] fn explicit_argument_wins() { check( @@ -435,4 +514,201 @@ mod tests { "#}, ); } + + #[test] + fn a_single_underscore_parameter_fills_a_context_parameter() { + check( + indoc! {r#" + def show(context label: str) -> str: + return label + + def relay(context _: str) -> str: + return show() + "#}, + indoc! {r#" + def show(label: str) -> str: + return label + + def relay(_: str) -> str: + return show(label=_) + "#}, + ); + } + + #[test] + fn a_repeated_underscore_parameter_is_refused_as_a_context_argument() { + // python binds only one of the parameters to `_`, and which one a read of it + // means is not decided. writing `label=_` answered the `int` argument where a + // `str` was declared + let source = indoc! {r#" + def show(context label: str) -> str: + return label + + def relay(context _: int, context _: str) -> str: + return show() + "#}; + let error = transpile(source, &Config::test_default()).unwrap_err(); + assert_eq!( + error, + "a repeated `_` parameter cannot supply the context argument `label`" + ); + } + + #[test] + fn a_repeated_underscore_is_refused_for_each_argument_of_one_call() { + // two arguments from repeated `_` parameters wrote `_=` twice, which python + // refuses to import + let source = indoc! {r#" + def show(context a: int, context b: str) -> str: + return b + + def relay(context _: int, context _: str) -> str: + return show() + "#}; + let error = transpile(source, &Config::test_default()).unwrap_err(); + assert_eq!( + error, + "a repeated `_` parameter cannot supply the context argument `a`" + ); + } + + #[test] + fn a_repeated_underscore_parameter_of_the_callee_is_refused_an_implicit_argument() { + // each value was written as `_=`, twice, which python refuses to import + let source = indoc! {r#" + def show(context _: int, context _: str) -> str: + return "ok" + + context n = 1 + context s = "a" + show() + "#}; + let error = transpile(source, &Config::test_default()).unwrap_err(); + assert_eq!( + error, + "a context argument cannot be supplied implicitly for a repeated `_` parameter" + ); + } + + #[test] + fn a_keyword_underscore_does_not_fill_a_repeated_underscore_parameter() { + // the keyword was taken to match both parameters named `_`, so nothing was + // written for the other one and the call raised `TypeError` + let source = indoc! {r#" + def show(context _: int, context _: str) -> str: + return "ok" + + context s = "a" + show(_=1) + "#}; + let error = transpile(source, &Config::test_default()).unwrap_err(); + assert_eq!( + error, + "a context argument cannot be supplied implicitly for a repeated `_` parameter" + ); + } + + #[test] + fn a_context_underscore_repeating_an_ordinary_parameter_is_refused_an_implicit_argument() { + let source = indoc! {r#" + def show(_: int, context _: str) -> str: + return "ok" + + context s = "a" + show(1) + "#}; + let error = transpile(source, &Config::test_default()).unwrap_err(); + assert_eq!( + error, + "a context argument cannot be supplied implicitly for a repeated `_` parameter" + ); + } + + #[test] + fn overloads_that_agree_on_a_keyword_only_parameter_receive_the_implicit_argument() { + // which overload a call selects is decided by its arguments, so an argument is + // written only where it is the right one for every overload at once + check( + indoc! {r#" + from typing import overload + + @overload + def f(a: int, *, context b: str) -> int: ... + @overload + def f(a: str, *, context b: str) -> str: ... + def f(a: int | str, *, context b: str) -> int | str: + return a + + context s1 = "asdf" + f(2) + "#}, + indoc! {r#" + from typing import overload + + @overload + def f(a: int, *, b: str) -> int: ... + @overload + def f(a: str, *, b: str) -> str: ... + def f(a: int | str, *, b: str) -> int | str: + return a + + s1 = "asdf" + f(2, b=s1) + "#}, + ); + } + + #[test] + fn overloads_that_disagree_on_a_parameter_receive_nothing() { + // the name means a different thing in each, so no one argument is right whichever + // overload the call selects + check( + indoc! {r#" + from typing import overload + + @overload + def f(a: int, *, context b: str = "d") -> int: ... + @overload + def f(a: str, *, context b: int = 0) -> str: ... + def f(a: int | str, *, context b: str | int = "d") -> int | str: + return a + + context s1 = "asdf" + f(2) + "#}, + indoc! {r#" + from typing import overload + + @overload + def f(a: int, *, b: str = "d") -> int: ... + @overload + def f(a: str, *, b: int = 0) -> str: ... + def f(a: int | str, *, b: str | int = "d") -> int | str: + return a + + s1 = "asdf" + f(2) + "#}, + ); + } + + #[test] + fn repeated_underscore_parameters_passed_positionally_need_no_implicit_argument() { + check( + indoc! {r#" + def show(context _: int, context _: str) -> str: + return "ok" + + context n = 1 + show(2, "b") + "#}, + indoc! {r#" + def show(_: int, _2: str, /) -> str: + return "ok" + + n = 1 + show(2, "b") + "#}, + ); + } } diff --git a/crates/by_transforms/src/transforms/conversion.rs b/crates/by_transforms/src/transforms/conversion.rs index 64d5098d4a..122ae80f80 100644 --- a/crates/by_transforms/src/transforms/conversion.rs +++ b/crates/by_transforms/src/transforms/conversion.rs @@ -68,6 +68,10 @@ fn shave_first_char(source: &str, start: TextSize, limit: TextSize) -> (String, } impl TypeAwarePass for ConversionPass<'_> { + fn lowering(&self) -> Option { + Some(super::ast_driver::Lowering::Conversion) + } + fn run(&self, stmts: &[Stmt], types: &dyn TypeInfo, ctx: &mut PassContext) { let mut collector = ConversionCollector { types, @@ -241,8 +245,9 @@ impl TypeAwarePass for ConversionPass<'_> { /// the conversions one site needs, and whether that site runs at import time struct SiteConversions { /// the span the edit claims: wide enough to strictly contain any peer edit - /// over one of the wrapped values — a call's whole argument list, or an - /// annotated assignment's value together with the `=` before it + /// over one of the wrapped values — a call's argument list from its `(`, or an + /// annotated assignment's value together with the `=` before it — and ending at + /// the last of them claim_range: TextRange, /// `(value range, conversion)` in source order wraps: Vec<(TextRange, ConversionInfo)>, @@ -325,8 +330,16 @@ impl<'ast> ast::visitor::Visitor<'ast> for ConversionCollector<'_> { let mut wraps = self.types.call_conversions(call); if !wraps.is_empty() { wraps.sort_by_key(|(range, _)| range.start()); + // from the `(` to the last converted argument. what follows it is left + // out: an argument a lowering adds before the `)` — a filled `context` + // parameter, a trailing lambda's block — goes after the conversion, not + // into it, and a rewrite of the call around the argument list still + // passes the claim through whole + let claim_end = wraps + .last() + .map_or(call.arguments.range().end(), |(range, _)| range.end()); self.sites.push(SiteConversions { - claim_range: call.arguments.range(), + claim_range: TextRange::new(call.arguments.range().start(), claim_end), wraps, runs_at_import: self.function_depth == 0, }); @@ -371,6 +384,34 @@ class Fahrenheit: ); } + /// a filled `context` parameter is an argument the lowering adds before the `)`, so + /// it follows the converted argument rather than joining the conversion's call + #[test] + fn a_filled_context_argument_follows_a_converted_one() { + let out = check(&format!( + "{TEMPERATURES}def report(t: Fahrenheit, context unit: str): ...\n\n\ + context unit = \"F\"\nreport(Celsius(1.0))\n" + )); + assert!( + out.contains("report(Fahrenheit.__from__(Celsius(1.0)), unit=unit)"), + "got:\n{out}" + ); + } + + /// a trailing lambda re-writes the call around its arguments to pass the block, and + /// the arguments it passes through keep their conversions + #[test] + fn a_trailing_lambda_call_converts_its_arguments() { + let out = check(&format!( + "{TEMPERATURES}def report(t: Fahrenheit, then: () -> None): ...\n\n\ + report(Celsius(1.0)):\n pass\n" + )); + assert!( + out.contains("report(Fahrenheit.__from__(Celsius(1.0)), then=_trailing_lambda_0)"), + "got:\n{out}" + ); + } + #[test] fn from_converts_an_annotated_assignment() { let out = check(&format!("{TEMPERATURES}x: Fahrenheit = Celsius(1.0)\n")); diff --git a/crates/by_transforms/src/transforms/debug_field.rs b/crates/by_transforms/src/transforms/debug_field.rs new file mode 100644 index 0000000000..061d98a38f --- /dev/null +++ b/crates/by_transforms/src/transforms/debug_field.rs @@ -0,0 +1,602 @@ +//! an f-string `=` field prints what the *author* wrote, not what it lowered to +//! +//! python renders `f"{expr = }"` as the source text between the `{` and the `=`, +//! followed by the value: +//! +//! ```python +//! a = 3 +//! f"{(a) = }" # '(a) = 3' — the parentheses and the spaces are the author's +//! ``` +//! +//! so every lowering that rewrites an expression inside such a field used to print +//! itself. `f"{(a!) = }"` came out as `(_force_unwrap(a)) = 3`, and the same held for +//! `??`, `?.`, a conversion, an extension call, a grapheme accessor and `typeof` +//! +//! there is no python spelling of "print this text, evaluate that expression" inside +//! one field, so a field whose expression lowers is taken apart: the author's text +//! becomes literal text of the string around it, and the lowered expression an +//! ordinary replacement field beside it. `f"{(a!) = }"` lowers to +//! `f"(a!) = {(_force_unwrap(a))!r}"`, which is what python prints for the source the +//! author wrote. the `!r` is the conversion python applies to a `=` field that names +//! none of its own +//! +//! a field nothing lowered is left exactly as written, so a `.py` file — and every +//! `f"{x = }"` in a `.by` one — still passes through untouched +//! +//! the taking-apart happens twice over, because a lowering reaches the output two +//! ways. a `TypeAwarePass` emits a text edit keyed on the expression's source range, +//! and [`claim_lowered`] answers those with an edit of its own over the whole field. +//! an `AstPass` instead replaces the expression node, and the statement around it is +//! re-rendered from the syntax tree — so [`rewrite_changed`] answers those in the +//! tree, where the re-render will read them +//! +//! "the expression lowers" is read off the edits, and a lowering whose python spelling +//! is the one the author already wrote still counts — a type test against `None` on an +//! optional is written `a is not None` either way. such a field is taken apart for no +//! gain: the text printed is the same either way, so the answer is right and only the +//! output is longer than it needed to be +//! +//! the reverse direction goes the other way and leaves these fields alone entirely, for +//! the same reason the forward one takes them apart — see [`printed_expressions`] + +use std::cell::RefCell; +use std::collections::HashMap; + +use ruff_python_ast::comparable::ComparableExpr; +use ruff_python_ast::visitor::source_order::{self, SourceOrderVisitor}; +use ruff_python_ast::visitor::transformer::{self, Transformer}; +use ruff_python_ast::{ + AnyStringFlags, AtomicNodeIndex, ConversionFlag, Expr, FString, InterpolatedElement, + InterpolatedStringElement, InterpolatedStringElements, InterpolatedStringLiteralElement, + ModModule, Stmt, StringFlags, TString, +}; +use ruff_text_size::{Ranged, TextRange}; + +use super::ast_driver::Fragment; + +/// the conversion python applies to a `=` field once the field is taken apart +/// +/// a `=` field that names no conversion of its own prints the value's `repr`, unless it +/// carries a format spec: `f"{a=}"` is `f"a={a!r}"`, and `f"{a=:>6}"` is `f"a={a:>6}"` +fn conversion_of(part: &InterpolatedElement) -> ConversionFlag { + match part.conversion { + ConversionFlag::None if part.format_spec.is_none() => ConversionFlag::Repr, + named => named, + } +} + +/// whether the caller writes the literal text into the output itself, or hands it to the +/// code generator +#[derive(Clone, Copy, PartialEq, Eq)] +enum Braces { + /// the text goes straight into the string, where a brace of its own would open a + /// replacement field + Double, + /// the text goes into the syntax tree, and the code generator doubles a brace when it + /// prints the literal element + Leave, +} + +/// the author's text written as literal text of a string with these flags +/// +/// the text moves out of the replacement field and into the string around it, where a +/// backslash starts an escape sequence and the string's own quote ends it — neither of +/// which it meant inside the field. `None` where the string's flags have no spelling for +/// it at all: a raw string escapes nothing, so it can hold neither its own quote nor, on +/// one line, a line break +fn as_literal_text(text: &str, flags: AnyStringFlags, braces: Braces) -> Option { + let raw = flags.prefix().is_raw(); + let quote = flags.quote_style().as_char(); + let triple = flags.is_triple_quoted(); + let mut out = String::with_capacity(text.len()); + for ch in text.chars() { + match ch { + '{' | '}' if braces == Braces::Double => { + out.push(ch); + out.push(ch); + } + '\\' if !raw => out.push_str("\\\\"), + '\n' if !raw => out.push_str("\\n"), + '\r' if !raw => out.push_str("\\r"), + _ if ch == quote && !raw => { + out.push('\\'); + out.push(quote); + } + // a raw string holds every byte it is given literally, which is what makes it + // faithful here and also what makes these two impossible in it + _ if ch == quote => return None, + '\n' | '\r' if !triple => return None, + _ => out.push(ch), + } + } + Some(out) +} + +/// the message for a field whose author text the string it sits in cannot spell +fn cannot_spell(text: &str) -> String { + format!( + "this f-string's `=` field prints `{text}`, the source text between its braces, and a \ + lowering rewrote the expression there — so the text has to be written beside the \ + field rather than inside it, which this string's own quotes cannot hold. Use a \ + different quote style, or drop the `r` prefix" + ) +} + +/// every replacement field of `elements`, its format specs included, with the flags of the +/// string it belongs to +fn fields_of<'a>( + elements: &'a InterpolatedStringElements, + flags: AnyStringFlags, + out: &mut Vec<(&'a InterpolatedElement, AnyStringFlags)>, +) { + for element in elements { + if let InterpolatedStringElement::Interpolation(part) = element { + out.push((part, flags)); + if let Some(spec) = &part.format_spec { + fields_of(&spec.elements, flags, out); + } + } + } +} + +/// every replacement field written anywhere in `suite` +fn fields(suite: &[Stmt]) -> Vec<(&InterpolatedElement, AnyStringFlags)> { + struct Fields<'a>(Vec<(&'a InterpolatedElement, AnyStringFlags)>); + + impl<'a> SourceOrderVisitor<'a> for Fields<'a> { + fn visit_expr(&mut self, expr: &'a Expr) { + match expr { + Expr::FString(fstring) => { + for part in fstring.value.f_strings() { + fields_of(&part.elements, part.flags.into(), &mut self.0); + } + } + Expr::TString(tstring) => { + for part in &tstring.value { + fields_of(&part.elements, part.flags.into(), &mut self.0); + } + } + _ => {} + } + source_order::walk_expr(self, expr); + } + } + + let mut found = Fields(Vec::new()); + for stmt in suite { + source_order::walk_stmt(&mut found, stmt); + } + found.0 +} + +/// whether `edit` rewrites the expression of a field spanning `field` +/// +/// a zero-width insertion counts at either end of the expression: a lowering that wraps an +/// expression inserts its opening text at the first byte and its closing text at the last. +/// an edit that covers the whole field, on the other hand, is a rewrite *of* the field — +/// an f-string in a type position replaced by the type it names, say — and leaves nothing +/// of the author's text to print +pub(super) fn rewrites_expression( + edit: TextRange, + expression: TextRange, + field: TextRange, +) -> bool { + if edit.contains_range(field) { + return false; + } + if edit.is_empty() { + expression.contains_inclusive(edit.start()) + } else { + edit.start() < expression.end() && edit.end() > expression.start() + } +} + +/// the source span of every expression a `=` field prints as its own text +/// +/// python renders `f"{expr = }"` by printing the text between the `{` and the `=`, so a +/// rewrite of the expression there changes what the program *prints*, not only how it is +/// written. the reverse direction leaves those spans alone for that reason: the `.by` it +/// writes has to print what the python it was given printed, and `f"{(a ?? b)=}"` prints +/// `(a ?? b)=` where `f"{(a if a is not None else b)=}"` prints the conditional +pub(crate) fn printed_expressions(suite: &[Stmt]) -> Vec { + fields(suite) + .into_iter() + .filter_map(|(part, _)| part.debug_text.as_ref().map(|_| part.expression.range())) + .collect() +} + +/// take apart every `=` field an edit rewrites the expression of +/// +/// `edited` is every range the passes claimed, and `claimed` every field +/// [`rewrite_changed`] already took apart in the syntax tree +pub(crate) fn claim_lowered( + suite: &[Stmt], + edited: &[TextRange], + claimed: &[TextRange], +) -> (Vec<(TextRange, Vec)>, Vec) { + let mut edits = Vec::new(); + let mut errors = Vec::new(); + for (part, flags) in fields(suite) { + let Some(debug) = &part.debug_text else { + continue; + }; + if claimed.contains(&part.range()) { + continue; + } + let expression = part.expression.range(); + let field = part.range(); + if !edited + .iter() + .any(|edit| rewrites_expression(*edit, expression, field)) + { + continue; + } + let Some(text) = as_literal_text(debug.as_str(), flags, Braces::Double) else { + errors.push(cannot_spell(debug.as_str())); + continue; + }; + // the expression is parenthesised so that whatever it lowered to still reads as + // the whole value of the field it moved into, ahead of the conversion + let mut fragments = vec![ + Fragment::Lit(format!("{text}{{(")), + Fragment::Src(expression), + ]; + let mut tail = String::from(")"); + if let Some(conversion) = conversion_of(part).to_char() { + tail.push('!'); + tail.push(conversion); + } + match &part.format_spec { + Some(spec) => { + tail.push(':'); + fragments.push(Fragment::Lit(tail)); + fragments.push(Fragment::Src(spec.range())); + fragments.push(Fragment::Lit("}".to_owned())); + } + None => { + tail.push('}'); + fragments.push(Fragment::Lit(tail)); + } + } + edits.push((part.range(), fragments)); + } + (edits, errors) +} + +/// take apart every `=` field whose expression an `AstPass` replaced +/// +/// `original` is the module body as it parsed, which is what says whether a field's +/// expression is still the one the author wrote. answers with the fields it took apart, so +/// [`claim_lowered`] leaves them alone +pub(crate) fn rewrite_changed( + module: &mut ModModule, + original: &[Stmt], + changed: &[usize], +) -> (Vec, Vec) { + // a field is matched to the one the author wrote by the range of the field itself, + // which an `AstPass` replacing an expression *inside* it leaves alone. matching by + // position in a walk instead would go wrong the moment a pass added or removed one + let mut written: HashMap> = HashMap::new(); + for &idx in changed { + let Some(stmt) = original.get(idx) else { + continue; + }; + for (part, _) in fields(std::slice::from_ref(stmt)) { + if part.debug_text.is_some() { + written.insert(part.range(), ComparableExpr::from(&*part.expression)); + } + } + } + + let rewrite = Rewrite { + written: &written, + claimed: RefCell::new(Vec::new()), + errors: RefCell::new(Vec::new()), + }; + for &idx in changed { + if let Some(stmt) = module.body.get_mut(idx) { + rewrite.visit_stmt(stmt); + } + } + (rewrite.claimed.into_inner(), rewrite.errors.into_inner()) +} + +struct Rewrite<'a> { + written: &'a HashMap>, + claimed: RefCell>, + errors: RefCell>, +} + +impl Rewrite<'_> { + /// whether this field still prints the expression the author wrote under it + fn lowered(&self, part: &InterpolatedElement) -> bool { + match self.written.get(&part.range()) { + Some(written) => *written != ComparableExpr::from(&*part.expression), + // a field the original has none of at this range is one a pass built itself, + // and its text is the pass's own to get right + None => false, + } + } + + fn rebuild(&self, elements: &mut InterpolatedStringElements, flags: AnyStringFlags) { + if !elements + .interpolations() + .any(|part| part.debug_text.is_some() && self.lowered(part)) + { + return; + } + let mut rebuilt: Vec = Vec::with_capacity(elements.len() + 1); + for element in elements.iter() { + match element { + InterpolatedStringElement::Interpolation(part) + if let Some(debug) = &part.debug_text + && self.lowered(part) => + { + // the code generator doubles a brace when it prints a literal element, + // so only what it does not know about is escaped here + let Some(text) = as_literal_text(debug.as_str(), flags, Braces::Leave) else { + self.errors.borrow_mut().push(cannot_spell(debug.as_str())); + rebuilt.push(element.clone()); + continue; + }; + self.claimed.borrow_mut().push(part.range()); + rebuilt.push(InterpolatedStringElement::Literal( + InterpolatedStringLiteralElement { + range: TextRange::default(), + node_index: AtomicNodeIndex::NONE, + value: text.into(), + }, + )); + let mut moved = part.clone(); + moved.conversion = conversion_of(part); + moved.debug_text = None; + rebuilt.push(InterpolatedStringElement::Interpolation(moved)); + } + kept => rebuilt.push(kept.clone()), + } + } + *elements = rebuilt.into(); + } +} + +impl Transformer for Rewrite<'_> { + fn visit_f_string(&self, f_string: &mut FString) { + transformer::walk_f_string(self, f_string); + self.rebuild(&mut f_string.elements, f_string.flags.into()); + } + + fn visit_t_string(&self, t_string: &mut TString) { + transformer::walk_t_string(self, t_string); + self.rebuild(&mut t_string.elements, t_string.flags.into()); + } +} + +#[cfg(test)] +mod tests { + use crate::python_passthrough::unchanged; + use crate::{Config, transpile}; + + /// the one line of the output the source's last line lowered to + fn lowered(source: &str) -> String { + lowered_with(source, &Config::test_default()) + } + + /// the same, for a source whose replacement fields need what 3.12 added to them — a + /// backslash, and the string's own quote nested inside a field + fn lowered_at_312(source: &str) -> String { + lowered_with( + source, + &Config { + min_version: crate::PythonVersion::PY312, + ..Config::test_default() + }, + ) + } + + fn lowered_with(source: &str, config: &Config) -> String { + let out = transpile(source, config).unwrap_or_else(|error| panic!("{source:?}: {error}")); + out.lines() + .last() + .expect("some output") + .trim_start() + .to_owned() + } + + /// every operator lowering that can reach inside a `=` field leaves the author's own + /// text in front of it, and puts what it lowered to in a field of its own + #[test] + fn an_operator_lowering_keeps_the_field_text() { + // a force-unwrap + assert_eq!( + lowered("def go(a: int?) -> None:\n print(f\"{(a!) = }\")\n"), + "print(f\"(a!) = {(_force_unwrap(a))!r}\")" + ); + // `??` + assert_eq!( + lowered("def go(a: int?, b: int) -> None:\n print(f\"{a ?? b = }\")\n"), + "print(f\"a ?? b = {(a if a is not None else b)!r}\")" + ); + // an optional chain + assert_eq!( + lowered("def go(a: int?) -> None:\n print(f\"{a?.bit_length() = }\")\n"), + "print(f\"a?.bit_length() = {((None if a is None else a.bit_length()))!r}\")" + ); + // a grapheme accessor, which re-emits its receiver + assert_eq!( + lowered("def go(s: str) -> None:\n print(f\"{s.first = }\")\n"), + "print(f\"s.first = {((Character(_by_graphemes(s)[0]) if s else None))!r}\")" + ); + } + + /// `typeof` is folded in the syntax tree rather than emitted as an edit, so the + /// statement around it is re-rendered — the other half of the rewrite + #[test] + fn a_syntax_tree_fold_keeps_the_field_text() { + assert_eq!( + lowered("def go(a: int) -> None:\n print(f\"{typeof(a) = }\")\n"), + "print(f\"typeof(a) = {TypeOf[a]!r}\")" + ); + } + + /// a soundness check inside a `=` field of a statement a syntax-tree fold already + /// rewrote is written into the tree last of all, so the field is taken apart after + /// that rather than before it + #[test] + fn a_check_written_into_a_re_rendered_statement_keeps_the_field_text() { + let checked = Config { + soundness: crate::config::SoundnessPositions::defaults(), + ..Config::test_default() + }; + let out = lowered_with( + "def go(d: dict[str, int]) -> None:\n print(f\"{d.get('k') = }\", typeof(d))\n", + &checked, + ); + assert_eq!( + out, + "print(f\"d.get('k') = {_soundness_check(d.get('k'), (int, type(None)))!r}\", \ + TypeOf[d])" + ); + } + + /// an extension call and a conversion are resolved from the checker's answer rather + /// than from the syntax, and reach the field the same way + #[test] + fn a_type_directed_rewrite_keeps_the_field_text() { + assert_eq!( + lowered( + "extension list:\n def second(self) -> Element:\n return self[1]\n\nxs = [1, 2, 3]\nprint(f\"{xs.second() = }\")\n" + ), + "print(f\"xs.second() = {(_by_ext__list__second(xs))!r}\")" + ); + // a conversion fires where the argument is written, which here is inside the field + assert_eq!( + lowered( + "class Meters:\n def __init__(self, value: float):\n self.value = value\n\n @classmethod\n def __of__(cls, value: int) -> Self:\n return cls(float(value))\n\ndef take(m: Meters) -> int:\n return 1\n\nprint(f\"{take(3) = }\")\n" + ), + "print(f\"take(3) = {(take(Meters.__of__(3)))!r}\")" + ); + } + + /// the conversion python applies to a `=` field is carried over: `repr` where the + /// field names none and has no format spec of its own, and the field's own otherwise + #[test] + fn the_field_keeps_the_conversion_python_would_apply() { + assert_eq!( + lowered("def go(a: int?) -> None:\n print(f\"{(a!)=!s}\")\n"), + "print(f\"(a!)={(_force_unwrap(a))!s}\")" + ); + // a format spec stands in for the repr, exactly as in python + assert_eq!( + lowered("def go(a: int?) -> None:\n print(f\"{(a!)=:>6}\")\n"), + "print(f\"(a!)={(_force_unwrap(a)):>6}\")" + ); + } + + /// a brace in the author's text opens no field of its own once the text is literal, + /// and a backslash starts no escape sequence — python prints both as written + #[test] + fn the_field_text_is_escaped_for_the_string_it_moves_into() { + assert_eq!( + lowered("def go(a: int?) -> None:\n print(f\"{({1: a}[1]!) = }\")\n"), + "print(f\"({{1: a}}[1]!) = {(_force_unwrap({1: a}[1]))!r}\")" + ); + assert_eq!( + lowered_at_312("def go() -> None:\n print(f\"{('a\\n'!) = }\")\n"), + "print(f\"('a\\\\n'!) = {(_force_unwrap('a\\n'))!r}\")" + ); + // the string's own quote, which the field allowed and the literal text does not + assert_eq!( + lowered_at_312( + "def go(d: dict[str, int]) -> None:\n print(f\"{(d[\"k\"]!) = }\")\n" + ), + "print(f\"(d[\\\"k\\\"]!) = {(_force_unwrap(d[\"k\"]))!r}\")" + ); + } + + /// a raw string escapes nothing, so it has no way to hold its own quote in literal + /// text — the field is refused rather than emitted as something that would not parse + #[test] + fn a_raw_string_that_cannot_spell_its_own_text_is_refused() { + let error = transpile( + "def go(d: dict[str, int]) -> None:\n print(rf\"{(d[\"k\"]!) = }\")\n", + &Config::test_default(), + ) + .expect_err("a raw f-string cannot hold its own quote in literal text"); + assert!(error.contains("`=` field"), "{error}"); + assert!(error.contains("quote"), "{error}"); + } + + /// the reverse direction leaves a `=` field's expression exactly as written. rewriting + /// it would change what the program prints — `f"{(a ?? b)=}"` prints `(a ?? b)=` where + /// the python it came from printed the conditional — so the `.by` written out keeps the + /// text the python had + #[test] + fn the_reverse_direction_leaves_a_field_text_alone() { + let out = crate::reverse_transpile( + "def go(a: int | None, b: int) -> None:\n print(f\"{(a if a is not None else b)=}\")\n", + &Config::test_default(), + ) + .expect("reverse should succeed"); + assert!( + out.contains("f\"{(a if a is not None else b)=}\""), + "got:\n{out}" + ); + // the same expression outside a `=` field is rewritten as it always was + let out = crate::reverse_transpile( + "def go(a: int | None, b: int) -> None:\n print(f\"{(a if a is not None else b)}\")\n", + &Config::test_default(), + ) + .expect("reverse should succeed"); + assert!(out.contains("f\"{(a ?? b)}\""), "got:\n{out}"); + } + + /// a lowering that writes back the text the author already wrote leaves the field + /// alone. a type test against `None` on an optional is spelled `a is not None` either + /// way, so taking the field apart would print the same characters out of a longer + /// f-string — and a `.py` file's own `=` field stopped surviving a round trip through + /// `.by` because of it + #[test] + fn a_lowering_that_writes_the_same_text_leaves_the_field_alone() { + unchanged_by_round_trip( + "def go(a: int | None, b: int) -> None:\n print(f\"{(a if a is not None else b)=}\")\n", + ); + assert_eq!( + lowered( + "def go(a: int?, b: int) -> None:\n print(f\"{(a if a is not None else b)=}\")\n" + ), + "print(f\"{(a if a is not None else b)=}\")" + ); + // the same test against a value that cannot be `None` folds to a constant, which + // is text the author did not write — that field is still taken apart + assert_eq!( + lowered("def go(a: int) -> None:\n print(f\"{(a is not None)=}\")\n"), + "print(f\"(a is not None)={(True)!r}\")" + ); + } + + /// `python` → `.by` → `python` gives back the file it started from + fn unchanged_by_round_trip(source: &str) { + let config = Config::test_default(); + let based = crate::reverse_transpile(source, &config).expect("reverse should succeed"); + let back = transpile(&based, &config).expect("forward should succeed"); + let tail = back + .lines() + .rev() + .take(source.lines().count()) + .collect::>(); + let tail = tail.into_iter().rev().collect::>().join("\n"); + assert_eq!(tail, source.trim_end(), "via:\n{based}"); + } + + /// a field nothing lowered keeps python's own compact spelling, in a `.by` file as in + /// a `.py` one + #[test] + fn an_unlowered_field_is_left_alone() { + unchanged("x = 1\nprint(f\"{x = }\")\n"); + unchanged("x = 1\nprint(f\"{ {1: 2}[1] = }\")\n"); + assert_eq!( + lowered("def go(a: int?, b: int) -> None:\n print(f\"{a ?? b} {b = }\")\n"), + "print(f\"{a if a is not None else b} {b = }\")" + ); + } +} diff --git a/crates/by_transforms/src/transforms/decl_site_variance.rs b/crates/by_transforms/src/transforms/decl_site_variance.rs index 933e3ced9e..c6484db3ec 100644 --- a/crates/by_transforms/src/transforms/decl_site_variance.rs +++ b/crates/by_transforms/src/transforms/decl_site_variance.rs @@ -36,6 +36,10 @@ impl<'src> VarianceStripPass<'src> { } impl TypeAwarePass for VarianceStripPass<'_> { + fn lowering(&self) -> Option { + Some(super::ast_driver::Lowering::VarianceStrip) + } + fn run(&self, stmts: &[Stmt], _types: &dyn TypeInfo, ctx: &mut PassContext) { let mut state = State { source: self.source, diff --git a/crates/by_transforms/src/transforms/decorator_keyword.rs b/crates/by_transforms/src/transforms/decorator_keyword.rs index c183ae3f82..203f1112e6 100644 --- a/crates/by_transforms/src/transforms/decorator_keyword.rs +++ b/crates/by_transforms/src/transforms/decorator_keyword.rs @@ -8,43 +8,149 @@ //! → //! ```python //! @overload -//! def d(fn: Callable[..., object], /) -> int: ... +//! def d(fn: Callable[..., object], *, option: bool = ...) -> int: ... //! @overload //! def d(*, option: bool = ...) -> Callable[[Callable[..., object]], int]: ... //! def d(fn=None, *, option=False): //! if fn is None: //! def inner(fn): -//! return d(fn, option=option) +//! return _by_d(fn, option=option) //! return inner //! return 1 if option else len(str(fn)) +//! _by_d = d //! ``` +//! +//! the two overloads are the pair ty models the declaration as +//! (`Signature::decorator_keyword_overloads`), so what a checker reading the +//! transpiled python makes of a decoration is what ty made of it: the decorated +//! parameter carries the type the source declared, the options are declared in +//! both shapes the decorator is applied in, and both state the return type — the +//! one the source wrote, or the one ty reads off the body when it wrote none +//! +//! the dispatcher calls itself under a name of its own, bound beside it, because the +//! options are free to take the name it is declared under: an option named `d` left +//! `d(fn, d=d)` calling the option. `inner` is a name of its own for the same reason, +//! and both are names the module does not spell +//! +//! a stub is never run, so it carries the two overloads alone use std::cell::RefCell; +use std::collections::HashMap; use ruff_python_ast::visitor::{Visitor, walk_stmt}; -use ruff_python_ast::{Expr, ModModule, ParameterWithDefault, Stmt, StmtFunctionDef}; +use ruff_python_ast::{ + Expr, ModModule, ParameterWithDefault, PythonVersion, Stmt, StmtFunctionDef, +}; use ruff_text_size::{Ranged, TextRange, TextSize}; -use super::ast_driver::{AstPass, PassContext}; +use super::ast_driver::{AstPass, Fragment, PassContext}; +use super::repeated_underscore::WrittenNames; +use crate::type_info::{SynthesizedType, TypeInfo}; + +/// The return type of each `decorator def` that wrote no return annotation, by the +/// function's source range. +/// +/// Both emitted overloads state the return type, and a `decorator def` need not have +/// written one — `decorator def d(fn): return len(str(fn))` returns `int`. The lowering +/// runs on the syntax tree and has no type information of its own, so the types are read +/// here, from the parse the checker answered about, in the way +/// [`literal_string::collect`](super::literal_string::collect) reads its own. +#[derive(Default)] +pub(crate) struct InferredReturnTypes { + by_function: HashMap, +} + +/// Read the inferred return type of every `decorator def` in `stmts` that left its return +/// annotation out. `stmts` must come from the same parse `types` answers for. +pub(crate) fn collect_return_types( + stmts: &[Stmt], + types: &dyn TypeInfo, + min_version: PythonVersion, +) -> InferredReturnTypes { + let mut collector = ReturnTypes { + types, + min_version, + found: InferredReturnTypes::default(), + }; + for stmt in stmts { + collector.visit_stmt(stmt); + } + collector.found +} + +struct ReturnTypes<'a> { + types: &'a dyn TypeInfo, + min_version: PythonVersion, + found: InferredReturnTypes, +} + +impl<'ast> Visitor<'ast> for ReturnTypes<'_> { + fn visit_stmt(&mut self, stmt: &'ast Stmt) { + if let Stmt::FunctionDef(function) = stmt + && has_decorator_keyword_marker(function) + && let Some(annotation) = self + .types + .inferred_return_annotation(function, self.min_version) + { + self.found.by_function.insert(function.range(), annotation); + } + walk_stmt(self, stmt); + } +} + +/// whether the parser marked `function` as a `decorator def`. The marker decorator is +/// synthesized, so it carries the invalid expression context rather than an `@` +fn has_decorator_keyword_marker(function: &StmtFunctionDef) -> bool { + function.decorator_list.iter().any(|decorator| { + matches!(&decorator.expression, Expr::Name(name) + if name.ctx.is_invalid() && name.id.as_str() == "decorator_keyword") + }) +} pub(crate) struct DecoratorKeyword<'src> { source: &'src str, + written: WrittenNames<'src>, + is_stub: bool, + return_types: InferredReturnTypes, } impl<'src> DecoratorKeyword<'src> { - pub(crate) fn new(source: &'src str) -> Self { - Self { source } + pub(crate) fn new( + source: &'src str, + written: WrittenNames<'src>, + is_stub: bool, + return_types: InferredReturnTypes, + ) -> Self { + Self { + source, + written, + is_stub, + return_types, + } } } impl AstPass for DecoratorKeyword<'_> { + /// a `decorator def` writes its options' signature itself, without their parameter + /// modifiers + fn subsumes(&self) -> &'static [super::ast_driver::Lowering] { + &[ + super::ast_driver::Lowering::LocalOnce, + super::ast_driver::Lowering::ContextParams, + ] + } + fn run(&self, module: &mut ModModule, ctx: &mut PassContext) { let mut state = State { source: self.source, - edits: RefCell::new(Vec::new()), + written: self.written, + is_stub: self.is_stub, + return_types: &self.return_types, + templates: RefCell::new(Vec::new()), errors: RefCell::new(Vec::new()), needs_callable: false, needs_overload: false, + synthesized_imports: RefCell::new(Vec::new()), class_depth: 0, }; for stmt in &module.body { @@ -58,25 +164,30 @@ impl AstPass for DecoratorKeyword<'_> { ctx.required_imports .push("from typing import overload".to_owned()); } - ctx.text_edits.extend(state.edits.into_inner()); + // the emitted overloads are read and never run, so a name only they need is + // imported where only a checker sees it + ctx.type_only_imports + .extend(state.synthesized_imports.into_inner()); + ctx.template_edits.extend(state.templates.into_inner()); ctx.errors.extend(state.errors.into_inner()); } } struct State<'src> { source: &'src str, - edits: RefCell>, + written: WrittenNames<'src>, + is_stub: bool, + return_types: &'src InferredReturnTypes, + templates: RefCell)>>, errors: RefCell>, needs_callable: bool, needs_overload: bool, + /// the imports a written-out return type names that the source never wrote + synthesized_imports: RefCell>, class_depth: u32, } impl State<'_> { - fn src(&self, range: TextRange) -> &str { - &self.source[usize::from(range.start())..usize::from(range.end())] - } - fn line_indent(&self, pos: TextSize) -> &str { super::source_util::line_indent(self.source, pos) } @@ -93,8 +204,8 @@ impl State<'_> { None } - fn push(&self, range: TextRange, repl: String) { - self.edits.borrow_mut().push((range, repl)); + fn template(&self, range: TextRange, frags: Vec) { + self.templates.borrow_mut().push((range, frags)); } fn error(&self, msg: String) { @@ -102,8 +213,6 @@ impl State<'_> { } fn process_function(&mut self, func: &StmtFunctionDef) { - use std::fmt::Write as _; - let Some(deco) = func.decorator_list.iter().find(|d| { super::source_util::is_synthetic_decorator(self.source, d) && matches!(&d.expression, Expr::Name(n) if n.id.as_str() == "decorator_keyword") @@ -147,100 +256,186 @@ impl State<'_> { } } - self.needs_callable = true; self.needs_overload = true; let fn_name = func.name.as_str(); - let ret_text = func - .returns - .as_ref() - .map(|r| self.src(r.range()).to_owned()) - .unwrap_or_else(|| "object".to_owned()); let base_indent = self.line_indent(func.range().start()).to_owned(); let body_indent = format!("{base_indent} "); - let mut header = String::new(); - header.push_str("@overload\n"); - header.push_str(&base_indent); - let _ = writeln!( - header, - "def {fn_name}(fn: Callable[..., object], /) -> {ret_text}: ..." - ); - header.push_str(&base_indent); - header.push_str("@overload\n"); - header.push_str(&base_indent); - let opt_sig: String = options - .iter() - .map(|o| { - let name = o.name().as_str(); - let ann = o - .annotation() - .map(|a| format!(": {}", self.src(a.range()))) - .unwrap_or_default(); - format!("{name}{ann} = ...") - }) - .collect::>() - .join(", "); - let opt_sig_args = if options.is_empty() { - String::new() - } else { - format!("*, {opt_sig}") + let python_name = |parameter: &ParameterWithDefault| { + crate::python_parameter_name(params, ¶meter.parameter, self.written) }; - let _ = writeln!( - header, - "def {fn_name}({opt_sig_args}) -> Callable[[Callable[..., object]], {ret_text}]: ..." - ); - header.push_str(&base_indent); - let impl_options: String = options - .iter() - .map(|o| { - let name = o.name().as_str(); - let default = o - .default - .as_ref() - .map(|d| self.src(d.range()).to_owned()) - .unwrap_or_else(|| "None".to_owned()); - format!("{name}={default}") - }) - .collect::>() - .join(", "); - let impl_sig_args = if options.is_empty() { - "fn=None".to_owned() + // the body reads the decorated callable by the name its parameter declares + let decorated = python_name(fn_param); + + // the declared types and defaults are passed through as source spans, so a + // lowering inside one — an arrow callable type, a `T?`, a `??` default — is + // materialized where this template re-emits it. written as text of our own + // they would be dropped, and the basedpython surface would reach the `.py` + let decorated_type = |frags: &mut Vec| match fn_param.annotation() { + Some(annotation) => frags.push(Fragment::Src(annotation.range())), + // an unannotated decorated parameter accepts any callable + None => frags.push(Fragment::Lit("Callable[..., object]".to_owned())), + }; + // a declaration that wrote no return annotation still has a return type, and both + // overloads state it: written as `object` the emitted python would be weaker than + // the source, and a caller would lose the type the decoration produces. `object` + // remains the fallback for a type with no python spelling, which is sound — it + // says less than the truth rather than something else + let inferred_return = self.return_types.by_function.get(&func.range()); + if let Some(inferred) = inferred_return { + let mut imports = self.synthesized_imports.borrow_mut(); + imports.extend( + inferred + .modules + .iter() + .map(|module| format!("import {module}")), + ); + imports.extend( + inferred + .typing_names + .iter() + .map(|name| format!("from typing import {name}")), + ); + } + let return_type = |frags: &mut Vec| match (&func.returns, inferred_return) { + (Some(returns), _) => frags.push(Fragment::Src(returns.range())), + (None, Some(inferred)) => frags.push(Fragment::Lit(inferred.text.clone())), + (None, None) => frags.push(Fragment::Lit("object".to_owned())), + }; + if fn_param.annotation().is_none() { + self.needs_callable = true; + } + // the decorated parameter is declared as the source declares it, so applying the + // decorator by keyword (`d(fn=f)`) is accepted here exactly where it is accepted + // of the declaration + // the first positional is the one in `posonlyargs` whenever there is one there + let decorated_marker = if params.posonlyargs.is_empty() { + "" } else { - format!("fn=None, *, {impl_options}") + ", /" + }; + // the options are keyword-only at every call site: one dispatcher serves each + // shape, and it tells them apart by whether the decorated function arrived + // positionally + let declared_options = |frags: &mut Vec| { + for option in &options { + frags.push(Fragment::Lit(format!( + ", {name}", + name = python_name(option) + ))); + if let Some(annotation) = option.annotation() { + frags.push(Fragment::Lit(": ".to_owned())); + frags.push(Fragment::Src(annotation.range())); + } + frags.push(Fragment::Lit(" = ...".to_owned())); + } }; - let _ = write!(header, "def {fn_name}({impl_sig_args}):"); + + // overload 1: applied to the function itself, with whatever options were given + let mut header: Vec = Vec::new(); + header.push(Fragment::Lit(format!( + "@overload\n{base_indent}def {fn_name}({decorated}: " + ))); + decorated_type(&mut header); + if !options.is_empty() { + header.push(Fragment::Lit(format!("{decorated_marker}, *"))); + } else if !decorated_marker.is_empty() { + header.push(Fragment::Lit(decorated_marker.to_owned())); + } + declared_options(&mut header); + header.push(Fragment::Lit(") -> ".to_owned())); + return_type(&mut header); + // overload 2: applied to the options alone, and what it hands back takes the + // function + header.push(Fragment::Lit(format!( + ": ...\n{base_indent}@overload\n{base_indent}def {fn_name}(" + ))); + if !options.is_empty() { + header.push(Fragment::Lit("*".to_owned())); + declared_options(&mut header); + } + header.push(Fragment::Lit(") -> Callable[[".to_owned())); + decorated_type(&mut header); + header.push(Fragment::Lit("], ".to_owned())); + return_type(&mut header); + header.push(Fragment::Lit("]: ...".to_owned())); + // the overloads alone are the declaration; everything past here is what runs + let declaration_end = header.len(); + + header.push(Fragment::Lit(format!( + "\n{base_indent}def {fn_name}({decorated}=None" + ))); + if !options.is_empty() { + header.push(Fragment::Lit(", *".to_owned())); + for option in &options { + header.push(Fragment::Lit(format!( + ", {name}=", + name = python_name(option) + ))); + match option.default.as_deref() { + Some(default) => header.push(Fragment::Src(default.range())), + None => header.push(Fragment::Lit("None".to_owned())), + } + } + } + header.push(Fragment::Lit("):".to_owned())); + + // a stub declares and never runs, so it carries the two overloads alone: the + // dispatcher and the name it reaches itself under are machinery for calls, and a + // `.pyi` holds none + if self.is_stub { + header.truncate(declaration_end); + self.template( + TextRange::new(deco.range().start(), func.range().end()), + header, + ); + return; + } let recursive_kw_args: String = options .iter() .map(|o| { - let name = o.name().as_str(); + let name = python_name(o); format!("{name}={name}") }) .collect::>() .join(", "); + // the dispatcher reaches itself through a name of its own rather than through the + // one it is declared under, which the options are free to take: an option named `d` + // left `d(fn, d=d)` calling the option + let self_name = self.written.fresh(&format!("_by_{fn_name}")); + // and the function it hands back is bound under a name of its own too, for the same + // reason — an option named `inner` was handed the dispatcher itself + let inner_name = self.written.fresh("inner"); let recursive_call = if options.is_empty() { - format!("{fn_name}(fn)") + format!("{self_name}({decorated})") } else { - format!("{fn_name}(fn, {recursive_kw_args})") + format!("{self_name}({decorated}, {recursive_kw_args})") }; let dispatch = |prefix: &str| { format!( - "{prefix}if fn is None:\n{body_indent} def inner(fn):\n{body_indent} return {recursive_call}\n{body_indent} return inner\n{body_indent}" + "{prefix}if {decorated} is None:\n{body_indent} def {inner_name}({decorated}):\n{body_indent} return {recursive_call}\n{body_indent} return {inner_name}\n{body_indent}" ) }; + // the name is bound after the declaration, where the dispatcher exists; a lowering + // inside it only reads the name when it runs + let bind_self = format!("\n{base_indent}{self_name} = {fn_name}"); // a `decorator def` with no body is a declaration — `decorator def d(fn)` // means the same as `decorator def d(fn): ...`, just as a bodyless plain // `def` does. the source carries no colon and no body statement to anchor // edits on, so the whole declaration is replaced in one go let Some(first_stmt) = func.body.first() else { - let body = dispatch(&format!("\n{body_indent}")); - self.push( + header.push(Fragment::Lit(format!( + "{body}...{bind_self}", + body = dispatch(&format!("\n{body_indent}")) + ))); + self.template( TextRange::new(deco.range().start(), func.range().end()), - format!("{header}{body}..."), + header, ); return; }; @@ -261,7 +456,7 @@ impl State<'_> { }; let header_end = colon_pos + TextSize::from(1); - self.push(TextRange::new(deco.range().start(), header_end), header); + self.template(TextRange::new(deco.range().start(), header_end), header); let is_inline_body = { let body_start = usize::from(body_first_pos); @@ -284,10 +479,12 @@ impl State<'_> { String::new() }; - self.push( + self.template( TextRange::new(body_first_pos, body_first_pos), - dispatch(&prefix), + vec![Fragment::Lit(dispatch(&prefix))], ); + let end = func.range().end(); + self.template(TextRange::new(end, end), vec![Fragment::Lit(bind_self)]); } } @@ -330,6 +527,21 @@ mod tests { assert_eq!(transpile(input, &Config::test_default()).unwrap(), expected); } + /// the options are passed as `@d(option=...)`, so a repeated `_` among them would + /// be keyword-only, and the name only a keyword reaches it by is not the author's + #[test] + fn a_repeated_underscore_option_is_refused() { + let error = transpile( + "decorator def d(fn: (...) -> object, _: int = 1, _: int = 2) -> int:\n return 7\n", + &Config::test_default(), + ) + .unwrap_err(); + assert!( + error.starts_with("a repeated `_` parameter cannot be keyword-only in `d`"), + "got:\n{error}" + ); + } + #[test] fn basic_decorator_with_option() { check( @@ -340,15 +552,125 @@ mod tests { indoc! {" from typing import Callable, overload @overload - def d(fn: Callable[..., object], /) -> int: ... + def d(fn: Callable[..., object], *, option: bool = ...) -> int: ... @overload def d(*, option: bool = ...) -> Callable[[Callable[..., object]], int]: ... def d(fn=None, *, option=False): if fn is None: def inner(fn): - return d(fn, option=option) + return _by_d(fn, option=option) return inner return 1 if option else len(str(fn)) + _by_d = d + "}, + ); + } + + /// the body reads the decorated callable under the name its parameter declares, which + /// is not always `fn`: lowered as `fn`, `func` was a `NameError` the first time the + /// decorator ran + #[test] + fn the_decorated_parameter_keeps_its_name() { + check( + indoc! {" + decorator def d(func: (...) -> object, option: bool = False) -> int: + return len(str(func)) + "}, + indoc! {" + from typing import Callable, overload + @overload + def d(func: Callable[..., object], *, option: bool = ...) -> int: ... + @overload + def d(*, option: bool = ...) -> Callable[[Callable[..., object]], int]: ... + def d(func=None, *, option=False): + if func is None: + def inner(func): + return _by_d(func, option=option) + return inner + return len(str(func)) + _by_d = d + "}, + ); + } + + /// the function the dispatcher hands back is bound under a name of its own: named + /// `inner`, it stood where an option of that name was declared, and decorating with + /// options handed the decorated function the dispatcher itself + #[test] + fn an_option_named_inner_keeps_its_value() { + check( + indoc! {" + decorator def d(fn: (...) -> object, inner: bool = False) -> str: + return f\"inner={inner}\" + "}, + indoc! {" + from typing import Callable, overload + @overload + def d(fn: Callable[..., object], *, inner: bool = ...) -> str: ... + @overload + def d(*, inner: bool = ...) -> Callable[[Callable[..., object]], str]: ... + def d(fn=None, *, inner=False): + if fn is None: + def inner2(fn): + return _by_d(fn, inner=inner) + return inner2 + return f\"inner={inner}\" + _by_d = d + "}, + ); + } + + /// the dispatcher reaches itself under a name of its own: an option named as the + /// decorator shadowed the decorator, and the dispatcher called the option instead + #[test] + fn an_option_named_as_the_decorator_does_not_shadow_it() { + check( + indoc! {" + decorator def d(fn: (...) -> object, d: bool = False) -> str: + return f\"d={d}\" + "}, + indoc! {" + from typing import Callable, overload + @overload + def d(fn: Callable[..., object], *, d: bool = ...) -> str: ... + @overload + def d(*, d: bool = ...) -> Callable[[Callable[..., object]], str]: ... + def d(fn=None, *, d=False): + if fn is None: + def inner(fn): + return _by_d(fn, d=d) + return inner + return f\"d={d}\" + _by_d = d + "}, + ); + } + + /// the name the dispatcher reaches itself under is one the module does not spell + #[test] + fn a_module_that_spells_the_dispatcher_name_is_numbered_around() { + check( + indoc! {" + _by_d = 1 + + decorator def d(fn: (...) -> object) -> int: + return _by_d + "}, + indoc! {" + from typing import Callable, overload + _by_d = 1 + + @overload + def d(fn: Callable[..., object]) -> int: ... + @overload + def d() -> Callable[[Callable[..., object]], int]: ... + def d(fn=None): + if fn is None: + def inner(fn): + return _by_d2(fn) + return inner + return _by_d + _by_d2 = d "}, ); } @@ -363,15 +685,16 @@ mod tests { indoc! {" from typing import Callable, overload @overload - def d(fn: Callable[..., object], /) -> int: ... + def d(fn: Callable[..., object]) -> int: ... @overload def d() -> Callable[[Callable[..., object]], int]: ... def d(fn=None): if fn is None: def inner(fn): - return d(fn) + return _by_d(fn) return inner return len(str(fn)) + _by_d = d "}, ); } @@ -386,15 +709,16 @@ mod tests { indoc! {" from typing import Callable, overload @overload - def d(fn: Callable[..., object], /) -> int: ... + def d(fn: Callable[..., object], *, a: int = ..., b: str = ...) -> int: ... @overload def d(*, a: int = ..., b: str = ...) -> Callable[[Callable[..., object]], int]: ... def d(fn=None, *, a=1, b=\"x\"): if fn is None: def inner(fn): - return d(fn, a=a, b=b) + return _by_d(fn, a=a, b=b) return inner return a + len(b) + _by_d = d "}, ); } @@ -406,15 +730,16 @@ mod tests { indoc! {" from typing import Callable, overload @overload - def d(fn: Callable[..., object], /) -> object: ... + def d(fn: Callable[[int], None]) -> None: ... @overload - def d() -> Callable[[Callable[..., object]], object]: ... + def d() -> Callable[[Callable[[int], None]], None]: ... def d(fn=None): if fn is None: def inner(fn): - return d(fn) + return _by_d(fn) return inner ... + _by_d = d "}, ); } @@ -426,15 +751,16 @@ mod tests { indoc! {" from typing import Callable, overload @overload - def d(fn: Callable[..., object], /) -> int: ... + def d(fn: Callable[..., object], *, option: bool = ...) -> int: ... @overload def d(*, option: bool = ...) -> Callable[[Callable[..., object]], int]: ... def d(fn=None, *, option=False): if fn is None: def inner(fn): - return d(fn, option=option) + return _by_d(fn, option=option) return inner ... + _by_d = d "}, ); } @@ -452,21 +778,282 @@ mod tests { indoc! {" from typing import Callable, overload @overload - def d(fn: Callable[..., object], /) -> object: ... + def d(fn: Callable[[int], None]) -> None: ... @overload - def d() -> Callable[[Callable[..., object]], object]: ... + def d() -> Callable[[Callable[[int], None]], None]: ... def d(fn=None): if fn is None: def inner(fn): - return d(fn) + return _by_d(fn) return inner ... + _by_d = d d(lambda i: None) "}, ); } + /// the declared types reach the overloads through the source, so a type written in + /// basedpython lowers on its way there. spelled as text of our own they were dropped: + /// the decorated parameter was declared `Callable[..., object]` however it was written, + /// and an option's `int?` reached the `.py` with the `?` still on it + #[test] + fn the_declared_types_lower_on_their_way_into_the_overloads() { + check( + indoc! {" + decorator def d(fn: (int) -> str, label: int? = None) -> str?: + return label + "}, + indoc! {" + from typing import Callable, overload + @overload + def d(fn: Callable[[int], str], *, label: int | None = ...) -> str | None: ... + @overload + def d(*, label: int | None = ...) -> Callable[[Callable[[int], str]], str | None]: ... + def d(fn=None, *, label=None): + if fn is None: + def inner(fn): + return _by_d(fn, label=label) + return inner + return label + _by_d = d + "}, + ); + } + + /// an option whose default is not a scalar is re-evaluated per call, so the signature + /// carries the `_MISSING` sentinel and the body carries the default itself. the header + /// template re-emits the whole signature, and re-emitted the sentinel in the guard too — + /// `if tags is _MISSING: tags = _MISSING` — so the option arrived as the sentinel object + /// and `@d` raised `TypeError: object of type 'object' has no len()` + #[test] + fn a_mutable_option_default_reaches_the_guard_and_not_the_signature() { + check( + indoc! {" + decorator def d(fn: (...) -> object, tags: list[str] = []) -> int: + return len(tags) + "}, + indoc! {" + from typing import Any, Callable, overload + _MISSING: Any = object() + @overload + def d(fn: Callable[..., object], *, tags: list[str] = ...) -> int: ... + @overload + def d(*, tags: list[str] = ...) -> Callable[[Callable[..., object]], int]: ... + def d(fn=None, *, tags=_MISSING): + if tags is _MISSING: + tags = [] + if fn is None: + def inner(fn): + return _by_d(fn, tags=tags) + return inner + return len(tags) + _by_d = d + "}, + ); + } + + /// and a lowering written inside that default still lands in the body copy: the guard + /// re-emits the default's own source, so only the sentinel itself stays behind + #[test] + fn a_lowering_inside_a_mutable_option_default_survives_the_relocation() { + let out = transpile( + "env: str? = None\n\ndecorator def d(fn: (...) -> object, tag: str = env ?? \"b\") -> int:\n return len(tag)\n", + &Config::test_default(), + ) + .unwrap(); + assert!( + out.contains("def d(fn=None, *, tag=_MISSING):"), + "got:\n{out}" + ); + assert!( + out.contains("tag = env if env is not None else \"b\""), + "got:\n{out}" + ); + } + + /// the header the overloads are written from replaces each parameter's name with text + /// of its own, so an edit a sibling pass made to the modifier prefix in front of that + /// name lands nowhere. every prefix a parameter can carry — `local`, `once`, `context` + /// — lowers by being deleted, which is what writing the bare name already does, so the + /// emitted python is the same either way. this pins that: a prefix that lowered to + /// anything else would be dropped here without a word + #[test] + fn a_parameter_modifier_leaves_the_overloads_as_they_were() { + let plain = transpile( + "decorator def d(fn: (...) -> object, tag: str = \"x\") -> int:\n return 1\n", + &Config::test_default(), + ) + .unwrap(); + for modified in [ + "decorator def d(local fn: (...) -> object, tag: str = \"x\") -> int:\n return 1\n", + "decorator def d(once fn: (...) -> object, tag: str = \"x\") -> int:\n return 1\n", + "decorator def d(fn: (...) -> object, context tag: str = \"x\") -> int:\n return 1\n", + ] { + assert_eq!( + transpile(modified, &Config::test_default()).unwrap(), + plain, + "modifiers changed the output of:\n{modified}" + ); + } + } + + /// the declaration is always the pair of overloads, and an option is keyword-only in + /// both of them, so a call site can fill it: the factory form `@d()` writes the keyword + /// into the argument list it already has, and a plain call writes it too. the bare `@d` + /// has no argument list, which is what the checker reports + #[test] + fn an_option_declared_context_is_filled_where_there_is_an_argument_list() { + check( + indoc! {r#" + decorator def d(fn: (...) -> object, context tag: str = "x") -> int: + return len(tag) + + context t: str = "hello" + + @d() + def g(): ... + + def plain(): ... + + k = d(plain) + "#}, + indoc! {r#" + from typing import Callable, overload + @overload + def d(fn: Callable[..., object], *, tag: str = ...) -> int: ... + @overload + def d(*, tag: str = ...) -> Callable[[Callable[..., object]], int]: ... + def d(fn=None, *, tag="x"): + if fn is None: + def inner(fn): + return _by_d(fn, tag=tag) + return inner + return len(tag) + _by_d = d + + t: str = "hello" + + @d(tag=t) + def g(): ... + + def plain(): ... + + k = d(plain, tag=t) + "#}, + ); + } + + /// a declaration that writes no return annotation still has a return type, and both + /// overloads state it. written as `object` — which is all the lowering can see on its + /// own — the emitted python would be weaker than the source: `@d` would produce + /// `object` where the checker reading the `.by` says `int` + #[test] + fn an_unwritten_return_type_reaches_the_overloads() { + check( + indoc! {" + decorator def d(fn: (...) -> object): + return len(str(fn)) + "}, + indoc! {" + from typing import Callable, overload + @overload + def d(fn: Callable[..., object]) -> int: ... + @overload + def d() -> Callable[[Callable[..., object]], int]: ... + def d(fn=None): + if fn is None: + def inner(fn): + return _by_d(fn) + return inner + return len(str(fn)) + _by_d = d + "}, + ); + } + + /// a `decorator def` with no body declares a signature and returns `None`, which is + /// what the checker reads there too + #[test] + fn a_bodyless_declaration_returns_none() { + let out = transpile( + "decorator def d(fn: (...) -> object)\n", + &Config::test_default(), + ) + .unwrap(); + assert!( + out.contains("def d(fn: Callable[..., object]) -> None: ...") + && out.contains("def d() -> Callable[[Callable[..., object]], None]: ..."), + "got:\n{out}" + ); + } + + /// a type with no spelling the emitted file could resolve is not written: `object` + /// stands, which says less than the truth rather than something else + #[test] + fn a_return_type_the_output_could_not_resolve_stays_object() { + // ty's answer here is `Literal[1]`, and `Literal` is not one of the names + // basedpython binds implicitly, so there is no import to write for it + let out = transpile( + "decorator def d(fn: (...) -> object):\n return 1\n", + &Config::test_default(), + ) + .unwrap(); + assert!( + out.contains("def d(fn: Callable[..., object]) -> object: ..."), + "got:\n{out}" + ); + } + + /// a stub is never run, so the dispatcher it would declare is a definition nothing + /// calls, and the name it reaches itself under is a binding nothing reads + #[test] + fn a_stub_declares_the_overloads_alone() { + let out = transpile( + "decorator def route(fn: (int) -> None, prefix: str = \"\")\n", + &Config { + is_stub: true, + ..Config::test_default() + }, + ) + .unwrap(); + assert_eq!( + out, + indoc! {" + from typing import Callable, overload + @overload + def route(fn: Callable[[int], None], *, prefix: str = ...) -> None: ... + @overload + def route(*, prefix: str = ...) -> Callable[[Callable[[int], None]], None]: ... + "} + ); + } + + /// and one written with a body drops that too — a `.pyi` declares, and the body the + /// dispatcher would call is not part of the declaration + #[test] + fn a_stub_with_a_body_declares_the_overloads_alone() { + let out = transpile( + "decorator def d(fn: (...) -> object) -> int:\n return 1\n", + &Config { + is_stub: true, + ..Config::test_default() + }, + ) + .unwrap(); + assert_eq!( + out, + indoc! {" + from typing import Callable, overload + @overload + def d(fn: Callable[..., object]) -> int: ... + @overload + def d() -> Callable[[Callable[..., object]], int]: ... + "} + ); + } + #[test] fn decorator_no_callable_fails() { let result = transpile( diff --git a/crates/by_transforms/src/transforms/destructure.rs b/crates/by_transforms/src/transforms/destructure.rs index bdd9a0492b..96e98d9ac2 100644 --- a/crates/by_transforms/src/transforms/destructure.rs +++ b/crates/by_transforms/src/transforms/destructure.rs @@ -66,6 +66,7 @@ use ruff_python_trivia::{SimpleTokenKind, SimpleTokenizer}; use ruff_text_size::{Ranged, TextRange, TextSize}; use super::ast_driver::{AstPass, Fragment, PassContext}; +use super::repeated_underscore::WrittenNames; use super::source_util::{line_indent, line_start, temporary_name}; pub(crate) struct DestructurePass<'src> { @@ -84,7 +85,7 @@ impl AstPass for DestructurePass<'_> { source: self.source, edits: Vec::new(), errors: Vec::new(), - names: NameGen::default(), + names: NameGen::new(self.source), }; for stmt in &module.body { lower.visit_stmt(stmt); @@ -98,7 +99,7 @@ struct DestructureLower<'src> { source: &'src str, edits: Vec<(TextRange, Vec)>, errors: Vec, - names: NameGen, + names: NameGen<'src>, } impl<'ast> Visitor<'ast> for DestructureLower<'_> { @@ -219,7 +220,11 @@ impl DestructureLower<'_> { )); return; }; - let selector = temporary_name("let", u32::from(let_stmt.range().start())); + let selector = temporary_name( + WrittenNames::new(self.source), + "let", + u32::from(let_stmt.range().start()), + ); let mut fragments = vec![Fragment::Lit(format!("{selector} = 0\n{indent}"))]; if !self.push_destructure( @@ -360,12 +365,23 @@ impl DestructureLower<'_> { let (header_end, span_end, body_indent) = match (leading_docstring, anchor_stmt) { // the block goes after the docstring, on its own line (Some(docstring), _) => { - let end = docstring.range().end(); - ( - end, - end, - line_indent(self.source, docstring.range().start()).to_owned(), - ) + let (start, end) = (docstring.range().start(), docstring.range().end()); + let same_line = + !self.source[usize::from(colon_end)..usize::from(start)].contains('\n'); + if same_line { + // the docstring was written on the clause's line, where nothing + // can follow it. it is given a line of its own first, as its own + // insertion so that a second destructuring parameter's block — + // which this edit passes through — still composes + let indent = format!("{} ", line_indent(self.source, colon_end)); + self.edits.push(( + TextRange::empty(start), + vec![Fragment::Lit(format!("\n{indent}"))], + )); + (end, end, indent) + } else { + (end, end, line_indent(self.source, start).to_owned()) + } } (None, Some(stmt)) => { let stmt_start = stmt.range().start(); @@ -431,8 +447,9 @@ impl DestructureLower<'_> { let indent = line_indent(self.source, match_stmt.range().start()).to_owned(); let case_indent = format!("{indent} "); let offset = u32::from(match_stmt.range().start()); - let subject_name = temporary_name("subject", offset); - let selector = temporary_name("case", offset); + let written = WrittenNames::new(self.source); + let subject_name = temporary_name(written, "subject", offset); + let selector = temporary_name(written, "case", offset); let Some(header_colon) = self.colon_end(match_stmt.subject.range().end()) else { self.errors.push(format!( @@ -561,7 +578,7 @@ pub(crate) fn push_destructure( pattern: &Pattern, guard: Option<&Expr>, on_match: &str, - names: &mut NameGen, + names: &mut NameGen<'_>, ) -> Result<(), String> { // every temporary this lowering makes, to be dropped once the nest is done let mut temporaries = Vec::new(); @@ -632,14 +649,23 @@ pub(crate) fn push_destructure( /// changes its output only where the formatting did. Two passes each holding one /// can name the same temporary, which is harmless: a temporary never outlives /// the `match` nest that made it — the nest drops it before any body runs. -#[derive(Default)] -pub(crate) struct NameGen(usize); +pub(crate) struct NameGen<'src> { + written: WrittenNames<'src>, + counter: usize, +} + +impl<'src> NameGen<'src> { + pub(crate) fn new(source: &'src str) -> Self { + Self { + written: WrittenNames::new(source), + counter: 0, + } + } -impl NameGen { fn next(&mut self, kind: &str) -> String { - let index = self.0; - self.0 += 1; - temporary_name(kind, index) + let index = self.counter; + self.counter += 1; + temporary_name(self.written, kind, index) } } @@ -655,7 +681,7 @@ struct Step { fn collect_steps( subject: &[Fragment], pattern: &Pattern, - names: &mut NameGen, + names: &mut NameGen<'_>, binders: &mut Vec, steps: &mut Vec, ) -> Result<(), String> { @@ -964,6 +990,44 @@ mod tests { ); } + /// and when that docstring is written on the `def`'s own line it is given a line of + /// its own first. nothing can follow a statement there, so the block was emitted at + /// the `def`'s own indentation — outside the function, where it ran at import and + /// raised `NameError` on a name no module binds + #[test] + fn a_parameter_destructure_opens_a_docstring_written_on_the_def_line() { + let out = check(&format!( + "{POINT}\ndef f(Point(x, y): Point): 'what f does'\n" + )); + assert!( + out.contains(indoc! {" + def f(__by_destructure_N__: Point): \n 'what f does' + match __by_destructure_N__: + case Point(x, y): + pass + del __by_destructure_N__ + "}), + "got:\n{out}" + ); + } + + /// the break is an insertion of its own, so a second destructuring parameter — whose + /// block this edit passes through — is still written + #[test] + fn two_parameters_destructure_under_a_docstring_on_the_def_line() { + let out = check(&format!( + "{POINT}\ndef f(Point(x, y): Point, Point(a, b): Point): 'what f does'\n" + )); + for pattern in ["Point(x, y)", "Point(a, b)"] { + assert!( + out.contains(&format!( + "\n match __by_destructure_N__:\n case {pattern}:\n pass\n del " + )), + "got:\n{out}" + ); + } + } + /// a conjunction is a match per conjunct, nested inside the previous one #[test] fn and_pattern_matches_every_conjunct() { diff --git a/crates/by_transforms/src/transforms/dynamic_keyword.rs b/crates/by_transforms/src/transforms/dynamic_keyword.rs index b4185e49ff..c73b84f281 100644 --- a/crates/by_transforms/src/transforms/dynamic_keyword.rs +++ b/crates/by_transforms/src/transforms/dynamic_keyword.rs @@ -77,6 +77,10 @@ impl DynamicKeywordPass { } impl TypeAwarePass for DynamicKeywordPass { + fn lowering(&self) -> Option { + Some(super::ast_driver::Lowering::DynamicKeyword) + } + fn run(&self, stmts: &[Stmt], types: &dyn TypeInfo, ctx: &mut PassContext) { let mut inner = DynamicKeyword::new(types); walk_type_positions(stmts, Some(types), &mut inner); @@ -208,8 +212,7 @@ mod tests { check( "from typing import Callable\nf: Callable[[dynamic], dynamic]\n", indoc! {" - from typing import Any - from typing import Callable + from typing import Callable, Any f: Callable[[Any], Any] "}, ); @@ -220,8 +223,7 @@ mod tests { check( "from typing import cast\nb = cast(dynamic, a)\n", indoc! {" - from typing import Any - from typing import cast + from typing import cast, Any b = cast(Any, a) "}, ); diff --git a/crates/by_transforms/src/transforms/enums.rs b/crates/by_transforms/src/transforms/enums.rs index e5e9e21a9d..c633be1db9 100644 --- a/crates/by_transforms/src/transforms/enums.rs +++ b/crates/by_transforms/src/transforms/enums.rs @@ -427,7 +427,7 @@ fn emit_sealed_hierarchy( imports.add("typing", "ClassVar"); } for variant in variants { - let variant_class = format!("_{name}_{}", variant.name); + let variant_class = variant_class_name(source, name, &variant.name); let declared = match variant.kind { // a payload variant is the class its call constructs VariantKind::Tuple => format!("type[{variant_class}]"), @@ -449,20 +449,37 @@ fn emit_sealed_hierarchy( // variant subclasses, emitted at module level and attached to the enum for variant in variants { out.push_gen("\n"); - emit_variant_class(out, name, variant, config); + emit_variant_class(out, source, name, variant, config); } } +/// the module-level name a variant's subclass is bound to. +/// +/// a module that spells `_Shape_Circle` itself — a class of its own by that name — +/// gets a different one, because the lowering binds this name at module scope and +/// would otherwise replace what the author wrote there without saying so. the name +/// is derived rather than carried, so every place that needs it asks for it here +/// and they cannot drift apart +fn variant_class_name(source: &str, enum_name: &str, variant: &str) -> String { + super::repeated_underscore::WrittenNames::new(source).fresh(&format!("_{enum_name}_{variant}")) +} + /// Emit one variant as a module-level subclass of the enum and attach it (or, for /// a unit variant, its singleton instance) as `EnumName.Variant`. /// /// A stub gets the subclass alone. The attachment and the name reset run as the /// module does, and the enum's body already declares `EnumName.Variant` to a /// checker -fn emit_variant_class(out: &mut Out, enum_name: &str, variant: &Variant, config: &Config) { +fn emit_variant_class( + out: &mut Out, + source: &str, + enum_name: &str, + variant: &Variant, + config: &Config, +) { // a private module-level name holds the subclass; the public binding is the // attached `EnumName.Variant` - let mangled = format!("_{enum_name}_{}", variant.name); + let mangled = variant_class_name(source, enum_name, &variant.name); match variant.kind { VariantKind::Unit => { // a payload-less variant is a *value*, not a class — `A.Baz` is the @@ -1038,9 +1055,8 @@ mod tests { PythonVersion::PY310, indoc! {" from __future__ import annotations - from typing import TypeVar, Generic from dataclasses import dataclass - from typing import final, ClassVar + from typing import final, ClassVar, TypeVar, Generic _T = TypeVar(\"_T\") _E = TypeVar(\"_E\") class Result(Generic[_T, _E]): @@ -1226,4 +1242,33 @@ mod tests { "} ); } + + /// the lowering binds the variant subclass at module scope, so a module that + /// already has a class of that name keeps its own and the variant takes another + #[test] + fn a_variant_subclass_gives_way_to_a_class_of_that_name() { + let out = transpile( + indoc! {" + class _Shape_Circle: + tag = \"mine\" + + enum class Shape: + case Circle(radius: int) + "}, + &Config::test_default(), + ) + .unwrap(); + assert!( + out.contains("class _Shape_Circle:\n tag: str = \"mine\""), + "the module's own class is gone:\n{out}" + ); + assert!( + out.contains("class _Shape_Circle2(Shape):"), + "the variant did not take another name:\n{out}" + ); + assert!( + out.contains("Circle: ClassVar[type[_Shape_Circle2]]"), + "the declaration names the wrong class:\n{out}" + ); + } } diff --git a/crates/by_transforms/src/transforms/extension.rs b/crates/by_transforms/src/transforms/extension.rs index b9ce87ae43..d5cc6b8c49 100644 --- a/crates/by_transforms/src/transforms/extension.rs +++ b/crates/by_transforms/src/transforms/extension.rs @@ -37,6 +37,7 @@ use ruff_text_size::{Ranged, TextRange}; use ty_python_semantic::ExtensionMemberKind; use super::ast_driver::{Fragment, PassContext, TypeAwarePass}; +use super::repeated_underscore::{WrittenNames, positional_only_count}; use super::source_util::{is_synthetic_decorator, line_start}; use crate::type_info::TypeInfo; @@ -124,7 +125,11 @@ pub(crate) fn parse_kind_word(word: &str) -> Option { /// render a member's parameter list without annotations (they reference type /// parameters with no runtime binding). defaults pass through as source spans /// so lowerings inside them compose -fn parameter_fragments(parameters: &ast::Parameters, fragments: &mut Vec) { +fn parameter_fragments( + parameters: &ast::Parameters, + written: WrittenNames, + fragments: &mut Vec, +) { let mut first = true; let mut separate = |fragments: &mut Vec| { if !first { @@ -132,36 +137,43 @@ fn parameter_fragments(parameters: &ast::Parameters, fragments: &mut Vec 0 && index == positional_only { + separate(&mut *fragments); + fragments.push(Fragment::Lit("/".to_owned())); + } separate(fragments); - fragments.push(Fragment::Lit(param.parameter.name.to_string())); + fragments.push(Fragment::Lit(name(¶m.parameter).to_string())); if let Some(default) = ¶m.default { fragments.push(Fragment::Lit("=".to_owned())); fragments.push(Fragment::Src(default.range())); } } - if !parameters.posonlyargs.is_empty() { + if positional_only > 0 + && positional_only == parameters.posonlyargs.len() + parameters.args.len() + { separate(&mut *fragments); fragments.push(Fragment::Lit("/".to_owned())); } - for param in ¶meters.args { - separate(fragments); - fragments.push(Fragment::Lit(param.parameter.name.to_string())); - if let Some(default) = ¶m.default { - fragments.push(Fragment::Lit("=".to_owned())); - fragments.push(Fragment::Src(default.range())); - } - } if let Some(vararg) = ¶meters.vararg { separate(&mut *fragments); - fragments.push(Fragment::Lit(format!("*{}", vararg.name))); + fragments.push(Fragment::Lit(format!("*{}", name(vararg)))); } else if !parameters.kwonlyargs.is_empty() { separate(&mut *fragments); fragments.push(Fragment::Lit("*".to_owned())); } for param in ¶meters.kwonlyargs { separate(fragments); - fragments.push(Fragment::Lit(param.parameter.name.to_string())); + fragments.push(Fragment::Lit(name(¶m.parameter).to_string())); if let Some(default) = ¶m.default { fragments.push(Fragment::Lit("=".to_owned())); fragments.push(Fragment::Src(default.range())); @@ -169,19 +181,24 @@ fn parameter_fragments(parameters: &ast::Parameters, fragments: &mut Vec { source: &'a str, + written: WrittenNames<'a>, is_stub: bool, } impl<'a> ExtensionBlockPass<'a> { - pub(crate) fn new(source: &'a str, is_stub: bool) -> Self { - Self { source, is_stub } + pub(crate) fn new(source: &'a str, written: WrittenNames<'a>, is_stub: bool) -> Self { + Self { + source, + written, + is_stub, + } } /// lower one extension block to its backing functions, in place. the block @@ -273,7 +290,7 @@ impl<'a> ExtensionBlockPass<'a> { "def {}(", backing_name(target, ordinal, func.name.as_str()) ))); - parameter_fragments(&func.parameters, &mut fragments); + parameter_fragments(&func.parameters, self.written, &mut fragments); fragments.push(Fragment::Lit(")".to_owned())); let marker = format!("{EXTENSION_MARKER} {} {header}", kind_word(kind)); @@ -330,6 +347,24 @@ impl<'a> ExtensionBlockPass<'a> { } impl TypeAwarePass for ExtensionBlockPass<'_> { + /// a member's backing function writes its own header: no annotations, no modifiers or + /// `context` prefixes, the extension's type parameters as the class's own, and each + /// parameter under the name ty decided, with the `/` a repeated `_` needs + fn subsumes(&self) -> &'static [super::ast_driver::Lowering] { + &[ + super::ast_driver::Lowering::Callable, + super::ast_driver::Lowering::OptionalType, + super::ast_driver::Lowering::LiteralType, + super::ast_driver::Lowering::JustFloat, + super::ast_driver::Lowering::DynamicKeyword, + super::ast_driver::Lowering::FloatConst, + super::ast_driver::Lowering::Modifiers, + super::ast_driver::Lowering::ContextParams, + super::ast_driver::Lowering::GenericPolyfill, + super::ast_driver::Lowering::RepeatedUnderscore, + ] + } + fn run(&self, stmts: &[Stmt], types: &dyn TypeInfo, ctx: &mut PassContext) { // occurrence index per target name, module-wide — the mangle // discriminator shared with ty's `backing_function_name` @@ -645,7 +680,14 @@ impl<'ast> Visitor<'ast> for ExtensionCallLower<'_> { // parameters have to be filled here — the injection // `context_params` anchors to the closing paren lands // inside the range being replaced and is dropped - for (parameter, variable) in self.types.implicit_context_arguments(call) { + let implicit = match self.types.implicit_context_arguments(call) { + Ok(implicit) => implicit, + Err(error) => { + self.errors.push(error); + Vec::new() + } + }; + for (parameter, variable) in implicit { if written { fragments.push(Fragment::Lit(", ".to_owned())); } @@ -822,6 +864,20 @@ mod tests { assert!(!out.contains("extension list"), "got:\n{out}"); } + /// the backing function declares the member's parameters as python binds them, and + /// two parameters named `_` are a `SyntaxError` there. the numbered one is + /// positional-only, as it is on any `def` + #[test] + fn a_repeated_underscore_in_a_member_is_numbered() { + let out = check( + "extension list:\n def pick(self, _: int, _: int, i: int) -> Element:\n return self[i]\n", + ); + assert!( + out.contains("def _by_ext__list__pick(self, _, _2, /, i):"), + "got:\n{out}" + ); + } + #[test] fn backing_function_is_hoisted_above_an_earlier_call() { // a member called before the block's source position must still resolve diff --git a/crates/by_transforms/src/transforms/float_const.rs b/crates/by_transforms/src/transforms/float_const.rs index 4e238dd244..cf4ca7c55d 100644 --- a/crates/by_transforms/src/transforms/float_const.rs +++ b/crates/by_transforms/src/transforms/float_const.rs @@ -103,6 +103,10 @@ impl FloatConstPass { } impl TypeAwarePass for FloatConstPass { + fn lowering(&self) -> Option { + Some(super::ast_driver::Lowering::FloatConst) + } + fn run(&self, stmts: &[Stmt], types: &dyn TypeInfo, ctx: &mut PassContext) { let mut inner = FloatConst::new(types); walk_type_positions(stmts, Some(types), &mut inner); diff --git a/crates/by_transforms/src/transforms/force_unwrap.rs b/crates/by_transforms/src/transforms/force_unwrap.rs index 849ec153e3..55073d0f58 100644 --- a/crates/by_transforms/src/transforms/force_unwrap.rs +++ b/crates/by_transforms/src/transforms/force_unwrap.rs @@ -14,21 +14,29 @@ //! return _v //! ``` //! -//! The rewrite uses narrow edits — a `_force_unwrap(` insertion at the operand -//! start and a `)` replacement of the trailing `!` — so the operand's bytes -//! are left untouched and any sibling operator lowering inside it (e.g. `?.` -//! or `??`) still applies. Nested `expr!!` composes: the two insertions at the -//! same offset concatenate, yielding `_force_unwrap(_force_unwrap(expr))`. +//! The rewrite is one template edit over the whole `expr!`, passing the operand +//! through: `_force_unwrap(` + everything but the trailing `!` + `)`. The +//! passthrough leaves the operand's bytes untouched — its own parentheses +//! included — so any sibling operator lowering inside it (`?.`, `??`, a grapheme +//! accessor) materializes within the wrap, and nested `expr!!` nests the +//! templates: `_force_unwrap(_force_unwrap(expr))`. +//! +//! One edit and not two. The wrap used to be an insertion at the operand's start +//! plus a `)` over the `!`, and a template edit on the operand — `s.first` +//! becoming `(Character(_by_graphemes(s)[0]) if s else None)` — separated them: +//! the insertion was absorbed into the template and re-emitted once per +//! passthrough of the receiver, while the `)` stayed outside, so `s.first!` +//! produced `_force_unwrap(s` twice and no matching close. use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; use ruff_python_ast::{Expr, Stmt, UnaryOp}; use ruff_text_size::{Ranged, TextRange, TextSize}; -use super::ast_driver::{PassContext, TypeAwarePass}; +use super::ast_driver::{Fragment, PassContext, TypeAwarePass}; use crate::type_info::TypeInfo; struct ForceUnwrap { - edits: Vec<(TextRange, String)>, + edits: Vec<(TextRange, Vec)>, used: bool, } @@ -50,19 +58,18 @@ impl<'ast> Visitor<'ast> for ForceUnwrap { if let Expr::UnaryOp(unary) = expr && unary.op == UnaryOp::Force { - // `_force_unwrap(` before the operand (a zero-width insertion; nested - // forces at the same offset concatenate in push order) - self.edits.push(( - TextRange::empty(expr.range().start()), - "_force_unwrap(".to_owned(), - )); - // `)` in place of the trailing `!` only — replacing the whole gap - // between the operand and `!` would swallow a parenthesised - // operand's own closing paren (`(a?.v)!` → unbalanced), and the - // operand's interior bytes are left for sibling lowerings (`?.`/`??`) + // everything but the trailing `!`, which is the operand as written — + // a parenthesised operand keeps its own parens, and the bytes inside + // are left for the sibling lowerings that key edits on them + let operand = + TextRange::new(expr.range().start(), expr.range().end() - TextSize::from(1)); self.edits.push(( - TextRange::new(expr.range().end() - TextSize::from(1), expr.range().end()), - ")".to_owned(), + expr.range(), + vec![ + Fragment::Lit("_force_unwrap(".to_owned()), + Fragment::Src(operand), + Fragment::Lit(")".to_owned()), + ], )); self.used = true; } @@ -93,7 +100,7 @@ impl TypeAwarePass for ForceUnwrapPass<'_> { ctx.runtime.insert(crate::runtime::OPTIONAL); ctx.runtime.insert(crate::runtime::FORCE_UNWRAP); } - ctx.text_edits.extend(inner.edits); + ctx.template_edits.extend(inner.edits); } } @@ -239,6 +246,85 @@ mod tests { ); } + /// the last line of the output, which is the statement the source's last line lowered to + fn lowered(source: &str) -> String { + let out = transpile(source, &Config::test_default()) + .unwrap_or_else(|error| panic!("{source:?}: {error}")); + out.lines().last().expect("some output").to_owned() + } + + /// `!` wraps whatever the operand lowered to, whether that lowering replaces the + /// operand with something wider (a grapheme accessor, an optional chain) or leaves + /// it alone. the wrap is one edit, so the operand's own rewrite materializes inside + /// it exactly once + #[test] + fn force_unwrap_wraps_a_lowered_operand() { + // a grapheme accessor, which re-emits its receiver twice: the reported shape + // (`Expected ',', found 'else'`) was the `_force_unwrap(` landing on both + assert_eq!( + lowered("def go(s: str) -> None:\n print(s.first!)\n"), + " print(_force_unwrap((Character(_by_graphemes(s)[0]) if s else None)))" + ); + // an optional chain + assert_eq!( + lowered("def go(a: int?) -> None:\n print(a?.bit_length()!)\n"), + " print(_force_unwrap((None if a is None else a.bit_length())))" + ); + // a subscript, a call and a comparison, none of which the operand rewrites + assert_eq!( + lowered("def go(xs: list[int?]) -> None:\n print(xs[0]!)\n"), + " print(_force_unwrap(xs[0]))" + ); + assert_eq!( + lowered("def f() -> int?:\n return 1\n\nprint(f()!)\n"), + "print(_force_unwrap(f()))" + ); + assert_eq!( + lowered("def go(a: int?) -> None:\n print(a! > 1)\n"), + " print(_force_unwrap(a) > 1)" + ); + } + + /// and inside an f-string replacement field, where the `!` may be bare: it is read as + /// the start of a conversion (`!r`) only where a conversion could follow it + #[test] + fn force_unwrap_inside_an_f_string() { + assert_eq!( + lowered("def go(s: str) -> None:\n print(f\"{(s.first!)}\")\n"), + " print(f\"{(_force_unwrap((Character(_by_graphemes(s)[0]) if s else None)))}\")" + ); + assert_eq!( + lowered("def go(a: int?) -> None:\n print(f\"{a!}\")\n"), + " print(f\"{_force_unwrap(a)}\")" + ); + // before the `:` of a format spec, and before the conversion's own `!` + assert_eq!( + lowered("def go(a: int?) -> None:\n print(f\"{a!:>4}\")\n"), + " print(f\"{_force_unwrap(a):>4}\")" + ); + assert_eq!( + lowered("def go(a: int?) -> None:\n print(f\"{a!!r}\")\n"), + " print(f\"{_force_unwrap(a)!r}\")" + ); + // a nested f-string, and a field inside a format spec + assert_eq!( + lowered("def go(a: int?) -> None:\n print(f\"{f'{a!}'}\")\n"), + " print(f\"{f'{_force_unwrap(a)}'}\")" + ); + assert_eq!( + lowered("def go(a: int?, b: int) -> None:\n print(f\"{b:{a!}}\")\n"), + " print(f\"{b:{_force_unwrap(a)}}\")" + ); + // the conversion the `!` is usually reaching for is untouched, and a `!` glued to + // `=` is still the comparison operator + assert_eq!( + lowered( + "def go(a: int, b: int) -> None:\n print(f\"{a!r} {a!s:>10} {a=!r} {a!=b}\")\n" + ), + " print(f\"{a!r} {a!s:>10} {a=!r} {a!=b}\")" + ); + } + #[test] fn plain_python_unchanged() { unchanged("x = a\n"); diff --git a/crates/by_transforms/src/transforms/generics.rs b/crates/by_transforms/src/transforms/generics.rs index 09708a6bed..715f2d2908 100644 --- a/crates/by_transforms/src/transforms/generics.rs +++ b/crates/by_transforms/src/transforms/generics.rs @@ -8,6 +8,7 @@ use ruff_text_size::{Ranged, TextRange, TextSize}; use crate::config::Config; use crate::transforms::callable::lower_type_expr_full; +use crate::transforms::repeated_underscore::WrittenNames; use crate::type_info::TypeInfo; use ruff_python_ast::PythonVersion; @@ -18,6 +19,7 @@ use ruff_python_ast::PythonVersion; /// - `type Alias = T` → `Alias: TypeAlias = T` pub(crate) struct GenericPolyfill<'src> { source: &'src str, + written: WrittenNames<'src>, types: &'src dyn TypeInfo, config: Config, edits: Vec, @@ -155,6 +157,7 @@ struct ProcessedTypeParams { impl<'src> GenericPolyfill<'src> { fn new( source: &'src str, + written: WrittenNames<'src>, types: &'src dyn TypeInfo, config: Config, symbolic_substitutions: Vec<(TextRange, String)>, @@ -162,6 +165,7 @@ impl<'src> GenericPolyfill<'src> { ) -> Self { Self { source, + written, types, config, edits: Vec::new(), @@ -473,6 +477,7 @@ impl<'src> GenericPolyfill<'src> { } else { lower_type_expr_full( self.source, + self.written, self.types, bound, &self.subsume_within(bound.range()), @@ -490,6 +495,7 @@ impl<'src> GenericPolyfill<'src> { if let Some(default) = &tv.default { let default_src = lower_type_expr_full( self.source, + self.written, self.types, default, &self.subsume_within(default.range()), @@ -934,6 +940,7 @@ impl<'src> GenericPolyfill<'src> { let value_src = lower_type_expr_full( self.source, + self.written, self.types, &alias.value, &substitutions, @@ -1294,16 +1301,41 @@ pub(crate) fn mangle(name: &str) -> String { pub(crate) struct GenericPolyfillPass<'src> { source: &'src str, + written: WrittenNames<'src>, config: Config, } impl<'src> GenericPolyfillPass<'src> { - pub(crate) fn new(source: &'src str, config: Config) -> Self { - Self { source, config } + pub(crate) fn new(source: &'src str, written: WrittenNames<'src>, config: Config) -> Self { + Self { + source, + written, + config, + } } } impl super::ast_driver::TypeAwarePass for GenericPolyfillPass<'_> { + fn lowering(&self) -> Option { + Some(super::ast_driver::Lowering::GenericPolyfill) + } + + /// a header or alias it polyfills is re-printed with its type parameters renamed, and it + /// re-renders what the lowerings inside wrote rather than their source — the symbolic + /// folds, the private alias names, the tuple, named tuple and match types, and the variance + /// keywords as `TypeVar` arguments + fn subsumes(&self) -> &'static [super::ast_driver::Lowering] { + &[ + super::ast_driver::Lowering::TupleLiteralType, + super::ast_driver::Lowering::AnonNamedTuple, + super::ast_driver::Lowering::MatchType, + super::ast_driver::Lowering::SymbolicTypeOp, + super::ast_driver::Lowering::Modifiers, + super::ast_driver::Lowering::VisibilityRename, + super::ast_driver::Lowering::VarianceStrip, + ] + } + fn run( &self, stmts: &[ruff_python_ast::Stmt], @@ -1312,6 +1344,7 @@ impl super::ast_driver::TypeAwarePass for GenericPolyfillPass<'_> { ) { let mut inner = GenericPolyfill::new( self.source, + self.written, types, self.config.clone(), ctx.symbolic_substitutions.clone(), @@ -2329,8 +2362,7 @@ mod tests { class A[P: (*: *, **: *)]: ... "}, indoc! {" - from typing import ParamSpec, Generic - from typing import TypeVar + from typing import TypeVar, ParamSpec, Generic _P = ParamSpec(\"_P\") class A(Generic[_P]): ... diff --git a/crates/by_transforms/src/transforms/if_let.rs b/crates/by_transforms/src/transforms/if_let.rs index f7c6ea1494..0545e4e7d8 100644 --- a/crates/by_transforms/src/transforms/if_let.rs +++ b/crates/by_transforms/src/transforms/if_let.rs @@ -41,6 +41,7 @@ use ruff_text_size::{Ranged, TextRange, TextSize}; use super::ast_driver::{Fragment, PassContext, TypeAwarePass}; use super::destructure::{NameGen, push_destructure}; +use super::repeated_underscore::WrittenNames; use super::source_util::{line_indent, temporary_name}; use crate::type_info::TypeInfo; @@ -62,7 +63,7 @@ impl TypeAwarePass for IfLetPass<'_> { edits: Vec::new(), errors: Vec::new(), counter: 0, - names: NameGen::default(), + names: NameGen::new(self.source), }; for stmt in stmts { lower.visit_stmt(stmt); @@ -88,7 +89,7 @@ struct IfLetLower<'a, 'src> { /// monotonic across the file so sibling chains get distinct selectors counter: usize, /// names the temporaries a clause's destructuring needs - names: NameGen, + names: NameGen<'src>, } impl<'ast> Visitor<'ast> for IfLetLower<'_, '_> { @@ -105,7 +106,7 @@ impl IfLetLower<'_, '_> { /// generated assignments can't shadow anything the source binds fn fresh_selector(&mut self, anchor: &Expr) -> String { loop { - let name = temporary_name("if_let", self.counter); + let name = temporary_name(WrittenNames::new(self.source), "if_let", self.counter); self.counter += 1; if self.types.is_unbound_at(&name, anchor) { return name; @@ -260,7 +261,7 @@ fn push_guard( value: usize, pattern: Option<&Pattern>, test: &Expr, - names: &mut NameGen, + names: &mut NameGen<'_>, ) -> Result<(), String> { match pattern { Some(pattern) => push_destructure( diff --git a/crates/by_transforms/src/transforms/inferred_annotation.rs b/crates/by_transforms/src/transforms/inferred_annotation.rs index 87fb73c3b2..43c9311224 100644 --- a/crates/by_transforms/src/transforms/inferred_annotation.rs +++ b/crates/by_transforms/src/transforms/inferred_annotation.rs @@ -29,18 +29,25 @@ use ruff_text_size::{Ranged, TextRange}; use crate::transforms::ast_driver::{PassContext, TypeAwarePass}; use crate::type_info::TypeInfo; -pub(crate) struct InferredAnnotationPass; +pub(crate) struct InferredAnnotationPass { + min_version: ruff_python_ast::PythonVersion, +} impl InferredAnnotationPass { - pub(crate) fn new() -> Self { - Self + pub(crate) fn new(min_version: ruff_python_ast::PythonVersion) -> Self { + Self { min_version } } } impl TypeAwarePass for InferredAnnotationPass { + fn lowering(&self) -> Option { + Some(super::ast_driver::Lowering::InferredAnnotation) + } + fn run(&self, stmts: &[Stmt], types: &dyn TypeInfo, ctx: &mut PassContext) { let mut state = State { types, + min_version: self.min_version, edits: Vec::new(), modules: BTreeSet::new(), typing_names: BTreeSet::new(), @@ -62,6 +69,7 @@ impl TypeAwarePass for InferredAnnotationPass { struct State<'a> { types: &'a dyn TypeInfo, + min_version: ruff_python_ast::PythonVersion, edits: Vec<(TextRange, String)>, /// modules a synthesized annotation names but the source never imported modules: BTreeSet, @@ -93,7 +101,10 @@ impl State<'_> { if is_dunder(name.id.as_str()) { return; } - let Some(annotation) = self.types.inferred_annotation(&assign.value) else { + let Some(annotation) = self + .types + .inferred_annotation(&assign.value, self.min_version) + else { return; }; let pos = name.range().end(); @@ -252,6 +263,33 @@ mod tests { ); } + /// and the block stays a block. it opens with a `from typing import TYPE_CHECKING`, + /// and merging the imports by module treated the whole thing as one such line — every + /// `typing` name that sorts after `TYPE_CHECKING` was then appended to the block's last + /// line, inside the block: `if TYPE_CHECKING:\n import decimal, final` + #[test] + fn a_typing_import_is_not_merged_into_the_type_checking_block() { + check( + indoc! {" + from decimal import Decimal as Dec + + final class A: + d = Dec(1) + "}, + indoc! {" + from typing import final + from typing import TYPE_CHECKING + if TYPE_CHECKING: + import decimal + from decimal import Decimal as Dec + + @final + class A: + d: decimal.Decimal = Dec(1) + "}, + ); + } + #[test] fn a_class_shadowed_by_its_own_module_is_qualified() { // `datetime` names the *module* in the output, so the bare display would @@ -295,6 +333,38 @@ mod tests { ); } + /// ty reports the modules a *type* lives in, which is not the same question as which + /// ones its spelling names: the type spelled `None` is reported as living in `types`, + /// and importing it bought the emitted file a `if TYPE_CHECKING: import types` block + /// that nothing in it read + #[test] + fn a_module_the_spelling_does_not_name_is_not_imported() { + check( + indoc! {" + class A: + a = None + "}, + indoc! {" + class A: + a: None = None + "}, + ); + } + + /// a `typing` name basedpython does not bind implicitly has no import to write, so the + /// annotation is refused rather than written. `Literal` is such a name, and a generator + /// keeps one inside its element type where promotion cannot reach it — written out, the + /// emitted file read a `Literal` it never imported + #[test] + fn an_annotation_naming_an_unimportable_typing_name_is_refused() { + let source = "class A:\n g = (x for x in [1])\n"; + let config = Config { + min_version: ruff_python_ast::PythonVersion::PY313, + ..Config::test_default() + }; + assert_eq!(transpile(source, &config).unwrap(), source); + } + #[test] fn a_class_local_to_a_function_is_left_bare() { // its qualified name is not a dotted path, so it has no spelling at all diff --git a/crates/by_transforms/src/transforms/init_method.rs b/crates/by_transforms/src/transforms/init_method.rs index 74748569ba..f910e29c8f 100644 --- a/crates/by_transforms/src/transforms/init_method.rs +++ b/crates/by_transforms/src/transforms/init_method.rs @@ -31,12 +31,14 @@ use ruff_text_size::{Ranged, TextRange, TextSize}; use super::ast_driver::{Fragment, PassContext, TypeAwarePass}; use super::callable::lower_type_expr_full; use super::mutable_defaults::{parameter_guards, undeclarable_error}; -use super::source_util::{PrologueStatement, first_body_statement}; +use super::repeated_underscore::WrittenNames; +use super::source_util::{PrologueStatement, body_prologue, first_body_statement}; use crate::config::FloatLiteralLowering; use crate::type_info::TypeInfo; pub(crate) struct InitMethod<'src> { source: &'src str, + written: WrittenNames<'src>, float_literals: FloatLiteralLowering, is_stub: bool, } @@ -44,11 +46,13 @@ pub(crate) struct InitMethod<'src> { impl<'src> InitMethod<'src> { pub(crate) fn new( source: &'src str, + written: WrittenNames<'src>, float_literals: FloatLiteralLowering, is_stub: bool, ) -> Self { Self { source, + written, float_literals, is_stub, } @@ -59,6 +63,7 @@ impl TypeAwarePass for InitMethod<'_> { fn run(&self, stmts: &[Stmt], types: &dyn TypeInfo, ctx: &mut PassContext) { let mut state = State { source: self.source, + written: self.written, types, symbolic_substitutions: ctx.symbolic_substitutions.clone(), float_literals: self.float_literals, @@ -73,7 +78,8 @@ impl TypeAwarePass for InitMethod<'_> { state.visit_stmt(stmt); } if state.needs_missing.into_inner() { - ctx.required_imports.push("_MISSING = object()".to_owned()); + ctx.required_imports + .extend(super::mutable_defaults::sentinel_definition(self.source)); } ctx.text_edits.extend(state.edits.into_inner()); ctx.template_edits.extend(state.templates.into_inner()); @@ -102,8 +108,34 @@ fn is_init_owned_modifier(word: &str) -> bool { matches!(word, "let" | "var" | "private" | "protected" | "public") } +/// One `self.: = ` line a declaring parameter stands for. The +/// line is written in full here and never continues onto a second one, so it +/// re-establishes no indentation of its own +struct SelfAssignment(String); + +impl PrologueStatement for SelfAssignment { + fn push(&self, frags: &mut Vec, _indent: &str) { + frags.push(Fragment::Lit(self.0.clone())); + } +} + +/// The text of fragments that pass no source through. A self-assignment is +/// written in full, so the insertion it sits in is plain text: a template would +/// absorb the zero-width insertions a sibling lowering left at the same offset, +/// and those belong to the statement the assignments precede +fn literal_text(frags: &[Fragment]) -> String { + frags + .iter() + .map(|frag| match frag { + Fragment::Lit(lit) => lit.as_str(), + Fragment::Src(_) => "", + }) + .collect() +} + struct State<'src> { source: &'src str, + written: WrittenNames<'src>, types: &'src dyn TypeInfo, /// `(range, rendered)` for every symbolic fold in the module. the `__init__` /// line is fresh output, so a fold inside a parameter annotation is dropped @@ -171,6 +203,7 @@ impl State<'_> { .collect(); lower_type_expr_full( self.source, + self.written, self.types, ann, &substitutions, @@ -225,7 +258,7 @@ impl State<'_> { // 2. collect attribute-declaring parameters (`let` / `var`, optionally // `private` / `public`) from every slot, strip the modifier prefix // from the source, and validate the combination - let mut let_assignments: Vec = Vec::new(); + let mut let_assignments: Vec = Vec::new(); let mut handle = |param: &Parameter| { let Some((prefix_range, words)) = self.modifier_prefix(param) else { return; @@ -278,13 +311,14 @@ impl State<'_> { // `protected`. the parameter itself keeps its declared name let attr = visibility_rename(name, visibility.name_prefix()) .unwrap_or_else(|| name.to_owned()); + let value = crate::python_parameter_name(&func.parameters, param, self.written); let line = if let Some(ann) = ¶m.annotation { let ann_src = self.lower_annotation(ann); - format!("self.{attr}: {ann_src} = {name}") + format!("self.{attr}: {ann_src} = {value}") } else { - format!("self.{attr} = {name}") + format!("self.{attr} = {value}") }; - let_assignments.push(line); + let_assignments.push(SelfAssignment(line)); }; for p in &func.parameters.posonlyargs { handle(&p.parameter); @@ -305,18 +339,18 @@ impl State<'_> { // 3. insert the self-assignments. whether there is a body at all is the // question `mutable_defaults` asks too, so the two never disagree about // which of them writes the guards a default needs - let first_user_stmt = first_body_statement(func); - if let Some(first) = first_user_stmt { - if !let_assignments.is_empty() { - let stmt_indent = self.line_indent(first.range().start()).to_owned(); - let mut text = String::new(); - for line in &let_assignments { - text.push_str(line); - text.push('\n'); - text.push_str(&stmt_indent); + if first_body_statement(func).is_some() { + // the same anchor `mutable_defaults` hangs its guards off, so the two + // agree about where the top of the body is: after a docstring, and on + // a line of its own when the source wrote the suite on the clause's + // line, where the class body's indentation would put the statements + // outside the method + if !let_assignments.is_empty() + && let Some(anchored) = body_prologue(self.source, func, &let_assignments) + { + for (at, frags) in anchored { + self.push(TextRange::new(at, at), literal_text(&frags)); } - let pos = first.range().start(); - self.push(TextRange::new(pos, pos), text); } } else { // the whole body is written here, so the guards a defaulted or @@ -327,7 +361,13 @@ impl State<'_> { written, guards, undeclarable, - } = parameter_guards(func, self.types, self.is_stub); + } = parameter_guards( + func, + self.written, + &super::mutable_defaults::sentinel_name(self.source), + self.types, + self.is_stub, + ); if let Some(parameter) = undeclarable.first() { self.errors .borrow_mut() @@ -345,7 +385,8 @@ impl State<'_> { guard.push(&mut frags, &body_indent); } for line in &let_assignments { - frags.push(Fragment::Lit(format!("\n{body_indent}{line}"))); + frags.push(Fragment::Lit(format!("\n{body_indent}"))); + line.push(&mut frags, &body_indent); } } // an inherited default is a signature edit that needs no guard, so the sentinels @@ -357,17 +398,15 @@ impl State<'_> { // no passthrough to carry, so emit plain text: a template // absorbs the zero-width insertions a sibling lowering left at // this offset, and here they belong to the signature - let text = frags - .iter() - .map(|frag| match frag { - Fragment::Lit(lit) => lit.as_str(), - Fragment::Src(_) => "", - }) - .collect::(); - self.push(TextRange::new(pos, pos), text); + self.push(TextRange::new(pos, pos), literal_text(&frags)); return; } - *self.needs_missing.borrow_mut() = true; + if guards + .iter() + .any(super::mutable_defaults::Guard::uses_sentinel) + { + *self.needs_missing.borrow_mut() = true; + } self.templates .borrow_mut() .push((TextRange::new(pos, pos), frags)); @@ -446,6 +485,25 @@ mod tests { ); } + /// a declaring parameter assigns its attribute from the parameter python binds, and + /// a repeated `_` is numbered there: read by its source name, the second assignment + /// would store the first argument again + #[test] + fn a_repeated_underscore_declares_from_its_numbered_name() { + check( + indoc! {" + class A: + init(let _: int, let _: int) + "}, + indoc! {" + class A: + def __init__(self, _: int, _2: int, /): + self._: int = _ + self._: int = _2 + "}, + ); + } + // `var` is accepted as the mutable counterpart of `let`; for an init // parameter it self-assigns identically #[test] @@ -640,6 +698,136 @@ mod tests { ); } + /// a body written on the `init` clause's own line has no line of its own for the + /// attribute declaration to precede it on. written at the body statement's + /// indentation — which is the *class body's* — the statement fell out of the method + /// and ran at class creation, where the parameter is not a name at all + #[test] + fn an_inline_body_stays_inside_the_method() { + check( + indoc! {" + class A: + init(let z: int): print(z) + "}, + indoc! {" + class A: + def __init__(self, z: int): \n self.z: int = z + print(z) + "}, + ); + } + + /// the declaration follows a docstring rather than displacing it: written above one, + /// the string stops being the first statement and the method loses its `__doc__` + #[test] + fn a_declaration_follows_the_docstring() { + check( + indoc! {" + class A: + init(let a: int): + \"\"\"doc\"\"\" + print(a) + "}, + indoc! {" + class A: + def __init__(self, a: int): + \"\"\"doc\"\"\" + self.a: int = a + print(a) + "}, + ); + } + + /// and it follows the guard that fills its parameter in, which `mutable_defaults` + /// writes at the same anchor — read before the guard runs, the attribute would be + /// stored as the sentinel + #[test] + fn a_declaration_follows_the_guard_for_its_own_default() { + check( + indoc! {" + class A: + init(let a: list[int] = []): + \"\"\"doc\"\"\" + print(a) + "}, + indoc! {" + from typing import Any + _MISSING: Any = object() + class A: + def __init__(self, a: list[int] = _MISSING): + \"\"\"doc\"\"\" + if a is _MISSING: + a = [] + self.a: list[int] = a + print(a) + "}, + ); + } + + /// a docstring written on the clause's line is a body of exactly one statement, and + /// nothing can follow a statement there. the docstring moves onto a line of its own so + /// the declaration has somewhere to go — written after it where it was, the emitted + /// line was indented under a suite that had already closed, and the whole file failed + /// to parse + #[test] + fn a_docstring_on_the_clause_line_is_given_a_line_of_its_own() { + check( + indoc! {" + class A: + init(let a: int): \"\"\"doc\"\"\" + "}, + indoc! {" + class A: + def __init__(self, a: int): \n \"\"\"doc\"\"\" + self.a: int = a + "}, + ); + } + + /// and the break is one insertion of its own, ahead of the docstring, so a second pass + /// writing into the same body still composes: `mutable_defaults` and this pass both + /// write at the offset after the docstring, in the order they run + #[test] + fn two_passes_write_under_a_docstring_on_the_clause_line() { + check( + indoc! {" + class A: + init(let a: list[int] = []): \"\"\"doc\"\"\" + "}, + indoc! {" + from typing import Any + _MISSING: Any = object() + class A: + def __init__(self, a: list[int] = _MISSING): \n \n \"\"\"doc\"\"\" + if a is _MISSING: + a = [] + self.a: list[int] = a + "}, + ); + } + + /// and a clause-line body that holds more than the docstring is broken the same way: + /// the break stays ahead of the docstring, and both passes write ahead of the statement + /// that follows it + #[test] + fn a_docstring_followed_by_statements_on_the_clause_line_is_given_a_line_of_its_own() { + check( + indoc! {" + class A: + init(let a: list[int] = []): \"\"\"doc\"\"\"; print(a) + "}, + indoc! {" + from typing import Any + _MISSING: Any = object() + class A: + def __init__(self, a: list[int] = _MISSING): \n \n \"\"\"doc\"\"\"; \n if a is _MISSING: + a = [] + \n self.a: list[int] = a + print(a) + "}, + ); + } + #[test] fn init_no_params_other_than_self() { check( diff --git a/crates/by_transforms/src/transforms/just_float.rs b/crates/by_transforms/src/transforms/just_float.rs index 85c128cb0c..22e3179130 100644 --- a/crates/by_transforms/src/transforms/just_float.rs +++ b/crates/by_transforms/src/transforms/just_float.rs @@ -98,6 +98,10 @@ impl JustFloatPass { } impl TypeAwarePass for JustFloatPass { + fn lowering(&self) -> Option { + Some(super::ast_driver::Lowering::JustFloat) + } + fn run(&self, stmts: &[Stmt], types: &dyn TypeInfo, ctx: &mut PassContext) { let mut inner = JustFloat::new(types); walk_type_positions(stmts, Some(types), &mut inner); diff --git a/crates/by_transforms/src/transforms/kw_subscript.rs b/crates/by_transforms/src/transforms/kw_subscript.rs index 6a7375e00b..1197f60a1e 100644 --- a/crates/by_transforms/src/transforms/kw_subscript.rs +++ b/crates/by_transforms/src/transforms/kw_subscript.rs @@ -322,6 +322,11 @@ impl<'src> KwSubscriptPass<'src> { } impl TypeAwarePass for KwSubscriptPass<'_> { + /// the value of a keyword subscript it rewrites is printed with its `T?` lowered + fn subsumes(&self) -> &'static [super::ast_driver::Lowering] { + &[super::ast_driver::Lowering::OptionalType] + } + fn run(&self, stmts: &[Stmt], types: &dyn TypeInfo, ctx: &mut PassContext) { let mut inner: KwSubscript<'_, dyn TypeInfo> = KwSubscript::new(self.source, Some(types), self.min_version); diff --git a/crates/by_transforms/src/transforms/lazy_import.rs b/crates/by_transforms/src/transforms/lazy_import.rs index 43306d6075..e9ff18b49a 100644 --- a/crates/by_transforms/src/transforms/lazy_import.rs +++ b/crates/by_transforms/src/transforms/lazy_import.rs @@ -26,7 +26,7 @@ //! A stub defers nothing — see [`Deferral::Never`]. use ruff_diagnostics::{Edit, Fix}; -use ruff_python_ast::visitor::{Visitor, walk_stmt}; +use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; use ruff_python_ast::{Stmt, StmtImport, StmtImportFrom}; use ruff_text_size::{Ranged, TextRange, TextSize}; @@ -60,7 +60,9 @@ pub(crate) struct LazyImport<'src> { /// bindings that must be bound to the real object rather than to a proxy. /// cpython's `except` checks that what it catches is a class inheriting /// `BaseException` and never consults `__instancecheck__`, so an exception - /// class reached through the proxy raises `TypeError` from the handler + /// class reached through the proxy raises `TypeError` from the handler; a + /// special form, and a name an annotation reads, are told apart by identity + /// by whatever reads them — see [`annotation_names`] eager_names: Vec, deferral: Deferral, pub(crate) edits: Vec, @@ -307,6 +309,10 @@ impl<'src> LazyImport<'src> { // generation), so bind them to the builtin instead let is_ty_ext = !is_relative && module_part == "ty_extensions"; let mut lines: Vec = Vec::new(); + // the names of this statement the pass leaves eager, and the slot in + // `lines` the one import line they share is written into + let mut eager_spellings: Vec = Vec::new(); + let mut eager_line: Option = None; // whether the statement imported a bare `Character` that contributed no // binding line — the import must still be *removed* (the preamble class // defines `Character`), so force an empty replacement below @@ -351,7 +357,14 @@ impl<'src> LazyImport<'src> { } else { format!("{name} as {bind}") }; - lines.push(format!("from {dots}{module_part} import {spelling}")); + // names that shared one import statement go on writing one line + // between them. the line is held at the position of the first of + // them, so it keeps its place among the lazy bindings around it + if eager_line.is_none() { + eager_line = Some(lines.len()); + lines.push(String::new()); + } + eager_spellings.push(spelling); } else if is_relative && module_part.is_empty() { // `from . import x` — `x` is a submodule of the current // package. Resolve the relative target at runtime @@ -373,6 +386,12 @@ impl<'src> LazyImport<'src> { )); } } + if let Some(index) = eager_line { + lines[index] = format!( + "from {dots}{module_part} import {}", + eager_spellings.join(", ") + ); + } // `character_only` means a bare `Character` import produced no lines but // must still be dropped (the preamble class defines `Character`) so it // doesn't survive as a runtime `from ty_extensions import Character` @@ -394,6 +413,160 @@ impl<'src> LazyImport<'src> { } } +/// every name the annotations python keeps for introspection read, strings included +/// +/// under the polyfill a `from` import is a proxy, and whatever reads an annotation back is +/// handed that proxy rather than the object: `dataclasses` tells a `KW_ONLY` pseudo-field +/// apart by asking whether the annotation *is* `dataclasses.KW_ONLY`, and +/// `typing.get_type_hints` answers with the proxy, which is not the class to anything +/// comparing by identity. a name read here is therefore bound eagerly +/// +/// the annotations are the ones python evaluates, or keeps as a string to evaluate on +/// request: a parameter's and a return's, of a `def` at any depth, and a variable's in a +/// module or class body, a class nested in a function included. a variable annotated +/// inside a function body is never evaluated nor kept, so nothing can read it back. the +/// value of a type alias is not an annotation, and neither is a type parameter's bound: +/// both are ordinary runtime values nothing tells apart by identity +/// +/// a string annotation is read as the expression it spells, because `typing.get_type_hints` +/// evaluates it against the module's globals — and under `from __future__ import +/// annotations` every annotation is one, which `dataclasses` resolves the same way. a +/// string that is not an expression names nothing +pub(crate) fn annotation_names(suite: &[Stmt], source: &str) -> Vec { + struct Names<'src> { + source: &'src str, + in_function: bool, + read: Vec, + } + + impl Names<'_> { + fn annotation(&mut self, annotation: &ruff_python_ast::Expr) { + struct Reads<'n> { + source: &'n str, + names: &'n mut Vec, + } + impl<'a> Visitor<'a> for Reads<'_> { + fn visit_expr(&mut self, expr: &'a ruff_python_ast::Expr) { + match expr { + ruff_python_ast::Expr::Name(name) => { + self.names.push(name.id.to_string()); + } + ruff_python_ast::Expr::StringLiteral(string) => { + if let Ok(parsed) = ruff_python_parser::typing::parse_type_annotation( + string, + self.source, + ) { + self.visit_expr(parsed.expression()); + } + } + _ => walk_expr(self, expr), + } + } + } + Reads { + source: self.source, + names: &mut self.read, + } + .visit_expr(annotation); + } + } + + impl<'a> Visitor<'a> for Names<'_> { + fn visit_stmt(&mut self, stmt: &'a Stmt) { + match stmt { + Stmt::FunctionDef(function) => { + for parameter in &*function.parameters { + if let Some(annotation) = parameter.annotation() { + self.annotation(annotation); + } + } + if let Some(returns) = &function.returns { + self.annotation(returns); + } + let outer = std::mem::replace(&mut self.in_function, true); + self.visit_body(&function.body); + self.in_function = outer; + } + Stmt::ClassDef(class) => { + let outer = std::mem::replace(&mut self.in_function, false); + self.visit_body(&class.body); + self.in_function = outer; + } + Stmt::AnnAssign(assign) if !self.in_function => { + self.annotation(&assign.annotation); + } + _ => walk_stmt(self, stmt), + } + } + + // an annotation is only ever found on a statement, so no expression needs a look + fn visit_expr(&mut self, _expr: &'a ruff_python_ast::Expr) {} + } + + let mut names = Names { + source, + in_function: false, + read: Vec::new(), + }; + names.visit_body(suite); + names.read +} + +/// The names every decorator in `suite` reads, which are bound eagerly. +/// +/// A decorator is the other thing read by identity rather than by value. `typing.overload` +/// says the `def` under it is one arm of an overload set — but only while it *is* +/// `typing.overload`; reached through the polyfill's proxy it is an unknown callable, and a +/// checker reading the emitted python sees two ordinary functions with `...` bodies instead +/// of the overloads the source declared. The same goes for `final`, `abstractmethod`, +/// `dataclass` and everything else whose whole meaning is which object it is. +/// +/// Binding these eagerly costs no laziness at all: a decorator is applied while the module +/// is executing, so a proxy standing in for one resolves at import time anyway — the defer +/// buys nothing and loses the declaration. +pub(crate) fn decorator_names(suite: &[Stmt]) -> Vec { + struct Reads<'n> { + names: &'n mut Vec, + } + + impl<'a> Visitor<'a> for Reads<'_> { + fn visit_expr(&mut self, expr: &'a ruff_python_ast::Expr) { + if let ruff_python_ast::Expr::Name(name) = expr { + self.names.push(name.id.to_string()); + } + walk_expr(self, expr); + } + } + + struct Names { + read: Vec, + } + + impl<'a> Visitor<'a> for Names { + fn visit_stmt(&mut self, stmt: &'a Stmt) { + let decorators = match stmt { + Stmt::FunctionDef(function) => function.decorator_list.as_slice(), + Stmt::ClassDef(class) => class.decorator_list.as_slice(), + _ => &[], + }; + for decorator in decorators { + Reads { + names: &mut self.read, + } + .visit_expr(&decorator.expression); + } + walk_stmt(self, stmt); + } + + // a decorator is only ever found on a statement, so no expression needs a look + fn visit_expr(&mut self, _expr: &'a ruff_python_ast::Expr) {} + } + + let mut names = Names { read: Vec::new() }; + names.visit_body(suite); + names.read +} + /// whether `test` is the `TYPE_CHECKING` flag, spelled bare or through the module it /// comes from fn is_type_checking_test(test: &ruff_python_ast::Expr) -> bool { @@ -859,6 +1032,227 @@ mod tests { ); } + #[test] + fn polyfill_special_forms_stay_eager() { + // a special form is recognised by what it *is*: a checker reading the output + // rejects `ClassVar[int]` once `ClassVar` names a variable, and `dataclasses` + // tells a bare `ClassVar` or `Final` annotation from a field by identity, which + // a proxy does not have. an ordinary function beside them is still deferred + let out = transpile_polyfill(indoc! {" + from typing import ClassVar, Final, Literal, Annotated as Note, cast + "}); + assert!( + out.ends_with(indoc! {" + from typing import ClassVar, Final, Literal, Annotated as Note + cast = _lazy_attr(\"typing\", \"cast\") + "}), + "got:\n{out}" + ); + } + + #[test] + fn polyfill_special_form_the_lowering_imported_stays_eager() { + // an enum with payloads lowers to classes whose members are annotated + // `ClassVar[...]`, and the import of `ClassVar` is the lowering's own — not in + // the source the transpile started from, so it is the output that is asked + let out = transpile_polyfill(indoc! {" + enum class Shape: + case Circle(radius: int) + case Empty + "}); + assert!( + out.contains("ClassVar[type[_Shape_Circle]]"), + "the enum no longer lowers through ClassVar, got:\n{out}" + ); + assert!( + !out.contains("ClassVar = _lazy_attr"), + "a special form was deferred, got:\n{out}" + ); + assert!( + out.contains("from typing import final, ClassVar\n"), + "got:\n{out}" + ); + } + + #[test] + fn polyfill_names_an_annotation_reads_stay_eager() { + // `dataclasses` tells a `KW_ONLY` annotation from a field by identity, and a + // type hint read back is compared by identity, so a name any annotation reads is + // bound to the real object. the decorator beside them is read by identity too — + // see `polyfill_a_decorator_stays_eager` + let out = transpile_polyfill(indoc! {" + from dataclasses import dataclass, KW_ONLY, InitVar + from fractions import Fraction + + @dataclass + class Row: + name: str + _: KW_ONLY + seed: InitVar[int] = 0 + + def half(value: Fraction) -> None: + pass + "}); + assert!( + out.contains(indoc! {" + from dataclasses import dataclass, KW_ONLY, InitVar + from fractions import Fraction + "}), + "got:\n{out}" + ); + } + + /// and the same rule is what keeps a `decorator def`'s own declaration readable: the + /// `overload` it imports is the whole reason the two stubs it emits are an overload set. + /// deferred, a checker read them as two ordinary functions and reported `empty-body` on + /// each, plus `invalid-parameter-default` on the `= ...` an overload stub writes + #[test] + fn polyfill_the_overload_a_decorator_def_imports_stays_eager() { + let out = transpile_polyfill(indoc! {" + decorator def d(fn: (...) -> object) -> int: + return 1 + "}); + assert!( + out.contains("from typing import Callable, overload\n"), + "got:\n{out}" + ); + assert!( + !out.contains("overload = _lazy_attr"), + "the overload declaration was deferred, got:\n{out}" + ); + } + + /// a decorator says what the definition under it *is*, and it says so by identity. + /// deferred, `@dataclass` is a proxy a checker reading the emitted python cannot + /// recognize: it read `Row` as an ordinary class and reported `Row("x")` as too many + /// positional arguments, where the source declares a one-field dataclass. deferring one + /// buys nothing either — a decorator is applied while the module executes, so the proxy + /// standing in for it resolves at import time regardless + #[test] + fn polyfill_a_decorator_stays_eager() { + let out = transpile_polyfill(indoc! {" + from functools import cache, reduce + + @cache + def f() -> int: + return reduce(lambda a, b: a + b, [1, 2], 0) + "}); + assert!(out.contains("from functools import cache\n"), "got:\n{out}"); + assert!( + out.contains("reduce = _lazy_attr(\"functools\", \"reduce\")"), + "a name only called stopped being deferred, got:\n{out}" + ); + } + + #[test] + fn polyfill_a_name_read_at_runtime_and_in_an_annotation_stays_eager() { + let out = transpile_polyfill(indoc! {" + from fractions import Fraction + + def half(value: Fraction) -> Fraction: + return value / Fraction(2) + "}); + assert!( + out.starts_with("from fractions import Fraction\n"), + "got:\n{out}" + ); + } + + #[test] + fn polyfill_a_name_an_annotation_the_lowering_wrote_reads_stays_eager() { + // a class-body assignment is given the annotation its value's type spells, so the + // source reads `Fraction` only at runtime and the output reads it in an annotation + let out = transpile_polyfill(indoc! {" + from fractions import Fraction + + class Price: + amount = Fraction(1, 2) + "}); + assert!( + out.contains("amount: Fraction = Fraction(1, 2)"), + "the class-body assignment is no longer annotated, got:\n{out}" + ); + assert!( + out.starts_with("from fractions import Fraction\n"), + "got:\n{out}" + ); + } + + #[test] + fn polyfill_a_name_only_read_at_runtime_stays_deferred() { + check_polyfill_body( + indoc! {" + from json import dumps + + def show(value: int) -> str: + return dumps(value) + "}, + indoc! {" + dumps = _lazy_attr(\"json\", \"dumps\") + + def show(value: int) -> str: + return dumps(value) + "}, + ); + } + + #[test] + fn polyfill_a_name_only_a_local_variable_annotation_reads_stays_deferred() { + // a local variable's annotation is never evaluated, and nothing can read it back + let out = transpile_polyfill(indoc! {" + from fractions import Fraction + + def half() -> None: + value: Fraction = 1 + "}); + assert!( + out.contains("Fraction = _lazy_attr(\"fractions\", \"Fraction\")"), + "got:\n{out}" + ); + } + + #[test] + fn polyfill_an_annotation_under_future_annotations_stays_eager() { + // every annotation is a string at runtime, and `typing.get_type_hints` evaluates + // it against the module's globals, where a proxy would be what it finds + let out = transpile_polyfill(indoc! {" + from __future__ import annotations + from fractions import Fraction + + def half(value: Fraction) -> None: + pass + "}); + assert!( + out.contains("from fractions import Fraction\n"), + "got:\n{out}" + ); + } + + #[test] + fn annotation_names_reads_the_annotations_python_keeps() { + use ruff_python_parser::parse_module; + + let source = indoc! {" + a: A = 1 + type Alias = NotAnAnnotation + + class C[T: NotABound]: + b: B + def method(self, c: C, *d: D, **e: E) -> F: + local: NotKept = 1 + class Nested: + g: G + + def quoted(h: \"list[H]\", i: \"not an expression(\") -> None: + pass + "}; + let parsed = parse_module(source).unwrap(); + let mut names = super::annotation_names(parsed.suite(), source); + names.sort(); + names.dedup(); + assert_eq!(names, ["A", "B", "C", "D", "E", "F", "G", "H", "list"]); + } + #[test] fn polyfill_multi_name_all_unlazifiable_stays_written() { // nothing to lazify, so the statement is left exactly as it was rather diff --git a/crates/by_transforms/src/transforms/literal_types.rs b/crates/by_transforms/src/transforms/literal_types.rs index e87a8d759f..f04d4f46e1 100644 --- a/crates/by_transforms/src/transforms/literal_types.rs +++ b/crates/by_transforms/src/transforms/literal_types.rs @@ -225,6 +225,10 @@ impl<'src> LiteralTypePass<'src> { } impl TypeAwarePass for LiteralTypePass<'_> { + fn lowering(&self) -> Option { + Some(super::ast_driver::Lowering::LiteralType) + } + fn run(&self, stmts: &[Stmt], types: &dyn TypeInfo, ctx: &mut PassContext) { let mut inner = LiteralType::new(self.source, types, self.float_literals); walk_type_positions_skipping(stmts, Some(types), &ctx.claimed_type_op_ranges, &mut inner); diff --git a/crates/by_transforms/src/transforms/local_once.rs b/crates/by_transforms/src/transforms/local_once.rs index d292ba8e36..639f1c849c 100644 --- a/crates/by_transforms/src/transforms/local_once.rs +++ b/crates/by_transforms/src/transforms/local_once.rs @@ -9,9 +9,9 @@ //! (mirroring the `let` init prefix), so //! [`parameter_modifiers`](ruff_python_ast::helpers::parameter_modifiers) //! recovers them from the source span and yields the ranges to delete. Deleting -//! is all this pass does; when another AST pass re-renders the whole enclosing -//! statement, the keyword is dropped for free (it was never in the AST) and this -//! edit is skipped by the splice's overlap dedup. +//! is all this pass does; when another AST pass rewrites the `def` itself, the +//! `def` is printed from the AST, where the keyword never was, and this edit +//! goes with the rest of its source. use ruff_python_ast::helpers::parameter_modifiers; use ruff_python_ast::visitor::{Visitor, walk_parameter}; @@ -68,6 +68,10 @@ impl<'src> LocalOncePass<'src> { } impl AstPass for LocalOncePass<'_> { + fn lowering(&self) -> Option { + Some(super::ast_driver::Lowering::LocalOnce) + } + fn run(&self, module: &mut ModModule, ctx: &mut PassContext) { let mut inner = LocalOnceStrip { source: self.source, @@ -217,9 +221,8 @@ mod tests { #[test] fn modifier_survives_body_rerender() { - // a body construct that forces the whole statement to be re-rendered - // (here a `??` coalesce) must still drop the parameter modifier — the - // keyword is never in the AST, so the re-render omits it + // a body construct another pass lowers (here a `??` coalesce) must still + // leave the parameter modifier dropped let out = transpile( "def f(local x: int | None) -> int:\n return x ?? 0\n", &Config::test_default(), diff --git a/crates/by_transforms/src/transforms/main_function.rs b/crates/by_transforms/src/transforms/main_function.rs index 95ef058ef9..3b6ceab5d6 100644 --- a/crates/by_transforms/src/transforms/main_function.rs +++ b/crates/by_transforms/src/transforms/main_function.rs @@ -15,20 +15,23 @@ //! existing `__main__` guard or a bare top-level `main()` call — so the entry //! point never runs twice +use ruff_python_ast::name::Name; use ruff_python_ast::{ self as ast, CmpOp, Expr, ModModule, ParameterWithDefault, Parameters, Stmt, StmtFunctionDef, }; use super::ast_driver::{AstPass, PassContext}; +use super::repeated_underscore::{WrittenNames, positional_only_count}; use super::source_util::{is_synthetic_decorator, python_string_literal}; pub(crate) struct MainFunction<'src> { source: &'src str, + written: WrittenNames<'src>, } impl<'src> MainFunction<'src> { - pub(crate) fn new(source: &'src str) -> Self { - Self { source } + pub(crate) fn new(source: &'src str, written: WrittenNames<'src>) -> Self { + Self { source, written } } } @@ -40,7 +43,7 @@ impl AstPass for MainFunction<'_> { } fn run(&self, module: &mut ModModule, ctx: &mut PassContext) { - let Some(entry) = entry_point(&module.body, self.source) else { + let Some(entry) = entry_point(&module.body, self.source, self.written) else { return; }; // a `private main` is renamed, a `main` the command line cannot fill @@ -160,6 +163,9 @@ impl ParameterKind { /// One non-variadic parameter of `main`. pub struct EntryParameter<'a> { pub parameter: &'a ParameterWithDefault, + /// the name python binds it to, which `main` is called with: a repeated `_` + /// is numbered by the lowering + pub python_name: Name, pub kind: ParameterKind, /// how the command line spells it; `None` when the annotation has no /// command-line spelling, so the parameter is not exposed @@ -168,7 +174,7 @@ pub struct EntryParameter<'a> { impl EntryParameter<'_> { pub fn name(&self) -> &str { - self.parameter.parameter.name.as_str() + self.python_name.as_str() } pub fn is_required(&self) -> bool { @@ -252,14 +258,19 @@ pub struct Choice { /// The module's entry point, or `None` when it has no top-level `main`. /// /// `source` is the text `body` was parsed from; the `private` modifier is read -/// against it. -pub fn entry_point<'a>(body: &'a [Stmt], source: &str) -> Option> { +/// against it. `written` is the module as its author wrote it, which names a +/// repeated `_` parameter the way the lowering does +pub fn entry_point<'a>( + body: &'a [Stmt], + source: &str, + written: WrittenNames, +) -> Option> { let function = last_top_level_main(body)?; Some(EntryPoint { function, is_private: is_private(source, function), module_invokes_main: module_invokes_main(body), - parameters: parameters(&function.parameters), + parameters: parameters(&function.parameters, written), extra_arguments: extra_arguments_converter(&function.parameters), }) } @@ -279,31 +290,39 @@ fn is_private(source: &str, func: &StmtFunctionDef) -> bool { /// so it keeps its default — or, when it has none, stops `main` being an entry /// point ([`EntryPoint::blocked_by`]). Variadics never require an argument, so /// they are left out. -fn parameters(params: &Parameters) -> Vec> { +fn parameters<'a>(params: &'a Parameters, written: WrittenNames) -> Vec> { let groups = [ (¶ms.posonlyargs, ParameterKind::Positional), (¶ms.args, ParameterKind::Any), (¶ms.kwonlyargs, ParameterKind::Keyword), ]; + // a repeated `_` makes the parameters up to the last of them positional-only in the + // python, whichever group the source wrote them in + let positional_only = positional_only_count(params, written); groups .into_iter() - .flat_map(|(group, kind)| { - group.iter().map(move |parameter| EntryParameter { - parameter, - kind, - spelling: parameter - .parameter - .annotation - .as_deref() - .and_then(cli_type) - .map(|(ty, choices)| CliSpelling { - converter: match ty { - CliType::Value(callable) => Some(callable), - CliType::Flag => None, - }, - choices, - }), - }) + .flat_map(|(group, kind)| group.iter().map(move |parameter| (parameter, kind))) + .enumerate() + .map(|(index, (parameter, kind))| EntryParameter { + parameter, + python_name: crate::python_parameter_name(params, ¶meter.parameter, written), + kind: if index < positional_only { + ParameterKind::Positional + } else { + kind + }, + spelling: parameter + .parameter + .annotation + .as_deref() + .and_then(cli_type) + .map(|(ty, choices)| CliSpelling { + converter: match ty { + CliType::Value(callable) => Some(callable), + CliType::Flag => None, + }, + choices, + }), }) .collect() } @@ -550,7 +569,7 @@ fn is_str(expr: &Expr, value: &str) -> bool { #[cfg(test)] mod tests { - use crate::{Config, transpile}; + use crate::{Config, WrittenNames, transpile}; use indoc::indoc; fn check(input: &str, expected: &str) { @@ -954,6 +973,21 @@ mod tests { ); } + /// `main` is called with each value under the name python binds its parameter to, + /// and a repeated `_` is numbered there. two specs named `_` register one option + /// twice, which `argparse` refuses before anything runs. the numbered parameters + /// are positional-only, so they are filled by position + #[test] + fn a_repeated_underscore_is_filled_by_its_numbered_name() { + let out = guard("def main(_: int, _: str):\n pass\n"); + assert!( + out.contains( + "(\"_\", int, \"positional\", True, None),\n (\"_2\", str, \"positional\", True, None)," + ), + "got:\n{out}" + ); + } + #[test] fn last_main_definition_decides() { // the trailing `def main` (with a required arg) is the live binding, @@ -980,7 +1014,8 @@ mod tests { let source = "def main[T](name: str, count: int = 1):\n pass\n"; assert!(guard(source).contains("(\"name\", str, \"any\", True, None),")); let module = parsed(source); - let entry = super::entry_point(&module.syntax().body, source).expect("a main"); + let entry = super::entry_point(&module.syntax().body, source, WrittenNames::new(source)) + .expect("a main"); assert!(entry.generates_guard()); let names: Vec<&str> = entry .parameters @@ -994,7 +1029,8 @@ mod tests { fn a_main_the_command_line_cannot_fill_names_what_blocks_it() { let source = "def main(a: int, argv, b: str = \"x\"):\n \"\"\"Adds.\"\"\"\n"; let module = parsed(source); - let entry = super::entry_point(&module.syntax().body, source).expect("a main"); + let entry = super::entry_point(&module.syntax().body, source, WrittenNames::new(source)) + .expect("a main"); assert_eq!( entry.blocked_by().map(super::EntryParameter::name), Some("argv") @@ -1007,7 +1043,8 @@ mod tests { fn a_main_call_inside_a_docstring_is_not_an_invocation() { let source = "\"\"\"\nmain()\n\"\"\"\ndef main():\n pass\n"; let module = parsed(source); - let entry = super::entry_point(&module.syntax().body, source).expect("a main"); + let entry = super::entry_point(&module.syntax().body, source, WrittenNames::new(source)) + .expect("a main"); assert!(!entry.module_invokes_main); assert!(guard(source).contains(" main()")); } @@ -1016,7 +1053,8 @@ mod tests { fn flags_are_the_spellings_the_runtime_registers() { let source = "def main(out_dir: Path, dry_run: bool = False, mode: \"a\" | \"b\" = \"a\"):\n pass\n"; let module = parsed(source); - let entry = super::entry_point(&module.syntax().body, source).expect("a main"); + let entry = super::entry_point(&module.syntax().body, source, WrittenNames::new(source)) + .expect("a main"); let [out_dir, dry_run, mode] = entry.parameters.as_slice() else { panic!("three parameters"); }; diff --git a/crates/by_transforms/src/transforms/match_polyfill.rs b/crates/by_transforms/src/transforms/match_polyfill.rs index 5dc0ee51e9..c7463d4428 100644 --- a/crates/by_transforms/src/transforms/match_polyfill.rs +++ b/crates/by_transforms/src/transforms/match_polyfill.rs @@ -67,6 +67,7 @@ use ruff_python_ast::{ use ruff_python_trivia::{SimpleTokenKind, SimpleTokenizer}; use ruff_text_size::{Ranged, TextRange, TextSize}; +use super::repeated_underscore::WrittenNames; use super::source_util::{preamble_offset, temporary_name}; use crate::Config; @@ -167,7 +168,7 @@ impl Lower<'_> { /// A temporary name nothing else in the file spells. fn fresh(&mut self) -> String { loop { - let name = temporary_name("match", self.counter); + let name = temporary_name(WrittenNames::new(self.source), "match", self.counter); self.counter += 1; if !self.source.contains(&name) { return name; diff --git a/crates/by_transforms/src/transforms/match_type.rs b/crates/by_transforms/src/transforms/match_type.rs index fd0cfeb7ef..23d29578e3 100644 --- a/crates/by_transforms/src/transforms/match_type.rs +++ b/crates/by_transforms/src/transforms/match_type.rs @@ -42,6 +42,16 @@ impl<'src> MatchTypePass<'src> { } impl AstPass for MatchTypePass<'_> { + fn lowering(&self) -> Option { + Some(super::ast_driver::Lowering::MatchType) + } + + /// a match type polyfilled as a `TypeAliasType` names its type parameters as the + /// parameter objects themselves, a pack's without the `*` or `Unpack` around it + fn subsumes(&self) -> &'static [super::ast_driver::Lowering] { + &[super::ast_driver::Lowering::Unpack] + } + fn run(&self, module: &mut ModModule, ctx: &mut PassContext) { let mut inner = MatchTypeLowering { source: self.source, diff --git a/crates/by_transforms/src/transforms/mod.rs b/crates/by_transforms/src/transforms/mod.rs index a3ece30213..f72039b3fb 100644 --- a/crates/by_transforms/src/transforms/mod.rs +++ b/crates/by_transforms/src/transforms/mod.rs @@ -8,12 +8,12 @@ pub(crate) mod character_type; pub(crate) mod checked_cast; pub(crate) mod class_pattern_star; pub(crate) mod coalesce; -pub(crate) mod coalesce_chain; pub(crate) mod compat; pub(crate) mod conformance; pub(crate) mod context_params; pub(crate) mod context_sensitive; pub(crate) mod conversion; +pub(crate) mod debug_field; pub(crate) mod decl_site_variance; pub(crate) mod decorated_binding; pub(crate) mod decorated_type; @@ -66,6 +66,7 @@ pub(crate) mod raises_clause; pub(crate) mod reified_class; pub(crate) mod reified_generic; pub(crate) mod repeated_underscore; +pub(crate) mod rerender; pub(crate) mod return_value_use; pub(crate) mod runtime_union; pub(crate) mod sentinel; @@ -77,6 +78,7 @@ pub(crate) mod static_resource; pub(crate) mod string_tag; pub(crate) mod super_keyword; pub(crate) mod symbolic_type_op; +pub(crate) mod template_expression; pub(crate) mod template_type; pub(crate) mod top_star; pub(crate) mod trailing_lambda; diff --git a/crates/by_transforms/src/transforms/modifiers.rs b/crates/by_transforms/src/transforms/modifiers.rs index 9475285aeb..37b53d5351 100644 --- a/crates/by_transforms/src/transforms/modifiers.rs +++ b/crates/by_transforms/src/transforms/modifiers.rs @@ -751,6 +751,16 @@ impl<'src> ModifiersPass<'src> { } impl AstPass for ModifiersPass<'_> { + fn lowering(&self) -> Option { + Some(super::ast_driver::Lowering::Modifiers) + } + + /// a declaration whose value a statement expression moves below its suite is re-written + /// from its `let` prefix, which carries the separator the move normalises + fn subsumes(&self) -> &'static [super::ast_driver::Lowering] { + &[super::ast_driver::Lowering::StatementExpression] + } + fn run(&self, module: &mut ruff_python_ast::ModModule, ctx: &mut PassContext) { let mut inner = Modifiers::new(self.source); for stmt in &module.body { @@ -1067,9 +1077,8 @@ mod tests { registry: ClassVar[list[str]] = [] "}, indoc! {" - from dataclasses import dataclass from typing import ClassVar - from dataclasses import field + from dataclasses import field, dataclass @dataclass(frozen=True, slots=True) class Config: diff --git a/crates/by_transforms/src/transforms/mutable_defaults.rs b/crates/by_transforms/src/transforms/mutable_defaults.rs index 1ad8f0d494..1be5ecb657 100644 --- a/crates/by_transforms/src/transforms/mutable_defaults.rs +++ b/crates/by_transforms/src/transforms/mutable_defaults.rs @@ -50,6 +50,7 @@ use ruff_python_ast::{Expr, ParameterWithDefault, Stmt, StmtFunctionDef}; use ruff_text_size::{Ranged, TextRange, TextSize}; use super::ast_driver::{Fragment, PassContext, TypeAwarePass}; +use super::repeated_underscore::{WrittenNames, reads_underscore, rebound_underscore}; use super::source_util::{PrologueStatement, body_prologue, first_body_statement}; use crate::type_info::TypeInfo; @@ -60,32 +61,67 @@ pub(crate) enum Guard { /// through a [`Fragment::Src`] passthrough so the lowerings written inside /// it (`?.`, `!`, an `is` test, a `context` argument) land in the body copy /// rather than being dropped with the signature they came from - Reevaluate { name: String, default: TextRange }, + Reevaluate { + name: String, + sentinel: String, + default: TextRange, + }, /// raise — the parameter is required, its sentinel default only exists /// because python rejects a required parameter after a defaulted one - Required { name: String, function: String }, + Required { + name: String, + sentinel: String, + function: String, + }, + /// bind `_` to the parameter that was the first `_`, for a body that reads it. a + /// repeated `_` that takes its name from an overridden method leaves no parameter + /// named `_`, and a read of `_` is the first one's value — after its default is + /// settled, so this comes after the guards above + Rebind { parameter: String }, +} + +impl Guard { + /// whether the guard tests for the `_MISSING` sentinel, which the output then defines + pub(crate) fn uses_sentinel(&self) -> bool { + match self { + Guard::Reevaluate { .. } | Guard::Required { .. } => true, + Guard::Rebind { .. } => false, + } + } } impl PrologueStatement for Guard { fn push(&self, frags: &mut Vec, base: &str) { match self { - Guard::Reevaluate { name, default } => { + Guard::Reevaluate { + name, + sentinel, + default, + } => { frags.push(Fragment::Lit(format!( - "if {name} is _MISSING:\n{base} {name} = " + "if {name} is {sentinel}:\n{base} {name} = " ))); frags.push(Fragment::Src(*default)); } - Guard::Required { name, function } => { + Guard::Required { + name, + sentinel, + function, + } => { frags.push(Fragment::Lit(format!( - "if {name} is _MISSING:\n{base} raise TypeError(\"{function}() missing required argument: '{name}'\")" + "if {name} is {sentinel}:\n{base} raise TypeError(\"{function}() missing required argument: '{name}'\")" ))); } + Guard::Rebind { parameter } => { + frags.push(Fragment::Lit(format!("_ = {parameter}"))); + } } } } struct MutableDefaults<'src> { source: &'src str, + written: WrittenNames<'src>, types: &'src dyn TypeInfo, edits: Vec<(TextRange, Vec)>, /// the `_MISSING` substitutions, whose defaults the guards re-evaluate @@ -106,8 +142,32 @@ struct MutableDefaults<'src> { /// it *absorbs* the zero-width insertions a lowering anchored to the default's /// first token left behind (`_force_unwrap(`), which a plain-text replacement /// leaves stranded in the signature. the guard re-emits them -fn sentinel_edit(default: TextRange) -> (TextRange, Vec) { - (default, vec![Fragment::Lit("_MISSING".to_owned())]) +fn sentinel_edit(default: TextRange, sentinel: &str) -> (TextRange, Vec) { + (default, vec![Fragment::Lit(sentinel.to_owned())]) +} + +/// The name the sentinel goes into the output under. +/// +/// It stands for "no argument was given", so it has to be a value nothing else can be — +/// which a name the module binds itself is not. Taken past whatever `source` spells, so a +/// module that writes `_MISSING` of its own keeps it and the sentinel goes somewhere else. +pub(crate) fn sentinel_name(source: &str) -> String { + WrittenNames::new(source).fresh("_MISSING") +} + +/// The lines the output needs before it can use the sentinel: the sentinel itself, and the +/// import its annotation names. +/// +/// The sentinel stands where the source wrote an annotated default, so an unannotated +/// `object()` there is a default the parameter cannot hold and a checker reading the output +/// says so. Declaring it `Any` is how typeshed declares its own sentinels, and it costs the +/// output nothing: the value never reaches the body, whose guard replaces it before anything +/// reads it. +pub(crate) fn sentinel_definition(source: &str) -> [String; 2] { + [ + "from typing import Any".to_owned(), + format!("{}: Any = object()", sentinel_name(source)), + ] } /// A default written into the signature at the end of the parameter it belongs to. @@ -151,6 +211,8 @@ pub(crate) struct ParameterGuards { /// calls. pub(crate) fn parameter_guards( f: &StmtFunctionDef, + written_names: WrittenNames, + sentinel: &str, types: &dyn TypeInfo, is_stub: bool, ) -> ParameterGuards { @@ -169,9 +231,11 @@ pub(crate) fn parameter_guards( Some(d) => { seen_default = true; if !is_stub && !is_immutable_scalar_default(d) && !body_cannot_evaluate(d) { - sentinels.push(sentinel_edit(d.range())); + sentinels.push(sentinel_edit(d.range(), sentinel)); guards.push(Guard::Reevaluate { - name: pw.parameter.name.id.to_string(), + name: crate::python_parameter_name(params, &pw.parameter, written_names) + .to_string(), + sentinel: sentinel.to_owned(), default: d.range(), }); } @@ -183,12 +247,16 @@ pub(crate) fn parameter_guards( written.push(written_default(pw, &value)); } None if seen_default && is_stub => { - undeclarable.push(pw.parameter.name.id.to_string()); + undeclarable.push( + crate::python_parameter_name(params, &pw.parameter, written_names).to_string(), + ); } None if seen_default => { - written.push(written_default(pw, "_MISSING")); + written.push(written_default(pw, sentinel)); guards.push(Guard::Required { - name: pw.parameter.name.id.to_string(), + name: crate::python_parameter_name(params, &pw.parameter, written_names) + .to_string(), + sentinel: sentinel.to_owned(), function: f.name.id.to_string(), }); } @@ -198,9 +266,11 @@ pub(crate) fn parameter_guards( for pw in ¶ms.kwonlyargs { match pw.default.as_deref() { Some(d) if !is_stub && !is_immutable_scalar_default(d) && !body_cannot_evaluate(d) => { - sentinels.push(sentinel_edit(d.range())); + sentinels.push(sentinel_edit(d.range(), sentinel)); guards.push(Guard::Reevaluate { - name: pw.parameter.name.id.to_string(), + name: crate::python_parameter_name(params, &pw.parameter, written_names) + .to_string(), + sentinel: sentinel.to_owned(), default: d.range(), }); } @@ -212,6 +282,14 @@ pub(crate) fn parameter_guards( } } } + if !is_stub + && let Some(parameter) = rebound_underscore(params, written_names) + && reads_underscore(&f.body) + { + guards.push(Guard::Rebind { + parameter: parameter.to_string(), + }); + } ParameterGuards { sentinels, written, @@ -289,7 +367,13 @@ impl MutableDefaults<'_> { written, guards, undeclarable, - } = parameter_guards(f, self.types, self.is_stub); + } = parameter_guards( + f, + self.written, + &sentinel_name(self.source), + self.types, + self.is_stub, + ); self.undeclarable.extend( undeclarable .into_iter() @@ -300,12 +384,12 @@ impl MutableDefaults<'_> { if guards.is_empty() { return; } - self.used = true; + self.used |= guards.iter().any(Guard::uses_sentinel); // the guards go at the start of the first body statement the source // actually wrote match body_prologue(self.source, f, &guards) { - Some(anchored) => self.guards.push(anchored), + Some(anchored) => self.guards.extend(anchored), // nothing in the body came from the source, so there is nowhere to // put the guard. say so rather than splice it at a synthesized // node's offset, which lands inside the signature @@ -325,12 +409,17 @@ impl<'ast> Visitor<'ast> for MutableDefaults<'_> { pub(crate) struct MutableDefaultsPass<'src> { source: &'src str, + written: WrittenNames<'src>, is_stub: bool, } impl<'src> MutableDefaultsPass<'src> { - pub(crate) fn new(source: &'src str, is_stub: bool) -> Self { - Self { source, is_stub } + pub(crate) fn new(source: &'src str, written: WrittenNames<'src>, is_stub: bool) -> Self { + Self { + source, + written, + is_stub, + } } } @@ -338,6 +427,7 @@ impl TypeAwarePass for MutableDefaultsPass<'_> { fn run(&self, stmts: &[Stmt], types: &dyn TypeInfo, ctx: &mut PassContext) { let mut inner = MutableDefaults { source: self.source, + written: self.written, types, edits: Vec::new(), relocating: Vec::new(), @@ -366,7 +456,8 @@ impl TypeAwarePass for MutableDefaultsPass<'_> { return; } if inner.used { - ctx.required_imports.push("_MISSING = object()".to_owned()); + ctx.required_imports + .extend(sentinel_definition(self.source)); } ctx.template_edits.extend(inner.edits); ctx.relocating_edits.extend(inner.relocating); @@ -482,7 +573,8 @@ mod tests { print(a) "}, indoc! {" - _MISSING = object() + from typing import Any + _MISSING: Any = object() class A: def f(self, a = _MISSING): if a is _MISSING: @@ -537,7 +629,8 @@ mod tests { init(items: list[int] = []) "}, indoc! {" - _MISSING = object() + from typing import Any + _MISSING: Any = object() class S: def __init__(self, items: list[int] = _MISSING): if items is _MISSING: @@ -550,7 +643,8 @@ mod tests { init(let items: list[int] = []) "}, indoc! {" - _MISSING = object() + from typing import Any + _MISSING: Any = object() class S: def __init__(self, items: list[int] = _MISSING): if items is _MISSING: @@ -570,7 +664,8 @@ mod tests { init(let first: int = 1, let second: str) "}, indoc! {" - _MISSING = object() + from typing import Any + _MISSING: Any = object() class S: def __init__(self, first: int = 1, second: str = _MISSING): if second is _MISSING: @@ -592,7 +687,8 @@ mod tests { print(items) "}, indoc! {" - _MISSING = object() + from typing import Any + _MISSING: Any = object() class S: def __init__(self, items: list[int] = _MISSING): if items is _MISSING: @@ -625,7 +721,8 @@ mod tests { self.items = items "}, indoc! {" - _MISSING = object() + from typing import Any + _MISSING: Any = object() class S: def __init__(self, items: list[int] = _MISSING) -> None: if items is _MISSING: @@ -643,7 +740,8 @@ mod tests { pass "}, indoc! {" - _MISSING = object() + from typing import Any + _MISSING: Any = object() def f(x=_MISSING): if x is _MISSING: x = [] @@ -660,7 +758,8 @@ mod tests { pass "}, indoc! {" - _MISSING = object() + from typing import Any + _MISSING: Any = object() def f(x=_MISSING): if x is _MISSING: x = {} @@ -677,7 +776,8 @@ mod tests { pass "}, indoc! {" - _MISSING = object() + from typing import Any + _MISSING: Any = object() def f(x=_MISSING): if x is _MISSING: x = {1, 2} @@ -730,7 +830,8 @@ mod tests { pass "}, indoc! {" - _MISSING = object() + from typing import Any + _MISSING: Any = object() def f(x=_MISSING, y=_MISSING): if x is _MISSING: x = [] @@ -750,7 +851,8 @@ mod tests { pass "#}, indoc! {r#" - _MISSING = object() + from typing import Any + _MISSING: Any = object() def f(x=_MISSING): """doc""" if x is _MISSING: @@ -760,6 +862,61 @@ mod tests { ); } + /// a docstring written on the `def`'s own line is a body of exactly one statement, and + /// nothing can follow a statement there. the guard needs a line, so the docstring is + /// given one first — written where it stood, the guard was indented under a suite that + /// had already closed and the emitted file did not parse + #[test] + fn a_docstring_on_the_def_line_is_given_a_line_of_its_own() { + check( + "def f(x=[]): \"\"\"doc\"\"\"\n", + "from typing import Any\n_MISSING: Any = object()\ndef f(x=_MISSING): \n \"\"\"doc\"\"\"\n if x is _MISSING:\n x = []\n", + ); + } + + /// a body written on the `def`'s own line can hold more than the one statement, and the + /// break and the guard then want different offsets: the break goes ahead of the + /// docstring, which has to stay the body's first statement to stay a docstring, and the + /// guard goes ahead of the statement that follows it. taking both from the guard's + /// offset left the docstring on the clause's line with an indented suite under it, and + /// the whole file failed to parse with `Unexpected indentation` + #[test] + fn a_docstring_followed_by_statements_on_the_def_line_is_given_a_line_of_its_own() { + check( + "def f(x=[]): \"\"\"doc\"\"\"; x.append(1); return x\n", + "from typing import Any\n_MISSING: Any = object()\ndef f(x=_MISSING): \n \"\"\"doc\"\"\"; \n if x is _MISSING:\n x = []\n x.append(1); return x\n", + ); + } + + /// the same body without a docstring breaks at the one offset, because the statement the + /// guard goes ahead of is the body's first + #[test] + fn statements_on_the_def_line_without_a_docstring_take_one_break() { + check( + "def f(x=[]): x.append(1); return x\n", + "from typing import Any\n_MISSING: Any = object()\ndef f(x=_MISSING): \n if x is _MISSING:\n x = []\n x.append(1); return x\n", + ); + } + + /// and a method reached the same way, since the break is indented from the `def` rather + /// than from the module + #[test] + fn a_method_body_on_the_clause_line_is_broken_at_its_own_indentation() { + check( + "class C:\n def m(self, x=[]): \"\"\"doc\"\"\"; x.append(1); return x\n", + "from typing import Any\n_MISSING: Any = object()\nclass C:\n def m(self, x=_MISSING): \n \"\"\"doc\"\"\"; \n if x is _MISSING:\n x = []\n x.append(1); return x\n", + ); + } + + /// with no guard to write, nothing breaks the line at all: a body on the clause's line + /// is left exactly as it was written + #[test] + fn a_body_on_the_def_line_is_left_alone_when_nothing_writes_into_it() { + unchanged("def f(x: int) -> int: \"\"\"doc\"\"\"; return x\n"); + unchanged("def f(x: int) -> int: return x\n"); + unchanged("class C:\n def m(self, x: int) -> int: \"\"\"doc\"\"\"; return x\n"); + } + #[test] fn sentinel_defined_once_for_multiple_functions() { check( @@ -770,7 +927,8 @@ mod tests { pass "}, indoc! {" - _MISSING = object() + from typing import Any + _MISSING: Any = object() def f(x=_MISSING): if x is _MISSING: x = [] @@ -791,7 +949,8 @@ mod tests { print(x, a) "}, indoc! {" - _MISSING = object() + from typing import Any + _MISSING: Any = object() def f(x=1, a=_MISSING): if a is _MISSING: raise TypeError(\"f() missing required argument: 'a'\") @@ -808,7 +967,8 @@ mod tests { print(x, a) "}, indoc! {" - _MISSING = object() + from typing import Any + _MISSING: Any = object() def f(x: int = 1, a: int = _MISSING): if a is _MISSING: raise TypeError(\"f() missing required argument: 'a'\") @@ -827,7 +987,8 @@ mod tests { print(x, a) "}, indoc! {" - _MISSING = object() + from typing import Any + _MISSING: Any = object() def f(x=_MISSING, a=_MISSING): if x is _MISSING: x = [] @@ -863,7 +1024,8 @@ mod tests { print(a) "#}, indoc! {r#" - _MISSING = object() + from typing import Any + _MISSING: Any = object() data = "fdsa" def f(a=_MISSING): if a is _MISSING: @@ -884,7 +1046,8 @@ mod tests { print(a) "#}, indoc! {r#" - _MISSING = object() + from typing import Any + _MISSING: Any = object() data = "fdsa" def f(a=_MISSING): if a is _MISSING: @@ -908,7 +1071,8 @@ mod tests { f(2) "}, indoc! {" - _MISSING = object() + from typing import Any + _MISSING: Any = object() def f(a, b = _MISSING): if b is _MISSING: b = a + 1 @@ -931,7 +1095,7 @@ mod tests { a: int = [] ) -> int: ... "}, - "_MISSING = object()\ndef f(\n a: int = _MISSING\n) -> int: \n if a is _MISSING:\n a = []\n ...\n", + "from typing import Any\n_MISSING: Any = object()\ndef f(\n a: int = _MISSING\n) -> int: \n if a is _MISSING:\n a = []\n ...\n", ); } @@ -939,7 +1103,7 @@ mod tests { fn inline_ellipsis_body() { check( "def f(x=[]): ...", - "_MISSING = object()\ndef f(x=_MISSING): \n if x is _MISSING:\n x = []\n ...", + "from typing import Any\n_MISSING: Any = object()\ndef f(x=_MISSING): \n if x is _MISSING:\n x = []\n ...", ); } @@ -955,7 +1119,8 @@ mod tests { return x "}, indoc! {" - _MISSING = object() + from typing import Any + _MISSING: Any = object() def f(x = _MISSING): if x is _MISSING: x = [True] @@ -976,7 +1141,8 @@ mod tests { return x "}, indoc! {" - _MISSING = object() + from typing import Any + _MISSING: Any = object() def f(x = _MISSING): if x is _MISSING: x = True @@ -1021,7 +1187,8 @@ mod tests { return a ?? len(xs) "}, indoc! {" - _MISSING = object() + from typing import Any + _MISSING: Any = object() def f(xs: list[int] = _MISSING) -> int: if xs is _MISSING: xs = [] @@ -1085,4 +1252,29 @@ mod tests { "got: {error}" ); } + + /// the sentinel stands for "no argument was given", which a value the module itself binds + /// is not — the guard would take a real argument for a missing one, and the module's own + /// binding would be the one overwritten + #[test] + fn the_sentinel_goes_past_a_name_the_module_binds() { + check( + indoc! {" + _MISSING = 5 + + def f(xs: list[int] = []) -> int: + return len(xs) + _MISSING + "}, + indoc! {" + from typing import Any + _MISSING2: Any = object() + _MISSING = 5 + + def f(xs: list[int] = _MISSING2) -> int: + if xs is _MISSING2: + xs = [] + return len(xs) + _MISSING + "}, + ); + } } diff --git a/crates/by_transforms/src/transforms/none_chain.rs b/crates/by_transforms/src/transforms/none_chain.rs index 1c2c14b974..e16893546d 100644 --- a/crates/by_transforms/src/transforms/none_chain.rs +++ b/crates/by_transforms/src/transforms/none_chain.rs @@ -5,6 +5,7 @@ use ruff_python_ast::{Expr, Stmt}; use ruff_text_size::{Ranged, TextRange}; use crate::transforms::ast_driver::{Fragment, PassContext, TypeAwarePass}; +use crate::transforms::repeated_underscore::WrittenNames; use crate::transforms::source_util::temporary_name; use crate::type_info::TypeInfo; @@ -32,15 +33,15 @@ impl<'src> NoneChain<'src> { const TEMP_VARS: usize = 10; /// The name a `?.` or `??` expansion walruses its receiver into. -pub(super) fn temp_var(index: usize) -> String { - temporary_name("t", index) +pub(super) fn temp_var(written: WrittenNames<'_>, index: usize) -> String { + temporary_name(written, "t", index) } -fn pick_temp_var(types: &dyn TypeInfo, anchor: &Expr) -> String { +fn pick_temp_var(written: WrittenNames<'_>, types: &dyn TypeInfo, anchor: &Expr) -> String { (0..TEMP_VARS) - .map(temp_var) + .map(|index| temp_var(written, index)) .find(|name| types.is_unbound_at(name, anchor)) - .unwrap_or_else(|| temp_var(TEMP_VARS)) + .unwrap_or_else(|| temp_var(written, TEMP_VARS)) } /// walks an attribute-access chain and returns `Some((python_form, guards, base))` @@ -179,6 +180,10 @@ impl<'src> NoneChainPass<'src> { } impl TypeAwarePass for NoneChainPass<'_> { + fn lowering(&self) -> Option { + Some(super::ast_driver::Lowering::NoneChain) + } + fn run(&self, stmts: &[Stmt], types: &dyn TypeInfo, ctx: &mut PassContext) { let mut inner = NoneChain::new(self.source, types); for stmt in stmts { @@ -229,7 +234,7 @@ impl<'ast> Visitor<'ast> for NoneChain<'_> { // `[i for i in a?.b]` would not even parse. the parentheses keep the emitted tree // the one that was parsed if let Some((head, form, guards, base)) = chain_head(expr, self.source, self.types) { - let temp = pick_temp_var(self.types, expr); + let temp = pick_temp_var(WrittenNames::new(self.source), self.types, expr); let trailers = TextRange::new(head.range().end(), expr.range().end()); let mut fragments = vec![Fragment::Lit("(".to_owned())]; fragments.extend(build_expansion(&guards, &form, &temp, base)); @@ -288,7 +293,7 @@ mod tests { fn double_chain_t_taken() { check( "__by_t_0__ = 1\nx = a?.a?.b\n", - "__by_t_0__ = 1\nx = (None if a is None else None if (__by_t_1__ := a.a) is None else __by_t_1__.b)\n", + "__by_t_0__ = 1\nx = (None if a is None else None if (__by_t_0__2 := a.a) is None else __by_t_0__2.b)\n", ); } diff --git a/crates/by_transforms/src/transforms/optional_type.rs b/crates/by_transforms/src/transforms/optional_type.rs index 34c4aee359..bdc7b30e9f 100644 --- a/crates/by_transforms/src/transforms/optional_type.rs +++ b/crates/by_transforms/src/transforms/optional_type.rs @@ -220,6 +220,10 @@ pub(crate) fn rewrite_type_expr( } impl TypeAwarePass for OptionalTypePass<'_> { + fn lowering(&self) -> Option { + Some(super::ast_driver::Lowering::OptionalType) + } + fn run(&self, stmts: &[Stmt], _types: &dyn TypeInfo, ctx: &mut PassContext) { let mut lower = OptionalLower::new(self.source, self.min_version); for stmt in stmts { diff --git a/crates/by_transforms/src/transforms/overload.rs b/crates/by_transforms/src/transforms/overload.rs index 15ed31e8db..8a4c361702 100644 --- a/crates/by_transforms/src/transforms/overload.rs +++ b/crates/by_transforms/src/transforms/overload.rs @@ -43,10 +43,12 @@ impl AstPass for Overload<'_> { source: self.source, is_stub: self.is_stub, edits: RefCell::new(Vec::new()), - needs_overload: false, + first_emitted: None, }; state.visit_body(&module.body); - if state.needs_overload { + if let Some(first_emitted) = state.first_emitted + && !imports_overload_before(&module.body, first_emitted) + { ctx.required_imports .push("from typing import overload".to_owned()); } @@ -54,11 +56,39 @@ impl AstPass for Overload<'_> { } } +/// whether the module already binds `overload` at `pos`, through a top-level +/// `from typing import overload` of its own. +/// +/// the name is read where a decorator runs, which is when the `def` below it is +/// evaluated — so an import after that `def` binds the name too late and the output +/// still needs its own. an alias binds some other name, and an import nested in a +/// function or under `if TYPE_CHECKING:` is not in scope at module level at all, +/// so neither is one of these +fn imports_overload_before(body: &[Stmt], pos: TextSize) -> bool { + body.iter().any(|stmt| { + let Stmt::ImportFrom(import) = stmt else { + return false; + }; + import.level == 0 + && import + .module + .as_ref() + .is_some_and(|m| m.as_str() == "typing") + && import.range().end() <= pos + && import + .names + .iter() + .any(|alias| alias.name.as_str() == "overload" && alias.asname.is_none()) + }) +} + struct State<'src> { source: &'src str, is_stub: bool, edits: RefCell>, - needs_overload: bool, + /// where the earliest `@overload` this pass writes goes, which is the position the + /// name has to be bound by. `None` when it writes none and needs no import + first_emitted: Option, } impl State<'_> { @@ -119,11 +149,29 @@ impl State<'_> { ) } + /// the source may write `@overload` itself and still use the bodyless shorthand for + /// the signature. the decorator it wrote is the one the output keeps — a second copy + /// applies `overload` twice, and `typing.overload` is not idempotent in what it + /// registers + fn wears_overload(func: &StmtFunctionDef) -> bool { + func.decorator_list.iter().any(|dec| match &dec.expression { + Expr::Name(n) => n.id.as_str() == "overload", + // `typing.overload`, or any module it was imported through + Expr::Attribute(a) => a.attr.as_str() == "overload", + _ => false, + }) + } + fn add_overload_stub(&mut self, func: &StmtFunctionDef) { - self.needs_overload = true; - let indent = self.line_indent(func.range().start()).to_owned(); - let start = func.range().start(); - self.push(TextRange::new(start, start), format!("@overload\n{indent}")); + if !Self::wears_overload(func) { + let start = func.range().start(); + self.first_emitted = Some(match self.first_emitted { + Some(first) => first.min(start), + None => start, + }); + let indent = self.line_indent(start).to_owned(); + self.push(TextRange::new(start, start), format!("@overload\n{indent}")); + } if func.body.is_empty() { let end = func.range().end(); self.push(TextRange::new(end, end), ": ...".to_owned()); @@ -308,6 +356,160 @@ mod tests { ); } + /// the two spellings can be mixed: `@overload` written out, with the signature left + /// bodyless. only the body is missing, so only the body is added + #[test] + fn written_overload_with_bodyless_shorthand() { + check( + indoc! {" + from typing import overload + @overload + def q(a: int) -> int + @overload + def q(a: str) -> str + def q(a: object) -> object: + return a + "}, + indoc! {" + from typing import overload + @overload + def q(a: int) -> int: ... + @overload + def q(a: str) -> str: ... + def q(a: object) -> object: + return a + "}, + ); + } + + /// a module reached through `typing` spells the decorator the same way + #[test] + fn qualified_overload_with_bodyless_shorthand() { + check( + indoc! {" + import typing + @typing.overload + def q(a: int) -> int + @typing.overload + def q(a: str) -> str + def q(a: object) -> object: + return a + "}, + indoc! {" + import typing + @typing.overload + def q(a: int) -> int: ... + @typing.overload + def q(a: str) -> str: ... + def q(a: object) -> object: + return a + "}, + ); + } + + /// a run the source decorates only in part still needs the decorator on the rest, + /// and takes the name from the import the source already wrote + #[test] + fn partly_written_overload_run() { + check( + indoc! {" + from typing import overload + @overload + def q(a: int) -> int + def q(a: str) -> str + def q(a: object) -> object: + return a + "}, + indoc! {" + from typing import overload + @overload + def q(a: int) -> int: ... + @overload + def q(a: str) -> str: ... + def q(a: object) -> object: + return a + "}, + ); + } + + /// a decorator is read when the `def` under it runs, so an import further down the + /// module binds the name too late and the output carries its own + #[test] + fn overload_imported_after_the_run_is_too_late() { + check( + indoc! {" + def q(a: int) -> int + def q(a: str) -> str + def q(a: object) -> object: + return a + + from typing import overload + "}, + indoc! {" + from typing import overload + @overload + def q(a: int) -> int: ... + @overload + def q(a: str) -> str: ... + def q(a: object) -> object: + return a + + from typing import overload + "}, + ); + } + + /// a run that writes `@overload` itself and also needs a name of the lowering's — the + /// `Callable` an arrow type becomes — puts both on the import the module already wrote, + /// rather than opening a second `from typing import` above it + #[test] + fn a_name_the_lowering_adds_joins_the_modules_own_typing_import() { + check( + indoc! {" + from typing import overload + @overload + def q(a: (int) -> int) -> int + @overload + def q(a: str) -> str + def q(a: object) -> object: + return a + "}, + indoc! {" + from typing import overload, Callable + @overload + def q(a: Callable[[int], int]) -> int: ... + @overload + def q(a: str) -> str: ... + def q(a: object) -> object: + return a + "}, + ); + } + + /// an alias binds some other name, so the decorator the output writes still needs + /// `overload` itself + #[test] + fn aliased_overload_import_does_not_count() { + check( + indoc! {" + from typing import overload as ol + def q(a: int) -> int + def q(a: str) -> str + def q(a: object) -> object: + return a + "}, + indoc! {" + from typing import overload as ol, overload + @overload + def q(a: int) -> int: ... + @overload + def q(a: str) -> str: ... + def q(a: object) -> object: + return a + "}, + ); + } + #[test] fn python_unchanged() { unchanged(indoc! {" @@ -329,8 +531,7 @@ mod tests { abstract def f(self) -> int "}, indoc! {" - from abc import abstractmethod - from abc import ABC + from abc import ABC, abstractmethod class A(ABC): @abstractmethod diff --git a/crates/by_transforms/src/transforms/parametric_is.rs b/crates/by_transforms/src/transforms/parametric_is.rs index 81c7b719ca..fed78e432b 100644 --- a/crates/by_transforms/src/transforms/parametric_is.rs +++ b/crates/by_transforms/src/transforms/parametric_is.rs @@ -692,6 +692,15 @@ impl<'src> ParametricIsPass<'src> { } impl TypeAwarePass for ParametricIsPass<'_> { + fn lowering(&self) -> Option { + Some(super::ast_driver::Lowering::ParametricIs) + } + + /// the type it tests against is printed as the runtime probe it builds, literals included + fn subsumes(&self) -> &'static [super::ast_driver::Lowering] { + &[super::ast_driver::Lowering::LiteralType] + } + fn run(&self, stmts: &[Stmt], types: &dyn TypeInfo, ctx: &mut PassContext) { let mut inner = ParametricIs { source: self.source, diff --git a/crates/by_transforms/src/transforms/properties.rs b/crates/by_transforms/src/transforms/properties.rs index 73385c001a..6b9088c15f 100644 --- a/crates/by_transforms/src/transforms/properties.rs +++ b/crates/by_transforms/src/transforms/properties.rs @@ -712,6 +712,16 @@ fn join_assignments(out: &mut Vec, assignments: Vec>, in } impl TypeAwarePass for PropertiesPass<'_> { + /// an accessor block it replaces is written as the `@property` members the parser + /// synthesized, with the modifiers as their decorators and the declared type as the + /// backing field's annotation + fn subsumes(&self) -> &'static [super::ast_driver::Lowering] { + &[ + super::ast_driver::Lowering::Modifiers, + super::ast_driver::Lowering::InferredAnnotation, + ] + } + fn run(&self, stmts: &[Stmt], _types: &dyn TypeInfo, ctx: &mut PassContext) { let mut finder = ClassFinder { classes: Vec::new(), diff --git a/crates/by_transforms/src/transforms/protocol_type.rs b/crates/by_transforms/src/transforms/protocol_type.rs index 74250c2391..e91aa06861 100644 --- a/crates/by_transforms/src/transforms/protocol_type.rs +++ b/crates/by_transforms/src/transforms/protocol_type.rs @@ -44,6 +44,7 @@ use crate::type_info::TypeInfo; use super::ast_driver::{PassContext, TypeAwarePass}; use super::callable::CallableSyntax; +use super::repeated_underscore::{WrittenNames, protocol_method_receiver}; use super::type_expr_walker::{Recurse, TypeExprVisitor, TypePos, walk_type_positions_skipping}; /// One inline protocol shape: the rendered class-body lines, in source order. @@ -79,12 +80,21 @@ impl Shape { pub(crate) struct ProtocolTypePass<'src> { source: &'src str, + written: WrittenNames<'src>, config: crate::Config, } impl<'src> ProtocolTypePass<'src> { - pub(crate) fn new(source: &'src str, config: crate::Config) -> Self { - Self { source, config } + pub(crate) fn new( + source: &'src str, + written: WrittenNames<'src>, + config: crate::Config, + ) -> Self { + Self { + source, + written, + config, + } } } @@ -110,13 +120,14 @@ struct ProtocolTypeLowering<'src> { impl<'src> ProtocolTypeLowering<'src> { fn new( source: &'src str, + written: WrittenNames<'src>, types: &'src dyn TypeInfo, claimed: &'src [TextRange], float_literals: FloatLiteralLowering, ) -> Self { Self { source, - callable: CallableSyntax::new(source, float_literals) + callable: CallableSyntax::new(source, written, float_literals) .with_types(types) .with_claimed_ranges(claimed), edits: Vec::new(), @@ -169,17 +180,10 @@ impl<'src> ProtocolTypeLowering<'src> { let Expr::CallableType(signature) = method.signature.as_ref() else { return None; }; - let receiver = signature.args.first().and_then(|first| match first { - Expr::Name(name) if name.ctx.is_invalid() => Some(name.id.as_str()), - _ => None, - }); - let offset = usize::from(receiver.is_some()); - let shift = |index: Option| index.map(|i| (i as usize).saturating_sub(offset)); - + let receiver = protocol_method_receiver(signature).map(|name| name.id.as_str()); let params = self.callable.render_protocol_params( - &signature.args[offset..], - shift(signature.parameter_slash()), - shift(signature.parameter_star()), + signature, + true, receiver.unwrap_or("self"), // a protocol method's receiver is its `self` parameter, never an implicit one None, @@ -378,12 +382,14 @@ impl<'ast> ruff_python_ast::visitor::Visitor<'ast> for TypevarScopeWalker<'_> { /// knows about and any inline protocol left in a value position. fn lower<'src>( source: &'src str, + written: WrittenNames<'src>, types: &'src dyn TypeInfo, claimed: &'src [TextRange], stmts: &[Stmt], config: &crate::Config, ) -> ProtocolTypeLowering<'src> { - let mut inner = ProtocolTypeLowering::new(source, types, claimed, config.float_literals); + let mut inner = + ProtocolTypeLowering::new(source, written, types, claimed, config.float_literals); { let mut walker = TypevarScopeWalker { config: config.clone(), @@ -404,17 +410,18 @@ fn lower<'src>( /// The preamble and edits needed to lower every inline protocol still present in /// post-transform output. /// -/// A pass that re-renders a whole statement from the AST re-emits the surface -/// `protocol(...)` syntax, after this pass's own text edits were computed — so -/// the driver runs the lowering again over the spliced output to catch them. +/// A node an AST pass rewrote is printed from the AST, which can re-emit the +/// surface `protocol(...)` syntax after this pass's own text edits were computed +/// — so the driver runs the lowering again over the spliced output to catch them. /// Returns `None` when nothing is left to lower. pub(crate) fn cleanup( source: &str, + written: WrittenNames, types: &dyn TypeInfo, stmts: &[Stmt], config: &crate::Config, ) -> Result, String)>, String> { - let mut inner = lower(source, types, &[], stmts, config); + let mut inner = lower(source, written, types, &[], stmts, config); if let Some(error) = inner.errors.first() { return Err(error.clone()); } @@ -456,9 +463,34 @@ pub(crate) fn cleanup( } impl TypeAwarePass for ProtocolTypePass<'_> { + fn lowering(&self) -> Option { + Some(super::ast_driver::Lowering::ProtocolType) + } + + /// an inline protocol it replaces moves into a class, whose member types the shared + /// type-expression lowerer prints, a `typeof` among them + fn subsumes(&self) -> &'static [super::ast_driver::Lowering] { + &[ + super::ast_driver::Lowering::Callable, + super::ast_driver::Lowering::OptionalType, + super::ast_driver::Lowering::LiteralType, + super::ast_driver::Lowering::JustFloat, + super::ast_driver::Lowering::DynamicKeyword, + super::ast_driver::Lowering::FloatConst, + super::ast_driver::Lowering::Typeof, + ] + } + fn run(&self, stmts: &[Stmt], types: &dyn TypeInfo, ctx: &mut PassContext) { let claimed = ctx.claimed_type_op_ranges.clone(); - let mut inner = lower(self.source, types, &claimed, stmts, &self.config); + let mut inner = lower( + self.source, + self.written, + types, + &claimed, + stmts, + &self.config, + ); if inner.needs_import { ctx.required_imports @@ -527,6 +559,44 @@ mod tests { ); } + /// the synthesized class is written into the preamble and names `Protocol` as it is + /// defined, so that import stays above it even when the module imports `Protocol` itself — + /// folding it into the module's own line would leave the class with no such name + #[test] + fn a_module_importing_protocol_itself_still_gets_the_name_above_the_class() { + check( + indoc! {" + from typing import Protocol + + def f(x: protocol(a: int)) -> None: ... + "}, + indoc! {r#" + from typing import Protocol + class _Protocol_2abdf9ec(Protocol): + a: "int" + from typing import Protocol + + def f(x: _Protocol_2abdf9ec) -> None: ... + "#}, + ); + } + + /// a member is declared as a `def` in the synthesized class, where a repeated `_` is + /// numbered and made positional-only as it is in any other `def`, and the receiver is + /// the first of them + #[test] + fn a_repeated_underscore_in_a_method_member_is_numbered() { + let out = transpile( + "def f(x: protocol(def m(_, _: int, _: str) -> int)) -> None: ...\n", + &Config::test_default(), + ) + .unwrap(); + assert!( + out.contains("def m(_, _2: \"int\", _3: \"str\", /) -> \"int\": ..."), + "got:\n{out}" + ); + } + #[test] fn identical_shapes_share_one_class() { check( @@ -668,9 +738,9 @@ mod tests { ); } - /// a pass that re-renders a whole statement from the AST re-emits the - /// surface `protocol(...)` syntax after this pass computed its edits — the - /// driver's cleanup loop catches what is left, without duplicating the class + /// an AST pass rewriting a node inside the `protocol(...)` this pass replaces + /// whole — the driver's cleanup loop catches what is left, without duplicating + /// the class #[test] fn survives_a_statement_re_rendered_by_another_pass() { check( diff --git a/crates/by_transforms/src/transforms/raises_clause.rs b/crates/by_transforms/src/transforms/raises_clause.rs index 3f334dfe79..95fe886bc0 100644 --- a/crates/by_transforms/src/transforms/raises_clause.rs +++ b/crates/by_transforms/src/transforms/raises_clause.rs @@ -105,26 +105,20 @@ impl super::ast_driver::TypeAwarePass for RaisesGuardPass<'_> { return; } + // a `def` another pass rewrote is printed from the syntax tree, where the guard + // — on the `def` line, inside the statement whenever it is decorated — has no + // source to land in. the driver refuses that loss rather than build without + // the check let mut guards = Vec::new(); - let mut conflicts = Vec::new(); - for (index, stmt) in stmts.iter().enumerate() { - // an AST pass that re-rendered this top-level statement rebuilds it - // from the AST, which discards any insertion inside its range — and - // the guard sits on the `def` line, inside it whenever the function - // is decorated. a runtime check that silently disappears is worse - // than one that refuses to build - let rerendered = ctx.changed.contains(&index); + for stmt in stmts { let mut collector = GuardCollector { source: self.source, types, - rerendered, guards: &mut guards, - conflicts: &mut conflicts, }; collector.visit_stmt(stmt); } - ctx.errors.extend(conflicts); if guards.is_empty() { return; } @@ -142,9 +136,7 @@ impl super::ast_driver::TypeAwarePass for RaisesGuardPass<'_> { struct GuardCollector<'a> { source: &'a str, types: &'a dyn TypeInfo, - rerendered: bool, guards: &'a mut Vec<(TextRange, String)>, - conflicts: &'a mut Vec, } impl<'ast> Visitor<'ast> for GuardCollector<'_> { @@ -152,15 +144,7 @@ impl<'ast> Visitor<'ast> for GuardCollector<'_> { if let Stmt::FunctionDef(function) = stmt && let Some(guard) = guard_for(self.source, function, self.types) { - if self.rerendered && !function.decorator_list.is_empty() { - self.conflicts.push(format!( - "`{}` declares `raises`, but another lowering re-rendered its \ - statement, which would drop the runtime guard", - function.name - )); - } else { - self.guards.push(guard); - } + self.guards.push(guard); } walk_stmt(self, stmt); @@ -331,9 +315,8 @@ mod tests { #[test] fn clause_survives_body_rerender() { - // a body construct that forces the whole statement to be re-rendered - // must still drop the clause — codegen emits python, which has no - // spelling for it + // a body construct an AST pass rewrites must still leave the clause + // dropped let out = transpile( "def f(x: int | None) -> int raises TypeError:\n return x ?? 0\n", &Config::test_default(), @@ -345,17 +328,15 @@ mod tests { #[test] fn clause_survives_a_whole_statement_rerender() { - // `typeof` in a parameter annotation makes an AST-mutation pass re-render - // the whole `def`, dropping this pass's deletion. the generator emits - // python, so the clause has to be erased there too — otherwise the - // construct leaks and the pipeline reports a transform conflict + // numbering a repeated `_` renames a parameter of the `def`, and the new + // name is emitted beside this pass's deletion rather than instead of it let out = transpile( - "def f(x: typeof(1)) raises TypeError:\n raise TypeError\n", + "def f(_: int, _: int) raises TypeError:\n raise TypeError\n", &Config::test_default(), ) .unwrap(); assert!(!out.contains("raises"), "clause leaked:\n{out}"); - assert!(out.contains("def f(x: TypeOf[1]):"), "got:\n{out}"); + assert!(out.contains("def f(_: int, _2: int, /):"), "got:\n{out}"); } #[test] @@ -507,21 +488,53 @@ mod tests { } #[test] - fn guard_that_would_be_dropped_is_an_error() { - // `typeof` makes an AST pass re-render the whole statement, discarding an - // insertion inside its range — and a decorated `def` puts the guard there. - // a runtime check that silently vanishes is worse than one that refuses - let error = transpile( - "class C:\n @staticmethod\n def m(x: typeof(1)) raises TypeError:\n raise TypeError\n", + fn guard_on_a_decorated_def_with_a_repeated_underscore() { + // numbering a repeated `_` renames a parameter, and a decorated `def` puts the + // guard between the decorator and the `def` keyword, in the same statement. + // the name is all the numbering re-emits, so the guard keeps its place + let out = transpile( + "class C:\n @staticmethod\n def m(_: int, _: int) raises TypeError:\n raise TypeError\n", + &guarded(), + ) + .unwrap(); + assert!( + out.contains( + " @staticmethod\n @_by_raises(TypeError, \"m\")\n def m(_: int, _2: int, /):" + ), + "got:\n{out}" + ); + } + + #[test] + fn guard_on_a_decorated_def_whose_body_a_pass_rewrote() { + // a `sentinel` in the body becomes an assignment, and the `def` holding it is + // printed from the tree. the text between the decorator and the parameter list + // is the same there as in the source, so the source is kept, and the guard + // inserted in it with it + let out = transpile( + "class C:\n @staticmethod\n def m() raises TypeError:\n sentinel A\n raise TypeError\n", &guarded(), ) - .unwrap_err(); + .unwrap(); assert!( - error.contains("would drop the runtime guard"), - "got: {error}" + out.contains(" @staticmethod\n @_by_raises(TypeError, \"m\")\n def m():"), + "got:\n{out}" ); } + #[test] + fn guard_beside_a_rewritten_annotation_is_kept() { + // `typeof` rewrites only the annotation, so the rest of the decorated `def` + // keeps its source and the guard its place in it + let out = transpile( + "class C:\n @staticmethod\n def m(x: typeof(1)) raises TypeError:\n raise TypeError\n", + &guarded(), + ) + .unwrap(); + assert!(out.contains("def m(x: TypeOf[1]):"), "got:\n{out}"); + assert!(out.contains("_by_raises("), "got:\n{out}"); + } + #[test] fn guard_wraps_a_declared_function() { let out = transpile( diff --git a/crates/by_transforms/src/transforms/reified_class.rs b/crates/by_transforms/src/transforms/reified_class.rs index 97ac91b667..b1e8912de4 100644 --- a/crates/by_transforms/src/transforms/reified_class.rs +++ b/crates/by_transforms/src/transforms/reified_class.rs @@ -156,7 +156,7 @@ impl<'src> ReifiedClass<'src> { }) .collect(); match body_prologue(self.source, method.function, &bindings) { - Some(anchored) => self.prologues.push(anchored), + Some(anchored) => self.prologues.extend(anchored), None => self.unanchored.push(method.function.name.id.to_string()), } } diff --git a/crates/by_transforms/src/transforms/repeated_underscore.rs b/crates/by_transforms/src/transforms/repeated_underscore.rs index 2909ee59bf..933eead6d3 100644 --- a/crates/by_transforms/src/transforms/repeated_underscore.rs +++ b/crates/by_transforms/src/transforms/repeated_underscore.rs @@ -1,106 +1,457 @@ -//! AST rewrite that renames repeated `_` parameters in function and lambda -//! definitions. +//! the lowering of a parameter list that repeats `_` //! -//! basedpython allows `def f(_, _, _): ...` as a shorthand for ignoring -//! multiple positional parameters. python rejects this with a duplicate- -//! parameter error, so each `_` after the first is renamed to a fresh -//! `_` (`_2`, `_3`, ...). references to `_` inside the body are left -//! alone and resolve to the first parameter +//! basedpython allows `def f(_, _, _): ...` as a shorthand for ignoring several +//! positional parameters. python rejects it with a duplicate-parameter error, so each +//! `_` after the first is renamed to a fresh `_` (`_2`, `_3`, ...), skipping any +//! name the module spells, and a `/` after the last of them makes them positional-only: +//! the numbered name is the lowering's rather than the author's, so no call may spell +//! it. a method that overrides one takes the names the overridden method gives those +//! positions instead, and their kinds with them +//! +//! which names and where the `/` goes is ty's answer, the one its signature of the +//! definition is built from (`ty_python_semantic::types::repeated_underscore`), so a +//! call ty accepts is one the python accepts. a shape ty refuses is refused here too. +//! references to `_` inside the body are left alone and resolve to the first parameter -use std::cell::Cell; -use std::collections::HashSet; +use std::cell::{Cell, RefCell}; +use std::collections::HashMap; use ruff_python_ast::name::Name; use ruff_python_ast::visitor::transformer::{Transformer, walk_expr, walk_stmt}; -use ruff_python_ast::{Expr, Parameter, Parameters, Stmt}; +use ruff_python_ast::visitor::{ + Visitor, walk_expr as visit_walk_expr, walk_stmt as visit_walk_stmt, +}; +use ruff_python_ast::{Expr, ModModule, Parameter, Parameters, Stmt}; +use ruff_python_trivia::{SimpleTokenKind, SimpleTokenizer}; +use ruff_text_size::{Ranged, TextRange}; +use ty_python_semantic::types::repeated_underscore::{ + self as decision, UnderscoreLowering, UnderscoreRefusal, callable_parameter_slots, + lower_repeated_underscores, lowered_parameter_names, parameter_slots, +}; -pub(crate) struct RepeatedUnderscore { - changed: Cell, +pub(crate) use ty_python_semantic::types::repeated_underscore::{ + ParameterSlot, protocol_method_receiver, +}; + +use super::ast_driver::{AstPass, PassContext}; +use crate::type_info::TypeInfo; + +type Decision = Result; + +/// how every parameter list of a module that repeats `_` is lowered, by the range of the +/// list — read off ty before any pass rewrites the syntax tree, so the passes that walk a +/// tree of their own can still ask +#[derive(Debug)] +pub(crate) struct UnderscoreLowerings { + decisions: HashMap, + /// whether the python the module targets has the `/` a repeated `_` needs + slash: bool, +} + +/// ty's answer for every parameter list in `suite` that repeats `_`, the module targeting a +/// python that has the `/` when `slash` +pub(crate) fn collect(suite: &[Stmt], types: &dyn TypeInfo, slash: bool) -> UnderscoreLowerings { + struct Collector<'a> { + types: &'a dyn TypeInfo, + lowerings: UnderscoreLowerings, + } + impl Visitor<'_> for Collector<'_> { + fn visit_stmt(&mut self, stmt: &Stmt) { + if let Stmt::FunctionDef(function) = stmt + && repeats_underscore(&function.parameters) + && let Some(decision) = self.types.repeated_underscore_lowering(function) + { + self.lowerings + .decisions + .insert(function.parameters.range, decision); + } + visit_walk_stmt(self, stmt); + } + + fn visit_expr(&mut self, expr: &Expr) { + if let Expr::Lambda(lambda) = expr + && let Some(parameters) = lambda.parameters.as_deref() + && let Some(decision) = standalone_lowering(parameters, self.lowerings.slash) + { + self.lowerings.decisions.insert(parameters.range, decision); + } + visit_walk_expr(self, expr); + } + } + let mut collector = Collector { + types, + lowerings: UnderscoreLowerings { + decisions: HashMap::new(), + slash, + }, + }; + collector.visit_body(suite); + collector.lowerings } -impl RepeatedUnderscore { - pub(crate) fn new() -> Self { +/// how `parameters` is lowered read on its own, with no method it overrides and no receiver +/// to consult. a lambda has neither, so this is the answer ty gives a lambda too +fn standalone_lowering(parameters: &Parameters, slash: bool) -> Option { + lower_repeated_underscores(¶meter_slots(parameters), false, None, |_| false, slash) +} + +fn repeats_underscore(parameters: &Parameters) -> bool { + parameters + .iter() + .filter(|parameter| parameter.name() == "_") + .nth(1) + .is_some() +} + +/// the names the module spells as written, and how the parameter lists in it that repeat +/// `_` are lowered — every name the lowering writes that the source did not +/// +/// a repeated `_` is numbered to a name outside the ones the module spells, so the +/// parameter cannot shadow a name the function reads from an enclosing scope: +/// +/// ```by +/// _2 = [9] +/// +/// def f(_: int, _: int) -> list[int]: +/// return _2 # the module's `_2`, so the second parameter is `_3` +/// ``` +/// +/// without the lowerings ty decided — the native compiler reads a module on its own — +/// a repeated `_` is numbered and nothing takes a name from an overridden method +#[derive(Clone, Copy, Debug)] +pub struct WrittenNames<'src> { + names: decision::WrittenNames<'src>, + lowerings: Option<&'src UnderscoreLowerings>, +} + +impl<'src> WrittenNames<'src> { + /// the names spelled in `source`, the text of a module as it was written + pub fn new(source: &'src str) -> Self { Self { - changed: Cell::new(false), + names: decision::WrittenNames::new(source), + lowerings: None, } } - pub(crate) fn changed_cell(&self) -> &Cell { - &self.changed + /// these names, with `lowerings` saying how each parameter list that repeats `_` is + /// lowered + pub(crate) fn with_lowerings<'a>(self, lowerings: &'a UnderscoreLowerings) -> WrittenNames<'a> + where + 'src: 'a, + { + WrittenNames { + names: self.names, + lowerings: Some(lowerings), + } } - fn rename(&self, params: &mut Parameters) { - let mut seen_underscore = false; - let mut taken: HashSet = HashSet::new(); - for p in params.iter() { - taken.insert(p.name().to_string()); + /// a name the module spells nowhere, `stem` itself when the module does not spell it + /// and `stem2`, `stem3`, … when it does + /// + /// a binding the lowering writes under a name the source wrote would shadow it — the + /// `inner` a `decorator def` writes standing where an option of that name is declared, + /// which the dispatcher then passed the dispatcher itself for + pub(crate) fn fresh(self, stem: &str) -> String { + self.names.fresh(stem) + } + + /// how `parameters` is lowered when it repeats `_`. without ty's answer, the list is + /// read on its own, as a lambda's is: it takes no name from a method it overrides + /// whether the python the module targets has the `/` a repeated `_` needs. without ty's + /// answers, as for a native build, it is assumed to + fn slash(self) -> bool { + self.lowerings.is_none_or(|lowerings| lowerings.slash) + } + + fn lowering(self, parameters: &Parameters) -> Option { + match self.lowerings { + Some(lowerings) => lowerings.decisions.get(¶meters.range).cloned(), + None => standalone_lowering(parameters, true), } - let mut next = 2u32; - let next_name = |next: &mut u32, taken: &mut HashSet| -> String { - loop { - let candidate = format!("_{}", *next); - *next += 1; - if !taken.contains(&candidate) { - taken.insert(candidate.clone()); - return candidate; - } + } + + /// the name each of `parameters` binds in the python, in declaration order + fn names(self, parameters: &Parameters) -> Vec { + let slots = parameter_slots(parameters); + match self.lowering(parameters) { + Some(Ok(lowering)) => lowering.names(&slots, self.names), + _ => lowered_parameter_names(&slots, self.names), + } + } +} + +/// the name `parameter`, one of `parameters`, binds in the python a basedpython +/// definition lowers to. `written` is the module the definition is in +/// +/// a native build publishes the same definition, and has to answer to the same +/// names: a keyword argument, the forwarder's signature and `__code__` all spell +/// them +pub fn python_parameter_name( + parameters: &Parameters, + parameter: &Parameter, + written: WrittenNames, +) -> Name { + parameters + .iter() + .zip(written.names(parameters)) + .find(|(candidate, _)| std::ptr::eq(candidate.as_parameter(), parameter)) + .map_or_else(|| parameter.name.id.clone(), |(_, name)| name) +} + +/// the parameters a callable type is declared with in the python — the `__call__` of the +/// protocol a callable type naming its parameters lowers to, or the method of an inline +/// protocol +pub(crate) struct LoweredCallableParameters { + /// the name each written parameter is declared under, in order, the receiver of a + /// protocol's method left out + pub(crate) names: Vec, + /// how many of the written parameters the `/` comes after, when a repeated `_` puts + /// one there + pub(crate) slash: Option, +} + +/// how the parameters of `ct` are declared, `method` saying it is the signature of an +/// inline protocol's method. a repeated `_` is lowered as in a `def`: every `_` after the +/// first is numbered, and made positional-only by a `/` after the last of them — ty's +/// answer for the same callable type. a shape it refuses is reported by the pass that +/// lowers parameter lists, and is written here with its `_`s numbered alone +pub(crate) fn lowered_callable_parameters( + ct: &ruff_python_ast::ExprCallableType, + method: bool, + written: WrittenNames, +) -> LoweredCallableParameters { + let slots = callable_parameter_slots(ct, method); + let spelled: Vec<(ParameterSlot, &str)> = slots + .iter() + .map(|(slot, name)| (*slot, name.as_str())) + .collect(); + let method_receiver = method && protocol_method_receiver(ct).is_some(); + let receivers = usize::from(method_receiver) + usize::from(ct.receiver.is_some()); + let (mut names, positional_only) = match lower_repeated_underscores( + &spelled, + method_receiver, + None, + |_| false, + written.slash(), + ) { + Some(Ok(lowering)) => ( + lowering.names(&spelled, written.names), + lowering.positional_only(), + ), + _ => (lowered_parameter_names(&spelled, written.names), 0), + }; + LoweredCallableParameters { + names: names.split_off(receivers.min(names.len())), + slash: positional_only + .checked_sub(receivers) + .filter(|&written| written > 0), + } +} + +/// how many of the leading positional `parameters` the python takes by position alone +pub(crate) fn positional_only_count(parameters: &Parameters, written: WrittenNames) -> usize { + match written.lowering(parameters) { + Some(Ok(lowering)) => lowering.positional_only(), + _ => parameters.posonlyargs.len(), + } +} + +/// the name a read of `_` in the body of a definition with `parameters` means, when the +/// lowering binds no parameter to `_` any more: the first `_` took a name from the method +/// the definition overrides, and a read of `_` is still the first `_`'s value +pub(crate) fn rebound_underscore(parameters: &Parameters, written: WrittenNames) -> Option { + let Some(Ok(lowering)) = written.lowering(parameters) else { + return None; + }; + if !lowering.is_inherited() { + return None; + } + parameters + .iter() + .zip(written.names(parameters)) + .find(|(parameter, _)| parameter.name() == "_") + .map(|(_, name)| name) +} + +/// whether `body` reads `_`, anywhere in it +pub(crate) fn reads_underscore(body: &[Stmt]) -> bool { + struct Reads { + found: bool, + } + impl Visitor<'_> for Reads { + fn visit_stmt(&mut self, stmt: &Stmt) { + if let Stmt::AugAssign(node) = stmt + && matches!(node.target.as_ref(), Expr::Name(name) if name.id == "_") + { + self.found = true; } - }; - let process = |p: &mut Parameter, - seen: &mut bool, - next: &mut u32, - taken: &mut HashSet, - changed: &Cell| { - if p.name.as_str() != "_" { - return; + visit_walk_stmt(self, stmt); + } + + fn visit_expr(&mut self, expr: &Expr) { + if let Expr::Name(name) = expr + && name.id == "_" + && !name.ctx.is_store() + { + self.found = true; } - if !*seen { - *seen = true; + visit_walk_expr(self, expr); + } + } + let mut reads = Reads { found: false }; + reads.visit_body(body); + reads.found +} + +pub(crate) struct RepeatedUnderscore<'src> { + source: &'src str, + written: WrittenNames<'src>, +} + +impl<'src> RepeatedUnderscore<'src> { + pub(crate) fn new(source: &'src str, written: WrittenNames<'src>) -> Self { + Self { source, written } + } +} + +impl AstPass for RepeatedUnderscore<'_> { + fn lowering(&self) -> Option { + Some(super::ast_driver::Lowering::RepeatedUnderscore) + } + + fn run(&self, module: &mut ModModule, ctx: &mut PassContext) { + for (index, stmt) in module.body.iter_mut().enumerate() { + let lowerer = Lowerer { + pass: self, + changed: Cell::new(false), + edits: RefCell::default(), + errors: RefCell::default(), + method_signatures: RefCell::default(), + }; + lowerer.visit_stmt(stmt); + if lowerer.changed.get() { + ctx.changed.push(index); + } + ctx.text_edits.extend(lowerer.edits.into_inner()); + ctx.errors.extend(lowerer.errors.into_inner()); + } + } +} + +struct Lowerer<'a, 'src> { + pass: &'a RepeatedUnderscore<'src>, + changed: Cell, + edits: RefCell>, + errors: RefCell>, + /// the signatures of the inline protocol methods met so far, which are not callable + /// types of their own + method_signatures: RefCell>, +} + +impl Lowerer<'_, '_> { + fn lower(&self, params: &mut Parameters, function: Option<&str>) { + let Some(decision) = self.pass.written.lowering(params) else { + return; + }; + let lowering = match decision { + Ok(lowering) => lowering, + Err(refusal) => { + let owner = + function.map_or_else(|| "a lambda".to_owned(), |name| format!("`{name}`")); + self.errors.borrow_mut().push(format!( + "{} in {owner}: {}", + refusal.message(), + refusal.help() + )); return; } - let new_name = next_name(next, taken); - p.name.id = Name::from(new_name.as_str()); - changed.set(true); }; - for pw in params + self.place_slash(params, lowering.positional_only()); + let names = self.pass.written.names(params); + let parameters = params .posonlyargs .iter_mut() .chain(params.args.iter_mut()) - .chain(params.kwonlyargs.iter_mut()) - { - process( - &mut pw.parameter, - &mut seen_underscore, - &mut next, - &mut taken, - &self.changed, - ); - } - if let Some(v) = params.vararg.as_deref_mut() { - process( - v, - &mut seen_underscore, - &mut next, - &mut taken, - &self.changed, - ); - } - if let Some(k) = params.kwarg.as_deref_mut() { - process( - k, - &mut seen_underscore, - &mut next, - &mut taken, - &self.changed, - ); - } - } -} - -impl Transformer for RepeatedUnderscore { + .map(|parameter| &mut parameter.parameter) + .chain(params.vararg.as_deref_mut()) + .chain( + params + .kwonlyargs + .iter_mut() + .map(|parameter| &mut parameter.parameter), + ) + .chain(params.kwarg.as_deref_mut()); + for (parameter, name) in parameters.zip(names) { + if parameter.name.id != name { + parameter.name.id = name; + self.changed.set(true); + } + } + } + + /// report the parameters of `callable` when they repeat `_` in a shape the lowering + /// refuses, `owner` naming what they belong to + fn refuse_callable( + &self, + callable: &ruff_python_ast::ExprCallableType, + method: bool, + owner: &str, + ) { + let slots = callable_parameter_slots(callable, method); + let spelled: Vec<(ParameterSlot, &str)> = slots + .iter() + .map(|(slot, name)| (*slot, name.as_str())) + .collect(); + let receiver = method && protocol_method_receiver(callable).is_some(); + if let Some(Err(refusal)) = lower_repeated_underscores( + &spelled, + receiver, + None, + |_| false, + self.pass.written.slash(), + ) { + self.errors.borrow_mut().push(format!( + "{} in {owner}: {}", + refusal.message(), + refusal.help() + )); + } + } + + /// move the `/` to after the first `positional_only` parameters, as edits of the + /// source. the header is where the passes before this one made their edits, and + /// printing it from the tree would drop them + fn place_slash(&self, params: &Parameters, positional_only: usize) { + let written = params.posonlyargs.len(); + if positional_only <= written { + return; + } + let Some(last) = params + .posonlyargs + .iter() + .chain(¶ms.args) + .nth(positional_only - 1) + else { + return; + }; + let mut edits = self.edits.borrow_mut(); + if let (Some(before), Some(after)) = (params.posonlyargs.last(), params.args.first()) { + let gap = TextRange::new(before.end(), after.start()); + if let Some(slash) = SimpleTokenizer::new(self.pass.source, gap) + .find(|token| token.kind() == SimpleTokenKind::Slash) + { + edits.push((TextRange::new(slash.start(), after.start()), String::new())); + } + } + edits.push((TextRange::empty(last.end()), ", /".to_owned())); + } +} + +impl Transformer for Lowerer<'_, '_> { fn visit_stmt(&self, stmt: &mut Stmt) { if let Stmt::FunctionDef(f) = stmt { - self.rename(&mut f.parameters); + let name = f.name.id.to_string(); + self.lower(&mut f.parameters, Some(&name)); } walk_stmt(self, stmt); } @@ -109,7 +460,24 @@ impl Transformer for RepeatedUnderscore { if let Expr::Lambda(l) = expr && let Some(params) = l.parameters.as_deref_mut() { - self.rename(params); + self.lower(params, None); + } + // a callable type or an inline protocol's method is written as a `__call__` or a + // method by the lowerings of those types, from the same answer. a shape it refuses + // is refused here, where every other parameter list's is + match expr { + Expr::ProtocolMethod(method) => { + if let Expr::CallableType(signature) = method.signature.as_ref() { + self.method_signatures.borrow_mut().push(signature.range); + self.refuse_callable(signature, true, &format!("`{}`", method.name.id)); + } + } + Expr::CallableType(callable) + if !self.method_signatures.borrow().contains(&callable.range) => + { + self.refuse_callable(callable, false, "a callable type"); + } + _ => {} } walk_expr(self, expr); } @@ -117,6 +485,8 @@ impl Transformer for RepeatedUnderscore { #[cfg(test)] mod tests { + use indoc::indoc; + use crate::python_passthrough::unchanged; use crate::{Config, transpile}; @@ -124,49 +494,149 @@ mod tests { assert_eq!(transpile(input, &Config::test_default()).unwrap(), expected); } + fn refused(input: &str) -> String { + transpile(input, &Config::test_default()).unwrap_err() + } + #[test] fn two_underscores() { check( "def f(_, _):\n print(_)\n", - "def f(_, _2):\n print(_)\n", + "def f(_, _2, /):\n print(_)\n", ); } + /// the `/` goes after the last `_`, so a named parameter after it keeps its keyword #[test] fn three_underscores() { check( "def g(_, _, _, x):\n return _\n", - "def g(_, _2, _3, x):\n return _\n", + "def g(_, _2, _3, /, x):\n return _\n", ); } #[test] - fn interleaved() { + fn lambda_underscores() { + check("f = lambda _, _: 1\n", "f = lambda _, _2, /: 1\n"); + } + + #[test] + fn nested_function() { check( - "def h(a, _, b, _):\n return a + b\n", - "def h(a, _, b, _2):\n return a + b\n", + "def outer(_, _):\n def inner(_, _):\n return _\n return _\n", + "def outer(_, _2, /):\n def inner(_, _2, /):\n return _\n return _\n", ); } + /// a method's receiver is never passed by keyword, so the `/` may pass it #[test] - fn lambda_underscores() { - check("f = lambda _, _: 1\n", "f = lambda _, _2: 1\n"); + fn a_method_receiver_is_made_positional_only() { + check( + "class A:\n def f(self, _, _):\n pass\n", + "class A:\n def f(self, _, _2, /):\n pass\n", + ); } #[test] - fn nested_function() { - // codegen emits a blank line before nested function defs + fn a_default_stays_before_the_slash() { check( - "def outer(_, _):\n def inner(_, _):\n return _\n return _\n", - "def outer(_, _2):\n\n def inner(_, _2):\n return _\n return _\n", + "def f(_: int, _: int = 1) -> None: ...\n", + "def f(_: int, _2: int = 1, /) -> None: ...\n", + ); + } + + /// a `/` the source wrote before the last `_` moves after it + #[test] + fn a_written_slash_moves_after_the_last_underscore() { + check( + "def f(a, /, _, _):\n return a\n", + "def f(a, _, _2, /):\n return a\n", + ); + } + + #[test] + fn a_written_slash_after_the_last_underscore_stays() { + check( + "def h(a, _, b, _, /):\n return a + b\n", + "def h(a, _, b, _2, /):\n return a + b\n", + ); + } + + /// `*_` is reached by no keyword, so nothing before it has to be positional-only + #[test] + fn vararg_underscore() { + check( + "def f(a, _, *_):\n return _\n", + "def f(a, _, *_2):\n return _\n", + ); + } + + /// a `/` would make `a` and `b` positional-only too, which is the author's to write + #[test] + fn a_named_parameter_before_the_last_underscore_is_refused() { + assert_eq!( + refused("def h(a, _, b, _):\n return a + b\n"), + "the repeated `_` parameters after `a` make it positional-only in `h`: write the `/` \ + after the last `_` to make them positional-only" + ); + assert_eq!( + refused("f = lambda a, _, _: a\n"), + "the repeated `_` parameters after `a` make it positional-only in a lambda: write the \ + `/` after the last `_` to make them positional-only" + ); + } + + #[test] + fn a_keyword_only_repeated_underscore_is_refused() { + assert_eq!( + refused("def f(_, *, _):\n pass\n"), + "a repeated `_` parameter cannot be keyword-only in `f`: only a keyword reaches it, \ + and it has no name of its own: move it before the `*`, or name it" ); } + /// the numbered name of a repeated `_` is a local of the function, so a name the + /// module binds under it would read the argument instead. python's answer here is + /// `[9]`, and the second parameter answered it — in `g`'s body, and in the guard + /// that re-evaluates `f`'s mutable default there + #[test] + fn a_numbered_name_skips_a_name_the_module_binds() { + check( + "_2: list[int] = [9]\n\ndef f(_: int, _: int, x: list[int] = _2) -> list[int]:\n return x\n\ndef g(_: int, _: int) -> list[int]:\n return _2\n", + "from typing import Any\n_MISSING: Any = object()\n_2: list[int] = [9]\n\ndef f(_: int, _3: int, /, x: list[int] = _MISSING) -> list[int]:\n if x is _MISSING:\n x = _2\n return x\n\ndef g(_: int, _3: int, /) -> list[int]:\n return _2\n", + ); + } + + /// what the module spells is read off its text, so a name reached any other way — + /// through `globals()`, or as an attribute — is skipped too, and so is one written + /// only in a comment + #[test] + fn a_numbered_name_skips_a_name_spelled_anywhere() { + check( + "def f(_, _):\n return globals()[\"_2\"]\n", + "def f(_, _3, /):\n return globals()[\"_2\"]\n", + ); + check( + "def f(_, _, _):\n return self._3 # _2\n", + "def f(_, _4, _5, /):\n return self._3 # _2\n", + ); + } + + /// a name is spelled only as a whole: `__2`, `a_2` and `_2b` spell no `_2` + #[test] + fn a_longer_name_does_not_spell_a_numbered_one() { + check( + "__2 = a_2 = _2b = 1\ndef f(_, _):\n return __2 + a_2 + _2b\n", + "__2 = a_2 = _2b = 1\ndef f(_, _2, /):\n return __2 + a_2 + _2b\n", + ); + } + + /// a name the source gives a parameter is not a repeated `_`, so the numbering skips it #[test] fn existing_collision() { check( - "def f(_, _2, _):\n return _\n", - "def f(_, _2, _3):\n return _\n", + "def f(_, _2, _, /):\n return _\n", + "def f(_, _2, _3, /):\n return _\n", ); } @@ -180,11 +650,100 @@ mod tests { unchanged("def f(a, b):\n return a + b\n"); } + /// an override takes the names the method it overrides gives those positions, so a + /// call that passes them by keyword works on it as it does on the base #[test] - fn vararg_underscore() { + fn an_override_takes_the_base_names() { + check( + indoc! {" + class A: + def f(self, x: int, y: int) -> None: ... + + class B(A): + override def f(self, _: int, _: int) -> None: ... + "}, + indoc! {" + from typing_extensions import override + class A: + def f(self, x: int, y: int) -> None: ... + + class B(A): + @override + def f(self, x: int, y: int) -> None: ... + "}, + ); + } + + /// a base parameter that is positional-only stays so + #[test] + fn an_override_keeps_a_positional_only_base_parameter() { check( - "def f(_, *_):\n return _\n", - "def f(_, *_2):\n return _\n", + indoc! {" + class A: + def f(self, x: int, /, y: int) -> None: ... + + class B(A): + override def f(self, _: int, _: int) -> None: ... + "}, + indoc! {" + from typing_extensions import override + class A: + def f(self, x: int, /, y: int) -> None: ... + + class B(A): + @override + def f(self, x: int, /, y: int) -> None: ... + "}, + ); + } + + /// no parameter is named `_` once they take the base's names, so a body that reads `_` + /// is handed the first of them, as it was before + #[test] + fn an_override_that_reads_underscore_binds_it() { + check( + indoc! {" + class A: + def f(self, x: int, y: int) -> int: + return x + + class B(A): + override def f(self, _: int, _: int) -> int: + return _ + "}, + indoc! {" + from typing_extensions import override + class A: + def f(self, x: int, y: int) -> int: + return x + + class B(A): + @override + def f(self, x: int, y: int) -> int: + _ = x + return _ + "}, + ); + } + + /// the name an override would take from its base is one its body reads from an + /// enclosing scope, where the parameter would shadow it + #[test] + fn an_inherited_name_the_body_reads_is_refused() { + assert_eq!( + refused(indoc! {" + x = 1 + + class A: + def f(self, x: int, y: int) -> int: + return x + + class B(A): + override def f(self, _: int, _: int) -> int: + return x + "}), + "a repeated `_` parameter named `x` shadows a `x` the body reads in `f`: the \ + parameter takes its name from the method it overrides; rename the `x` the body reads" ); } } diff --git a/crates/by_transforms/src/transforms/rerender.rs b/crates/by_transforms/src/transforms/rerender.rs new file mode 100644 index 0000000000..8a563e9174 --- /dev/null +++ b/crates/by_transforms/src/transforms/rerender.rs @@ -0,0 +1,2080 @@ +//! the edits that carry an AST pass's rewrite of a statement into the output +//! +//! an AST pass rewrites the syntax tree, and every other pass lowers what it +//! lowers as an edit keyed on a range of the source. printing a rewritten +//! statement from its tree would drop each of those edits inside it — a +//! conversion, an extension call, a quoted forward reference — and nothing +//! would say so wherever the construct they lowered is valid python as written. +//! +//! so no statement is printed whole. a node that kept its source range is one +//! the passes did not synthesize, and it is emitted as its source, with the +//! edits inside it applied. a node whose own text a pass changed — `typeof x` +//! become `TypeOf[x]`, a `def` with a `sentinel` in its body become an +//! assignment — is re-emitted at its range as a template that prints what the +//! pass made of it and passes each node below it that kept a range through as +//! `Src`. a node a pass made has no range, and is printed where it stands. +//! +//! whether a node's own text changed is decided by printing it twice, from the +//! rewritten tree and from the parse, with every node below it that kept a range +//! printed as a placeholder naming that range. the two agree exactly when the +//! passes left the node itself alone. +//! +//! between two nodes it passes through, a template prints tokens of its own, and +//! keeps the source's text wherever those tokens are the source's — see +//! [`Rerendering::segment`]. +//! +//! a name the node spells — a parameter's, a function's, an attribute's — is +//! compared as the source spelled it, so a node a pass only renamed something in +//! is not re-emitted: the name is, at its own range. numbering a repeated `_` +//! changes nothing else about a `def`, and every edit elsewhere in its signature +//! lands in the source around the new name. +//! +//! a compound statement whose clauses a pass left alone, and changed only the +//! statements of, is not re-emitted either: each suite a pass changed is, over the +//! source its statements span. a header is where the parser stands basedpython up +//! as syntax python has no spelling for — a modifier as a decorator, `init(...)` +//! as a `def __init__` carrying a `self` and a `-> None` the source never wrote, a +//! `raises` clause — and the passes that lower it do so with edits in the header's +//! source, which printing the header again would drop. a suite written on the line +//! of its clause is re-emitted the same way, as a block below the clause: it has no +//! line of its own to keep, and what it prints need not fit on one. +//! +//! what a template prints is indented as the source around it is: by the line the +//! node starts on, and each block it opens by the step the node's own source +//! indents its blocks by, which a file need not use throughout. + +use std::collections::HashMap; + +use ruff_python_ast::name::Name; +use ruff_python_ast::visitor::source_order::{SourceOrderVisitor, TraversalSignal}; +use ruff_python_ast::visitor::transformer::{self, Transformer}; +use ruff_python_ast::visitor::{self, Visitor}; +use ruff_python_ast::{ + Alias, AnyNodeRef, AtomicNodeIndex, ExceptHandler, Expr, ExprContext, ExprName, Identifier, + Keyword, NodeKind, Parameter, Pattern, PatternKeyword, Stmt, StmtExpr, Suite, TypeParam, +}; +use ruff_python_codegen::{Generator, Indentation, Mode}; +use ruff_python_trivia::{ + BackwardsTokenizer, CommentRanges, SimpleToken, SimpleTokenKind, SimpleTokenizer, +}; +use ruff_source_file::LineEnding; +use ruff_text_size::{Ranged, TextLen, TextRange, TextSize}; + +use super::ast_driver::Fragment; + +/// the re-emission of one statement an AST pass rewrote +pub(crate) struct Rerendering<'a> { + source: &'a str, + comments: &'a CommentRanges, + /// where the edits the other passes made stand + edited: &'a [TextRange], + indentation: &'a Indentation, + /// what every placeholder starts with, which the source spells nowhere + label: String, +} + +/// a node below the one being printed that kept its range +enum Hole<'m> { + Expr(&'m Expr), + Stmt(&'m Stmt), +} + +impl Hole<'_> { + fn range(&self) -> TextRange { + match self { + Hole::Expr(expr) => expr.range(), + Hole::Stmt(stmt) => stmt.range(), + } + } +} + +/// whether a node at `range` of `kind` is one the passes kept from the parse +/// +/// a node a pass synthesized carries no range. one a pass built by parsing text of its +/// own should have forgotten the ranges into that text (see [`forget_stmt_ranges`]), +/// and one that did not is still not passed through as source it never came from +fn is_kept(parsed: &Parsed<'_>, range: TextRange, kind: NodeKind) -> bool { + range != TextRange::default() && parsed.at(range, kind).is_some() +} + +fn expr_kind(expr: &Expr) -> NodeKind { + AnyNodeRef::from(expr).kind() +} + +fn stmt_kind(stmt: &Stmt) -> NodeKind { + AnyNodeRef::from(stmt).kind() +} + +impl<'a> Rerendering<'a> { + pub(crate) fn new( + source: &'a str, + comments: &'a CommentRanges, + edited: &'a [TextRange], + indentation: &'a Indentation, + ) -> Self { + let mut label = "__by_src_".to_string(); + while source.contains(&label) { + label.push('_'); + } + Self { + source, + comments, + edited, + indentation, + label, + } + } + + /// the edits that turn `original`, the statement as parsed at `range`, into + /// `rewritten`, what the passes made of it + pub(crate) fn edits( + &self, + original: &Stmt, + rewritten: &Stmt, + range: TextRange, + ) -> Vec<(TextRange, Vec)> { + let mut parsed = Parsed::default(); + parsed.visit_stmt(original); + let mut edits = Vec::new(); + self.node(&parsed, &Hole::Stmt(rewritten), range, &mut edits); + edits + } + + /// the edits for `node`, standing at `range` in the source, and for every node + /// below it that kept a range + fn node( + &self, + parsed: &Parsed<'_>, + node: &Hole<'_>, + range: TextRange, + edits: &mut Vec<(TextRange, Vec)>, + ) { + let Below { holes, names } = below(parsed, node); + let original = parsed.at(range, kind_of(node)); + let unchanged = original.as_ref().is_some_and(|original| { + self.print( + parsed, + Printed::Node(original), + Spelling::Parsed, + self.indentation, + ) == self.print( + parsed, + Printed::Node(node), + Spelling::Parsed, + self.indentation, + ) + }); + if unchanged { + rename(parsed, &names, &[], edits); + } else if let (Some(Hole::Stmt(original)), Hole::Stmt(rewritten)) = (&original, node) + && let Some(suites) = self.changed_suites(parsed, original, rewritten) + { + let regions: Vec = suites.iter().map(|suite| suite.region).collect(); + rename(parsed, &names, ®ions, edits); + for suite in suites { + if edits.iter().any(|(at, _)| *at == suite.region) { + continue; + } + let indentation = self.indentation_between(range, suite.region); + let printed = Printed::Suite(&suite.statements); + let text = self.print(parsed, printed, Spelling::Rewritten, &indentation); + let holes = below_statements(parsed, &suite.statements).holes; + edits.push(( + suite.region, + self.fragments(parsed, printed, suite.region, &text, &holes, &indentation), + )); + } + } else if !edits.iter().any(|(at, _)| *at == range) { + let indentation = match &original { + Some(Hole::Stmt(original)) => self.indentation_of(original), + _ => self.indentation.clone(), + }; + let printed = Printed::Node(node); + let text = self.print(parsed, printed, Spelling::Rewritten, &indentation); + edits.push(( + range, + self.fragments(parsed, printed, range, &text, &holes, &indentation), + )); + } + for hole in &holes { + self.node(parsed, hole, hole.range(), edits); + } + } + + /// the suites a pass changed the statements of, when it changed nothing else about + /// `rewritten`, the statement `original` was parsed as + /// + /// `None` when its clauses differ, or when a suite that changed cannot be re-emitted + /// over source of its own: one the source wrote no statement in, or one the passes + /// emptied + fn changed_suites<'m>( + &self, + parsed: &Parsed<'_>, + original: &Stmt, + rewritten: &'m Stmt, + ) -> Option>> { + let original_suites = suites(original); + let rewritten_suites = suites(rewritten); + if original_suites.is_empty() || original_suites.len() != rewritten_suites.len() { + return None; + } + let original_clauses = self.without_suites(original); + let rewritten_clauses = self.without_suites(rewritten); + let clauses = |stmt: &Stmt| { + self.print( + parsed, + Printed::Node(&Hole::Stmt(stmt)), + Spelling::Parsed, + self.indentation, + ) + }; + if clauses(&original_clauses) != clauses(&rewritten_clauses) { + return None; + } + // the parser builds some statements of a suite out of its clause — the field an + // `init(...)` parameter declares — and gives them the range of what they were + // built from, or none. the source does not write them in the suite + let mut clause_ranges = ClauseRanges::default(); + clause_ranges.visit_stmt(&original_clauses); + let written = |stmt: &Stmt| { + let range = stmt.range(); + !range.is_empty() + && !clause_ranges + .ranges + .iter() + .any(|clause| clause.contains_range(range)) + }; + let mut changed = Vec::new(); + for (original_suite, rewritten_suite) in original_suites.into_iter().zip(rewritten_suites) { + let original_statements: Vec<&Stmt> = + original_suite.iter().filter(|stmt| written(stmt)).collect(); + let statements: Vec<&'m Stmt> = rewritten_suite + .iter() + .filter(|stmt| !is_kept(parsed, stmt.range(), stmt_kind(stmt)) || written(stmt)) + .collect(); + let same = original_statements.len() == statements.len() + && original_statements + .iter() + .zip(&statements) + .all(|(original, stmt)| { + stmt.range() == original.range() + && is_kept(parsed, stmt.range(), stmt_kind(stmt)) + }); + if same { + continue; + } + let (Some(first), Some(last)) = + (original_statements.first(), original_statements.last()) + else { + return None; + }; + if statements.is_empty() { + return None; + } + let region = TextRange::new(first.start(), last.end()); + changed.push(ChangedSuite { region, statements }); + } + Some(changed) + } + + /// `stmt` with each of its suites a placeholder, so it prints its clauses alone + fn without_suites(&self, stmt: &Stmt) -> Stmt { + let mut copy = stmt.clone(); + for suite in suites_mut(&mut copy) { + *suite = Suite::from([Stmt::Expr(StmtExpr { + node_index: AtomicNodeIndex::NONE, + range: TextRange::default(), + value: Box::new(Expr::Name(ExprName { + node_index: AtomicNodeIndex::NONE, + range: TextRange::default(), + id: Name::from(format!("{}suite", self.label)), + ctx: ExprContext::Load, + })), + })]); + } + copy + } + + /// the step `stmt`'s source indents its blocks by, read off the first statement of + /// its first suite that starts a line + fn indentation_of(&self, stmt: &Stmt) -> Indentation { + suites(stmt) + .into_iter() + .flatten() + .find(|body| !body.range().is_empty() && self.line_prefix(body.range()).is_some()) + .map_or_else( + || self.indentation.clone(), + |body| self.indentation_between(stmt.range(), body.range()), + ) + } + + /// the step from the indentation of the line `outer` starts on to that of `inner`, + /// or the file's when that is not a step deeper + fn indentation_between(&self, outer: TextRange, inner: TextRange) -> Indentation { + let outer_line = &self.source[self.line_start(outer.start())..]; + let outer_prefix = &outer_line[..outer_line + .find(|c: char| c != ' ' && c != '\t') + .unwrap_or(outer_line.len())]; + match self.line_prefix(inner) { + Some(inner) if inner.len() > outer_prefix.len() && inner.starts_with(outer_prefix) => { + Indentation::new(inner[outer_prefix.len()..].to_string()) + } + _ => self.indentation.clone(), + } + } + + /// the whitespace the line holding `offset` starts with + fn line_indentation(&self, offset: TextSize) -> &str { + let line = &self.source[self.line_start(offset)..]; + &line[..line + .find(|c: char| c != ' ' && c != '\t') + .unwrap_or(line.len())] + } + + /// where the line holding `offset` starts + fn line_start(&self, offset: TextSize) -> usize { + self.source[..usize::from(offset)] + .rfind('\n') + .map_or(0, |at| at + 1) + } + + /// `printed` with each node below it that kept a range as a placeholder + fn print( + &self, + parsed: &Parsed<'_>, + printed: Printed<'_, '_>, + spelling: Spelling, + indentation: &Indentation, + ) -> String { + let placeholders = Placeholders { + label: &self.label, + parsed, + spelling, + }; + let statement = |stmt: &Stmt| { + let mut copy = stmt.clone(); + placeholders.stmt_names(&mut copy); + transformer::walk_stmt(&placeholders, &mut copy); + generator(indentation) + .stmt(©) + .trim_end_matches('\n') + .to_string() + }; + match printed { + Printed::Node(Hole::Expr(expr)) => { + let mut copy = (*expr).clone(); + placeholders.expr_names(&mut copy); + transformer::walk_expr(&placeholders, &mut copy); + generator(indentation).expr(©) + } + Printed::Node(Hole::Stmt(stmt)) => statement(stmt), + // each statement is printed where a placeholder for the suite would be, so + // a statement a pass kept is still a placeholder rather than printed + Printed::Suite(statements) => statements + .iter() + .map(|stmt| { + if is_kept(parsed, stmt.range(), stmt_kind(stmt)) { + placeholder(&self.label, stmt.range()) + } else { + statement(stmt) + } + }) + .collect::>() + .join("\n"), + } + } + + /// the template `printed` stands for: its text, with each placeholder passing its + /// node's source through + fn fragments( + &self, + parsed: &Parsed<'_>, + node: Printed<'_, '_>, + range: TextRange, + printed: &str, + holes: &[Hole<'_>], + indentation: &Indentation, + ) -> Vec { + let by_label: HashMap> = holes + .iter() + .map(|hole| (placeholder(&self.label, hole.range()), hole)) + .collect(); + // a statement below the first line of the source is indented by the line it + // starts on, which the generator, printing it on its own, knows nothing of + let indent = match node { + Printed::Node(Hole::Stmt(_)) | Printed::Suite(_) => self.line_prefix(range), + Printed::Node(Hole::Expr(_)) => None, + }; + // a suite written on the line of its clause — `def f(): x` — has no line of its + // own to take an indentation from, and what it prints does not fit on one: it is + // written as a block below the clause instead, a step deeper than the line the + // clause starts on, which leaves the clause and its lowerings the source they are + // written in + let opens_a_block = indent.is_none() && matches!(node, Printed::Suite(_)); + let indent = match (indent, opens_a_block) { + (indent, false) => indent, + (_, true) => Some(format!( + "{}{}", + self.line_indentation(range.start()), + indentation.as_str() + )), + }; + let mut indented = String::new(); + if opens_a_block { + indented.push('\n'); + indented.push_str(indent.as_deref().unwrap_or_default()); + } + push_indented(&mut indented, printed, indent.as_deref()); + let mut fragments: Vec = Vec::new(); + let mut text = String::new(); + // where the source the text printed so far stands in for begins + let mut after = range.start(); + let mut rest = indented.as_str(); + while let Some(at) = rest.find(&self.label) { + let tail = &rest[at..]; + let end = tail + .find(|c: char| !(c.is_ascii_alphanumeric() || c == '_')) + .unwrap_or(tail.len()); + let Some(hole) = by_label.get(&tail[..end]) else { + text.push_str(&rest[..at + end]); + rest = &rest[at + end..]; + continue; + }; + text.push_str(&rest[..at]); + let parenthesize = match hole { + Hole::Expr(expr) => self.needs_parentheses(parsed, node, expr, indentation), + Hole::Stmt(_) => false, + }; + if parenthesize { + text.push('('); + } + let mut segment = self.segment(std::mem::take(&mut text), after, hole.range().start()); + // the statement's own lines keep the source's indentation, so its first line + // does as well + if segment.tail.is_none() + && let Hole::Stmt(stmt) = hole + && let Some(prefix) = self.line_prefix(stmt.range()) + { + let trimmed = segment.text.trim_end_matches([' ', '\t']).len(); + if segment.text[..trimmed].ends_with('\n') { + segment.text.truncate(trimmed); + segment.text.push_str(&prefix); + } + } + segment.push(&mut fragments); + push_source(&mut fragments, hole.range()); + if parenthesize { + text.push(')'); + } + after = hole.range().end(); + rest = &rest[at + end..]; + } + text.push_str(rest); + self.segment(text, after, range.end()).push(&mut fragments); + fragments + } + + /// how `text`, printed between two nodes a template passes through, stands in for + /// the source between them, from `start` to `end` + /// + /// where the printed tokens begin as the source's do, the source is kept up to the + /// last of them, and likewise where they end as the source's do. that keeps what + /// the printer has no spelling for — the comments, the blank lines, a signature + /// laid out over several lines — and an edit another pass made there, such as the + /// guard a `raises` clause inserts ahead of a decorated `def`. what is left between + /// is printed, with the comments at either end of the source it stands for + fn segment(&self, text: String, start: TextSize, end: TextSize) -> Segment { + if start > end { + // the passes reordered the nodes around it, and it stands for no source + return Segment { + head: None, + text, + tail: None, + }; + } + let gap = TextRange::new(start, end); + let printed = TextRange::up_to(text.text_len()); + let same = |ours: &SimpleToken, theirs: &SimpleToken| { + ours.kind() == theirs.kind() && text[ours.range()] == self.source[theirs.range()] + }; + let (front, whole) = forward_tokens(&text, printed); + let (source_front, source_whole) = forward_tokens(self.source, gap); + let mut head = front + .iter() + .zip(&source_front) + .take_while(|(ours, theirs)| same(ours, theirs)) + .count(); + if whole && source_whole && head == front.len() && head == source_front.len() { + return Segment { + head: Some(gap), + text: String::new(), + tail: None, + }; + } + // an edit the source is split through, or an insertion where it is split, could + // belong to either side: to the source, or to what the printer spelled in its + // place. it is left whole on the printed side, where it is reported if it is lost + while head > 0 && !self.splits_cleanly(source_front[head - 1].end()) { + head -= 1; + } + let head_end = front[..head].last().map_or(printed.start(), Ranged::end); + let source_head_end = source_front[..head].last().map_or(start, Ranged::end); + let back = backward_tokens(&text, printed, &[]); + let source_back = backward_tokens(self.source, gap, self.comments); + let mut tail = back + .iter() + .zip(&source_back) + .take_while(|(ours, theirs)| { + ours.start() >= head_end && theirs.start() >= source_head_end && same(ours, theirs) + }) + .count(); + while tail > 0 && !self.splits_cleanly(source_back[tail - 1].start()) { + tail -= 1; + } + let tail_start = back[..tail].last().map_or(printed.end(), Ranged::start); + let source_tail_start = source_back[..tail].last().map_or(end, Ranged::start); + Segment { + head: (head > 0).then(|| TextRange::new(start, source_head_end)), + text: self.with_comments( + &text[TextRange::new(head_end, tail_start)], + TextRange::new(source_head_end, source_tail_start), + ), + tail: (tail > 0).then(|| TextRange::new(source_tail_start, end)), + } + } + + /// whether no edit stands across `offset`, or is inserted at it + fn splits_cleanly(&self, offset: TextSize) -> bool { + !self.edited.iter().any(|edit| { + if edit.is_empty() { + edit.start() == offset + } else { + edit.start() < offset && offset < edit.end() + } + }) + } + + /// `text`, printed where `gap` stands in the source, with the comments and the blank + /// lines of `gap` that lie ahead of its first token and after its last + /// + /// a comment ahead of the first token ends a line of the source or sits on one of + /// its own, and is kept where the printed text breaks the line there too; likewise + /// after the last token. a comment between two tokens the printer spelled + /// differently has no place in the printed line it would have to sit in, and is not + /// kept + fn with_comments(&self, text: &str, gap: TextRange) -> String { + let core_start = text.len() - text.trim_start().len(); + let core_end = text.trim_end().len(); + if core_start >= core_end { + return text.to_owned(); + } + let source = &self.source[gap]; + let (lead, trail) = match self.token_bounds(gap) { + Some((first, last)) => ( + &source[..usize::from(first - gap.start())], + &source[usize::from(last - gap.start())..], + ), + // no token between two statements: what ends the line of the first belongs to + // it, and the lines after that to the second + None => match source.find('\n') { + Some(newline) => (&source[..=newline], &source[newline..]), + None => return text.to_owned(), + }, + }; + let mut out = respaced(&text[..core_start], lead); + out.push_str(&text[core_start..core_end]); + out.push_str(&respaced(&text[core_end..], trail)); + out + } + + /// where the first token in `gap` begins and the last one ends, looking past + /// whitespace and comments. a line continuation counts as a token, so what is + /// looked past is only ever something a line break may be put around + fn token_bounds(&self, gap: TextRange) -> Option<(TextSize, TextSize)> { + let comments = self.comments.comments_in_range(gap); + let is_blank = |c: char| matches!(c, ' ' | '\t' | '\n' | '\r' | '\x0c'); + let mut first = gap.start(); + loop { + if first >= gap.end() { + return None; + } + if let Some(comment) = comments.iter().find(|comment| comment.start() == first) { + first = comment.end(); + continue; + } + let c = self.source[usize::from(first)..].chars().next()?; + if !is_blank(c) { + break; + } + first += c.text_len(); + } + let mut last = gap.end(); + loop { + if let Some(comment) = comments.iter().find(|comment| comment.end() == last) { + last = comment.start(); + continue; + } + let c = self.source[..usize::from(last)].chars().next_back()?; + if !is_blank(c) { + break; + } + last -= c.text_len(); + } + Some((first, last)) + } + + /// the whitespace ahead of `range` on its line, where nothing else is + fn line_prefix(&self, range: TextRange) -> Option { + let start = usize::from(range.start()); + let line_start = self.source[..start].rfind('\n').map_or(0, |at| at + 1); + let prefix = &self.source[line_start..start]; + prefix + .chars() + .all(|c| c == ' ' || c == '\t') + .then(|| prefix.to_string()) + } + + /// whether `hole`'s source has to be parenthesized where `node` prints it + /// + /// a placeholder is a name, which binds tighter than anything, so the printed + /// text says nothing about the expression it stands for. that is asked by + /// printing `node` again with the expression itself in the placeholder's place, + /// its own operands as placeholders, and reading whether the generator put + /// parentheses around it + fn needs_parentheses( + &self, + parsed: &Parsed<'_>, + node: Printed<'_, '_>, + hole: &Expr, + indentation: &Indentation, + ) -> bool { + if binds_tightest(hole) { + return false; + } + let placeholders = Placeholders { + label: &self.label, + parsed, + spelling: Spelling::Rewritten, + }; + let mut operand = hole.clone(); + transformer::walk_expr(&placeholders, &mut operand); + let alone = generator(indentation).expr(&operand); + let probe = Probe { + label: &self.label, + parsed, + range: hole.range(), + operand: &operand, + }; + let statement = |stmt: &Stmt| { + let mut copy = stmt.clone(); + transformer::walk_stmt(&probe, &mut copy); + generator(indentation).stmt(©) + }; + let printed = match node { + Printed::Node(Hole::Expr(expr)) => { + let mut copy = (*expr).clone(); + transformer::walk_expr(&probe, &mut copy); + generator(indentation).expr(©) + } + Printed::Node(Hole::Stmt(stmt)) => statement(stmt), + Printed::Suite(statements) => statements + .iter() + .map(|stmt| statement(stmt)) + .collect::(), + }; + match printed.find(&alone) { + Some(at) => { + printed[..at].ends_with('(') && printed[at + alone.len()..].starts_with(')') + } + // printed differently in place than alone, so nothing can be read off it + None => true, + } + } +} + +fn generator(indentation: &Indentation) -> Generator<'_> { + Generator::new(indentation, LineEnding::Lf).with_mode(Mode::BasedPython) +} + +/// what a template prints: a node, or the statements of a suite +#[derive(Clone, Copy)] +enum Printed<'p, 'm> { + Node(&'p Hole<'m>), + Suite(&'p [&'m Stmt]), +} + +/// a suite whose statements a pass changed, to be re-emitted over `region`, the source +/// from the first statement the source wrote in it to the last +struct ChangedSuite<'m> { + region: TextRange, + statements: Vec<&'m Stmt>, +} + +/// the edits that re-emit each of `names` a pass renamed, at its own range, other than +/// those inside `regions`, which a template re-emits +fn rename( + parsed: &Parsed<'_>, + names: &[&Identifier], + regions: &[TextRange], + edits: &mut Vec<(TextRange, Vec)>, +) { + for name in names { + if regions + .iter() + .any(|region| region.contains_range(name.range)) + { + continue; + } + if parsed + .identifier(name.range) + .is_some_and(|original| original.id != name.id) + { + edits.push((name.range, vec![Fragment::Lit(name.id.to_string())])); + } + } +} + +/// the suites `stmt` holds, in source order +fn suites(stmt: &Stmt) -> Vec<&Suite> { + match stmt { + Stmt::FunctionDef(node) => vec![&node.body], + Stmt::ClassDef(node) => vec![&node.body], + Stmt::For(node) => vec![&node.body, &node.orelse], + Stmt::While(node) => vec![&node.body, &node.orelse], + Stmt::If(node) => std::iter::once(&node.body) + .chain(node.elif_else_clauses.iter().map(|clause| &clause.body)) + .collect(), + Stmt::Let(node) => vec![&node.orelse], + Stmt::With(node) => vec![&node.body], + Stmt::Match(node) => node.cases.iter().map(|case| &case.body).collect(), + Stmt::Try(node) => std::iter::once(&node.body) + .chain( + node.handlers + .iter() + .map(|ExceptHandler::ExceptHandler(handler)| &handler.body), + ) + .chain([&node.orelse, &node.finalbody]) + .collect(), + Stmt::Return(_) + | Stmt::Delete(_) + | Stmt::TypeAlias(_) + | Stmt::Assign(_) + | Stmt::AugAssign(_) + | Stmt::AnnAssign(_) + | Stmt::Raise(_) + | Stmt::Assert(_) + | Stmt::Import(_) + | Stmt::ImportFrom(_) + | Stmt::Global(_) + | Stmt::Nonlocal(_) + | Stmt::Expr(_) + | Stmt::Pass(_) + | Stmt::Break(_) + | Stmt::Continue(_) + | Stmt::IpyEscapeCommand(_) => Vec::new(), + } +} + +/// as [`suites`], to be replaced +fn suites_mut(stmt: &mut Stmt) -> Vec<&mut Suite> { + match stmt { + Stmt::FunctionDef(node) => vec![&mut node.body], + Stmt::ClassDef(node) => vec![&mut node.body], + Stmt::For(node) => vec![&mut node.body, &mut node.orelse], + Stmt::While(node) => vec![&mut node.body, &mut node.orelse], + Stmt::If(node) => std::iter::once(&mut node.body) + .chain( + node.elif_else_clauses + .iter_mut() + .map(|clause| &mut clause.body), + ) + .collect(), + Stmt::Let(node) => vec![&mut node.orelse], + Stmt::With(node) => vec![&mut node.body], + Stmt::Match(node) => node.cases.iter_mut().map(|case| &mut case.body).collect(), + Stmt::Try(node) => std::iter::once(&mut node.body) + .chain( + node.handlers + .iter_mut() + .map(|ExceptHandler::ExceptHandler(handler)| &mut handler.body), + ) + .chain([&mut node.orelse, &mut node.finalbody]) + .collect(), + Stmt::Return(_) + | Stmt::Delete(_) + | Stmt::TypeAlias(_) + | Stmt::Assign(_) + | Stmt::AugAssign(_) + | Stmt::AnnAssign(_) + | Stmt::Raise(_) + | Stmt::Assert(_) + | Stmt::Import(_) + | Stmt::ImportFrom(_) + | Stmt::Global(_) + | Stmt::Nonlocal(_) + | Stmt::Expr(_) + | Stmt::Pass(_) + | Stmt::Break(_) + | Stmt::Continue(_) + | Stmt::IpyEscapeCommand(_) => Vec::new(), + } +} + +/// the range of every node a statement's clauses hold, below the statement itself +#[derive(Default)] +struct ClauseRanges { + ranges: Vec, + entered: bool, +} + +impl<'a> SourceOrderVisitor<'a> for ClauseRanges { + fn enter_node(&mut self, node: AnyNodeRef<'a>) -> TraversalSignal { + if std::mem::replace(&mut self.entered, true) && !node.range().is_empty() { + self.ranges.push(node.range()); + } + TraversalSignal::Traverse + } +} + +/// `stmt` as a node a pass made, rather than one read from this file's source +/// +/// a pass that builds a node by parsing text of its own is handed ranges into that +/// text, which name nothing here — and a range is what says a node came from the +/// source, see the module docs +pub(crate) fn forget_stmt_ranges(stmt: &mut Stmt) { + Forget.visit_stmt(stmt); + ForgetNames.visit_stmt(stmt); +} + +/// as [`forget_stmt_ranges`], for an expression +pub(crate) fn forget_expr_ranges(expr: &mut Expr) { + Forget.visit_expr(expr); + ForgetNames.visit_expr(expr); +} + +/// forgets the range of every name, which [`Forget`]'s expression relocation leaves +struct ForgetNames; + +impl NameTransformer for ForgetNames { + fn name(&self, name: &mut Identifier) { + name.range = TextRange::default(); + } +} + +impl Transformer for ForgetNames { + fn visit_stmt(&self, stmt: &mut Stmt) { + self.stmt_names(stmt); + transformer::walk_stmt(self, stmt); + } + + fn visit_expr(&self, expr: &mut Expr) { + self.expr_names(expr); + transformer::walk_expr(self, expr); + } + + fn visit_parameter(&self, parameter: &mut Parameter) { + self.name(&mut parameter.name); + transformer::walk_parameter(self, parameter); + } + + fn visit_keyword(&self, keyword: &mut Keyword) { + self.keyword_names(keyword); + transformer::walk_keyword(self, keyword); + } + + fn visit_alias(&self, alias: &mut Alias) { + self.alias_names(alias); + } + + fn visit_type_param(&self, type_param: &mut TypeParam) { + self.type_param_names(type_param); + transformer::walk_type_param(self, type_param); + } + + fn visit_except_handler(&self, except_handler: &mut ExceptHandler) { + self.except_handler_names(except_handler); + transformer::walk_except_handler(self, except_handler); + } + + fn visit_pattern(&self, pattern: &mut Pattern) { + self.pattern_names(pattern); + transformer::walk_pattern(self, pattern); + } + + fn visit_pattern_keyword(&self, pattern_keyword: &mut PatternKeyword) { + self.name(&mut pattern_keyword.attr); + transformer::walk_pattern_keyword(self, pattern_keyword); + } +} + +/// what is done to each name a node spells itself, as opposed to the names spelled by +/// the nodes below it +trait NameTransformer { + fn name(&self, name: &mut Identifier); + + fn stmt_names(&self, stmt: &mut Stmt) { + match stmt { + Stmt::FunctionDef(function) => self.name(&mut function.name), + Stmt::ClassDef(class) => self.name(&mut class.name), + Stmt::ImportFrom(import) => { + if let Some(module) = &mut import.module { + self.name(module); + } + } + Stmt::Global(global) => global.names.iter_mut().for_each(|name| self.name(name)), + Stmt::Nonlocal(nonlocal) => nonlocal.names.iter_mut().for_each(|name| self.name(name)), + _ => {} + } + } + + fn expr_names(&self, expr: &mut Expr) { + match expr { + Expr::Attribute(attribute) => self.name(&mut attribute.attr), + Expr::ProtocolMethod(method) => self.name(&mut method.name), + _ => {} + } + } + + fn keyword_names(&self, keyword: &mut Keyword) { + if let Some(arg) = &mut keyword.arg { + self.name(arg); + } + } + + fn alias_names(&self, alias: &mut Alias) { + self.name(&mut alias.name); + if let Some(asname) = &mut alias.asname { + self.name(asname); + } + } + + fn type_param_names(&self, type_param: &mut TypeParam) { + match type_param { + TypeParam::TypeVar(type_var) => self.name(&mut type_var.name), + TypeParam::TypeVarTuple(type_var_tuple) => self.name(&mut type_var_tuple.name), + TypeParam::ParamSpec(param_spec) => self.name(&mut param_spec.name), + } + } + + fn except_handler_names(&self, except_handler: &mut ExceptHandler) { + let ExceptHandler::ExceptHandler(handler) = except_handler; + if let Some(name) = &mut handler.name { + self.name(name); + } + } + + fn pattern_names(&self, pattern: &mut Pattern) { + let name = match pattern { + Pattern::MatchAs(pattern) => pattern.name.as_mut(), + Pattern::MatchStar(pattern) => pattern.name.as_mut(), + Pattern::MatchMapping(pattern) => pattern.rest.as_mut(), + _ => None, + }; + if let Some(name) = name { + self.name(name); + } + } +} + +/// the names a node spells itself, as [`NameTransformer`] reaches them +fn own_names(node: AnyNodeRef<'_>) -> Vec<&Identifier> { + match node { + AnyNodeRef::StmtFunctionDef(function) => vec![&function.name], + AnyNodeRef::StmtClassDef(class) => vec![&class.name], + AnyNodeRef::StmtImportFrom(import) => import.module.iter().collect(), + AnyNodeRef::StmtGlobal(global) => global.names.iter().collect(), + AnyNodeRef::StmtNonlocal(nonlocal) => nonlocal.names.iter().collect(), + AnyNodeRef::ExprAttribute(attribute) => vec![&attribute.attr], + AnyNodeRef::ExprProtocolMethod(method) => vec![&method.name], + AnyNodeRef::Parameter(parameter) => vec![¶meter.name], + AnyNodeRef::Keyword(keyword) => keyword.arg.iter().collect(), + AnyNodeRef::Alias(alias) => std::iter::once(&alias.name).chain(&alias.asname).collect(), + AnyNodeRef::TypeParamTypeVar(type_var) => vec![&type_var.name], + AnyNodeRef::TypeParamTypeVarTuple(type_var_tuple) => vec![&type_var_tuple.name], + AnyNodeRef::TypeParamParamSpec(param_spec) => vec![¶m_spec.name], + AnyNodeRef::ExceptHandlerExceptHandler(handler) => handler.name.iter().collect(), + AnyNodeRef::PatternMatchAs(pattern) => pattern.name.iter().collect(), + AnyNodeRef::PatternMatchStar(pattern) => pattern.name.iter().collect(), + AnyNodeRef::PatternMatchMapping(pattern) => pattern.rest.iter().collect(), + AnyNodeRef::PatternKeyword(keyword) => vec![&keyword.attr], + _ => Vec::new(), + } +} + +struct Forget; + +impl Transformer for Forget { + fn visit_expr(&self, expr: &mut Expr) { + ruff_python_ast::relocate::relocate_expr(expr, TextRange::default()); + } + + fn visit_stmt(&self, stmt: &mut Stmt) { + let range = match stmt { + Stmt::FunctionDef(node) => &mut node.range, + Stmt::ClassDef(node) => &mut node.range, + Stmt::Return(node) => &mut node.range, + Stmt::Delete(node) => &mut node.range, + Stmt::TypeAlias(node) => &mut node.range, + Stmt::Assign(node) => &mut node.range, + Stmt::AugAssign(node) => &mut node.range, + Stmt::AnnAssign(node) => &mut node.range, + Stmt::For(node) => &mut node.range, + Stmt::While(node) => &mut node.range, + Stmt::If(node) => &mut node.range, + Stmt::Let(node) => &mut node.range, + Stmt::With(node) => &mut node.range, + Stmt::Match(node) => &mut node.range, + Stmt::Raise(node) => &mut node.range, + Stmt::Try(node) => &mut node.range, + Stmt::Assert(node) => &mut node.range, + Stmt::Import(node) => &mut node.range, + Stmt::ImportFrom(node) => &mut node.range, + Stmt::Global(node) => &mut node.range, + Stmt::Nonlocal(node) => &mut node.range, + Stmt::Expr(node) => &mut node.range, + Stmt::Pass(node) => &mut node.range, + Stmt::Break(node) => &mut node.range, + Stmt::Continue(node) => &mut node.range, + Stmt::IpyEscapeCommand(node) => &mut node.range, + }; + *range = TextRange::default(); + transformer::walk_stmt(self, stmt); + } +} + +/// the source a template keeps around the text it prints between two passthroughs, and +/// that text +struct Segment { + head: Option, + text: String, + tail: Option, +} + +impl Segment { + fn push(self, fragments: &mut Vec) { + if let Some(head) = self.head { + push_source(fragments, head); + } + if !self.text.is_empty() { + fragments.push(Fragment::Lit(self.text)); + } + if let Some(tail) = self.tail { + push_source(fragments, tail); + } + } +} + +/// the tokens of `range` in `text` from its start, up to the first the simple tokenizer +/// cannot read — a string, a number — and whether there was none +fn forward_tokens(text: &str, range: TextRange) -> (Vec, bool) { + readable(SimpleTokenizer::new(text, range)) +} + +/// as [`forward_tokens`], from the end of `range`, last first +fn backward_tokens(text: &str, range: TextRange, comments: &[TextRange]) -> Vec { + readable(BackwardsTokenizer::new(text, range, comments)).0 +} + +fn readable(tokens: impl Iterator) -> (Vec, bool) { + let mut read = Vec::new(); + for token in tokens { + match token.kind() { + SimpleTokenKind::Other | SimpleTokenKind::Bogus => return (read, false), + kind if kind.is_trivia() => {} + _ => read.push(token), + } + } + (read, true) +} + +/// `range` of the source passed through after `fragments`, as one span with a +/// passthrough of the source just before it +/// +/// an edit that spans the two is inside the one span, and materializes there +fn push_source(fragments: &mut Vec, range: TextRange) { + if let Some(Fragment::Src(previous)) = fragments.last_mut() + && previous.end() == range.start() + { + *previous = previous.cover(range); + } else if !range.is_empty() { + fragments.push(Fragment::Src(range)); + } +} + +/// `whitespace`, printed between two tokens, holding the lines `trivia`, the source +/// between the same two tokens, holds: its comments and blank lines, followed by the +/// indentation the printed text continues at +/// +/// only a line break the printed text makes is given them — anywhere else, a comment +/// would run into the code after it +fn respaced(whitespace: &str, trivia: &str) -> String { + match (whitespace.rfind('\n'), trivia.rfind('\n')) { + (Some(printed), Some(source)) => { + format!("{}{}", &trivia[..=source], &whitespace[printed + 1..]) + } + _ => whitespace.to_owned(), + } +} + +/// `text` appended to `out`, each line after the first indented by `indent` +fn push_indented(out: &mut String, text: &str, indent: Option<&str>) { + match indent { + Some(indent) if !indent.is_empty() => { + let mut lines = text.split('\n'); + if let Some(first) = lines.next() { + out.push_str(first); + } + for line in lines { + out.push('\n'); + if !line.is_empty() { + out.push_str(indent); + } + out.push_str(line); + } + } + _ => out.push_str(text), + } +} + +/// an expression whose source needs no parentheses wherever an expression can stand, +/// or which cannot be given any +fn binds_tightest(expr: &Expr) -> bool { + match expr { + Expr::Tuple(tuple) => { + tuple.parenthesized + || tuple + .elts + .iter() + .any(|element| matches!(element, Expr::Slice(_) | Expr::Starred(_))) + } + Expr::Generator(generator) => generator.parenthesized, + Expr::Name(_) + | Expr::Attribute(_) + | Expr::Subscript(_) + | Expr::Call(_) + | Expr::StringLiteral(_) + | Expr::BytesLiteral(_) + | Expr::FString(_) + | Expr::TString(_) + | Expr::NumberLiteral(_) + | Expr::BooleanLiteral(_) + | Expr::NoneLiteral(_) + | Expr::EllipsisLiteral(_) + | Expr::List(_) + | Expr::Dict(_) + | Expr::Set(_) + | Expr::ListComp(_) + | Expr::SetComp(_) + | Expr::DictComp(_) + | Expr::Starred(_) + | Expr::Slice(_) => true, + _ => false, + } +} + +fn placeholder(label: &str, range: TextRange) -> String { + format!( + "{label}{}x{}", + u32::from(range.start()), + u32::from(range.end()) + ) +} + +fn placeholder_name(label: &str, range: TextRange) -> Expr { + Expr::Name(ExprName { + node_index: AtomicNodeIndex::NONE, + range: TextRange::default(), + id: Name::from(placeholder(label, range)), + ctx: ExprContext::Load, + }) +} + +/// how a printed node spells the names it kept from the source +#[derive(Clone, Copy)] +enum Spelling { + /// as the passes left them + Rewritten, + /// as the source spelled them, so a node whose one change is a name prints as it + /// was parsed + Parsed, +} + +/// replaces each node the passes kept with a placeholder naming it +struct Placeholders<'l> { + label: &'l str, + parsed: &'l Parsed<'l>, + spelling: Spelling, +} + +impl NameTransformer for Placeholders<'_> { + fn name(&self, name: &mut Identifier) { + if let Spelling::Parsed = self.spelling + && let Some(original) = self.parsed.identifier(name.range) + { + name.id = original.id.clone(); + } + } +} + +impl Transformer for Placeholders<'_> { + fn visit_expr(&self, expr: &mut Expr) { + if is_kept(self.parsed, expr.range(), expr_kind(expr)) { + *expr = placeholder_name(self.label, expr.range()); + } else { + self.expr_names(expr); + transformer::walk_expr(self, expr); + } + } + + fn visit_parameter(&self, parameter: &mut Parameter) { + self.name(&mut parameter.name); + transformer::walk_parameter(self, parameter); + } + + fn visit_keyword(&self, keyword: &mut Keyword) { + self.keyword_names(keyword); + transformer::walk_keyword(self, keyword); + } + + fn visit_alias(&self, alias: &mut Alias) { + self.alias_names(alias); + } + + fn visit_type_param(&self, type_param: &mut TypeParam) { + self.type_param_names(type_param); + transformer::walk_type_param(self, type_param); + } + + fn visit_except_handler(&self, except_handler: &mut ExceptHandler) { + self.except_handler_names(except_handler); + transformer::walk_except_handler(self, except_handler); + } + + fn visit_pattern(&self, pattern: &mut Pattern) { + self.pattern_names(pattern); + transformer::walk_pattern(self, pattern); + } + + fn visit_pattern_keyword(&self, pattern_keyword: &mut PatternKeyword) { + self.name(&mut pattern_keyword.attr); + transformer::walk_pattern_keyword(self, pattern_keyword); + } + + fn visit_stmt(&self, stmt: &mut Stmt) { + if !is_kept(self.parsed, stmt.range(), stmt_kind(stmt)) { + self.stmt_names(stmt); + transformer::walk_stmt(self, stmt); + } else { + *stmt = Stmt::Expr(StmtExpr { + node_index: AtomicNodeIndex::NONE, + range: TextRange::default(), + value: Box::new(placeholder_name(self.label, stmt.range())), + }); + } + } +} + +/// as [`Placeholders`], but the node at `range` becomes `operand` +struct Probe<'p> { + label: &'p str, + parsed: &'p Parsed<'p>, + range: TextRange, + operand: &'p Expr, +} + +impl Transformer for Probe<'_> { + fn visit_expr(&self, expr: &mut Expr) { + if !is_kept(self.parsed, expr.range(), expr_kind(expr)) { + transformer::walk_expr(self, expr); + } else if expr.range() == self.range { + *expr = self.operand.clone(); + } else { + *expr = placeholder_name(self.label, expr.range()); + } + } + + fn visit_stmt(&self, stmt: &mut Stmt) { + if !is_kept(self.parsed, stmt.range(), stmt_kind(stmt)) { + transformer::walk_stmt(self, stmt); + } else { + *stmt = Stmt::Expr(StmtExpr { + node_index: AtomicNodeIndex::NONE, + range: TextRange::default(), + value: Box::new(placeholder_name(self.label, stmt.range())), + }); + } + } +} + +/// what `node` holds that it does not print itself +struct Below<'m> { + /// the nodes directly below it, looking through what a pass synthesized, that + /// kept their ranges + holes: Vec>, + /// the names it spells, its own and those of the nodes between it and its holes + names: Vec<&'m Identifier>, +} + +fn below<'m>(parsed: &Parsed<'_>, node: &Hole<'m>) -> Below<'m> { + struct Collect<'m, 'p> { + parsed: &'p Parsed<'p>, + below: Below<'m>, + } + impl<'m> Visitor<'m> for Collect<'m, '_> { + fn visit_expr(&mut self, expr: &'m Expr) { + if is_kept(self.parsed, expr.range(), expr_kind(expr)) { + self.below.holes.push(Hole::Expr(expr)); + } else { + self.below.names.extend(own_names(expr.into())); + visitor::walk_expr(self, expr); + } + } + + fn visit_stmt(&mut self, stmt: &'m Stmt) { + if is_kept(self.parsed, stmt.range(), stmt_kind(stmt)) { + self.below.holes.push(Hole::Stmt(stmt)); + } else { + self.below.names.extend(own_names(stmt.into())); + visitor::walk_stmt(self, stmt); + } + } + + fn visit_parameter(&mut self, parameter: &'m Parameter) { + self.below.names.extend(own_names(parameter.into())); + visitor::walk_parameter(self, parameter); + } + + fn visit_keyword(&mut self, keyword: &'m Keyword) { + self.below.names.extend(own_names(keyword.into())); + visitor::walk_keyword(self, keyword); + } + + fn visit_alias(&mut self, alias: &'m Alias) { + self.below.names.extend(own_names(alias.into())); + } + + fn visit_type_param(&mut self, type_param: &'m TypeParam) { + self.below.names.extend(own_names(type_param.into())); + visitor::walk_type_param(self, type_param); + } + + fn visit_except_handler(&mut self, except_handler: &'m ExceptHandler) { + self.below.names.extend(own_names(except_handler.into())); + visitor::walk_except_handler(self, except_handler); + } + + fn visit_pattern(&mut self, pattern: &'m Pattern) { + self.below.names.extend(own_names(pattern.into())); + visitor::walk_pattern(self, pattern); + } + + fn visit_pattern_keyword(&mut self, pattern_keyword: &'m PatternKeyword) { + self.below.names.extend(own_names(pattern_keyword.into())); + visitor::walk_pattern_keyword(self, pattern_keyword); + } + } + let mut collect = Collect { + parsed, + below: Below { + holes: Vec::new(), + names: Vec::new(), + }, + }; + match node { + Hole::Expr(expr) => { + collect.below.names.extend(own_names((*expr).into())); + visitor::walk_expr(&mut collect, expr); + } + Hole::Stmt(stmt) => { + collect.below.names.extend(own_names((*stmt).into())); + visitor::walk_stmt(&mut collect, stmt); + } + } + collect.below +} + +/// as [`below`], for the statements of a suite +fn below_statements<'m>(parsed: &Parsed<'_>, statements: &[&'m Stmt]) -> Below<'m> { + let mut below = Below { + holes: Vec::new(), + names: Vec::new(), + }; + for stmt in statements { + if is_kept(parsed, stmt.range(), stmt_kind(stmt)) { + below.holes.push(Hole::Stmt(stmt)); + } else { + let inner = self::below(parsed, &Hole::Stmt(stmt)); + below.holes.extend(inner.holes); + below.names.extend(inner.names); + } + } + below +} + +fn kind_of(node: &Hole<'_>) -> NodeKind { + match node { + Hole::Expr(expr) => expr_kind(expr), + Hole::Stmt(stmt) => stmt_kind(stmt), + } +} + +/// every expression, statement and name of the parse, by range +#[derive(Default)] +struct Parsed<'o> { + nodes: HashMap>>, + names: HashMap, +} + +impl<'o> Parsed<'o> { + /// the name the source spells at `range` + /// + /// a name a pass made carries no range, and neither does one a pass parsed out of + /// text of its own, once [`forget_stmt_ranges`] has run over it + fn identifier(&self, range: TextRange) -> Option<&'o Identifier> { + if range.is_empty() { + return None; + } + self.names.get(&range).copied() + } + + fn record(&mut self, names: Vec<&'o Identifier>) { + for name in names { + self.names.insert(name.range, name); + } + } + + fn at(&self, range: TextRange, kind: NodeKind) -> Option> { + self.nodes.get(&range)?.iter().find_map(|node| { + (kind_of(node) == kind).then_some(match node { + Hole::Expr(expr) => Hole::Expr(expr), + Hole::Stmt(stmt) => Hole::Stmt(stmt), + }) + }) + } +} + +impl<'o> Visitor<'o> for Parsed<'o> { + fn visit_expr(&mut self, expr: &'o Expr) { + self.nodes + .entry(expr.range()) + .or_default() + .push(Hole::Expr(expr)); + self.record(own_names(expr.into())); + visitor::walk_expr(self, expr); + } + + fn visit_stmt(&mut self, stmt: &'o Stmt) { + self.nodes + .entry(stmt.range()) + .or_default() + .push(Hole::Stmt(stmt)); + self.record(own_names(stmt.into())); + visitor::walk_stmt(self, stmt); + } + + fn visit_parameter(&mut self, parameter: &'o Parameter) { + self.record(own_names(parameter.into())); + visitor::walk_parameter(self, parameter); + } + + fn visit_keyword(&mut self, keyword: &'o Keyword) { + self.record(own_names(keyword.into())); + visitor::walk_keyword(self, keyword); + } + + fn visit_alias(&mut self, alias: &'o Alias) { + self.record(own_names(alias.into())); + } + + fn visit_type_param(&mut self, type_param: &'o TypeParam) { + self.record(own_names(type_param.into())); + visitor::walk_type_param(self, type_param); + } + + fn visit_except_handler(&mut self, except_handler: &'o ExceptHandler) { + self.record(own_names(except_handler.into())); + visitor::walk_except_handler(self, except_handler); + } + + fn visit_pattern(&mut self, pattern: &'o Pattern) { + self.record(own_names(pattern.into())); + visitor::walk_pattern(self, pattern); + } + + fn visit_pattern_keyword(&mut self, pattern_keyword: &'o PatternKeyword) { + self.record(own_names(pattern_keyword.into())); + visitor::walk_pattern_keyword(self, pattern_keyword); + } +} + +#[cfg(test)] +mod tests { + use crate::{Config, transpile}; + use indoc::indoc; + + fn check(input: &str) -> String { + transpile(input, &Config::test_default()).unwrap() + } + + const TEMPERATURES: &str = indoc! {" + class Celsius: + def __init__(self, degrees: int): + self.degrees = degrees + + class Fahrenheit: + def __init__(self, degrees: int): + self.degrees = degrees + + @classmethod + def __from__(cls, value: Celsius) -> Self: + return cls(value.degrees * 9 // 5 + 32) + + def report(t: Fahrenheit) -> int: + return t.degrees + + "}; + + /// numbering a repeated `_` rewrites the `def`, and the conversion its body makes + /// is an edit the type-aware pass keyed on the body's source. a call the printer + /// would spell differently from the source went missing without a word + #[test] + fn a_conversion_in_a_def_whose_parameters_were_renamed_is_kept() { + let out = check(&format!( + "{TEMPERATURES}def f(c: Celsius, _: int, _: int, /) -> int:\n return report(\n c,\n )\n" + )); + assert!( + out.contains("def f(c: Celsius, _: int, _2: int, /) -> int:"), + "got:\n{out}" + ); + assert!(out.contains("Fahrenheit.__from__(c)"), "got:\n{out}"); + } + + /// `typeof` is lowered in the syntax tree, and the extension call beside it is an + /// edit — which printing the `def` whole reported as a transform conflict + #[test] + fn an_extension_call_beside_a_typeof_annotation_is_kept() { + let out = check(indoc! {" + marker: int = 1 + + extension list: + def second(self) -> Element: + return self[1] + + def f(xs: list[int], tag: typeof marker) -> int: + return xs.second() + "}); + assert!( + out.contains("def f(xs: list[int], tag: TypeOf[marker]) -> int:"), + "got:\n{out}" + ); + assert!(out.contains("_by_ext__list__second(xs)"), "got:\n{out}"); + } + + /// python before 3.14 evaluates a parameter's annotation where the `def` runs, and + /// `later` is not bound yet there. the quote around the annotation is an edit, and + /// the `TypeOf` inside it is the syntax tree's + #[test] + fn a_forward_reference_through_typeof_is_quoted() { + let out = check(indoc! {" + def f(x: typeof later) -> typeof later: + return x + + later: int = 1 + "}); + assert!( + out.contains("def f(x: \"TypeOf[later]\") -> \"TypeOf[later]\":"), + "got:\n{out}" + ); + } + + /// only what a pass rewrote is printed from the tree: a node it left alone is the + /// source, its comments and spelling included + #[test] + fn a_rewritten_statement_keeps_the_source_it_did_not_rewrite() { + let out = check(indoc! {" + def f(x: int | None, y: int = 3) -> int: + sentinel A + return ( # a comment the printer has no spelling for + 3 + ) + "}); + assert!( + out.contains(indoc! {" + def f(x: int | None, y: int = 3) -> int: + A = Sentinel(\"A\") + return ( # a comment the printer has no spelling for + 3 + ) + "}), + "got:\n{out}" + ); + } + + /// the source between two nodes a rewritten statement kept is kept too, wherever the + /// statement prints the same tokens there: a comment on the header's line, above the + /// first statement, between two statements and at the end of one, the blank lines + /// between them, and the layout of a signature spread over several lines + #[test] + fn comments_between_the_kept_nodes_of_a_rewritten_statement_are_kept() { + let out = check(indoc! {" + def f( + a: int, # after a parameter + b: int + ) -> int: # on the header line + # before the first statement + x = a # at the end of a statement + + # between two statements + sentinel A + return x + "}); + assert!( + out.contains(indoc! {" + def f( + a: int, # after a parameter + b: int + ) -> int: # on the header line + # before the first statement + x = a # at the end of a statement + + # between two statements + A = Sentinel(\"A\") + return x + "}), + "got:\n{out}" + ); + } + + /// a decorator is a node the statement keeps, and the source between it and the + /// `def` is kept with its comments + #[test] + fn comments_around_a_decorator_of_a_rewritten_statement_are_kept() { + let out = check(indoc! {" + class C: + @staticmethod # on the decorator line + # between the decorator and the def + def m() -> int: + sentinel A + return 1 + "}); + assert!( + out.contains(indoc! {" + class C: + @staticmethod # on the decorator line + # between the decorator and the def + def m() -> int: + A = Sentinel(\"A\") + return 1 + "}), + "got:\n{out}" + ); + } + + /// a comment beside a statement a pass replaced has no source left to be kept in, and + /// stays with the statements it sat between: one ending the line before it stays + /// there, one ending its own line ends the line of what replaced it, and one on a + /// line of its own after it stays ahead of the next statement + #[test] + fn comments_beside_a_replaced_statement_keep_their_place() { + let out = check(indoc! {" + def f() -> int: + x = 1 # at the end of the statement before + # before the replaced statement + sentinel A # at the end of the replaced statement + # before the statement after + return x + "}); + assert!( + out.contains(indoc! {" + def f() -> int: + x = 1 # at the end of the statement before + # before the replaced statement + A = Sentinel(\"A\") # at the end of the replaced statement + # before the statement after + return x + "}), + "got:\n{out}" + ); + } + + /// a comment after the last statement of a block, and one on the line of the clause + /// that follows it, lie between two statements the rewritten `if` keeps + #[test] + fn comments_around_a_clause_of_a_rewritten_statement_are_kept() { + let out = check(indoc! {" + def f(flag: bool) -> int: + if flag: + sentinel A + return 1 + # after the last statement of the block + else: # on the line of the clause + return 2 + # after the last statement of the function + "}); + assert!( + out.contains(indoc! {" + def f(flag: bool) -> int: + if flag: + A = Sentinel(\"A\") + return 1 + # after the last statement of the block + else: # on the line of the clause + return 2 + # after the last statement of the function + "}), + "got:\n{out}" + ); + } + + /// numbering a repeated `_` changes a parameter's name and nothing else about the + /// `def`, so the name is all that is re-emitted: the signature keeps its layout and + /// its comments + #[test] + fn a_renamed_parameter_is_all_that_is_re_emitted() { + let out = check(indoc! {" + def f( + _: int, # the first + _: int = 3, # the second + ) -> int: + return 1 + "}); + assert!( + out.contains(indoc! {" + def f( + _: int, # the first + _2: int = 3, /, # the second + ) -> int: + return 1 + "}), + "got:\n{out}" + ); + } + + /// the `context` prefix is deleted by an edit, which has to land beside the name + /// numbering a repeated `_` rewrote. printing the `def` whole spelled the prefix + /// again, and the transpile was refused as a conflict + #[test] + fn an_edit_beside_a_renamed_parameter_is_applied() { + let out = check(indoc! {" + def show(context _: int, context _: str) -> str: + return \"ok\" + "}); + assert!( + out.contains("def show(_: int, _2: str, /) -> str:"), + "got:\n{out}" + ); + } + + /// the shorthand `init(...)` is lowered in its header's source, which stands for a + /// `def __init__` with a `self` and a `-> None` the source never wrote. a pass that + /// rewrites a statement of its body re-emits the body alone, and the header's + /// lowering lands; printing the whole `def` spelled the header from the syntax tree, + /// and the transpile was refused + #[test] + fn an_init_shorthand_whose_body_a_pass_rewrote_is_lowered() { + let out = check(indoc! {" + class A: + init(let x: int, y: int): + sentinel S + self.z = x + y + "}); + assert!( + out.contains(indoc! {" + class A: + def __init__(self, x: int, y: int): + self.x: int = x + S = Sentinel(\"S\") + self.z = x + y + "}), + "got:\n{out}" + ); + } + + /// a modifier is a decorator in the syntax tree and a keyword in the source, which + /// the modifiers pass lowers where it is written. printing the `def` whole spelled + /// it `@private`, which is not python + #[test] + fn modifiers_of_a_def_whose_body_a_pass_rewrote_are_lowered() { + let out = check(indoc! {" + class A: + private def f(self) -> int: + sentinel S + return 1 + + static def g() -> int: + sentinel T + return 2 + "}); + assert!( + out.contains(indoc! {" + class A: + def __f(self) -> int: + S = Sentinel(\"S\") + return 1 + + @staticmethod + def g() -> int: + T = Sentinel(\"T\") + return 2 + "}), + "got:\n{out}" + ); + } + + /// a `raises` clause is deleted from the header's source, and the header of a `def` + /// whose body a pass rewrote is passed through with the deletion applied + #[test] + fn a_raises_clause_of_a_def_whose_body_a_pass_rewrote_is_deleted() { + let out = check(indoc! {" + def f() -> int raises ValueError: + sentinel S + raise ValueError + "}); + assert!( + out.contains(indoc! {" + def f() -> int: + S = Sentinel(\"S\") + raise ValueError + "}), + "got:\n{out}" + ); + } + + /// a lowering that names a parameter names the one python binds, which a repeated + /// `_` is not + #[test] + fn a_default_guard_names_the_renamed_parameter() { + let out = check(indoc! {" + def f(_: list[int] = [], _: list[int] = []) -> int: + return len(_) + "}); + assert!(out.contains("if _ is _MISSING:"), "got:\n{out}"); + assert!( + out.contains("if _2 is _MISSING:\n _2 = []"), + "got:\n{out}" + ); + } + + /// a suite written on the line of its clause is re-emitted like any other suite: + /// only what changed, as a block below the clause, so the header keeps the source + /// the `raises` clause was deleted from. printing the whole statement instead spelled + /// the clause again and dropped that deletion, and the transpile was refused + #[test] + fn a_suite_on_the_line_of_its_clause_is_re_emitted_as_a_block() { + let out = transpile( + indoc! {" + marker: int = 1 + + def f(s: str, t: typeof marker) -> int raises ValueError: return len(s) + t + "}, + &Config { + soundness: crate::SoundnessPositions::all(), + ..Config::test_default() + }, + ) + .unwrap(); + assert!( + out.contains(concat!( + "def f(s: str, t: TypeOf[marker]) -> int: \n", + " _soundness_check(s, str)\n", + " _soundness_check(t, int)\n", + " return len(s) + t\n", + )), + "got:\n{out}" + ); + } + + /// the block a suite of its clause's line is re-emitted as is indented below the + /// line the clause starts on, whatever that line is indented by + #[test] + fn a_re_emitted_inline_suite_is_indented_below_its_clause() { + let out = transpile( + indoc! {" + marker: int = 1 + + class A: + def f(self, s: str, t: typeof marker) -> int: return len(s) + t + "}, + &Config { + soundness: crate::SoundnessPositions::all(), + ..Config::test_default() + }, + ) + .unwrap(); + assert!( + out.contains(concat!( + "class A:\n", + " def f(self, s: str, t: TypeOf[marker]) -> int: \n", + " _soundness_check(s, str)\n", + " _soundness_check(t, int)\n", + " return len(s) + t\n", + )), + "got:\n{out}" + ); + } + + /// a rewritten statement below the first column keeps the indentation of the lines + /// it passes through, whatever the source indents by + #[test] + fn a_rewritten_method_keeps_its_indentation() { + let out = check(indoc! {" + class A: + def f(self) -> int: + sentinel B + if True: + return 1 + return 2 + "}); + assert!( + out.contains(indoc! {" + class A: + def f(self) -> int: + B = Sentinel(\"B\") + if True: + return 1 + return 2 + "}), + "got:\n{out}" + ); + } + + /// a file may indent one block by four spaces and another by two. a `def` whose body + /// a pass rewrote is re-emitted as its own source is indented, where printing it with + /// the four spaces the file starts with put its statements out of step with the + /// ones it passes through, and the output did not parse + #[test] + fn a_rewritten_body_is_indented_as_its_own_source_is() { + let out = check(indoc! {" + class A: + def g(self) -> int: + return 1 + + + def f() -> int: + sentinel B + if True: + return 1 + return 2 + "}); + assert!( + out.contains(indoc! {" + def f() -> int: + B = Sentinel(\"B\") + if True: + return 1 + return 2 + "}), + "got:\n{out}" + ); + } + + /// a statement printed whole opens its blocks by the step its own source indents + /// them by, so a block it prints holds the statements it passes through at the + /// depth they are written at. printed by the four spaces given for the file, the + /// `if` it adds would be deeper than the `y = 1` beside it + #[test] + fn a_block_a_rewritten_statement_prints_is_indented_as_its_source_is() { + use ruff_python_ast::{Expr, ExprUnaryOp, Stmt, UnaryOp}; + use ruff_python_codegen::Indentation; + use ruff_text_size::Ranged; + + use super::{Rerendering, forget_stmt_ranges}; + use crate::transforms::ast_driver::Fragment; + + let source = "class A:\n def g(self):\n return 1\n\nif x:\n y = 1\n"; + let parsed = ruff_python_parser::parse_module(source).unwrap(); + let comments = ruff_python_trivia::CommentRanges::from(parsed.tokens()); + let parsed = parsed.into_syntax(); + let original = &parsed.body[1]; + let mut rewritten = original.clone(); + let Stmt::If(conditional) = &mut rewritten else { + panic!("an `if`") + }; + let test = conditional.test.clone(); + *conditional.test = Expr::UnaryOp(ExprUnaryOp { + node_index: ruff_python_ast::AtomicNodeIndex::NONE, + range: ruff_text_size::TextRange::default(), + op: UnaryOp::Not, + operand: test, + }); + let mut added = ruff_python_parser::parse_module("if y:\n pass\n") + .unwrap() + .into_syntax() + .body + .remove(0); + forget_stmt_ranges(&mut added); + conditional.body.push(added); + let indentation = Indentation::new(" ".to_string()); + let edits = Rerendering::new(source, &comments, &[], &indentation).edits( + original, + &rewritten, + original.range(), + ); + let spelled: Vec = edits + .iter() + .map(|(_, fragments)| { + fragments + .iter() + .map(|fragment| match fragment { + Fragment::Lit(text) => text.clone(), + Fragment::Src(range) => source[*range].to_string(), + }) + .collect() + }) + .collect(); + assert_eq!(spelled, ["if not x:\n y = 1\n if y:\n pass"]); + } + + /// a placeholder binds tighter than any operator, so whether its source needs + /// parentheses is read off how the printer spells the operand in its place: `a + b` + /// moved from a call's argument to under a `*` needs them + #[test] + fn a_moved_operand_is_parenthesized_where_it_has_to_be() { + use ruff_python_ast::{Expr, ExprBinOp, ExprCall, Operator, Stmt}; + use ruff_python_codegen::Indentation; + use ruff_text_size::{Ranged, TextRange}; + + use super::Rerendering; + use crate::transforms::ast_driver::Fragment; + + let source = "x = f(a + b, c)\n"; + let parsed = ruff_python_parser::parse_module(source).unwrap(); + let comments = ruff_python_trivia::CommentRanges::from(parsed.tokens()); + let parsed = parsed.into_syntax(); + let original = &parsed.body[0]; + let mut rewritten = original.clone(); + let Stmt::Assign(assign) = &mut rewritten else { + panic!("an assignment") + }; + let Expr::Call(ExprCall { arguments, .. }) = assign.value.as_ref() else { + panic!("a call") + }; + let (sum, c) = (arguments.args[0].clone(), arguments.args[1].clone()); + *assign.value = Expr::BinOp(ExprBinOp { + node_index: ruff_python_ast::AtomicNodeIndex::NONE, + range: TextRange::default(), + left: Box::new(sum), + op: Operator::Mult, + right: Box::new(c), + }); + let indentation = Indentation::default(); + let edits = Rerendering::new(source, &comments, &[], &indentation).edits( + original, + &rewritten, + original.range(), + ); + let spelled: Vec = edits + .iter() + .map(|(_, fragments)| { + fragments + .iter() + .map(|fragment| match fragment { + Fragment::Lit(text) => text.clone(), + Fragment::Src(range) => source[*range].to_string(), + }) + .collect() + }) + .collect(); + assert_eq!(spelled, ["x = (a + b) * c"]); + } +} diff --git a/crates/by_transforms/src/transforms/soundness.rs b/crates/by_transforms/src/transforms/soundness.rs index 0646bcdd16..ca780537f6 100644 --- a/crates/by_transforms/src/transforms/soundness.rs +++ b/crates/by_transforms/src/transforms/soundness.rs @@ -46,9 +46,11 @@ //! wrapped expression (coalesce, force-unwrap, generic-call stripping) are //! materialized inside the check's passthrough span, and nested checks //! (`t()[0]`, an argument inside a checked call) compose by template -//! recursion. Statements an AST-mutation pass re-rendered (casts, coalesce -//! chains, `typeof`) are skipped wholesale — their source ranges no longer -//! align with the output. +//! recursion. In a top-level statement an AST-mutation pass rewrote (a +//! coalesce chain, `typeof`, a repeated `_` parameter) the checks are written +//! into the syntax tree instead — see [`place_in_rerendered`]: a check made at a +//! node the pass replaced has no source left to land on as a text edit, and +//! written into the tree it is either placed or reported. //! //! The `returns`/`assignments`/`arguments` gates share a "value is unhelpful" //! rule — the value is a plain `Any`, or a gated projection whose own type is @@ -62,19 +64,23 @@ //! defined later in the module can raise `NameError` if the checked line runs //! at import time before the class body. +use std::cell::Cell; use std::collections::HashMap; -use std::fmt::Write as _; +use ruff_python_ast::name::Name; +use ruff_python_ast::visitor::transformer::{self, Transformer}; use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; use ruff_python_ast::{ - Comprehension, Expr, ExprCall, HasNodeIndex, Parameter, Stmt, StmtFunctionDef, UnaryOp, + AtomicNodeIndex, Comprehension, Expr, ExprCall, ExprName, HasNodeIndex, Parameter, Stmt, + StmtFunctionDef, UnaryOp, }; use ruff_text_size::{Ranged, TextRange, TextSize}; use ty_python_semantic::types::soundness::CheckTarget; use super::ast_driver::{Fragment, PassContext, TypeAwarePass}; use super::parametric_is::variance_tuple; -use super::source_util::{line_indent, line_start}; +use super::repeated_underscore::WrittenNames; +use super::source_util::{PrologueStatement, body_prologue}; use crate::Config; use crate::config::SoundnessPositions; use crate::type_info::{SoundnessCheck, TypeInfo}; @@ -108,7 +114,9 @@ enum Site<'ast> { /// `function`'s own parameters are checked where its body begins Parameters { function: &'ast StmtFunctionDef, - guards: Vec<(&'ast str, SoundnessCheck)>, + /// each checked parameter by the name python binds it to, which for a repeated + /// `_` is not the one the source spells + guards: Vec<(Name, SoundnessCheck)>, }, } @@ -117,28 +125,30 @@ enum Site<'ast> { struct Soundness<'a, 'ast> { types: &'a dyn TypeInfo, positions: SoundnessPositions, + /// the module as written, which a repeated `_` parameter's name is numbered around + written: WrittenNames<'a>, sites: Vec>, /// gated expressions already covered by a wrap on an enclosing `!` /// force-unwrap — wrapping them directly would splice the check's second /// argument into `_force_unwrap`'s call parens consumed: Vec, - /// spans whose *source text* is user-visible, so nothing may be inserted into - /// them: an f-string's `{expr=}` renders the text between the braces, and a - /// check wrapped there would print itself - verbatim: Vec, /// declared-return-type check plans of the enclosing functions (innermost /// last); `None` when a function has no annotation or an uncheckable one return_targets: Vec>, } impl<'a, 'ast> Soundness<'a, 'ast> { - fn new(types: &'a dyn TypeInfo, positions: SoundnessPositions) -> Self { + fn new( + types: &'a dyn TypeInfo, + positions: SoundnessPositions, + written: WrittenNames<'a>, + ) -> Self { Self { types, positions, + written, sites: Vec::new(), consumed: Vec::new(), - verbatim: Vec::new(), return_targets: Vec::new(), } } @@ -279,7 +289,10 @@ impl<'a, 'ast> Soundness<'a, 'ast> { { let parameter = &pwd.parameter; if let Some(plan) = self.parameter_plan(parameter) { - guards.push((parameter.name.as_str(), plan)); + guards.push(( + crate::python_parameter_name(params, parameter, self.written), + plan, + )); } } if !guards.is_empty() { @@ -363,25 +376,6 @@ impl<'ast> Visitor<'ast> for Soundness<'_, 'ast> { } fn visit_expr(&mut self, expr: &'ast Expr) { - // `f"{x=}"` renders the source between the braces, so a check inserted - // there would show up in the output. nothing inside one is wrapped - if let Expr::FString(fstring) = expr { - for element in fstring.value.elements() { - if let ruff_python_ast::InterpolatedStringElement::Interpolation(part) = element - && part.debug_text.is_some() - { - self.verbatim.push(part.expression.range()); - } - } - } - if self - .verbatim - .iter() - .any(|span| span.contains_range(expr.range())) - { - walk_expr(self, expr); - return; - } if let Expr::UnaryOp(unary) = expr && unary.op == UnaryOp::Force { @@ -465,8 +459,8 @@ impl SoundnessSites { self.elements.get(&SiteKey::of(iterable)?) } - /// the checks made on `function`'s own parameters where its body begins, by - /// parameter name, in the order the parameters are written + /// the checks made on `function`'s own parameters where its body begins, by the + /// name python binds each parameter to, in the order the parameters are written pub fn parameters(&self, function: &StmtFunctionDef) -> &[(String, SoundnessCheck)] { SiteKey::of(function) .and_then(|key| self.parameters.get(&key)) @@ -474,9 +468,11 @@ impl SoundnessSites { } } -/// decide every soundness check `suite` makes under `positions` +/// decide every soundness check `suite` makes under `positions`. `written` is the +/// module `suite` is parsed from, as it was written pub fn soundness_sites( model: &ty_python_semantic::SemanticModel<'_>, + written: WrittenNames, suite: &[Stmt], positions: SoundnessPositions, ) -> SoundnessSites { @@ -484,7 +480,7 @@ pub fn soundness_sites( if !positions.any() { return sites; } - let mut walker = Soundness::new(model, positions); + let mut walker = Soundness::new(model, positions, written); walker.visit_body(suite); for site in walker.sites { match site { @@ -502,7 +498,7 @@ pub fn soundness_sites( if let Some(key) = SiteKey::of(function) { let guards = guards .into_iter() - .map(|(name, plan)| (name.to_owned(), plan)) + .map(|(name, plan)| (name.to_string(), plan)) .collect(); sites.parameters.insert(key, guards); } @@ -512,6 +508,17 @@ pub fn soundness_sites( sites } +/// One entry guard written at the top of a body. The guard is a call on a line +/// of its own and never continues onto a second one, so it re-establishes no +/// indentation +struct EntryGuard(String); + +impl PrologueStatement for EntryGuard { + fn push(&self, frags: &mut Vec, _indent: &str) { + frags.push(Fragment::Lit(self.0.clone())); + } +} + /// what the checks a pass decided on are written as #[derive(Default)] #[expect( @@ -519,9 +526,14 @@ pub fn soundness_sites( reason = "independent runtime-helper usage flags, not a state machine" )] struct Rendered { - edits: Vec<(TextRange, Vec)>, + /// each value passed through a call: the span it is at, the text opening the call + /// and the text closing it + wraps: Vec<(TextRange, String, String)>, /// the entry-guard suites, anchored at the body statement they precede guards: Vec<(TextSize, Vec)>, + /// functions whose body holds nothing the source wrote, so a guard has no + /// position to anchor to + unanchored: Vec, used_iter: bool, used_aiter: bool, used_iter_p: bool, @@ -535,14 +547,8 @@ impl Rendered { /// wrap `source[range]` in `helper(, )`. `trailing` /// carries its own leading `, ` (e.g. `", str"` or `", A[int], (0,)"`) fn wrap_call(&mut self, range: TextRange, helper: &str, trailing: &str) { - self.edits.push(( - range, - vec![ - Fragment::Lit(format!("{helper}(")), - Fragment::Src(range), - Fragment::Lit(format!("{trailing})")), - ], - )); + self.wraps + .push((range, format!("{helper}("), format!("{trailing})"))); } /// wrap `range` in the scalar check (`_soundness_check` / @@ -592,6 +598,14 @@ impl Rendered { self.wrap_call(iterable.range(), helper, &trailing); } + /// the guard statements that validate each of `checks`, in order + fn guard_stmts(&mut self, checks: &[(Name, SoundnessCheck)]) -> Vec { + checks + .iter() + .map(|(name, plan)| self.guard_stmt(name, plan)) + .collect() + } + /// the guard statement that validates parameter `name` against `plan` fn guard_stmt(&mut self, name: &str, plan: &SoundnessCheck) -> String { match plan { @@ -611,61 +625,38 @@ impl Rendered { &mut self, source: &str, func: &StmtFunctionDef, - checks: &[(&str, SoundnessCheck)], + checks: &[(Name, SoundnessCheck)], ) { - let guards: Vec = checks - .iter() - .map(|(name, plan)| self.guard_stmt(name, plan)) + let guards: Vec = self + .guard_stmts(checks) + .into_iter() + .map(EntryGuard) .collect(); - - // insert after a leading docstring (which must stay the first - // statement); mirrors `mutable_defaults`' body-prologue insertion - let docstring_count = usize::from(matches!( - func.body.first(), - Some(Stmt::Expr(e)) if matches!(e.value.as_ref(), Expr::StringLiteral(_)) - )); - let mut text = String::new(); - let insert_at = if let Some(stmt) = func.body.get(docstring_count) { - let at = stmt.range().start(); - let prefix = &source[usize::from(line_start(source, at))..usize::from(at)]; - if prefix.trim().is_empty() { - // multi-line body: each guard sits at the body indent and - // re-establishes it for the statement that follows - for guard in &guards { - let _ = write!(text, "{guard}\n{prefix}"); - } - } else { - // single-line body (`def f(a: A[int]): ...`) — break the body - // onto its own indented line after the guards - let base = format!("{} ", line_indent(source, func.range().start())); - for guard in &guards { - let _ = write!(text, "\n{base}{guard}"); - } - let _ = write!(text, "\n{base}"); - } - at - } else { - // docstring-only body: append the guards after it - let base = format!("{} ", line_indent(source, func.range().start())); - for guard in &guards { - let _ = write!(text, "\n{base}{guard}"); - } - func.body[docstring_count - 1].range().end() - }; - self.guards.push((insert_at, vec![Fragment::Lit(text)])); + // the same anchor `mutable_defaults` and `init_method` hang their own body + // insertions off, so the four agree about where the top of a body is + match body_prologue(source, func, &guards) { + Some(anchored) => self.guards.extend(anchored), + // nothing in the body came from the source. a check with nowhere to go + // is an error rather than a check dropped: a native build of the module + // makes it from the same site, and the two builds would answer + // differently + None => self.unanchored.push(func.name.to_string()), + } } } pub(crate) struct SoundnessPass<'src> { positions: SoundnessPositions, source: &'src str, + written: WrittenNames<'src>, } impl<'src> SoundnessPass<'src> { - pub(crate) fn new(source: &'src str, config: &Config) -> Self { + pub(crate) fn new(source: &'src str, written: WrittenNames<'src>, config: &Config) -> Self { Self { positions: config.soundness, source, + written, } } } @@ -680,30 +671,70 @@ impl TypeAwarePass for SoundnessPass<'_> { if !self.positions.any() { return; } - let mut walker = Soundness::new(types, self.positions); + let mut inner = Rendered::default(); + let mut rerendered = Vec::new(); for (idx, stmt) in stmts.iter().enumerate() { - // an AST-mutation pass re-rendered this statement; edits into its - // original source range would be dropped and flagged as leaks + let mut walker = Soundness::new(types, self.positions, self.written); + walker.visit_stmt(stmt); + // an AST-mutation pass rewrote this statement, and an edit at a node it + // replaced would be dropped: its checks go into the tree it is printed from if ctx.changed.contains(&idx) { + let mut rendered = Rendered::default(); + let mut checks = Vec::new(); + for site in &walker.sites { + match site { + Site::Value { expr, plan } => rendered.wrap_check(expr.range(), plan), + Site::Elements { + iterable, + is_async, + plan, + } => rendered.wrap_iteration(iterable, *is_async, plan), + Site::Parameters { function, guards } => { + checks.push(RerenderedCheck::Guards { + function: function.range(), + statements: rendered.guard_stmts(guards), + }); + } + } + } + checks.extend(rendered.wraps.drain(..).map(|(range, before, after)| { + RerenderedCheck::Wrap { + range, + before, + after, + } + })); + inner.used_iter |= rendered.used_iter; + inner.used_aiter |= rendered.used_aiter; + inner.used_iter_p |= rendered.used_iter_p; + inner.used_aiter_p |= rendered.used_aiter_p; + inner.used_parametric |= rendered.used_parametric; + if !checks.is_empty() { + rerendered.push((idx, checks)); + } continue; } - walker.visit_stmt(stmt); - } - let mut inner = Rendered::default(); - for site in &walker.sites { - match site { - Site::Value { expr, plan } => inner.wrap_check(expr.range(), plan), - Site::Elements { - iterable, - is_async, - plan, - } => inner.wrap_iteration(iterable, *is_async, plan), - Site::Parameters { function, guards } => { - inner.insert_param_guards(self.source, function, guards); + for site in &walker.sites { + match site { + Site::Value { expr, plan } => inner.wrap_check(expr.range(), plan), + Site::Elements { + iterable, + is_async, + plan, + } => inner.wrap_iteration(iterable, *is_async, plan), + Site::Parameters { function, guards } => { + inner.insert_param_guards(self.source, function, guards); + } } } } - if inner.edits.is_empty() && inner.guards.is_empty() { + if let Some(name) = inner.unanchored.first() { + ctx.errors.push(format!( + "`{name}` has a body nothing in the source anchors, so an entry guard has nowhere to go" + )); + return; + } + if inner.wraps.is_empty() && inner.guards.is_empty() && rerendered.is_empty() { return; } ctx.runtime.insert(crate::runtime::SOUNDNESS_CHECK); @@ -725,8 +756,172 @@ impl TypeAwarePass for SoundnessPass<'_> { if inner.used_aiter_p { ctx.runtime.insert(crate::runtime::SOUNDNESS_AITER_P); } - ctx.template_edits.extend(inner.edits); + ctx.template_edits + .extend(inner.wraps.into_iter().map(|(range, before, after)| { + ( + range, + vec![ + Fragment::Lit(before), + Fragment::Src(range), + Fragment::Lit(after), + ], + ) + })); ctx.statement_inserts.extend(inner.guards); + ctx.rerendered_checks.extend(rerendered); + } +} + +/// a check to write into a top-level statement that is printed from its syntax tree, +/// found by the source range of the node it is made at +pub(crate) enum RerenderedCheck { + /// the value the expression at `range` produces, passed through the call + /// `before` opens and `after` closes + Wrap { + range: TextRange, + before: String, + after: String, + }, + /// the entry guards of the function at `function` + Guards { + function: TextRange, + statements: Vec, + }, +} + +/// write `checks` into `stmt`, a top-level statement an AST-mutation pass rewrote +/// +/// a mutation leaves every node it did not replace at its source range, and a check +/// is made at a node the source holds, so each is found where the text edit would +/// have landed. a check with nowhere to go is an error rather than a check dropped: +/// a native build of the module makes it from the same sites, and the two builds +/// would answer differently +pub(crate) fn place_in_rerendered( + stmt: &mut Stmt, + checks: Vec, +) -> Result<(), String> { + let mut wraps = Vec::new(); + let mut guards = Vec::new(); + for check in checks { + match check { + RerenderedCheck::Wrap { + range, + before, + after, + } => { + let placeholder = "__by_soundness_value__"; + let Ok(parsed) = + ruff_python_parser::parse_expression(&format!("{before}{placeholder}{after}")) + else { + return Err(format!( + "a soundness check could not be written: `{before}…{after}`" + )); + }; + let mut parsed = *parsed.into_syntax().body; + super::rerender::forget_expr_ranges(&mut parsed); + let Expr::Call(call) = parsed else { + return Err(format!( + "a soundness check is not a call: `{before}…{after}`" + )); + }; + wraps.push((range, call, Cell::new(false))); + } + RerenderedCheck::Guards { + function, + statements, + } => { + let Ok(parsed) = ruff_python_parser::parse_module(&statements.join("\n")) else { + return Err(format!( + "a parameter guard could not be written: `{}`", + statements.join("; ") + )); + }; + let mut statements: Vec = parsed.into_syntax().body.into_iter().collect(); + statements + .iter_mut() + .for_each(super::rerender::forget_stmt_ranges); + guards.push((function, statements, Cell::new(false))); + } + } + } + let placing = Placing { wraps, guards }; + placing.visit_stmt(stmt); + let unplaced = placing + .wraps + .iter() + .filter(|(_, _, placed)| !placed.get()) + .map(|(range, _, _)| *range) + .chain( + placing + .guards + .iter() + .filter(|(_, _, placed)| !placed.get()) + .map(|(range, _, _)| *range), + ) + .next(); + match unplaced { + Some(range) => Err(format!( + "a soundness check at {range:?} could not be written: a lowering rewrote the \ + expression it checks" + )), + None => Ok(()), + } +} + +struct Placing { + wraps: Vec<(TextRange, ExprCall, Cell)>, + guards: Vec<(TextRange, Vec, Cell)>, +} + +impl Transformer for Placing { + fn visit_stmt(&self, stmt: &mut Stmt) { + transformer::walk_stmt(self, stmt); + let Stmt::FunctionDef(function) = stmt else { + return; + }; + let Some((_, statements, placed)) = self + .guards + .iter() + .find(|(range, _, _)| *range == function.range) + else { + return; + }; + // after a docstring, which has to stay the first statement + let at = super::source_util::body_prologue_index(function); + function.body.splice(at..at, statements.iter().cloned()); + placed.set(true); + } + + fn visit_annotation(&self, _expr: &mut Expr) {} + + fn visit_expr(&self, expr: &mut Expr) { + // the children first, so a check made inside this value is inside its check + transformer::walk_expr(self, expr); + // a node a lowering left twice — `a ?? b` repeats a side-effect-free `a` — + // is checked wherever it is evaluated + let at = expr.range(); + for (range, call, placed) in &self.wraps { + if at != *range { + continue; + } + let value = std::mem::replace( + expr, + Expr::Name(ExprName { + node_index: AtomicNodeIndex::NONE, + range: TextRange::default(), + id: ruff_python_ast::name::Name::new_static("None"), + ctx: ruff_python_ast::ExprContext::Load, + }), + ); + // a second check made at the same node is written around this one, as the + // text edits compose + let mut call = call.clone(); + if let Some(first) = call.arguments.args.first_mut() { + *first = value; + } + *expr = Expr::Call(call); + placed.set(true); + } } } @@ -816,13 +1011,18 @@ mod tests { ); } + /// a `=` field is checked like any other expression. the check used to be skipped + /// there, because `f"{x=}"` prints the source between the braces and the wrap would + /// print itself — but the field is now taken apart around the value + /// ([`super::debug_field`]), so the author's own text is what is printed and the value + /// is free to be wrapped #[test] - fn a_debug_interpolation_is_left_verbatim() { - // `f"{x=}"` renders the source between the braces, so a check inserted - // there would print itself + fn a_debug_interpolation_is_checked_beside_the_text_it_prints() { let out = check("def f(d: dict[str, int]):\n return f\"{d.get('k')=}\"\n"); - assert!(!out.contains("_soundness_check"), "got:\n{out}"); - assert!(out.contains("f\"{d.get('k')=}\""), "got:\n{out}"); + assert!( + out.contains("f\"d.get('k')={(_soundness_check(d.get('k'), (int, type(None))))!r}\""), + "got:\n{out}" + ); } #[test] @@ -1355,6 +1555,21 @@ mod tests { ); } + /// an `init(…)` shorthand's body opens with the attribute declarations the parser + /// synthesized from its parameters, whose ranges point back into the signature. + /// anchored at one of those, the guard was spliced into the parameter list + #[test] + fn entry_guard_in_an_init_shorthand_sits_in_the_body() { + let out = check_with( + "class A:\n init(let s: str):\n print(s)\n", + params_only(), + ); + assert!( + out.contains(" def __init__(self, s: str):\n _soundness_check(s, str)\n self.s: str = s\n print(s)\n"), + "got:\n{out}" + ); + } + #[test] fn entry_guard_follows_docstring() { // the docstring must stay the first statement; the guard goes after it @@ -1426,12 +1641,87 @@ mod tests { ); } + /// a repeated `_` is numbered in the python its `def` lowers to, so each guard checks + /// the parameter python binds: spelled by its source name, the second `_` would be + /// checked as the first, and an `int` refused where a `str` was declared + #[test] + fn a_repeated_underscore_is_guarded_by_its_numbered_name() { + let out = check_with( + "def f(_: int, _: str) -> int:\n return 1\n", + params_only(), + ); + assert!(out.contains("_soundness_check(_, int)"), "got:\n{out}"); + assert!(out.contains("_soundness_check(_2, str)"), "got:\n{out}"); + } + #[test] fn all_includes_parameters() { let out = check_with("def f(s: str): ...\n", crate::SoundnessPositions::all()); assert!(out.contains("_soundness_check(s, str)"), "got:\n{out}"); } + // ── statements a syntax-tree pass prints again ────────────────────── + + /// `typeof` is lowered by rewriting the syntax tree, and a top-level statement + /// holding one is printed again from that tree rather than edited in place. the + /// checks everywhere else in the statement are made all the same — a native build + /// of the module makes them, from the same sites + #[test] + fn a_statement_printed_from_its_syntax_tree_keeps_its_checks() { + let out = check( + "from typing import Any\n\ndef f(x: Any, a: list[str], n: int) -> int:\n b: int = x\n for s in a:\n print(s)\n m: typeof(n) = n\n return m\n", + ); + assert!( + out.contains("TypeOf[n]"), + "`typeof` no longer rewrites the tree, got:\n{out}" + ); + assert!( + out.contains("b: int = _soundness_check(x, int)"), + "got:\n{out}" + ); + assert!( + out.contains("for s in _soundness_iter(a, str):"), + "got:\n{out}" + ); + } + + #[test] + fn a_function_printed_from_its_syntax_tree_keeps_its_entry_guards() { + // a repeated `_` parameter is renamed on the syntax tree + let out = check_with( + "def f(_: int, _: int, v: str) -> str:\n return v\n", + params_only(), + ); + assert!( + out.contains("_2"), + "the parameters are no longer renamed, got:\n{out}" + ); + assert!(out.contains("_soundness_check(v, str)"), "got:\n{out}"); + } + + /// and it keeps its docstring. the parser writes an `init(…)` shorthand's attribute + /// declarations ahead of everything the source wrote, so the docstring is not the + /// body's first statement in the tree — read as though it were, the guard landed + /// above the string and the method lost its `__doc__` + #[test] + fn an_entry_guard_follows_the_docstring_of_an_init_shorthand() { + // `typeof` rewrites the syntax tree, so the whole class is printed from it + let out = check_with( + "class C:\n init(let x: int):\n \"\"\"doc\"\"\"\n print(x)\n\n def m(self):\n y: typeof(self.x) = 1\n print(y)\n", + params_only(), + ); + assert!( + out.contains("TypeOf[self.x]"), + "`typeof` no longer rewrites the tree, got:\n{out}" + ); + assert!( + out.contains( + " \"\"\"doc\"\"\"\n _soundness_check(x, int)\n self.x: int = x\n" + ), + "got:\n{out}" + ); + } + /// a stub is never run, so it produces no value to check #[test] fn a_stub_gets_no_checks() { diff --git a/crates/by_transforms/src/transforms/source_util.rs b/crates/by_transforms/src/transforms/source_util.rs index 8b52ed99df..341945a68a 100644 --- a/crates/by_transforms/src/transforms/source_util.rs +++ b/crates/by_transforms/src/transforms/source_util.rs @@ -2,6 +2,7 @@ use std::fmt::Display; use std::fmt::Write as _; use super::ast_driver::Fragment; +use super::repeated_underscore::WrittenNames; use ruff_python_ast::helpers::consumed_keywords; use ruff_python_ast::visitor::{Visitor, walk_stmt}; @@ -69,6 +70,42 @@ pub(crate) fn preamble_offset(text: &str) -> usize { at } +/// Render `value` as a python string literal. +/// +/// A conservative escaper that always emits a double-quoted form, so the result +/// re-lexes as one token whatever the text holds. +pub(crate) fn string_repr(value: &str) -> String { + let mut out = String::with_capacity(value.len() + 2); + out.push('"'); + for ch in value.chars() { + match ch { + '\\' => out.push_str("\\\\"), + '"' => out.push_str("\\\""), + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + '\t' => out.push_str("\\t"), + c => out.push(c), + } + } + out.push('"'); + out +} + +/// `value` as a string literal node, for a lowering that writes into the syntax +/// tree rather than into the source. +pub(crate) fn string_literal(value: &str) -> ruff_python_ast::ExprStringLiteral { + ruff_python_ast::ExprStringLiteral { + node_index: ruff_python_ast::AtomicNodeIndex::NONE, + range: TextRange::default(), + value: ruff_python_ast::StringLiteralValue::single(ruff_python_ast::StringLiteral { + node_index: ruff_python_ast::AtomicNodeIndex::NONE, + range: TextRange::default(), + value: value.into(), + flags: ruff_python_ast::StringLiteralFlags::empty(), + }), + } +} + /// the offset just past the end of the line `end` falls on fn line_end(text: &str, end: usize) -> usize { text[end..] @@ -76,8 +113,15 @@ fn line_end(text: &str, end: usize) -> usize { .map_or(text.len(), |newline| end + newline + 1) } -pub(crate) fn temporary_name(kind: &str, index: impl Display) -> String { - format!("__by_{kind}_{index}__") +/// A name for a value the lowering needs to hold on to and the source never named. +/// +/// The spelling is chosen to be one nobody writes, but "nobody writes it" is not something +/// the transpiler gets to assume: a module that does write it has the temporary land on top +/// of its own binding, and a read of that binding inside the function the temporary is +/// written in becomes an `UnboundLocalError`. So the name is taken past whatever the module +/// spells, the same way a numbered `_` parameter is. +pub(crate) fn temporary_name(written: WrittenNames<'_>, kind: &str, index: impl Display) -> String { + written.fresh(&format!("__by_{kind}_{index}__")) } /// Where the text between a statement's own words and its value begins. @@ -477,9 +521,23 @@ pub(crate) trait PrologueStatement { fn push(&self, frags: &mut Vec, indent: &str); } -/// The insertion that writes `statements` at the top of `f`'s body — after a +/// The insertions that write `statements` at the top of `f`'s body — after a /// docstring, before everything else the source wrote there. /// +/// Usually one insertion. A body the source wrote on the clause's line +/// (`def f(a): """what f does"""`, `def f(a): return 1`) needs a second one: +/// nothing indented can follow a statement there, so the body has to be broken +/// onto a line of its own before anything can be written under it. The break is +/// an insertion of its own, ahead of the body, so that two passes writing into +/// the same body still compose — their statements coalesce at the one offset, +/// exactly as they do for a body that already had a line to itself. +/// +/// Whether the body is on the clause's line and where the statements go are two +/// separate questions, because a clause-line body can hold several statements: +/// `def f(a): """doc"""; a.append("x"); return len(a)` breaks at the docstring, +/// which has to stay the body's first statement to stay a docstring, but writes +/// the prologue ahead of the `a.append` that follows it. +/// /// `None` when the body holds nothing the source wrote at all: the `init(…)` /// shorthand generates its whole body, and an offset taken from a statement the /// parser synthesized from a parameter points back into the signature. @@ -487,44 +545,78 @@ pub(crate) fn body_prologue( source: &str, f: &StmtFunctionDef, statements: &[impl PrologueStatement], -) -> Option<(TextSize, Vec)> { +) -> Option)>> { + let header_end = header_end(f); + let body_start = first_body_statement(f) + .and_then(|stmt| body_range(stmt, header_end))? + .start(); + let mut inserts = Vec::new(); + let body_own_line = + &source[usize::from(line_start(source, body_start))..usize::from(body_start)]; + let indent = if body_own_line.trim().is_empty() { + // the insertion lands after the body's own indentation; each statement + // re-establishes it for the line that follows + body_own_line.to_owned() + } else { + let indent = format!("{} ", line_indent(source, f.range().start())); + inserts.push((body_start, vec![Fragment::Lit(format!("\n{indent}"))])); + indent + }; + let mut frags: Vec = Vec::new(); - if let Some(range) = first_source_statement(f).and_then(|s| body_range(s, header_end(f))) { - let insert_at = range.start(); - let prefix = &source[usize::from(line_start(source, insert_at))..usize::from(insert_at)]; - if prefix.trim().is_empty() { - // the insertion lands after the statement's own indentation; each - // statement re-establishes it for the line that follows + match first_source_statement(f).and_then(|stmt| body_range(stmt, header_end)) { + Some(range) => { + let insert_at = range.start(); + // that statement follows a docstring on one line when the break + // above went somewhere else and left text before it, so a line has + // to be opened for the prologue as well + let prefix = + &source[usize::from(line_start(source, insert_at))..usize::from(insert_at)]; + if insert_at != body_start && !prefix.trim().is_empty() { + frags.push(Fragment::Lit(format!("\n{indent}"))); + } for statement in statements { - statement.push(&mut frags, prefix); - frags.push(Fragment::Lit(format!("\n{prefix}"))); + statement.push(&mut frags, &indent); + frags.push(Fragment::Lit(format!("\n{indent}"))); } - } else { - // single-line body (`def f(self): return T`) — break it onto its - // own indented line after the insertion - let indent = format!("{} ", line_indent(source, f.range().start())); + inserts.push((insert_at, frags)); + } + // a docstring is the only thing the source wrote: the insertion follows + // it, because a docstring is only a docstring while it is the body's + // first statement + None => { + let docstring = source_statements(f) + .next() + .filter(|stmt| is_docstring(stmt)) + .and_then(|stmt| body_range(stmt, header_end))?; for statement in statements { frags.push(Fragment::Lit(format!("\n{indent}"))); statement.push(&mut frags, &indent); } - frags.push(Fragment::Lit(format!("\n{indent}"))); + inserts.push((docstring.end(), frags)); } - return Some((insert_at, frags)); } - // a docstring is the only thing the source wrote: the insertion follows it - let docstring = f - .body - .first() - .filter(|stmt| { - matches!(stmt, Stmt::Expr(e) if matches!(e.value.as_ref(), Expr::StringLiteral(_))) - }) - .and_then(|stmt| body_range(stmt, header_end(f)))?; - let indent = format!("{} ", line_indent(source, f.range().start())); - for statement in statements { - frags.push(Fragment::Lit(format!("\n{indent}"))); - statement.push(&mut frags, &indent); - } - Some((docstring.end(), frags)) + Some(inserts) +} + +/// The index in `f`'s body that a statement written at the top of it goes at: +/// past a docstring the source wrote, and at 0 when it wrote none. +/// +/// This is [`body_prologue`]'s counterpart for a statement printed from its +/// syntax tree rather than spliced into the source. There is no offset to +/// anchor to there, only a position in the body — but the question is the same +/// one, and so is the trap: the parser writes an `init(…)` shorthand's +/// attribute declarations *ahead* of everything the source wrote, so asking +/// whether `body[0]` is a docstring answers no for every `init(…)` that +/// declares an attribute, and the statement lands above the docstring, where +/// the string stops being one and the method loses its `__doc__`. +pub(crate) fn body_prologue_index(f: &StmtFunctionDef) -> usize { + let header_end = header_end(f); + f.body + .iter() + .position(|stmt| body_range(stmt, header_end).is_some()) + .filter(|&index| is_docstring(&f.body[index])) + .map_or(0, |index| index + 1) } /// The first statement in `f`'s body that came from the source, skipping a @@ -536,16 +628,31 @@ pub(crate) fn body_prologue( /// /// [`init_method`]: super::init_method fn first_source_statement(f: &StmtFunctionDef) -> Option<&Stmt> { - let header_end = header_end(f); - let docstring_count = if let Some(Stmt::Expr(e)) = f.body.first() { - usize::from(matches!(e.value.as_ref(), Expr::StringLiteral(_))) + let mut statements = source_statements(f); + let first = statements.next()?; + if is_docstring(first) { + statements.next() } else { - 0 - }; + Some(first) + } +} + +/// The statements in `f`'s body the source wrote, in order. +/// +/// A body can hold statements the parser synthesized as well — the `init(…)` +/// shorthand's attribute declarations, which it writes *ahead* of everything the +/// source wrote. Asking whether the body's first statement is a docstring would +/// therefore answer no for every `init(…)` that declares an attribute, and put a +/// generated line above the docstring, where it stops being one. +fn source_statements(f: &StmtFunctionDef) -> impl Iterator { + let header_end = header_end(f); f.body .iter() - .skip(docstring_count) - .find(|s| body_range(s, header_end).is_some()) + .filter(move |s| body_range(s, header_end).is_some()) +} + +fn is_docstring(stmt: &Stmt) -> bool { + matches!(stmt, Stmt::Expr(e) if matches!(e.value.as_ref(), Expr::StringLiteral(_))) } /// The first statement in `f`'s body that came from the source, a docstring diff --git a/crates/by_transforms/src/transforms/statement_expression.rs b/crates/by_transforms/src/transforms/statement_expression.rs index c70c183db5..b8a8a7348e 100644 --- a/crates/by_transforms/src/transforms/statement_expression.rs +++ b/crates/by_transforms/src/transforms/statement_expression.rs @@ -54,6 +54,7 @@ use ruff_text_size::{Ranged, TextRange}; use std::collections::HashSet; use super::ast_driver::{AstPass, Fragment, PassContext}; +use super::repeated_underscore::WrittenNames; use super::source_util::{ line_indent, line_start, parenthesized_value_range, temporary_name, value_separator_start, }; @@ -69,6 +70,16 @@ impl<'src> StatementExpressionPass<'src> { } impl AstPass for StatementExpressionPass<'_> { + fn lowering(&self) -> Option { + Some(super::ast_driver::Lowering::StatementExpression) + } + + /// a `??` whose fallback is a statement (`?? break`, `?? raise …`) is written as the + /// suite that runs it + fn subsumes(&self) -> &'static [super::ast_driver::Lowering] { + &[super::ast_driver::Lowering::NoneCoalesce] + } + fn run(&self, module: &mut ModModule, ctx: &mut PassContext) { let mut misplaced = Vec::new(); let mut lower = Lower { @@ -135,7 +146,7 @@ impl<'ast> Visitor<'ast> for Lower<'_> { impl Lower<'_> { fn next_temp(&mut self) -> String { - let name = temporary_name("stmt_expr", self.counter); + let name = temporary_name(WrittenNames::new(self.source), "stmt_expr", self.counter); self.counter += 1; name } @@ -957,6 +968,28 @@ mod tests { ); } + /// the temporary is written in the function the statement expression stands in, so a + /// module that binds the same name has its own binding turned into a local of that + /// function — and the read the branch was written to make raises `UnboundLocalError` + #[test] + fn a_temporary_goes_past_a_name_the_module_binds() { + let out = check(indoc! {" + __by_stmt_expr_0__ = 7 + + def f(x: int) -> int: + a = if x > 0: + __by_stmt_expr_0__ + else: + 0 + return a + "}); + assert!( + out.contains("__by_stmt_expr_0__2 = __by_stmt_expr_0__"), + "got:\n{out}" + ); + assert!(out.contains("a = __by_stmt_expr_0__2"), "got:\n{out}"); + } + #[test] fn comments_inside_a_branch_survive() { let out = check(indoc! {" diff --git a/crates/by_transforms/src/transforms/static_resource.rs b/crates/by_transforms/src/transforms/static_resource.rs index 2f555f7cbf..31095945b0 100644 --- a/crates/by_transforms/src/transforms/static_resource.rs +++ b/crates/by_transforms/src/transforms/static_resource.rs @@ -45,7 +45,6 @@ use ruff_python_ast::visitor::{Visitor, walk_stmt}; use ruff_python_ast::{Alias, ModModule, Stmt, StmtImport}; use ruff_text_size::{Ranged, TextRange, TextSize}; -use thin_vec::ThinVec; use super::ast_driver::{AstPass, PassContext}; use crate::type_info::TypeInfo; @@ -160,17 +159,12 @@ fn indented(source: &str, indent: &str) -> String { /// Replaces each static resource import with the document it names. /// -/// The import goes from the source text *and* from the AST, because either one -/// alone leaves the other standing. A statement no pass mutated keeps its source -/// bytes, and ignores the AST; a statement some pass did mutate is re-rendered -/// from the AST, and the text edit inside it is dropped. An `import` statement -/// that survived to the output is not python at all — a path is not a module -/// name — so phase 3 rejects the whole file. -/// -/// The AST rewrite deliberately does not declare the statement changed. Saying -/// so would re-render every enclosing `def` through the code generator, losing -/// the comments and the formatting of everything else in it; leaving it undeclared -/// means the rewrite matters only when something else already forced a re-render. +/// The import is replaced in the source text alone. A statement an AST pass +/// rewrote passes the source of every statement it kept through, with the edits +/// inside applied, so the import is replaced there as well. Replacing it in the +/// syntax tree too would make the statement around it one a pass rewrote, and the +/// edit a second lowering of what the tree already says — which is refused, since +/// nothing tells it apart from an edit the rewrite lost. pub(crate) struct StaticResource { lowerings: ResourceLowerings, source: String, @@ -183,82 +177,10 @@ impl StaticResource { source: source.to_string(), } } - - fn rendering_for(&self, range: TextRange) -> Option<&str> { - self.lowerings - .replacements - .iter() - .find_map(|(replaced, rendering)| (*replaced == range).then_some(rendering.as_str())) - } - - /// Replace every lowered import in `body` with the statements it renders to. - fn rewrite_body(&self, body: &mut ThinVec) { - let mut index = 0; - while index < body.len() { - let rendered = match &body[index] { - Stmt::Import(import) if import.names.iter().any(|alias| alias.is_resource) => { - self.rendering_for(import.range()).and_then(|rendering| { - let parsed = ruff_python_parser::parse_module(rendering).ok()?; - Some(parsed.into_syntax().body) - }) - } - _ => None, - }; - - if let Some(rendered) = rendered { - let count = rendered.len(); - body.splice(index..=index, rendered); - index += count; - continue; - } - - self.rewrite_within(&mut body[index]); - index += 1; - } - } - - /// Rewrite the bodies `stmt` holds, if it holds any. - fn rewrite_within(&self, stmt: &mut Stmt) { - match stmt { - Stmt::FunctionDef(node) => self.rewrite_body(&mut node.body), - Stmt::ClassDef(node) => self.rewrite_body(&mut node.body), - Stmt::For(node) => { - self.rewrite_body(&mut node.body); - self.rewrite_body(&mut node.orelse); - } - Stmt::While(node) => { - self.rewrite_body(&mut node.body); - self.rewrite_body(&mut node.orelse); - } - Stmt::If(node) => { - self.rewrite_body(&mut node.body); - for clause in &mut node.elif_else_clauses { - self.rewrite_body(&mut clause.body); - } - } - Stmt::With(node) => self.rewrite_body(&mut node.body), - Stmt::Match(node) => { - for case in &mut node.cases { - self.rewrite_body(&mut case.body); - } - } - Stmt::Try(node) => { - self.rewrite_body(&mut node.body); - for handler in &mut node.handlers { - let ruff_python_ast::ExceptHandler::ExceptHandler(handler) = handler; - self.rewrite_body(&mut handler.body); - } - self.rewrite_body(&mut node.orelse); - self.rewrite_body(&mut node.finalbody); - } - // every other statement holds expressions, and an import is not one - _ => {} - } - } } impl AstPass for StaticResource { - fn run(&self, module: &mut ModModule, ctx: &mut PassContext) { + fn run(&self, _module: &mut ModModule, ctx: &mut PassContext) { for error in &self.lowerings.errors { ctx.errors.push(error.clone()); } @@ -277,8 +199,6 @@ impl AstPass for StaticResource { .to_string(); ctx.text_edits.push((*range, replacement)); } - - self.rewrite_body(&mut module.body); } } @@ -341,10 +261,9 @@ mod tests { ); } - /// an ast-mutating pass re-renders the whole top-level statement it touched, - /// through the code generator rather than from source. a resource import - /// inside one is re-spelled by the generator, and a path written as a name - /// is not python + /// numbering a repeated `_` rewrites the `def`, and the import in its body is + /// passed through as source with the replacement applied. printed from the + /// syntax tree instead, a path written as a name is not python #[test] fn an_import_inside_a_statement_another_pass_rewrites_survives() { let output = transpile(&[ @@ -356,14 +275,13 @@ mod tests { ]) .expect("transpile should succeed"); - assert!(output.contains("def f(_, _2):"), "{output}"); + assert!(output.contains("def f(_, _2, /):"), "{output}"); assert!(output.contains("class config:"), "{output}"); assert!(!output.contains("import data/config.json"), "{output}"); } - /// the statements spliced into the AST are parsed from the rendering and - /// carry its ranges, which name nothing in this file. a pass reading one as - /// a range in the source slices out of bounds + /// a rewritten `def` holding a resource import composes with the passes that + /// rewrite the statements around it #[test] fn a_rewritten_statement_does_not_poison_the_passes_around_it() { let output = transpile(&[ diff --git a/crates/by_transforms/src/transforms/string_tag.rs b/crates/by_transforms/src/transforms/string_tag.rs index 1c8d952b28..66627f0918 100644 --- a/crates/by_transforms/src/transforms/string_tag.rs +++ b/crates/by_transforms/src/transforms/string_tag.rs @@ -12,10 +12,11 @@ //! ``` //! //! On Python 3.14+, `Template` is `string.templatelib.Template` and `t"..."` -//! is native, so the call lowers to `tag(t"...")` with two narrow edits (a -//! `(t` insertion before the quote and a `)` after the close) that keep the -//! literal's source verbatim — any sibling lowering inside an interpolation -//! still applies. +//! is native, so the call lowers to `tag(t"...")`: the tag and the literal pass +//! through as source, so any sibling lowering inside an interpolation still +//! applies. the literal's `t` is written with the rest of what a t-string needs +//! (see [`template_expression`]), since a field a lowering reached is rebuilt +//! around the whole literal, prefix included. //! //! Below 3.14 there is no runtime t-string, so the literal is rewritten to an //! explicit `_Template(...)` constructor over a polyfill with the same @@ -35,6 +36,8 @@ use ruff_text_size::{Ranged, TextRange}; use crate::Config; use super::ast_driver::{AstPass, Fragment, PassContext}; +use super::source_util::string_repr; +use super::template_expression; pub(crate) struct StringTagPass<'src> { source: &'src str, @@ -107,13 +110,16 @@ impl State<'_> { let func_end = call.func.range().end(); let lit_range = tstring.range(); if self.native { - // wrap the verbatim literal: `tag` `(t` `"..."` `)`. two narrow - // edits keep the source bytes between them, so a lowering inside an - // interpolation still applies - self.text_edits - .push((TextRange::empty(func_end), "(t".to_owned())); - self.text_edits - .push((TextRange::empty(call.range().end()), ")".to_owned())); + // `tag` `(` `"..."` `)`, the literal's `t` left to the t-string lowering + self.template_edits.push(( + call.range(), + vec![ + Fragment::Src(TextRange::new(call.range().start(), func_end)), + Fragment::Lit("(".to_owned()), + Fragment::Src(lit_range), + Fragment::Lit(")".to_owned()), + ], + )); return; } @@ -150,7 +156,7 @@ impl State<'_> { frags.push(Fragment::Src(interp.expression.range())); frags.push(Fragment::Lit(format!( ", {}", - string_repr(self.src(interp.expression.range())) + string_repr(template_expression::author_text(self.source, interp)) ))); let conversion = conversion_arg(interp.conversion); let format_spec = interp @@ -203,25 +209,6 @@ impl<'ast> Visitor<'ast> for State<'_> { } } -/// render `value` as a python string literal. defers to a conservative escaper -/// that always emits a double-quoted form so the result re-lexes as one token -fn string_repr(value: &str) -> String { - let mut out = String::with_capacity(value.len() + 2); - out.push('"'); - for ch in value.chars() { - match ch { - '\\' => out.push_str("\\\\"), - '"' => out.push_str("\\\""), - '\n' => out.push_str("\\n"), - '\r' => out.push_str("\\r"), - '\t' => out.push_str("\\t"), - c => out.push(c), - } - } - out.push('"'); - out -} - /// the PEP 750 conversion argument: `None`, or the conversion char as a string fn conversion_arg(flag: ConversionFlag) -> String { match flag { diff --git a/crates/by_transforms/src/transforms/super_keyword.rs b/crates/by_transforms/src/transforms/super_keyword.rs index e0f5e4d376..a17281ef43 100644 --- a/crates/by_transforms/src/transforms/super_keyword.rs +++ b/crates/by_transforms/src/transforms/super_keyword.rs @@ -13,18 +13,22 @@ use ruff_python_ast::{Expr, ExprContext, ModModule, Stmt}; use ruff_text_size::{Ranged, TextRange}; use super::ast_driver::{AstPass, PassContext, render_expr}; +use super::repeated_underscore::WrittenNames; -pub(crate) struct SuperKeyword; +pub(crate) struct SuperKeyword<'src> { + written: WrittenNames<'src>, +} -impl SuperKeyword { - pub(crate) fn new() -> Self { - Self +impl<'src> SuperKeyword<'src> { + pub(crate) fn new(written: WrittenNames<'src>) -> Self { + Self { written } } } -impl AstPass for SuperKeyword { +impl AstPass for SuperKeyword<'_> { fn run(&self, module: &mut ModModule, ctx: &mut PassContext) { let state = State { + written: self.written, edits: RefCell::new(Vec::new()), class_stack: RefCell::new(Vec::new()), self_stack: RefCell::new(Vec::new()), @@ -37,22 +41,27 @@ impl AstPass for SuperKeyword { } } -struct State { +struct State<'src> { + written: WrittenNames<'src>, edits: RefCell>, class_stack: RefCell>, self_stack: RefCell>, } -fn first_param_name(func: &ruff_python_ast::StmtFunctionDef) -> Option { +/// the name the receiver is bound to in the python, which a repeated `_` may have changed +fn first_param_name( + func: &ruff_python_ast::StmtFunctionDef, + written: WrittenNames, +) -> Option { let params = &func.parameters; params .posonlyargs .first() .or_else(|| params.args.first()) - .map(|p| p.parameter.name.as_str().to_owned()) + .map(|p| crate::python_parameter_name(params, &p.parameter, written).to_string()) } -impl<'ast> Visitor<'ast> for &State { +impl<'ast> Visitor<'ast> for &State<'_> { fn visit_stmt(&mut self, stmt: &'ast Stmt) { match stmt { Stmt::ClassDef(c) => { @@ -63,7 +72,7 @@ impl<'ast> Visitor<'ast> for &State { self.class_stack.borrow_mut().pop(); } Stmt::FunctionDef(f) => { - let pushed = first_param_name(f); + let pushed = first_param_name(f, self.written); if let Some(name) = &pushed { self.self_stack.borrow_mut().push(name.clone()); } diff --git a/crates/by_transforms/src/transforms/symbolic_type_op.rs b/crates/by_transforms/src/transforms/symbolic_type_op.rs index 96f515fa5c..fb8cf0a76e 100644 --- a/crates/by_transforms/src/transforms/symbolic_type_op.rs +++ b/crates/by_transforms/src/transforms/symbolic_type_op.rs @@ -16,9 +16,9 @@ //! in the working AST without marking the statement changed. that mutation is //! what lets it run before `typeof` lowering: a `typeof` operand (`1 + typeof //! d`) disappears from the AST here, so the `typeof` pass never sees it and -//! never claims the statement out from under the text edit. if some *other* -//! pass does end up re-rendering the statement, the AST already carries the -//! resolved type, so the result stays correct either way. +//! never claims the operation out from under the text edit. if some *other* +//! pass rewrites a node around it, which is printed from the AST, the AST +//! already carries the resolved type, so the result stays correct either way. use std::cell::RefCell; use std::collections::HashMap; @@ -206,13 +206,9 @@ impl TypeExprVisitor for FoldCollector<'_> { if rendered == "Any" { self.needs_any_import = true; } - self.folds.insert( - expr.range(), - Fold { - node: *parsed.into_syntax().body, - rendered, - }, - ); + let mut node = *parsed.into_syntax().body; + super::rerender::forget_expr_ranges(&mut node); + self.folds.insert(expr.range(), Fold { node, rendered }); // the whole operation is replaced; its operands are gone from the output Recurse::Stop } @@ -249,6 +245,24 @@ impl Transformer for SymbolicTypeOp { } impl AstPass for SymbolicTypeOp { + fn lowering(&self) -> Option { + Some(super::ast_driver::Lowering::SymbolicTypeOp) + } + + /// a fold replaces the operation with the type ty computed for it, which no longer + /// holds any of the operands' own spelling + fn subsumes(&self) -> &'static [super::ast_driver::Lowering] { + &[ + super::ast_driver::Lowering::Callable, + super::ast_driver::Lowering::OptionalType, + super::ast_driver::Lowering::LiteralType, + super::ast_driver::Lowering::JustFloat, + super::ast_driver::Lowering::DynamicKeyword, + super::ast_driver::Lowering::FloatConst, + super::ast_driver::Lowering::Typeof, + ] + } + fn run(&self, module: &mut ModModule, ctx: &mut PassContext) { // mutate the working AST (so `typeof` and other AST passes never see the // consumed operands) but drive the output through text edits, leaving the diff --git a/crates/by_transforms/src/transforms/template_expression.rs b/crates/by_transforms/src/transforms/template_expression.rs new file mode 100644 index 0000000000..3300e4c3e3 --- /dev/null +++ b/crates/by_transforms/src/transforms/template_expression.rs @@ -0,0 +1,557 @@ +//! a t-string field hands a reader its source text, which has to be the author's +//! +//! python 3.14 gives every replacement field of a `t"..."` to the program as a +//! `string.templatelib.Interpolation`, whose `expression` attribute is the *source text* +//! of the field: everything between the `{` and the end of the expression, its grouping +//! parentheses kept and its trailing whitespace dropped. +//! +//! ```python +//! a = 3 +//! t"{ (a) !r}".interpolations[0].expression # ' (a)' +//! ``` +//! +//! that text is what the attribute is for — a template library names a cache entry by it, +//! shows it in an error, or renders a `=`-style debug view from it. python reads it off +//! the file it compiled, which here is the lowered one, so `t"{a!}"` handed a reader +//! `_force_unwrap(a)`. that is not what the author wrote, and it is not something the +//! reader could evaluate either: the helper it names is private to the generated module. +//! so the author's text is the answer, the same answer +//! [`debug_field`](super::debug_field) reached for `f"{x = }"` +//! +//! there is no way to spell "this field's text is that" in a t-string literal, because +//! python derives it from the source. a t-string whose fields a lowering reached is +//! therefore rebuilt once, at runtime: [`_by_template`] takes the lowered template and +//! the author's text for each field it changed, and returns a template carrying the same +//! values under the author's own text. a t-string nothing lowered is left exactly as +//! written, and so is every field of a rebuilt one that nothing lowered — those keep the +//! interpolation python built, so their text stays byte-identical to python's own +//! +//! a `tag"..."` literal is a t-string only at 3.14, where the tag is called with it. the +//! source writes it without the `t`, so the prefix is written here too — inside the rebuild +//! when there is one, and on its own when there is not. below 3.14 it is not a t-string in +//! the output at all: [`string_tag`](super::string_tag) builds its template explicitly, and +//! gives each field the author's text as it does +//! +//! [`_by_template`]: crate::runtime + +use ruff_python_ast::comparable::ComparableExpr; +use ruff_python_ast::visitor::source_order::{self, SourceOrderVisitor}; +use ruff_python_ast::visitor::transformer::{self, Transformer}; +use ruff_python_ast::{ + Arguments, AtomicNodeIndex, Expr, ExprCall, ExprName, ExprNoneLiteral, ExprTString, ExprTuple, + InterpolatedElement, ModModule, Stmt, +}; +use ruff_python_ast::{ExprContext, name::Name}; +use ruff_text_size::{Ranged, TextRange, TextSize}; + +use super::ast_driver::Fragment; +use super::debug_field::rewrites_expression; +use super::source_util::string_repr; +use crate::runtime; + +/// a t-string to rebuild, and what each of its fields is to say it was written as +/// +/// `texts` runs parallel to the template's interpolations at runtime. `None` is a field +/// nothing lowered, which keeps the interpolation python built for it +pub(crate) struct Wrap { + /// the t-string literal, as the author's source spans it + pub(crate) range: TextRange, + texts: Vec>, + /// the literal is a string tag's, which the source writes without its `t` + pub(crate) tagged: bool, +} + +impl Wrap { + /// the call that rebuilds this template, re-emitting the literal from source so a + /// lowering inside any of its fields still applies + pub(crate) fn fragments(&self) -> Vec { + let prefix = if self.tagged { "t" } else { "" }; + if !self.rebuilds() { + return vec![Fragment::Lit(prefix.to_owned()), Fragment::Src(self.range)]; + } + vec![ + Fragment::Lit(format!("{}({prefix}", runtime::TEMPLATE_TEXT.name())), + Fragment::Src(self.range), + Fragment::Lit(format!(", {})", self.texts_tuple())), + ] + } + + /// whether the template is rebuilt at runtime: a field of it was lowered. a string tag's + /// literal nothing lowered only has its `t` written + pub(crate) fn rebuilds(&self) -> bool { + self.texts.iter().any(Option::is_some) + } + + /// the author's texts as a python tuple literal + /// + /// a template that reached this pass has at least one field, and a one-element tuple + /// needs a trailing comma to be a tuple at all + fn texts_tuple(&self) -> String { + let mut out = String::from("("); + for (index, text) in self.texts.iter().enumerate() { + if index > 0 { + out.push_str(", "); + } + match text { + Some(text) => out.push_str(&string_repr(text)), + None => out.push_str("None"), + } + } + if self.texts.len() == 1 { + out.push(','); + } + out.push(')'); + out + } + + /// the same call as a syntax tree, for a statement the driver re-renders + fn call(&self, template: Expr) -> Expr { + let texts = self + .texts + .iter() + .map(|text| match text { + Some(text) => Expr::StringLiteral(super::source_util::string_literal(text)), + None => Expr::NoneLiteral(ExprNoneLiteral { + node_index: AtomicNodeIndex::NONE, + range: TextRange::default(), + }), + }) + .collect(); + Expr::Call(ExprCall { + node_index: AtomicNodeIndex::NONE, + range_start: TextSize::default(), + func: Box::new(Expr::Name(ExprName { + node_index: AtomicNodeIndex::NONE, + range: TextRange::default(), + id: Name::from(runtime::TEMPLATE_TEXT.name()), + ctx: ExprContext::Load, + })), + arguments: Arguments { + node_index: AtomicNodeIndex::NONE, + range: TextRange::default(), + args: Box::new([ + template, + Expr::Tuple(ExprTuple { + node_index: AtomicNodeIndex::NONE, + range: TextRange::default(), + elts: texts, + ctx: ExprContext::Load, + parenthesized: true, + is_anon_named_tuple: false, + is_anon_named_tuple_value: false, + callable_shape: None, + is_parameter_shape: false, + }), + ]), + keywords: thin_vec::ThinVec::new(), + }, + cast_kind: None, + is_string_tag: false, + }) + } +} + +/// the source text python hands a reader as a field's `expression` +/// +/// python takes it from just past the `{` to the end of the expression, keeping whatever +/// grouping parentheses the expression was written inside and dropping the whitespace +/// after them. the expression node's own range holds neither, so both ends are read off +/// the field: the start from its brace, and the end by stepping over the parentheses that +/// close between the expression's last token and the `=`, conversion, format spec or `}` +/// that ends the field — nothing else can stand there +pub(crate) fn author_text<'src>(source: &'src str, field: &InterpolatedElement) -> &'src str { + let start = usize::from(field.range().start()) + '{'.len_utf8(); + let expression_end = usize::from(field.expression.range().end()); + let Some(tail) = source.get(expression_end..usize::from(field.range().end())) else { + return ""; + }; + let mut closing = 0; + for (offset, ch) in tail.char_indices() { + match ch { + ')' => closing = offset + ch.len_utf8(), + ch if ch.is_whitespace() => {} + _ => break, + } + } + source + .get(start..expression_end + closing) + .unwrap_or_default() +} + +/// a t-string of the source, the interpolations it hands a reader in the order it hands +/// them over, and whether it is a string tag's +type Template<'a> = (&'a ExprTString, Vec<&'a InterpolatedElement>, bool); + +/// every t-string of `suite` whose text a reader would get from the lowering +/// +/// a `tag"..."` template is one only when `tags` says the target calls its tag with a +/// t-string — see the module documentation +fn templates(suite: &[Stmt], tags: bool) -> Vec> { + struct Found<'a> { + templates: Vec>, + tagged: Vec, + } + + impl<'a> SourceOrderVisitor<'a> for Found<'a> { + fn visit_expr(&mut self, expr: &'a Expr) { + match expr { + Expr::Call(call) if call.is_string_tag => { + if let [Expr::TString(tstring)] = &*call.arguments.args { + self.tagged.push(tstring.range()); + } + } + Expr::TString(tstring) => { + // a format spec's own fields are not interpolations of the template: + // python evaluates them where they stand and hands the field one + // finished spec string, so only the top level is walked here + let fields = tstring + .value + .iter() + .flat_map(|part| part.elements.interpolations()) + .collect(); + self.templates.push((tstring, fields, false)); + } + _ => {} + } + source_order::walk_expr(self, expr); + } + } + + let mut found = Found { + templates: Vec::new(), + tagged: Vec::new(), + }; + for stmt in suite { + source_order::walk_stmt(&mut found, stmt); + } + let Found { + mut templates, + tagged, + } = found; + templates.retain_mut(|(tstring, fields, is_tagged)| { + *is_tagged = tagged.contains(&tstring.range()); + if *is_tagged { tags } else { !fields.is_empty() } + }); + templates +} + +/// the source span of every expression a t-string reports to the program that reads it +/// +/// the reverse direction leaves these alone. the `.by` it writes has to report what the +/// python it was given reported, and a `t"{a ?? b}"` says `a ?? b` where the +/// `t"{a if a is not None else b}"` it came from says the conditional. a `tag"..."` +/// template is in here too: its fields reach a reader the same way, whichever of the two +/// shapes [`string_tag`](super::string_tag) lowers it to +pub(crate) fn read_expressions(suite: &[Stmt]) -> Vec { + struct Read(Vec); + + impl<'a> SourceOrderVisitor<'a> for Read { + fn visit_expr(&mut self, expr: &'a Expr) { + if let Expr::TString(tstring) = expr { + self.0.extend( + tstring + .value + .iter() + .flat_map(|part| part.elements.interpolations()) + .map(|field| field.expression.range()), + ); + } + source_order::walk_expr(self, expr); + } + } + + let mut found = Read(Vec::new()); + for stmt in suite { + source_order::walk_stmt(&mut found, stmt); + } + found.0 +} + +/// the fields of `suite` whose expression an [`AstPass`](super::ast_driver::AstPass) +/// replaced, by the range of the field the author wrote +/// +/// read before any later rewrite of a field reaches the tree, so the tree still holds one +/// field per field the author wrote. a field the original has none of at this range is one +/// a pass built for itself, and its text is that pass's own to get right +pub(crate) fn replaced_in_tree( + module: &ModModule, + original: &[Stmt], + changed: &[usize], +) -> Vec { + let mut replaced = Vec::new(); + for &idx in changed { + let (Some(before), Some(after)) = (original.get(idx), module.body.get(idx)) else { + continue; + }; + let written = templates(std::slice::from_ref(before), true); + for (tstring, fields, _) in templates(std::slice::from_ref(after), true) { + let Some((_, was, _)) = written + .iter() + .find(|(other, _, _)| other.range() == tstring.range()) + else { + continue; + }; + for field in fields { + if was + .iter() + .find(|earlier| earlier.range() == field.range()) + .is_some_and(|earlier| { + ComparableExpr::from(&*earlier.expression) + != ComparableExpr::from(&*field.expression) + }) + { + replaced.push(field.range()); + } + } + } + } + replaced +} + +/// every t-string that has to be rebuilt, and the author's text for each of its fields +/// +/// `edited` is every range the passes claimed that writes something other than what it +/// covers, and `replaced` the fields [`replaced_in_tree`] found. `tags` says the target +/// calls a string tag with a t-string, whose `t` is then written here whether or not a +/// field needs rebuilding +pub(crate) fn claim( + source: &str, + suite: &[Stmt], + edited: &[TextRange], + replaced: &[TextRange], + tags: bool, +) -> Vec { + let mut wraps = Vec::new(); + for (tstring, fields, tagged) in templates(suite, tags) { + let texts: Vec> = fields + .iter() + .map(|field| { + let lowered = replaced.contains(&field.range()) + || edited.iter().any(|edit| { + rewrites_expression(*edit, field.expression.range(), field.range()) + }); + lowered.then(|| author_text(source, field).to_owned()) + }) + .collect(); + if tagged || texts.iter().any(Option::is_some) { + wraps.push(Wrap { + range: tstring.range(), + texts, + tagged, + }); + } + } + wraps +} + +/// rebuild in the syntax tree every t-string of `wraps` that stands in `module` +/// +/// the driver re-renders a statement an `AstPass` changed from the tree, where an edit +/// keyed on source ranges does not land — so a t-string in such a statement is wrapped +/// here instead. a t-string keeps its own range while a pass rewrites an expression +/// inside it, which is what matches one to its wrap +pub(crate) fn rewrite_changed(module: &mut ModModule, changed: &[usize], wraps: &[Wrap]) { + let rewrite = Rewrite { wraps }; + for &idx in changed { + if let Some(stmt) = module.body.get_mut(idx) { + rewrite.visit_stmt(stmt); + } + } +} + +struct Rewrite<'a> { + wraps: &'a [Wrap], +} + +impl Transformer for Rewrite<'_> { + fn visit_expr(&self, expr: &mut Expr) { + transformer::walk_expr(self, expr); + let Expr::TString(tstring) = expr else { + return; + }; + let Some(wrap) = self.wraps.iter().find(|wrap| wrap.range == tstring.range()) else { + return; + }; + let template = std::mem::replace( + expr, + Expr::NoneLiteral(ExprNoneLiteral { + node_index: AtomicNodeIndex::NONE, + range: TextRange::default(), + }), + ); + *expr = wrap.call(template); + } +} + +#[cfg(test)] +mod tests { + use crate::python_passthrough::unchanged; + use crate::{Config, PythonVersion, transpile}; + + /// the one line of the output the source's last line lowered to + /// + /// t-strings are 3.14 syntax, so every source here is transpiled for 3.14 + fn lowered(source: &str) -> String { + let out = transpile( + source, + &Config { + min_version: PythonVersion::PY314, + ..Config::test_default() + }, + ) + .unwrap_or_else(|error| panic!("{source:?}: {error}")); + out.lines() + .last() + .expect("some output") + .trim_start() + .to_owned() + } + + /// every operator lowering that can reach inside a t-string field says what the + /// author wrote rather than what it lowered to + /// a string tag's literal is written without its `t`, which the rebuild writes inside + /// it, so the tag is handed the template with the author's texts + #[test] + fn a_string_tag_is_handed_the_authors_text() { + assert_eq!( + lowered("def tag(t: object) -> object:\n return t\na: int? = 1\nx = tag\"{a!}\"\n"), + "x = tag(_by_template(t\"{_force_unwrap(a)}\", (\"a!\",)))" + ); + } + + /// a string tag's literal nothing lowered keeps python's own template, and only has its + /// `t` written + #[test] + fn a_string_tag_nothing_lowered_is_called_with_the_literal() { + assert_eq!( + lowered("def tag(t: object) -> object:\n return t\na = 1\nx = tag\"{a}\"\n"), + "x = tag(t\"{a}\")" + ); + } + + #[test] + fn an_operator_lowering_reports_the_author_text() { + // a force-unwrap + assert_eq!( + lowered("def go(a: int?):\n print(t\"{a!}\")\n"), + "print(_by_template(t\"{_force_unwrap(a)}\", (\"a!\",)))" + ); + // `??` + assert_eq!( + lowered("def go(a: int?, b: int):\n print(t\"{a ?? b}\")\n"), + "print(_by_template(t\"{a if a is not None else b}\", (\"a ?? b\",)))" + ); + // an optional chain + assert_eq!( + lowered("def go(a: int?):\n print(t\"{a?.bit_length()}\")\n"), + "print(_by_template(t\"{(None if a is None else a.bit_length())}\", \ + (\"a?.bit_length()\",)))" + ); + // a grapheme accessor, which re-emits its receiver + assert_eq!( + lowered("def go(s: str):\n print(t\"{s.first}\")\n"), + "print(_by_template(t\"{(Character(_by_graphemes(s)[0]) if s else None)}\", \ + (\"s.first\",)))" + ); + } + + /// `typeof` is folded in the syntax tree rather than emitted as an edit, so the + /// statement around it is re-rendered — the other half of the rewrite + #[test] + fn a_syntax_tree_fold_reports_the_author_text() { + assert_eq!( + lowered("def go(a: int):\n print(t\"{typeof(a)}\")\n"), + "print(_by_template(t\"{TypeOf[a]}\", ('typeof(a)',)))" + ); + } + + /// an extension call and a conversion are resolved from the checker's answer rather + /// than from the syntax, and reach the field the same way + #[test] + fn a_type_directed_rewrite_reports_the_author_text() { + assert_eq!( + lowered( + "extension list:\n def second(self) -> Element:\n return self[1]\n\nxs = [1, 2, 3]\nprint(t\"{xs.second()}\")\n" + ), + "print(_by_template(t\"{_by_ext__list__second(xs)}\", (\"xs.second()\",)))" + ); + // a conversion fires where the argument is written, which here is inside the field + assert_eq!( + lowered( + "class Meters:\n def __init__(self, value: float):\n self.value = value\n\n @classmethod\n def __of__(cls, value: int) -> Self:\n return cls(float(value))\n\ndef take(m: Meters) -> int:\n return 1\n\nprint(t\"{take(3)}\")\n" + ), + "print(_by_template(t\"{take(Meters.__of__(3))}\", (\"take(3)\",)))" + ); + } + + /// a `=` field of a t-string is taken apart by `debug_field` into the literal text + /// python itself would have put there and a field beside it, and that field reports + /// the author's text too — the parentheses the taking-apart adds are not the author's + #[test] + fn a_debug_field_reports_the_author_text() { + assert_eq!( + lowered("def go(a: int?):\n print(t\"{(a!) = }\")\n"), + "print(_by_template(t\"(a!) = {(_force_unwrap(a))!r}\", (\"(a!)\",)))" + ); + } + + /// only the fields a lowering reached are rebuilt. the rest keep the interpolation + /// python built, so their own text stays byte-identical to python's + #[test] + fn an_untouched_field_keeps_pythons_own_interpolation() { + assert_eq!( + lowered("def go(a: int?, b: int):\n print(t\"{b} {a!} {b}\")\n"), + "print(_by_template(t\"{b} {_force_unwrap(a)} {b}\", (None, \"a!\", None)))" + ); + } + + /// python reads the text from just past the `{` to the end of the expression, keeping + /// the grouping parentheses and dropping the whitespace after them. `fields_carry_the_ + /// text_python_would_have_reported` in the runtime test holds these to a real 3.14 + #[test] + fn the_author_text_is_the_span_python_would_have_reported() { + assert_eq!( + lowered("def go(a: int?):\n print(t\"{ (a!) }\")\n"), + "print(_by_template(t\"{ (_force_unwrap(a)) }\", (\" (a!)\",)))" + ); + // a conversion and a format spec end the text the same way the closing brace does + assert_eq!( + lowered("def go(a: int?):\n print(t\"{ (a!) !r:>6}\")\n"), + "print(_by_template(t\"{ (_force_unwrap(a)) !r:>6}\", (\" (a!)\",)))" + ); + } + + /// a t-string nothing lowered is left exactly as written, in a `.by` file as in a + /// `.py` one — there is nothing to correct, and python's own text is already right + #[test] + fn an_unlowered_template_is_left_alone() { + unchanged("x = 1\nprint(t\"{x}\")\n"); + assert_eq!( + lowered("def go(a: int?, b: int):\n print(t\"{b}\")\n"), + "print(t\"{b}\")" + ); + } + + /// the reverse direction leaves a t-string's fields exactly as written. rewriting one + /// would change what the program reports — `t"{a ?? b}"` reports `a ?? b` where the + /// python it came from reports the conditional + #[test] + fn the_reverse_direction_leaves_a_field_alone() { + let out = crate::reverse_transpile( + "def go(a: int | None, b: int):\n print(t\"{a if a is not None else b}\")\n", + &Config::test_default(), + ) + .expect("reverse should succeed"); + assert!( + out.contains("t\"{a if a is not None else b}\""), + "got:\n{out}" + ); + // the same expression outside a t-string is rewritten as it always was + let out = crate::reverse_transpile( + "def go(a: int | None, b: int):\n print(f\"{a if a is not None else b}\")\n", + &Config::test_default(), + ) + .expect("reverse should succeed"); + assert!(out.contains("f\"{a ?? b}\""), "got:\n{out}"); + } +} diff --git a/crates/by_transforms/src/transforms/trailing_lambda.rs b/crates/by_transforms/src/transforms/trailing_lambda.rs index 036dde0fcb..b2165168ed 100644 --- a/crates/by_transforms/src/transforms/trailing_lambda.rs +++ b/crates/by_transforms/src/transforms/trailing_lambda.rs @@ -382,7 +382,12 @@ impl TrailingLambdaLower<'_, '_> { // separate from. so a call that writes no arguments but // fills a `context` parameter still needs one here, or the // two insertions run together: `f(theme=themeblock=...)` - let fills_context = !self.types.implicit_context_arguments(call).is_empty(); + // a refused `context` argument is reported by the `context` lowering, + // which writes nothing for it + let fills_context = self + .types + .implicit_context_arguments(call) + .is_ok_and(|implicit| !implicit.is_empty()); let separator = match last_argument_end { None if fills_context => ", ", None => "", @@ -842,10 +847,10 @@ mod tests { } #[test] - fn ast_pass_inside_body_falls_back_to_rendered_lowering() { - // `typeof` is rewritten in the AST, so the driver re-renders the whole - // statement through the generator, which has no type info — the - // rendered lowering appends the function positionally + fn an_ast_pass_inside_the_body_keeps_the_template_lowering() { + // `typeof` is rewritten in the syntax tree, and only the node it rewrote is + // printed from there, so the block keeps the template lowering and its + // keyword binding let out = check(indoc! {" def f(a: (int) -> None): a(1) @@ -854,14 +859,17 @@ mod tests { x: typeof(1) = 1 print(x, it) "}); - assert!(out.contains("def __trailing_lambda__(it):"), "got:\n{out}"); - assert!(out.contains("f(__trailing_lambda__)"), "got:\n{out}"); + assert!( + out.contains("def _trailing_lambda_0(it=None):"), + "got:\n{out}" + ); + assert!(out.contains("f(a=_trailing_lambda_0)"), "got:\n{out}"); + assert!(out.contains("x: TypeOf[1] = 1"), "got:\n{out}"); } - /// a typed lambda used to force that fallback too. it lowers by deleting - /// its basedpython surface now (see [`typed_lambda`](super::typed_lambda)), - /// leaving the statement un-re-rendered, so the block keeps the template - /// lowering and its keyword binding + /// a typed lambda lowers by deleting its basedpython surface (see + /// [`typed_lambda`](super::typed_lambda)), leaving the statement's source in + /// place, so the block keeps the template lowering and its keyword binding #[test] fn typed_lambda_inside_body_keeps_the_template_lowering() { let out = check(indoc! {" diff --git a/crates/by_transforms/src/transforms/type_is.rs b/crates/by_transforms/src/transforms/type_is.rs index 49ddc402e0..25b2f9fbeb 100644 --- a/crates/by_transforms/src/transforms/type_is.rs +++ b/crates/by_transforms/src/transforms/type_is.rs @@ -35,6 +35,12 @@ impl<'src> TypeIs<'src> { } impl AstPass for TypeIs<'_> { + /// a return guard it rewrites into `TypeIs[T]` is the `is` pair the parametric lowering + /// would otherwise have tested + fn subsumes(&self) -> &'static [super::ast_driver::Lowering] { + &[super::ast_driver::Lowering::ParametricIs] + } + fn run(&self, module: &mut ModModule, ctx: &mut PassContext) { let mut state = State { edits: Vec::new(), diff --git a/crates/by_transforms/src/transforms/typed_dict_literal.rs b/crates/by_transforms/src/transforms/typed_dict_literal.rs index ce917c4e18..c33dd8c795 100644 --- a/crates/by_transforms/src/transforms/typed_dict_literal.rs +++ b/crates/by_transforms/src/transforms/typed_dict_literal.rs @@ -448,6 +448,12 @@ fn flatten_bitand(expr: &Expr) -> Vec<&Expr> { } impl AstPass for TypedDictLiteralPass<'_> { + /// a typed dict type it replaces moves into a class, whose field types the shared + /// type-expression lowerer prints + fn subsumes(&self) -> &'static [super::ast_driver::Lowering] { + super::callable::TYPE_EXPRESSION + } + fn run(&self, module: &mut ModModule, ctx: &mut PassContext) { let mut inner = TypedDictLiteral::new(self.source); let body: &[Stmt] = &module.body; diff --git a/crates/by_transforms/src/transforms/typed_lambda.rs b/crates/by_transforms/src/transforms/typed_lambda.rs index 88953a5f1f..1c740e7031 100644 --- a/crates/by_transforms/src/transforms/typed_lambda.rs +++ b/crates/by_transforms/src/transforms/typed_lambda.rs @@ -8,15 +8,14 @@ //! annotation on them. //! //! All of that is removed as **source deletions** rather than by clearing the -//! AST: a cleared node would make the driver re-render the whole enclosing -//! statement, and a re-render drops every sub-statement edit inside it. A -//! lambda is a value, so what surrounds it is ordinary code that other passes -//! rewrite — a loop's [per-iteration binding](super::unique_loop_bindings) -//! wraps it, `??` lowers inside its body — and those edits have to survive. -//! Deletions leave the rest of the statement's bytes alone, so they do. +//! AST: a cleared node would be printed from the syntax tree rather than from +//! the source. A lambda is a value, so what surrounds it is ordinary code that +//! other passes rewrite — a loop's [per-iteration binding](super::unique_loop_bindings) +//! wraps it, `??` lowers inside its body — and deletions leave every byte of it +//! but the surface they remove, so those edits apply as written. //! -//! A shape with no parenthesized parameter list to anchor to keeps the old AST -//! rewrite, and the re-render that comes with it. +//! A shape with no parenthesized parameter list to anchor to keeps the AST +//! rewrite. use std::cell::{Cell, RefCell}; diff --git a/crates/by_transforms/src/transforms/typeof_keyword.rs b/crates/by_transforms/src/transforms/typeof_keyword.rs index 690859aa23..e94421de73 100644 --- a/crates/by_transforms/src/transforms/typeof_keyword.rs +++ b/crates/by_transforms/src/transforms/typeof_keyword.rs @@ -183,8 +183,7 @@ mod tests { "}, indoc! {" from ty_extensions import TypeOf - def f(x: TypeOf[y]) -> TypeOf[z]: - ... + def f(x: TypeOf[y]) -> TypeOf[z]: ... "}, ); } diff --git a/crates/by_transforms/src/transforms/unique_loop_bindings.rs b/crates/by_transforms/src/transforms/unique_loop_bindings.rs index 338445c7e9..480ff0bc1a 100644 --- a/crates/by_transforms/src/transforms/unique_loop_bindings.rs +++ b/crates/by_transforms/src/transforms/unique_loop_bindings.rs @@ -649,7 +649,8 @@ mod tests { return value "}, indoc! {" - _MISSING = object() + from typing import Any + _MISSING: Any = object() def build(): for i in items: @_by_loop_bind(i=i) @@ -820,7 +821,8 @@ mod tests { return cb "}, indoc! {" - _MISSING = object() + from typing import Any + _MISSING: Any = object() def build(): for i in items: @_by_loop_bind(i=i) diff --git a/crates/by_transforms/src/transforms/unpack.rs b/crates/by_transforms/src/transforms/unpack.rs index ec25f0ddb2..b3bcbf5384 100644 --- a/crates/by_transforms/src/transforms/unpack.rs +++ b/crates/by_transforms/src/transforms/unpack.rs @@ -32,6 +32,10 @@ impl UnpackSyntax { } impl AstPass for UnpackSyntax { + fn lowering(&self) -> Option { + Some(super::ast_driver::Lowering::Unpack) + } + fn run(&self, module: &mut ModModule, ctx: &mut PassContext) { // a parameter pack is a `ParamSpec` at runtime, and neither `**kwargs: **Pack` nor // `*args: *P` is valid python at any version, so this lowering is not version-gated diff --git a/crates/by_transforms/src/transforms/visibility_rename.rs b/crates/by_transforms/src/transforms/visibility_rename.rs index a2f165b912..ffb3633cbd 100644 --- a/crates/by_transforms/src/transforms/visibility_rename.rs +++ b/crates/by_transforms/src/transforms/visibility_rename.rs @@ -48,6 +48,16 @@ use crate::type_info::TypeInfo; pub(crate) struct VisibilityRenamePass; impl TypeAwarePass for VisibilityRenamePass { + fn lowering(&self) -> Option { + Some(super::ast_driver::Lowering::VisibilityRename) + } + + /// a private definition it renames is written under its mangled name, without the + /// `private` keyword + fn subsumes(&self) -> &'static [super::ast_driver::Lowering] { + &[super::ast_driver::Lowering::Modifiers] + } + fn run(&self, stmts: &[Stmt], types: &dyn TypeInfo, ctx: &mut PassContext) { let module_private: HashSet = types.private_module_symbols().into_iter().collect(); let mut declarations = RenamedDeclarations::default(); diff --git a/crates/by_transforms/src/type_info.rs b/crates/by_transforms/src/type_info.rs index 8eb41561cc..0e050bb813 100644 --- a/crates/by_transforms/src/type_info.rs +++ b/crates/by_transforms/src/type_info.rs @@ -4,6 +4,7 @@ use crate::transforms::trailing_lambda::RECEIVER_PARAMETER; use ruff_python_ast::helpers::is_dotted_name; use ruff_python_ast::{ Expr, ExprCall, ExprName, ExprRef, Parameter, ParameterWithDefault, Stmt, StmtClassDef, + StmtFunctionDef, }; use ruff_python_parser::parse_expression; use ruff_python_stdlib::basedpython::IMPLICIT_TYPING_NAMES; @@ -12,6 +13,7 @@ use ty_python_core::scope::ScopeKind; use ty_python_core::{global_scope, place_table, semantic_index}; use ty_python_semantic::types::call_type_forms::CallTypeForm; use ty_python_semantic::types::exceptions::RaisesRuntimeTarget; +use ty_python_semantic::types::repeated_underscore::{UnderscoreLowering, UnderscoreRefusal}; use ty_python_semantic::types::{ DisplaySettings, DynamicType, KnownClass, KnownInstanceType, Type, UnpackedKwargs, character, }; @@ -124,6 +126,21 @@ pub(crate) trait TypeInfo { function: &ruff_python_ast::StmtFunctionDef, ) -> Option; + /// the return type of a `def` that wrote no return annotation, spelled for the + /// emitted file — what its body returns, which is what ty reads there. + /// + /// `None` when the source wrote an annotation (the annotation itself is what to + /// re-emit, and it may carry a lowering of its own), when the type carries a + /// dynamic part, when it has no python type-expression spelling, or when that + /// spelling reads a name the emitted file would not resolve — `Literal[1]` is + /// ty's exact answer for `return 1` and `Literal` is not a name basedpython binds + /// implicitly, so there is no import to write for it + fn inferred_return_annotation( + &self, + function: &ruff_python_ast::StmtFunctionDef, + min_version: ruff_python_ast::PythonVersion, + ) -> Option; + /// whether `name` resolves to a basedpython return-value marker — /// `ignorable_return_value` or `must_use_return_value`. both are pure /// declarations about how a caller may treat a result, with nothing behind @@ -375,7 +392,11 @@ pub(crate) trait TypeInfo { /// example: a literal `20` inferred as `Literal[20]` is promoted to /// `"int"` here so two value-forms with structurally compatible fields /// hash to the same class shape. - fn promoted_type_display(&self, expr: &Expr) -> Option; + fn promoted_type_display( + &self, + expr: &Expr, + min_version: ruff_python_ast::PythonVersion, + ) -> Option; /// whether `expr` is an application of a `type def` — `F[bool]` where `F` is /// a type function. such an application lowers to the type the type function @@ -539,8 +560,12 @@ pub(crate) trait TypeInfo { /// lambda block binds implicitly, when the call is inside one. the variable /// name is already the *emitted* one, so a block receiver comes back as the /// lowering's receiver parameter rather than as `self`. empty when the - /// callee has no context parameters or nothing resolves - fn implicit_context_arguments(&self, call: &ExprCall) -> Vec<(String, String)>; + /// callee has no context parameters or nothing resolves. an error, naming the + /// parameter, when a value that would fill one is a repeated `_` parameter: + /// which of those a read of `_` means is not decided, so no argument can be + /// written for it. an error too when a parameter to fill is a `_` the callee + /// repeats, since which of those a keyword `_` names is not decided either + fn implicit_context_arguments(&self, call: &ExprCall) -> Result, String>; /// whether `expr`'s inferred type is a string — a `str` / `Character` / /// `LiteralString` / string-literal / `str`-subclass instance. dynamic @@ -576,7 +601,11 @@ pub(crate) trait TypeInfo { /// the promoted type carries a dynamic part (`Unknown` / `Any`), or it has /// no python type-expression spelling (a module, a callable in arrow form, /// a class local to a function) - fn inferred_annotation(&self, expr: &Expr) -> Option; + fn inferred_annotation( + &self, + expr: &Expr, + min_version: ruff_python_ast::PythonVersion, + ) -> Option; /// basedpython: the default this parameter takes from the method its `def` overrides, /// written the way a signature writes it — `1`, `"a"`, `None`. @@ -587,6 +616,14 @@ pub(crate) trait TypeInfo { /// overrides declares one, or when the base's default is an expression rather than a value fn inherited_parameter_default(&self, parameter: &ParameterWithDefault) -> Option; + /// basedpython: how the lowering writes `function`'s parameters when they repeat `_`: the + /// names and kinds ty's signature of the definition is built from, or why there are none. + /// `None` when they do not repeat it + fn repeated_underscore_lowering( + &self, + function: &StmtFunctionDef, + ) -> Option>; + /// whether adding a type annotation to a bare `name = value` assignment in /// `class_def`'s body would change the class's runtime semantics — true for /// dataclass-like classes, framework models, `NamedTuple`s, `TypedDict`s @@ -694,6 +731,30 @@ impl TypeInfo for SemanticModel<'_> { ) } + fn inferred_return_annotation( + &self, + function: &ruff_python_ast::StmtFunctionDef, + min_version: ruff_python_ast::PythonVersion, + ) -> Option { + if function.returns.is_some() { + return None; + } + let db = self.db(); + let env = self.program_environment(); + let ty = + ty_python_semantic::types::declared_return_type(db, function.inferred_type(self)?)?; + // a dynamic part (`Unknown` from an unresolved import, `Any`) has no faithful + // annotation + if ty.has_dynamic(db, &env) { + return None; + } + // the promoted form, as for a synthesized assignment annotation: `A()` infers + // `final A`, which has no python spelling at all, and the annotation a reader + // would write for it is `A` + let promoted = ty.promote(db, &env).promote_class_literals(db, &env); + spell_for_python(db, &env, self, promoted, min_version) + } + fn is_return_value_marker(&self, name: &ExprName) -> bool { name.inferred_type(self) .and_then(Type::as_function_literal) @@ -1001,12 +1062,16 @@ impl TypeInfo for SemanticModel<'_> { }) } - fn promoted_type_display(&self, expr: &Expr) -> Option { + fn promoted_type_display( + &self, + expr: &Expr, + min_version: ruff_python_ast::PythonVersion, + ) -> Option { let db = self.db(); let env = self.program_environment(); let ty = expr.inferred_type(self)?; let promoted = ty.promote(db, &env).promote_class_literals(db, &env); - spell_for_python(db, &env, self, promoted) + spell_for_python(db, &env, self, promoted, min_version) } fn template_literal_strings(&self, expr: &Expr) -> Option> { @@ -1177,6 +1242,13 @@ impl TypeInfo for SemanticModel<'_> { ty_python_semantic::types::ide_support::inherited_parameter_default(self, parameter) } + fn repeated_underscore_lowering( + &self, + function: &StmtFunctionDef, + ) -> Option> { + ty_python_semantic::types::ide_support::repeated_underscore_lowering(self, function) + } + fn parameter_check_plan(&self, parameter: &Parameter) -> Option { let ty = parameter.inferred_type(self)?; ty_python_semantic::types::soundness::parameter_runtime_check_plan( @@ -1257,31 +1329,44 @@ impl TypeInfo for SemanticModel<'_> { SemanticModel::trailing_lambda_callee_is_once(self, callee) } - fn implicit_context_arguments(&self, call: &ExprCall) -> Vec<(String, String)> { + fn implicit_context_arguments(&self, call: &ExprCall) -> Result, String> { let Some(callee) = call.func.inferred_type(self) else { - return Vec::new(); + return Ok(Vec::new()); }; - ty_python_semantic::types::context_params::implicit_context_arguments( + let implicit = ty_python_semantic::types::context_params::implicit_context_arguments( self.db(), &self.program_environment(), self.file(), callee, call, - ) - .into_iter() - .map(|argument| { - let variable = if argument.is_block_receiver { - RECEIVER_PARAMETER.to_string() - } else if argument.is_module_private { - // the binding is a module-level `private` variable, which the - // lowering emits under its underscored name - crate::transforms::modifiers::module_private_name(&argument.variable) - } else { - argument.variable.to_string() - }; - (argument.parameter.to_string(), variable) - }) - .collect() + ); + if let Some(parameter) = implicit.from_repeated_underscore.first() { + return Err(format!( + "a repeated `_` parameter cannot supply the context argument `{parameter}`" + )); + } + if implicit.for_repeated_underscore { + return Err( + "a context argument cannot be supplied implicitly for a repeated `_` parameter" + .to_string(), + ); + } + Ok(implicit + .arguments + .into_iter() + .map(|argument| { + let variable = if argument.is_block_receiver { + RECEIVER_PARAMETER.to_string() + } else if argument.is_module_private { + // the binding is a module-level `private` variable, which the + // lowering emits under its underscored name + crate::transforms::modifiers::module_private_name(&argument.variable) + } else { + argument.variable.to_string() + }; + (argument.parameter.to_string(), variable) + }) + .collect()) } fn is_string_like(&self, expr: &Expr) -> bool { @@ -1319,7 +1404,11 @@ impl TypeInfo for SemanticModel<'_> { ty_python_semantic::types::class_framework_role(self.db(), class) } - fn inferred_annotation(&self, expr: &Expr) -> Option { + fn inferred_annotation( + &self, + expr: &Expr, + min_version: ruff_python_ast::PythonVersion, + ) -> Option { let db = self.db(); let env = self.program_environment(); let ty = expr.inferred_type(self)?; @@ -1332,7 +1421,7 @@ impl TypeInfo for SemanticModel<'_> { // ``); `type[int]` is its promoted form, and the type ty // itself reads back off the undeclared attribute through an instance let promoted = ty.promote(db, &env).promote_class_literals(db, &env); - spell_for_python(db, &env, self, promoted) + spell_for_python(db, &env, self, promoted, min_version) } fn class_body_annotation_is_semantic(&self, class_def: &StmtClassDef) -> bool { @@ -1405,11 +1494,19 @@ fn display_for_python<'db>( /// qualifying them and reporting the module to import; the `typing` names ride the same /// table [`implicit_typing`](super::transforms::implicit_typing) uses for names the source /// does write, which never sees a synthesized annotation. +/// +/// A name it can place neither way is a refusal, not an omission: dropped, it reaches the +/// emitted file unresolved — ty spells a generator's element type `types.GeneratorType[ +/// Literal[1], None, None]`, and `Literal` is not one of the names basedpython binds. And +/// the modules reported are only those the spelling actually names, because the reporting +/// is per-type rather than per-name: ty names `types` for the type spelled `None`, which +/// bought an emitted file a `if TYPE_CHECKING: import types` it never read. fn spell_for_python<'db>( db: &'db dyn Db, env: &ProgramEnvironment<'db>, model: &SemanticModel<'db>, ty: Type<'db>, + min_version: ruff_python_ast::PythonVersion, ) -> Option { let spelling = ty.source_spelling_in( db, @@ -1418,24 +1515,53 @@ fn spell_for_python<'db>( DisplaySettings::default().with_reduced_symbolic_operations(), )?; let text = strip_binding_context_suffix(&spelling.text); - let parsed = parse_expression(&text).ok()?; + let read = names_read(&text)?; - let mut names = NameCollector(Vec::new()); - ruff_python_ast::visitor::Visitor::visit_expr(&mut names, parsed.expr()); - let typing_names = names - .0 - .into_iter() + let typing_names: Vec<&'static str> = read + .iter() .filter(|name| !model.is_bound_globally(name)) .filter_map(|name| IMPLICIT_TYPING_NAMES.iter().copied().find(|n| *n == name)) .collect(); + let mut modules = spelling.modules; + modules.retain(|module| read.iter().any(|name| name == module)); - Some(SynthesizedType { + let spelled = SynthesizedType { text, - modules: spelling.modules, + modules, typing_names, + }; + resolvable_in_output(model, min_version, &spelled, &read).then_some(spelled) +} + +/// Whether every name `spelled` reads will resolve in the emitted file. +/// +/// [`spell_for_python`] accounts for two kinds of name it has to import — a qualified +/// class's module, and a `typing` name basedpython binds implicitly. What is left is fine +/// only if the output binds it anyway: a builtin, or a name the source's own global scope +/// binds, which the emitted file keeps. Anything else would reach the output unresolved, +/// so the spelling is refused rather than written with a name that is not there. +fn resolvable_in_output( + model: &SemanticModel<'_>, + min_version: ruff_python_ast::PythonVersion, + spelled: &SynthesizedType, + read: &[String], +) -> bool { + read.iter().all(|name| { + spelled.modules.iter().any(|module| module == name) + || spelled.typing_names.contains(&name.as_str()) + || model.is_bound_globally(name) + || ruff_python_stdlib::builtins::is_python_builtin(name, min_version.minor, false) }) } +/// the root names `text` reads, or `None` when it is not an expression at all +fn names_read(text: &str) -> Option> { + let parsed = parse_expression(text).ok()?; + let mut names = NameCollector(Vec::new()); + ruff_python_ast::visitor::Visitor::visit_expr(&mut names, parsed.expr()); + Some(names.0) +} + /// The names a spelled type reads at its roots — `decimal` in `decimal.Decimal`, `Never` /// in `Never | None`. An attribute's member is not a `Name`, so only roots are collected struct NameCollector(Vec); diff --git a/crates/by_transforms/tests/conversion_runtime.rs b/crates/by_transforms/tests/conversion_runtime.rs index e372300066..717bd1dcd0 100644 --- a/crates/by_transforms/tests/conversion_runtime.rs +++ b/crates/by_transforms/tests/conversion_runtime.rs @@ -125,6 +125,30 @@ assert [t.degrees for t in temperatures] == [32.0, 212.0] mapping: dict[str, Fahrenheit] = {"boiling": Celsius(100.0)} assert mapping["boiling"].degrees == 212.0 +# a conversion inside a `def` another lowering rewrites in the syntax tree — here the +# numbering of a repeated `_` — spelled the way the source spells it rather than the +# way the tree would be printed +def ignoring(c: Celsius, _: int, _: int, /) -> object: + return report( + c, + ) + +assert ignoring(Celsius(100.0), 1, 2) == 212.0, ignoring(Celsius(100.0), 1, 2) + +# a call that also passes a trailing lambda's block, or fills a `context` parameter: +# each is an argument the lowering adds after the converted one +def report_then(t: Fahrenheit, then: (float) -> None): + then(t.degrees) + +report_then(Celsius(100.0)): + assert it == 212.0, it + +def report_in(t: Fahrenheit, context unit: str) -> str: + return f"{t.degrees}{unit}" + +context unit = "F" +assert report_in(Celsius(100.0)) == "212.0F", report_in(Celsius(100.0)) + print("ok") "#; diff --git a/crates/by_transforms/tests/fstring_debug_runtime.rs b/crates/by_transforms/tests/fstring_debug_runtime.rs new file mode 100644 index 0000000000..db9cea3ded --- /dev/null +++ b/crates/by_transforms/tests/fstring_debug_runtime.rs @@ -0,0 +1,115 @@ +//! Runtime test for the f-string `=` field, whose text is the author's own. +//! +//! Python renders `f"{expr = }"` by printing the source text between the `{` and +//! the `=` and then the value, so the text a lowered field prints is a claim about +//! the *source*, not about the lowering. Only an interpreter can settle it: the +//! transform's unit tests can say what text was emitted, but not what python makes +//! of it — a literal whose braces or backslashes were escaped wrongly still looks +//! plausible in the emitted source and prints something else. +//! +//! Every assertion here is written twice over: the basedpython program asserts what +//! the field prints, and [`PYTHON_REFERENCE`] asserts that the same expectation +//! holds for the plain-python spelling of the same field. So a wrong expectation +//! fails rather than being blessed. + +use std::process::Command; + +use by_transforms::{Config, PythonVersion, transpile}; + +mod common; + +/// basedpython whose `=` fields all hold an expression that lowers +const PROGRAM: &str = r#" +def show(a: int?, b: int, d: dict[str, int]) -> None: + # a force-unwrap + assert f"{(a!) = }" == "(a!) = 3", f"force-unwrap: {f'{(a!) = }'}" + # a `??` + assert f"{a ?? b = }" == "a ?? b = 3", "coalesce" + # an optional chain + assert f"{a?.bit_length() = }" == "a?.bit_length() = 2", "optional chain" + # the author's own spacing, which python reproduces byte for byte + assert f"{ (a!) = }" == " (a!) = 3", "spacing" + # a format spec, which python prints instead of the repr + assert f"{(a!)=:>6}" == "(a!)= 3", "format spec" + # a conversion the field names for itself + assert f"{(a!)=!s}" == "(a!)=3", "conversion" + # braces in the author's text, which are not a field of their own + assert f"{({1: a}[1]!) = }" == "({1: a}[1]!) = 3", "braces" + # a backslash, which python prints as the two characters it was written as + assert f"{('a\n'!) = }" == "('a\\n'!) = 'a\\n'", "backslash" + # the string's own quote, nested the way 3.12 allows + assert f"{(d["k"]!) = }" == '(d["k"]!) = 5', "nested quote" + # and a field nothing lowered, which keeps python's own compact rendering + assert f"{b = }" == "b = 7", "unlowered" + assert f"{ {1: 2}[1] = }" == " {1: 2}[1] = 2", "unlowered braces" + +show(3, 7, {"k": 5}) +print("ok") +"#; + +/// the same fields with the basedpython operators taken out, so the expectations above +/// are checked against python's own rendering rather than against themselves +const PYTHON_REFERENCE: &str = r#" +def show(a, b, d): + assert f"{(a) = }" == "(a) = 3", "force-unwrap" + assert f"{ (a) = }" == " (a) = 3", "spacing" + assert f"{(a)=:>6}" == "(a)= 3", "format spec" + assert f"{(a)=!s}" == "(a)=3", "conversion" + assert f"{({1: a}[1]) = }" == "({1: a}[1]) = 3", "braces" + assert f"{('a\n') = }" == "('a\\n') = 'a\\n'", "backslash" + assert f"{(d["k"]) = }" == '(d["k"]) = 5', "nested quote" + assert f"{b = }" == "b = 7", "unlowered" + assert f"{ {1: 2}[1] = }" == " {1: 2}[1] = 2", "unlowered braces" + +show(3, 7, {"k": 5}) +print("ok") +"#; + +fn run(python: &str, source: &str, what: &str) { + let output = Command::new(python) + .arg("-c") + .arg(source) + .output() + .expect("failed to spawn python"); + assert!( + output.status.success(), + "{what} failed on {python}:\n--- stderr ---\n{}\n--- source ---\n{source}", + String::from_utf8_lossy(&output.stderr), + ); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); +} + +#[test] +#[expect( + clippy::print_stderr, + reason = "a skipped test must say why it skipped, or it reads as a pass" +)] +fn a_debug_field_prints_the_source_the_author_wrote() { + let Some(python) = common::python() else { + eprintln!("skipping f-string `=` field runtime test: no interpreter found"); + return; + }; + // a replacement field may nest the string's own quote from 3.12 on, and hold a + // backslash from the same release + let probe = Command::new(&python) + .args([ + "-c", + "import sys; sys.exit(0 if sys.version_info >= (3, 12) else 1)", + ]) + .status() + .map(|status| status.success()) + .unwrap_or(false); + if !probe { + eprintln!("skipping f-string `=` field runtime test: needs python 3.12 or later"); + return; + } + + run(&python, PYTHON_REFERENCE, "the plain-python reference"); + + let config = Config { + min_version: PythonVersion::PY313, + ..Config::default() + }; + let transpiled = transpile(PROGRAM, &config).expect("transpile should succeed"); + run(&python, &transpiled, "the transpiled program"); +} diff --git a/crates/by_transforms/tests/lazy_import_runtime.rs b/crates/by_transforms/tests/lazy_import_runtime.rs index dadeec7496..14aadef40e 100644 --- a/crates/by_transforms/tests/lazy_import_runtime.rs +++ b/crates/by_transforms/tests/lazy_import_runtime.rs @@ -201,6 +201,61 @@ assert textwrap.indent("a", " ") == " a", "and the alias after that one binds to print("ok") "#; +/// the readers of an annotation, each asserting the answer python gives for the names it +/// finds. a proxy is not the object it stands for, and no forwarded operator makes it one: +/// `dataclasses` tells a `KW_ONLY` annotation from a field by identity, and a type hint is +/// compared by identity by whatever consumes it. so a name an annotation reads is imported +/// eagerly. `InitVar[int]` and `singledispatch`'s registration by annotation are readers +/// the proxy happened to satisfy, and are kept here as readers all the same +const ANNOTATIONS_MAIN: &str = r#" +from dataclasses import dataclass, fields, KW_ONLY, InitVar +from fractions import Fraction +from functools import singledispatch +import fractions +import typing + +@dataclass +class Row: + name: str + _: KW_ONLY + extra: int = 0 + seed: InitVar[int] = 0 + +assert [field.name for field in fields(Row)] == ["name", "extra"], "`KW_ONLY` and `InitVar` are not fields" +assert Row("a", extra=1, seed=2).extra == 1, "the row constructs with its keyword-only field" + +@singledispatch +def describe(value: object) -> str: + return "object" + +@describe.register +def _(value: Fraction) -> str: + return "fraction" + +assert describe(Fraction(1, 2)) == "fraction", "an annotation registers a class" + +def half(value: Fraction) -> Fraction: + return value / Fraction(2) + +assert typing.get_type_hints(half)["value"] === fractions.Fraction, "the hint is the class itself" +print("ok") +"#; + +/// under `from __future__ import annotations` every annotation is a string, and +/// `typing.get_type_hints` evaluates it against the module's globals +const FUTURE_ANNOTATIONS_MAIN: &str = r#" +from __future__ import annotations +from fractions import Fraction +import fractions +import typing + +def half(value: Fraction) -> None: + pass + +assert typing.get_type_hints(half)["value"] === fractions.Fraction, "the string resolves to the class" +print("ok") +"#; + /// an interpreter to run the transpiled output on. `$PYTHON` first, then the /// usual names; `None` (test skips) when none is found fn python() -> Option { @@ -219,6 +274,15 @@ fn python() -> Option { }) } +/// whether `python` runs what the transpiler targets by default, 3.10 — a +/// dataclass's `KW_ONLY` among it +fn runs_the_default_target(python: &str) -> bool { + Command::new(python) + .args(["-c", "import sys; sys.exit(sys.version_info < (3, 10))"]) + .status() + .is_ok_and(|status| status.success()) +} + /// transpile `sources` (`module stem` → basedpython source) into a fresh /// directory under the cargo temp dir fn build_case(case: &str, sources: &[(&str, &str)]) -> PathBuf { @@ -322,3 +386,34 @@ fn a_submodule_resolves_through_the_proxy() { let dir = build_case("lazy_submodule", &[("main", SUBMODULE_MAIN)]); run_main(&python, &dir); } + +#[test] +fn an_annotation_reads_the_imported_object() { + let Some(python) = python() else { + eprintln!("skipping lazy-import runtime test: no python interpreter found"); + return; + }; + if !runs_the_default_target(&python) { + eprintln!("skipping lazy-import runtime test: {python} is older than python 3.10"); + return; + } + let dir = build_case("lazy_annotations", &[("main", ANNOTATIONS_MAIN)]); + run_main(&python, &dir); +} + +#[test] +fn a_string_annotation_resolves_to_the_imported_object() { + let Some(python) = python() else { + eprintln!("skipping lazy-import runtime test: no python interpreter found"); + return; + }; + if !runs_the_default_target(&python) { + eprintln!("skipping lazy-import runtime test: {python} is older than python 3.10"); + return; + } + let dir = build_case( + "lazy_future_annotations", + &[("main", FUTURE_ANNOTATIONS_MAIN)], + ); + run_main(&python, &dir); +} diff --git a/crates/by_transforms/tests/tstring_expression_runtime.rs b/crates/by_transforms/tests/tstring_expression_runtime.rs new file mode 100644 index 0000000000..6baa95bc30 --- /dev/null +++ b/crates/by_transforms/tests/tstring_expression_runtime.rs @@ -0,0 +1,138 @@ +//! Runtime test for a t-string field's `expression`, whose text is the author's own. +//! +//! Python 3.14 hands every replacement field of a `t"..."` to the program as a +//! `string.templatelib.Interpolation` whose `expression` is the *source text* of +//! that field, so the text a lowered field reports is a claim about the *source*, +//! not about the lowering. Only an interpreter can settle it: the transform's unit +//! tests can say what was emitted, but not what python makes of it — and the exact +//! span python reports (its leading whitespace, its grouping parentheses, its +//! dropped trailing whitespace) is python's own rule, which is easy to state +//! wrongly from the outside. +//! +//! Every assertion here is written twice over: the basedpython program asserts what +//! the field reports, and [`PYTHON_REFERENCE`] asserts that the same expectation +//! holds for the plain-python spelling of the same field. So a wrong expectation +//! fails rather than being blessed. + +use std::process::Command; + +use by_transforms::{Config, PythonVersion, transpile}; + +/// basedpython whose t-string fields hold expressions that lower +const PROGRAM: &str = r#" +from string.templatelib import Template + +def show(a: int?, b: int, d: dict[str, int]) -> None: + def text(t) -> list: + return [i.expression for i in t.interpolations] + + # a force-unwrap + assert text(t"{a!}") == ["a!"], text(t"{a!}") + # a `??` + assert text(t"{a ?? b}") == ["a ?? b"], text(t"{a ?? b}") + # an optional chain + assert text(t"{a?.bit_length()}") == ["a?.bit_length()"], text(t"{a?.bit_length()}") + # `typeof`, which is folded in the syntax tree rather than emitted as an edit + assert text(t"{typeof(b)}") == ["typeof(b)"], text(t"{typeof(b)}") + # the author's own spacing and grouping parentheses, which python keeps + assert text(t"{ (a!) }") == [" (a!)"], text(t"{ (a!) }") + # a conversion and a format spec end the reported text where the brace would + assert text(t"{ (a!) !r:>6}") == [" (a!)"], text(t"{ (a!) !r:>6}") + # a field nothing lowered keeps the interpolation python built for it + assert text(t"{b} {a!} { b }") == ["b", "a!", " b"], text(t"{b} {a!} { b }") + # the values and the rendering are the ones the lowering produced + assert t"{a!}".values == (3,), t"{a!}".values + assert t"{ (a!) !r:>6}".values == (3,), "spec values" + assert t"{a!}".strings == ("", ""), t"{a!}".strings + # a `=` field, which python itself splits into literal text and a field beside it + assert list(t"{(a!) = }")[0] == "(a!) = ", list(t"{(a!) = }")[0] + assert text(t"{(a!) = }") == ["(a!)"], text(t"{(a!) = }") + # a template nothing lowered is left exactly as written + assert text(t"{b}") == ["b"], text(t"{b}") + # a string tag is handed the template with the same texts + assert text(same"{a!}") == ["a!"], text(same"{a!}") + assert text(same"{b} {a ?? b}") == ["b", "a ?? b"], text(same"{b} {a ?? b}") + assert text(same"{b}") == ["b"], text(same"{b}") + +def same(t: Template) -> Template: + return t + +show(3, 7, {"k": 5}) +print("ok") +"#; + +/// the same fields with the basedpython operators taken out, so the expectations above +/// are checked against python's own reporting rather than against themselves +const PYTHON_REFERENCE: &str = r#" +def show(a, b, d): + def text(t): + return [i.expression for i in t.interpolations] + + assert text(t"{a}") == ["a"], text(t"{a}") + assert text(t"{ (a) }") == [" (a)"], text(t"{ (a) }") + assert text(t"{ (a) !r:>6}") == [" (a)"], text(t"{ (a) !r:>6}") + assert text(t"{b} {a} { b }") == ["b", "a", " b"], text(t"{b} {a} { b }") + assert t"{a}".values == (3,), t"{a}".values + assert t"{a}".strings == ("", ""), t"{a}".strings + assert list(t"{(a) = }")[0] == "(a) = ", list(t"{(a) = }")[0] + assert text(t"{(a) = }") == ["(a)"], text(t"{(a) = }") + assert text(t"{b}") == ["b"], text(t"{b}") + +show(3, 7, {"k": 5}) +print("ok") +"#; + +/// An interpreter that has t-strings. `$PYTHON` is taken only when it is one, because +/// the suite's usual interpreter is older and would skip every assertion here silently. +fn python_314() -> Option { + let mut candidates = Vec::new(); + if let Ok(python) = std::env::var("PYTHON") { + candidates.push(python); + } + candidates.extend(["python3.14", "python3"].map(String::from)); + candidates.into_iter().find(|python| { + Command::new(python) + .args([ + "-c", + "import sys; sys.exit(0 if sys.version_info >= (3, 14) else 1)", + ]) + .status() + .map(|status| status.success()) + .unwrap_or(false) + }) +} + +fn run(python: &str, source: &str, what: &str) { + let output = Command::new(python) + .arg("-c") + .arg(source) + .output() + .expect("failed to spawn python"); + assert!( + output.status.success(), + "{what} failed on {python}:\n--- stderr ---\n{}\n--- source ---\n{source}", + String::from_utf8_lossy(&output.stderr), + ); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); +} + +#[test] +#[expect( + clippy::print_stderr, + reason = "a skipped test must say why it skipped, or it reads as a pass" +)] +fn a_field_reports_the_source_the_author_wrote() { + let Some(python) = python_314() else { + eprintln!("skipping t-string `expression` runtime test: needs python 3.14 or later"); + return; + }; + + run(&python, PYTHON_REFERENCE, "the plain-python reference"); + + let config = Config { + min_version: PythonVersion::PY314, + ..Config::default() + }; + let transpiled = transpile(PROGRAM, &config).expect("transpile should succeed"); + run(&python, &transpiled, "the transpiled program"); +} diff --git a/crates/ruff_db/src/diagnostic/mod.rs b/crates/ruff_db/src/diagnostic/mod.rs index 5b0dde0a52..4dcb46fd5b 100644 --- a/crates/ruff_db/src/diagnostic/mod.rs +++ b/crates/ruff_db/src/diagnostic/mod.rs @@ -1147,6 +1147,25 @@ pub enum DiagnosticId { /// /// This indicates a bug in the program rather than a user error. InternalError, + + /// A module the transpiler declined to lower. + /// + /// ## Why is this bad? + /// Some basedpython is well-formed but has no correct python to lower it to — a module + /// that binds one of the names its own runtime helpers are provided under, say. The + /// transpiler reports the module rather than emitting python that would do the wrong + /// thing, and the report names what to change. + /// + /// ## Example + /// ```python + /// def go() -> str: + /// _soundness_check = "mine" + /// return pick(xs, 0) + /// ``` + /// + /// The checked `return` lowers to a `_soundness_check(...)` call, which in this scope + /// finds the local string. Rename the local. + TranspileRefused, } impl DiagnosticId { @@ -1202,6 +1221,7 @@ impl DiagnosticId { DiagnosticId::InvalidCliOption => "invalid-cli-option", DiagnosticId::PreviewFeature => "preview-feature", DiagnosticId::InternalError => "internal-error", + DiagnosticId::TranspileRefused => "transpile-refused", } } diff --git a/crates/ruff_python_formatter/resources/test/fixtures/ruff/force_unwrap.by b/crates/ruff_python_formatter/resources/test/fixtures/ruff/force_unwrap.by new file mode 100644 index 0000000000..58ee17c18e --- /dev/null +++ b/crates/ruff_python_formatter/resources/test/fixtures/ruff/force_unwrap.by @@ -0,0 +1,19 @@ +x = a! +y = a!.bit_length() +z = a?.b! + +print(f"{a!}") +print(f"{a!:>4}") +print(f"{a!!r}") +print(f"{(a!)!r}") +print(f"{(a!)=}") +print(f"{a?.b()!}") +print(f"{ [a!, b] }") +print(f"{f'{a!}'}") +print(f"{b:{a!}}") + +print(f"{a!r}") +print(f"{a!s:>10}") +print(f"{a!a}") +print(f"{a=!r}") +print(f"{a!=b}") diff --git a/crates/ruff_python_formatter/tests/snapshots/format@force_unwrap.by.snap b/crates/ruff_python_formatter/tests/snapshots/format@force_unwrap.by.snap new file mode 100644 index 0000000000..ff22728f0b --- /dev/null +++ b/crates/ruff_python_formatter/tests/snapshots/format@force_unwrap.by.snap @@ -0,0 +1,49 @@ +--- +source: crates/ruff_python_formatter/tests/fixtures.rs +input_file: crates/ruff_python_formatter/resources/test/fixtures/ruff/force_unwrap.by +--- +## Input +```bython +x = a! +y = a!.bit_length() +z = a?.b! + +print(f"{a!}") +print(f"{a!:>4}") +print(f"{a!!r}") +print(f"{(a!)!r}") +print(f"{(a!)=}") +print(f"{a?.b()!}") +print(f"{ [a!, b] }") +print(f"{f'{a!}'}") +print(f"{b:{a!}}") + +print(f"{a!r}") +print(f"{a!s:>10}") +print(f"{a!a}") +print(f"{a=!r}") +print(f"{a!=b}") +``` + +## Output +```bython +x = a! +y = a!.bit_length() +z = a?.b! + +print(f"{a!}") +print(f"{a!:>4}") +print(f"{a!!r}") +print(f"{(a!)!r}") +print(f"{(a!)=}") +print(f"{a?.b()!}") +print(f"{[a!, b]}") +print(f"{f'{a!}'}") +print(f"{b:{a!}}") + +print(f"{a!r}") +print(f"{a!s:>10}") +print(f"{a!a}") +print(f"{a=!r}") +print(f"{a != b}") +``` diff --git a/crates/ruff_python_parser/src/parser/expression.rs b/crates/ruff_python_parser/src/parser/expression.rs index ca974a91d0..5fb91a0057 100644 --- a/crates/ruff_python_parser/src/parser/expression.rs +++ b/crates/ruff_python_parser/src/parser/expression.rs @@ -136,6 +136,23 @@ pub(super) const fn starts_statement_expression(kind: TokenKind) -> bool { ) } +/// basedpython: the refusal for a force-unwrap written in front of a replacement field's +/// `=` debug marker. +/// +/// Both spellings of it — `f"{a!=}"`, which the lexer reads as one `!=`, and `f"{a! = }"`, +/// which it reads as two tokens — are refused, because accepting either would make the +/// meaning of `!=` inside a field depend on the whitespace around it. `f"{a!=b}"` is a +/// comparison and has to stay one, and python's grammar has no other place where a space +/// decides which operator was written. +fn force_unwrap_before_debug_equal() -> ParseErrorType { + ParseErrorType::BasedPythonOnly( + "`!=` is the comparison operator inside a replacement field too, so a `!` written \ + in front of the field's `=` cannot be the force-unwrap. Parenthesise the operand: \ + `f\"{(a!) = }\"`" + .to_string(), + ) +} + impl<'src> Parser<'src> { /// Returns `true` if the parser is at a name or keyword (including soft keyword) token. pub(super) fn at_name_or_keyword(&self) -> bool { @@ -1428,11 +1445,41 @@ impl<'src> Parser<'src> { node_index: AtomicNodeIndex::NONE, }) } + // basedpython: a `!` in front of a replacement field's `=` is the + // one place the force-unwrap cannot be claimed. `!=` is the + // comparison operator everywhere, `f"{a!=b}"` included, and the + // lexer glues it — so `f"{a!=}"` and `f"{a! = }"` would differ + // only by a space, which python's grammar never uses to tell two + // operators apart. neither is claimed, and this says so where + // python's "invalid conversion character" would not. + TokenKind::Exclamation + if self.options.is_basedpython + && context.is_in_interpolation() + && self.peek() == TokenKind::Equal => + { + let range = self.current_token_range(); + self.add_error(force_unwrap_before_debug_equal(), range); + break lhs; + } // basedpython postfix `!` force-unwrap. Inside an interpolated - // string replacement field a trailing `!` is the conversion flag - // (`f"{x!r}"`), so it is suppressed there. + // string replacement field a `!` introduces python's conversion + // flag (`f"{x!r}"`), so there it is the force-unwrap operator + // only where no conversion can follow it: the field's closing + // `}`, the `:` that opens its format spec, or another `!`, + // which is then the conversion's own (`f"{x!!r}"` unwraps `x` + // and prints its repr). python rejects all three spellings, so + // this takes nothing away from it — but a `.py` file keeps + // python's "invalid conversion character", which is what a + // mistyped `!r` needs to hear rather than a message about an + // operator that language does not have. TokenKind::Exclamation - if self.options.mode != Mode::Ipython && !context.is_in_interpolation() => + if self.options.mode != Mode::Ipython + && (!context.is_in_interpolation() + || (self.options.is_basedpython + && matches!( + self.peek(), + TokenKind::Rbrace | TokenKind::Colon | TokenKind::Exclamation + ))) => { self.error_if_not_basedpython( "`!` (force-unwrap) operator is not valid in .py files".to_string(), @@ -2269,6 +2316,26 @@ impl<'src> Parser<'src> { identity_ops.resize(index, false); identity_ops.push(identity); }; + // basedpython: the other spelling of a force-unwrap in front of a replacement + // field's `=`, where the lexer glued the two into one `!=`. a comparison with + // nothing on its right is never valid, so the author meant the debug form — + // but claiming it here would make `f"{a!=}"` and `f"{a!=b}"` different + // operators, told apart by what follows. see the `Exclamation` arm in + // `parse_lhs_expression` for the other half + if self.options.is_basedpython + && op == CmpOp::NotEq + && context.is_in_interpolation() + && self.at(TokenKind::NotEqual) + { + let followed_by_end_of_field = matches!( + self.peek(), + TokenKind::Rbrace | TokenKind::Colon | TokenKind::Exclamation + ); + if followed_by_end_of_field { + let range = self.current_token_range(); + self.add_error(force_unwrap_before_debug_equal(), range); + } + } record_identity(&mut identity_ops, 0, self.at_identity_operator(op)); self.bump_cmp_op(op); @@ -5535,9 +5602,10 @@ bitflags! { const EXCLUDE_FOR = 1 << 5; /// basedpython: set while parsing the top-level value of an interpolated - /// string replacement field (`f"{value!r}"`). Suppresses the postfix - /// `!` force-unwrap operator so the trailing `!` stays available as the - /// conversion flag. A parenthesised `(value!)` resets the context. + /// string replacement field (`f"{value!r}"`). Keeps a trailing `!` available + /// as the conversion flag rather than reading it as the postfix force-unwrap + /// operator, except where no conversion can follow it. A parenthesised + /// `(value!)` resets the context. const IN_INTERPOLATION = 1 << 6; /// basedpython: set while parsing the lower end of a type-parameter bound range @@ -5661,8 +5729,8 @@ impl ExpressionContext { } /// basedpython: returns `true` if parsing the value of an interpolated-string - /// replacement field, where a trailing `!` is the conversion flag rather - /// than the postfix force-unwrap operator + /// replacement field, where a `!` that could introduce a conversion flag is + /// read as that rather than as the postfix force-unwrap operator const fn is_in_interpolation(self) -> bool { self.0.contains(ExpressionContextFlags::IN_INTERPOLATION) } diff --git a/crates/ruff_python_parser/src/parser/tests.rs b/crates/ruff_python_parser/src/parser/tests.rs index c9e126d609..b12df92e0c 100644 --- a/crates/ruff_python_parser/src/parser/tests.rs +++ b/crates/ruff_python_parser/src/parser/tests.rs @@ -1888,6 +1888,186 @@ fn fstring_conversion_after_ternary_is_not_force_unwrap() { } } +/// The value of the first replacement field of the first f-string in `source`, parsed as +/// basedpython, together with its conversion flag. +fn first_interpolation(source: &str) -> (Expr, ruff_python_ast::ConversionFlag) { + let parsed = parse_basedpython_module(source); + let Some(Stmt::Expr(stmt)) = parsed.syntax().body.first() else { + panic!("expected an expression statement for {source:?}"); + }; + let Expr::FString(fstring) = &*stmt.value else { + panic!("expected an f-string for {source:?}, got {:?}", stmt.value); + }; + let element = fstring + .value + .elements() + .find_map(InterpolatedStringElement::as_interpolation) + .unwrap_or_else(|| panic!("expected a replacement field in {source:?}")); + ((*element.expression).clone(), element.conversion) +} + +#[test] +fn force_unwrap_in_interpolation_where_no_conversion_can_follow() { + // inside a replacement field a `!` normally introduces python's conversion + // flag, so the postfix force-unwrap has to give way to it. but a conversion + // is a single `s`/`r`/`a` before the field's `}` or its format spec, so a + // `!` sitting directly in front of `}`, `:` or another `!` cannot be one — + // there the `!` is the force-unwrap, and `f"{a!}"` no longer has to be + // written `f"{(a!)}"` to be understood + for source in [ + r#"f"{a!}""#, + r#"f"{a!:>4}""#, + r#"f"{a!!}""#, + r#"f"{a?.b()!}""#, + ] { + let (expr, conversion) = first_interpolation(source); + assert!( + matches!(&expr, Expr::UnaryOp(unary) if unary.op == UnaryOp::Force), + "expected the field of {source:?} to be a force-unwrap, got {expr:?}" + ); + assert!( + conversion.is_none(), + "expected no conversion flag on {source:?}" + ); + } + + // the conversion the `!` is usually reaching for keeps every spelling it has + for (source, expected) in [ + (r#"f"{a!r}""#, ruff_python_ast::ConversionFlag::Repr), + (r#"f"{a!s:>10}""#, ruff_python_ast::ConversionFlag::Str), + (r#"f"{a!a}""#, ruff_python_ast::ConversionFlag::Ascii), + (r#"f"{a=!r}""#, ruff_python_ast::ConversionFlag::Repr), + ] { + let (expr, conversion) = first_interpolation(source); + assert_eq!(conversion, expected, "conversion flag of {source:?}"); + assert!( + matches!(expr, Expr::Name(_)), + "expected a plain name under the conversion of {source:?}, got {expr:?}" + ); + } + + // `!` glued to `=` is the comparison operator, in a replacement field as + // everywhere else, so `f"{a!=b}"` keeps meaning `a != b` and a force-unwrap + // the author wants to print with `=` still needs `f"{(a!)=}"` + let (expr, _) = first_interpolation(r#"f"{a!=b}""#); + assert!( + matches!(&expr, Expr::Compare(_)), + "expected a comparison for `f\"{{a!=b}}\"`, got {expr:?}" + ); + + // a force-unwrap in front of the conversion's own `!` leaves the conversion + // alone, so `f"{a!!r}"` unwraps `a` and prints its repr + let (expr, conversion) = first_interpolation(r#"f"{a!!r}""#); + assert_eq!(conversion, ruff_python_ast::ConversionFlag::Repr); + assert!( + matches!(&expr, Expr::UnaryOp(unary) if unary.op == UnaryOp::Force), + "expected a force-unwrap under the conversion, got {expr:?}" + ); + + // a `.py` file has no force-unwrap to reach for, and `f"{x!}"` there is a + // mistyped conversion — so python's own diagnostic is what it keeps + for source in [r#"f"{a!}""#, r#"f"{a!:>4}""#] { + let parsed = crate::parse_unchecked(source, ParseOptions::from(Mode::Module)); + let messages: Vec = parsed.errors().iter().map(ToString::to_string).collect(); + assert!( + messages + .iter() + .any(|message| message.contains("invalid conversion character")), + "expected {source:?} to keep python's conversion diagnostic, got {messages:?}" + ); + assert!( + !messages + .iter() + .any(|message| message.contains("force-unwrap")), + "expected {source:?} not to be reported as a force-unwrap, got {messages:?}" + ); + } +} + +/// the basedpython parse errors of `source`, in the order the parser raised them +fn basedpython_errors(source: &str) -> Vec { + let options = ParseOptions::from(Mode::Module).with_basedpython(true); + crate::parse_unchecked(source, options) + .errors() + .iter() + .map(ToString::to_string) + .collect() +} + +#[test] +fn force_unwrap_in_front_of_a_debug_equal_is_refused() { + // a `!` in front of a replacement field's `=` is the one place the force-unwrap + // cannot be claimed. the lexer glues `!=`, so `f"{a!=}"` and `f"{a! = }"` would + // differ only by a space — and `f"{a!=b}"` is a comparison that has to stay one, + // so a space would be deciding which operator was written. python's grammar never + // does that, and neither do we: both spellings are refused, and the refusal names + // the parenthesised form that does work + for source in [ + r#"f"{a!=}""#, + r#"f"{a!= }""#, + r#"f"{a!=:>4}""#, + r#"f"{a!=!r}""#, + r#"f"{a! = }""#, + r#"f"{a! =}""#, + r#"f"{a ! = }""#, + r#"f"{a! =:>4}""#, + ] { + let messages = basedpython_errors(source); + assert!( + messages + .first() + .is_some_and(|message| message.contains(r#"f"{(a!) = }""#)), + "expected {source:?} to be refused with the parenthesised spelling, got {messages:?}" + ); + } + + // every comparison keeps its meaning, however it is spaced and whatever follows it + for source in [ + r#"f"{a!=b}""#, + r#"f"{a != b}""#, + r#"f"{a!=-1}""#, + r#"f"{d[a!=b] = }""#, + r#"f"{(a != b)}""#, + ] { + assert_eq!( + basedpython_errors(source), + Vec::::new(), + "expected {source:?} to parse" + ); + } + + // the parenthesised spelling the refusal names really does work, and a field with + // no `!` in front of its `=` is python's own + for source in [ + r#"f"{(a!) = }""#, + r#"f"{(a!)=}""#, + r#"f"{a = }""#, + r#"f"{a=!r}""#, + ] { + assert_eq!( + basedpython_errors(source), + Vec::::new(), + "expected {source:?} to parse" + ); + } + + // a `.py` file has no force-unwrap to reach for, so it keeps python's own wording + for source in [r#"f"{a!=}""#, r#"f"{a! = }""#] { + let messages: Vec = + crate::parse_unchecked(source, ParseOptions::from(Mode::Module)) + .errors() + .iter() + .map(ToString::to_string) + .collect(); + assert!( + !messages + .iter() + .any(|message| message.contains("force-unwrap")), + "expected {source:?} to keep python's own diagnostic, got {messages:?}" + ); + } +} + #[test] fn nested_parens_grow_stack() { let src = format!("{}1{}", "(".repeat(1_000), ")".repeat(1_000)); diff --git a/crates/ty/src/by_commands.rs b/crates/ty/src/by_commands.rs index 3e4a8a51d5..903f446d2c 100644 --- a/crates/ty/src/by_commands.rs +++ b/crates/ty/src/by_commands.rs @@ -22,7 +22,9 @@ use walkdir::WalkDir; use crate::ExitStatus; use crate::args::LoweringArgs; use crate::by_lowering::SettledLowering; -use by_stage::emit::{CheckGate, Emit, Transpiled, is_unusable_source, transpile_bug_diagnostic}; +use by_stage::emit::{ + CheckGate, Emit, Transpiled, is_unusable_source, transpile_failure_diagnostic, +}; use by_stage::project::{ BY_SOURCES, COMPILABLE_SOURCES, Rebuilder, build_project_db, may_contain_sources, module_roots, source_files, @@ -1578,7 +1580,7 @@ pub(crate) fn cmd_transpile( out } Err(e) => { - diagnostics.push(transpile_bug_diagnostic(file, &e)); + diagnostics.push(transpile_failure_diagnostic(file, &e)); render_diagnostics(&db, &diagnostics)?; return Ok(ExitStatus::Failure); } @@ -1761,8 +1763,9 @@ fn py_source_files(root: &Path) -> Vec { const BY_RUNNER_FILENAME: &str = "_by_runner.py"; /// write the sourcemap module + runner shim into the run dir. the shim runs the -/// target module and, on an uncaught exception, rewrites traceback frames in -/// generated files back to their `.by` source location +/// target module compiled under its `.by` path and lines, and, on an uncaught +/// exception, rewrites any frame still in a generated file back to its `.by` +/// source location fn write_traceback_runtime( staging: &mut Staging, entries: &[TracebackEntry], @@ -1775,7 +1778,9 @@ fn write_traceback_runtime( } const BY_RUNNER_SRC: &str = r#"# generated by `by run` — runs the target module with .by-aware tracebacks +import ast import hashlib +import importlib.machinery import importlib.util import linecache import os @@ -1789,6 +1794,9 @@ from _by_sourcemap import DIGESTS, SOURCEMAP # still match the filenames python reports in frames. the key the entry was # written under travels with it, because DIGESTS is keyed the unresolved way _BY_MAP = {os.path.realpath(py): (py, info) for py, info in SOURCEMAP.items()} +# the .by files themselves, which a module compiled by `_by_code` names in its frames, +# with the key of the entry each was transpiled under +_BY_SOURCES = {os.path.realpath(info[0]): py for py, info in SOURCEMAP.items()} # entries already checked against their digests, so each file is read at most # once no matter how many frames it contributes @@ -1813,6 +1821,14 @@ def _digest_matches(path, spec): return digest.hexdigest() == expected +def _stale(key, by_path): + _STALE.add(key) + sys.stderr.write( + f"note: {by_path} no longer matches what it was transpiled from, " + "so no frame names a line of it\n" + ) + + def _is_current(key, by_path): # the map describes a *pair* of files, and a .by line read out of it is only # true while both are still the ones it was built from. a source saved since @@ -1831,20 +1847,53 @@ def _is_current(key, by_path): ): _CURRENT.add(key) return True - _STALE.add(key) - sys.stderr.write( - f"note: {by_path} no longer matches what it was transpiled from — " - "its frames are left as generated python\n" - ) + _stale(key, by_path) + return False + + +def _still_current(key, by_path): + # a module `_by_code` compiled has its `.by` lines in its frames already, checked + # when it was loaded. a `.by` saved over since then no longer has those lines, so + # it is checked again before a frame names one + if key in _STALE: + return False + digests = DIGESTS.get(key) + if digests is not None and _digest_matches(by_path, digests["by"]): + return True + _stale(key, by_path) return False def _rewrite(frames): # drop the runner/runpy bootstrap above the first user frame - first = next((i for i, f in enumerate(frames) if os.path.realpath(f.filename) in _BY_MAP), None) + first = next( + ( + i + for i, f in enumerate(frames) + if os.path.realpath(f.filename) in _BY_MAP + or os.path.realpath(f.filename) in _BY_SOURCES + ), + None, + ) frames = frames[first:] if first is not None else frames out = [] for f in frames: + compiled_from = _BY_SOURCES.get(os.path.realpath(f.filename)) + if compiled_from is not None and not _still_current(compiled_from, f.filename): + # its line is one of the file that was transpiled, which is gone. the + # generated line it came from is named instead when it is the only one + # that `.by` line was written as, and no line at all when it is not + _, generated = SOURCEMAP[compiled_from] + written_as = [ + index + 1 + for index, line in enumerate(generated) + if f.lineno and line == f.lineno - 1 + ] + if len(written_as) == 1: + out.append(traceback.FrameSummary(compiled_from, written_as[0], f.name)) + else: + out.append(f' File "{f.filename}", in {f.name}\n') + continue entry = _BY_MAP.get(os.path.realpath(f.filename)) if entry is not None and f.lineno is not None: key, (by_path, lines) = entry @@ -1862,7 +1911,17 @@ def _rewrite(frames): def _excepthook(etype, evalue, tb): frames = _rewrite(traceback.extract_tb(tb)) sys.stderr.write("Traceback (most recent call last):\n") - sys.stderr.write("".join(traceback.StackSummary.from_list(frames).format())) + # a frame `_rewrite` could only name is already text. the others are formatted a + # run at a time, so python still folds a repeated one + run = [] + for frame in frames + [None]: + if isinstance(frame, traceback.FrameSummary): + run.append(frame) + continue + sys.stderr.write("".join(traceback.StackSummary.from_list(run).format())) + run = [] + if frame is not None: + sys.stderr.write(frame) sys.stderr.write("".join(traceback.format_exception_only(etype, evalue))) @@ -1875,6 +1934,45 @@ def _by_source_of(origin): return None if entry is None else entry[1][0] +def _by_code(origin): + """The staged `.py` at `origin` compiled as the `.by` it was transpiled from. + + A program reads its own locations back — `traceback.format_exc()`, the file + and line `warnings` attributes a warning to, `co_filename`, `inspect` — and + python answers each from the code object. So the code object is compiled under + the `.by` path, with every line the `.by` line it came from: the answer is then + the author's wherever it is read, and the source line python shows beside it is + read from the `.by` too. A line the transpiler wrote on its own — a runtime + helper — has no `.by` line, and says so as line 0. Columns are the generated + file's, so every node spans the whole of its `.by` line instead: python marks + no part of a line a node covers, and reads what it can from the `.by` text. + + `None` when there is no `.by` to answer with, or it has changed since it was + transpiled, which leaves the module to python's own loader. + """ + entry = _BY_MAP.get(os.path.realpath(origin)) if origin else None + if entry is None: + return None + key, (by_path, lines) = entry + if not _is_current(key, by_path): + return None + with open(origin, "rb") as handle: + tree = ast.parse(handle.read(), origin) + with open(by_path, "rb") as handle: + widths = [len(line.rstrip(b"\r\n")) for line in handle] + + def by_line(line): + mapped = lines[line - 1] if line is not None and 0 < line <= len(lines) else None + return 0 if mapped is None else mapped + 1 + + for node in ast.walk(tree): + if "lineno" in node._attributes: + node.lineno = node.end_lineno = by_line(node.lineno) + node.col_offset = 0 + node.end_col_offset = widths[node.lineno - 1] if node.lineno else 0 + return compile(tree, by_path, "exec", dont_inherit=True) + + class _ByLoader: """Wraps a staged module's loader to name its `.by` source as `__file__`. @@ -1884,9 +1982,11 @@ class _ByLoader: when the run ends. The `.by` the module was written as is the answer to every question those paths are asked, so it is what the module reports. - Only `__file__` moves. The code object still comes from the `.py`, so the - filename in a traceback frame is still the one `SOURCEMAP` is keyed by and - frames are still rewritten to `.by` lines the same way. + A module python's own loader would compile is compiled by `_by_code` instead, + under the `.by` path and lines. One any other loader stands behind — a test + runner that rewrites assertions as it compiles — is left to it, and only its + `__file__` moves; its frames are rewritten to `.by` lines when they reach the + excepthook. """ def __init__(self, inner, by_path): @@ -1896,8 +1996,20 @@ class _ByLoader: def create_module(self, spec): return self._by_inner.create_module(spec) + def get_code(self, fullname): + if type(self._by_inner) is importlib.machinery.SourceFileLoader: + code = _by_code(self._by_inner.get_filename(fullname)) + if code is not None: + return code + return self._by_inner.get_code(fullname) + def exec_module(self, module): module.__file__ = self._by_path + if type(self._by_inner) is importlib.machinery.SourceFileLoader: + code = _by_code(self._by_inner.get_filename(module.__name__)) + if code is not None: + exec(code, module.__dict__) + return self._by_inner.exec_module(module) def __getattr__(self, name): diff --git a/crates/ty/tests/by_e2e.rs b/crates/ty/tests/by_e2e.rs index c42826333e..9bf728f67d 100644 --- a/crates/ty/tests/by_e2e.rs +++ b/crates/ty/tests/by_e2e.rs @@ -1148,6 +1148,67 @@ fn run_still_rewrites_traceback_frames_to_by_lines() { ); } +/// what a running program reads back about its own locations is the `.by` it was +/// written as: a traceback it formats itself, the place a warning is attributed to, +/// and a code object's file, which agrees with the module's `__file__` +#[test] +fn run_hands_a_program_its_own_by_locations() { + let dir = tempfile::tempdir().expect("tempdir"); + fs::write( + dir.path().join("main.by"), + "\ +import traceback +import warnings + +def boom(a: int?) -> int: + return a! + +def main(): + try: + _ = boom(None) + except RuntimeError: + print(traceback.format_exc()) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter(\"always\") + warnings.warn(\"careful\") + print(\"warned at\", caught[0].filename, caught[0].lineno) + print(\"same file\", main.__code__.co_filename == __file__) +", + ) + .unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") + .args(["run", "main"]) + .current_dir(dir.path()) + .output() + .expect("failed to spawn by"); + + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(output.status.success(), "by run failed:\n{stderr}"); + assert!( + stdout.contains("main.by\", line 9, in main") && stdout.contains("_ = boom(None)"), + "a formatted traceback should name the .by line of the call:\n{stdout}" + ); + assert!( + stdout.contains("main.by\", line 5, in boom") && stdout.contains("return a!"), + "and the .by line that raised, as it is written:\n{stdout}" + ); + assert!( + !stdout.contains(".py\"") && !stdout.contains("_force_unwrap(a)"), + "no frame may name the generated file or its text:\n{stdout}" + ); + assert!( + stdout.contains("main.by 14"), + "a warning should be attributed to its .by line:\n{stdout}" + ); + assert!( + stdout.contains("same file True"), + "a code object's file should be the module's own:\n{stdout}" + ); +} + /// a frame inside a trailing-lambda block is reported at the block's own `.by` /// line. the block's suite is hoisted into a `def` ahead of the call it hung /// off, and the map used to charge every line of that `def` — header and body @@ -2571,6 +2632,43 @@ fn reverse_paren_tuple_in_type_subscript() { ); } +/// a module the transpiler declines to lower is not a syntax error, and does not read as one: +/// the author's `.by` parses, and the report names what to rename. `invalid-syntax` is kept for +/// the failures that really are syntax, which `transpile_renders_parse_error_with_location` +/// covers +#[test] +fn a_refused_transpile_is_not_reported_as_a_syntax_error() { + let dir = tempfile::tempdir().expect("tempdir"); + let by_path = dir.path().join("clash.by"); + fs::write( + &by_path, + "from typing import Optional\n\nx = Some(3)\nprint(x!)\n", + ) + .unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") + .arg("transpile") + .arg(&by_path) + .output() + .expect("failed to spawn by"); + + assert!(!output.status.success(), "expected non-zero exit"); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("error[transpile-refused]"), + "the refusal should carry its own code:\n{stderr}" + ); + assert!( + !stderr.contains("invalid-syntax"), + "and must not read as a syntax error:\n{stderr}" + ); + assert!( + stderr.contains("`Optional`"), + "naming what to rename:\n{stderr}" + ); +} + #[test] fn transpile_renders_parse_error_with_location() { // file-based transpile should surface ty-style diagnostics on invalid @@ -2614,6 +2712,24 @@ fn transpile_renders_parse_error_with_location() { ); } +/// The lowering asks for the names it needs one at a time and the driver puts them on one +/// `from typing import …` line. The lazy-import rewrite reads that line again and re-emits the +/// names it leaves eager, which it has to do without taking the line apart. `by_transforms`' +/// own tests run with lazy imports switched off, so only a test of the command sees this. +#[test] +fn transpile_writes_one_import_line_for_the_names_it_adds() { + let out = transpile( + "def apply(f: (int) -> int, xs: list[int] = []) -> int:\n\ + \x20 return f(xs[0] if xs else 0)\n", + ); + assert!(out.contains("from typing import Any, Callable\n"), "{out}"); + assert_eq!( + out.matches("from typing import").count(), + 1, + "the names the lowering added should share one import line:\n{out}" + ); +} + #[test] fn transpile_malformed_inputs_never_panic() { // adversarial / truncated basedpython snippets must produce a clean @@ -6394,6 +6510,21 @@ fn a_built_package_runs_off_one_copy_of_the_runtime() { !one.contains("_lazy_attr(\"app._by_runtime\""), "the runtime import is eager:\n{one}" ); + // the lowering asks for helpers in two places — `cast!` during the AST rewrite, the + // lazified `import json` after it — and both sets belong on the one import line + assert_eq!( + one.matches("from app._by_runtime import").count(), + 1, + "every helper the module calls should share one runtime import:\n{one}" + ); + for helper in ["_checked_cast_pred", "_lazy_module"] { + assert!( + one.lines() + .find(|line| line.starts_with("from app._by_runtime import")) + .is_some_and(|line| line.contains(helper)), + "`{helper}` should be on that line:\n{one}" + ); + } assert_eq!( run_python( diff --git a/crates/ty/tests/mdtest_divergence.rs b/crates/ty/tests/mdtest_divergence.rs index 426427624a..c824375074 100644 --- a/crates/ty/tests/mdtest_divergence.rs +++ b/crates/ty/tests/mdtest_divergence.rs @@ -259,6 +259,13 @@ fn clean_mdtest_blocks_run() { .args(["-c", "frozendict"]) .output() .is_ok_and(|o| o.status.success()); + // `string.templatelib` arrived in 3.14, and a block importing a name from it to + // annotate with imports the module where the statement stands — the lazy-import + // polyfill keeps a name an annotation reads eager + let has_templatelib = Command::new(&python) + .args(["-c", "import string.templatelib"]) + .output() + .is_ok_and(|o| o.status.success()); let dir = mdtest_dir(); let mut files: Vec = fs::read_dir(&dir) @@ -292,6 +299,7 @@ fn clean_mdtest_blocks_run() { let unavailable: Vec<&str> = [ (!has_typing_extensions).then_some("typing_extensions"), (!has_frozendict).then_some("frozendict"), + (!has_templatelib).then_some("string.templatelib"), (!has_pydantic).then_some("pydantic"), (!has_sqlalchemy).then_some("sqlalchemy"), (!has_pytest).then_some("pytest"), diff --git a/crates/ty_ide/src/inlay_hints.rs b/crates/ty_ide/src/inlay_hints.rs index 8693f69e83..2d653371b4 100644 --- a/crates/ty_ide/src/inlay_hints.rs +++ b/crates/ty_ide/src/inlay_hints.rs @@ -1684,7 +1684,8 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { return; }; - let arguments = implicit_context_arguments(self.db, env, self.model.file(), callee, call); + let arguments = + implicit_context_arguments(self.db, env, self.model.file(), callee, call).arguments; if arguments.is_empty() { return; } @@ -12418,6 +12419,34 @@ def each(fn: (int) -> None) -> None: })); } + /// An override that repeats `_` takes the names its base gives those positions, so a + /// call is hinted with the base's names. The numbered `_` of a function that overrides + /// nothing is positional-only, and hinted with nothing. + #[test] + fn basedpython_repeated_underscore_parameter_names() { + let mut test = basedpython_inlay_hint_test( + " + class A: + def f(self, x: int = 1, y: str = 'a') -> None: ... + + class B(A): + override def f(self, _, _) -> None: ... + + def g(_: int, _: int) -> None: ... + + B().f(1, 'b') + g(1, 2) + ", + ); + + assert_snapshot!(test.inlay_hints_with_settings(&InlayHintSettings { + call_argument_names: true, + inherited_parameter_defaults: true, + inherited_parameter_types: true, + ..InlayHintSettings::none() + })); + } + /// An overload implementation takes its parameter types from the overloads /// it implements, the same way an override takes them from its base. #[test] diff --git a/crates/ty_ide/src/snapshots/ty_ide__inlay_hints__tests__basedpython_repeated_underscore_parameter_names.snap b/crates/ty_ide/src/snapshots/ty_ide__inlay_hints__tests__basedpython_repeated_underscore_parameter_names.snap new file mode 100644 index 0000000000..712d7df22f --- /dev/null +++ b/crates/ty_ide/src/snapshots/ty_ide__inlay_hints__tests__basedpython_repeated_underscore_parameter_names.snap @@ -0,0 +1,48 @@ +--- +source: crates/ty_ide/src/inlay_hints.rs +expression: "test.inlay_hints_with_settings(&InlayHintSettings\n{\n call_argument_names: true, inherited_parameter_defaults: true,\n inherited_parameter_types: true, ..InlayHintSettings::none()\n})" +--- + +class A: + def f(self, x: int = 1, y: str = 'a') -> None: ... + +class B(A): + override def f(self, _[=1][: int], _[="a"][: str]) -> None: ... + +def g(_: int, _: int) -> None: ... + +B().f([x=]1, [y=]'b') +g(1, 2) + +--------------------------------------------- +info[inlay-hint-location]: Inlay Hint Target + --> main.by:6:26 + | +6 | override def f(self, _, _) -> None: ... + | ^ +info: Source + --> main2.py:10:8 + | +10 | B().f([x=]1, [y=]'b') + | ^ + +info[inlay-hint-location]: Inlay Hint Target + --> main.by:6:29 + | +6 | override def f(self, _, _) -> None: ... + | ^ +info: Source + --> main2.py:10:15 + | +10 | B().f([x=]1, [y=]'b') + | ^ + +--------------------------------------------- +info[inlay-hint-edit]: Inlay hint edits +--> main.by:1:1 + | +9 | + - B().f(1, 'b') +10 + B().f(1, y='b') +11 | g(1, 2) + | diff --git a/crates/ty_python_semantic/resources/lint_docs/invalid-decorator-def.md b/crates/ty_python_semantic/resources/lint_docs/invalid-decorator-def.md new file mode 100644 index 0000000000..2aed9f07df --- /dev/null +++ b/crates/ty_python_semantic/resources/lint_docs/invalid-decorator-def.md @@ -0,0 +1,57 @@ +## What it does + +Checks for a `decorator def` whose parameters, or whose position, leave it with no decorator to +expand into. + +## Why is this bad? + +A `decorator def` is lowered to a pair of overloads and a runtime dispatcher, built from a first +parameter that receives the decorated function and options that each carry a default. A declaration +that does not have that shape has nothing to build from, and the lowering refuses it. Reported here, +the refusal arrives while the file is being checked rather than when it is transpiled. + +## Examples + +The first parameter is the one a decoration hands the function to, so there has to be one: + +```by +# error: [invalid-decorator-def] +decorator def logged() -> int: + return 1 +``` + +Every shape the decorator is applied in supplies that function, so the parameter never falls back to +a default: + +```by +def nothing() -> None: ... + +# error: [invalid-decorator-def] +decorator def retry(fn: (...) -> object = nothing) -> int: + return 1 +``` + +The options are what `@traced` decorates without, so each of them needs a default: + +```by +# error: [invalid-decorator-def] +decorator def traced(fn: (...) -> object, label: str) -> int: + return 1 +``` + +`decorator def` is module-scope only. A decorator written in a class body is a plain `def` that +returns a callable: + +```by +class Registry: + # error: [invalid-decorator-def] + decorator def register(fn: (...) -> object) -> int: + return 1 +``` + +The form the lowering accepts is a decorated parameter followed by options that each have a default: + +```by +decorator def traced(fn: (...) -> object, label: str = "") -> int: + return len(label) +``` diff --git a/crates/ty_python_semantic/resources/lint_docs/invalid-repeated-underscore.md b/crates/ty_python_semantic/resources/lint_docs/invalid-repeated-underscore.md new file mode 100644 index 0000000000..936ec1cdfc --- /dev/null +++ b/crates/ty_python_semantic/resources/lint_docs/invalid-repeated-underscore.md @@ -0,0 +1,57 @@ +## What it does + +Checks for a parameter list that repeats `_` in a shape the lowering has no python spelling for. + +## Why is this bad? + +Python refuses two parameters of one name, so every `_` after the first is given a name of its own. +That name is the lowering's rather than the author's, so a call reaches the parameter by position +alone: it is positional-only. Where that cannot be written, or would change a parameter the author +named, the lowering refuses the definition. Reported here, the refusal arrives while the file is +being checked rather than when it is transpiled. + +## Examples + +Python's `/` makes every parameter before it positional-only, so a named parameter ahead of a +repeated `_` would stop being reachable by keyword: + +```by +# error: [invalid-repeated-underscore] +def pair(a: int, _: int, b: int, _: int) -> int: + return a + b +``` + +Writing the `/` says so, and is accepted: + +```by +def explicit(a: int, _: int, b: int, _: int, /) -> int: + return a + b +``` + +A repeated `_` after `*` is keyword-only, and only a keyword reaches it: + +```by +# error: [invalid-repeated-underscore] +def keyword(_: int, *, _: int) -> None: ... +``` + +Python before 3.8 has no `/` at all, so a module targeting it cannot make a repeated `_` +positional-only, and every definition that needs the `/` is refused there. + +A method that overrides one takes the names the overridden method gives those positions. A name the +body reads from an enclosing scope would then read the parameter instead. Refused, the override +keeps no names a caller of the base could pass by keyword, which is reported too: + +```by +x = 1 + +class Base: + def f(self, x: int, y: int) -> int: + return x + y + +class Override(Base): + # error: [invalid-repeated-underscore] + # error: [invalid-method-override] + override def f(self, _: int, _: int) -> int: + return x +``` diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_context_parameters.md b/crates/ty_python_semantic/resources/mdtest/basedpython_context_parameters.md index abbffa54d6..dd3f3efbf3 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_context_parameters.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_context_parameters.md @@ -156,6 +156,92 @@ def g(): f() # error: [missing-context-argument] ``` +## a `context _` parameter supplies a context argument + +A `context` parameter named `_` is an ordinary name when it is the function's only parameter named +`_`. + +```by +def show(context label: str) -> str: + return label + +def relay(context _: str) -> str: + return show() + +print(relay("a")) +``` + +## a repeated `_` parameter does not supply a context argument + +A function may name several parameters `_`, but Python binds only one of them to the name `_`. Which +one a call site would read is not decided, so a `context` parameter whose value would come from a +repeated `_` is reported rather than filled from either of them. + +```by +def show(context label: str) -> str: + return label + +def relay(context _: int, context _: str) -> str: + # error: [missing-context-argument] "a repeated `_` parameter cannot supply context parameter `label` of function `show`" + return show() +``` + +This holds for every repeated `_` that matches, not only the last one, and for a `context` `_` +repeated by an ordinary parameter. + +```by +def count(context n: int) -> int: + return n + +def first(context _: int, context _: str) -> int: + # error: [missing-context-argument] "a repeated `_` parameter cannot supply context parameter `n` of function `count`" + return count() + +def mixed(_: str, context _: int) -> int: + # error: [missing-context-argument] "a repeated `_` parameter cannot supply context parameter `n` of function `count`" + return count() +``` + +## a repeated `_` parameter is not filled implicitly + +A function may name several parameters `_`, `context` ones among them. They are positional-only, so +a call passes them by position, and an implicit argument is written as a keyword, so none can be +written for one. Each such parameter the call leaves unmatched is reported, whatever is in scope. + +```by +def show(context _: int, context _: str) -> str: + return "ok" + +context n = 1 +context s = "a" + +show(2, "b") + +# error: [missing-context-argument] "repeated `_` parameter 1 of function `show` cannot be supplied implicitly" +# error: [missing-context-argument] "repeated `_` parameter 2 of function `show` cannot be supplied implicitly" +show() +``` + +A keyword `_` fills neither of them. + +```by +# error: [positional-only-parameter-as-kwarg] +# error: [missing-context-argument] "repeated `_` parameter 2 of function `show` cannot be supplied implicitly" +show(_=2) +``` + +This holds for a `context` parameter `_` that repeats the name of an ordinary one. + +```by +def mixed(_: int, context _: str) -> str: + return "ok" + +mixed(1, "b") + +# error: [missing-context-argument] "repeated `_` parameter 2 of function `mixed` cannot be supplied implicitly" +mixed(1) +``` + ## a trailing lambda block's `it` is a candidate A block binds the value its callback is called with as `it`, and nobody writes that binding. It is @@ -345,7 +431,7 @@ def f(context b: str, *args: int): ... # error: [invalid-syntax] "`*` parameter ## resolution is limited to plain functions and bound methods -The transpiler can only inject implicit arguments where it can see a single signature, so +The transpiler can only inject implicit arguments where it can see the callee's own signature, so constructors (and other indirect callables) keep the plain missing-argument behaviour and require explicit arguments. @@ -361,6 +447,175 @@ a = A(s) a.m() # ok — bound methods resolve ``` +## an overloaded callee fills a parameter every overload agrees on + +Which overload a call selects is decided by the arguments it is given, so an argument may be written +only where it is the right one for every overload at once: the same name, resolved to the same +value, and keyword-only in all of them so that whether the call already supplies it cannot depend on +which one is selected. + +```by +from typing import overload + +@overload +def f(a: int, *, context b: str) -> int: ... +@overload +def f(a: str, *, context b: str) -> str: ... +def f(a: int | str, *, context b: str) -> int | str: + return a + +context t: str = "hello" + +reveal_type(f(1)) # revealed: int +``` + +## overloads that disagree on a `context` parameter fill nothing + + + +The name means a different thing in each overload, so no one argument is right whichever is +selected. Nothing is written, and a parameter with no default is the missing argument it is. + +The call then goes without an argument nothing was ever going to write, so the report says which +overloads parted and what each of them would have done. + +```by +from typing import overload + +@overload +def f(a: int, *, context b: str) -> int: ... +@overload +def f(a: str, *, context b: int) -> str: ... +def f(a: int | str, *, context b: str | int) -> int | str: + return a + +context t: str = "hello" +context u: int = 1 + +f(1) # error: [no-matching-overload] +``` + +## an overload that leaves the `context` parameter out + + + +Declaring the parameter in only some overloads is a disagreement of the same kind: a call that +selects the overload without it would be given an argument it never asked for. + +```by +from typing import overload + +@overload +def g(a: int, *, context b: str) -> int: ... +@overload +def g(a: int, x: int) -> str: ... +def g(a: int, x: int = 0, *, context b: str = "") -> int | str: + return a + +context t: str = "hello" + +g(1) # error: [no-matching-overload] +``` + +## a positional `context` parameter of an overload set is not filled + + + +A positional slot can sit at a different index in each overload, so whether the call already fills +it is not the same question for all of them. Only a keyword-only parameter is read through an +overload set. + +The call then goes without an argument nothing was ever going to write, which on its own reads as an +ordinary failure to match. So the report names the parameter and the spelling that would have +worked. + +```by +from typing import overload + +@overload +def f(a: int, context b: str) -> int: ... +@overload +def f(a: str, context b: str) -> str: ... +def f(a: int | str, context b: str) -> int | str: + return a + +context t: str = "hello" + +f(1) # error: [no-matching-overload] +``` + +## a decoration cannot fill a `context` parameter + +`@deco` is a call — it runs `deco(g)` — but it is the one call the source writes no argument list +for, so there is nowhere to put the implicit argument. A parameter with a default would quietly take +it rather than the ambient value the declaration promised, so the decoration is reported and left as +written. The call form is where the argument can be written. + +```by +def deco(fn: (...) -> object, context b: str = "default") -> object: + return fn + +context t: str = "hello" + +# error: [missing-context-argument] "context parameter `b` cannot be filled at a decoration" +@deco +def g(): ... + +def undecorated(): ... + +def call_it() -> object: + return deco(undecorated) # ok — `t` is passed implicitly +``` + +The decorated definition is the decorator's first positional argument, so a `context` parameter +standing in that slot is filled by it like any other. + +```by +def only(context fn: object) -> object: + return fn + +@only +def h(): ... +``` + +## a decoration of a `decorator def` cannot fill one either + +A `decorator def` is declared as the pair of overloads it is applied in, and the option is unfilled +in both — so which one the decoration selects does not change the answer. + +```by +decorator def d(fn: (...) -> object, context tag: str = "default") -> object: + print("decorating with", tag) + return fn + +context t: str = "hello" + +# error: [missing-context-argument] "context parameter `tag` cannot be filled at a decoration" +@d +def g(): ... +``` + +## an overload that fills the parameter leaves the decoration alone + +Only a parameter unfilled in every overload is reported: which overload a decoration selects is not +decided here, so a report has to be true whichever it is. + +```by +from typing import overload + +@overload +def deco(fn: (...) -> object, context b: str = "default") -> object: ... +@overload +def deco(fn: int) -> object: ... +def deco(fn: object, b: str = "default") -> object: + return fn + +context t: str = "hello" + +@deco +def g(): ... +``` + ## reveal_type of the parameter inside the body The `context` prefix does not change the parameter's declared type. diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_decorator_keyword.md b/crates/ty_python_semantic/resources/mdtest/basedpython_decorator_keyword.md index 1dd7e25e0d..6c58989736 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_decorator_keyword.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_decorator_keyword.md @@ -52,6 +52,54 @@ def h() -> None: ... reveal_type(trace(h)) # revealed: int ``` +## applied with options + +A `decorator def` is applied in three shapes: bare, with empty parentheses, and with options. The +parentheses call it with the options alone, and what that returns is what decorates the function, so +every shape gives the function the decorator's declared return type. + +```by +decorator def d(fn: (int) -> None, option: bool = False) -> str: + return "on" if option else "off" + +def plain(i: int) -> None: ... + +@d +def f1(i): ... + +@d() +def f2(i): ... + +@d(option=True) +def f3(i): ... + +reveal_type(f1) # revealed: str +reveal_type(f2) # revealed: str +reveal_type(f3) # revealed: str + +reveal_type(d(plain)) # revealed: str +reveal_type(d(option=True)) # revealed: (fn: (int, /) -> None, /) -> str +``` + +An option is still checked, in either shape. + +```by +# error: [invalid-argument-type] "Expected `bool`, found `"yes"`" +# error: [dynamic-function-decorator-return] +@d(option="yes") +def f4(i): ... + +# error: [invalid-argument-type] "Expected `bool`, found `"yes"`" +d(plain, option="yes") +``` + +The options are keyword-only at the call site: one dispatcher serves every shape, and it tells them +apart by whether it was given the decorated function positionally. + +```by +d(plain, True) # error: [no-matching-overload] +``` + ## the decorated function's unannotated parameters take their declared types a decoration hands the function to the decorator, so the callable the declaration says it accepts is @@ -67,6 +115,17 @@ def f(i): reveal_type(i) # revealed: int ``` +Applying it with options declares them just as directly: the options are matched first, and what the +call returns takes the function. + +```by +decorator def e(fn: (str) -> None, option: bool = False): ... + +@e(option=True) +def g(s): + reveal_type(s) # revealed: str +``` + ## multiple options ```by @@ -84,3 +143,79 @@ def target() -> None: ... reveal_type(configure(target)) # revealed: str reveal_type(configure(target, name="hello", count=2)) # revealed: str ``` + +## a declaration with no parameter for the decorated function + +The lowering builds the dispatcher and its two overloads out of a first parameter that receives the +decorated function. A declaration with no parameters has nothing to hand the function to, and is +refused rather than transpiled into a decorator that cannot be applied. + +```by +# error: [invalid-decorator-def] "`decorator def logged` declares no parameter for the function it decorates" +decorator def logged() -> int: + return 1 +``` + +## a default on the decorated parameter + +Every shape a `decorator def` is applied in supplies the decorated function, so a default on the +parameter that receives it stands for a call that never happens. + +```by +def nothing() -> None: ... + +# error: [invalid-decorator-def] "The decorated parameter `fn` of `decorator def retry` cannot have a default" +decorator def retry(fn: (...) -> object = nothing) -> int: + return 1 +``` + +## an option with no default + +The options are what `@d` decorates *without*, so each of them has to have a default. One that does +not would leave the bare and empty-parenthesis shapes with an argument missing. + +```by +# error: [invalid-decorator-def] "Option `label` of `decorator def traced` has no default" +decorator def traced(fn: (...) -> object, label: str) -> int: + return 1 +``` + +## a variadic parameter + +Neither the decorated function nor an option arrives through `*args` or `**kwargs`: the dispatcher +tells the two call shapes apart by whether it was given the function positionally, and a variadic +absorbs both. + +```by +# error: [invalid-decorator-def] "`decorator def timed` cannot declare `*args` or `**kwargs`" +decorator def timed(fn: (...) -> object, *rest: int) -> int: + return 1 +``` + +## inside a class body + +`decorator def` is module-scope only. In a class body the overloads the lowering writes would stand +in the class's own namespace, as attributes rather than declarations, so the keyword is refused +there and a decorator written as a method is a plain `def` that returns a callable. + +```by +class Registry: + # error: [invalid-decorator-def] "`decorator def register` is only valid at module scope" + decorator def register(fn: (...) -> object) -> int: + return 1 +``` + +## a declaration the lowering refuses keeps the signature it was written with + +Nothing is expanded, so the name stands for the function the source declared. Reading it that way is +what makes the refusal above the only thing reported. + +```by +# error: [invalid-decorator-def] +decorator def logged(fn: (...) -> object, level: int) -> str: + return "x" + +def target() -> None: ... + +reveal_type(logged(target, 3)) # revealed: str +``` diff --git a/crates/ty_python_semantic/resources/mdtest/repeated_underscore.md b/crates/ty_python_semantic/resources/mdtest/repeated_underscore.md index a328f6f8ee..0d837d6ffe 100644 --- a/crates/ty_python_semantic/resources/mdtest/repeated_underscore.md +++ b/crates/ty_python_semantic/resources/mdtest/repeated_underscore.md @@ -1,9 +1,15 @@ # Repeated `_` parameters basedpython relaxes the python rule that all parameter names must be unique: when the duplicated -name is `_`, the parameters are allowed and the transpiler renames trailing occurrences (`_2`, `_3`, -...) so the lowered python is valid. references to `_` inside the body always resolve to the first -parameter +name is `_`, the parameters are allowed. Python itself refuses a repeated parameter, so the +transpiler gives every `_` after the first a name of its own (`_2`, `_3`, ...) that the module +spells nowhere. Those names are the transpiler's rather than the author's, so a call cannot spell +them: the numbered parameters are positional-only, and the signature ty checks a call against is the +one the transpiled python has. The signature shows each of them as the `_` the author wrote. +References to `_` inside the body resolve to the first parameter. + +A module that spells `_2` anywhere has its `_`s numbered past it, so the examples below that pass a +numbered name by keyword declare the function in a module of its own. ## allowed in basedpython @@ -13,9 +19,6 @@ def f(_, _): def g(_, _, _, x: int): reveal_type(x) # revealed: int - -h = lambda _, _: 1 -reveal_type(h) # revealed: (_, _) -> 1 ``` ## still rejected for non-`_` names @@ -33,3 +36,348 @@ def f(x: int, x: str) -> int: def f(_, _): return 1 ``` + +## numbered parameters are positional-only + +A `/` follows the last numbered `_`, so every parameter up to it is passed by position: + +`pairs.by`: + +```by +def pair(_: int, _: str) -> None: ... +``` + +`main.by`: + +```by +from pairs import pair + +reveal_type(pair) # revealed: def pair(_: int, _: str, /) + +pair(1, "a") +# error: [positional-only-parameter-as-kwarg] +pair(1, _2="a") +# error: [positional-only-parameter-as-kwarg] +# error: [positional-only-parameter-as-kwarg] +pair(_=1, _2="a") +``` + +## a diagnostic names a repeated `_` by its position + +Several parameters are called `_`, and every one after the first has a name in the python the author +never wrote, so a diagnostic about one says which it is by position: + +```by +def pair(_: int, _: str) -> None: ... + +# error: [missing-argument] "No argument provided for required parameter 2 (`_`) of function `pair`" +pair(1) +# error: [missing-argument] "No arguments provided for required parameters 1 (`_`), 2 (`_`)" +pair() +``` + +## a named parameter after the last `_` keeps its keyword + +The `/` goes after the last `_`, so a parameter declared after it is still reachable by name: + +`ignoring.by`: + +```by +def ignore(_: int, _: int, label: str) -> None: ... +``` + +`main.by`: + +```by +from ignoring import ignore + +reveal_type(ignore) # revealed: def ignore(_: int, _: int, /, label: str) + +ignore(1, 2, label="a") +``` + +## a variadic `_` needs no `/` + +No keyword reaches a `*_` or a `**_`, so a `_` repeated only by one of them stays as written: + +`rests.by`: + +```by +def rest(_: int, *_: int) -> None: ... +``` + +`main.by`: + +```by +from rests import rest + +reveal_type(rest) # revealed: def rest(_: int, *_: int) + +rest(_=1) +``` + +## a method's receiver + +A method's receiver is never passed by keyword, so the `/` making it positional-only changes nothing +a caller writes: + +`handlers.by`: + +```by +class Handler: + def handle(self, _: int, _: int) -> None: ... +``` + +`main.by`: + +```by +from handlers import Handler + +reveal_type(Handler.handle) # revealed: def handle(self, _: int, _: int, /) + +Handler().handle(1, 2) +``` + +## a lambda + +`lambdas.by`: + +```by +h = lambda _, _: 1 +``` + +`main.by`: + +```by +from lambdas import h + +reveal_type(h) # revealed: (_, _, /) -> 1 +``` + +## a callable type + +A callable type that names its parameters is declared in python as a protocol's `__call__`, whose +parameter list follows the same rule, so a call reaches the numbered `_` by position alone: + +`callbacks.by`: + +```by +type Callback = (_: int, _: str, name: bytes) -> None +``` + +`main.by`: + +```by +from callbacks import Callback + +def call(f: Callback) -> None: + reveal_type(f) # revealed: (_: int, _: str, /, name: bytes) -> None + f(1, "a", name=b"") + # error: [positional-only-parameter-as-kwarg] + f(1, _2="a", name=b"") +``` + +## an inline protocol's method + +A method of an inline protocol is declared as a method of the protocol class it lowers to, and its +receiver is exempt as a `def`'s is: + +`handlers.by`: + +```by +type Handler = protocol(def handle(self, _: int, _: str) -> None) +``` + +`main.by`: + +```by +from handlers import Handler + +def call(h: Handler) -> None: + reveal_type(h.handle) # revealed: (_: int, _: str, /) -> None + h.handle(1, "a") + # error: [positional-only-parameter-as-kwarg] + h.handle(1, _2="a") +``` + +## a keyword-only repeated `_` in a callable type is refused + +A parameter after `*args` is reached only by keyword, as it is in a `def`: + +```by +# error: [invalid-repeated-underscore] "a repeated `_` parameter cannot be keyword-only" +type Keyword = (int, /, _: int, *args: int, _: str) -> None +``` + +## a named parameter before the last `_` is refused + +Python's `/` makes every parameter before it positional-only. A parameter the author named would +stop being reachable by that name, which is theirs to decide, so the definition is refused: + +```by +# error: [invalid-repeated-underscore] "the repeated `_` parameters after `a` make it positional-only" +def pair(a: int, _: int, b: int, _: int) -> None: ... +``` + +A lambda is refused by the same rule: + +```by +# error: [invalid-repeated-underscore] "the repeated `_` parameters after `a` make it positional-only" +first = lambda a, _, _: a +``` + +## writing the `/` is accepted + +`explicit.by`: + +```by +def pair(a: int, _: int, b: int, _: int, /) -> None: ... +``` + +`main.by`: + +```by +from explicit import pair + +reveal_type(pair) # revealed: def pair(a: int, _: int, b: int, _: int, /) +``` + +## a `/` written before the last `_` + +A `/` the source wrote before the last `_` ends up after it: + +`moved.by`: + +```by +def pair(a: int, /, _: int, _: int) -> None: ... +``` + +`main.by`: + +```by +from moved import pair + +reveal_type(pair) # revealed: def pair(a: int, _: int, _: int, /) +``` + +## a keyword-only repeated `_` is refused + +A parameter after `*` is reached only by keyword, and a repeated `_` has no name of its own to be +reached by: + +```by +# error: [invalid-repeated-underscore] "a repeated `_` parameter cannot be keyword-only" +def keyword(_: int, *, _: int) -> None: ... +``` + +## a target without the `/` is refused + +Python 3.8 added the `/`, and before it nothing makes a parameter positional-only. A numbered `_` +there would be reached by the number the lowering gave it, so the definition is refused instead: + +```toml +[environment] +python-version = "3.7" +``` + +```by +# error: [invalid-repeated-underscore] "a repeated `_` parameter needs a `/`, which python before 3.8 does not have" +def pair(_: int, _: str) -> None: ... +``` + +## an override takes the base method's names + +A method that overrides one takes the names the overridden method gives those positions, so a call +written against the base keeps working on the override: + +```by +class Base: + def f(self, x: int, y: str) -> None: ... + +class Override(Base): + override def f(self, _: int, _: str) -> None: ... + +reveal_type(Override.f) # revealed: def f(self, x: int, y: str) + +Override().f(x=1, y="a") +base: Base = Override() +base.f(x=1, y="a") +``` + +## a positional-only base parameter stays so + +The kind comes from the base along with the name: + +```by +class Base: + def f(self, x: int, /, y: str) -> None: ... + +class Override(Base): + override def f(self, _: int, _: str) -> None: ... + +reveal_type(Override.f) # revealed: def f(self, x: int, /, y: str) + +Override().f(1, y="a") +# error: [positional-only-parameter-as-kwarg] +Override().f(x=1, y="a") +``` + +## an override with defaults + +Parameters named after the base take its defaults too, as any override that re-declares one does: + +```by +class Base: + def f(self, x: int = 1, y: str = "a") -> None: ... + +class Override(Base): + override def f(self, _: int, _: str) -> None: ... + +reveal_type(Override.f) # revealed: def f(self, x: int = 1, y: str = "a") + +Override().f() +``` + +## an override that does not line up with its base is numbered + +Where the base has no parameter at a `_`'s position, none of the `_`s take a name from it. The +override is then an invalid one, and is reported as such: + +`overrides.by`: + +```by +class Base: + def f(self, x: int) -> None: ... + +class Override(Base): + # error: [invalid-method-override] + override def f(self, _: int, _: int) -> None: ... +``` + +`main.by`: + +```by +from overrides import Override + +reveal_type(Override.f) # revealed: def f(self, _: int, _: int, /) +``` + +## an inherited name the body reads is refused + +A parameter named after the base would shadow a name the body reads from an enclosing scope. +Refused, the override keeps no names a caller of the base could pass by keyword, which is reported +too: + +```by +x = 1 + +class Base: + def f(self, x: int, y: int) -> int: + return x + +class Override(Base): + # error: [invalid-repeated-underscore] "a repeated `_` parameter named `x` shadows a `x` the body reads" + # error: [invalid-method-override] + override def f(self, _: int, _: int) -> int: + return x +``` diff --git "a/crates/ty_python_semantic/resources/mdtest/snapshots/basedpython_context_\342\200\246_-_basedpython___context\342\200\246_-_a_positional_`contex\342\200\246_(7854bce3d4fdb653).snap" "b/crates/ty_python_semantic/resources/mdtest/snapshots/basedpython_context_\342\200\246_-_basedpython___context\342\200\246_-_a_positional_`contex\342\200\246_(7854bce3d4fdb653).snap" new file mode 100644 index 0000000000..a37abc8712 --- /dev/null +++ "b/crates/ty_python_semantic/resources/mdtest/snapshots/basedpython_context_\342\200\246_-_basedpython___context\342\200\246_-_a_positional_`contex\342\200\246_(7854bce3d4fdb653).snap" @@ -0,0 +1,55 @@ +--- +source: crates/mdtest/src/lib.rs +expression: snapshot +--- + +--- +mdtest name: basedpython_context_parameters.md - basedpython: context parameters - a positional `context` parameter of an overload set is not filled +mdtest path: crates/ty_python_semantic/resources/mdtest/basedpython_context_parameters.md +--- + +# Source files + +## mdtest_snippet.by + +``` + 1 | from typing import overload + 2 | + 3 | @overload + 4 | def f(a: int, context b: str) -> int: ... + 5 | @overload + 6 | def f(a: str, context b: str) -> str: ... + 7 | def f(a: int | str, context b: str) -> int | str: + 8 | return a + 9 | +10 | context t: str = "hello" +11 | +12 | f(1) # error: [no-matching-overload] +``` + +# Diagnostics + +``` +error[no-matching-overload]: No overload of function `f` matches arguments + --> src/mdtest_snippet.by:12:1 + | +12 | f(1) # error: [no-matching-overload] + | ^^^^ +info: `b` is a `context` parameter, and an overload set is given one implicitly only where it is keyword-only in every overload +info: Declare `b` after a `*` in every overload, or pass it explicitly +info: First overload defined here + --> src/mdtest_snippet.by:3:1 + | +3 | / @overload +4 | | def f(a: int, context b: str) -> int: ... + | |_________________________________________^ First overload defined here +info: Possible overloads for function `f`: +info: (a: int, b: str) -> int +info: (a: str, b: str) -> str +info: Overload implementation defined here + --> src/mdtest_snippet.by:7:5 + | +7 | def f(a: int | str, context b: str) -> int | str: + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``` diff --git "a/crates/ty_python_semantic/resources/mdtest/snapshots/basedpython_context_\342\200\246_-_basedpython___context\342\200\246_-_an_overload_that_lea\342\200\246_(39d2635c95b04ae2).snap" "b/crates/ty_python_semantic/resources/mdtest/snapshots/basedpython_context_\342\200\246_-_basedpython___context\342\200\246_-_an_overload_that_lea\342\200\246_(39d2635c95b04ae2).snap" new file mode 100644 index 0000000000..45f3035152 --- /dev/null +++ "b/crates/ty_python_semantic/resources/mdtest/snapshots/basedpython_context_\342\200\246_-_basedpython___context\342\200\246_-_an_overload_that_lea\342\200\246_(39d2635c95b04ae2).snap" @@ -0,0 +1,56 @@ +--- +source: crates/mdtest/src/lib.rs +expression: snapshot +--- + +--- +mdtest name: basedpython_context_parameters.md - basedpython: context parameters - an overload that leaves the `context` parameter out +mdtest path: crates/ty_python_semantic/resources/mdtest/basedpython_context_parameters.md +--- + +# Source files + +## mdtest_snippet.by + +``` + 1 | from typing import overload + 2 | + 3 | @overload + 4 | def g(a: int, *, context b: str) -> int: ... + 5 | @overload + 6 | def g(a: int, x: int) -> str: ... + 7 | def g(a: int, x: int = 0, *, context b: str = "") -> int | str: + 8 | return a + 9 | +10 | context t: str = "hello" +11 | +12 | g(1) # error: [no-matching-overload] +``` + +# Diagnostics + +``` +error[no-matching-overload]: No overload of function `g` matches arguments + --> src/mdtest_snippet.by:12:1 + | +12 | g(1) # error: [no-matching-overload] + | ^^^^ +info: `b` is a `context` parameter, and an overload set is given one implicitly only where every overload would be given the same argument +info: overload 1 would take `t`; overload 2 does not declare `b` +info: Pass `b` explicitly +info: First overload defined here + --> src/mdtest_snippet.by:3:1 + | +3 | / @overload +4 | | def g(a: int, *, context b: str) -> int: ... + | |____________________________________________^ First overload defined here +info: Possible overloads for function `g`: +info: (a: int, *, b: str) -> int +info: (a: int, x: int) -> str +info: Overload implementation defined here + --> src/mdtest_snippet.by:7:5 + | +7 | def g(a: int, x: int = 0, *, context b: str = "") -> int | str: + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``` diff --git "a/crates/ty_python_semantic/resources/mdtest/snapshots/basedpython_context_\342\200\246_-_basedpython___context\342\200\246_-_overloads_that_disag\342\200\246_(420a7d0ca57c17a8).snap" "b/crates/ty_python_semantic/resources/mdtest/snapshots/basedpython_context_\342\200\246_-_basedpython___context\342\200\246_-_overloads_that_disag\342\200\246_(420a7d0ca57c17a8).snap" new file mode 100644 index 0000000000..23af2591ac --- /dev/null +++ "b/crates/ty_python_semantic/resources/mdtest/snapshots/basedpython_context_\342\200\246_-_basedpython___context\342\200\246_-_overloads_that_disag\342\200\246_(420a7d0ca57c17a8).snap" @@ -0,0 +1,57 @@ +--- +source: crates/mdtest/src/lib.rs +expression: snapshot +--- + +--- +mdtest name: basedpython_context_parameters.md - basedpython: context parameters - overloads that disagree on a `context` parameter fill nothing +mdtest path: crates/ty_python_semantic/resources/mdtest/basedpython_context_parameters.md +--- + +# Source files + +## mdtest_snippet.by + +``` + 1 | from typing import overload + 2 | + 3 | @overload + 4 | def f(a: int, *, context b: str) -> int: ... + 5 | @overload + 6 | def f(a: str, *, context b: int) -> str: ... + 7 | def f(a: int | str, *, context b: str | int) -> int | str: + 8 | return a + 9 | +10 | context t: str = "hello" +11 | context u: int = 1 +12 | +13 | f(1) # error: [no-matching-overload] +``` + +# Diagnostics + +``` +error[no-matching-overload]: No overload of function `f` matches arguments + --> src/mdtest_snippet.by:13:1 + | +13 | f(1) # error: [no-matching-overload] + | ^^^^ +info: `b` is a `context` parameter, and an overload set is given one implicitly only where every overload would be given the same argument +info: overload 1 would take `t`; overload 2 would take `u` +info: Pass `b` explicitly +info: First overload defined here + --> src/mdtest_snippet.by:3:1 + | +3 | / @overload +4 | | def f(a: int, *, context b: str) -> int: ... + | |____________________________________________^ First overload defined here +info: Possible overloads for function `f`: +info: (a: int, *, b: str) -> int +info: (a: str, *, b: int) -> str +info: Overload implementation defined here + --> src/mdtest_snippet.by:7:5 + | +7 | def f(a: int | str, *, context b: str | int) -> int | str: + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``` diff --git a/crates/ty_python_semantic/src/semantic_model.rs b/crates/ty_python_semantic/src/semantic_model.rs index e9ac85e0da..0ca4100f4a 100644 --- a/crates/ty_python_semantic/src/semantic_model.rs +++ b/crates/ty_python_semantic/src/semantic_model.rs @@ -330,14 +330,20 @@ impl<'db> SemanticModel<'db> { } /// basedpython: the names this file imports that must be bound to the real - /// object rather than to a lazy proxy — the exception classes. + /// object rather than to a lazy proxy — the exception classes and the special + /// forms. /// - /// `except` is the one place cpython refuses a stand-in: it checks that what - /// it catches is a class inheriting `BaseException`, and never consults + /// `except` is one place cpython refuses a stand-in: it checks that what it + /// catches is a class inheriting `BaseException`, and never consults /// `__instancecheck__`. A proxy there raises `TypeError` from the handler — /// the line least likely to be covered by a happy-path test — so an /// exception class is imported eagerly and the proxy never stands where it /// cannot work. + /// + /// A special form is the other. What reads one tells it apart by identity: + /// `dataclasses` decides that a bare `ClassVar` annotation is not a field by + /// asking whether it *is* `typing.ClassVar`, and a checker reading the emitted + /// python rejects `ClassVar[int]` once `ClassVar` names a variable. pub fn eagerly_imported_names(&self) -> Vec { let db = self.db; let env = self.program_environment(); @@ -345,14 +351,18 @@ impl<'db> SemanticModel<'db> { let mut names = Vec::new(); let mut collect = |import: &ast::StmtImportFrom| { for alias in &import.names { - let Some(Type::ClassLiteral(class)) = alias.inferred_type(self) else { - continue; + let eager = match alias.inferred_type(self) { + Some(Type::SpecialForm(_)) => true, + Some(Type::ClassLiteral(class)) => { + Type::instance(db, &env, class.default_specialization(db)).is_assignable_to( + db, + &env, + crate::types::KnownClass::BaseException.to_instance(db, &env), + ) + } + _ => false, }; - if Type::instance(db, &env, class.default_specialization(db)).is_assignable_to( - db, - &env, - crate::types::KnownClass::BaseException.to_instance(db, &env), - ) { + if eager { let bound = alias.asname.as_ref().unwrap_or(&alias.name); names.push(bound.id.to_string()); } diff --git a/crates/ty_python_semantic/src/types.rs b/crates/ty_python_semantic/src/types.rs index 976729315d..c72682cec6 100644 --- a/crates/ty_python_semantic/src/types.rs +++ b/crates/ty_python_semantic/src/types.rs @@ -98,6 +98,7 @@ pub use crate::types::display::{DisplaySettings, SourceSpelling, TypeDetail, Typ pub(crate) use crate::types::enums::basedpython_is_plain_value; pub(crate) use crate::types::enums::{EnumClassLiteral, EnumComplementType, enum_metadata}; pub(crate) use crate::types::equality::{ComparisonSoundnessPolicy, equality_truthiness}; +pub use crate::types::function::declared_return_type; use crate::types::function::{ DataclassTransformerFlags, DataclassTransformerParams, FunctionDecorators, FunctionSpans, FunctionType, KnownFunction, OverloadLiteral, @@ -229,6 +230,7 @@ mod regex; pub(crate) mod reified_infer; pub(crate) mod relation; mod relation_error; +pub mod repeated_underscore; mod restricted; mod safe_variance; mod set_theoretic; diff --git a/crates/ty_python_semantic/src/types/call/bind.rs b/crates/ty_python_semantic/src/types/call/bind.rs index 88ff6add30..c6274514dd 100644 --- a/crates/ty_python_semantic/src/types/call/bind.rs +++ b/crates/ty_python_semantic/src/types/call/bind.rs @@ -38,7 +38,10 @@ use crate::types::constraints::{ Solutions, }; use crate::types::context::LintDiagnosticGuardBuilder; -use crate::types::context_params::{ContextResolution, resolve_context_argument}; +use crate::types::context_params::{ + ContextDisagreement, ContextResolution, overload_fillable_context_parameters, + resolve_context_argument, +}; use crate::types::dedicated::django; use crate::types::dedicated::pydantic::{self, ConfigBoolean}; use crate::types::diagnostic::{ @@ -1086,10 +1089,11 @@ impl<'db> Bindings<'db> { /// basedpython: after parameter matching, fill unmatched `context` /// parameters from the `context` declarations visible at the call site, /// rewriting each binding's missing-argument errors. must run before - /// `check_types` so a fully resolved call binds cleanly. overloaded - /// callables are skipped — the transpiler's injection is limited to - /// single-signature callees (`single_signature`), and checking must not - /// accept a call the lowering cannot complete + /// `check_types` so a fully resolved call binds cleanly. + /// + /// An overloaded callable is filled only where every one of its overloads agrees, since + /// checking must not accept a call the lowering cannot complete and the lowering writes + /// only what they agree on — see [`overload_fillable_context_parameters`] pub(crate) fn resolve_context_arguments( &mut self, db: &'db dyn Db, @@ -1098,10 +1102,32 @@ impl<'db> Bindings<'db> { call_offset: ruff_text_size::TextSize, ) { for callable_binding in self.iter_flat_mut() { - let [binding] = callable_binding.overloads.as_mut_slice() else { - continue; - }; - binding.resolve_context_arguments(db, env, scope, call_offset); + let signatures: Vec<_> = callable_binding + .overloads + .iter() + .map(|binding| binding.signature.clone()) + .collect(); + let agreed = overload_fillable_context_parameters( + db, + env, + scope, + call_offset, + signatures.as_slice(), + ); + for binding in &mut callable_binding.overloads { + binding.resolve_context_arguments( + db, + env, + scope, + call_offset, + agreed.as_ref().map(|agreed| &agreed.fillable), + ); + } + // kept for the report: a parameter the overloads could not agree on is one the + // call was never going to be given, and nothing else says so + callable_binding.context_disagreements = agreed + .map(|agreed| agreed.disagreements) + .unwrap_or_default(); } } @@ -3664,6 +3690,7 @@ impl<'db> From> for Bindings<'db> { overload_call_result: None, matching_overload_before_type_checking: None, overloads: smallvec_inline![from], + context_disagreements: Vec::new(), }; Bindings { callable_type, @@ -3741,6 +3768,10 @@ pub(crate) struct CallableBinding<'db> { /// By using `SmallVec`, we avoid an extra heap allocation for the common case of a /// non-overloaded callable. overloads: SmallVec<[Binding<'db>; 1]>, + + /// basedpython: the keyword-only `context` parameters this overload set could not fill + /// implicitly because its overloads do not agree on them. Empty for everything else. + context_disagreements: Vec, } #[derive(Copy, Clone)] @@ -3801,6 +3832,7 @@ impl<'db> CallableBinding<'db> { overload_call_result: None, matching_overload_before_type_checking: None, overloads, + context_disagreements: Vec::new(), } } @@ -3813,6 +3845,7 @@ impl<'db> CallableBinding<'db> { overload_call_result: None, matching_overload_before_type_checking: None, overloads: smallvec![], + context_disagreements: Vec::new(), } } @@ -4052,6 +4085,93 @@ impl<'db> CallableBinding<'db> { }) } + /// basedpython: the `context` parameters this call was reported missing that are the ones + /// an overload set never writes an argument for. + /// + /// A `context` parameter is filled through an overload set only where every overload agrees + /// on it, and agreement needs the parameter to be keyword-only: a keyword names the same + /// parameter whichever overload is selected, where a positional slot can sit at a different + /// index in each — so whether the call already supplies it is a different question for each + /// overload, and no one argument is right for all of them. + /// + /// Left as it is, the call fails for want of an argument that was never going to be written, + /// and "no overload matches arguments" gives an author no way to see why. So the name is + /// reported alongside it. + fn unfilled_positional_context_parameters(&self) -> Vec { + let mut names: Vec = Vec::new(); + for binding in &self.overloads { + let parameters = binding.signature.parameters(); + for error in &binding.errors { + let BindingError::MissingArguments { + parameters: missing, + .. + } = error + else { + continue; + }; + for missing in &missing.0 { + let Some(parameter) = parameters.get(missing.signature_parameter_index) else { + continue; + }; + if !parameter.is_context() || parameter.is_keyword_only() { + continue; + } + let Some(name) = parameter.name() else { + continue; + }; + if !names.contains(name) { + names.push(name.clone()); + } + } + } + } + names + } + + /// basedpython: the `context` parameters this call was reported missing that its overloads + /// could not agree on. + /// + /// A keyword-only `context` parameter is filled through an overload set only where every + /// overload would be given the same argument for it — the name has to mean the same thing in + /// all of them, since which overload the call selects is decided by its arguments and not + /// here. Where they part, nothing is written, and the call goes without an argument that was + /// never going to be there. So the report says which overloads parted and over what. + fn unagreed_context_parameters(&self) -> Vec<&ContextDisagreement> { + if self.context_disagreements.is_empty() { + return Vec::new(); + } + let mut missing_names: Vec<&Name> = Vec::new(); + for binding in &self.overloads { + let parameters = binding.signature.parameters(); + for error in &binding.errors { + let BindingError::MissingArguments { + parameters: missing, + .. + } = error + else { + continue; + }; + for missing in &missing.0 { + let Some(parameter) = parameters.get(missing.signature_parameter_index) else { + continue; + }; + if !parameter.is_context() || !parameter.is_keyword_only() { + continue; + } + if let Some(name) = parameter.name() + && !missing_names.contains(&name) + { + missing_names.push(name); + } + } + } + } + self.context_disagreements + .iter() + .filter(|disagreement| missing_names.contains(&&disagreement.parameter)) + .collect() + } + /// Returns the distinct source overload indexes that should be shown in diagnostics. /// /// Bound method overloads preserve their source indexes after receiver filtering. Receiver @@ -5160,6 +5280,35 @@ impl<'db> CallableBinding<'db> { )); } + for parameter in self.unfilled_positional_context_parameters() { + diag.info(format_args!( + "`{parameter}` is a `context` parameter, and an overload set is given one \ + implicitly only where it is keyword-only in every overload" + )); + diag.info(format_args!( + "Declare `{parameter}` after a `*` in every overload, or pass it explicitly" + )); + } + + for disagreement in self.unagreed_context_parameters() { + let parameter = &disagreement.parameter; + diag.info(format_args!( + "`{parameter}` is a `context` parameter, and an overload set is given one \ + implicitly only where every overload would be given the same argument" + )); + let parted = disagreement + .per_overload + .iter() + .enumerate() + .map(|(index, choice)| { + format!("overload {} {}", index + 1, choice.describe(parameter)) + }) + .collect::>() + .join("; "); + diag.info(format_args!("{parted}")); + diag.info(format_args!("Pass `{parameter}` explicitly")); + } + if let Some((kind, function)) = function_type_and_kind { let (overloads, implementation) = function.overloads_and_implementation(context.db()); @@ -9252,6 +9401,7 @@ impl<'db> Binding<'db> { env: &ProgramEnvironment<'db>, scope: ScopeId<'db>, call_offset: ruff_text_size::TextSize, + fillable: Option<&Vec>, ) { let parameters = self.signature.parameters(); if !parameters.iter().any(Parameter::is_context) { @@ -9274,13 +9424,15 @@ impl<'db> Binding<'db> { if !parameter.is_context() { return true; } - match resolve_context_argument( - db, - env, - scope, - call_offset, - parameter.annotated_type(), - ) { + // an overload set fills only what all of its overloads agree on; a parameter + // outside that keeps its missing-argument error, because nothing will be + // written for it + if fillable.is_some_and(|fillable| { + parameter.name().is_none_or(|name| !fillable.contains(name)) + }) { + return true; + } + match resolve_context_argument(db, env, scope, call_offset, parameters, parameter) { ContextResolution::Resolved { .. } => false, ContextResolution::NotFound => { context_errors.push(BindingError::NoContextArgument { @@ -9295,6 +9447,18 @@ impl<'db> Binding<'db> { }); false } + ContextResolution::RepeatedUnderscore => { + context_errors.push(BindingError::RepeatedUnderscoreContextArgument { + parameter: parameter_context.clone(), + }); + false + } + ContextResolution::RepeatedUnderscoreParameter => { + context_errors.push(BindingError::RepeatedUnderscoreContextParameter { + parameter: parameter_context.clone(), + }); + false + } } }); } @@ -9868,7 +10032,9 @@ impl ParameterContext { signature_parameter_index: index, source_parameter_index: parameter.source_parameter_index(), is_receiver: parameter.is_receiver(), - positional, + // basedpython: a repeated `_` is one of several of that name, so its position + // is what tells it apart + positional: positional || parameter.is_repeated_underscore(), } } } @@ -10021,6 +10187,16 @@ pub(crate) enum BindingError<'db> { parameter: ParameterContext, candidates: Vec, }, + /// basedpython: a `context` parameter is unmatched and the value in scope + /// that would fill it is a `_` parameter its function repeats. + RepeatedUnderscoreContextArgument { + parameter: ParameterContext, + }, + /// basedpython: a `context` parameter is unmatched and is a `_` its function + /// repeats, which no implicit argument can be written for. + RepeatedUnderscoreContextParameter { + parameter: ParameterContext, + }, /// A call argument can't be matched to any parameter. UnknownArgument { argument_name: ast::name::Name, @@ -10200,6 +10376,8 @@ impl BindingError<'_> { | BindingError::MissingArguments { .. } | BindingError::NoContextArgument { .. } | BindingError::AmbiguousContextArgument { .. } + | BindingError::RepeatedUnderscoreContextArgument { .. } + | BindingError::RepeatedUnderscoreContextParameter { .. } | BindingError::UnmatchedOverload | BindingError::PropertyHasNoGetter(..) | BindingError::PropertyHasNoSetter(..) @@ -10277,6 +10455,8 @@ impl<'db> BindingError<'db> { | Self::MissingArguments { .. } | Self::NoContextArgument { .. } | Self::AmbiguousContextArgument { .. } + | Self::RepeatedUnderscoreContextArgument { .. } + | Self::RepeatedUnderscoreContextParameter { .. } | Self::UnknownArgument { .. } | Self::UnknownKeywordVariadicArgument { .. } | Self::PositionalOnlyParameterAsKwarg { .. } @@ -10680,6 +10860,45 @@ impl<'db> BindingError<'db> { } } + Self::RepeatedUnderscoreContextArgument { parameter } => { + let range = all_arguments_range(node); + if let Some(builder) = context.report_lint(&MISSING_CONTEXT_ARGUMENT, range) { + let mut diag = builder.into_diagnostic(format_args!( + "a repeated `_` parameter cannot supply context parameter {parameter}{}", + callable_description + .map(|description| format!(" of {description}")) + .unwrap_or_default() + )); + diag.sub(SubDiagnostic::new( + SubDiagnosticSeverity::Info, + format_args!( + "python binds only one of the parameters named `_`; give the one \ + that supplies it a name of its own" + ), + )); + } + } + + Self::RepeatedUnderscoreContextParameter { parameter } => { + let range = all_arguments_range(node); + if let Some(builder) = context.report_lint(&MISSING_CONTEXT_ARGUMENT, range) { + let mut diag = builder.into_diagnostic(format_args!( + "repeated `_` parameter {}{} cannot be supplied implicitly", + parameter.signature_parameter_index + 1, + callable_description + .map(|description| format!(" of {description}")) + .unwrap_or_default() + )); + diag.sub(SubDiagnostic::new( + SubDiagnosticSeverity::Info, + format_args!( + "which of the parameters named `_` a keyword names is not decided; \ + pass the argument positionally, or give the parameter a name of its own" + ), + )); + } + } + Self::UnknownArgument { argument_name, argument_index, diff --git a/crates/ty_python_semantic/src/types/context_params.rs b/crates/ty_python_semantic/src/types/context_params.rs index 79ec4d02d4..b517715293 100644 --- a/crates/ty_python_semantic/src/types/context_params.rs +++ b/crates/ty_python_semantic/src/types/context_params.rs @@ -30,13 +30,14 @@ use ruff_python_ast as ast; use ruff_python_ast::name::Name; use ruff_text_size::{Ranged, TextRange, TextSize}; use ty_python_core::definition::Definition; +use ty_python_core::place::ScopedPlaceId; use ty_python_core::scope::{NodeWithScopeKind, ScopeId, ScopeKind}; use ty_python_core::{place_table, semantic_index}; use crate::Db; use crate::types::ProgramEnvironment; use crate::types::receivers::{ImplicitReceiverName, implicit_receiver_name}; -use crate::types::soundness::single_signature; +use crate::types::signatures::{Parameter, Parameters, Signature}; use crate::types::trailing_lambda::{enclosing_block_callee, trailing_lambda_it_type}; use crate::types::{Type, binding_type}; @@ -53,6 +54,15 @@ pub(crate) enum ContextResolution<'db> { NotFound, /// several candidates in the winning scope match, in source order Ambiguous(Vec), + /// a candidate that matches in the winning scope is a `_` parameter its + /// function repeats. python binds only one of those parameters to the name, + /// and which one a read of `_` means is not decided, so the value is refused + /// rather than guessed + RepeatedUnderscore, + /// the parameter being filled is a `_` its function repeats. which of those + /// parameters a keyword `_` names is not decided, so no argument can be + /// written for it + RepeatedUnderscoreParameter, } /// a value that can fill a `context` parameter, found in one scope @@ -64,6 +74,8 @@ struct Candidate<'db> { /// the body runs, and has no value expression a call could sit inside range: Option, binding: CandidateBinding<'db>, + /// whether this is a `_` parameter of a function that has several + is_repeated_underscore: bool, } /// where a candidate's type comes from, and how a call site can name it @@ -90,15 +102,20 @@ impl<'db> CandidateBinding<'db> { } } -/// resolve the implicit argument for one unmatched `context` parameter of a -/// call at `call_offset` inside `scope` +/// resolve the implicit argument for `parameter`, an unmatched `context` +/// parameter among `parameters`, of a call at `call_offset` inside `scope` pub(crate) fn resolve_context_argument<'db>( db: &'db dyn Db, env: &ProgramEnvironment<'db>, scope: ScopeId<'db>, call_offset: TextSize, - parameter_ty: Type<'db>, + parameters: &Parameters<'db>, + parameter: &Parameter<'db>, ) -> ContextResolution<'db> { + if is_repeated_underscore(db, parameters, parameter) { + return ContextResolution::RepeatedUnderscoreParameter; + } + let parameter_ty = parameter.annotated_type(); let file = scope.file(db); let index = semantic_index(db, db.program_file(file)); let module = parsed_module(db, db.program_file(file).python_file(db)).load(db); @@ -144,10 +161,15 @@ pub(crate) fn resolve_context_argument<'db>( }) }); - // a name redeclared later shadows its earlier declaration + // a name redeclared later shadows its earlier declaration. a repeated `_` + // parameter is not a redeclaration: python binds only one of them to the + // name, and which one is not decided, so each stays a candidate candidates.reverse(); let mut seen = Vec::new(); candidates.retain(|candidate| { + if candidate.is_repeated_underscore { + return true; + } let fresh = !seen.contains(&candidate.name); if fresh { seen.push(candidate.name.clone()); @@ -156,27 +178,36 @@ pub(crate) fn resolve_context_argument<'db>( }); candidates.reverse(); - let matching: Vec<(Name, Type<'db>, CandidateBinding<'db>)> = candidates + let matching: Vec<(Candidate<'db>, Type<'db>)> = candidates .into_iter() .filter_map(|candidate| { let ty = candidate.binding.ty(db); - ty.is_assignable_to(db, env, parameter_ty).then_some(( - candidate.name, - ty, - candidate.binding, - )) + ty.is_assignable_to(db, env, parameter_ty) + .then_some((candidate, ty)) }) .collect(); - match matching.len() { - 0 => {} - 1 => { - let (name, ty, binding) = matching.into_iter().next().expect("length checked"); - return ContextResolution::Resolved { name, ty, binding }; + if matching + .iter() + .any(|(candidate, _)| candidate.is_repeated_underscore) + { + return ContextResolution::RepeatedUnderscore; + } + match <[_; 1]>::try_from(matching) { + Ok([(candidate, ty)]) => { + return ContextResolution::Resolved { + name: candidate.name, + ty, + binding: candidate.binding, + }; } - _ => { + Err(matching) if matching.is_empty() => {} + Err(matching) => { return ContextResolution::Ambiguous( - matching.into_iter().map(|(name, _, _)| name).collect(), + matching + .into_iter() + .map(|(candidate, _)| candidate.name) + .collect(), ); } } @@ -187,8 +218,43 @@ pub(crate) fn resolve_context_argument<'db>( ContextResolution::NotFound } +/// whether `parameter` is one of several `parameters` the source names `_`, and no keyword +/// but an ambiguous `_` reaches it +/// +/// the lowering names such a parameter and makes it positional-only, so the signature does +/// not call it `_` any more. one that takes its name from the method it overrides is +/// reached by that name, and is filled like any other +fn is_repeated_underscore<'db>( + db: &'db dyn Db, + parameters: &Parameters<'db>, + parameter: &Parameter<'db>, +) -> bool { + let is_underscore = + |parameter: &Parameter<'db>| written_name(db, parameter).is_some_and(|name| name == "_"); + is_underscore(parameter) + && parameters + .iter() + .filter(|other| is_underscore(other)) + .count() + > 1 + && parameter.keyword_name().is_none_or(|name| name == "_") +} + +/// the name `parameter` is written with, which the signature need not share +fn written_name<'db>(db: &'db dyn Db, parameter: &Parameter<'db>) -> Option { + match parameter.definition() { + Some(definition) if let ScopedPlaceId::Symbol(symbol) = definition.place(db) => Some( + place_table(db, definition.scope(db)) + .symbol(symbol) + .name() + .clone(), + ), + _ => parameter.name().cloned(), + } +} + /// one `context` parameter of a call site and the value filling it -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq, Eq)] pub struct ImplicitContextArgument { /// the `context` parameter left unmatched by the explicit arguments pub parameter: Name, @@ -209,39 +275,231 @@ pub struct ImplicitContextArgument { pub is_module_private: bool, } +/// the `context` arguments a call site leaves implicit +#[derive(Debug, Clone, Default)] +pub struct ImplicitContextArguments { + /// the arguments the transpiler must append to the call, in parameter order + pub arguments: Vec, + /// the `context` parameters whose value would come from a repeated `_` + /// parameter, which checking reports and the transpiler must refuse. which of + /// the repeated parameters a read of `_` means is not decided, so no argument + /// can be written for them + pub from_repeated_underscore: Vec, + /// whether a `context` parameter left to be filled is a `_` the callee + /// repeats, which checking reports and the transpiler must refuse. which of + /// those parameters a keyword `_` names is not decided, so no argument can be + /// written for it + pub for_repeated_underscore: bool, +} + +impl ImplicitContextArguments { + /// whether no `context` parameter of the call is decided by what is in scope: + /// nothing is filled, and nothing is refused for coming from a repeated `_` + pub fn is_empty(&self) -> bool { + self.arguments.is_empty() + && self.from_repeated_underscore.is_empty() + && !self.for_repeated_underscore + } +} + /// the implicit arguments the transpiler must append to `call`: for each /// `context` parameter of `callee` that no explicit argument matches, the /// in-scope declaration that fills it, in parameter order. parameters that /// fail to resolve are skipped — checking already reported them. calls that /// use `*` / `**` unpacking are skipped entirely: whether the unpacking covers /// a parameter is not knowable statically +/// +/// An overloaded callee is read through its first overload, restricted to the parameters +/// `overload_fillable_context_parameters` says every overload agrees on. Which overload a +/// call selects is decided elsewhere, so only an argument that is the right one for each of +/// them separately may be written. A parameter left out falls back to the default it +/// declares, which is what it does when no candidate is in scope at all. pub fn implicit_context_arguments<'db>( db: &'db dyn Db, env: &ProgramEnvironment<'db>, file: File, callee: Type<'db>, call: &ast::ExprCall, -) -> Vec { +) -> ImplicitContextArguments { let has_unpacking = call.arguments.args.iter().any(ast::Expr::is_starred_expr) || call.arguments.keywords.iter().any(|kw| kw.arg.is_none()); if has_unpacking { - return Vec::new(); + return ImplicitContextArguments::default(); } - let Some(signature) = single_signature(db, callee) else { - return Vec::new(); + let signatures = callee_signatures(db, callee); + let Some(first) = signatures.first() else { + return ImplicitContextArguments::default(); }; - let parameters = signature.parameters(); let index = semantic_index(db, db.program_file(file)); let Some(file_scope_id) = index.try_expression_scope_id(&ast::ExprRef::from(call)) else { - return Vec::new(); + return ImplicitContextArguments::default(); }; let scope = file_scope_id.to_scope_id(db, db.program_file(file)); + let fillable = + overload_fillable_context_parameters(db, env, scope, call.range().start(), &signatures); + implicit_context_arguments_for( + db, + env, + file, + scope, + first.parameters(), + fillable.as_ref().map(|agreed| &agreed.fillable), + call, + ) +} + +/// the signatures `callee` may be called through, in declaration order +fn callee_signatures<'db>(db: &'db dyn Db, callee: Type<'db>) -> Box<[Signature<'db>]> { + match callee { + Type::FunctionLiteral(function) => { + function.signature(db).overloads.iter().cloned().collect() + } + Type::BoundMethod(method) => method + .bound_signatures(db) + .overloads + .iter() + .cloned() + .collect(), + _ => Box::default(), + } +} + +/// the `context` parameters of an overload set that a call site may fill, or `None` when +/// `signatures` is a single signature and the question does not arise. +/// +/// A call selects one overload, and which one is decided by the arguments it is given — not +/// here. So a parameter may be filled only where doing so is right for every overload at +/// once: the name has to mean the same thing in all of them, and it has to be one the call +/// either gives or does not give in all of them alike. +/// +/// That second half is what restricts this to **keyword-only** parameters. A keyword names +/// the same parameter in every overload, so whether the call already supplies it cannot +/// depend on which one is selected; a positional slot can sit at a different index in each, +/// and then an argument written for one of them is an argument the others never asked for. +/// A `decorator def`'s options are keyword-only, which is the shape this is here for. +pub(crate) fn overload_fillable_context_parameters<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + scope: ScopeId<'db>, + call_offset: TextSize, + signatures: &[Signature<'db>], +) -> Option { + let (first, rest) = signatures.split_first()?; + if rest.is_empty() { + return None; + } + let resolutions = |signature: &Signature<'db>| { + let parameters = signature.parameters(); + parameters + .iter() + .filter(|parameter| parameter.is_context() && parameter.is_keyword_only()) + .filter_map(|parameter| { + let name = parameter.name()?.clone(); + let resolution = + resolve_context_argument(db, env, scope, call_offset, parameters, parameter); + Some((name, resolution)) + }) + .collect::>() + }; + let per_overload: Vec<_> = signatures.iter().map(resolutions).collect(); + let mut agreed = resolutions(first); + for signature in rest { + let other = resolutions(signature); + agreed.retain(|entry| other.contains(entry)); + } + let fillable: Vec = agreed.into_iter().map(|(name, _)| name).collect(); + + // every keyword-only `context` parameter any overload declares, in the order they are + // first written. the ones that are not fillable are exactly the ones the overloads did + // not agree on, and an author has no way to see that from "no overload matches arguments" + let mut disagreements: Vec = Vec::new(); + for (name, _) in per_overload.iter().flatten() { + if fillable.contains(name) || disagreements.iter().any(|d| d.parameter == *name) { + continue; + } + disagreements.push(ContextDisagreement { + parameter: name.clone(), + per_overload: per_overload + .iter() + .map(|resolutions| { + match resolutions.iter().find(|(declared, _)| declared == name) { + None => ContextChoice::Undeclared, + Some((_, ContextResolution::Resolved { name, .. })) => { + ContextChoice::Takes(name.clone()) + } + Some(_) => ContextChoice::NoValue, + } + }) + .collect(), + }); + } + Some(OverloadContextParameters { + fillable, + disagreements, + }) +} + +/// what an overload set makes of its keyword-only `context` parameters at one call site +#[derive(Debug, Clone)] +pub(crate) struct OverloadContextParameters { + /// the names every overload agrees on, which are the ones a call may be given implicitly + pub(crate) fillable: Vec, + /// the names they do not agree on, with what each overload made of each + pub(crate) disagreements: Vec, +} + +/// one `context` parameter an overload set could not fill implicitly, because its overloads do +/// not agree on it. +/// +/// Nothing is written for such a parameter, so a call that does not pass it explicitly goes +/// without an argument — which on its own reads as an ordinary failure to match. What each +/// overload made of the parameter is kept so the report can say where they part. +#[derive(Debug, Clone)] +pub(crate) struct ContextDisagreement { + pub(crate) parameter: Name, + /// one entry per overload, in declaration order + pub(crate) per_overload: Vec, +} + +/// what one overload would have done with a `context` parameter +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum ContextChoice { + /// the overload does not declare the parameter as a keyword-only `context` one at all + Undeclared, + /// it declares it, and would take the value this declaration holds + Takes(Name), + /// it declares it, and no visible declaration supplies a value for it + NoValue, +} + +impl ContextChoice { + /// how this overload's answer reads in a report, as the tail of "overload N …" + pub(crate) fn describe(&self, parameter: &Name) -> String { + match self { + ContextChoice::Undeclared => format!("does not declare `{parameter}`"), + ContextChoice::Takes(name) => format!("would take `{name}`"), + ContextChoice::NoValue => format!("finds no value for `{parameter}`"), + } + } +} + +/// [`implicit_context_arguments`] against one signature's parameter list. `fillable` +/// restricts it to the names an overload set agrees on; `None` places no restriction +fn implicit_context_arguments_for<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + file: File, + scope: ScopeId<'db>, + parameters: &Parameters<'db>, + fillable: Option<&Vec>, + call: &ast::ExprCall, +) -> ImplicitContextArguments { + let mut implicit = ImplicitContextArguments::default(); let positional_count = call.arguments.args.len(); let mut positional_index = 0; - let mut implicit = Vec::new(); for parameter in parameters { let fills_positional_slot = parameter.is_positional(); let matched_positionally = fills_positional_slot && positional_index < positional_count; @@ -254,25 +512,33 @@ pub fn implicit_context_arguments<'db>( let Some(name) = parameter.name() else { continue; }; + if fillable.is_some_and(|fillable| !fillable.contains(name)) { + continue; + } let matched_by_keyword = call .arguments .keywords .iter() .any(|kw| kw.arg.as_ref().is_some_and(|arg| arg.id == *name)); - if matched_positionally || matched_by_keyword { + // a keyword `_` is not taken to match a `_` the callee repeats: which of + // those parameters it names is not decided + if matched_positionally + || (matched_by_keyword && !is_repeated_underscore(db, parameters, parameter)) + { continue; } - if let ContextResolution::Resolved { + let resolution = + resolve_context_argument(db, env, scope, call.range().start(), parameters, parameter); + if resolution == ContextResolution::RepeatedUnderscore { + implicit.from_repeated_underscore.push(name.clone()); + } else if resolution == ContextResolution::RepeatedUnderscoreParameter { + implicit.for_repeated_underscore = true; + } else if let ContextResolution::Resolved { name: variable, binding, .. - } = resolve_context_argument( - db, - env, - scope, - call.range().start(), - parameter.annotated_type(), - ) { + } = resolution + { let declaration = match binding { CandidateBinding::Written(definition) => Some(definition.focus_range( db, @@ -287,7 +553,7 @@ pub fn implicit_context_arguments<'db>( } CandidateBinding::BlockArgument(_) | CandidateBinding::BlockReceiver(_) => false, }; - implicit.push(ImplicitContextArgument { + implicit.arguments.push(ImplicitContextArgument { parameter: name.clone(), variable, declaration, @@ -299,6 +565,60 @@ pub fn implicit_context_arguments<'db>( implicit } +/// the `context` parameters of `callee` that a decoration cannot fill. +/// +/// `@deco` is a call — it runs `deco(g)` — but it is the one call whose argument list the +/// source has nowhere to write. The lowering completes a call by appending the resolved +/// argument after the ones the source wrote, and a decoration has none to append to: the +/// parameter would quietly take its default, or, with no default, the decoration would +/// raise `TypeError`. Neither is the ambient value the declaration promised, so each such +/// parameter is reported and the decoration is left as written — the same bargain every +/// other unresolvable `context` parameter gets. +/// +/// The decorated definition fills the first positional slot, exactly as it does at runtime, +/// so a `context` parameter that *is* that slot is not among these. +/// +/// An overloaded callee is read through every one of its overloads, and only a parameter +/// left unfilled in all of them is reported — which overload a decoration selects is not +/// decided here, and a report true of every one of them is true whichever it is. This is +/// what reaches a `decorator def`, whose declaration is always the pair of overloads it is +/// applied in. +pub fn unfilled_decoration_context_parameters<'db>( + db: &'db dyn Db, + callee: Type<'db>, +) -> Vec { + let signatures = match callee { + Type::FunctionLiteral(function) => function.signature(db).clone(), + Type::BoundMethod(method) => method.bound_signatures(db).clone(), + _ => return Vec::new(), + }; + let mut unfilled: Option> = None; + for signature in &signatures.overloads { + let mut decorated_slot_taken = false; + let mut in_this_one = Vec::new(); + for parameter in signature.parameters() { + if parameter.is_positional() && !decorated_slot_taken { + decorated_slot_taken = true; + continue; + } + if !parameter.is_context() { + continue; + } + if let Some(name) = parameter.name() { + in_this_one.push(name.clone()); + } + } + unfilled = Some(match unfilled { + None => in_this_one, + Some(so_far) => so_far + .into_iter() + .filter(|name| in_this_one.contains(name)) + .collect(), + }); + } + unfilled.unwrap_or_default() +} + /// collect the names the trailing lambda block containing `scope` binds /// implicitly: `self` when its callback declares a receiver, and `it`. /// @@ -328,6 +648,7 @@ fn collect_block_candidates<'db>( name: Name::new_static("self"), range: None, binding: CandidateBinding::BlockReceiver(ty), + is_repeated_underscore: false, }); } @@ -338,6 +659,7 @@ fn collect_block_candidates<'db>( name: Name::new_static("it"), range: None, binding: CandidateBinding::BlockArgument(ty), + is_repeated_underscore: false, }); } } @@ -352,6 +674,10 @@ fn collect_candidates<'db>( out: &mut Vec>, ) { let mut push_params = |parameters: &ast::Parameters| { + let underscores = parameters + .iter() + .filter(|parameter| parameter.name() == "_") + .count(); for parameter in parameters.iter().map(ast::AnyParameterRef::as_parameter) { if parameter.is_context && let Some(definition) = index.try_definition(parameter) @@ -360,6 +686,7 @@ fn collect_candidates<'db>( name: parameter.name.id.clone(), range: Some(parameter.range()), binding: CandidateBinding::Written(definition), + is_repeated_underscore: parameter.name.id == "_" && underscores > 1, }); } } @@ -403,6 +730,7 @@ fn collect_declarations<'db>( name: target.id.clone(), range: Some(decl.range()), binding: CandidateBinding::Written(definition), + is_repeated_underscore: false, }); } } diff --git a/crates/ty_python_semantic/src/types/diagnostic.rs b/crates/ty_python_semantic/src/types/diagnostic.rs index 2395681c23..e7d32a5cb3 100644 --- a/crates/ty_python_semantic/src/types/diagnostic.rs +++ b/crates/ty_python_semantic/src/types/diagnostic.rs @@ -23,6 +23,7 @@ use crate::types::function::{FunctionDecorators, FunctionType, KnownFunction, Ov use crate::types::infer::UnsupportedComparisonError; use crate::types::overrides::MethodKind; use crate::types::protocol_class::ProtocolMember; +use crate::types::repeated_underscore::UnderscoreRefusal; use crate::types::special_form::TypeQualifier; use crate::types::string_annotation::{ ESCAPE_CHARACTER_IN_FORWARD_ANNOTATION, IMPLICIT_CONCATENATED_STRING_TYPE_ANNOTATION, @@ -108,6 +109,8 @@ pub(crate) fn register_lints(registry: &mut LintRegistryBuilder) { registry.register_lint(&INVALID_BASE); registry.register_lint(&INVALID_CONTEXT_MANAGER); registry.register_lint(&INVALID_DECLARATION); + registry.register_lint(&INVALID_DECORATOR_DEF); + registry.register_lint(&INVALID_REPEATED_UNDERSCORE); registry.register_lint(&INVALID_EXCEPTION_CAUGHT); registry.register_lint(&INVALID_ENUM_MEMBER_ANNOTATION); registry.register_lint(&INVALID_GENERIC_ENUM); @@ -580,6 +583,26 @@ declare_lint! { } } +declare_lint! { + #[doc = include_str!("../../resources/lint_docs/invalid-decorator-def.md")] + pub(crate) static INVALID_DECORATOR_DEF = { + summary: "detects a `decorator def` whose shape has no decorator to expand into", + status: LintStatus::stable("0.0.82"), + default_level: Level::Error, + ty_compat: TyCompat::BasedPython, + } +} + +declare_lint! { + #[doc = include_str!("../../resources/lint_docs/invalid-repeated-underscore.md")] + pub(crate) static INVALID_REPEATED_UNDERSCORE = { + summary: "detects a repeated `_` parameter the lowering has no python spelling for", + status: LintStatus::stable("0.0.83"), + default_level: Level::Error, + ty_compat: TyCompat::BasedPython, + } +} + declare_lint! { #[doc = include_str!("../../resources/lint_docs/implicit-declaration.md")] pub(crate) static IMPLICIT_DECLARATION = { @@ -2871,11 +2894,14 @@ declare_lint! { /// ## What it does /// Checks for calls that leave a basedpython `context` parameter unfilled: /// no explicit argument matches it and no `context` declaration in scope - /// has a type assignable to it. + /// has a type assignable to it. A decoration is also reported, because it + /// is a call with no argument list to supply the value in. /// /// ## Why is this bad? /// A `context` parameter has no default — the call raises `TypeError` at - /// runtime unless something supplies the argument. + /// runtime unless something supplies the argument. One that does have a + /// default quietly takes it instead of the ambient value the declaration + /// promised. /// /// ## Examples /// ```python @@ -2884,6 +2910,11 @@ declare_lint! { /// f(1) # error: no context value in scope /// context s = "hello" /// f(1) # ok — `s` is passed implicitly + /// + /// def deco(fn: (...) -> object, context b: str = "d") -> object: ... + /// + /// @deco # error: `b` cannot be filled at a decoration + /// def g(): ... /// ``` pub(crate) static MISSING_CONTEXT_ARGUMENT = { summary: "detects calls whose `context` parameter has no matching context value", @@ -5357,6 +5388,48 @@ pub(super) fn report_bad_dunder_delattr_call( } } +/// basedpython: report each `context` parameter a decoration leaves unfilled. +/// +/// `@deco` is a call with no argument list to append to, so the lowering has nowhere to +/// write the ambient value and the parameter takes its default instead. Reporting it is +/// what keeps that from happening quietly; the call form (`deco(f, b=...)`, or a decorator +/// factory `@deco(b=...)`) is where the argument can be written. +pub(super) fn report_unfilled_decoration_context_parameters<'db>( + context: &InferContext<'db, '_>, + decorator: &ast::Decorator, + decorator_ty: Type<'db>, +) { + // a modifier keyword (`final def f()`) parses as a synthetic decorator that decorates + // nothing, and resolves to no signature of its own + if matches!(&decorator.expression, ast::Expr::Name(name) if name.ctx.is_invalid()) { + return; + } + if !matches!( + decorator_ty, + Type::FunctionLiteral(_) | Type::BoundMethod(_) + ) { + return; + } + for parameter in crate::types::context_params::unfilled_decoration_context_parameters( + context.db(), + decorator_ty, + ) { + let Some(builder) = context.report_lint(&MISSING_CONTEXT_ARGUMENT, decorator) else { + return; + }; + let mut diagnostic = builder.into_diagnostic(format_args!( + "context parameter `{parameter}` cannot be filled at a decoration" + )); + diagnostic.sub(SubDiagnostic::new( + SubDiagnosticSeverity::Info, + format_args!( + "a decoration has no argument list to write it in; call the decorator \ + explicitly, or give it a factory form the decoration can pass it to" + ), + )); + } +} + pub(super) fn report_dynamic_function_decorator_return<'db>( context: &InferContext<'db, '_>, decorator: &ast::Decorator, @@ -5876,6 +5949,94 @@ pub(super) fn report_missing_function_body( diagnostic.help("Write the body, or `: ...` if the function is meant to do nothing"); } +/// basedpython: report a parameter list that repeats `_` in a shape the lowering refuses — see +/// [`UnderscoreRefusal`]. the transpiler refuses the same definition for the same reason, so +/// the refusal arrives while the file is being checked +pub(super) fn report_invalid_repeated_underscore( + context: &InferContext, + parameter: AnyNodeRef, + refusal: &UnderscoreRefusal, +) { + let Some(builder) = context.report_lint(&INVALID_REPEATED_UNDERSCORE, parameter) else { + return; + }; + let mut diagnostic = builder.into_diagnostic(refusal.message()); + diagnostic.help(refusal.help()); +} + +/// basedpython: what is wrong with a `decorator def` the lowering cannot expand. +pub(super) enum InvalidDecoratorDef<'a> { + /// written in a class body, where the keyword is not available at all + InClassBody, + /// no parameter to receive the decorated function + NoDecoratedParameter, + /// a variadic parameter, which neither the function nor an option can arrive through + Variadic, + /// a default on the parameter that receives the decorated function + DecoratedParameterDefault { parameter: &'a str }, + /// an option with no default, which decorating without options cannot supply + OptionWithoutDefault { parameter: &'a str }, +} + +/// basedpython: report a `decorator def` whose shape has no decorator to expand into. +/// +/// The lowering builds the two overloads and the runtime dispatcher out of a first parameter that +/// receives the decorated function and options that each carry a default. Without that shape there +/// is nothing to build, and the transpile refuses the file. ty reports the same refusal, so a +/// `decorator def` that cannot be lowered is caught where the rest of the file's problems are. +pub(super) fn report_invalid_decorator_def( + context: &InferContext, + node: AnyNodeRef, + name: &str, + problem: &InvalidDecoratorDef, +) { + let Some(builder) = context.report_lint(&INVALID_DECORATOR_DEF, node) else { + return; + }; + let mut diagnostic = match problem { + InvalidDecoratorDef::InClassBody => { + let mut diagnostic = builder.into_diagnostic(format_args!( + "`decorator def {name}` is only valid at module scope" + )); + diagnostic.help("Write a plain `def` that returns a callable"); + diagnostic + } + InvalidDecoratorDef::NoDecoratedParameter => { + let mut diagnostic = builder.into_diagnostic(format_args!( + "`decorator def {name}` declares no parameter for the function it decorates" + )); + diagnostic.help(format_args!( + "Add a first parameter, which `@{name}` hands the decorated function" + )); + diagnostic + } + InvalidDecoratorDef::Variadic => { + let mut diagnostic = builder.into_diagnostic(format_args!( + "`decorator def {name}` cannot declare `*args` or `**kwargs`" + )); + diagnostic.help("Declare the decorated function and each option as a named parameter"); + diagnostic + } + InvalidDecoratorDef::DecoratedParameterDefault { parameter } => { + let mut diagnostic = builder.into_diagnostic(format_args!( + "The decorated parameter `{parameter}` of `decorator def {name}` cannot have a default" + )); + diagnostic.help("Remove the default"); + diagnostic + } + InvalidDecoratorDef::OptionWithoutDefault { parameter } => { + let mut diagnostic = builder.into_diagnostic(format_args!( + "Option `{parameter}` of `decorator def {name}` has no default" + )); + diagnostic.help(format_args!( + "Give it one: `@{name}` decorates with no options at all" + )); + diagnostic + } + }; + diagnostic.info("A `decorator def` takes the decorated function first, then options that each have a default"); +} + pub(super) fn report_invalid_type_checking_constant(context: &InferContext, node: AnyNodeRef) { let Some(builder) = context.report_lint(&INVALID_TYPE_CHECKING_CONSTANT, node) else { return; diff --git a/crates/ty_python_semantic/src/types/function.rs b/crates/ty_python_semantic/src/types/function.rs index 01852929ed..ae1cf9c8b7 100644 --- a/crates/ty_python_semantic/src/types/function.rs +++ b/crates/ty_python_semantic/src/types/function.rs @@ -61,7 +61,7 @@ use ruff_db::source::source_text; use ruff_diagnostics::{Edit, Fix}; use ruff_python_ast::find_node::covering_node; use ruff_python_ast::helpers::{last_bound_parameter, parameter_modifiers}; -use ruff_python_ast::{self as ast, ParameterWithDefault}; +use ruff_python_ast::{self as ast, ParameterWithDefault, PythonVersion}; use ruff_python_edits::unwrapped_call_argument; use ruff_text_size::Ranged; use salsa::plumbing::AsId; @@ -97,6 +97,10 @@ use crate::types::known_instance::DeprecatedInstance; use crate::types::list_members::all_members; use crate::types::narrow::ClassInfoConstraintFunction; use crate::types::relation::TypeRelationChecker; +use crate::types::repeated_underscore::{ + BaseParameter, LoweredParameters, ParameterSlot, UnderscoreLowering, UnderscoreRefusal, + WrittenNames, lower_repeated_underscores, parameter_slots, reads_from_enclosing_scope, +}; use crate::types::signatures::{ CallableSignature, DeferredAssertions, NarrowingGuard, NarrowingGuardKind, Parameters, ReturnCallableTypeVarScope, Signature, @@ -634,6 +638,76 @@ impl<'db> OverloadLiteral<'db> { false } + /// basedpython: how the lowering writes this definition's parameters when it spells `_` + /// more than once — see [`lower_repeated_underscores`]. `None` when it does not, and + /// outside basedpython files, where a repeated parameter is a syntax error + /// + /// ty's signature of the definition and the transpiler's output are both built from this + /// answer, so a call ty accepts is one the emitted python accepts + #[salsa::tracked( + returns(ref), + cycle_initial=|_, _, _| None, + heap_size=ruff_memory_usage::heap_size, + )] + pub(crate) fn repeated_underscores( + self, + db: &'db dyn Db, + ) -> Option> { + if !self.file(db).source_type(db).is_basedpython() { + return None; + } + let module = parsed_module(db, self.python_file(db)).load(db); + let node = self.body_scope(db).node(db).expect_function().node(&module); + let mut slots = parameter_slots(&node.parameters); + if slots.iter().filter(|(_, name)| *name == "_").count() < 2 { + return None; + } + // a `decorator def` is passed its options as `@d(option=...)`, so the python it + // lowers to declares every parameter after the decorated one keyword-only + let is_decorator_keyword = node.decorator_list.iter().any(|decorator| { + matches!(&decorator.expression, ast::Expr::Name(name) + if name.ctx.is_invalid() && name.id == "decorator_keyword") + }); + if is_decorator_keyword { + for (slot, _) in slots.iter_mut().skip(1) { + if matches!( + slot, + ParameterSlot::PositionalOnly | ParameterSlot::PositionalOrKeyword + ) { + *slot = ParameterSlot::KeywordOnly; + } + } + } + + let env = ProgramEnvironment::from_scope(self.body_scope(db)); + let base: Option>> = self + .overridden_method(db, &env) + .filter(|base| base.signature(db).overloads.len() == 1) + .map(|base| { + base_raw_signature(db, base) + .parameters() + .iter() + .take_while(|parameter| parameter.is_positional()) + .map(|parameter| { + Some(BaseParameter { + name: parameter.name()?.clone(), + positional_only: parameter.is_positional_only(), + }) + }) + .collect() + }); + + let index = semantic_index(db, self.program_file(db)); + let body_scope = self.body_scope(db).file_scope_id(db); + lower_repeated_underscores( + &slots, + self.binds_first_parameter(db), + base.as_deref(), + |name| reads_from_enclosing_scope(index, body_scope, name), + env.python_version(db) >= PythonVersion::PY38, + ) + } + /// basedpython: names of the pep 695 type parameters this definition /// reifies (declared `reified`, or referenced in a value position in the /// body), in declaration order. always empty outside basedpython files @@ -966,6 +1040,43 @@ impl<'db> OverloadLiteral<'db> { signature } + /// basedpython: the signatures a `decorator def` is called with, if this is one. + /// + /// A `decorator def` declares one function and is applied in two shapes — `@d`, which hands it + /// the decorated function, and `@d(option=...)`, which hands it the options and decorates with + /// what it returns. The lowering writes an overload for each, and this is the same pair. + /// + /// `None` for a `decorator def` the lowering refuses: one inside a class body, or one whose + /// parameters are not a decorated function followed by options with defaults. Its declared + /// signature stands there, as it did before. + fn decorator_keyword_signatures(self, db: &'db dyn Db) -> Option<[Signature<'db>; 2]> { + let scope = self.body_scope(db); + let program_file = self.program_file(db); + let module = parsed_module(db, program_file.python_file(db)).load(db); + let function_node = scope.node(db).expect_function().node(&module); + let is_decorator_keyword = function_node.decorator_list.iter().any(|decorator| { + matches!(&decorator.expression, ast::Expr::Name(name) + if name.ctx.is_invalid() && name.id == "decorator_keyword") + }); + if !is_decorator_keyword { + return None; + } + let index = semantic_index(db, program_file); + if index + .ancestor_scopes(scope.file_scope_id(db)) + .skip(1) + .any(|(_, ancestor)| ancestor.kind().is_class()) + { + return None; + } + Signature::decorator_keyword_overloads( + db, + &self.signature(db), + &self.raw_signature(db, ReturnCallableTypeVarScope::Lexical), + self.definition(db), + ) + } + /// Returns the effective signatures of this overload after applying decorators. fn decorated_signatures( self, @@ -1071,6 +1182,19 @@ impl<'db> OverloadLiteral<'db> { index, ); + let lowered_parameters = match self.repeated_underscores(db) { + Some(Ok(lowering)) => { + let source = source_text(db, self.file(db)); + let slots = parameter_slots(&function_stmt_node.parameters); + Some(LoweredParameters::new( + lowering, + &slots, + WrittenNames::new(source.as_str()), + )) + } + _ => None, + }; + let mut raw_signature = Signature::from_function( db, env, @@ -1079,6 +1203,7 @@ impl<'db> OverloadLiteral<'db> { function_stmt_node, has_implicitly_positional_first_parameter, return_callable_typevar_scope, + lowered_parameters.as_ref(), ); let generic_context = raw_signature.generic_context; @@ -1388,12 +1513,20 @@ impl<'db> OverloadLiteral<'db> { ) -> Option> { let decorator = self.innermost_applied_decorator(db, function_stmt_node)?; let decorator_callable = decorator.try_upcast_to_callable(db, env)?.exactly_one()?; - // an overloaded decorator would take the function as a different parameter depending on - // how it is called, and which overload a call picks is decided by the very type this is - // trying to fill in - let [decorator_signature] = decorator_callable.signatures(db).overloads.as_slice() else { + // a decoration calls the decorator with the function as its one positional argument, so an + // overload that cannot be called that way is never the one it picks. when more than one + // can be, which of them a decoration picks is decided by the types — that is, by the very + // type this is trying to fill in + let decoration = CallArguments::positional([Type::unknown()]); + let bindings = Type::Callable(decorator_callable) + .bindings(db, env) + .match_parameters(db, env, &decoration); + let mut matching = bindings.single_element()?.matching_overloads(); + let (Some((index, _)), None) = (matching.next(), matching.next()) else { return None; }; + let signatures = decorator_callable.signatures(db); + let decorator_signature = signatures.overloads.get(index)?; let decorated = decorator_signature.parameters().iter().next()?; if !decorated.is_positional() { return None; @@ -1953,6 +2086,11 @@ impl<'db> FunctionLiteral<'db> { if let Some(implementation) = implementation && overloads.is_empty() { + // basedpython: a `decorator def` is written once and called in two shapes, so it is + // overloaded even though the source spells no `@overload` + if let Some(signatures) = implementation.decorator_keyword_signatures(db) { + return CallableSignature::from_overloads(signatures); + } return CallableSignature::single(implementation.signature(db)); } @@ -4457,6 +4595,20 @@ fn revealed_declared_type<'db>( ) } +/// The return type `function` declares — the annotation it wrote, or what its body returns +/// when it wrote none. +/// +/// This is the function's own signature, before a `decorator def` is expanded into the two +/// shapes it is applied in (`Signature::decorator_keyword_overloads`): both of those state +/// this same type, and the transpiler writes them out, so it has to be able to ask for it. +/// `None` when `function` is not a function literal. +pub fn declared_return_type<'db>(db: &'db dyn Db, function: Type<'db>) -> Option> { + let Type::FunctionLiteral(function) = function else { + return None; + }; + Some(function.literal(db).last_definition.signature(db).return_ty) +} + #[cfg(test)] pub(crate) mod tests { use strum::IntoEnumIterator; diff --git a/crates/ty_python_semantic/src/types/ide_support.rs b/crates/ty_python_semantic/src/types/ide_support.rs index 482aff3aa7..6d207fcd6a 100644 --- a/crates/ty_python_semantic/src/types/ide_support.rs +++ b/crates/ty_python_semantic/src/types/ide_support.rs @@ -24,8 +24,9 @@ use crate::types::list_members::all_end_of_scope_members; use crate::types::literal::LiteralValueTypeKind; use crate::types::overrides::is_constructor_like_method; use crate::types::receivers; +use crate::types::repeated_underscore::{UnderscoreLowering, UnderscoreRefusal}; use crate::types::signatures::{ - ParameterKind, ParametersKind, ReturnCallableTypeVarScope, Signature, + Parameter, ParameterKind, ParametersKind, ReturnCallableTypeVarScope, Signature, }; use crate::types::{ CallDunderError, CallableTypes, ClassBase, ClassLiteral, KnownClass, KnownFunction, KnownUnion, @@ -3178,10 +3179,7 @@ pub fn inherited_parameter_annotation<'db>( let function = nearest_enclosing_function(db, index, definition.scope(db))?; let signature = function.last_definition_raw_signature(db, ReturnCallableTypeVarScope::Public); - let matched = signature - .parameters() - .iter() - .find(|candidate| candidate.name() == Some(¶meter.name.id))?; + let matched = declared_parameter(model, definition, signature, parameter)?; // an annotation the source wrote is displayed by the source; an implicit receiver and an // anonymous hole are the two types a signature holds that were never written anywhere @@ -3213,12 +3211,51 @@ pub fn inherited_parameter_default( let function = nearest_enclosing_function(db, index, definition.scope(db))?; let signature = function.last_definition_raw_signature(db, ReturnCallableTypeVarScope::Public); + declared_parameter(model, definition, signature, ¶meter.parameter)? + .default_type(db)? + .display_default_value(db, &model.program_environment()) +} + +/// the parameter of `signature`, the signature of the function whose parameter `definition` +/// defines, that `parameter` declares +/// +/// it is found by where it stands in the function's own list rather than by its name, which +/// the signature need not share: a parameter list that repeats `_` names its parameters as the +/// lowering writes them +fn declared_parameter<'s, 'db>( + model: &SemanticModel<'db>, + definition: Definition<'db>, + signature: &'s Signature<'db>, + parameter: &ast::Parameter, +) -> Option<&'s Parameter<'db>> { + let db = model.db(); + let module = parsed_module(db, db.program_file(model.file()).python_file(db)).load(db); + let function = definition.scope(db).node(db).as_function()?.node(&module); + let position = function + .parameters + .iter() + .position(|candidate| candidate.as_parameter().range == parameter.range)?; signature .parameters() .iter() - .find(|candidate| candidate.name() == Some(¶meter.parameter.name.id))? - .default_type(db)? - .display_default_value(db, &model.program_environment()) + .find(|candidate| candidate.source_parameter_index() == Some(position)) +} + +/// basedpython: how the lowering writes `function`'s parameters when they repeat `_` — the +/// answer ty's signature of the definition is built from. `None` when they do not +pub fn repeated_underscore_lowering( + model: &SemanticModel<'_>, + function: &ast::StmtFunctionDef, +) -> Option> { + let db = model.db(); + let index = semantic_index(db, db.program_file(model.file())); + let definition = index.try_definition(function)?; + infer_definition_types(db, definition) + .function_type(definition)? + .literal(db) + .last_definition + .repeated_underscores(db) + .clone() } /// basedpython: the return type recovered for a `def` that leaves its annotation out. diff --git a/crates/ty_python_semantic/src/types/infer/builder.rs b/crates/ty_python_semantic/src/types/infer/builder.rs index 58d4f9cdd5..86b1f58c02 100644 --- a/crates/ty_python_semantic/src/types/infer/builder.rs +++ b/crates/ty_python_semantic/src/types/infer/builder.rs @@ -102,13 +102,14 @@ use crate::types::diagnostic::{ report_invalid_exception_cause, report_invalid_exception_raised, report_invalid_exception_tuple_caught, report_invalid_generator_yield_type, report_invalid_key_on_typed_dict, report_invalid_match_args_type, - report_invalid_type_checking_constant, + report_invalid_repeated_underscore, report_invalid_type_checking_constant, report_match_pattern_against_non_runtime_checkable_protocol, report_match_pattern_against_typed_dict, report_mismatched_type_name, report_possibly_missing_attribute, report_possibly_unresolved_reference, report_restricted_constructor, report_too_many_positional_patterns_for_class_pattern, - report_unplaceable_starred_class_pattern, report_unsound_assignment, report_unsound_yield, - report_unsupported_augmented_assignment, report_unsupported_comparison, + report_unfilled_decoration_context_parameters, report_unplaceable_starred_class_pattern, + report_unsound_assignment, report_unsound_yield, report_unsupported_augmented_assignment, + report_unsupported_comparison, }; use crate::types::enums::{enum_ignored_names, is_enum_class_by_inheritance}; use crate::types::exceptions::CallSolution; @@ -138,6 +139,9 @@ use crate::types::newtype::NewType; use crate::types::receivers; use crate::types::regex; use crate::types::reified_infer::{self, ErasedTargetReason, ReifiedInferenceError}; +use crate::types::repeated_underscore::{ + LoweredParameters, WrittenNames, lower_repeated_underscores, parameter_slots, +}; use crate::types::set_theoretic::RecursivelyDefined; use crate::types::signatures::{CallableSignature, ReturnCallableTypeVarScope}; use crate::types::soundness::{ @@ -6892,6 +6896,11 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { }), }; + // basedpython: a `context` parameter of the decorator itself is not filled here — + // there is no argument list to write it in — so say so rather than let it take its + // default + report_unfilled_decoration_context_parameters(&self.context, decorator_node, decorator_ty); + let call_arguments = CallArguments::positional([decorated_ty]); let (return_ty, decorator_bindings) = match decorator_ty.try_call(db, env, &call_arguments) { @@ -10640,6 +10649,40 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { self.infer_expression(&lambda_expression.body, tcx); } + /// basedpython: the names and kinds the lowering gives a lambda's parameters when they + /// repeat `_`, reporting a shape it refuses. a lambda overrides nothing and has no receiver, + /// so the answer is the parameter list's alone + fn lambda_repeated_underscores( + &self, + parameters: &ast::Parameters, + ) -> Option { + if !self.is_basedpython_file() { + return None; + } + let slots = parameter_slots(parameters); + let slash = self.program_environment().python_version(self.db()) >= PythonVersion::PY38; + match lower_repeated_underscores(&slots, false, None, |_| false, slash)? { + Ok(lowering) => { + let source = source_text(self.db(), self.file()); + Some(LoweredParameters::new( + &lowering, + &slots, + WrittenNames::new(source.as_str()), + )) + } + Err(refusal) => { + if let Some(parameter) = parameters.iter().nth(refusal.index()) { + report_invalid_repeated_underscore( + &self.context, + parameter.as_parameter().into(), + &refusal, + ); + } + None + } + } + } + fn infer_lambda_expression( &mut self, lambda_expression: &ast::ExprLambda, @@ -10715,7 +10758,8 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { } }; - let parameters = if let Some(parameters) = parameters { + let parameters = if let Some(parameters_node) = parameters { + let parameters = parameters_node; let positional_only = parameters .posonlyargs .iter() @@ -10793,7 +10837,14 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { .chain(keyword_only) .chain(keyword_variadic); - Parameters::from_annotation(db, env, parameters) + // basedpython: a lambda that repeats `_` takes the names and kinds the lowering + // writes it with, as a `def` does + match self.lambda_repeated_underscores(parameters_node) { + Some(lowered) => { + Parameters::from_annotation(db, env, lowered.apply(parameters.collect())) + } + None => Parameters::from_annotation(db, env, parameters), + } } else { Parameters::empty() }; diff --git a/crates/ty_python_semantic/src/types/infer/builder/function.rs b/crates/ty_python_semantic/src/types/infer/builder/function.rs index b5e0ff9e2c..95a0dc5606 100644 --- a/crates/ty_python_semantic/src/types/infer/builder/function.rs +++ b/crates/ty_python_semantic/src/types/infer/builder/function.rs @@ -10,13 +10,14 @@ use crate::{ diagnostic::{ ABSTRACT_AND_FINAL_METHOD, FINAL_ON_NON_METHOD, INEFFECTIVE_PRIVATE, INVALID_FIXTURE_TYPE, INVALID_PARAMETER_DEFAULT, INVALID_PARAMETRIZE, - INVALID_PARAMSPEC, INVALID_TYPE_FORM, REDUNDANT_RETURN_ANNOTATION, REIFIED_CLASSMETHOD, - TRAILING_LAMBDA_PARAMETERS, TRAILING_LAMBDA_RETURN_TYPE, UNKNOWN_FIXTURE, - UNSOUND_RETURN_STATEMENT, USELESS_OVERLOAD_BODY, add_type_expression_reference_link, - is_invalid_typed_dict_literal, report_bool_as_int, report_implicit_return_type, - report_invalid_generator_function_return_type, report_invalid_return_type, - report_missing_function_body, report_shadowed_type_variable, - report_unsound_return_statement, + INVALID_PARAMSPEC, INVALID_TYPE_FORM, InvalidDecoratorDef, REDUNDANT_RETURN_ANNOTATION, + REIFIED_CLASSMETHOD, TRAILING_LAMBDA_PARAMETERS, TRAILING_LAMBDA_RETURN_TYPE, + UNKNOWN_FIXTURE, UNSOUND_RETURN_STATEMENT, USELESS_OVERLOAD_BODY, + add_type_expression_reference_link, is_invalid_typed_dict_literal, report_bool_as_int, + report_implicit_return_type, report_invalid_decorator_def, + report_invalid_generator_function_return_type, report_invalid_repeated_underscore, + report_invalid_return_type, report_missing_function_body, + report_shadowed_type_variable, report_unsound_return_statement, }, extensions, function::{ @@ -58,6 +59,7 @@ use ty_python_core::{ use ruff_db::diagnostic::{Annotation, Span}; use ruff_db::parsed::parsed_module; use ruff_python_ast as ast; +use ruff_python_ast::AnyNodeRef; use ruff_python_ast::helpers::{MemberVisibility, ReturnGuardForm, return_guards}; use ruff_text_size::Ranged; use rustc_hash::FxHashSet; @@ -86,6 +88,19 @@ fn is_synthesized_from_another_construct(function: &ast::StmtFunctionDef) -> boo }) } +/// basedpython: whether this function was written `decorator def`, which the parser records with a +/// synthetic `decorator_keyword` marker decorator. +fn is_decorator_def(function: &ast::StmtFunctionDef) -> bool { + function.decorator_list.iter().any(|decorator| { + matches!( + &decorator.expression, + ast::Expr::Name(marker) + if matches!(marker.ctx, ast::ExprContext::Invalid) + && marker.id.as_str() == "decorator_keyword" + ) + }) +} + fn function_has_deferred_annotations(function: &ast::StmtFunctionDef) -> bool { function.type_params.is_none() && (function.returns.is_some() @@ -338,6 +353,14 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { // position asks for an implementation, the missing body is the whole of the problem self.check_missing_function_body(function); + // basedpython: a `decorator def` the lowering cannot expand is refused when the file is + // transpiled; reported here, the refusal arrives while the file is being checked + self.check_decorator_def(function); + + // basedpython: a parameter list that repeats `_` in a shape the lowering has no python + // spelling for is refused when the file is transpiled, and reported here + self.check_repeated_underscores(function); + let enclosing_function_for_return_check = nearest_enclosing_function(db, self.index, self.scope()); @@ -648,6 +671,105 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { report_missing_function_body(&self.context, function); } + /// basedpython: report a `decorator def` that has no decorator to expand into. + /// + /// The lowering builds the two overloads and the runtime dispatcher out of a first parameter + /// that receives the decorated function and options that each carry a default, and refuses + /// anything else. What it refuses is what is reported here, so the shape is settled while the + /// file is checked rather than when it is transpiled. ty models the accepted shape as the same + /// pair of signatures — see [`Signature::decorator_keyword_overloads`] — and reads a + /// declaration it refuses as the plain function it was written as. + /// + /// [`Signature::decorator_keyword_overloads`]: crate::types::signatures::Signature::decorator_keyword_overloads + fn check_decorator_def(&self, function: &ast::StmtFunctionDef) { + if !is_decorator_def(function) { + return; + } + let name = function.name.id.as_str(); + let report = |node: AnyNodeRef, problem: &InvalidDecoratorDef| { + report_invalid_decorator_def(&self.context, node, name, problem); + }; + + // the keyword is module-scope only: the overloads the lowering writes would stand in the + // enclosing class's namespace, where they are attributes rather than declarations + if self + .index + .ancestor_scopes(self.scope().file_scope_id(self.db())) + .skip(1) + .any(|(_, ancestor)| ancestor.kind().is_class()) + { + report((&function.name).into(), &InvalidDecoratorDef::InClassBody); + return; + } + + let parameters = &function.parameters; + if let Some(variadic) = parameters.vararg.as_deref().or(parameters.kwarg.as_deref()) { + report(variadic.into(), &InvalidDecoratorDef::Variadic); + return; + } + + let mut positional = parameters.posonlyargs.iter().chain(¶meters.args); + let Some(decorated) = positional.next() else { + report( + parameters.as_ref().into(), + &InvalidDecoratorDef::NoDecoratedParameter, + ); + return; + }; + if let Some(default) = decorated.default.as_deref() { + report( + default.into(), + &InvalidDecoratorDef::DecoratedParameterDefault { + parameter: decorated.name().as_str(), + }, + ); + return; + } + + for option in positional.chain(¶meters.kwonlyargs) { + if option.default.is_none() { + report( + (&option.parameter).into(), + &InvalidDecoratorDef::OptionWithoutDefault { + parameter: option.name().as_str(), + }, + ); + return; + } + } + } + + /// basedpython: report a parameter list that repeats `_` in a shape the lowering refuses. + /// the refusal is the one the definition's signature is built from — see + /// [`OverloadLiteral::repeated_underscores`] + fn check_repeated_underscores(&self, function: &ast::StmtFunctionDef) { + let db = self.db(); + if function + .parameters + .iter() + .filter(|parameter| parameter.name() == "_") + .count() + < 2 + { + return; + } + let Some(function_type) = nearest_enclosing_function(db, self.index, self.scope()) else { + return; + }; + let Some(Err(refusal)) = function_type + .literal(db) + .last_definition + .repeated_underscores(db) + else { + return; + }; + let node = function.parameters.iter().nth(refusal.index()).map_or_else( + || AnyNodeRef::from(function.parameters.as_ref()), + |parameter| AnyNodeRef::from(parameter.as_parameter()), + ); + report_invalid_repeated_underscore(&self.context, node, refusal); + } + /// basedpython: whether a `def` written with no body at all declares a signature in this /// position, rather than leaving out an implementation. fn bodyless_def_declares_a_signature(&self, function: &ast::StmtFunctionDef) -> bool { diff --git a/crates/ty_python_semantic/src/types/infer/builder/type_expression.rs b/crates/ty_python_semantic/src/types/infer/builder/type_expression.rs index ad82b4dcd6..ab7b819e5c 100644 --- a/crates/ty_python_semantic/src/types/infer/builder/type_expression.rs +++ b/crates/ty_python_semantic/src/types/infer/builder/type_expression.rs @@ -19,13 +19,17 @@ use crate::types::diagnostic::{ self, EXPERIMENTAL_SYNTAX, INVALID_PARAMSPEC, INVALID_TYPE_FORM, NOT_SUBSCRIPTABLE, UNBOUND_TYPE_VARIABLE, UNRESOLVED_ATTRIBUTE, UNSUPPORTED_OPERATOR, report_invalid_argument_number_to_special_form, report_invalid_arguments_to_callable, - report_invalid_concatenate_last_arg, report_missing_type_arguments, - report_unsupported_binary_operation, + report_invalid_concatenate_last_arg, report_invalid_repeated_underscore, + report_missing_type_arguments, report_unsupported_binary_operation, }; use crate::types::function::{FunctionDecorators, FunctionType}; use crate::types::infer::builder::binary_expressions::BinaryInferenceState; use crate::types::infer::builder::subscript::AnnotatedExprContext; use crate::types::infer::{InferenceFlags, TypeExpressionFlags}; +use crate::types::repeated_underscore::{ + LoweredParameters, ParameterSlot, WrittenNames, callable_parameter_slots, + lower_repeated_underscores, +}; use crate::types::signatures::{ConcatenateTail, Signature}; use crate::types::special_form::{AliasSpec, LegacyStdlibAlias}; use crate::types::string_annotation::parse_string_annotation; @@ -1989,6 +1993,7 @@ impl<'db> TypeInferenceBuilder<'db, '_> { |index| callable.parameter_borrow(index + receiver_offset), )) .collect(); + let params = self.lower_callable_repeated_underscores(callable, receiver.is_some(), params); let parameters = Parameters::from_annotation(db, env, params); let return_type = self.infer_type_expression(&callable.returns); let previous = self @@ -2000,6 +2005,48 @@ impl<'db> TypeInferenceBuilder<'db, '_> { result } + /// basedpython: `params`, the parameters of `callable`, under the names and kinds the + /// lowering writes them with when they repeat `_` — the rule a `def`'s parameter list + /// follows, so a call is checked against the `__call__` or method the transpiler + /// declares. a shape the lowering refuses is reported, and its parameters are left + /// as written + fn lower_callable_repeated_underscores( + &mut self, + callable: &ast::ExprCallableType, + method_receiver: bool, + params: Vec>, + ) -> Vec> { + let slots = callable_parameter_slots(callable, method_receiver); + let spelled: Vec<(ParameterSlot, &str)> = slots + .iter() + .map(|(slot, name)| (*slot, name.as_str())) + .collect(); + let slash = self.program_environment().python_version(self.db()) >= PythonVersion::PY38; + match lower_repeated_underscores(&spelled, method_receiver, None, |_| false, slash) { + Some(Ok(lowering)) if spelled.len() == params.len() => { + let source = source_text(self.db(), self.file()); + LoweredParameters::new(&lowering, &spelled, WrittenNames::new(source.as_str())) + .apply(params) + } + Some(Err(refusal)) => { + // the slots are the receiver, then the written parameters. a method's + // receiver is written as the first of them, an implicit one apart + let written = refusal + .index() + .checked_sub(usize::from(callable.receiver.is_some())); + let node = match written { + Some(index) => callable.args.get(index).map(ast::AnyNodeRef::from), + None => callable.receiver.as_deref().map(ast::AnyNodeRef::from), + }; + if let Some(node) = node { + report_invalid_repeated_underscore(&self.context, node, &refusal); + } + params + } + Some(Ok(_)) | None => params, + } + } + /// basedpython: the leading parameter an implicit receiver contributes — the /// `int` of `int.() -> str`. It is positional-only (it is passed by position /// when the callable is called directly) and marked as the receiver, which is diff --git a/crates/ty_python_semantic/src/types/repeated_underscore.rs b/crates/ty_python_semantic/src/types/repeated_underscore.rs new file mode 100644 index 0000000000..0a9fab5a4c --- /dev/null +++ b/crates/ty_python_semantic/src/types/repeated_underscore.rs @@ -0,0 +1,788 @@ +//! basedpython: a parameter list that spells `_` more than once +//! +//! python refuses two parameters of one name, so the lowering gives every `_` after the +//! first a name of its own. those names are the lowering's rather than the author's, so no +//! call reaches such a parameter by keyword: it is positional-only in the python. a method +//! that overrides one is the exception, and takes the names the overridden method gives +//! those positions, so a call written against the base keeps working on the override +//! +//! what each parameter is called in the python, and how many of them a call reaches by +//! position alone, is decided here once — for ty's signature of the definition and for the +//! transpiler's output alike + +use std::collections::HashSet; + +use ruff_python_ast as ast; +use ruff_python_ast::name::Name; +use ruff_python_stdlib::identifiers::is_identifier_continuation; +use ty_python_core::{FileScopeId, SemanticIndex}; + +use crate::types::signatures::Parameter; + +/// the names a module spells anywhere in the text its author wrote — every run of +/// identifier characters, in code, strings and comments alike +/// +/// a repeated `_` is numbered to a name outside these, so the parameter cannot +/// shadow a name the function reads from an enclosing scope: +/// +/// ```by +/// _2 = [9] +/// +/// def f(_: int, _: int) -> list[int]: +/// return _2 # the module's `_2`, so the second parameter is `_3` +/// ``` +/// +/// it is read off the written text rather than out of any scope, so it takes no +/// analysis to answer and every reader of a definition answers it the same way. the +/// text is the one written, before any rewrite: the transpiler's passes see a module +/// its enum lowering has added names to, and the native compiler does not +#[derive(Clone, Copy, Debug)] +pub struct WrittenNames<'src> { + source: &'src str, +} + +impl<'src> WrittenNames<'src> { + /// the names spelled in `source`, the text of a module as it was written + pub fn new(source: &'src str) -> Self { + Self { source } + } + + /// a name the module spells nowhere, `stem` itself when the module does not spell it + /// and `stem2`, `stem3`, … when it does + /// + /// a binding the lowering writes under a name the source wrote would shadow it — the + /// `inner` a `decorator def` writes standing where an option of that name is declared, + /// which the dispatcher then passed the dispatcher itself for + pub fn fresh(self, stem: &str) -> String { + if !self.spells(stem) { + return stem.to_owned(); + } + // candidates only ever count up, so one never repeats an earlier one. a module + // spells finitely many names, and each candidate is longer than the last, so one + // of them is free long before the numbers run out + (2u32..u32::MAX) + .map(|number| format!("{stem}{number}")) + .find(|candidate| !self.spells(candidate)) + .unwrap_or_else(|| stem.to_owned()) + } + + /// whether `name` is spelled anywhere in the module, as a whole run of identifier + /// characters rather than as a part of a longer one + pub fn spells(self, name: &str) -> bool { + self.source.match_indices(name).any(|(start, _)| { + let before = self.source[..start].chars().next_back(); + let after = self.source[start + name.len()..].chars().next(); + !before.is_some_and(is_identifier_continuation) + && !after.is_some_and(is_identifier_continuation) + }) + } +} + +/// where a parameter stands in a parameter list. the order is the one the repeated `_`s +/// are numbered in, which is not the order they are declared in: a `*args` is declared +/// before the keyword-only parameters and numbered after them +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub enum ParameterSlot { + PositionalOnly, + PositionalOrKeyword, + KeywordOnly, + Variadic, + KeywordVariadic, +} + +impl ParameterSlot { + const fn is_positional(self) -> bool { + matches!(self, Self::PositionalOnly | Self::PositionalOrKeyword) + } +} + +/// each of `parameters` as its slot and the name the source spells, in declaration order +pub fn parameter_slots(parameters: &ast::Parameters) -> Vec<(ParameterSlot, &str)> { + let mut slots = Vec::with_capacity(parameters.len()); + for parameter in ¶meters.posonlyargs { + slots.push((ParameterSlot::PositionalOnly, parameter.name().as_str())); + } + for parameter in ¶meters.args { + slots.push(( + ParameterSlot::PositionalOrKeyword, + parameter.name().as_str(), + )); + } + if let Some(parameter) = ¶meters.vararg { + slots.push((ParameterSlot::Variadic, parameter.name.as_str())); + } + for parameter in ¶meters.kwonlyargs { + slots.push((ParameterSlot::KeywordOnly, parameter.name().as_str())); + } + if let Some(parameter) = ¶meters.kwarg { + slots.push((ParameterSlot::KeywordVariadic, parameter.name.as_str())); + } + slots +} + +/// the receiver an inline protocol's method writes first, `self` in +/// `protocol(def m(self, x: int) -> str)`. the parser marks it as a label, since it is a +/// parameter's name where every other element of the list is a type +pub fn protocol_method_receiver(signature: &ast::ExprCallableType) -> Option<&ast::ExprName> { + match signature.args.first() { + Some(ast::Expr::Name(name)) if name.ctx.is_invalid() => Some(name), + _ => None, + } +} + +/// each parameter of a callable type, as its slot and the name it is written with, in the +/// order its signature lists them: the receiver of an inline protocol's method when +/// `method_receiver` says `callable` is one's signature and writes one, an implicit +/// receiver (`int.() -> str`), then each parameter written after them +/// +/// a parameter written as a bare type is named `_`, `i` its position among the written +/// ones. no call names it, and the `__call__` the transpiler declares a callable type with +/// writes it under that name, which a repeated `_` is then numbered around +pub fn callable_parameter_slots( + callable: &ast::ExprCallableType, + method_receiver: bool, +) -> Vec<(ParameterSlot, String)> { + let receiver = if method_receiver { + protocol_method_receiver(callable) + } else { + None + }; + let offset = usize::from(receiver.is_some()); + let slash = callable + .parameter_slash() + .map(|index| (index as usize).saturating_sub(offset)); + let star = callable + .parameter_star() + .map(|index| (index as usize).saturating_sub(offset)); + let mut slots = Vec::with_capacity(callable.args.len() + 1); + if let Some(receiver) = receiver { + slots.push((ParameterSlot::PositionalOrKeyword, receiver.id.to_string())); + } + if callable.receiver.is_some() { + slots.push((ParameterSlot::PositionalOnly, String::new())); + } + let mut keyword_only = false; + for (index, element) in callable.args[offset..].iter().enumerate() { + keyword_only |= Some(index) == star; + let named = if slash.is_some_and(|slash| index < slash) { + ParameterSlot::PositionalOnly + } else if keyword_only { + ParameterSlot::KeywordOnly + } else { + ParameterSlot::PositionalOrKeyword + }; + let slot = match element { + ast::Expr::Named(parameter) => match parameter.target.as_ref() { + ast::Expr::Starred(starred) => match starred.value.as_ref() { + ast::Expr::Starred(inner) => ( + ParameterSlot::KeywordVariadic, + inner + .value + .as_name_expr() + .map_or("kwargs", |name| name.id.as_str()) + .to_owned(), + ), + value => { + keyword_only = true; + // the anonymous `*: *Ts` carries an empty name + let name = value + .as_name_expr() + .map(|name| name.id.as_str()) + .filter(|name| !name.is_empty()) + .unwrap_or("args"); + (ParameterSlot::Variadic, name.to_owned()) + } + }, + target => ( + named, + target + .as_name_expr() + .map_or_else(|| "_".to_owned(), |name| name.id.to_string()), + ), + }, + ast::Expr::Starred(starred) => match starred.value.as_ref() { + ast::Expr::Starred(_) => (ParameterSlot::KeywordVariadic, "kwargs".to_owned()), + _ => { + keyword_only = true; + (ParameterSlot::Variadic, "args".to_owned()) + } + }, + _ => (ParameterSlot::PositionalOnly, format!("_{index}")), + }; + slots.push(slot); + } + slots +} + +/// a positional parameter of the method a definition overrides +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct BaseParameter { + pub name: Name, + pub positional_only: bool, +} + +/// how the lowering writes a parameter list that repeats `_` +#[derive(Clone, Debug, PartialEq, Eq, Hash, get_size2::GetSize, salsa::SalsaValue)] +pub struct UnderscoreLowering { + /// the name each positional `_` takes from the overridden method, in declaration order. + /// `None` when they are numbered instead + inherited: Option>, + /// how many of the leading positional parameters a call reaches by position alone + positional_only: usize, +} + +impl UnderscoreLowering { + /// how many of the leading positional parameters a call reaches by position alone + pub fn positional_only(&self) -> usize { + self.positional_only + } + + /// whether the positional `_`s take their names from the overridden method. no + /// parameter is called `_` in the python then, so a read of `_` in the body finds + /// nothing of the definition's own + pub fn is_inherited(&self) -> bool { + self.inherited.is_some() + } + + /// the name each of `slots` binds in the python, in declaration order. `slots` is the + /// parameter list this lowering was decided for, and `written` the module it is in + pub fn names(&self, slots: &[(ParameterSlot, &str)], written: WrittenNames) -> Vec { + let mut spelled: Vec<(ParameterSlot, Name)> = slots + .iter() + .map(|(slot, name)| (*slot, Name::from(*name))) + .collect(); + if let Some(inherited) = &self.inherited { + let underscores = spelled + .iter_mut() + .filter(|(slot, name)| slot.is_positional() && name == "_"); + for ((_, name), base) in underscores.zip(inherited) { + name.clone_from(base); + } + } + let spelled: Vec<(ParameterSlot, &str)> = spelled + .iter() + .map(|(slot, name)| (*slot, name.as_str())) + .collect(); + number_underscores(&spelled, !self.is_inherited(), written) + } +} + +/// the name each parameter of a list takes in the python, in declaration order, which of them +/// are one of its `_`s, and how many of the leading positional ones a call reaches by position +/// alone +pub(crate) struct LoweredParameters { + pub(crate) names: Vec, + /// which of the parameters are written `_`. every one after the first has a python name + /// the author never wrote, so the signature shows each as the `_` written, and a + /// diagnostic about one says which it is by position + pub(crate) underscores: Vec, + pub(crate) positional_only: usize, +} + +impl LoweredParameters { + /// how the parameter list `slots`, written in the module `written`, is checked when + /// `lowering` writes it + pub(crate) fn new( + lowering: &UnderscoreLowering, + slots: &[(ParameterSlot, &str)], + written: WrittenNames, + ) -> Self { + // a name taken from an overridden method is a name the author can spell + let underscores = slots + .iter() + .map(|(_, spelled)| *spelled == "_" && !lowering.is_inherited()) + .collect(); + Self { + names: lowering.names(slots, written), + underscores, + positional_only: lowering.positional_only(), + } + } + + /// `parameters`, the ones of the list in declaration order, under the names and kinds + /// the lowering writes them with + pub(crate) fn apply<'db>(&self, parameters: Vec>) -> Vec> { + if self.names.len() != parameters.len() { + return parameters; + } + parameters + .into_iter() + .zip(self.names.iter().zip(&self.underscores)) + .enumerate() + .map(|(index, (parameter, (name, underscore)))| { + parameter.lowered_as(name.clone(), index < self.positional_only, *underscore) + }) + .collect() + } +} + +/// why the lowering cannot write a parameter list that repeats `_` +#[derive(Clone, Debug, PartialEq, Eq, Hash, get_size2::GetSize, salsa::SalsaValue)] +pub enum UnderscoreRefusal { + /// a `_` after `*` or `*args` that is not the one keeping the name `_`. only a keyword + /// reaches a keyword-only parameter, and its name is not the author's + KeywordOnly { index: usize }, + /// a named parameter before a `_` that has to be positional-only. python's `/` makes + /// every parameter before it positional-only, and that is a change to a parameter the + /// author named, which is theirs to write + DraggedPositionalOnly { index: usize, name: Name }, + /// the name a `_` takes from the overridden method is one the body reads from an + /// enclosing scope, which the parameter would then shadow + ShadowsEnclosingName { index: usize, name: Name }, + /// the `_`s have to be positional-only, and the python the module targets has no `/` + /// to make them so + NoPositionalOnlySyntax { index: usize }, +} + +impl UnderscoreRefusal { + /// the declaration index of the parameter the refusal is about + pub fn index(&self) -> usize { + match self { + Self::KeywordOnly { index } + | Self::DraggedPositionalOnly { index, .. } + | Self::ShadowsEnclosingName { index, .. } + | Self::NoPositionalOnlySyntax { index } => *index, + } + } + + /// what is wrong, in the words a diagnostic and a transpile error both use + pub fn message(&self) -> String { + match self { + Self::KeywordOnly { .. } => { + "a repeated `_` parameter cannot be keyword-only".to_owned() + } + Self::DraggedPositionalOnly { name, .. } => { + format!("the repeated `_` parameters after `{name}` make it positional-only") + } + Self::ShadowsEnclosingName { name, .. } => { + format!("a repeated `_` parameter named `{name}` shadows a `{name}` the body reads") + } + Self::NoPositionalOnlySyntax { .. } => { + "a repeated `_` parameter needs a `/`, which python before 3.8 does not have" + .to_owned() + } + } + } + + /// how to write the definition so it is accepted + pub fn help(&self) -> String { + match self { + Self::KeywordOnly { .. } => { + "only a keyword reaches it, and it has no name of its own: move it before the `*`, \ + or name it" + .to_owned() + } + Self::DraggedPositionalOnly { .. } => { + "write the `/` after the last `_` to make them positional-only".to_owned() + } + Self::ShadowsEnclosingName { name, .. } => format!( + "the parameter takes its name from the method it overrides; rename the `{name}` \ + the body reads" + ), + Self::NoPositionalOnlySyntax { .. } => { + "name the parameters, or require python 3.8 or later".to_owned() + } + } + } +} + +/// how the lowering writes `slots`, a parameter list in declaration order, when it spells +/// `_` more than once. `None` when it does not +/// +/// `slash` says whether the python the module targets has the `/`: python 3.8 added it, and +/// before it nothing makes a parameter positional-only — so a list whose `_`s need one is +/// refused there, rather than lowered to parameters a call reaches by the number +/// +/// `receiver` says whether the first parameter is the receiver a method binds, which a call +/// never passes by keyword, so making it positional-only changes nothing a caller wrote. +/// `base` is the overridden method's positional parameters, by position, when the definition +/// overrides one. `reads_from_enclosing_scope` says whether the body reads a name from an +/// enclosing scope, which a parameter of that name would then shadow +pub fn lower_repeated_underscores( + slots: &[(ParameterSlot, &str)], + receiver: bool, + base: Option<&[Option]>, + reads_from_enclosing_scope: impl Fn(&str) -> bool, + slash: bool, +) -> Option> { + let underscores: Vec = (0..slots.len()) + .filter(|&index| slots[index].1 == "_") + .collect(); + if underscores.len() < 2 { + return None; + } + let written_positional_only = slots + .iter() + .filter(|(slot, _)| *slot == ParameterSlot::PositionalOnly) + .count(); + let positional_underscores: Vec = underscores + .iter() + .copied() + .filter(|&index| slots[index].0.is_positional()) + .collect(); + + // every named parameter up to the `/` has to be positional-only as written, bar the + // receiver + let dragged = |positional_only: usize| { + (0..positional_only).find_map(|index| { + let (slot, name) = slots[index]; + let exempt = + slot == ParameterSlot::PositionalOnly || name == "_" || (index == 0 && receiver); + (!exempt).then(|| UnderscoreRefusal::DraggedPositionalOnly { + index, + name: Name::from(name), + }) + }) + }; + + // a `/` the lowering adds, where the target has none to add + let missing_slash = |positional_only: usize| { + (!slash && positional_only > written_positional_only).then(|| { + UnderscoreRefusal::NoPositionalOnlySyntax { + index: positional_only - 1, + } + }) + }; + + if let Some(base) = base + && let Some(inherited) = inherited_names(slots, &positional_underscores, base) + { + // no `_` keeps its name, so a keyword-only one would be numbered + if let Some(&index) = underscores + .iter() + .find(|&&index| slots[index].0 == ParameterSlot::KeywordOnly) + { + return Some(Err(UnderscoreRefusal::KeywordOnly { index })); + } + if let Some((index, parameter)) = inherited + .iter() + .find(|(_, parameter)| reads_from_enclosing_scope(¶meter.name)) + { + return Some(Err(UnderscoreRefusal::ShadowsEnclosingName { + index: *index, + name: parameter.name.clone(), + })); + } + let positional_only = inherited + .iter() + .filter(|(_, parameter)| parameter.positional_only) + .map(|(index, _)| index + 1) + .max() + .unwrap_or(0) + .max(written_positional_only); + if let Some(refusal) = dragged(positional_only).or_else(|| missing_slash(positional_only)) { + return Some(Err(refusal)); + } + return Some(Ok(UnderscoreLowering { + inherited: Some( + inherited + .into_iter() + .map(|(_, parameter)| parameter.name.clone()) + .collect(), + ), + positional_only, + })); + } + + // the first `_` in numbering order keeps its name, and every other one is numbered + let kept = underscores + .iter() + .copied() + .min_by_key(|&index| (slots[index].0, index)); + let numbered = underscores + .iter() + .copied() + .filter(|&index| Some(index) != kept); + if let Some(index) = numbered + .clone() + .find(|&index| slots[index].0 == ParameterSlot::KeywordOnly) + { + return Some(Err(UnderscoreRefusal::KeywordOnly { index })); + } + let positional_only = numbered + .filter(|&index| slots[index].0.is_positional()) + .map(|index| index + 1) + .max() + .unwrap_or(0) + .max(written_positional_only); + if let Some(refusal) = dragged(positional_only).or_else(|| missing_slash(positional_only)) { + return Some(Err(refusal)); + } + Some(Ok(UnderscoreLowering { + inherited: None, + positional_only, + })) +} + +/// the parameter of `base` each of `positional_underscores` stands in for, by position +/// +/// `None` unless every one of them has a counterpart with a name the definition does not +/// already give a parameter of its own. an override that lines up with its base in part +/// does not line up with it, and is reported as an invalid override +fn inherited_names<'a>( + slots: &[(ParameterSlot, &str)], + positional_underscores: &[usize], + base: &'a [Option], +) -> Option> { + if positional_underscores.is_empty() { + return None; + } + let own: HashSet<&str> = slots + .iter() + .map(|(_, name)| *name) + .filter(|name| *name != "_") + .collect(); + positional_underscores + .iter() + .map(|&index| { + base.get(index)? + .as_ref() + .filter(|parameter| !own.contains(parameter.name.as_str())) + .map(|parameter| (index, parameter)) + }) + .collect() +} + +/// whether a parameter named `name` would change what the body of the function whose +/// scope is `scope` reads: the body mentions the name itself, or a scope nested in it reads +/// the name without binding it, which python resolves through the function +/// +/// a nested scope that binds the name reads its own, but one nested in *that* is still +/// asked, which can only answer yes where the answer did not matter +pub(crate) fn reads_from_enclosing_scope( + index: &SemanticIndex, + scope: FileScopeId, + name: &str, +) -> bool { + if index.place_table(scope).symbol_by_name(name).is_some() { + return true; + } + let mut pending: Vec = index.child_scopes(scope).map(|(id, _)| id).collect(); + while let Some(scope) = pending.pop() { + if let Some(symbol) = index.place_table(scope).symbol_by_name(name) + && !symbol.is_global() + && (!symbol.is_bound() || symbol.is_nonlocal()) + { + return true; + } + pending.extend(index.child_scopes(scope).map(|(id, _)| id)); + } + false +} + +/// the name each of `parameters`, a parameter list written as each one's slot and source +/// name, binds in the python it lowers to, where no parameter takes a name from an +/// overridden method +/// +/// a parameter list the transpiler writes out of something other than a `def` — the +/// `__call__` of a callable type naming its parameters — numbers a repeated `_` as the +/// `def` it declares would. a number is never one the list or the module in `written` +/// already spells +pub fn lowered_parameter_names( + parameters: &[(ParameterSlot, &str)], + written: WrittenNames, +) -> Vec { + number_underscores(parameters, true, written) +} + +/// `parameters` with every `_` numbered, bar the first in numbering order when `keep_first` +fn number_underscores( + parameters: &[(ParameterSlot, &str)], + keep_first: bool, + written: WrittenNames, +) -> Vec { + let taken: HashSet<&str> = parameters.iter().map(|(_, name)| *name).collect(); + let mut order: Vec = (0..parameters.len()).collect(); + order.sort_by_key(|&index| parameters[index].0); + let mut names: Vec = parameters + .iter() + .map(|(_, name)| Name::from(*name)) + .collect(); + let mut keep = keep_first; + // candidates only ever count up, so one never repeats an earlier one + let mut next = 2u32; + for index in order { + if parameters[index].1 != "_" || std::mem::replace(&mut keep, false) { + continue; + } + let candidate = loop { + let candidate = format!("_{next}"); + next += 1; + if !taken.contains(candidate.as_str()) && !written.spells(&candidate) { + break candidate; + } + }; + names[index] = Name::from(candidate); + } + names +} + +#[cfg(test)] +mod tests { + use super::*; + + use ParameterSlot::{KeywordOnly, PositionalOnly, PositionalOrKeyword, Variadic}; + + fn lower( + slots: &[(ParameterSlot, &str)], + receiver: bool, + base: Option<&[Option]>, + ) -> Option> { + lower_repeated_underscores(slots, receiver, base, |_| false, true) + } + + fn base(parameters: &[(&str, bool)]) -> Vec> { + parameters + .iter() + .map(|(name, positional_only)| { + Some(BaseParameter { + name: Name::from(*name), + positional_only: *positional_only, + }) + }) + .collect() + } + + #[test] + fn a_single_underscore_is_left_alone() { + assert_eq!(lower(&[(PositionalOrKeyword, "_")], false, None), None); + } + + #[test] + fn numbered_underscores_are_positional_only() { + let slots = [(PositionalOrKeyword, "_"), (PositionalOrKeyword, "_")]; + let lowering = lower(&slots, false, None).unwrap().unwrap(); + assert_eq!(lowering.positional_only(), 2); + assert_eq!(lowering.names(&slots, WrittenNames::new("")), ["_", "_2"]); + } + + #[test] + fn a_named_parameter_after_the_last_underscore_stays_a_keyword() { + let slots = [ + (PositionalOrKeyword, "_"), + (PositionalOrKeyword, "_"), + (PositionalOrKeyword, "a"), + ]; + let lowering = lower(&slots, false, None).unwrap().unwrap(); + assert_eq!(lowering.positional_only(), 2); + } + + #[test] + fn a_named_parameter_before_the_last_underscore_is_refused() { + let slots = [ + (PositionalOrKeyword, "a"), + (PositionalOrKeyword, "_"), + (PositionalOrKeyword, "_"), + ]; + assert_eq!( + lower(&slots, false, None), + Some(Err(UnderscoreRefusal::DraggedPositionalOnly { + index: 0, + name: Name::from("a") + })) + ); + } + + #[test] + fn the_receiver_is_not_dragged() { + let slots = [ + (PositionalOrKeyword, "self"), + (PositionalOrKeyword, "_"), + (PositionalOrKeyword, "_"), + ]; + let lowering = lower(&slots, true, None).unwrap().unwrap(); + assert_eq!(lowering.positional_only(), 3); + } + + #[test] + fn a_written_slash_is_accepted() { + let slots = [ + (PositionalOnly, "a"), + (PositionalOnly, "_"), + (PositionalOnly, "_"), + ]; + let lowering = lower(&slots, false, None).unwrap().unwrap(); + assert_eq!(lowering.positional_only(), 3); + } + + #[test] + fn a_numbered_keyword_only_underscore_is_refused() { + let slots = [(PositionalOrKeyword, "_"), (KeywordOnly, "_")]; + assert_eq!( + lower(&slots, false, None), + Some(Err(UnderscoreRefusal::KeywordOnly { index: 1 })) + ); + } + + /// python before 3.8 has no `/`, so a list whose `_`s need one is refused there, and a + /// `/` the author wrote asks for nothing more + #[test] + fn a_slash_the_target_cannot_write_is_refused() { + let slots = [(PositionalOrKeyword, "_"), (PositionalOrKeyword, "_")]; + assert_eq!( + lower_repeated_underscores(&slots, false, None, |_| false, false), + Some(Err(UnderscoreRefusal::NoPositionalOnlySyntax { index: 1 })) + ); + let written = [(PositionalOnly, "_"), (PositionalOnly, "_")]; + assert!(matches!( + lower_repeated_underscores(&written, false, None, |_| false, false), + Some(Ok(_)) + )); + } + + #[test] + fn a_variadic_underscore_needs_no_slash() { + let slots = [ + (PositionalOrKeyword, "a"), + (PositionalOrKeyword, "_"), + (Variadic, "_"), + ]; + let lowering = lower(&slots, false, None).unwrap().unwrap(); + assert_eq!(lowering.positional_only(), 0); + assert_eq!( + lowering.names(&slots, WrittenNames::new("")), + ["a", "_", "_2"] + ); + } + + #[test] + fn an_override_takes_the_base_names_and_kinds() { + let slots = [ + (PositionalOrKeyword, "self"), + (PositionalOrKeyword, "_"), + (PositionalOrKeyword, "_"), + ]; + let base = base(&[("self", false), ("x", true), ("y", false)]); + let lowering = lower(&slots, true, Some(&base)).unwrap().unwrap(); + assert_eq!(lowering.positional_only(), 2); + assert_eq!( + lowering.names(&slots, WrittenNames::new("")), + ["self", "x", "y"] + ); + } + + #[test] + fn an_override_that_does_not_line_up_is_numbered() { + let slots = [ + (PositionalOrKeyword, "self"), + (PositionalOrKeyword, "_"), + (PositionalOrKeyword, "_"), + ]; + let base = base(&[("self", false), ("x", false)]); + let lowering = lower(&slots, true, Some(&base)).unwrap().unwrap(); + assert!(!lowering.is_inherited()); + } + + #[test] + fn an_inherited_name_the_body_reads_is_refused() { + let slots = [(PositionalOrKeyword, "_"), (PositionalOrKeyword, "_")]; + let base = base(&[("x", false), ("y", false)]); + assert_eq!( + lower_repeated_underscores(&slots, false, Some(&base), |name| name == "y", true), + Some(Err(UnderscoreRefusal::ShadowsEnclosingName { + index: 1, + name: Name::from("y") + })) + ); + } +} diff --git a/crates/ty_python_semantic/src/types/signatures.rs b/crates/ty_python_semantic/src/types/signatures.rs index cf8cef907a..540e6400d2 100644 --- a/crates/ty_python_semantic/src/types/signatures.rs +++ b/crates/ty_python_semantic/src/types/signatures.rs @@ -42,6 +42,7 @@ use crate::types::instance::ProtocolInstanceType; use crate::types::relation::{ HasRelationToVisitor, IsDisjointVisitor, TypeRelation, TypeRelationChecker, TypeVarEvaluation, }; +use crate::types::repeated_underscore::LoweredParameters; use crate::types::tuple::{Tuple, TupleSpec, TupleSpecBuilder, TupleType, VariableSegment}; use crate::types::typed_dict::extract_unpacked_typed_dict_keys_from_kwargs_annotation; use crate::types::typevar::{ @@ -1018,6 +1019,7 @@ impl<'db> Signature<'db> { } /// Return a typed signature from a function definition. + #[expect(clippy::too_many_arguments)] pub(super) fn from_function( db: &'db dyn Db, env: &ProgramEnvironment<'db>, @@ -1026,12 +1028,14 @@ impl<'db> Signature<'db> { function_node: &ast::StmtFunctionDef, has_implicitly_positional_first_parameter: bool, return_callable_typevar_scope: ReturnCallableTypeVarScope, + lowered: Option<&LoweredParameters>, ) -> Self { let parameters = Parameters::from_parameters( db, definition, function_node.parameters.as_ref(), has_implicitly_positional_first_parameter, + lowered, ); let return_ty = if function_node.is_asserts_return { // basedpython: `-> asserts x` names a place, not a type. such a function @@ -2707,6 +2711,77 @@ impl<'db> Signature<'db> { pub(crate) fn with_return_type(self, return_ty: Type<'db>) -> Self { Self { return_ty, ..self } } + + /// basedpython: the two ways a `decorator def` declared by this signature is called. + /// + /// `decorator def d(fn: (int) -> None, option: bool = False) -> str` is applied as `@d`, which + /// calls it with the decorated function, and as `@d(option=True)`, which calls it with the + /// options alone and decorates with what that returns. So it is called as + /// + /// ```text + /// (fn: (int) -> None, *, option: bool = False) -> str + /// (*, option: bool = False) -> ((int) -> None) -> str + /// ``` + /// + /// The options are keyword-only in both: the lowering writes one dispatcher for the two + /// shapes, and it can only tell them apart by whether it was given the function positionally. + /// + /// `self` is the signature as callers see it and `lexical` the same signature with the type + /// variables of a returned callable still bound to the function — the form the second shape's + /// returned callable is built from, since its parameter and return type move into it. + /// + /// `None` when the declaration is not one the lowering can write a dispatcher for: it must + /// take the decorated function as its first parameter, positionally and without a default, and + /// every other parameter must be one a keyword can name and must have a default. + pub(super) fn decorator_keyword_overloads( + db: &'db dyn Db, + public: &Self, + lexical: &Self, + definition: Definition<'db>, + ) -> Option<[Self; 2]> { + let shape = |signature: &Self| { + let mut parameters = signature.parameters().iter(); + let decorated = parameters.next()?; + if !decorated.is_positional() || decorated.has_default() { + return None; + } + let options = parameters + .map(|option| { + option + .has_default() + .then(|| option.clone().into_keyword_only()) + .flatten() + }) + .collect::>>()?; + Some((decorated.clone(), options)) + }; + + let (decorated, options) = shape(public)?; + let called_with_the_function = public.clone().with_parameters(Parameters::standard( + std::iter::once(decorated).chain(options.clone()), + )); + + let (decorated, options) = shape(lexical)?; + let decorating = Type::Callable(CallableType::single( + db, + Self::new( + Parameters::standard([decorated.into_positional_only()?]), + lexical.return_ty, + ), + )); + let options = Parameters::standard(options); + let (generic_context, decorating) = GenericContext::remove_callable_only_typevars( + db, + lexical.generic_context, + &options, + decorating, + definition, + ); + let called_with_the_options = Self::new_generic(generic_context, options, decorating) + .with_definition(Some(definition)); + + Some([called_with_the_function, called_with_the_options]) + } } impl<'db> VarianceInferable<'db> for &Signature<'db> { @@ -5721,6 +5796,7 @@ impl<'db> Parameters<'db> { definition: Definition<'db>, parameters: &ast::Parameters, has_implicitly_positional_first_parameter: bool, + lowered: Option<&LoweredParameters>, ) -> Self { let ast::Parameters { posonlyargs, @@ -5853,18 +5929,24 @@ impl<'db> Parameters<'db> { ) }); - Self::from_annotation( - db, - &env, - positional_only - .into_iter() - .chain(positional_or_keyword) - .chain(variadic) - .chain(keyword_only) - .chain(keywords) - .enumerate() - .map(|(index, parameter)| parameter.with_source_parameter_index(Some(index))), - ) + let parameters: Vec> = positional_only + .into_iter() + .chain(positional_or_keyword) + .chain(variadic) + .chain(keyword_only) + .chain(keywords) + .enumerate() + .map(|(index, parameter)| parameter.with_source_parameter_index(Some(index))) + .collect(); + + // basedpython: a parameter list that repeats `_` takes the names and kinds the + // lowering writes it with, so a call is checked against the python it runs as + let parameters = match lowered { + Some(lowered) => lowered.apply(parameters), + None => parameters, + }; + + Self::from_annotation(db, &env, parameters) } fn apply_type_mapping_impl<'a>( @@ -6269,10 +6351,9 @@ pub(crate) struct Parameter<'db> { /// declarations in scope at the call site is_context: bool, - /// basedpython: the parameter is the implicit receiver of a - /// `T.() -> R` callable — it is bound by an `x.fn()` call and supplies the - /// implicit member scope of a trailing lambda block - is_receiver: bool, + /// basedpython: whether the parameter is the implicit receiver of a + /// `T.() -> R` callable, or one of a list's repeated `_`s + role: ParameterRole, /// basedpython: the `local` / `once` modifier the parameter was declared /// with in a callable type — the `local` of `(local int) -> None`. @@ -6294,6 +6375,19 @@ pub(crate) struct Parameter<'db> { kind: ParameterKind<'db>, } +/// basedpython: what a parameter is, beyond its kind, that a signature shows or a +/// diagnostic says about it +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, get_size2::GetSize)] +enum ParameterRole { + Plain, + /// the implicit receiver of a `T.() -> R` callable — it is bound by an `x.fn()` call and + /// supplies the implicit member scope of a trailing lambda block + Receiver, + /// one of a parameter list's repeated `_`s. every one after the first has a name the + /// lowering gave it, which the author never wrote + RepeatedUnderscore, +} + #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, get_size2::GetSize)] enum ParameterAnnotationKind { Normal, @@ -6323,7 +6417,7 @@ impl<'db> Parameter<'db> { inferred_annotation: true, annotation_kind: ParameterAnnotationKind::Normal, is_context: false, - is_receiver: false, + role: ParameterRole::Plain, borrow: ParameterBorrow::None, source_parameter_index: None, kind: ParameterKind::PositionalOnly { @@ -6340,7 +6434,7 @@ impl<'db> Parameter<'db> { inferred_annotation: true, annotation_kind: ParameterAnnotationKind::Normal, is_context: false, - is_receiver: false, + role: ParameterRole::Plain, borrow: ParameterBorrow::None, source_parameter_index: None, kind: ParameterKind::PositionalOrKeyword { @@ -6357,7 +6451,7 @@ impl<'db> Parameter<'db> { inferred_annotation: true, annotation_kind: ParameterAnnotationKind::Normal, is_context: false, - is_receiver: false, + role: ParameterRole::Plain, borrow: ParameterBorrow::None, source_parameter_index: None, kind: ParameterKind::Variadic { name }, @@ -6371,7 +6465,7 @@ impl<'db> Parameter<'db> { inferred_annotation: true, annotation_kind: ParameterAnnotationKind::Normal, is_context: false, - is_receiver: false, + role: ParameterRole::Plain, borrow: ParameterBorrow::None, source_parameter_index: None, kind: ParameterKind::KeywordOnly { @@ -6388,7 +6482,7 @@ impl<'db> Parameter<'db> { inferred_annotation: true, annotation_kind: ParameterAnnotationKind::Normal, is_context: false, - is_receiver: false, + role: ParameterRole::Plain, borrow: ParameterBorrow::None, source_parameter_index: None, kind: ParameterKind::KeywordVariadic { name }, @@ -6451,6 +6545,44 @@ impl<'db> Parameter<'db> { .flatten() } + /// basedpython: this parameter made keyword-only, keeping its name, annotation and default. + /// + /// `None` for a parameter no keyword can name: a variadic one, or a positional-only one that + /// a `Callable` annotation left unnamed. + pub(crate) fn into_keyword_only(self) -> Option { + let (name, default_type) = match self.kind { + ParameterKind::PositionalOnly { + name: Some(name), + default_type, + } + | ParameterKind::PositionalOrKeyword { name, default_type } + | ParameterKind::KeywordOnly { name, default_type } => (name, default_type), + ParameterKind::PositionalOnly { name: None, .. } + | ParameterKind::Variadic { .. } + | ParameterKind::KeywordVariadic { .. } => return None, + }; + Some(Self { + kind: ParameterKind::KeywordOnly { name, default_type }, + ..self + }) + } + + /// basedpython: this parameter made positional-only, keeping its name, annotation and default. + /// + /// `None` for a variadic parameter, which stands for a run of arguments rather than one. + pub(crate) fn into_positional_only(self) -> Option { + let (name, default_type) = match self.kind { + ParameterKind::PositionalOnly { name, default_type } => (name, default_type), + ParameterKind::PositionalOrKeyword { name, default_type } + | ParameterKind::KeywordOnly { name, default_type } => (Some(name), default_type), + ParameterKind::Variadic { .. } | ParameterKind::KeywordVariadic { .. } => return None, + }; + Some(Self { + kind: ParameterKind::PositionalOnly { name, default_type }, + ..self + }) + } + pub(crate) fn with_default_type(mut self, default: Type<'db>) -> Self { self.set_default_type(default); self @@ -6528,7 +6660,7 @@ impl<'db> Parameter<'db> { inferred_annotation: self.inferred_annotation, annotation_kind: self.annotation_kind, is_context: self.is_context, - is_receiver: self.is_receiver, + role: self.role, borrow: self.borrow, source_parameter_index: self.source_parameter_index, } @@ -6553,7 +6685,7 @@ impl<'db> Parameter<'db> { inferred_annotation: self.inferred_annotation, annotation_kind: self.annotation_kind, is_context: self.is_context, - is_receiver: self.is_receiver, + role: self.role, borrow: self.borrow, source_parameter_index: self.source_parameter_index, kind, @@ -6573,7 +6705,7 @@ impl<'db> Parameter<'db> { annotation_kind, inferred_annotation, is_context, - is_receiver, + role, borrow, source_parameter_index, kind, @@ -6607,7 +6739,7 @@ impl<'db> Parameter<'db> { inferred_annotation: *inferred_annotation, annotation_kind: *annotation_kind, is_context: *is_context, - is_receiver: *is_receiver, + role: *role, borrow: *borrow, source_parameter_index: *source_parameter_index, kind, @@ -6661,7 +6793,7 @@ impl<'db> Parameter<'db> { inferred_annotation, annotation_kind, is_context: parameter.is_context, - is_receiver: false, + role: ParameterRole::Plain, // a `def`'s own `local` / `once` prefix is recovered from the source // span, not carried on the signature — only a callable type records // the modifier here @@ -6671,6 +6803,50 @@ impl<'db> Parameter<'db> { } } + /// basedpython: this parameter under `name`, the one the lowering writes it with, and + /// positional-only when the lowering puts it before a `/`. `underscore` says it is one of + /// the list's repeated `_`s + pub(crate) fn lowered_as( + mut self, + name: Name, + positional_only: bool, + underscore: bool, + ) -> Self { + if underscore { + self.role = ParameterRole::RepeatedUnderscore; + } + self.kind = match self.kind { + ParameterKind::PositionalOnly { + name: written, + default_type, + } => ParameterKind::PositionalOnly { + name: written.map(|_| name), + default_type, + }, + ParameterKind::PositionalOrKeyword { default_type, .. } if positional_only => { + ParameterKind::PositionalOnly { + name: Some(name), + default_type, + } + } + ParameterKind::PositionalOrKeyword { default_type, .. } => { + ParameterKind::PositionalOrKeyword { name, default_type } + } + ParameterKind::Variadic { .. } => ParameterKind::Variadic { name }, + ParameterKind::KeywordOnly { default_type, .. } => { + ParameterKind::KeywordOnly { name, default_type } + } + ParameterKind::KeywordVariadic { .. } => ParameterKind::KeywordVariadic { name }, + }; + self + } + + /// basedpython: whether this parameter is one of a list's repeated `_`s. every one after + /// the first has a name the lowering gave it, which the author never wrote + pub(crate) fn is_repeated_underscore(&self) -> bool { + self.role == ParameterRole::RepeatedUnderscore + } + /// Returns `true` if this is a keyword-only parameter. pub(crate) fn is_keyword_only(&self) -> bool { matches!(self.kind, ParameterKind::KeywordOnly { .. }) @@ -6745,7 +6921,7 @@ impl<'db> Parameter<'db> { /// basedpython: whether this is the implicit receiver parameter of a /// `T.() -> R` callable pub(crate) fn is_receiver(&self) -> bool { - self.is_receiver + self.role == ParameterRole::Receiver } /// basedpython: mark this parameter as the implicit receiver of a @@ -6763,7 +6939,7 @@ impl<'db> Parameter<'db> { } pub(crate) fn with_receiver(mut self) -> Self { - self.is_receiver = true; + self.role = ParameterRole::Receiver; self } @@ -6802,13 +6978,19 @@ impl<'db> Parameter<'db> { /// Display name of the parameter, if it has one. pub(crate) fn display_name(&self) -> Option> { + // basedpython: a repeated `_` is shown as the `_` the author wrote + static UNDERSCORE: Name = Name::new_static("_"); let prefix = match self.kind { ParameterKind::Variadic { .. } => ParameterNamePrefix::Variadic, ParameterKind::KeywordVariadic { .. } => ParameterNamePrefix::KeywordVariadic, _ => ParameterNamePrefix::None, }; - self.name() - .map(|name| ParameterDisplayName { name, prefix }) + let name = if self.is_repeated_underscore() { + Some(&UNDERSCORE) + } else { + self.name() + }; + name.map(|name| ParameterDisplayName { name, prefix }) } /// Returns whether this parameter has a default without inferring its type. diff --git a/crates/ty_python_semantic/src/types/state_reads.rs b/crates/ty_python_semantic/src/types/state_reads.rs index ff3aa0c689..06695c445c 100644 --- a/crates/ty_python_semantic/src/types/state_reads.rs +++ b/crates/ty_python_semantic/src/types/state_reads.rs @@ -787,6 +787,7 @@ impl<'a, 'db> ReadsCollector<'a, 'db> { // name in scope here, exactly as if the call had written it for implicit in implicit_context_arguments(db, &self.env, self.program_file.file(db), callee, call) + .arguments { if implicit.is_block_receiver { continue; diff --git a/crates/ty_server/src/server/api/requests/entry_point.rs b/crates/ty_server/src/server/api/requests/entry_point.rs index cf0096c0c6..08db32b9ab 100644 --- a/crates/ty_server/src/server/api/requests/entry_point.rs +++ b/crates/ty_server/src/server/api/requests/entry_point.rs @@ -149,7 +149,13 @@ impl BackgroundRequestHandler for EntryPointHandler { let source_type = file.source_type(db); let generates_entry_points = source_type.is_basedpython() && !source_type.is_stub(); let main = generates_entry_points - .then(|| by_transforms::entry_point::entry_point(body, source.as_str())) + .then(|| { + by_transforms::entry_point::entry_point( + body, + source.as_str(), + by_transforms::WrittenNames::new(source.as_str()), + ) + }) .flatten() .and_then(|entry| { Some(MainFunction { diff --git a/docs/basedpython/development/how-transpilation-works.md b/docs/basedpython/development/how-transpilation-works.md index 9d13ddbe48..fef6a28959 100644 --- a/docs/basedpython/development/how-transpilation-works.md +++ b/docs/basedpython/development/how-transpilation-works.md @@ -20,8 +20,9 @@ source (.by) │ │ sentinel, mutable-defaults, …) │ ├─ run TypeAwarePasses: read the SemanticModel and emit text edits │ │ (intersection, callable, generics, literal-types, anon-NT, …) - │ └─ splice it together: re-render changed statements, apply text edits - │ (ruff-style first-wins overlap skip), emit hoisted class defs, prepend + │ └─ splice it together: re-emit the nodes the AST passes changed, apply + │ text edits (ruff-style first-wins overlap skip, composed inside the + │ nodes both touch), emit hoisted class defs, prepend │ required imports and the runtime helpers the emitted code calls │ (see "the runtime" below), append `__all__` epilogue │ @@ -140,8 +141,10 @@ and gets the single-file db: correct, just blind past the file - **`AstPass`** — mutates the AST in place via the [`Transformer`](https://docs.rs/ruff_python_ast) protocol. the driver tracks - which top-level statements changed and re-renders them through - `ruff_python_codegen` (basedpython mode) + which top-level statements changed, and re-emits only the nodes in them the + pass changed, through `ruff_python_codegen` (basedpython mode). a node it made + carries no source range; a node it kept carries its own, and that is how the + two are told apart - **`TypeAwarePass`** — reads the shared `SemanticModel` and emits sub-statement text edits keyed by `TextRange`. it never mutates the AST, because `inferred_type` binds to the exact parsed node identities @@ -170,11 +173,43 @@ keeps a mutable default without its guard after the passes run, the driver assembles the output in one pass: -1. whole-statement replacements for mutated statements (re-rendered) and hoisted - statements (synthesized class defs inserted before the statement that needs - them) +1. hoisted statements (synthesized class defs inserted before the statement that + needs them) +1. a template for each node an AST pass changed. it is printed from the syntax + tree with every node below it that kept its source range as a placeholder, + and each placeholder passes that node's source through — so a conversion, a + quoted forward reference or an extension call a `TypeAwarePass` lowered inside + it is applied there, where printing the statement whole would have dropped it. + whether a node changed is read by printing it the same way from the parse and + comparing, with the names it spells as the source spelled them: a node whose + one change is a name, a parameter a repeated `_` numbered, re-emits only that + name, and a compound statement whose one change is the statements of a suite + re-emits only that suite — its header keeps its source, where the parser + stands up what python has no spelling for (a modifier as a decorator, + `init(...)` as a `def __init__`, a `raises` clause) and the passes that lower + it edit that source. a suite written on the line of its clause (`def f(): x`) + is re-emitted as a block below it, since it has no line of its own to keep. a node a pass builds by parsing text of its own has to forget the + ranges into that text, or they would name source they did not come from. + between two nodes it passes through, a template keeps the source wherever it + prints the same tokens there — the comments, the blank lines, the layout of a + signature, and the edits other passes made in that text — and elsewhere keeps + the comments ahead of the first token it prints differently and after the + last, where it breaks the line too. a comment between two tokens it prints + differently has no line of its own in the output, and is not kept. what a + template prints is indented by the step its node's own source indents its + blocks by, since a file may indent one block by two spaces and another by four 1. sub-statement text edits, applied with ruff-style first-wins overlap skip — - a wider edit wins over a narrower one nested inside it + a wider edit wins over a narrower one nested inside it, and a template + applies the edits inside the source it passes through. the splice records + every edit it writes out, and one it leaves out has been lost — whatever the + wider edit prints there stands in for that source, however it spells it — + and is a transpile error, unless the wider edit accounts for it: it deletes + what it covers, it writes the same thing at the same span, one pass wrote + both, or its pass declares that it writes the lost edit's lowering itself + (`TypeAwarePass::subsumes`, a list of `Lowering`s). a changed node accounts + for nothing, and a lowering no pass declares is refused wherever another + edit covers it, so a new lowering that lands inside a construct some pass + writes itself fails the transpile until that pass takes it on 1. `required_imports` prepended (deduped, `from`-imports merged) 1. `__all__` epilogue appended for `export`/`public` modifiers diff --git a/docs/basedpython/development/sourcemaps.md b/docs/basedpython/development/sourcemaps.md index e8500b017b..f8f464c6ed 100644 --- a/docs/basedpython/development/sourcemaps.md +++ b/docs/basedpython/development/sourcemaps.md @@ -93,6 +93,16 @@ writes a note saying which file no longer matches. a frame pointing at `build/main.py` is a worse answer, but a frame quoting a `.by` that has been rewritten since is a false one +`by run` also reads the map when it loads a staged module: python's own loader +would compile the `.py`, so the shim compiles it instead, under the `.by` path and +with each line the `.by` line it came from (a generated line is line 0). every +location the running program reads back — a traceback it formats, the place a +warning is attributed to, `co_filename`, `inspect` — is then the author's. the +digests are checked then, and a mismatch leaves the module to python's loader. a +`.by` saved over during the run is caught when a frame is reported: the frame +names the one generated line that `.by` line was written as, or no line when +there was more than one + both tables are keyed by the generated path exactly as the map spells it. the shim resolves symlinks before matching a frame's filename against it — a temporary tree under `/tmp` on macOS is reached by a different path than the one diff --git a/docs/basedpython/features/context-parameters.md b/docs/basedpython/features/context-parameters.md index a098561f0e..c2dde0968b 100644 --- a/docs/basedpython/features/context-parameters.md +++ b/docs/basedpython/features/context-parameters.md @@ -133,23 +133,64 @@ pass the argument themselves ## diagnostics - `missing-context-argument` — a `context` parameter is unmatched and nothing - in scope fits it + in scope fits it, what fits it is a `_` parameter its function repeats, the + parameter is itself a `_` its function repeats, or the call is a decoration, + which has no argument list to write the value in - `ambiguous-context-argument` — several declarations in the winning scope fit it ## limitations - resolution happens only at direct call expressions (`f(...)`) whose callee - is a plain function or bound method with a single signature. constructors, - `__call__` instances, overloaded functions, union-typed callees, dunder - dispatch, and calls with `*` / `**` unpacking all keep the plain - missing-argument behaviour — pass the argument explicitly there + is a plain function or bound method. constructors, `__call__` instances, + union-typed callees, dunder dispatch, and calls with `*` / `**` unpacking + all keep the plain missing-argument behaviour — pass the argument + explicitly there + +- an **overloaded** callee is filled only where its overloads agree. which + overload a call selects is decided by the arguments it is given, so the + parameter has to mean the same thing in every overload — the same name, + resolving to the same value, and keyword-only in all of them, so that + whether the call already supplies it cannot depend on which one is chosen. + a `decorator def`'s options are keyword-only in both of its overloads, so + they are filled; anything the overloads disagree on keeps the plain + missing-argument behaviour + +- a **decoration** cannot supply one. `@deco` is a call, but it is the one call + the source writes no argument list for, so there is nowhere to put the + implicit argument. a `context` parameter left unfilled there is reported + rather than quietly taking its default: + + ```by + def deco(fn: (...) -> object, context b: str = "default") -> object: ... + + context t: str = "hello" + + @deco # error: `b` cannot be filled at a decoration + def g(): ... + + g = deco(g) # ok — `t` is passed implicitly + ``` + + the decorated definition is the decorator's first positional argument, so a + `context` parameter standing in that slot is filled by it like any other + - candidates are typed at their declaration site: reassigning a context variable to a different (assignable) type between declaration and call is not tracked + - a declaration inside a conditional branch counts lexically; whether the branch executed is not tracked +- a function that repeats `_` as a parameter name cannot supply a `context` + argument from any of those parameters: python binds only one of them to + `_`, and which one is not decided yet. give the parameter a name of its own + +- a `context` parameter named `_` whose function repeats `_` as a parameter + name is not filled implicitly either: which of those parameters a keyword + `_` names is not decided yet. pass the argument positionally, or give the + parameter a name of its own + ## inlay hints a call site shows the arguments it fills implicitly, written where the lowering diff --git a/docs/basedpython/features/decorated-parameters.md b/docs/basedpython/features/decorated-parameters.md index 2556e8c35c..3a07542303 100644 --- a/docs/basedpython/features/decorated-parameters.md +++ b/docs/basedpython/features/decorated-parameters.md @@ -73,8 +73,11 @@ this covers the gradual `(...) -> T`, a parameter pack, and any declared paramet terms of the decorator's own type variables — a type variable belongs to the scope that declared it, so it cannot stand as a parameter type here -an overloaded decorator declares nothing either: which overload a decoration picks depends on the -shape of the function, which is the very thing being inferred +an overloaded decorator declares what the overload a decoration can call declares. a decoration +passes the function as one positional argument, so an overload that cannot be called that way is +never the one it picks — the option-only overload of a [`decorator def`](decorator-keyword.md), say. +when more than one of them can take the function, which one a decoration picks depends on the shape +of the function, which is the very thing being inferred, and nothing is declared ## more than one decorator diff --git a/docs/basedpython/features/differences-from-python.md b/docs/basedpython/features/differences-from-python.md index 0a2b060927..8cc2af8d6f 100644 --- a/docs/basedpython/features/differences-from-python.md +++ b/docs/basedpython/features/differences-from-python.md @@ -75,6 +75,25 @@ module scope — no longer happens just because the importing module was loaded. `from __future__ import ...`, `from x import *`, and an unaliased `import a.b` stay eager. see [lazy imports](lazy-imports.md) +### an annotation read at runtime is its python spelling + +a program that reads its own annotations reads the python they were lowered to: + +```by +def f(x: int?, cb: (int) -> str): ... + +print(f.__annotations__) +``` + +prints `{'x': int | None, 'cb': typing.Callable[[int], str]}`. +where python keeps annotations as strings — below python 3.10, or under +`from __future__ import annotations` — the strings are the lowered text too, +`'int | None'` or `'Union[int, None]'` rather than `'int?'`. there is no string +that says what you wrote: python evaluates that string to get the type, and +`int?` is not python. `typing.get_type_hints` and `inspect.signature` read the +same annotations, and a dataclass's fields carry them, so all of them show the +python spelling + ## what an annotation means the same annotation denotes a different type diff --git a/docs/basedpython/features/exceptions.md b/docs/basedpython/features/exceptions.md index 4e3c0ba1f0..c759e9ae52 100644 --- a/docs/basedpython/features/exceptions.md +++ b/docs/basedpython/features/exceptions.md @@ -307,9 +307,10 @@ one reified changes how the program is built, and turning a check on must not do that. an unreified parameter stays on its ceiling, where a `PermissionError` passes -a decorated function whose statement another lowering re-renders cannot carry -the guard — the insertion sits inside the range being rebuilt — and that is a -transpile error rather than a silently missing check +a decorated function whose `def` another lowering rewrites — numbering a +repeated `_` parameter is one — cannot carry the guard: the `def` is printed from +its syntax tree, and the insertion has nowhere in it to go. that is a transpile +error rather than a silently missing check ## known gaps diff --git a/docs/basedpython/features/lazy-imports.md b/docs/basedpython/features/lazy-imports.md index 79faf48c83..facea98208 100644 --- a/docs/basedpython/features/lazy-imports.md +++ b/docs/basedpython/features/lazy-imports.md @@ -91,6 +91,62 @@ from errors import ParseError decode = _lazy_attr("errors", "decode") ``` +nor is a **special form** — `ClassVar`, `Final`, `Literal`, `Annotated`, `Protocol` and +the rest of what a type checker gives a meaning of its own. what reads one tells it +apart by identity: `dataclasses` decides a bare `ClassVar` annotation is not a field +by asking whether it *is* `typing.ClassVar`, and a checker reading the emitted python +rejects `ClassVar[int]` once `ClassVar` names a variable. that holds for the imports the +transpiler writes itself, such as the `ClassVar` an `enum class` lowers through: + +```py +from typing import ClassVar +cast = _lazy_attr("typing", "cast") +``` + +nor is a name an **annotation** reads. whatever reads an annotation back is handed what +the name is bound to, and a proxy is not the object it stands for: `dataclasses` decides +that `_: KW_ONLY` is not a field by asking whether the annotation *is* +`dataclasses.KW_ONLY`, and `typing.get_type_hints` answers with the proxy rather than the +class. so a `from` import whose name any annotation reads stays an ordinary import — a +parameter's or a return's annotation, a variable's in a module or a class body, or one +that is a string, as every annotation is under `from __future__ import annotations`: + +```by +from fractions import Fraction +from json import dumps + +def show(value: Fraction) -> str: + return dumps(str(value)) +``` + +```py +from fractions import Fraction +dumps = _lazy_attr("json", "dumps") +``` + +this is an edge of the polyfill: **an import used only in annotations is not deferred on +python before 3.15**, and the module it names runs where the import is written — so one +this python does not have fails there, as an ordinary import does. an +annotation on a variable inside a function body is never evaluated, so it does not count. +on 3.15 and later the `lazy` keyword binds the real object, and such an import is deferred +like any other + +the functional forms build annotations out of values instead: `NamedTuple("P", [("x", T)])`, +`TypedDict("TD", {"a": T})` and `dataclasses.make_dataclass(...)` are handed their types as +arguments, when the call runs. a name imported with `from` and read only there is read as a +value, so under the polyfill it is still a proxy, and whatever reads those annotations back is +handed the proxy: `typing.get_type_hints(P)["x"]` is the proxy rather than the class, and +`make_dataclass` makes a field of the `_` in this `Row`, and leaves `extra` positional: + +```by +from dataclasses import KW_ONLY, make_dataclass + +Row = make_dataclass("Row", [("name", str), ("_", KW_ONLY), ("extra", int)]) +``` + +reach such a name through its module — `import dataclasses` and `dataclasses.KW_ONLY` — which +binds the real module rather than a proxy, or target `--min-version 3.15` + two things a proxy cannot emulate, and which are therefore limitations of the polyfill only: diff --git a/docs/basedpython/features/repeated-underscore.md b/docs/basedpython/features/repeated-underscore.md index d312aea1dd..551b88114b 100644 --- a/docs/basedpython/features/repeated-underscore.md +++ b/docs/basedpython/features/repeated-underscore.md @@ -1,16 +1,137 @@ # repeated `_` parameters -basedpython lets `_` appear more than once in a function or lambda signature. -python rejects this as a duplicate-parameter error; basedpython only -exempts `_` (the unused-argument convention), so signatures that ignore -several arguments don't need invented placeholders: +basedpython lets `_` appear more than once in a function or lambda signature, so +a signature that ignores several arguments doesn't need invented placeholders. +the `_`s are passed by position: ```by -def f(_, _): - print(_) +def on_event(_: int, _: str): + print("fired") -g = lambda _, _: 1 +on_event(1, "a") ``` -references to `_` in the body resolve to the first parameter. duplicates of -any other name remain a syntax error +transpiles to: + +```python +def on_event(_: int, _2: str, /): + print("fired") + +on_event(1, "a") +``` + +python rejects two parameters of one name, so each `_` after the first is given +a name of its own — `_2`, `_3` and so on, skipping any name the module spells. +that name is the transpiler's rather than yours, so no call may spell it: a `/` +after the last `_` makes them positional-only, and passing one by keyword is a +`positional-only-parameter-as-kwarg` error + +python only exempts `_`: duplicates of any other name remain a syntax error + +## parameters after the last `_` + +the `/` goes after the last `_`, so a parameter declared after it is still +passed by name: + +```by +def ignore(_: int, _: int, label: str): ... + +ignore(1, 2, label="a") +``` + +a `*_` or `**_` is reached by no keyword, so a `_` repeated only by one of them +is left as it is + +## parameters before the last `_` + +python's `/` makes every parameter before it positional-only, so a parameter you +named ahead of a repeated `_` would stop being reachable by its name. that is +refused rather than done quietly, as `invalid-repeated-underscore`: + +```by +def pair(a: int, _: int, b: int, _: int): ... # error +``` + +writing the `/` yourself says it is what you meant: + +```by +def pair(a: int, _: int, b: int, _: int, /): ... +``` + +a method's receiver is the exception: nothing passes `self` by keyword, so +`def handle(self, _: int, _: int)` is accepted, with `self` before the `/` + +a repeated `_` after `*` would be keyword-only, which only a keyword reaches, and +it has no name of its own to be reached by. that is refused as well + +python before 3.8 has no `/`, so a project that targets it cannot make the `_`s +positional-only, and a signature that needs the `/` is refused there + +## callable types and protocol methods + +a callable type that names its parameters, and a method of an inline protocol, +follow the same rules, so a callback that ignores several arguments is called by +position: + +```by +type Handler = (_: int, _: str) -> None +``` + +transpiles to: + +```python +from typing import Protocol +class _Callable_f9c80db4(Protocol): + def __call__(self, _: "int", _2: "str", /) -> "None": ... + +type Handler = _Callable_f9c80db4 +``` + +## overrides + +a method that overrides another takes the names the overridden method gives +those positions, so a call written against the base keeps working on the +override: + +```by +class Handler: + def handle(self, event: int, source: str): ... + +class Quiet(Handler): + override def handle(self, _: int, _: str): ... + +Quiet().handle(event=1, source="a") +``` + +transpiles to: + +```python +from typing_extensions import override + +class Handler: + def handle(self, event: int, source: str): ... + +class Quiet(Handler): + @override + def handle(self, event: int, source: str): ... + +Quiet().handle(event=1, source="a") +``` + +a base parameter that is positional-only stays so, and a default the base gives +a parameter comes with it, as it does for any override (see +[inherited default values](inherited-defaults.md)) + +where the base has no parameter at some `_`'s position, none of the `_`s take a +name from it and they are numbered as above. that override is reported as an +`invalid-method-override` anyway + +a name taken from the base would shadow the same name read from an enclosing +scope, so an override whose body reads one is refused as +`invalid-repeated-underscore` + +## reading `_` + +references to `_` in the body resolve to the first `_` parameter. in an override +that took its names from the base, the first of them is bound to `_` at the start +of the body, so a read of `_` still finds it diff --git a/docs/basedpython/features/wrapped-results.md b/docs/basedpython/features/wrapped-results.md index 19de7de318..229a3af2c4 100644 --- a/docs/basedpython/features/wrapped-results.md +++ b/docs/basedpython/features/wrapped-results.md @@ -152,6 +152,20 @@ x = foo()! # raises if foo() is None_ or Err(_) panics raise `RuntimeError("force-unwrap of absent value")` and include the wrapped error as `__cause__` when the value was an `Err` +inside an f-string replacement field a `!` is python's conversion flag when +one can follow it, and the force-unwrap when none can: + +```by +print(f"{foo()!}") # force unwrap, then format +print(f"{foo()!:>4}") # force unwrap, then the format spec +print(f"{foo()!!r}") # force unwrap, then the `!r` conversion +print(f"{x!r}") # the conversion alone, as in python +``` + +`!=` is the comparison operator there as everywhere else, so `f"{x!=y}"` +compares. a force-unwrap printed with `=` needs the parenthesised form, +`f"{(x!)=}"` + ### `??` — coalesce > against a genuine wrapped type (`int??`, `T ? E`) the wrapper-aware behaviour diff --git a/ty.schema.json b/ty.schema.json index 2454eb75a5..db22a28089 100644 --- a/ty.schema.json +++ b/ty.schema.json @@ -1281,6 +1281,16 @@ } ] }, + "invalid-decorator-def": { + "title": "detects a `decorator def` whose shape has no decorator to expand into", + "description": "## What it does\n\nChecks for a `decorator def` whose parameters, or whose position, leave it with no decorator to\nexpand into.\n\n## Why is this bad?\n\nA `decorator def` is lowered to a pair of overloads and a runtime dispatcher, built from a first\nparameter that receives the decorated function and options that each carry a default. A declaration\nthat does not have that shape has nothing to build from, and the lowering refuses it. Reported here,\nthe refusal arrives while the file is being checked rather than when it is transpiled.\n\n## Examples\n\nThe first parameter is the one a decoration hands the function to, so there has to be one:\n\n```by\n# error: [invalid-decorator-def]\ndecorator def logged() -> int:\n return 1\n```\n\nEvery shape the decorator is applied in supplies that function, so the parameter never falls back to\na default:\n\n```by\ndef nothing() -> None: ...\n\n# error: [invalid-decorator-def]\ndecorator def retry(fn: (...) -> object = nothing) -> int:\n return 1\n```\n\nThe options are what `@traced` decorates without, so each of them needs a default:\n\n```by\n# error: [invalid-decorator-def]\ndecorator def traced(fn: (...) -> object, label: str) -> int:\n return 1\n```\n\n`decorator def` is module-scope only. A decorator written in a class body is a plain `def` that\nreturns a callable:\n\n```by\nclass Registry:\n # error: [invalid-decorator-def]\n decorator def register(fn: (...) -> object) -> int:\n return 1\n```\n\nThe form the lowering accepts is a decorated parameter followed by options that each have a default:\n\n```by\ndecorator def traced(fn: (...) -> object, label: str = \"\") -> int:\n return len(label)\n```", + "default": "error", + "oneOf": [ + { + "$ref": "#/definitions/Level" + } + ] + }, "invalid-enum-member-annotation": { "title": "detects type annotations on enum members", "description": "## What it does\n\nChecks for enum members that have explicit type annotations.\n\n## Why is this bad?\n\nThe [typing spec] states that type checkers should infer a literal type for all enum members. An\nexplicit type annotation on an enum member is misleading because the annotated type will be\nincorrect — the actual runtime type is the enum class itself, not the annotated type.\n\nIn CPython's `enum` module, annotated assignments with values are still treated as members at\nruntime, but the annotation will confuse readers of the code.\n\n## Examples\n\n```python\nfrom enum import Enum\n\n\nclass Pet(Enum):\n CAT = 1 # OK\n # enum members should not be annotated\n DOG: int = 2 # error\n```\n\nUse instead:\n\n```python\nfrom enum import Enum\n\n\nclass Pet(Enum):\n CAT = 1\n DOG = 2\n```\n\n## References\n\n- [Typing spec: Enum members](https://typing.python.org/en/latest/spec/enums.html#enum-members)\n\n[typing spec]: https://typing.python.org/en/latest/spec/enums.html#enum-members", @@ -1601,6 +1611,16 @@ } ] }, + "invalid-repeated-underscore": { + "title": "detects a repeated `_` parameter the lowering has no python spelling for", + "description": "## What it does\n\nChecks for a parameter list that repeats `_` in a shape the lowering has no python spelling for.\n\n## Why is this bad?\n\nPython refuses two parameters of one name, so every `_` after the first is given a name of its own.\nThat name is the lowering's rather than the author's, so a call reaches the parameter by position\nalone: it is positional-only. Where that cannot be written, or would change a parameter the author\nnamed, the lowering refuses the definition. Reported here, the refusal arrives while the file is\nbeing checked rather than when it is transpiled.\n\n## Examples\n\nPython's `/` makes every parameter before it positional-only, so a named parameter ahead of a\nrepeated `_` would stop being reachable by keyword:\n\n```by\n# error: [invalid-repeated-underscore]\ndef pair(a: int, _: int, b: int, _: int) -> int:\n return a + b\n```\n\nWriting the `/` says so, and is accepted:\n\n```by\ndef explicit(a: int, _: int, b: int, _: int, /) -> int:\n return a + b\n```\n\nA repeated `_` after `*` is keyword-only, and only a keyword reaches it:\n\n```by\n# error: [invalid-repeated-underscore]\ndef keyword(_: int, *, _: int) -> None: ...\n```\n\nPython before 3.8 has no `/` at all, so a module targeting it cannot make a repeated `_`\npositional-only, and every definition that needs the `/` is refused there.\n\nA method that overrides one takes the names the overridden method gives those positions. A name the\nbody reads from an enclosing scope would then read the parameter instead. Refused, the override\nkeeps no names a caller of the base could pass by keyword, which is reported too:\n\n```by\nx = 1\n\nclass Base:\n def f(self, x: int, y: int) -> int:\n return x + y\n\nclass Override(Base):\n # error: [invalid-repeated-underscore]\n # error: [invalid-method-override]\n override def f(self, _: int, _: int) -> int:\n return x\n```", + "default": "error", + "oneOf": [ + { + "$ref": "#/definitions/Level" + } + ] + }, "invalid-return-type": { "title": "detects returned values that can't be assigned to the function's annotated return type", "description": "## What it does\n\nDetects returned values that can't be assigned to the function's annotated return type.\n\nNote that the special case of a function with a non-`None` return type and an empty body is handled\nby the separate `empty-body` error code.\n\n## Why is this bad?\n\nReturning an object of a type incompatible with the annotated return type is unsound, and will lead\nto ty inferring incorrect types elsewhere.\n\n## Examples\n\n```python\ndef func() -> int:\n return \"a\" # error: [invalid-return-type]\n```", @@ -1883,7 +1903,7 @@ }, "missing-context-argument": { "title": "detects calls whose `context` parameter has no matching context value", - "description": "## What it does\nChecks for calls that leave a basedpython `context` parameter unfilled:\nno explicit argument matches it and no `context` declaration in scope\nhas a type assignable to it.\n\n## Why is this bad?\nA `context` parameter has no default — the call raises `TypeError` at\nruntime unless something supplies the argument.\n\n## Examples\n```python\ndef f(a: int, context b: str): ...\n\nf(1) # error: no context value in scope\ncontext s = \"hello\"\nf(1) # ok — `s` is passed implicitly\n```", + "description": "## What it does\nChecks for calls that leave a basedpython `context` parameter unfilled:\nno explicit argument matches it and no `context` declaration in scope\nhas a type assignable to it. A decoration is also reported, because it\nis a call with no argument list to supply the value in.\n\n## Why is this bad?\nA `context` parameter has no default — the call raises `TypeError` at\nruntime unless something supplies the argument. One that does have a\ndefault quietly takes it instead of the ambient value the declaration\npromised.\n\n## Examples\n```python\ndef f(a: int, context b: str): ...\n\nf(1) # error: no context value in scope\ncontext s = \"hello\"\nf(1) # ok — `s` is passed implicitly\n\ndef deco(fn: (...) -> object, context b: str = \"d\") -> object: ...\n\n@deco # error: `b` cannot be filled at a decoration\ndef g(): ...\n```", "default": "error", "oneOf": [ { From e4b79a457cbb0b9b18d628e2288bedf73838e77e Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Wed, 23 Sep 2026 22:46:53 +1000 Subject: [PATCH 12/16] transpiler: hold a .by file to its target python, and spell what the runtime evaluates for it version checks run for .by files except where the lowering removes the syntax; an evaluated `|` is `Union` below 3.10 and `T?` a tuple of classes in isinstance; a `some` parameter is a TypeVar at every version, refused off a parameter annotation and on variadics; the generics polyfill imports its constructors and declares its type variables under names the module does not spell and passes ParamSpec and TypeVarTuple defaults through; by run compiles from text so 3.9-3.11 do not recurse, and a -W error warning names the .by line; polyfill runtime tests run on an interpreter that can run them --- .config/nextest.toml | 8 + crates/by_transforms/src/lib.rs | 157 +++- .../src/transforms/annotation.rs | 21 +- .../src/transforms/anon_named_tuple.rs | 9 +- .../src/transforms/ast_driver.rs | 450 ++++++++++- .../by_transforms/src/transforms/callable.rs | 228 +++++- .../by_transforms/src/transforms/generics.rs | 727 ++++++++++++++---- .../src/transforms/inferred_annotation.rs | 90 ++- .../src/transforms/init_method.rs | 27 +- .../src/transforms/literal_types.rs | 25 + .../by_transforms/src/transforms/modifiers.rs | 22 +- .../src/transforms/optional_type.rs | 25 +- .../src/transforms/properties.rs | 83 +- .../src/transforms/protocol_type.rs | 16 +- .../src/transforms/reified_generic.rs | 21 + .../src/transforms/repeated_underscore.rs | 93 ++- .../src/transforms/runtime_union.rs | 102 ++- .../by_transforms/src/transforms/soundness.rs | 66 +- .../src/transforms/source_util.rs | 13 + .../src/transforms/symbolic_type_op.rs | 102 ++- .../src/transforms/type_expr_walker.rs | 51 +- .../src/transforms/typed_dict_literal.rs | 112 ++- .../src/transforms/typing_redirect.rs | 27 + crates/by_transforms/src/transforms/unpack.rs | 108 ++- .../by_transforms/tests/polyfill_runtime.rs | 414 ++++++++-- crates/ruff_python_parser/src/error.rs | 60 ++ .../src/parser/expression.rs | 19 + crates/ruff_python_parser/src/parser/mod.rs | 8 +- .../src/parser/statement.rs | 35 +- crates/ruff_python_parser/src/parser/tests.rs | 107 +++ crates/ty/docs/cli.md | 2 +- crates/ty/src/args.rs | 3 +- crates/ty/src/by_commands.rs | 178 ++++- crates/ty/tests/by_e2e.rs | 166 ++++ .../mdtest/basedpython_optional_type.md | 63 ++ .../mdtest/basedpython_reified_generics.md | 14 + .../mdtest/basedpython_some_parameters.md | 126 +++ .../version_related_syntax_errors.md | 138 ++++ .../resources/mdtest/narrow/isinstance.md | 48 ++ .../resources/mdtest/repeated_underscore.md | 122 +++ crates/ty_python_semantic/src/place.rs | 5 +- crates/ty_python_semantic/src/reified.rs | 6 + .../src/types/diagnostic.rs | 4 + .../ty_python_semantic/src/types/function.rs | 62 ++ .../src/types/infer/builder.rs | 67 +- .../src/types/infer/builder/typeguard.rs | 181 +++-- crates/ty_python_semantic/src/types/narrow.rs | 23 +- .../src/types/repeated_underscore.rs | 94 ++- .../development/how-transpilation-works.md | 8 +- docs/basedpython/development/sourcemaps.md | 10 +- docs/basedpython/features/index.md | 1 + docs/basedpython/features/polyfills.md | 38 +- .../features/repeated-underscore.md | 4 + docs/basedpython/features/some-parameters.md | 73 ++ ty.schema.json | 2 +- zensical.toml | 1 + 56 files changed, 4165 insertions(+), 500 deletions(-) create mode 100644 crates/ty_python_semantic/resources/mdtest/basedpython_some_parameters.md create mode 100644 docs/basedpython/features/some-parameters.md diff --git a/.config/nextest.toml b/.config/nextest.toml index a1265b39ed..94d5783739 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -7,6 +7,14 @@ serial = { max-threads = 1 } filter = 'binary(file_watching)' test-group = 'serial' +# the polyfill runtime tests run their output on an old interpreter with +# `typing_extensions`, which a machine often does not have. each says which +# interpreter it ran on, or that it found none and ran nothing, and a passing +# test's output is otherwise hidden, so a skip would read as a pass +[[profile.default.overrides]] +filter = 'binary(polyfill_runtime)' +success-output = "final" + [profile.ci] # Print out output for failing tests as soon as they fail, and also at the end # of the run (for easy scrollability). diff --git a/crates/by_transforms/src/lib.rs b/crates/by_transforms/src/lib.rs index 90bddfd38e..65cd4239c2 100644 --- a/crates/by_transforms/src/lib.rs +++ b/crates/by_transforms/src/lib.rs @@ -6,7 +6,10 @@ mod transforms; pub(crate) mod type_info; pub use config::{Config, FloatLiteralLowering, PythonVersion, SoundnessPositions}; -pub use transforms::repeated_underscore::{WrittenNames, python_parameter_name}; +pub use transforms::repeated_underscore::{ + UnderscoreLowerings, WrittenNames, positional_only_count, python_parameter_name, + repeated_underscore_lowerings, +}; pub use transforms::soundness::{SoundnessSites, soundness_sites}; /// A module's `main` as the program's command line — the reading the entry-point @@ -2085,6 +2088,8 @@ mod python_parse_errors { #[cfg(test)] mod transpile_error { use super::*; + use ruff_python_ast::PySourceType; + use ruff_python_parser::UnsupportedSyntaxErrorKind as Kind; use ruff_text_size::TextSize; /// basedpython holds back python's two `match` checks for a `.by` source, @@ -2122,6 +2127,156 @@ mod transpile_error { .unwrap(); } + /// how many kinds of syntax the parser holds a python version to + const KINDS: usize = 22; + + /// the name of `kind`'s variant. a new kind fails to compile here until it is counted in + /// [`KINDS`] and given an example in `the_target_syntax_the_lowering_keeps_is_refused` + fn variant(kind: Kind) -> &'static str { + match kind { + Kind::Match => "Match", + Kind::Walrus => "Walrus", + Kind::ExceptStar => "ExceptStar", + Kind::UnparenthesizedNamedExpr(_) => "UnparenthesizedNamedExpr", + Kind::ParenthesizedKeywordArgumentName => "ParenthesizedKeywordArgumentName", + Kind::StarTuple(_) => "StarTuple", + Kind::RelaxedDecorator(_) => "RelaxedDecorator", + Kind::PositionalOnlyParameter => "PositionalOnlyParameter", + Kind::TypeParameterList => "TypeParameterList", + Kind::LazyImportStatement => "LazyImportStatement", + Kind::TypeAliasStatement => "TypeAliasStatement", + Kind::TypeParamDefault => "TypeParamDefault", + Kind::Pep701FString(_) => "Pep701FString", + Kind::ParenthesizedContextManager => "ParenthesizedContextManager", + Kind::StarExpressionInIndex => "StarExpressionInIndex", + Kind::StarAnnotation => "StarAnnotation", + Kind::UnpackingInComprehension(_) => "UnpackingInComprehension", + Kind::UnparenthesizedUnpackInFor => "UnparenthesizedUnpackInFor", + Kind::UnparenthesizedExceptionTypes => "UnparenthesizedExceptionTypes", + Kind::TemplateStrings => "TemplateStrings", + Kind::UnaryPlusMatchPattern => "UnaryPlusMatchPattern", + Kind::Destructuring => "Destructuring", + } + } + + /// the parser holds a `.by` file to the python it targets for every kind of syntax the + /// lowering leaves standing, and to none the lowering rewrites + /// (`UnsupportedSyntaxErrorKind::is_lowered_by_basedpython`). so that a checker reports + /// exactly what the transpile refuses, each kind is written here at a python too old for + /// it, and the transpile refuses it exactly when the parser says the lowering keeps it + #[test] + fn the_target_syntax_the_lowering_keeps_is_refused() { + // the python spelling of each kind, and the basedpython forms written as one + let python = PySourceType::Python; + let basedpython = PySourceType::BasedPython; + let examples = [ + ("let (a, b) := x\n", PythonVersion::PY37, basedpython), + ( + "if let [a] := x:\n pass\n", + PythonVersion::PY37, + basedpython, + ), + ( + "match x:\n case 1:\n pass\n", + PythonVersion::PY37, + python, + ), + ( + "match x:\n case 1:\n pass\n", + PythonVersion::PY39, + python, + ), + ( + "match x:\n case +1:\n pass\n", + PythonVersion::PY39, + python, + ), + ( + "match x:\n case +1:\n pass\n", + PythonVersion::PY312, + python, + ), + ("if (y := 1):\n pass\n", PythonVersion::PY37, python), + ( + "try:\n pass\nexcept* ValueError:\n pass\n", + PythonVersion::PY310, + python, + ), + ("print(a[y := 0])\n", PythonVersion::PY38, python), + ("f((a)=1)\n", PythonVersion::PY38, python), + ("def f(a):\n return 1, *a\n", PythonVersion::PY37, python), + ("@d[0]\ndef f():\n pass\n", PythonVersion::PY38, python), + ("def f(a, /):\n pass\n", PythonVersion::PY37, python), + ( + "def f[T](x: T) -> T:\n return x\n", + PythonVersion::PY311, + python, + ), + ("lazy import json\n", PythonVersion::PY314, python), + ("type A = int\n", PythonVersion::PY311, python), + ( + "class C[T = int]:\n pass\n", + PythonVersion::PY312, + python, + ), + ("f\"{d[\"a\"]}\"\n", PythonVersion::PY311, python), + ( + "with (open(a) as x, open(b) as y):\n pass\n", + PythonVersion::PY38, + python, + ), + ( + "def f[*Ts](x: tuple[*Ts]) -> None:\n pass\n", + PythonVersion::PY310, + python, + ), + ( + "def f[*Ts](*args: *Ts) -> None:\n pass\n", + PythonVersion::PY310, + python, + ), + ("[*x for x in a]\n", PythonVersion::PY314, python), + ("for x in *a, *b:\n pass\n", PythonVersion::PY38, python), + ( + "try:\n pass\nexcept ValueError, TypeError:\n pass\n", + PythonVersion::PY313, + python, + ), + ("t\"{a}\"\n", PythonVersion::PY313, python), + ]; + + let mut covered = HashSet::new(); + for (source, version, source_type) in examples { + let options = + ruff_python_parser::ParseOptions::from(source_type).with_target_version(version); + let parsed = ruff_python_parser::parse_unchecked(source, options); + let kinds: Vec = parsed + .unsupported_syntax_errors() + .iter() + .map(|error| error.kind) + .collect(); + assert!(!kinds.is_empty(), "{source:?} is fine on {version}"); + covered.extend(kinds.iter().map(|kind| variant(*kind))); + let kept = kinds + .iter() + .any(|kind| !kind.is_lowered_by_basedpython(version)); + // every real transpile lowers imports, which is what rewrites a written `lazy` + let config = Config { + min_version: version, + lazy_imports: true, + ..Config::test_default() + }; + let refused = transpile(source, &config).is_err(); + assert_eq!( + refused, + kept, + "{source:?} on {version}: the parser says {kinds:?} is {}", + if kept { "kept" } else { "lowered" } + ); + } + assert_eq!(covered.len(), KINDS, "covered only {covered:?}"); + } + #[test] fn verify_syntax_message_has_no_byte_range() { let err = verify_syntax("def f(:\n pass\n", &AuthorNames::none()).unwrap_err(); diff --git a/crates/by_transforms/src/transforms/annotation.rs b/crates/by_transforms/src/transforms/annotation.rs index e1d6c921dc..4f8621ae04 100644 --- a/crates/by_transforms/src/transforms/annotation.rs +++ b/crates/by_transforms/src/transforms/annotation.rs @@ -5,12 +5,13 @@ use ruff_python_ast::{Expr, PythonVersion, Stmt}; use ruff_text_size::{Ranged, TextRange}; use crate::Config; -use crate::config::FloatLiteralLowering; use crate::transforms::ast_driver::{PassContext, TypeAwarePass}; use crate::transforms::callable::CallableSyntax; use crate::transforms::optional_type; use crate::transforms::repeated_underscore::WrittenNames; -use crate::transforms::type_expr_walker::{Recurse, TypeExprVisitor, TypePos, walk_type_positions}; +use crate::transforms::type_expr_walker::{ + Recurse, RootKind, TypeExprVisitor, TypePos, walk_type_positions, +}; use crate::type_info::TypeInfo; /// The element type an unpacked tuple element wraps, whichever way the target @@ -64,18 +65,17 @@ impl<'src> TupleLiteralType<'src> { source: &'src str, written: WrittenNames<'src>, types: &'src dyn TypeInfo, - min_version: PythonVersion, - float_literals: FloatLiteralLowering, + config: &Config, symbolic_substitutions: &'src [(TextRange, String)], ) -> Self { - let mut leaves = CallableSyntax::new(source, written, float_literals).with_types(types); + let mut leaves = CallableSyntax::new(source, written, config).with_types(types); for (range, rendered) in symbolic_substitutions { leaves.add_substitution(*range, rendered.clone()); } Self { source, types, - min_version, + min_version: config.min_version, needs_unpack_import: Cell::new(false), leaves: RefCell::new(leaves), symbolic_substitutions, @@ -374,6 +374,12 @@ impl TypeExprVisitor for TupleLiteralType<'_> { } Recurse::Stop } + + fn enter_root(&mut self, kind: RootKind) { + self.leaves + .borrow_mut() + .set_evaluated(kind == RootKind::Evaluated); + } } pub(crate) struct TupleLiteralTypePass<'src> { @@ -409,8 +415,7 @@ impl TypeAwarePass for TupleLiteralTypePass<'_> { self.source, self.written, types, - self.config.min_version, - self.config.float_literals, + &self.config, &symbolic_substitutions, ); walk_type_positions(stmts, Some(types), &mut inner); diff --git a/crates/by_transforms/src/transforms/anon_named_tuple.rs b/crates/by_transforms/src/transforms/anon_named_tuple.rs index 133025cf22..a8abcfe1ae 100644 --- a/crates/by_transforms/src/transforms/anon_named_tuple.rs +++ b/crates/by_transforms/src/transforms/anon_named_tuple.rs @@ -97,6 +97,7 @@ impl Shape { pub(crate) struct AnonNamedTuple<'src> { source: &'src str, + written: WrittenNames<'src>, types: &'src dyn TypeInfo, config: crate::Config, /// One lowerer drives every field type in the run, so its imports and the @@ -173,8 +174,9 @@ impl<'src> AnonNamedTuple<'src> { ) -> Self { Self { source, + written, types, - callable: CallableSyntax::new(source, written, config.float_literals).with_types(types), + callable: CallableSyntax::new(source, written, &config).with_types(types), config, edits: Vec::new(), shapes: indexmap::IndexMap::new(), @@ -704,7 +706,10 @@ impl<'src> AnonNamedTuple<'src> { ruff_python_ast::TypeParam::TypeVarTuple(tvt) => tvt.name.id.as_str(), ruff_python_ast::TypeParam::ParamSpec(ps) => ps.name.id.as_str(), }; - frame.insert(name.to_owned(), super::generics::mangle(name)); + frame.insert( + name.to_owned(), + super::generics::polyfilled_name(self.written, name), + ); } if frame.is_empty() { return false; diff --git a/crates/by_transforms/src/transforms/ast_driver.rs b/crates/by_transforms/src/transforms/ast_driver.rs index 4449ab45e4..bb75c6fbb3 100644 --- a/crates/by_transforms/src/transforms/ast_driver.rs +++ b/crates/by_transforms/src/transforms/ast_driver.rs @@ -190,6 +190,7 @@ pub(crate) enum Lowering { ParametricIs, ProtocolType, RepeatedUnderscore, + RuntimeUnion, StatementExpression, SymbolicTypeOp, TupleLiteralType, @@ -912,8 +913,19 @@ struct Authorship { marks: Vec<[usize; 5]>, } +#[cfg(test)] +thread_local! { + /// the lowerings each pass declares it writes itself, by the pass's name, recorded as + /// the passes run: the test holding every declaration to a case showing it reads them + static DECLARED_SUBSUMPTIONS: RefCell> = + const { RefCell::new(Vec::new()) }; +} + impl Authorship { fn finished(&mut self, author: Author, ctx: &PassContext) { + #[cfg(test)] + DECLARED_SUBSUMPTIONS + .with(|declared| declared.borrow_mut().push((author.name, author.subsumes))); self.authors.push(author); self.marks.push([ ctx.text_edits.len(), @@ -1244,8 +1256,11 @@ pub(crate) fn run_against_source<'a>( source_ref, ); - let symbolic_folds = - symbolic_type_op::collect_symbolic_folds(parsed_handle.suite(), &semantic_model); + let symbolic_folds = symbolic_type_op::collect_symbolic_folds( + parsed_handle.suite(), + &semantic_model, + config.float_literals, + ); let symbolic_needs_literal_import = symbolic_folds.needs_literal_import; let symbolic_needs_any_import = symbolic_folds.needs_any_import; ctx.claimed_type_op_ranges = symbolic_folds.claimed_ranges(); @@ -1319,9 +1334,9 @@ pub(crate) fn run_against_source<'a>( config.min_version, config.inject_future_annotations, ); - let init_method_pass = - init_method::InitMethod::new(source_ref, written, config.float_literals, config.is_stub); - let properties_pass = properties::PropertiesPass::new(source_ref, accessor_value_ranges); + let init_method_pass = init_method::InitMethod::new(source_ref, written, config.clone()); + let properties_pass = + properties::PropertiesPass::new(source_ref, accessor_value_ranges, config.min_version); let local_once_pass = local_once::LocalOncePass::new(source_ref); let raises_strip_pass = raises_clause::RaisesStripPass::new(source_ref); let return_value_use_pass = return_value_use::ReturnValueUsePass::new( @@ -1347,8 +1362,17 @@ pub(crate) fn run_against_source<'a>( config.min_version, ), ); - let unpack_pass = unpack::UnpackSyntax::new(config.clone()); - let typed_dict_literal_pass = typed_dict_literal::TypedDictLiteralPass::new(source_ref); + let unpack_pass = unpack::UnpackSyntax::new( + config.clone(), + unpack::collect_type_subscripts(parsed_handle.suite(), &semantic_model), + ); + let typed_dict_literal_pass = typed_dict_literal::TypedDictLiteralPass::new( + source_ref, + written, + config.clone(), + parsed_handle.suite(), + &semantic_model, + ); let just_float_pass = just_float::JustFloatPass::new(); let float_const_pass = float_const::FloatConstPass::new(); let kw_subscript_pass = kw_subscript::KwSubscriptPass::new(source_ref, config.min_version); @@ -1366,8 +1390,7 @@ pub(crate) fn run_against_source<'a>( let tuple_types_pass = annotation::TupleLiteralTypePass::new(source_ref, written, config.clone()); let literal_types_pass = literal_types::LiteralTypePass::new(source_ref, config.float_literals); - let callable_pass = - callable::CallableSyntaxPass::new(source_ref, written, config.float_literals); + let callable_pass = callable::CallableSyntaxPass::new(source_ref, written, config.clone()); let protocol_type_pass = protocol_type::ProtocolTypePass::new(source_ref, written, config.clone()); let coalesce_text_pass = coalesce::NoneCoalescePass::new(source_ref); @@ -2390,3 +2413,412 @@ mod driver_tests { assert_eq!(out.text(), "[1])W"); } } + +/// a pass that declares it writes another lowering itself +/// ([`TypeAwarePass::subsumes`]) is trusted to: the edits that lowering makes inside +/// one of the pass's own are left out of the output without an error. a declaration +/// the pass does not live up to drops the lowering in silence, so each one is shown +/// here written, by a construct of the declaring pass with the lowered construct +/// inside it, unless [`SHOWN_ELSEWHERE`](subsumed_lowerings::SHOWN_ELSEWHERE) says where +/// it is shown. a declaration with neither fails +/// `every_declared_lowering_is_shown_written` +#[cfg(test)] +mod subsumed_lowerings { + use ruff_python_ast::PythonVersion; + + use super::{DECLARED_SUBSUMPTIONS, Lowering}; + use crate::{Config, transpile}; + + /// the declared pairs shown written somewhere other than a case here, by the declaring + /// pass's name: the test of the pass's own that shows it, or why it holds + const SHOWN_ELSEWHERE: &[(&str, Lowering, &str)] = &[ + ( + "GenericPolyfillPass", + Lowering::MatchType, + "polyfilled_match_type_lowers_to_object", + ), + ( + "GenericPolyfillPass", + Lowering::SymbolicTypeOp, + "arithmetic_type_alias_is_folded_too", + ), + ( + "GenericPolyfillPass", + Lowering::Modifiers, + "private_match_type_takes_the_underscore_name", + ), + ( + "GenericPolyfillPass", + Lowering::VisibilityRename, + "private_match_type_takes_the_underscore_name", + ), + ( + "ModifiersPass", + Lowering::StatementExpression, + "a_declaration_takes_a_suite_bearing_value", + ), + ( + "ExtensionBlockPass", + Lowering::Modifiers, + "static_and_class_members_bind_the_class_object", + ), + ( + "ExtensionBlockPass", + Lowering::ContextParams, + "a_context_parameter_of_an_extension_member_is_filled", + ), + ( + "ExtensionBlockPass", + Lowering::GenericPolyfill, + "conditional_extension_keeps_bounds_in_marker", + ), + ( + "ExtensionBlockPass", + Lowering::RepeatedUnderscore, + "a_repeated_underscore_in_a_member_is_numbered", + ), + ( + "DecoratorKeyword", + Lowering::LocalOnce, + "a_parameter_modifier_leaves_the_overloads_as_they_were", + ), + ( + "DecoratorKeyword", + Lowering::ContextParams, + "a_parameter_modifier_leaves_the_overloads_as_they_were", + ), + ( + "KwSubscriptPass", + Lowering::OptionalType, + "getitem_kw_value_optional_lowers", + ), + ( + "MatchTypePass", + Lowering::Unpack, + "polyfilled_match_type_lowers_to_object", + ), + ( + "VisibilityRenamePass", + Lowering::Modifiers, + "private_type_alias_polyfilled", + ), + ("PropertiesPass", Lowering::Modifiers, "stored_var_property"), + ( + "SymbolicTypeOp", + Lowering::LiteralType, + "plain_int_addition", + ), + ( + "SymbolicTypeOp", + Lowering::DynamicKeyword, + "dynamic_operand_folds_to_any", + ), + ("SymbolicTypeOp", Lowering::Typeof, "typeof_operand"), + ( + "SymbolicTypeOp", + Lowering::Callable, + "by construction: a fold writes the type ty computed and none of its operands", + ), + ( + "SymbolicTypeOp", + Lowering::OptionalType, + "by construction: a fold writes the type ty computed and none of its operands", + ), + ( + "SymbolicTypeOp", + Lowering::JustFloat, + "by construction: a fold writes the type ty computed and none of its operands", + ), + ( + "SymbolicTypeOp", + Lowering::FloatConst, + "by construction: a fold writes the type ty computed and none of its operands", + ), + ( + "ParametricIsPass", + Lowering::LiteralType, + "protocol_int_literal_argument", + ), + // the runtime-union lowering runs only below 3.10, where the cases above never go + ( + "CallableSyntaxPass", + Lowering::RuntimeUnion, + "a_cast_target_and_a_typevar_bound_are_spelled_out_below_310", + ), + ]; + + /// `source`, which puts a construct of the pass named `pass` around one of each of + /// `lowerings`, transpiles to python holding each of `written` and none of `unwritten` + struct Case { + pass: &'static str, + lowerings: Vec, + source: String, + written: Vec, + unwritten: Vec, + } + + fn case( + pass: &'static str, + lowerings: &[Lowering], + source: &str, + written: &[&str], + unwritten: &[&str], + ) -> Case { + Case { + pass, + lowerings: lowerings.to_vec(), + source: source.to_owned(), + written: written.iter().map(|text| (*text).to_owned()).collect(), + unwritten: unwritten.iter().map(|text| (*text).to_owned()).collect(), + } + } + + /// a type expression of each of `lowerings`, as `wrap` puts it inside a construct of + /// `pass`, is printed lowered where `show` says the construct puts it + fn leaves( + pass: &'static str, + lowerings: &[Lowering], + wrap: impl Fn(&str) -> String, + show: impl Fn(&str) -> String, + ) -> Vec { + let leaves = [ + (Lowering::LiteralType, "1", "Literal[1]", None), + (Lowering::JustFloat, "float", "JustFloat", None), + (Lowering::DynamicKeyword, "dynamic", "Any", Some("dynamic")), + (Lowering::FloatConst, "1.5", "float", Some("1.5")), + (Lowering::OptionalType, "int?", "int | None", Some("int?")), + ( + Lowering::Callable, + "(int) -> str", + "Callable[[int], str]", + Some("(int) ->"), + ), + (Lowering::Typeof, "typeof y", "TypeOf[y]", Some("typeof")), + ]; + leaves + .into_iter() + .filter(|(lowering, ..)| lowerings.contains(lowering)) + .map(|(lowering, leaf, lowered, unwritten)| Case { + pass, + lowerings: vec![lowering], + source: format!("y = 1\n{}\n", wrap(leaf)), + written: vec![show(lowered)], + unwritten: unwritten.map(str::to_owned).into_iter().collect(), + }) + .collect() + } + + fn check(cases: &[Case], version: PythonVersion) { + let config = Config { + min_version: version, + ..Config::test_default() + }; + let mut failures = Vec::new(); + for case in cases { + let what = format!("`{}` around {:?}", case.pass, case.lowerings); + let output = match transpile(&case.source, &config) { + Ok(output) => output, + Err(error) => { + failures.push(format!("{what}: refused: {error}")); + continue; + } + }; + for text in &case.written { + if !output.contains(text.as_str()) { + failures.push(format!("{what}: `{text}` is missing from\n{output}")); + } + } + for text in &case.unwritten { + if output.contains(text.as_str()) { + failures.push(format!("{what}: `{text}` is left in\n{output}")); + } + } + } + assert!(failures.is_empty(), "{}", failures.join("\n\n")); + } + + fn type_expression_cases() -> Vec { + let mut cases = Vec::new(); + cases.extend(leaves( + "TupleLiteralTypePass", + super::super::callable::TYPE_EXPRESSION, + |leaf| format!("def f(x: ({leaf}, str)) -> None: ..."), + |lowered| format!("tuple[{lowered}, str]"), + )); + cases.extend(leaves( + "CallableSyntaxPass", + super::super::callable::TYPE_EXPRESSION, + |leaf| format!("def f(x: ({leaf}) -> str) -> None: ..."), + |lowered| format!("Callable[[{lowered}], str]"), + )); + let fields = [ + Lowering::Callable, + Lowering::OptionalType, + Lowering::LiteralType, + Lowering::JustFloat, + Lowering::DynamicKeyword, + Lowering::FloatConst, + Lowering::Typeof, + ]; + cases.extend(leaves( + "AnonNamedTuplePass", + &fields, + |leaf| format!("def f(x: (a: {leaf}, b: str)) -> None: ..."), + |lowered| format!("a: {lowered}"), + )); + cases.extend(leaves( + "ProtocolTypePass", + &fields, + |leaf| format!("def f(x: protocol(a: {leaf})) -> None: ..."), + |lowered| format!("a: \"{lowered}\""), + )); + cases.extend(leaves( + "TypedDictLiteralPass", + &fields, + |leaf| format!("def f(x: {{\"a\": {leaf}}}) -> None: ..."), + |lowered| format!("a: \"{lowered}\""), + )); + cases.push(case( + "CallableSyntaxPass", + &[Lowering::ProtocolType], + "def f(x: (protocol(a: int)) -> str) -> None: ...\n", + &["Callable[[_Protocol_"], + &["protocol("], + )); + cases + } + + fn whole_construct_cases() -> Vec { + vec![ + case( + "GenericPolyfillPass", + &[Lowering::TupleLiteralType], + "type Pair = (int, str)\n", + &["TypeAliasType(\"Pair\", tuple[int, str])"], + &["(int, str)"], + ), + case( + "GenericPolyfillPass", + &[Lowering::AnonNamedTuple], + "type Point = (x: int, y: int)\n", + &["TypeAliasType(\"Point\", _AnonNamedTuple_"], + &["(x: int"], + ), + case( + "GenericPolyfillPass", + &[Lowering::VarianceStrip], + "class Box[out T]:\n def get(self) -> T: ...\n", + &["covariant=True"], + &["out T"], + ), + case( + "ExtensionBlockPass", + &[ + Lowering::LiteralType, + Lowering::JustFloat, + Lowering::DynamicKeyword, + Lowering::FloatConst, + Lowering::OptionalType, + Lowering::Callable, + ], + "extension int:\n def scaled(self, by: 1, f: float, d: dynamic, c: 1.5, o: int?, k: (int) -> str) -> float:\n return 1.0\n", + &["(self, by, f, d, c, o, k)"], + &["dynamic", "1.5", "int?", "(int) ->"], + ), + case( + "PropertiesPass", + &[Lowering::InferredAnnotation], + "class A:\n var v = 0\n get() = field\n set(value):\n field = value\n", + &["self.__v: int = 0"], + &["var v"], + ), + case( + "TypeIs", + &[Lowering::ParametricIs], + "def f(x: object) -> x is list[int]:\n return isinstance(x, list)\n", + &["TypeIs[list[int]]"], + &["is list"], + ), + case( + "StatementExpressionPass", + &[Lowering::NoneCoalesce], + "def f(xs: list[int?]) -> None:\n for x in xs:\n y = x ?? continue\n print(y)\n", + &["continue"], + &["??"], + ), + case( + "NoneCoalescePass", + &[Lowering::NoneChain], + "class A:\n b: int? = None\n\ndef f(a: A?) -> int:\n return a?.b ?? 1\n", + &[], + &["?.", "??"], + ), + ] + } + + #[test] + fn a_type_expression_a_pass_moves_is_written_lowered() { + check(&type_expression_cases(), PythonVersion::PY310); + } + + #[test] + fn a_construct_a_pass_writes_whole_is_written_lowered() { + check(&whole_construct_cases(), PythonVersion::PY310); + } + + /// every lowering a pass declares it writes has a case above or an entry in + /// [`SHOWN_ELSEWHERE`], so a declaration added later fails here until something shows + /// it written — and an entry whose declaration is gone fails too + #[test] + fn every_declared_lowering_is_shown_written() { + DECLARED_SUBSUMPTIONS.with(|declared| declared.borrow_mut().clear()); + transpile("x = 1\n", &Config::test_default()).expect("transpile failed"); + let declared = DECLARED_SUBSUMPTIONS.with(std::cell::RefCell::take); + let cases: Vec = type_expression_cases() + .into_iter() + .chain(whole_construct_cases()) + .collect(); + + let mut failures = Vec::new(); + for &(pass, lowerings) in &declared { + for lowering in lowerings { + let in_a_case = cases + .iter() + .any(|case| case.pass == pass && case.lowerings.contains(lowering)); + let elsewhere = SHOWN_ELSEWHERE + .iter() + .any(|(shown, shown_lowering, _)| *shown == pass && shown_lowering == lowering); + if !in_a_case && !elsewhere { + failures.push(format!( + "`{pass}` declares it writes {lowering:?}, and nothing shows it" + )); + } + } + } + for (pass, lowering, _) in SHOWN_ELSEWHERE { + if !declared + .iter() + .any(|(name, lowerings)| name == pass && lowerings.contains(lowering)) + { + failures.push(format!( + "`{pass}` no longer declares {lowering:?}, which `SHOWN_ELSEWHERE` lists" + )); + } + } + for case in &cases { + if !declared.iter().any(|(name, lowerings)| { + *name == case.pass + && case + .lowerings + .iter() + .all(|lowering| lowerings.contains(lowering)) + }) { + failures.push(format!( + "a case shows `{}` writing {:?}, which it does not declare", + case.pass, case.lowerings + )); + } + } + assert!(failures.is_empty(), "{}", failures.join("\n")); + } +} diff --git a/crates/by_transforms/src/transforms/callable.rs b/crates/by_transforms/src/transforms/callable.rs index 8a47207ced..aa396feb9a 100644 --- a/crates/by_transforms/src/transforms/callable.rs +++ b/crates/by_transforms/src/transforms/callable.rs @@ -25,14 +25,15 @@ use std::hash::{Hash, Hasher}; use ruff_diagnostics::{Edit, Fix}; use ruff_python_ast::helpers::{is_classvar_marker_id, is_final_marker_id, is_let_marker_id}; -use ruff_python_ast::{Expr, ExprCallableType, Stmt, UnaryOp}; +use ruff_python_ast::{Expr, ExprCallableType, Operator, PythonVersion, Stmt, UnaryOp}; use ruff_text_size::{Ranged, TextRange}; use super::ast_driver::{Lowering, PassContext, TypeAwarePass}; use super::intersection::{collect_intersect, collect_union, is_intersection_node}; use super::just_float::rewrite_type_expr_with_imports; use super::repeated_underscore::{WrittenNames, lowered_callable_parameters}; -use crate::config::FloatLiteralLowering; +use super::type_expr_walker::RootKind; +use crate::config::{Config, FloatLiteralLowering}; use crate::type_info::{TypeInfo, UnpackedKwargsLowering}; /// the lowerings [`CallableSyntax::lower_type_expr`] writes itself, wherever it prints a type @@ -64,8 +65,17 @@ pub(crate) struct CallableSyntax<'src> { /// `typeof` fold lowers the rest in the syntax tree. `None` owns every one owned_typeof: Option<&'src [TextRange]>, float_literals: FloatLiteralLowering, + /// whether a union the runtime evaluates can be written `A | B`: from 3.10, where + /// `type.__or__` arrived, and in a stub, which nothing evaluates + union_operator_runs: bool, + /// whether the runtime evaluates the type expression being lowered where it + /// stands. an annotation it does not: below 3.10, the only targets where the + /// spelling matters, annotations are always deferred, so a union the author + /// wrote with the operator can stay as written there + evaluated: bool, edits: Vec, needs_import: bool, + needs_union_import: bool, needs_concatenate_import: bool, needs_protocol_import: bool, needs_intersection_import: bool, @@ -100,20 +110,19 @@ struct ProtocolShape { } impl<'src> CallableSyntax<'src> { - pub(crate) fn new( - source: &'src str, - written: WrittenNames<'src>, - float_literals: FloatLiteralLowering, - ) -> Self { + pub(crate) fn new(source: &'src str, written: WrittenNames<'src>, config: &Config) -> Self { Self { source, written, types: None, claimed_ranges: &[], owned_typeof: None, - float_literals, + float_literals: config.float_literals, + union_operator_runs: config.is_stub || config.min_version >= PythonVersion::PY310, + evaluated: false, edits: Vec::new(), needs_import: false, + needs_union_import: false, needs_concatenate_import: false, needs_protocol_import: false, needs_intersection_import: false, @@ -143,6 +152,27 @@ impl<'src> CallableSyntax<'src> { .map(|(_, name)| name.as_str()) } + /// say whether the runtime evaluates the type expressions lowered from here on + /// where they stand, which decides whether a union written with the operator has + /// to be spelled out + pub(crate) fn set_evaluated(&mut self, evaluated: bool) { + self.evaluated = evaluated; + } + + /// a union of the rendered `arms`, which is `Union[A, B]` below 3.10, since + /// `A | B` calls `type.__or__`. a union this lowerer spells for a `?` or an `or` + /// is written so wherever it stands, as the optional lowering writes a bare `T?`: + /// even a deferred annotation is evaluated by whatever reads it with + /// `get_type_hints` + fn spell_union(&mut self, arms: &[String]) -> String { + if self.union_operator_runs { + arms.join(" | ") + } else { + self.needs_union_import = true; + format!("Union[{}]", arms.join(", ")) + } + } + pub(crate) fn with_types(mut self, types: &'src dyn TypeInfo) -> Self { self.types = Some(types); self @@ -186,6 +216,7 @@ impl<'src> CallableSyntax<'src> { (self.needs_typeof_import, "from ty_extensions import TypeOf"), (self.needs_not_import, "from ty_extensions import Not"), (self.needs_annotated_import, "from typing import Annotated"), + (self.needs_union_import, "from typing import Union"), ] { if needed { lines.push(line.to_owned()); @@ -204,6 +235,7 @@ impl<'src> CallableSyntax<'src> { self.needs_typeof_import = false; self.needs_not_import = false; self.needs_annotated_import = false; + self.needs_union_import = false; self.needs_optional_runtime = false; (lines, helpers) } @@ -259,7 +291,8 @@ impl<'src> CallableSyntax<'src> { .filter(|(candidate, _)| range.contains_range(*candidate)) .map(|(candidate, name)| (*candidate, name.as_str())) .collect(); - subs.sort_by_key(|(candidate, _)| candidate.start()); + // an insertion at an offset goes ahead of a replacement starting there + subs.sort_by_key(|(candidate, _)| (candidate.start(), candidate.end())); let mut out = String::new(); let mut cursor = range.start(); @@ -621,7 +654,18 @@ impl<'src> CallableSyntax<'src> { let mut parts: Vec = Vec::new(); collect_union(expr, &mut parts); let rendered: Vec = parts.iter().map(|p| self.rewrite_or_leaf(p)).collect(); - Some(rendered.join(" | ")) + Some(self.spell_union(&rendered)) + } + + // a union written with the operator is kept as written, unless the runtime + // evaluates it on a target without the operator + Expr::BinOp(b) + if b.op == Operator::BitOr && self.evaluated && !self.union_operator_runs => + { + let mut parts: Vec = Vec::new(); + collect_union(expr, &mut parts); + let rendered: Vec = parts.iter().map(|p| self.rewrite_or_leaf(p)).collect(); + Some(self.spell_union(&rendered)) } // `not T` → `Not[T]` @@ -643,12 +687,23 @@ impl<'src> CallableSyntax<'src> { depth += 1; inner = u2.operand.as_ref(); } + // the `?` already rewritten by the optional lowering, in a spelling for the + // target's version, is written with that rewrite: its insertion ahead of the + // operand would otherwise land inside the operand's rendering, with no tail + if self.substitutions.iter().any(|(range, _)| { + !range.is_empty() + && range.start() >= inner.end() + && expr.range().contains_range(*range) + }) { + return Some(self.sweep_substitutions(expr.range())); + } let inner_str = self.rewrite_or_leaf(inner); if depth >= 2 { self.needs_optional_runtime = true; } + let union = self.spell_union(&[inner_str, "None".to_owned()]); Some(format!( - "{}{inner_str} | None{}", + "{}{union}{}", "Optional[".repeat(depth - 1), "]".repeat(depth - 1) )) @@ -841,15 +896,19 @@ fn is_named(expr: &Expr, ident: &str) -> bool { /// `substitutions` are ranges another transform has already lowered — a symbolic /// fold (`T.a` → `int`), a typevar rename (`T` → `_T`). They are honoured on every /// rendering path, which is what lets one wide edit carry rewrites it subsumes. +/// +/// `root` says where the text is written, which decides how a union in it is spelled pub(crate) fn lower_type_expr_full( source: &str, written: WrittenNames, types: &dyn TypeInfo, expr: &Expr, substitutions: &[(TextRange, String)], - float_literals: FloatLiteralLowering, + config: &Config, + root: RootKind, ) -> Option { - let mut inner = CallableSyntax::new(source, written, float_literals).with_types(types); + let mut inner = CallableSyntax::new(source, written, config).with_types(types); + inner.set_evaluated(root == RootKind::Evaluated); for (range, name) in substitutions { inner.add_substitution(*range, name.clone()); } @@ -866,7 +925,7 @@ pub(crate) fn lower_type_expr_full( } // no structural type-form — fall back to the per-leaf composer // (`float` → `JustFloat`, a literal → `Literal[…]`, `dynamic` → `Any`) - rewrite_type_expr_with_imports(source, types, expr, float_literals).map(|(text, _)| text) + rewrite_type_expr_with_imports(source, types, expr, config.float_literals).map(|(text, _)| text) } /// if `expr` is `Subscript(Name("__let__"|"__classvar__"|"__final__"), slice)`, @@ -945,24 +1004,24 @@ impl crate::transforms::type_expr_walker::TypeExprVisitor for CallableSyntax<'_> } crate::transforms::type_expr_walker::Recurse::Stop } + + fn enter_root(&mut self, kind: RootKind) { + self.set_evaluated(kind == RootKind::Evaluated); + } } pub(crate) struct CallableSyntaxPass<'src> { source: &'src str, written: WrittenNames<'src>, - float_literals: FloatLiteralLowering, + config: Config, } impl<'src> CallableSyntaxPass<'src> { - pub(crate) fn new( - source: &'src str, - written: WrittenNames<'src>, - float_literals: FloatLiteralLowering, - ) -> Self { + pub(crate) fn new(source: &'src str, written: WrittenNames<'src>, config: Config) -> Self { Self { source, written, - float_literals, + config, } } } @@ -977,6 +1036,11 @@ struct ValueCallableWalker<'a, 'src> { } impl<'ast> ruff_python_ast::visitor::Visitor<'ast> for ValueCallableWalker<'_, '_> { + /// every annotation is a type position the type-position walk has lowered, as + /// the annotation it is. lowered again here as a value, a union in it would be + /// spelled for a runtime that never evaluates it + fn visit_annotation(&mut self, _expr: &'ast Expr) {} + fn visit_expr(&mut self, expr: &'ast Expr) { // an inline protocol is lowered whole by `protocol_type`, which renders // its members through its own lowerer — rewriting a method member's @@ -1003,7 +1067,8 @@ impl TypeAwarePass for CallableSyntaxPass<'_> { } /// a type expression it replaces is printed by the shared lowerer, which writes every - /// leaf inside it too, an inline protocol as the name of the class it hoists to + /// leaf inside it too, an inline protocol as the name of the class it hoists to, and a + /// union the runtime evaluates in the spelling the target can run fn subsumes(&self) -> &'static [Lowering] { &[ Lowering::Callable, @@ -1013,6 +1078,7 @@ impl TypeAwarePass for CallableSyntaxPass<'_> { Lowering::DynamicKeyword, Lowering::FloatConst, Lowering::ProtocolType, + Lowering::RuntimeUnion, ] } @@ -1021,7 +1087,7 @@ impl TypeAwarePass for CallableSyntaxPass<'_> { // `required_imports` / `edits` uses below let claimed = ctx.claimed_type_op_ranges.clone(); let structural_typeof = ctx.structural_typeof_ranges.clone(); - let mut inner = CallableSyntax::new(self.source, self.written, self.float_literals) + let mut inner = CallableSyntax::new(self.source, self.written, &self.config) .with_types(types) .with_claimed_ranges(&claimed) .with_owned_typeof(&structural_typeof); @@ -1034,6 +1100,8 @@ impl TypeAwarePass for CallableSyntaxPass<'_> { // also lower callable types appearing in value positions; duplicate // edits over type-position callables dedup in the splice { + // a callable type standing as a value is evaluated there + inner.set_evaluated(true); let mut walker = ValueCallableWalker { inner: &mut inner }; for stmt in stmts { ruff_python_ast::visitor::Visitor::visit_stmt(&mut walker, stmt); @@ -1077,6 +1145,120 @@ mod tests { ); } + /// transpile for python 3.9, which has no `type.__or__` + fn check_py39(input: &str, expected: &str) { + let config = Config { + min_version: crate::PythonVersion::PY39, + ..Config::test_default() + }; + assert_eq!(transpile(input, &config).unwrap(), expected); + } + + /// a type written where the runtime evaluates it is spelled with `Union` below 3.10, + /// at any depth, whether the union was written with `|`, `or` or `?` + #[test] + fn an_evaluated_union_is_spelled_out_below_310() { + check_py39( + indoc! {" + Optionals = list[int?] + Values = dict[str, int | None] + Either = list[int or str] + Pairs = list[(int, str | None)] + Handlers = list[(int | None) -> str] + class Items(list[int | str]): ... + "}, + indoc! {" + from __future__ import annotations + from typing import Callable, Union + Optionals = list[Union[int, None]] + Values = dict[str, Union[int, None]] + Either = list[Union[int, str]] + Pairs = list[tuple[int, Union[str, None]]] + Handlers = list[Callable[[Union[int, None]], str]] + class Items(list[Union[int, str]]): ... + "}, + ); + } + + /// a `cast` target and a type argument of a typing construct are evaluated by the + /// call. a union at the top of a `cast` target is also a value the runtime-union + /// lowering reads, and this pass writes it in its place + #[test] + fn a_cast_target_and_a_typevar_bound_are_spelled_out_below_310() { + check_py39( + indoc! {" + from typing import TypeVar, cast + Bound = TypeVar(\"Bound\", bound=int | str) + def f(v: object): + return cast(int | str, v), cast(list[int?], v) + "}, + indoc! {" + from __future__ import annotations + from typing import TypeVar, cast, Union + Bound = TypeVar(\"Bound\", bound=Union[int, str]) + def f(v: object): + return cast(Union[int, str], v), cast(list[Union[int, None]], v) + "}, + ); + } + + /// a bound, a default and an alias value are call arguments once polyfilled, so the + /// union the author wrote with `|` is spelled out there too + #[test] + fn a_polyfilled_bound_default_and_alias_are_spelled_out_below_310() { + check_py39( + indoc! {" + def f[T: int | str](x: T) -> T: + return x + def g[T = int | None](): + pass + type A = list[int?] | str + "}, + indoc! {" + from __future__ import annotations + from typing import Union + from typing_extensions import TypeVar, TypeAliasType + _T = TypeVar(\"_T\", bound=Union[int, str]) + def f(x: _T) -> _T: + return x + _T_1 = TypeVar(\"_T_1\", default=Union[int, None]) + def g(): + pass + A = TypeAliasType(\"A\", Union[list[Union[int, None]], str]) + "}, + ); + } + + /// an annotation is deferred, so a union the author wrote with `|` keeps its spelling. + /// one this lowerer spells for a `?` or an `or` is written as the optional lowering + /// writes a bare `T?`, which `get_type_hints` can evaluate + #[test] + fn an_annotation_keeps_a_written_operator_below_310() { + check_py39( + "def f(a: list[int?], b: int | str, c: int or str, d: (int | str) -> str) -> list[int] | None: ...\n", + indoc! {" + from __future__ import annotations + from typing import Callable, Union + def f(a: list[Union[int, None]], b: int | str, c: Union[int, str], d: Callable[[int | str], str]) -> list[int] | None: ... + "}, + ); + } + + /// nothing in a stub is evaluated, so a written union keeps its spelling whatever the + /// target + #[test] + fn a_stub_keeps_a_written_operator_below_310() { + let config = Config { + min_version: crate::PythonVersion::PY39, + is_stub: true, + ..Config::test_default() + }; + assert_eq!( + transpile("Values = dict[str, int | None]\n", &config).unwrap(), + "Values = dict[str, int | None]\n" + ); + } + /// a callable type naming its parameters is declared as a `__call__` in the /// synthesized protocol, where a repeated `_` is numbered and made positional-only as /// it is in any `def` diff --git a/crates/by_transforms/src/transforms/generics.rs b/crates/by_transforms/src/transforms/generics.rs index 715f2d2908..7f49f9a1ed 100644 --- a/crates/by_transforms/src/transforms/generics.rs +++ b/crates/by_transforms/src/transforms/generics.rs @@ -2,13 +2,14 @@ use std::collections::{HashMap, HashSet}; use std::fmt::Write as _; use ruff_diagnostics::{Edit, Fix}; -use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; +use ruff_python_ast::visitor::{Visitor, walk_body, walk_expr, walk_stmt}; use ruff_python_ast::{Expr, Stmt, StmtClassDef, StmtFunctionDef, StmtTypeAlias, TypeParam}; use ruff_text_size::{Ranged, TextRange, TextSize}; use crate::config::Config; use crate::transforms::callable::lower_type_expr_full; use crate::transforms::repeated_underscore::WrittenNames; +use crate::transforms::type_expr_walker::RootKind; use crate::type_info::TypeInfo; use ruff_python_ast::PythonVersion; @@ -25,18 +26,20 @@ pub(crate) struct GenericPolyfill<'src> { edits: Vec, // Imports to inject at the top of the file. needed_imports: ImportNeeds, + /// the names the typing constructors this pass writes are called by + constructors: Constructors, /// `TypeVar` definitions already emitted at module scope. Polyfilling each /// generic class/function emits its own `_T = TypeVar("_T")` line; without /// dedup, a module with several generics over the same name produces /// repeated identical declarations (and an `F811 redefinition` warning). emitted_typevar_defs: std::collections::HashSet, - /// names already bound to a `TypeVar` at module scope along with the - /// arguments used. when a later class needs a `TypeVar` with the same name - /// but different shape (different bound/default/variance) we generate a - /// fresh suffix to avoid shadowing the earlier definition - emitted_typevar_signatures: std::collections::HashMap, - /// counter for fresh-suffix typevar names (`_T_2`, `_T_3`, …) - typevar_suffix_counter: usize, + /// each name a `TypeVar` definition this pass writes binds, with the arguments it is + /// called with and the suite it is written into. a name is only ever bound in one suite: + /// a definition written into a class body is not visible to a function the module + /// declares later, and one written into an `if` does not run when the `if` does not + emitted_typevar_signatures: HashMap, + /// the suite the statement being visited stands in + suite: Suite, /// names of classes/functions whose first type parameter has a top-parameters /// bound (i.e. `class A[P: (*: *, **: *)]`). subscript sites for these /// targets get tuple slices rewritten to list form so paramspec @@ -92,37 +95,52 @@ pub(crate) struct ImportNeeds { paramspec: bool, typealias_type: bool, typevar_needs_ext: bool, // TypeVar(default=) on < 3.13 + /// a `default=` below 3.13, which `typing`'s own `ParamSpec` does not take + paramspec_needs_ext: bool, + /// a `default=` below 3.13, which `typing`'s own `TypeVarTuple` does not take + typevar_tuple_needs_ext: bool, } impl ImportNeeds { /// Build the import lines to prepend to the file. - fn into_lines(self) -> Vec { + fn into_lines(self, constructors: &Constructors) -> Vec { let mut lines = Vec::new(); - let mut typing_names: Vec<&str> = Vec::new(); - let mut ext_names: Vec<&str> = Vec::new(); + let mut typing_names: Vec = Vec::new(); + let mut ext_names: Vec = Vec::new(); if self.typevar { + let name = imported("TypeVar", &constructors.type_var); if self.typevar_needs_ext { - ext_names.push("TypeVar"); + ext_names.push(name); } else { - typing_names.push("TypeVar"); + typing_names.push(name); } } if self.typevar_tuple { - typing_names.push("TypeVarTuple"); + let name = imported("TypeVarTuple", &constructors.type_var_tuple); + if self.typevar_tuple_needs_ext { + ext_names.push(name); + } else { + typing_names.push(name); + } } if self.unpack { - typing_names.push("Unpack"); + typing_names.push(imported("Unpack", &constructors.unpack)); } if self.paramspec { - typing_names.push("ParamSpec"); + let name = imported("ParamSpec", &constructors.param_spec); + if self.paramspec_needs_ext { + ext_names.push(name); + } else { + typing_names.push(name); + } } if self.generic { - typing_names.push("Generic"); + typing_names.push(imported("Generic", &constructors.generic)); } if self.typealias_type { - ext_names.push("TypeAliasType"); + ext_names.push(imported("TypeAliasType", &constructors.type_alias_type)); } if !typing_names.is_empty() { @@ -139,6 +157,41 @@ impl ImportNeeds { } } +/// `name` imported so that it is bound to `local` +fn imported(name: &str, local: &str) -> String { + if name == local { + name.to_owned() + } else { + format!("{name} as {local}") + } +} + +/// the names the typing constructors the polyfill writes are called by. one the module spells +/// is imported under a name it does not: the module's own binding need not be the one the +/// polyfill needs. `from typing import TypeVar` binds a `TypeVar` that takes no `default=` +/// below 3.13, and nothing stops a module binding `Generic` to a class of its own +struct Constructors { + type_var: String, + generic: String, + type_var_tuple: String, + unpack: String, + param_spec: String, + type_alias_type: String, +} + +impl Constructors { + fn new(written: WrittenNames) -> Self { + Self { + type_var: written.fresh("TypeVar"), + generic: written.fresh("Generic"), + type_var_tuple: written.fresh("TypeVarTuple"), + unpack: written.fresh("Unpack"), + param_spec: written.fresh("ParamSpec"), + type_alias_type: written.fresh("TypeAliasType"), + } + } +} + /// the pieces a polyfilled type-parameter list lowers to struct ProcessedTypeParams { /// each parameter as it is written *inside a subscript* — a variadic carries @@ -170,9 +223,10 @@ impl<'src> GenericPolyfill<'src> { config, edits: Vec::new(), needed_imports: ImportNeeds::default(), + constructors: Constructors::new(written), emitted_typevar_defs: std::collections::HashSet::new(), - emitted_typevar_signatures: std::collections::HashMap::new(), - typevar_suffix_counter: 0, + emitted_typevar_signatures: HashMap::new(), + suite: Suite::Module, parameters_targets: HashSet::new(), needed_imports_any: false, generic_class_renames: HashMap::new(), @@ -309,33 +363,32 @@ impl<'src> GenericPolyfill<'src> { self.config.min_version >= required } - /// Pick a unique mangled name for a `TypeVar`. + /// the name the `TypeVar` a parameter declared as `source_name` is bound to /// - /// First emission of a given source name returns the standard mangled - /// form (`T` → `_T`). A *later* emission with a different signature gets - /// a numeric suffix (`_T_2`, `_T_3`, …) so the per-class `TypeVar` object - /// isn't shadowed by a later one with a different bound / default / - /// variance. A later emission whose signature *matches* the existing one - /// reuses the original mangled name (Python identity is preserved) + /// the first definition is named [`polyfilled_name`] (`T` → `_T`). a later one written + /// with the same arguments into the same suite reuses that name, so python sees one + /// object; any other takes a numeric suffix (`_T_1`, `_T_2`, …) the module does not spell, + /// so it shadows neither an earlier definition nor a binding of the module's own fn unique_typevar_name(&mut self, source_name: &str, signature_args: &str) -> String { - let base = mangle(source_name); - let key_existing = self.emitted_typevar_signatures.get(&base).cloned(); - if let Some(existing_sig) = key_existing { - if existing_sig == signature_args { - return base; - } - self.typevar_suffix_counter += 1; - let mangled = format!("{base}_{}", self.typevar_suffix_counter); - self.emitted_typevar_signatures - .insert(mangled.clone(), signature_args.to_owned()); - return mangled; - } + let base = polyfilled_name(self.written, source_name); + let signature = (signature_args.to_owned(), self.suite); + let name = std::iter::once(base.clone()) + .chain((1u32..u32::MAX).map(|number| format!("{base}_{number}"))) + .find( + |candidate| match self.emitted_typevar_signatures.get(candidate) { + Some(existing) => *existing == signature, + None => !self.written.spells(candidate), + }, + ) + .unwrap_or(base); self.emitted_typevar_signatures - .insert(base.clone(), signature_args.to_owned()); - base + .entry(name.clone()) + .or_insert(signature); + name } - /// Skip `TypeVar` declarations already emitted elsewhere in the module + /// Skip `TypeVar` declarations already written into the suite being visited. a name is + /// bound in one suite only, so a definition seen before was written into this one fn dedupe_defs(&mut self, defs: &[String], indent: &str) -> String { use std::fmt::Write as _; let mut prefix = String::new(); @@ -390,6 +443,65 @@ impl<'src> GenericPolyfill<'src> { (start, indent) } + /// `default`, a type parameter's default, as the python the `default=` argument of its + /// declaration is written with. `visible` renames the type parameters in scope + fn default_arg(&mut self, default: &Expr, visible: &HashMap) -> String { + let rendered = lower_type_expr_full( + self.source, + self.written, + self.types, + default, + &self.subsume_within(default.range()), + &self.config, + RootKind::Evaluated, + ) + .unwrap_or_else(|| self.src(default.range()).to_owned()); + apply_renames_to_rendered(&rendered, visible) + } + + /// the `default=` argument of a `ParamSpec`: a list of types, `...`, or another parameter + /// specification + fn param_spec_default(&mut self, default: &Expr, visible: &HashMap) -> String { + match default { + Expr::Tuple(tuple) if tuple.parenthesized => { + self.parameter_shape_list(tuple, Some(visible)) + } + Expr::List(list) => { + let elements: Vec = list + .elts + .iter() + .map(|element| self.default_arg(element, visible)) + .collect(); + format!("[{}]", elements.join(", ")) + } + other => self.default_arg(other, visible), + } + } + + /// write the declaration of a `ParamSpec` a parameter named `name` lowers to, with + /// `default` as its `default=` argument, and answer the name it is bound to + fn declare_param_spec( + &mut self, + name: &str, + default: Option, + defs: &mut Vec, + ) -> String { + let arguments = default + .map(|default| { + self.needed_imports.paramspec_needs_ext |= + self.config.min_version < PythonVersion::PY313; + format!(", default={default}") + }) + .unwrap_or_default(); + let mangled = self.unique_typevar_name(name, &format!("ParamSpec({arguments})")); + defs.push(format!( + "{mangled} = {}(\"{mangled}\"{arguments})", + self.constructors.param_spec + )); + self.needed_imports.paramspec = true; + mangled + } + fn process_type_params(&mut self, params: &[TypeParam]) -> ProcessedTypeParams { let mut generic_args: Vec = Vec::new(); let mut param_names: Vec = Vec::new(); @@ -416,11 +528,13 @@ impl<'src> GenericPolyfill<'src> { if let Some(bound) = &tv.bound && is_parameters_bound(bound) { - let mangled = self.unique_typevar_name(name, "ParamSpec"); + let default = tv + .default + .as_deref() + .map(|default| self.param_spec_default(default, &visible)); + let mangled = self.declare_param_spec(name, default, &mut defs); renames.insert(name.to_owned(), mangled.clone()); visible.insert(name.to_owned(), mangled.clone()); - defs.push(format!("{mangled} = ParamSpec(\"{mangled}\")")); - self.needed_imports.paramspec = true; param_names.push(mangled.clone()); generic_args.push(mangled); continue; @@ -481,7 +595,8 @@ impl<'src> GenericPolyfill<'src> { self.types, bound, &self.subsume_within(bound.range()), - self.config.float_literals, + &self.config, + RootKind::Evaluated, ) .unwrap_or_else(|| self.src(bound.range()).to_owned()) }; @@ -493,22 +608,10 @@ impl<'src> GenericPolyfill<'src> { } if let Some(default) = &tv.default { - let default_src = lower_type_expr_full( - self.source, - self.written, - self.types, - default, - &self.subsume_within(default.range()), - self.config.float_literals, - ) - .unwrap_or_else(|| self.src(default.range()).to_owned()); if self.config.min_version < PythonVersion::PY313 { self.needed_imports.typevar_needs_ext = true; } - extra_args.push(format!( - "default={}", - apply_renames_to_rendered(&default_src, &visible) - )); + extra_args.push(format!("default={}", self.default_arg(default, &visible))); } // basedpython variance keywords: forward `out`/`in`/`in out` @@ -538,7 +641,11 @@ impl<'src> GenericPolyfill<'src> { visible.insert(name.to_owned(), mangled.clone()); let mut args: Vec = vec![format!("\"{mangled}\"")]; args.extend(extra_args); - let def = format!("{mangled} = TypeVar({})", args.join(", ")); + let def = format!( + "{mangled} = {}({})", + self.constructors.type_var, + args.join(", ") + ); self.needed_imports.typevar = true; param_names.push(mangled.clone()); @@ -548,10 +655,39 @@ impl<'src> GenericPolyfill<'src> { TypeParam::TypeVarTuple(tvt) => { let name = tvt.name.id.as_str(); - let mangled = self.unique_typevar_name(name, "TypeVarTuple"); + // a keyword argument cannot be starred, so the default `*tuple[int, str]` + // is passed as `Unpack[tuple[int, str]]`, the spelling the call accepts + let default = tvt.default.as_deref().map(|default| { + let unpacked = match default { + Expr::Starred(starred) => starred.value.as_ref(), + other => other, + }; + let rendered = self.default_arg(unpacked, &visible); + if matches!(default, Expr::Starred(_)) { + self.needed_imports.unpack = true; + format!("{}[{rendered}]", self.constructors.unpack) + } else { + rendered + } + }); + let signature = default + .as_ref() + .map_or_else(String::new, |default| format!("default={default}")); + let mangled = + self.unique_typevar_name(name, &format!("TypeVarTuple({signature})")); renames.insert(name.to_owned(), mangled.clone()); visible.insert(name.to_owned(), mangled.clone()); - defs.push(format!("{mangled} = TypeVarTuple(\"{mangled}\")")); + let arguments = if default.is_some() { + self.needed_imports.typevar_tuple_needs_ext |= + self.config.min_version < PythonVersion::PY313; + format!(", {signature}") + } else { + String::new() + }; + defs.push(format!( + "{mangled} = {}(\"{mangled}\"{arguments})", + self.constructors.type_var_tuple + )); self.needed_imports.typevar_tuple = true; self.needed_imports.unpack = true; // star-in-subscript (`Generic[*T]`) is only valid syntax @@ -560,7 +696,7 @@ impl<'src> GenericPolyfill<'src> { let arg = if self.config.min_version >= PythonVersion::PY311 { format!("*{mangled}") } else { - format!("Unpack[{mangled}]") + format!("{}[{mangled}]", self.constructors.unpack) }; param_names.push(mangled); generic_args.push(arg); @@ -568,11 +704,13 @@ impl<'src> GenericPolyfill<'src> { TypeParam::ParamSpec(ps) => { let name = ps.name.id.as_str(); - let mangled = self.unique_typevar_name(name, "ParamSpec"); + let default = ps + .default + .as_deref() + .map(|default| self.param_spec_default(default, &visible)); + let mangled = self.declare_param_spec(name, default, &mut defs); renames.insert(name.to_owned(), mangled.clone()); visible.insert(name.to_owned(), mangled.clone()); - defs.push(format!("{mangled} = ParamSpec(\"{mangled}\")")); - self.needed_imports.paramspec = true; param_names.push(mangled.clone()); generic_args.push(mangled); } @@ -600,8 +738,15 @@ impl<'src> GenericPolyfill<'src> { // top-parameters bound → `**T` (PEP 695 paramspec syntax) if is_parameters_bound(bound) { let name = tv.name.id.as_str(); + let default = tv + .default + .as_deref() + .map(|default| { + format!(" = {}", self.param_spec_default(default, &HashMap::new())) + }) + .unwrap_or_default(); self.edits.push(Fix::safe_edit(Edit::range_replacement( - format!("**{name}"), + format!("**{name}{default}"), param.range(), ))); continue; @@ -689,7 +834,7 @@ impl<'src> GenericPolyfill<'src> { // record for module-level variant subclasses that reference these params self.generic_class_renames .insert(class.name.id.as_str().to_owned(), rename_map.clone()); - let generic_str = format!("Generic[{}]", generic_args.join(", ")); + let generic_str = format!("{}[{}]", self.constructors.generic, generic_args.join(", ")); self.needed_imports.generic = true; // Modify or add base classes. @@ -787,8 +932,13 @@ impl<'src> GenericPolyfill<'src> { self.parameters_targets .insert(func.name.id.as_str().to_owned()); } + // basedpython: `some T` declares a type parameter the source writes nowhere in the + // list, so native syntax has no place for it, and it is always declared as a `TypeVar` + let is_hole = + |param: &TypeParam| matches!(param, TypeParam::TypeVar(tv) if tv.is_some_hole); + let has_hole = tp.type_params.iter().any(is_hole); // PEP 695 function type params are native syntax in 3.12+ (3.13+ with defaults) - if self.supports_native_type_params(&tp.type_params) { + if !has_hole && self.supports_native_type_params(&tp.type_params) { self.lower_type_param_bounds(&tp.type_params); return HashMap::new(); } @@ -799,9 +949,13 @@ impl<'src> GenericPolyfill<'src> { .. } = self.process_type_params(&tp.type_params); - // Remove `[T, ...]` from the function signature. - self.edits - .push(Fix::safe_edit(Edit::range_deletion(tp.range()))); + // Remove `[T, ...]` from the function signature. a list of holes alone is the + // parser's, and there are no brackets in the source to remove: its range is the + // parameter list's + if !tp.type_params.iter().all(is_hole) { + self.edits + .push(Fix::safe_edit(Edit::range_deletion(tp.range()))); + } // Insert TypeVar definitions before the function. let (line_start, indent) = self.line_start_of(func.range().start()); @@ -837,11 +991,21 @@ impl<'src> GenericPolyfill<'src> { if let Some(ret) = &func.returns { rename_in_expr(ret, &rename_map, &mut self.edits); } - for stmt in &func.body { - rename_in_stmt(stmt, &rename_map, &mut self.edits); + // the body reads a parameter under its own name, which a type parameter of that name + // does not reach: the parameter is bound in the body's scope, the type parameter in + // the one around it. a `some` hole always has its parameter's name + let mut body_renames = rename_map.clone(); + for parameter in &func.parameters { + body_renames.remove(parameter.name().as_str()); } self.reconcile_pending(&rename_map, mark); - rename_map + let mark = self.edits.len(); + for stmt in &func.body { + rename_in_stmt(stmt, &body_renames, &mut self.edits); + } + self.reconcile_pending(&body_renames, mark); + // what the rest of the body sees, a nested definition among it + body_renames } fn process_type_alias(&mut self, alias: &StmtTypeAlias) { @@ -900,13 +1064,11 @@ impl<'src> GenericPolyfill<'src> { self.needed_imports.typealias_type = true; let (_line_start, indent) = self.line_start_of(alias.range().start()); let indent = indent.to_owned(); - let mut replacement = String::new(); - for d in &defs { - let _ = writeln!(replacement, "{indent}{d}"); - } + let mut replacement = self.dedupe_defs(&defs, &indent); let _ = write!( replacement, - "{indent}{name_src} = TypeAliasType(\"{name_src}\", object{type_params_arg})" + "{indent}{name_src} = {}(\"{name_src}\", object{type_params_arg})", + self.constructors.type_alias_type ); self.edits.push(Fix::safe_edit(Edit::range_replacement( replacement, @@ -944,7 +1106,8 @@ impl<'src> GenericPolyfill<'src> { self.types, &alias.value, &substitutions, - self.config.float_literals, + &self.config, + RootKind::Evaluated, ) .unwrap_or(raw_value_src); @@ -953,13 +1116,11 @@ impl<'src> GenericPolyfill<'src> { let (_line_start, indent) = self.line_start_of(alias.range().start()); let indent = indent.to_owned(); - let mut replacement = String::new(); - for d in &defs { - let _ = writeln!(replacement, "{indent}{d}"); - } + let mut replacement = self.dedupe_defs(&defs, &indent); let _ = write!( replacement, - "{indent}{name_src} = TypeAliasType(\"{name_src}\", {value_src}{type_params_arg})" + "{indent}{name_src} = {}(\"{name_src}\", {value_src}{type_params_arg})", + self.constructors.type_alias_type ); self.edits.push(Fix::safe_edit(Edit::range_replacement( @@ -975,6 +1136,61 @@ impl GenericPolyfill<'_> { /// the substitution. Parameters spec syntax (`(int, str, /, name: T)`) /// drops the `/` and `*` markers and replaces named-only fields with /// `Any` since runtime `ParamSpec` only carries positional types + /// a parameter-shape tuple (`(int, name: str)`) as the list a `ParamSpec` is specialized by + /// or defaults to at runtime. each element is written as the source spells it, or, given + /// the type parameters `visible` in scope, lowered as a default is + fn parameter_shape_list( + &mut self, + t: &ruff_python_ast::ExprTuple, + visible: Option<&HashMap>, + ) -> String { + // the inner structure (markers, named, variadic, kwargs) doesn't map + // 1:1 to runtime ParamSpec list elements, so we lower each + // element to a positional Python type. mapping: + // `int` → `int` + // `name: T` → `Any` (named-only has no positional slot) + // `*: T` → `Any` (variadic flattened to one element) + // `*name: T` → `Any` + // `**: T` → dropped + // `**name: T` → dropped + let mut parts: Vec = Vec::new(); + for elt in &t.elts { + match elt { + Expr::Named(named) => { + if let Expr::Starred(starred) = named.target.as_ref() { + // `**name: T` — Starred(Starred(...)) target → drop + if matches!(starred.value.as_ref(), Expr::Starred(_)) { + continue; + } + // `*name: T` + parts.push("Any".to_owned()); + self.needed_imports_any = true; + } else { + // `name: T` + parts.push("Any".to_owned()); + self.needed_imports_any = true; + } + } + Expr::Starred(s) => { + if matches!(s.value.as_ref(), Expr::Starred(_)) { + // `**: T` — drop + continue; + } + // `*: T` + parts.push("Any".to_owned()); + self.needed_imports_any = true; + } + _ => { + parts.push(match visible { + Some(visible) => self.default_arg(elt, visible), + None => self.src(elt.range()).to_owned(), + }); + } + } + } + format!("[{}]", parts.join(", ")) + } + fn rewrite_parameters_subscript(&mut self, sub: &ruff_python_ast::ExprSubscript) { let Expr::Name(name) = sub.value.as_ref() else { return; @@ -990,52 +1206,9 @@ impl GenericPolyfill<'_> { } if t.has_parameter_shape() { - // emit a single replacement for the whole tuple — the inner - // structure (markers, named, variadic, kwargs) doesn't map - // 1:1 to runtime ParamSpec list elements, so we lower each - // element to a positional Python type. mapping: - // `int` → `int` - // `name: T` → `Any` (named-only has no positional slot) - // `*: T` → `Any` (variadic flattened to one element) - // `*name: T` → `Any` - // `**: T` → dropped - // `**name: T` → dropped - let mut parts: Vec = Vec::new(); - for elt in &t.elts { - match elt { - Expr::Named(named) => { - if let Expr::Starred(starred) = named.target.as_ref() { - // `**name: T` — Starred(Starred(...)) target → drop - if matches!(starred.value.as_ref(), Expr::Starred(_)) { - continue; - } - // `*name: T` - parts.push("Any".to_owned()); - self.needed_imports_any = true; - } else { - // `name: T` - parts.push("Any".to_owned()); - self.needed_imports_any = true; - } - } - Expr::Starred(s) => { - if matches!(s.value.as_ref(), Expr::Starred(_)) { - // `**: T` — drop - continue; - } - // `*: T` - parts.push("Any".to_owned()); - self.needed_imports_any = true; - } - _ => { - parts.push(self.src(elt.range()).to_owned()); - } - } - } - self.edits.push(Fix::safe_edit(Edit::range_replacement( - format!("[{}]", parts.join(", ")), - t.range(), - ))); + let list = self.parameter_shape_list(t, None); + self.edits + .push(Fix::safe_edit(Edit::range_replacement(list, t.range()))); return; } @@ -1077,6 +1250,15 @@ impl<'ast> Visitor<'ast> for GenericPolyfill<'_> { } } + fn visit_body(&mut self, body: &'ast [Stmt]) { + let outer = self.suite; + if let Some(first) = body.first() { + self.suite = Suite::Nested(first.start()); + } + walk_body(self, body); + self.suite = outer; + } + fn visit_expr(&mut self, expr: &'ast Expr) { if let Expr::Subscript(sub) = expr { self.rewrite_parameters_subscript(sub); @@ -1291,14 +1473,26 @@ fn rename_in_stmt(stmt: &Stmt, renames: &HashMap, edits: &mut Ve } } -pub(crate) fn mangle(name: &str) -> String { +/// the name the polyfill first tries for the `TypeVar` a type parameter named `name` is +/// declared as: `_T` for `T`, spelled so that the module spells it nowhere. the definition is +/// written into the scope the generic stands in, where a name the module binds would be +/// overwritten by it +pub(crate) fn polyfilled_name(written: WrittenNames, name: &str) -> String { if name.starts_with('_') { - name.to_owned() + written.fresh(name) } else { - format!("_{name}") + written.fresh(&format!("_{name}")) } } +/// the statement list a definition the polyfill writes lands in, told apart by where its +/// first statement starts +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +enum Suite { + Module, + Nested(TextSize), +} + pub(crate) struct GenericPolyfillPass<'src> { source: &'src str, written: WrittenNames<'src>, @@ -1360,7 +1554,7 @@ impl super::ast_driver::TypeAwarePass for GenericPolyfillPass<'_> { ctx.text_edits .retain(|(range, _)| !superseded.contains(range)); let emits_any = inner.needed_imports_any; - for line in std::mem::take(&mut inner.needed_imports).into_lines() { + for line in std::mem::take(&mut inner.needed_imports).into_lines(&inner.constructors) { ctx.required_imports.push(line); } if emits_any { @@ -1888,25 +2082,166 @@ mod tests { #[test] fn paramspec_and_typevartuple_defaults_downlevel_on_312() { - // defaults on any parameter kind make the header 3.13-only syntax, so - // the whole declaration polyfills. the polyfill itself drops paramspec - // and typevartuple defaults, matching the pre-3.12 path + // defaults on any parameter kind make the header 3.13-only syntax, so the whole + // declaration polyfills. `typing`'s own `ParamSpec` and `TypeVarTuple` take no + // `default=` below 3.13, so both come from `typing_extensions`. a keyword argument + // cannot be starred, so a variadic's default is passed unpacked with `Unpack` check_at( indoc! {" class A[**P = [int]]: ... class B[*Ts = *tuple[int, str]]: ... "}, indoc! {" - from typing import TypeVarTuple, Unpack, ParamSpec, Generic - _P = ParamSpec(\"_P\") + from typing import Unpack, Generic + from typing_extensions import TypeVarTuple, ParamSpec + _P = ParamSpec(\"_P\", default=[int]) class A(Generic[_P]): ... - _Ts = TypeVarTuple(\"_Ts\") + _Ts = TypeVarTuple(\"_Ts\", default=Unpack[tuple[int, str]]) class B(Generic[*_Ts]): ... "}, PythonVersion::PY312, ); } + /// a parameter specification spelled with the top-parameters bound takes its default as a + /// parameter list, which python spells as a list + #[test] + fn a_parameters_bound_default_is_a_list() { + check_at( + "class A[P: (*: *, **: *) = (int, str)]: ...\n", + indoc! {" + from typing import Generic + from typing_extensions import ParamSpec + _P = ParamSpec(\"_P\", default=[int, str]) + class A(Generic[_P]): ... + "}, + PythonVersion::PY312, + ); + check_at( + "class A[P: (*: *, **: *) = (int, str)]: ...\n", + "class A[**P = [int, str]]: ...\n", + PythonVersion::PY313, + ); + } + + /// a default needs the `TypeVar` from `typing_extensions` below 3.13, and the one a module + /// imports from `typing` itself takes none. so the polyfill calls its own by a name the + /// module does not spell, and the module's `TypeVar` keeps meaning what it imported + #[test] + fn a_default_calls_a_type_var_the_module_does_not_bind() { + check_at( + indoc! {" + from typing import TypeVar + U = TypeVar(\"U\") + class A[T = int]: ... + "}, + indoc! {" + from typing_extensions import TypeVar as TypeVar2 + from typing import TypeVar, Generic + U = TypeVar(\"U\") + _T = TypeVar2(\"_T\", default=int) + class A(Generic[_T]): ... + "}, + PythonVersion::PY312, + ); + } + + /// every constructor the polyfill writes is called by a name the module does not spell, + /// whatever the module binds that name to + #[test] + fn a_constructor_the_module_binds_is_imported_under_another_name() { + check_at( + indoc! {" + class Generic: ... + class A[T, *Ts, **P]: ... + "}, + indoc! {" + from typing import TypeVar, TypeVarTuple, Unpack, ParamSpec, Generic as Generic2 + class Generic: ... + _T = TypeVar(\"_T\") + _Ts = TypeVarTuple(\"_Ts\") + _P = ParamSpec(\"_P\") + class A(Generic2[_T, *_Ts, _P]): ... + "}, + PythonVersion::PY311, + ); + } + + /// the `TypeVar` a type parameter is declared as is written into the module, where a + /// binding of the module's own under the same name would be overwritten by it + #[test] + fn a_type_variable_the_module_binds_keeps_its_name() { + check_at( + indoc! {" + from typing import TypeVar + _T = TypeVar(\"_T\", bound=int) + def legacy(x: _T) -> _T: ... + def modern[T](x: T) -> T: ... + class A[_T]: ... + "}, + indoc! {" + from typing import TypeVar, TypeVar as TypeVar2, Generic + _T = TypeVar(\"_T\", bound=int) + def legacy(x: _T) -> _T: ... + _T2 = TypeVar2(\"_T2\") + def modern(x: _T2) -> _T2: ... + class A(Generic[_T2]): ... + "}, + PythonVersion::PY311, + ); + } + + /// a definition written into a class body or an `if` is not in scope where the module + /// declares a function later, so that function gets a definition of its own, under a name + /// that shadows neither + #[test] + fn a_type_variable_is_bound_in_one_suite() { + check_at( + indoc! {" + class C: + def m[T](self, x: T) -> T: ... + if flag: + def g[T](x: T) -> T: ... + def f[T](x: T) -> T: ... + def h[T](x: T) -> T: ... + "}, + indoc! {" + from typing import TypeVar + class C: + _T = TypeVar(\"_T\") + def m(self, x: _T) -> _T: ... + if flag: + _T_1 = TypeVar(\"_T_1\") + def g(x: _T_1) -> _T_1: ... + _T_2 = TypeVar(\"_T_2\") + def f(x: _T_2) -> _T_2: ... + def h(x: _T_2) -> _T_2: ... + "}, + PythonVersion::PY311, + ); + } + + /// a numbered name is skipped when the module spells it, as the first name is + #[test] + fn a_numbered_type_variable_skips_a_name_the_module_binds() { + check_at( + indoc! {" + _T_1 = 1 + def f[T: int](x: T) -> T: ... + def g[T](x: T) -> T: ... + "}, + indoc! {" + from typing import TypeVar + _T_1 = 1 + _T = TypeVar(\"_T\", bound=int) + def f(x: _T) -> _T: ... + _T_2 = TypeVar(\"_T_2\") + def g(x: _T_2) -> _T_2: ... + "}, + PythonVersion::PY311, + ); + } + #[test] fn type_alias_default_downleveled_on_312() { check_at( @@ -2382,4 +2717,112 @@ mod tests { "}, ); } + + /// `some int` declares a type parameter named after its parameter. the parser gives the + /// list it synthesizes the parameter list's range, which has no brackets to remove + #[test] + fn some_parameter() { + check( + indoc! {" + def inc(n: some int) -> int: + return n + 1 + "}, + indoc! {" + from typing import TypeVar + _n = TypeVar(\"_n\", bound=int) + def inc(n: _n) -> int: + return n + 1 + "}, + ); + } + + /// native syntax has nowhere to declare a `some` hole, so it is a `TypeVar` at every + /// version, and the signature's other reads of it are renamed with it + #[test] + fn some_parameter_read_by_the_return_type() { + check_at( + indoc! {" + def echo[T](x: T, n: some int) -> n: + return n + "}, + indoc! {" + from typing import TypeVar + _T = TypeVar(\"_T\") + _n = TypeVar(\"_n\", bound=int) + def echo(x: _T, n: _n) -> _n: + return n + "}, + PythonVersion::PY313, + ); + } + + /// the body reads a `some` parameter under its own name, which the hole shares, in a + /// nested definition too + #[test] + fn a_some_parameter_is_not_renamed_in_the_body() { + check( + indoc! {" + def show(n: some int) -> n: + def inner() -> None: + print(n) + inner() + return n + "}, + indoc! {" + from typing import TypeVar + _n = TypeVar(\"_n\", bound=int) + def show(n: _n) -> _n: + def inner() -> None: + print(n) + inner() + return n + "}, + ); + } + + /// a bound is evaluated when the `TypeVar` is made, so below 3.10 an optional in it is + /// the `Union` the optional lowering spells for that version, written whole + #[test] + fn an_optional_bound_below_310() { + check_at( + indoc! {" + def f[T: int?](x: T) -> T: + return x + def g(n: some int?) -> int: + return 0 + type B = int? + "}, + indoc! {" + from __future__ import annotations + from typing import TypeVar, Union + from typing_extensions import TypeAliasType + _T = TypeVar(\"_T\", bound=Union[int, None]) + def f(x: _T) -> _T: + return x + _n = TypeVar(\"_n\", bound=Union[int, None]) + def g(n: _n) -> int: + return 0 + B = TypeAliasType(\"B\", Union[int, None]) + "}, + PythonVersion::PY39, + ); + } + + /// an optional of an optional keeps its outer layer as the runtime `Optional` wrapper, + /// which the bound writes once + #[test] + fn a_nested_optional_bound() { + let output = crate::transpile( + "def f[T: int??](x: T) -> T:\n return x\n", + &crate::Config { + min_version: PythonVersion::PY311, + ..crate::Config::test_default() + }, + ) + .expect("transpile failed"); + assert!( + output.contains("TypeVar(\"_T\", bound=Optional[int | None])"), + "unexpected output:\n{output}" + ); + } } diff --git a/crates/by_transforms/src/transforms/inferred_annotation.rs b/crates/by_transforms/src/transforms/inferred_annotation.rs index 43c9311224..1626abf587 100644 --- a/crates/by_transforms/src/transforms/inferred_annotation.rs +++ b/crates/by_transforms/src/transforms/inferred_annotation.rs @@ -10,7 +10,9 @@ //! (`Literal[1]` → `int`) and spliced in as an annotation, so the class's //! implicit attribute types become declared. //! -//! only a bare ` = ` directly in a class body is rewritten. +//! only a bare ` = ` directly in a class body is rewritten, and a +//! declaration that states no type, `var = `, which is the same +//! assignment. //! multi-target (`a = b = 1`) and unpacking (`a, b = ...`) assignments are //! left alone, as are assignments whose value has no faithful annotation: one //! with an `Unknown` / `Any` part, and one whose type python cannot spell in a @@ -22,8 +24,9 @@ use std::collections::BTreeSet; +use ruff_python_ast::helpers::{DeclarationMarker, DeclarationMarkerKind}; use ruff_python_ast::visitor::{Visitor, walk_stmt}; -use ruff_python_ast::{Expr, Stmt, StmtAssign, StmtClassDef}; +use ruff_python_ast::{Expr, ExprName, Stmt, StmtClassDef}; use ruff_text_size::{Ranged, TextRange}; use crate::transforms::ast_driver::{PassContext, TypeAwarePass}; @@ -83,28 +86,34 @@ impl State<'_> { return; } for stmt in &class.body { - if let Stmt::Assign(assign) = stmt { - self.process_assign(assign); + match stmt { + Stmt::Assign(assign) => { + // only a single ` = ` — chained (`a = b = 1`) and + // unpacking (`a, b = ...`) targets have no single declared type + if let [Expr::Name(name)] = assign.targets.as_slice() { + self.process_assign(name, &assign.value); + } + } + Stmt::AnnAssign(declaration) + if let Expr::Name(name) = declaration.target.as_ref() + && let Some(value) = &declaration.value + && states_no_type(&declaration.annotation) => + { + self.process_assign(name, value); + } + _ => {} } } } - fn process_assign(&mut self, assign: &StmtAssign) { - // only a single ` = ` — chained (`a = b = 1`) and - // unpacking (`a, b = ...`) targets have no single declared type - let [Expr::Name(name)] = assign.targets.as_slice() else { - return; - }; + fn process_assign(&mut self, name: &ExprName, value: &Expr) { // dunders (`__slots__`, `__match_args__`, ...) are class machinery, not // typed attributes — annotating them is noise (and `__slots__ = ()` is // exactly what the enum lowering re-feeds through this pipeline) if is_dunder(name.id.as_str()) { return; } - let Some(annotation) = self - .types - .inferred_annotation(&assign.value, self.min_version) - else { + let Some(annotation) = self.types.inferred_annotation(value, self.min_version) else { return; }; let pos = name.range().end(); @@ -115,6 +124,13 @@ impl State<'_> { } } +/// whether a declaration's annotation is the marker of `[modifiers] = `, +/// which declares no type of its own +fn states_no_type(annotation: &Expr) -> bool { + matches!(annotation, Expr::Name(marker) if DeclarationMarker::from_id(marker.id.as_str()) + .is_some_and(|marker| marker.kind == DeclarationMarkerKind::Assign)) +} + fn is_dunder(name: &str) -> bool { name.len() > 4 && name.starts_with("__") && name.ends_with("__") } @@ -155,6 +171,52 @@ mod tests { ); } + /// `var` and a modifier chain declare no type, so the member is the bare assignment + /// it lowers to and gets the same annotation + #[test] + fn a_declaration_stating_no_type_is_annotated() { + check( + indoc! {" + class A: + var a = 0 + b = 0 + var c = 1.5 + private var d = \"x\" + protected e = [1] + "}, + indoc! {" + class A: + a: int = 0 + b: int = 0 + c: float = 1.5 + __d: str = \"x\" + _e: list[int] = [1] + "}, + ); + } + + /// outside a class body a declaration is left as the assignment it lowers to, as a + /// bare one is + #[test] + fn a_declaration_outside_a_class_is_left_bare() { + check( + indoc! {" + var a = 0 + b = 0 + def f(): + var c = 0 + return c + "}, + indoc! {" + a = 0 + b = 0 + def f(): + c = 0 + return c + "}, + ); + } + #[test] fn str_and_bool_attributes() { check( diff --git a/crates/by_transforms/src/transforms/init_method.rs b/crates/by_transforms/src/transforms/init_method.rs index f910e29c8f..06bd908398 100644 --- a/crates/by_transforms/src/transforms/init_method.rs +++ b/crates/by_transforms/src/transforms/init_method.rs @@ -33,28 +33,22 @@ use super::callable::lower_type_expr_full; use super::mutable_defaults::{parameter_guards, undeclarable_error}; use super::repeated_underscore::WrittenNames; use super::source_util::{PrologueStatement, body_prologue, first_body_statement}; -use crate::config::FloatLiteralLowering; +use super::type_expr_walker::RootKind; +use crate::config::Config; use crate::type_info::TypeInfo; pub(crate) struct InitMethod<'src> { source: &'src str, written: WrittenNames<'src>, - float_literals: FloatLiteralLowering, - is_stub: bool, + config: Config, } impl<'src> InitMethod<'src> { - pub(crate) fn new( - source: &'src str, - written: WrittenNames<'src>, - float_literals: FloatLiteralLowering, - is_stub: bool, - ) -> Self { + pub(crate) fn new(source: &'src str, written: WrittenNames<'src>, config: Config) -> Self { Self { source, written, - float_literals, - is_stub, + config, } } } @@ -66,8 +60,7 @@ impl TypeAwarePass for InitMethod<'_> { written: self.written, types, symbolic_substitutions: ctx.symbolic_substitutions.clone(), - float_literals: self.float_literals, - is_stub: self.is_stub, + config: &self.config, edits: RefCell::new(Vec::new()), templates: RefCell::new(Vec::new()), relocating: RefCell::new(Vec::new()), @@ -141,8 +134,7 @@ struct State<'src> { /// line is fresh output, so a fold inside a parameter annotation is dropped /// unless it is spliced in here symbolic_substitutions: Vec<(TextRange, String)>, - float_literals: FloatLiteralLowering, - is_stub: bool, + config: &'src Config, edits: RefCell>, templates: RefCell)>>, /// the `_MISSING` substitutions, whose defaults the guards re-evaluate @@ -207,7 +199,8 @@ impl State<'_> { self.types, ann, &substitutions, - self.float_literals, + self.config, + RootKind::Annotation, ) .unwrap_or_else(|| { self.source[usize::from(ann.range().start())..usize::from(ann.range().end())].to_owned() @@ -366,7 +359,7 @@ impl State<'_> { self.written, &super::mutable_defaults::sentinel_name(self.source), self.types, - self.is_stub, + self.config.is_stub, ); if let Some(parameter) = undeclarable.first() { self.errors diff --git a/crates/by_transforms/src/transforms/literal_types.rs b/crates/by_transforms/src/transforms/literal_types.rs index f04d4f46e1..a5e0957409 100644 --- a/crates/by_transforms/src/transforms/literal_types.rs +++ b/crates/by_transforms/src/transforms/literal_types.rs @@ -284,6 +284,31 @@ fn is_literal_expr(expr: &Expr, float_literals: FloatLiteralLowering) -> bool { } } +/// how a float or complex literal type written as `text` is spelled in the output, or +/// `None` when `expr` is no such literal: what [`LiteralType`] writes for one the source wrote +pub(crate) fn float_literal_spelling( + expr: &Expr, + text: &str, + float_literals: FloatLiteralLowering, +) -> Option { + if let Some(nominal) = nominal_float_type(expr, float_literals) { + return Some(nominal.to_owned()); + } + let is_float = + |number: &ruff_python_ast::Number| !matches!(number, ruff_python_ast::Number::Int(_)); + let number = match expr { + Expr::NumberLiteral(n) => &n.value, + Expr::UnaryOp(u) if matches!(u.op, UnaryOp::USub | UnaryOp::UAdd) => { + match u.operand.as_ref() { + Expr::NumberLiteral(n) => &n.value, + _ => return None, + } + } + _ => return None, + }; + is_float(number).then(|| format!("Literal[{text}]")) +} + /// the builtin a float or complex literal type is one of, when the project /// spells such a literal with its nominal type rather than with `Literal[...]`. /// diff --git a/crates/by_transforms/src/transforms/modifiers.rs b/crates/by_transforms/src/transforms/modifiers.rs index 37b53d5351..d83e30366e 100644 --- a/crates/by_transforms/src/transforms/modifiers.rs +++ b/crates/by_transforms/src/transforms/modifiers.rs @@ -579,9 +579,17 @@ impl<'src> Modifiers<'src> { id if DeclarationMarker::from_id(id) .is_some_and(|marker| marker.kind == DeclarationMarkerKind::Assign) => { + // the name and the `=` are two edits, so the type the + // inferred-annotation lowering gives a class member lands + // between them + let target_end = node.target.range().end(); self.edits.push(Fix::safe_edit(Edit::range_replacement( - format!("{name} = "), - prefix_range, + name, + TextRange::new(stmt_start, target_end), + ))); + self.edits.push(Fix::safe_edit(Edit::range_replacement( + " = ".to_owned(), + TextRange::new(target_end, value_range.start()), ))); } id if is_classvar_marker_id(id) => { @@ -1324,7 +1332,7 @@ mod tests { "}, indoc! {" class A: - a = 1 + a: int = 1 "}, ); } @@ -1357,7 +1365,7 @@ mod tests { "}, indoc! {" class A: - a = 1 + a: int = 1 b: int = 2 c: str "}, @@ -1823,7 +1831,7 @@ mod tests { "}, indoc! {" class Foo: - a = 1 + a: int = 1 "}, ); } @@ -1994,8 +2002,8 @@ mod tests { "}, indoc! {" class Outer: - __count = 0 - _step = 2 + __count: int = 0 + _step: int = 2 "}, ); } diff --git a/crates/by_transforms/src/transforms/optional_type.rs b/crates/by_transforms/src/transforms/optional_type.rs index bdc7b30e9f..7d0da61513 100644 --- a/crates/by_transforms/src/transforms/optional_type.rs +++ b/crates/by_transforms/src/transforms/optional_type.rs @@ -6,8 +6,9 @@ //! //! A target older than 3.10 has no `type.__or__`, and an optional is written in //! plenty of places the runtime evaluates — a `cast` target, an alias — so for -//! those the union is spelled `Union[T, None]` instead. Which spelling is used -//! never changes what the type means. +//! those the union is spelled `Union[T, None]` instead, and `(T, type(None),)` +//! where `isinstance` expects classes, as the runtime union lowering spells a +//! `|` there. Which spelling is used never changes what the type means. //! //! It emits narrow text edits rather than mutating the AST so it composes with //! the value-position operator lowerings that share a statement — e.g. a @@ -24,6 +25,8 @@ //! postfix `^` / `!` operators are intentionally left for a later pass — their //! runtime representation is still being settled. +use std::collections::HashSet; + use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; use ruff_python_ast::{Expr, PythonVersion, Stmt, UnaryOp}; use ruff_text_size::{Ranged, TextRange}; @@ -43,6 +46,9 @@ struct OptionalLower<'src> { needs_union: bool, /// whether the target can spell a union with `|` (python 3.10) native_union: bool, + /// the optionals standing where `isinstance` expects classes, which a target + /// without `|` spells as a tuple of classes, see [`super::runtime_union`] + classinfo: HashSet, source: &'src str, /// stack of in-scope PEP 695 type-parameter names. `?` over a bare type /// variable lowers to the *wrapped* form (`Optional[T | None]`) — a plain @@ -58,6 +64,7 @@ impl<'src> OptionalLower<'src> { needs_runtime: false, needs_union: false, native_union: min_version >= PythonVersion::PY310, + classinfo: HashSet::new(), source, typevar_scopes: Vec::new(), } @@ -134,8 +141,11 @@ impl<'ast> Visitor<'ast> for OptionalLower<'_> { if wrap_layers >= 1 { self.needs_runtime = true; } + let classinfo = !self.native_union && wrap_layers == 0 && self.classinfo.contains(&node); let mut prefix = "Optional[".repeat(wrap_layers); - if !self.native_union { + if classinfo { + prefix.push('('); + } else if !self.native_union { self.needs_union = true; prefix.push_str("Union["); } @@ -143,7 +153,9 @@ impl<'ast> Visitor<'ast> for OptionalLower<'_> { self.edits.push((TextRange::empty(node.start()), prefix)); } let mut replacement = close_parens; - if self.native_union { + if classinfo { + replacement.push_str(", type(None),)"); + } else if self.native_union { replacement.push_str(" | None"); } else { replacement.push_str(", None]"); @@ -224,8 +236,11 @@ impl TypeAwarePass for OptionalTypePass<'_> { Some(super::ast_driver::Lowering::OptionalType) } - fn run(&self, stmts: &[Stmt], _types: &dyn TypeInfo, ctx: &mut PassContext) { + fn run(&self, stmts: &[Stmt], types: &dyn TypeInfo, ctx: &mut PassContext) { let mut lower = OptionalLower::new(self.source, self.min_version); + if !lower.native_union { + lower.classinfo = super::runtime_union::classinfo_optionals(stmts, types); + } for stmt in stmts { lower.visit_stmt(stmt); } diff --git a/crates/by_transforms/src/transforms/properties.rs b/crates/by_transforms/src/transforms/properties.rs index 6b9088c15f..e184b9726a 100644 --- a/crates/by_transforms/src/transforms/properties.rs +++ b/crates/by_transforms/src/transforms/properties.rs @@ -129,13 +129,19 @@ impl<'ast> Visitor<'ast> for ValueRangeCollector<'_> { pub(crate) struct PropertiesPass<'src> { source: &'src str, value_ranges: ValueRanges, + min_version: ruff_python_ast::PythonVersion, } impl<'src> PropertiesPass<'src> { - pub(crate) fn new(source: &'src str, value_ranges: ValueRanges) -> Self { + pub(crate) fn new( + source: &'src str, + value_ranges: ValueRanges, + min_version: ruff_python_ast::PythonVersion, + ) -> Self { Self { source, value_ranges, + min_version, } } @@ -197,9 +203,21 @@ impl<'ast> Visitor<'ast> for FieldAccessFinder<'_> { /// and/or an initialiser, each emitted as a source passthrough. struct Backing<'a> { annotation: Option<&'a Expr>, + /// the type ty infers for an initialiser declared with no type, written where + /// `inferred_annotation` writes one for a bare class-body assignment + inferred: Option, value: Option<&'a Expr>, } +impl Backing<'_> { + /// the backing field's annotation, after the `: ` it follows + fn annotation_fragment(&self) -> Option { + self.annotation + .map(|annotation| Fragment::Src(annotation.range())) + .or_else(|| self.inferred.clone().map(Fragment::Lit)) + } +} + /// the leading whitespace of the line `text` starts at fn leading_whitespace(text: &str) -> &str { let line = text.split_once('\n').map_or(text, |(line, _)| line); @@ -473,7 +491,7 @@ impl PropertiesPass<'_> { spans } - fn process_class(&self, class: &StmtClassDef, ctx: &mut PassContext) { + fn process_class(&self, class: &StmtClassDef, types: &dyn TypeInfo, ctx: &mut PassContext) { let placement = self.init_placement(class); // assignment templates for every backing field whose initialiser moved into // `__init__`, in class-body order @@ -545,6 +563,7 @@ impl PropertiesPass<'_> { .then_some(assign.annotation.as_ref()); Some(Backing { annotation, + inferred: None, value: assign.value.as_deref(), }) } @@ -552,8 +571,26 @@ impl PropertiesPass<'_> { if within(assign.range()) && matches!(assign.targets.first(), Some(Expr::Name(name)) if name.id.as_str() == backing_name) => { + // a declaration with no type is a bare class-body assignment to ty, + // which gets the type ty infers for it where a class body's + // annotations mean nothing more than a type + let inferred = (!types.class_body_annotation_is_semantic(class)) + .then(|| types.inferred_annotation(&assign.value, self.min_version)) + .flatten() + .map(|synthesized| { + ctx.type_only_imports + .extend(synthesized.modules.into_iter().map(|m| format!("import {m}"))); + ctx.type_only_imports.extend( + synthesized + .typing_names + .into_iter() + .map(|name| format!("from typing import {name}")), + ); + synthesized.text + }); Some(Backing { annotation: None, + inferred, value: Some(assign.value.as_ref()), }) } @@ -596,9 +633,9 @@ impl PropertiesPass<'_> { backing.value.is_some() && !matches!(placement, InitPlacement::ClassLevel); if moves { let mut assign = vec![Fragment::Lit(format!("self.{backing_name}"))]; - if let Some(annotation) = backing.annotation { + if let Some(annotation) = backing.annotation_fragment() { assign.push(Fragment::Lit(": ".to_owned())); - assign.push(Fragment::Src(annotation.range())); + assign.push(annotation); } if let Some(value) = backing.value { assign.push(Fragment::Lit(" = ".to_owned())); @@ -607,9 +644,9 @@ impl PropertiesPass<'_> { moved.push(assign); } else { frags.push(Fragment::Lit(backing_name.clone())); - if let Some(annotation) = backing.annotation { + if let Some(annotation) = backing.annotation_fragment() { frags.push(Fragment::Lit(": ".to_owned())); - frags.push(Fragment::Src(annotation.range())); + frags.push(annotation); } if let Some(value) = backing.value { frags.push(Fragment::Lit(" = ".to_owned())); @@ -713,8 +750,8 @@ fn join_assignments(out: &mut Vec, assignments: Vec>, in impl TypeAwarePass for PropertiesPass<'_> { /// an accessor block it replaces is written as the `@property` members the parser - /// synthesized, with the modifiers as their decorators and the declared type as the - /// backing field's annotation + /// synthesized, with the modifiers as their decorators and the declared type — or, + /// for a declaration with none, the type ty infers — as the backing field's annotation fn subsumes(&self) -> &'static [super::ast_driver::Lowering] { &[ super::ast_driver::Lowering::Modifiers, @@ -722,7 +759,7 @@ impl TypeAwarePass for PropertiesPass<'_> { ] } - fn run(&self, stmts: &[Stmt], _types: &dyn TypeInfo, ctx: &mut PassContext) { + fn run(&self, stmts: &[Stmt], types: &dyn TypeInfo, ctx: &mut PassContext) { let mut finder = ClassFinder { classes: Vec::new(), }; @@ -730,7 +767,7 @@ impl TypeAwarePass for PropertiesPass<'_> { finder.visit_stmt(stmt); } for class in finder.classes { - self.process_class(class, ctx); + self.process_class(class, types, ctx); } } } @@ -969,6 +1006,32 @@ mod tests { ); } + /// a declaration with no type gives the backing field the type ty infers for it, as a + /// bare class-body assignment gets one + #[test] + fn an_untyped_property_backing_field_takes_the_inferred_type() { + check( + indoc! {" + class Counter: + var count = 0 + get() = field + set(value): + field = value + "}, + indoc! {" + class Counter: + def __init__(self) -> None: + self.__count: int = 0 + @property + def count(self): + return self.__count + @count.setter + def count(self, value) -> None: + self.__count = value + "}, + ); + } + /// an accessor that never mentions `field` is computed — no backing storage #[test] fn computed_property() { diff --git a/crates/by_transforms/src/transforms/protocol_type.rs b/crates/by_transforms/src/transforms/protocol_type.rs index e91aa06861..146d93e577 100644 --- a/crates/by_transforms/src/transforms/protocol_type.rs +++ b/crates/by_transforms/src/transforms/protocol_type.rs @@ -39,7 +39,7 @@ use ruff_diagnostics::{Edit, Fix}; use ruff_python_ast::{Expr, ExprProtocolType, Stmt}; use ruff_text_size::{Ranged, TextRange}; -use crate::config::FloatLiteralLowering; +use crate::config::Config; use crate::type_info::TypeInfo; use super::ast_driver::{PassContext, TypeAwarePass}; @@ -123,11 +123,11 @@ impl<'src> ProtocolTypeLowering<'src> { written: WrittenNames<'src>, types: &'src dyn TypeInfo, claimed: &'src [TextRange], - float_literals: FloatLiteralLowering, + config: &Config, ) -> Self { Self { source, - callable: CallableSyntax::new(source, written, float_literals) + callable: CallableSyntax::new(source, written, config) .with_types(types) .with_claimed_ranges(claimed), edits: Vec::new(), @@ -338,6 +338,7 @@ impl<'ast> ruff_python_ast::visitor::Visitor<'ast> for ValueProtocolWalker<'_, ' /// Collects the typevar renames the PEP 695 polyfill will apply, keyed by the /// range of the generic scope that declares them. struct TypevarScopeWalker<'a> { + written: WrittenNames<'a>, config: crate::Config, scopes: &'a mut Vec<(TextRange, HashMap)>, } @@ -356,7 +357,10 @@ impl TypevarScopeWalker<'_> { .iter() .map(|param| { let name = param.name().id.as_str(); - (name.to_owned(), super::generics::mangle(name)) + ( + name.to_owned(), + super::generics::polyfilled_name(self.written, name), + ) }) .collect(); if !frame.is_empty() { @@ -388,10 +392,10 @@ fn lower<'src>( stmts: &[Stmt], config: &crate::Config, ) -> ProtocolTypeLowering<'src> { - let mut inner = - ProtocolTypeLowering::new(source, written, types, claimed, config.float_literals); + let mut inner = ProtocolTypeLowering::new(source, written, types, claimed, config); { let mut walker = TypevarScopeWalker { + written, config: config.clone(), scopes: &mut inner.typevar_scopes, }; diff --git a/crates/by_transforms/src/transforms/reified_generic.rs b/crates/by_transforms/src/transforms/reified_generic.rs index bb15e42279..5f143e4dd6 100644 --- a/crates/by_transforms/src/transforms/reified_generic.rs +++ b/crates/by_transforms/src/transforms/reified_generic.rs @@ -558,6 +558,27 @@ mod tests { ); } + /// the body reads the parameter `n`, not the type parameter of that name, so nothing is + /// reified and the function lowers like any other generic one + #[test] + fn a_parameter_named_like_the_type_parameter_is_not_a_read_of_it() { + let output = transpile( + indoc! {" + def f[n](n: n) -> n: + return n + "}, + &Config { + min_version: PythonVersion::PY311, + ..Config::test_default() + }, + ) + .expect("transpile failed"); + assert!( + output.contains("def f(n: _n) -> _n:\n return n\n"), + "unexpected output:\n{output}" + ); + } + #[test] fn default_on_312_is_an_error() { // pep 696 defaults are 3.13-only syntax, and a reified function can't diff --git a/crates/by_transforms/src/transforms/repeated_underscore.rs b/crates/by_transforms/src/transforms/repeated_underscore.rs index 933eead6d3..48d7234273 100644 --- a/crates/by_transforms/src/transforms/repeated_underscore.rs +++ b/crates/by_transforms/src/transforms/repeated_underscore.rs @@ -42,7 +42,7 @@ type Decision = Result; /// list — read off ty before any pass rewrites the syntax tree, so the passes that walk a /// tree of their own can still ask #[derive(Debug)] -pub(crate) struct UnderscoreLowerings { +pub struct UnderscoreLowerings { decisions: HashMap, /// whether the python the module targets has the `/` a repeated `_` needs slash: bool, @@ -89,6 +89,20 @@ pub(crate) fn collect(suite: &[Stmt], types: &dyn TypeInfo, slash: bool) -> Unde collector.lowerings } +/// ty's answer for every parameter list in `suite` that repeats `_`, for a native build of +/// the module `model` is of. that build runs on the interpreter it is compiled for, which +/// has the `/` +/// +/// the native build publishes the same definitions the transpiled module does, so it +/// reads the same answer: [`WrittenNames::with_lowerings`] these, and every name and `/` +/// it writes is the transpiler's +pub fn repeated_underscore_lowerings( + model: &ty_python_semantic::SemanticModel<'_>, + suite: &[Stmt], +) -> UnderscoreLowerings { + collect(suite, model, true) +} + /// how `parameters` is lowered read on its own, with no method it overrides and no receiver /// to consult. a lambda has neither, so this is the answer ty gives a lambda too fn standalone_lowering(parameters: &Parameters, slash: bool) -> Option { @@ -116,8 +130,8 @@ fn repeats_underscore(parameters: &Parameters) -> bool { /// return _2 # the module's `_2`, so the second parameter is `_3` /// ``` /// -/// without the lowerings ty decided — the native compiler reads a module on its own — -/// a repeated `_` is numbered and nothing takes a name from an overridden method +/// without the lowerings ty decided, a repeated `_` is numbered and nothing takes a name +/// from an overridden method #[derive(Clone, Copy, Debug)] pub struct WrittenNames<'src> { names: decision::WrittenNames<'src>, @@ -135,7 +149,7 @@ impl<'src> WrittenNames<'src> { /// these names, with `lowerings` saying how each parameter list that repeats `_` is /// lowered - pub(crate) fn with_lowerings<'a>(self, lowerings: &'a UnderscoreLowerings) -> WrittenNames<'a> + pub fn with_lowerings<'a>(self, lowerings: &'a UnderscoreLowerings) -> WrittenNames<'a> where 'src: 'a, { @@ -155,10 +169,15 @@ impl<'src> WrittenNames<'src> { self.names.fresh(stem) } + /// whether the module spells `name` anywhere + pub(crate) fn spells(self, name: &str) -> bool { + self.names.spells(name) + } + /// how `parameters` is lowered when it repeats `_`. without ty's answer, the list is /// read on its own, as a lambda's is: it takes no name from a method it overrides /// whether the python the module targets has the `/` a repeated `_` needs. without ty's - /// answers, as for a native build, it is assumed to + /// answers it is assumed to fn slash(self) -> bool { self.lowerings.is_none_or(|lowerings| lowerings.slash) } @@ -249,7 +268,7 @@ pub(crate) fn lowered_callable_parameters( } /// how many of the leading positional `parameters` the python takes by position alone -pub(crate) fn positional_only_count(parameters: &Parameters, written: WrittenNames) -> usize { +pub fn positional_only_count(parameters: &Parameters, written: WrittenNames) -> usize { match written.lowering(parameters) { Some(Ok(lowering)) => lowering.positional_only(), _ => parameters.posonlyargs.len(), @@ -257,8 +276,8 @@ pub(crate) fn positional_only_count(parameters: &Parameters, written: WrittenNam } /// the name a read of `_` in the body of a definition with `parameters` means, when the -/// lowering binds no parameter to `_` any more: the first `_` took a name from the method -/// the definition overrides, and a read of `_` is still the first `_`'s value +/// lowering binds `_` to anything but the first `_`: the first `_` took a name from the +/// method the definition overrides, and a read of `_` is still the first `_`'s value pub(crate) fn rebound_underscore(parameters: &Parameters, written: WrittenNames) -> Option { let Some(Ok(lowering)) = written.lowering(parameters) else { return None; @@ -271,6 +290,7 @@ pub(crate) fn rebound_underscore(parameters: &Parameters, written: WrittenNames) .zip(written.names(parameters)) .find(|(parameter, _)| parameter.name() == "_") .map(|(_, name)| name) + .filter(|name| name != "_") } /// whether `body` reads `_`, anywhere in it @@ -726,6 +746,63 @@ mod tests { ); } + /// a base that repeats `_` numbers its own, and the override takes those names. its + /// first is `_`, which the body's `_` already reads + #[test] + fn an_override_of_numbered_underscores_keeps_them() { + check( + indoc! {" + class A: + def f(self, _: int, _: int) -> int: + return _ + + class B(A): + override def f(self, _: int, _: int) -> int: + return _ + "}, + indoc! {" + from typing_extensions import override + class A: + def f(self, _: int, _2: int, /) -> int: + return _ + + class B(A): + @override + def f(self, _: int, _2: int, /) -> int: + return _ + "}, + ); + } + + /// a base parameter its author named `_` is reached by that keyword, so the `_` in its + /// position keeps the name + #[test] + fn a_base_parameter_named_underscore_keeps_it() { + check( + indoc! {" + class A: + def f(self, x: int, _: int) -> int: + return x + + class B(A): + override def f(self, _: int, _: int) -> int: + return _ + "}, + indoc! {" + from typing_extensions import override + class A: + def f(self, x: int, _: int) -> int: + return x + + class B(A): + @override + def f(self, x: int, _: int) -> int: + _ = x + return _ + "}, + ); + } + /// the name an override would take from its base is one its body reads from an /// enclosing scope, where the parameter would shadow it #[test] diff --git a/crates/by_transforms/src/transforms/runtime_union.rs b/crates/by_transforms/src/transforms/runtime_union.rs index 3a13037598..02385561ee 100644 --- a/crates/by_transforms/src/transforms/runtime_union.rs +++ b/crates/by_transforms/src/transforms/runtime_union.rs @@ -23,12 +23,20 @@ //! value, not a class, and only the union operator accepts it as shorthand for //! `NoneType`. //! +//! An optional `T?` the runtime evaluates is the union `T | None` too, and is +//! spelled by the same rule: `(T, type(None),)` where `isinstance` expects +//! classes, `Union[T, None]` everywhere else. The optional lowering writes it, and +//! asks [`classinfo_optionals`] which of its optionals stand in the classinfo +//! argument. +//! //! Whether a `|` is a union at all is asked of the checker rather than guessed //! from the shape: `a | b` is overwhelmingly a bitwise or, and only the types of //! its operands tell the two apart. +use std::collections::HashSet; + use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; -use ruff_python_ast::{Expr, ExprCall, Operator, PythonVersion, Stmt}; +use ruff_python_ast::{Expr, ExprCall, Operator, PythonVersion, Stmt, UnaryOp}; use ruff_text_size::{Ranged, TextRange}; use super::ast_driver::{Fragment, PassContext, TypeAwarePass}; @@ -48,6 +56,10 @@ impl RuntimeUnionPass { } impl TypeAwarePass for RuntimeUnionPass { + fn lowering(&self) -> Option { + Some(super::ast_driver::Lowering::RuntimeUnion) + } + // only a union the runtime evaluates needs the older spelling. a checker // reads `X | Y` in a stub whatever version the stub is for fn runtime_only(&self) -> bool { @@ -58,11 +70,7 @@ impl TypeAwarePass for RuntimeUnionPass { if self.min_version >= MIN_VERSION { return; } - let mut lower = Lower { - types, - edits: Vec::new(), - needs_import: false, - }; + let mut lower = Lower::new(types); for stmt in stmts { lower.visit_stmt(stmt); } @@ -74,10 +82,33 @@ impl TypeAwarePass for RuntimeUnionPass { } } +/// The optionals in `stmts` that stand where `isinstance` / `issubclass` expects +/// classes, and so are spelled as a tuple of classes below 3.10. +pub(crate) fn classinfo_optionals(stmts: &[Stmt], types: &dyn TypeInfo) -> HashSet { + let mut lower = Lower::new(types); + for stmt in stmts { + lower.visit_stmt(stmt); + } + lower.classinfo_optionals +} + struct Lower<'a> { types: &'a dyn TypeInfo, edits: Vec<(TextRange, Vec)>, needs_import: bool, + /// every optional the classinfo walk reached, for the optional lowering + classinfo_optionals: HashSet, +} + +impl<'a> Lower<'a> { + fn new(types: &'a dyn TypeInfo) -> Self { + Self { + types, + edits: Vec::new(), + needs_import: false, + classinfo_optionals: HashSet::new(), + } + } } impl<'ast> Visitor<'ast> for Lower<'_> { @@ -162,7 +193,8 @@ impl<'ast> Lower<'_> { /// Visit an expression standing where `isinstance` expects classes. A union /// here becomes a tuple, and so does one nested inside a tuple or list the - /// argument already spells; everything else is ordinary value context. + /// argument already spells, or inside an optional; everything else is + /// ordinary value context. fn visit_classinfo(&mut self, expr: &'ast Expr) { if let Some(arms) = self.union_arms(expr) { let fragments = spell(&arms, Form::ClassInfo); @@ -183,6 +215,12 @@ impl<'ast> Lower<'_> { self.visit_classinfo(element); } } + Expr::UnaryOp(optional) + if optional.op == UnaryOp::Optional && self.types.is_runtime_union(expr) => + { + self.classinfo_optionals.insert(expr.range()); + self.visit_classinfo(&optional.operand); + } _ => self.visit_expr(expr), } } @@ -291,6 +329,56 @@ mod tests { ); } + /// an optional is the union `T | None`, and where `isinstance` expects classes it + /// is spelled as that union is there + #[test] + fn an_optional_classinfo_is_a_tuple() { + let out = lowered(indoc! {" + def f(x: object, t: type): + return isinstance(x, int?) or issubclass(t, int?) + "}); + assert!( + out.contains("isinstance(x, (int, type(None),)) or issubclass(t, (int, type(None),))"), + "got:\n{out}" + ); + assert!(!out.contains("Union"), "got:\n{out}"); + } + + /// an optional inside a classinfo union or tuple is still a classinfo, and so is a + /// union an optional wraps + #[test] + fn a_nested_classinfo_optional_is_a_tuple_too() { + let out = lowered(indoc! {" + def f(x: object): + return isinstance(x, int? | str) or isinstance(x, (float?, bytes)) or isinstance(x, (int | str)?) + "}); + assert!( + out.contains("isinstance(x, ((int, type(None),), str,))"), + "got:\n{out}" + ); + assert!( + out.contains("isinstance(x, ((float, type(None),), bytes))"), + "got:\n{out}" + ); + assert!( + out.contains("isinstance(x, (((int, str,)), type(None),))"), + "got:\n{out}" + ); + assert!(!out.contains("Union"), "got:\n{out}"); + } + + /// an optional anywhere else keeps the `Union` spelling, a subscript inside the + /// classinfo argument among them + #[test] + fn an_optional_outside_a_classinfo_is_a_union() { + let out = lowered(indoc! {" + from typing import cast + def f(v: object): + return cast(int?, v) + "}); + assert!(out.contains("cast(Union[int, None], v)"), "got:\n{out}"); + } + /// a `cast` target is a type expression the runtime still evaluates #[test] fn a_cast_target_is_spelled_out() { diff --git a/crates/by_transforms/src/transforms/soundness.rs b/crates/by_transforms/src/transforms/soundness.rs index ca780537f6..a8f83f7346 100644 --- a/crates/by_transforms/src/transforms/soundness.rs +++ b/crates/by_transforms/src/transforms/soundness.rs @@ -80,7 +80,7 @@ use ty_python_semantic::types::soundness::CheckTarget; use super::ast_driver::{Fragment, PassContext, TypeAwarePass}; use super::parametric_is::variance_tuple; use super::repeated_underscore::WrittenNames; -use super::source_util::{PrologueStatement, body_prologue}; +use super::source_util::{PrologueStatement, body_prologue, needs_parentheses_as_argument}; use crate::Config; use crate::config::SoundnessPositions; use crate::type_info::{SoundnessCheck, TypeInfo}; @@ -544,24 +544,32 @@ struct Rendered { } impl Rendered { - /// wrap `source[range]` in `helper(, )`. `trailing` + /// wrap `expr` in `helper(, )`. `trailing` /// carries its own leading `, ` (e.g. `", str"` or `", A[int], (0,)"`) - fn wrap_call(&mut self, range: TextRange, helper: &str, trailing: &str) { - self.wraps - .push((range, format!("{helper}("), format!("{trailing})"))); + fn wrap_call(&mut self, expr: &Expr, helper: &str, trailing: &str) { + let (open, close) = if needs_parentheses_as_argument(expr) { + ("(", ")") + } else { + ("", "") + }; + self.wraps.push(( + expr.range(), + format!("{helper}({open}"), + format!("{close}{trailing})"), + )); } - /// wrap `range` in the scalar check (`_soundness_check` / + /// wrap `expr` in the scalar check (`_soundness_check` / /// `_soundness_parametric`) named by `plan` - fn wrap_check(&mut self, range: TextRange, plan: &SoundnessCheck) { + fn wrap_check(&mut self, expr: &Expr, plan: &SoundnessCheck) { match plan { SoundnessCheck::Isinstance(target) => { - self.wrap_call(range, "_soundness_check", &format!(", {target}")); + self.wrap_call(expr, "_soundness_check", &format!(", {target}")); } SoundnessCheck::Parametric { alias, variances } => { self.used_parametric = true; self.wrap_call( - range, + expr, "_soundness_parametric", &format!(", {alias}, {}", variance_tuple(variances)), ); @@ -595,7 +603,7 @@ impl Rendered { (helper, format!(", {alias}, {}", variance_tuple(variances))) } }; - self.wrap_call(iterable.range(), helper, &trailing); + self.wrap_call(iterable, helper, &trailing); } /// the guard statements that validate each of `checks`, in order @@ -683,7 +691,7 @@ impl TypeAwarePass for SoundnessPass<'_> { let mut checks = Vec::new(); for site in &walker.sites { match site { - Site::Value { expr, plan } => rendered.wrap_check(expr.range(), plan), + Site::Value { expr, plan } => rendered.wrap_check(expr, plan), Site::Elements { iterable, is_async, @@ -716,7 +724,7 @@ impl TypeAwarePass for SoundnessPass<'_> { } for site in &walker.sites { match site { - Site::Value { expr, plan } => inner.wrap_check(expr.range(), plan), + Site::Value { expr, plan } => inner.wrap_check(expr, plan), Site::Elements { iterable, is_async, @@ -1044,6 +1052,40 @@ mod tests { assert!(out.contains("def _soundness_iter"), "got:\n{out}"); } + /// a tuple written without parentheses is one iterable, and stays one inside the + /// check's call rather than becoming its arguments + #[test] + fn an_unparenthesized_tuple_is_checked_as_one_iterable() { + let out = check( + "def f(a: list[int], b: list[int]):\n for x in *a, *b:\n print(x)\n for y in a, b:\n print(y)\n", + ); + assert!( + out.contains("for x in _soundness_iter((*a, *b), int):"), + "got:\n{out}" + ); + assert!( + out.contains("for y in _soundness_iter((a, b), list):"), + "got:\n{out}" + ); + } + + /// a call takes a `yield` only parenthesized, and a `yield` the source parenthesized + /// has its parentheses outside the check + #[test] + fn a_yield_is_checked_parenthesized() { + let out = check( + "from typing import Any, Generator\ndef g() -> Generator[int, Any, None]:\n x: int = yield 1\n y: int = (yield 2)\n", + ); + assert!( + out.contains("x: int = _soundness_check((yield 1), int)"), + "got:\n{out}" + ); + assert!( + out.contains("y: int = (_soundness_check((yield 2), int))"), + "got:\n{out}" + ); + } + #[test] fn comprehension_iterable_checked() { let out = check("def f(a: list[int]):\n b = [x + 1 for x in a]\n"); diff --git a/crates/by_transforms/src/transforms/source_util.rs b/crates/by_transforms/src/transforms/source_util.rs index 341945a68a..6e3f36456a 100644 --- a/crates/by_transforms/src/transforms/source_util.rs +++ b/crates/by_transforms/src/transforms/source_util.rs @@ -12,6 +12,19 @@ use ruff_python_ast::{ use ruff_python_trivia::SimpleTokenizer; use ruff_text_size::{Ranged, TextRange, TextSize}; +/// Whether `expr`, written as it is, has to be parenthesized to stand as one argument +/// of a call among others: a tuple or generator expression written without its own +/// parentheses (`for x in a, b:`), or a `yield`, which a call does not accept bare +/// even where the source parenthesized it, since those parentheses are outside its range +pub(crate) fn needs_parentheses_as_argument(expr: &Expr) -> bool { + match expr { + Expr::Tuple(tuple) => !tuple.parenthesized, + Expr::Generator(generator) => !generator.parenthesized, + Expr::Yield(_) | Expr::YieldFrom(_) => true, + _ => false, + } +} + /// Names a temporary a lowering needs in its output. /// /// The name is a dunder because a lowering fires wherever its construct was diff --git a/crates/by_transforms/src/transforms/symbolic_type_op.rs b/crates/by_transforms/src/transforms/symbolic_type_op.rs index fb8cf0a76e..1f41529159 100644 --- a/crates/by_transforms/src/transforms/symbolic_type_op.rs +++ b/crates/by_transforms/src/transforms/symbolic_type_op.rs @@ -24,14 +24,17 @@ use std::cell::RefCell; use std::collections::HashMap; use ruff_python_ast::visitor::transformer::{Transformer, walk_expr}; +use ruff_python_ast::visitor::{self, Visitor}; use ruff_python_ast::{CmpOp, Expr, ModModule, Operator, Stmt, UnaryOp}; use ruff_python_parser::parse_expression; use ruff_text_size::{Ranged, TextRange}; use super::ast_driver::{AstPass, PassContext}; +use super::literal_types::float_literal_spelling; use super::type_expr_walker::{ Recurse, TypeExprVisitor, TypePos, walk_one_type_expr, walk_type_positions, }; +use crate::config::FloatLiteralLowering; use crate::type_info::TypeInfo; /// One resolved operation: the replacement node (spliced into the working AST) @@ -76,9 +79,14 @@ impl SymbolicFolds { /// Walk every type position and resolve each non-union/non-intersection binary /// operation to the type ty inferred for it, parsed into a replacement node. -pub(crate) fn collect_symbolic_folds(stmts: &[Stmt], types: &dyn TypeInfo) -> SymbolicFolds { +pub(crate) fn collect_symbolic_folds( + stmts: &[Stmt], + types: &dyn TypeInfo, + float_literals: FloatLiteralLowering, +) -> SymbolicFolds { let mut collector = FoldCollector { types, + float_literals, folds: HashMap::new(), needs_literal_import: false, needs_any_import: false, @@ -93,6 +101,7 @@ pub(crate) fn collect_symbolic_folds(stmts: &[Stmt], types: &dyn TypeInfo) -> Sy struct FoldCollector<'a> { types: &'a dyn TypeInfo, + float_literals: FloatLiteralLowering, folds: HashMap, needs_literal_import: bool, needs_any_import: bool, @@ -200,6 +209,16 @@ impl TypeExprVisitor for FoldCollector<'_> { ("Any".to_string(), parsed) } }; + // ty spells a float or complex literal type as the bare number, which python reads + // as no type at all. it is written the way the same literal written in the source is + let (rendered, parsed) = + match spell_float_literals(&rendered, parsed.expr(), self.float_literals) { + Some(spelled) => match parse_expression(&spelled) { + Ok(parsed) => (spelled, parsed), + Err(_) => return Recurse::Descend, + }, + None => (rendered, parsed), + }; if rendered.contains("Literal[") { self.needs_literal_import = true; } @@ -214,6 +233,54 @@ impl TypeExprVisitor for FoldCollector<'_> { } } +/// `rendered` with each float or complex literal type in `parsed` (its parse) spelled as +/// [`float_literal_spelling`] spells a written one, or `None` when it holds none +fn spell_float_literals( + rendered: &str, + parsed: &Expr, + float_literals: FloatLiteralLowering, +) -> Option { + struct Spellings<'a> { + rendered: &'a str, + float_literals: FloatLiteralLowering, + edits: Vec<(TextRange, String)>, + } + impl<'a> Visitor<'a> for Spellings<'_> { + fn visit_expr(&mut self, expr: &'a Expr) { + // what `Literal[...]` holds is already a python literal + if let Expr::Subscript(subscript) = expr + && matches!(subscript.value.as_ref(), Expr::Name(name) if name.id.as_str() == "Literal") + { + return; + } + let text = &self.rendered[expr.range()]; + if let Some(spelling) = float_literal_spelling(expr, text, self.float_literals) { + self.edits.push((expr.range(), spelling)); + return; + } + visitor::walk_expr(self, expr); + } + } + let mut spellings = Spellings { + rendered, + float_literals, + edits: Vec::new(), + }; + spellings.visit_expr(parsed); + if spellings.edits.is_empty() { + return None; + } + let mut out = String::new(); + let mut cursor = 0; + for (range, spelling) in spellings.edits { + out.push_str(&rendered[cursor..usize::from(range.start())]); + out.push_str(&spelling); + cursor = usize::from(range.end()); + } + out.push_str(&rendered[cursor..]); + Some(out) +} + pub(crate) struct SymbolicTypeOp { folds: HashMap, edits: RefCell>, @@ -309,6 +376,39 @@ mod tests { ); } + /// a fold to a float literal type is written as the same literal written in the source + /// is: `float` by default, since python has no spelling for the literal itself + #[test] + fn a_float_fold_is_written_as_a_written_float() { + check( + "a: 1.0 + 1.5\nb: 2.5\nc: list[0.5 * 3]\nd: 1 + 1 | 0.5 + 0.5\n", + indoc! {" + from typing import Literal + a: float + b: float + c: list[float] + d: Literal[2] | float + "}, + ); + } + + /// the project that keeps a float literal's precision keeps a folded one's too + #[test] + fn a_float_fold_kept_as_a_literal() { + let config = Config { + float_literals: crate::config::FloatLiteralLowering::Literal, + ..Config::test_default() + }; + assert_eq!( + transpile("a: 1.0 + 1.5\nb: -(1.0 + 1.5)\n", &config).unwrap(), + crate::python_passthrough::lazify_expected(indoc! {" + from typing import Literal + a: Literal[2.5] + b: Literal[-2.5] + "}) + ); + } + #[test] fn unary_operations() { // `~` and a multiply-negated literal are genuine unary operations that diff --git a/crates/by_transforms/src/transforms/type_expr_walker.rs b/crates/by_transforms/src/transforms/type_expr_walker.rs index fa92e88d41..adf7466f29 100644 --- a/crates/by_transforms/src/transforms/type_expr_walker.rs +++ b/crates/by_transforms/src/transforms/type_expr_walker.rs @@ -70,8 +70,24 @@ pub(crate) enum Recurse { Stop, } +/// whether the runtime evaluates a root type expression where it stands +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum RootKind { + /// a variable, parameter or return annotation, which a target without + /// `type.__or__` always defers through `from __future__ import annotations` + Annotation, + /// every other root: an alias value, a type parameter's bound or default (each + /// a call argument once polyfilled), a class base, a `cast` target, a + /// value-position type application, the type argument of a typing construct + Evaluated, +} + pub(crate) trait TypeExprVisitor { fn visit(&mut self, expr: &Expr, pos: TypePos) -> Recurse; + + /// told ahead of each root the walk visits what kind of root it is, which holds + /// for everything nested inside it + fn enter_root(&mut self, _kind: RootKind) {} } /// walk every type-position expression in `stmts`. `types` enables detection @@ -125,6 +141,11 @@ struct TypePosWalker<'a> { } impl TypePosWalker<'_> { + fn visit_root(&mut self, expr: &Expr, kind: RootKind) { + self.visitor.enter_root(kind); + self.visit_type_expr(expr, TypePos::Root); + } + fn visit_type_expr(&mut self, expr: &Expr, pos: TypePos) { // a claimed subtree has already been resolved by an earlier pass; don't // visit or descend into it @@ -241,7 +262,7 @@ impl TypePosWalker<'_> { fn visit_parameters(&mut self, params: &Parameters) { for p in params.iter_non_variadic_params() { if let Some(ann) = &p.parameter.annotation { - self.visit_type_expr(ann, TypePos::Root); + self.visit_root(ann, RootKind::Annotation); } if let Some(default) = &p.default { self.visit_expr(default); @@ -249,12 +270,12 @@ impl TypePosWalker<'_> { } if let Some(v) = ¶ms.vararg { if let Some(ann) = &v.annotation { - self.visit_type_expr(ann, TypePos::Root); + self.visit_root(ann, RootKind::Annotation); } } if let Some(k) = ¶ms.kwarg { if let Some(ann) = &k.annotation { - self.visit_type_expr(ann, TypePos::Root); + self.visit_root(ann, RootKind::Annotation); } } } @@ -268,24 +289,24 @@ impl TypePosWalker<'_> { match b.as_ref() { Expr::Tuple(t) if tv.is_type_mapping && t.parenthesized => { for member in &t.elts { - self.visit_type_expr(member, TypePos::Root); + self.visit_root(member, RootKind::Evaluated); } } - _ => self.visit_type_expr(b, TypePos::Root), + _ => self.visit_root(b, RootKind::Evaluated), } } if let Some(d) = &tv.default { - self.visit_type_expr(d, TypePos::Root); + self.visit_root(d, RootKind::Evaluated); } } TypeParam::TypeVarTuple(tvt) => { if let Some(d) = &tvt.default { - self.visit_type_expr(d, TypePos::Root); + self.visit_root(d, RootKind::Evaluated); } } TypeParam::ParamSpec(ps) => { if let Some(d) = &ps.default { - self.visit_type_expr(d, TypePos::Root); + self.visit_root(d, RootKind::Evaluated); } } } @@ -310,13 +331,13 @@ impl<'ast> Visitor<'ast> for TypePosWalker<'_> { // name being declared (`var a: str?` → `var [str | None]`) let annotation = declaration_annotation_type(&a.annotation).unwrap_or(&a.annotation); - self.visit_type_expr(annotation, TypePos::Root); + self.visit_root(annotation, RootKind::Annotation); if let Some(v) = &a.value { self.visit_expr(v); } } Stmt::TypeAlias(a) => { - self.visit_type_expr(&a.value, TypePos::Root); + self.visit_root(&a.value, RootKind::Evaluated); if let Some(tp) = &a.type_params { for p in &tp.type_params { self.visit_type_param(p); @@ -329,7 +350,7 @@ impl<'ast> Visitor<'ast> for TypePosWalker<'_> { if let Some(ret) = &f.returns && !f.is_asserts_return { - self.visit_type_expr(ret, TypePos::Root); + self.visit_root(ret, RootKind::Annotation); } if let Some(tp) = &f.type_params { for p in &tp.type_params { @@ -343,7 +364,7 @@ impl<'ast> Visitor<'ast> for TypePosWalker<'_> { Stmt::ClassDef(c) => { if let Some(arguments) = &c.arguments { for base in &arguments.args { - self.visit_type_expr(base, TypePos::Root); + self.visit_root(base, RootKind::Evaluated); } for kw in &arguments.keywords { // class kwargs (e.g. `metaclass=Meta`) are runtime @@ -368,7 +389,7 @@ impl<'ast> Visitor<'ast> for TypePosWalker<'_> { // value-position type application: `list[T]` outside annotation if let Expr::Subscript(s) = expr { if self.is_known_type_subscript(&s.value) { - self.visit_type_expr(expr, TypePos::Root); + self.visit_root(expr, RootKind::Evaluated); return; } } @@ -376,7 +397,7 @@ impl<'ast> Visitor<'ast> for TypePosWalker<'_> { if let Expr::Call(c) = expr { if is_cast_name(&c.func) { if let Some(t) = c.arguments.args.first() { - self.visit_type_expr(t, TypePos::Root); + self.visit_root(t, RootKind::Evaluated); } for arg in c.arguments.args.iter().skip(1) { self.visit_expr(arg); @@ -400,7 +421,7 @@ impl<'ast> Visitor<'ast> for TypePosWalker<'_> { .any(|argument| expr.range().contains_range(argument.range())) }; for argument in &type_arguments { - self.visit_type_expr(argument, TypePos::Root); + self.visit_root(argument, RootKind::Evaluated); } // everything else the call holds is an ordinary value for arg in &c.arguments.args { diff --git a/crates/by_transforms/src/transforms/typed_dict_literal.rs b/crates/by_transforms/src/transforms/typed_dict_literal.rs index c33dd8c795..a6dd5ba1c1 100644 --- a/crates/by_transforms/src/transforms/typed_dict_literal.rs +++ b/crates/by_transforms/src/transforms/typed_dict_literal.rs @@ -42,6 +42,10 @@ use ruff_text_size::Ranged; use super::type_expr_walker::{Recurse, TypeExprVisitor, TypePos, walk_type_positions}; use super::ast_driver::{AstPass, PassContext}; +use super::callable::CallableSyntax; +use super::repeated_underscore::WrittenNames; +use crate::config::Config; +use crate::type_info::TypeInfo; /// Sorted list of `(field_name, type_source)` pairs identifying a unique /// shape. Sorted so that `{"a": int, "b": str}` and `{"b": str, "a": int}` @@ -126,12 +130,22 @@ pub(crate) struct TypedDictLiteral<'src> { needs_literal_import: bool, /// set when a nested `T??` field needs the runtime `Optional[...]` wrapper needs_optional_runtime: bool, + /// the shared type-expression lowerer, which prints every field type this pass does + /// not take apart itself. the class is hoisted out of the statement the field type + /// was written in, so no other pass's edit reaches the field + callable: CallableSyntax<'src>, } impl<'src> TypedDictLiteral<'src> { - fn new(source: &'src str) -> Self { + fn new( + source: &'src str, + written: WrittenNames<'src>, + config: &Config, + types: &'src dyn TypeInfo, + ) -> Self { Self { source, + callable: CallableSyntax::new(source, written, config).with_types(types), edits: Vec::new(), errors: Vec::new(), shapes: IndexMap::new(), @@ -295,7 +309,7 @@ impl<'src> TypedDictLiteral<'src> { self.needs_literal_import = true; format!("Literal[{}]", self.render_subbed(expr.range())) } - _ => self.render_subbed(expr.range()), + _ => self.callable.lower_type_expr(expr), } } @@ -333,6 +347,7 @@ impl<'src> TypedDictLiteral<'src> { n }; self.range_to_class.push((source_range, name.clone())); + self.callable.add_substitution(source_range, name.clone()); name } @@ -403,17 +418,27 @@ impl TypeExprVisitor for TypedDictLiteral<'_> { /// `Literal[…]` in a type expression: int, str/bytes, bool, hex/neg-int, or /// `None` fn is_literal_value(expr: &Expr) -> bool { + // a float or complex is no `Literal`, and the shared lowerer prints it the way the + // project spells one matches!( expr, - Expr::NumberLiteral(_) - | Expr::StringLiteral(_) + Expr::NumberLiteral(ruff_python_ast::ExprNumberLiteral { + value: ruff_python_ast::Number::Int(_), + .. + }) | Expr::StringLiteral(_) | Expr::BytesLiteral(_) | Expr::BooleanLiteral(_) | Expr::NoneLiteral(_) ) || matches!( expr, Expr::UnaryOp(u) if matches!(u.op, ruff_python_ast::UnaryOp::USub) - && matches!(u.operand.as_ref(), Expr::NumberLiteral(_)) + && matches!( + u.operand.as_ref(), + Expr::NumberLiteral(ruff_python_ast::ExprNumberLiteral { + value: ruff_python_ast::Number::Int(_), + .. + }) + ) ) } @@ -449,15 +474,25 @@ fn flatten_bitand(expr: &Expr) -> Vec<&Expr> { impl AstPass for TypedDictLiteralPass<'_> { /// a typed dict type it replaces moves into a class, whose field types the shared - /// type-expression lowerer prints + /// type-expression lowerer prints, a `typeof` among them fn subsumes(&self) -> &'static [super::ast_driver::Lowering] { - super::callable::TYPE_EXPRESSION - } - - fn run(&self, module: &mut ModModule, ctx: &mut PassContext) { - let mut inner = TypedDictLiteral::new(self.source); - let body: &[Stmt] = &module.body; - walk_type_positions(body, None, &mut inner); + &[ + super::ast_driver::Lowering::Callable, + super::ast_driver::Lowering::OptionalType, + super::ast_driver::Lowering::LiteralType, + super::ast_driver::Lowering::JustFloat, + super::ast_driver::Lowering::DynamicKeyword, + super::ast_driver::Lowering::FloatConst, + super::ast_driver::Lowering::Typeof, + ] + } + + fn run(&self, _module: &mut ModModule, ctx: &mut PassContext) { + let mut inner = TypedDictLiteral::new(self.source, self.written, &self.config, self.types); + // the module as parsed, not the one the passes before this rewrote: the type + // information the field lowerer asks is about those nodes, and the edits are keyed + // on the source ranges both share + walk_type_positions(self.suite, None, &mut inner); if inner.needs_literal_import { ctx.required_imports .push("from typing import Literal".to_owned()); @@ -476,12 +511,16 @@ impl AstPass for TypedDictLiteralPass<'_> { // them into required_imports as a single non-`from` line so // merge_from_imports leaves them untouched and they end up in // the preamble verbatim - let defs = inner.class_defs(); + // the field lowerer's own classes come first: a field type may name one + let defs = format!("{}{}", inner.callable.class_defs(), inner.class_defs()); if !defs.is_empty() { ctx.required_imports .push(defs.trim_end_matches('\n').to_owned()); } } + let (imports, helpers) = inner.callable.take_requirements(); + ctx.required_imports.extend(imports); + ctx.runtime.extend(helpers); for fix in inner.edits { for edit in fix.edits() { let range = edit.range(); @@ -498,11 +537,28 @@ impl AstPass for TypedDictLiteralPass<'_> { /// import is emitted alongside the synthesized class definitions pub(crate) struct TypedDictLiteralPass<'src> { source: &'src str, + written: WrittenNames<'src>, + config: Config, + /// the module as parsed, which is what `types` answers for + suite: &'src [Stmt], + types: &'src dyn TypeInfo, } impl<'src> TypedDictLiteralPass<'src> { - pub(crate) fn new(source: &'src str) -> Self { - Self { source } + pub(crate) fn new( + source: &'src str, + written: WrittenNames<'src>, + config: Config, + suite: &'src [Stmt], + types: &'src dyn TypeInfo, + ) -> Self { + Self { + source, + written, + config, + suite, + types, + } } } @@ -810,4 +866,28 @@ mod tests { let err = result.unwrap_err(); assert!(err.contains("must be a string literal"), "got: {err}"); } + + /// the class is hoisted out of the annotation, so no other pass's edit reaches a field + /// type: each is printed lowered here, where a field read back by `get_type_hints` + /// has to name a real type + #[test] + fn field_types_are_printed_lowered() { + let out = transpile( + indoc! {" + y = 1 + a: {\"f\": float, \"d\": dynamic, \"c\": 1.5, \"k\": (int) -> str, \"t\": typeof y} + "}, + &Config::test_default(), + ) + .unwrap(); + for field in [ + "f: \"JustFloat\"", + "d: \"Any\"", + "c: \"float\"", + "k: \"Callable[[int], str]\"", + "t: \"TypeOf[y]\"", + ] { + assert!(out.contains(field), "`{field}` is missing from:\n{out}"); + } + } } diff --git a/crates/by_transforms/src/transforms/typing_redirect.rs b/crates/by_transforms/src/transforms/typing_redirect.rs index eac2d6811c..563fd4bd7b 100644 --- a/crates/by_transforms/src/transforms/typing_redirect.rs +++ b/crates/by_transforms/src/transforms/typing_redirect.rs @@ -156,6 +156,33 @@ mod tests { ); } + /// a target without 3.10's `typing` names imports them from `typing_extensions`, the + /// `ParamSpec` a polyfilled `**P` declares among them + #[test] + fn redirects_310_names_below_it() { + let config = Config { + min_version: ruff_python_ast::PythonVersion::PY39, + ..Config::test_default() + }; + assert_eq!( + transpile( + "from typing import Optional, ParamSpec, Concatenate, TypeAlias, TypeGuard\n", + &config + ) + .unwrap(), + indoc! {" + from __future__ import annotations + from typing import Optional + from typing_extensions import ParamSpec, Concatenate, TypeAlias, TypeGuard + "} + ); + let out = transpile("class Deco[**P]:\n pass\n", &config).unwrap(); + assert!( + out.contains("from typing_extensions import ParamSpec\n"), + "got:\n{out}" + ); + } + #[test] fn redirects_312_override() { check( diff --git a/crates/by_transforms/src/transforms/unpack.rs b/crates/by_transforms/src/transforms/unpack.rs index b3bcbf5384..61be3afb9a 100644 --- a/crates/by_transforms/src/transforms/unpack.rs +++ b/crates/by_transforms/src/transforms/unpack.rs @@ -5,12 +5,17 @@ //! `tuple[*Ts]` → `tuple[Unpack[Ts]]` //! `class Stack(Generic[*Ts]):` → `class Stack(Generic[Unpack[Ts]]):` //! +//! A starred subscript that is not a type is the tuple python 3.11 reads it as: +//! +//! `d[*a]` → `d[(*a,)]` +//! //! Also lowers basedpython's pack forwarding, which no python version accepts: //! //! `def f(**kwargs: **Kwargs)` → `def f(**kwargs: Kwargs.kwargs)` //! `def f(*args: *P, **kwargs: **P)` → `def f(*args: P.args, **kwargs: P.kwargs)` use std::cell::RefCell; +use std::collections::HashSet; use ruff_python_ast::PythonVersion; use ruff_python_ast::helpers::top_star_slice_elements; @@ -19,15 +24,42 @@ use ruff_python_ast::{Expr, ModModule, Stmt}; use ruff_text_size::{Ranged, TextRange, TextSize}; use super::ast_driver::{AstPass, PassContext}; +use super::type_expr_walker::{Recurse, TypeExprVisitor, TypePos, walk_type_positions}; use crate::config::Config; +use crate::type_info::TypeInfo; pub(crate) struct UnpackSyntax { config: Config, + type_subscripts: TypeSubscripts, } impl UnpackSyntax { - pub(crate) fn new(config: Config) -> Self { - Self { config } + pub(crate) fn new(config: Config, type_subscripts: TypeSubscripts) -> Self { + Self { + config, + type_subscripts, + } + } +} + +/// the ranges of the subscripts that are type expressions — the ones whose starred elements +/// are spelled `Unpack[...]` rather than as the tuple python reads a starred subscript as +#[derive(Default)] +pub(crate) struct TypeSubscripts(HashSet); + +/// which subscripts in `stmts` are type expressions, as the type checker reads them +pub(crate) fn collect_type_subscripts(stmts: &[Stmt], types: &dyn TypeInfo) -> TypeSubscripts { + let mut subscripts = TypeSubscripts::default(); + walk_type_positions(stmts, Some(types), &mut subscripts); + subscripts +} + +impl TypeExprVisitor for TypeSubscripts { + fn visit(&mut self, expr: &Expr, _pos: TypePos) -> Recurse { + if let Expr::Subscript(subscript) = expr { + self.0.insert(subscript.range()); + } + Recurse::Descend } } @@ -56,6 +88,7 @@ impl AstPass for UnpackSyntax { edits: RefCell::new(Vec::new()), needs_import: false, lowered_varargs: pack.lowered_varargs, + type_subscripts: &self.type_subscripts, }; for stmt in &module.body { state.visit_stmt(stmt); @@ -144,13 +177,25 @@ fn star_token_range(unpack: TextRange) -> TextRange { TextRange::at(unpack.start(), TextSize::from(1)) } -struct State { +struct State<'a> { edits: RefCell>, needs_import: bool, lowered_varargs: Vec, + type_subscripts: &'a TypeSubscripts, } -impl State { +impl State<'_> { + /// `d[*a]` → `d[(*a,)]` and `d[*a, b]` → `d[(*a, b)]`: the tuple python 3.11 builds + /// for a starred subscript, which earlier versions only accept parenthesized + fn parenthesize_value_slice(&self, slice: &ruff_python_ast::ExprTuple) { + let mut edits = self.edits.borrow_mut(); + edits.push((TextRange::empty(slice.start()), "(".to_owned())); + // a tuple of one element needs a comma, which its range takes in when written + let needs_comma = matches!(&*slice.elts, [only] if only.end() == slice.end()); + let closing = if needs_comma { ",)" } else { ")" }; + edits.push((TextRange::empty(slice.end()), closing.to_owned())); + } + fn rewrite_subscript_starred(&mut self, starred: &ruff_python_ast::ExprStarred) { self.needs_import = true; let star_range = star_token_range(starred.range()); @@ -183,7 +228,7 @@ impl State { } } -impl<'ast> Visitor<'ast> for State { +impl<'ast> Visitor<'ast> for State<'_> { fn visit_stmt(&mut self, stmt: &'ast Stmt) { if let Stmt::FunctionDef(f) = stmt && let Some(vararg) = &f.parameters.vararg @@ -200,7 +245,16 @@ impl<'ast> Visitor<'ast> for State { walk_expr(self, expr); return; } + let is_type = self.type_subscripts.0.contains(&s.range()); match s.slice.as_ref() { + Expr::Tuple(t) + if !is_type + && !t.parenthesized + && !t.has_parameter_shape() + && t.elts.iter().any(Expr::is_starred_expr) => + { + self.parenthesize_value_slice(t); + } Expr::Starred(st) => self.rewrite_subscript_starred(st), Expr::Tuple(t) if !t.has_parameter_shape() => { for elt in &t.elts { @@ -334,6 +388,50 @@ mod tests { ); } + /// a starred subscript of a value is the tuple of what it unpacks, which python before + /// 3.11 spells with parentheses. `Unpack[a]` there would look `a` up as a type + #[test] + fn a_value_subscript_is_the_tuple_python_reads() { + check( + indoc! {" + d = {(1, 2): \"x\", (1, 2, 3): \"y\"} + a = (1, 2) + print(d[*a], d[*a,], d[*a, 3], d[*a, 3,]) + "}, + indoc! {" + d = {(1, 2): \"x\", (1, 2, 3): \"y\"} + a = (1, 2) + print(d[(*a,)], d[(*a,)], d[(*a, 3)], d[(*a, 3,)]) + "}, + ); + } + + /// whether a subscript is a type is the type checker's answer, so a generic type + /// applied outside an annotation is still an unpack, and a value in the metadata of + /// an annotation is still a tuple + #[test] + fn a_type_subscript_outside_an_annotation_is_an_unpack() { + check( + indoc! {" + from typing import Annotated, TypeVarTuple + Ts = TypeVarTuple(\"Ts\") + d = {(1,): 1} + a = (1,) + Alias = tuple[int, *Ts] + x: Annotated[tuple[*Ts], d[*a]] + "}, + indoc! {" + from typing import Annotated + from typing_extensions import TypeVarTuple, Unpack + Ts = TypeVarTuple(\"Ts\") + d = {(1,): 1} + a = (1,) + Alias = tuple[int, Unpack[Ts]] + x: Annotated[tuple[Unpack[Ts]], d[(*a,)]] + "}, + ); + } + #[test] fn regular_arg_annotation_unchanged() { check("def f(x: int): ...\n", "def f(x: int): ...\n"); diff --git a/crates/by_transforms/tests/polyfill_runtime.rs b/crates/by_transforms/tests/polyfill_runtime.rs index ec36992495..3283944f25 100644 --- a/crates/by_transforms/tests/polyfill_runtime.rs +++ b/crates/by_transforms/tests/polyfill_runtime.rs @@ -16,13 +16,10 @@ //! Asserting on the lowered *text* is what let all three through, so these //! tests execute it instead. -use std::process::Command; +use std::process::{Command, Stdio}; use by_transforms::{Config, PythonVersion, transpile}; -mod common; -use common::python; - /// exercises the polyfill's typevar rename across every position that renders /// replacement text rather than patching source bytes const RENAMES: &str = r#" @@ -104,23 +101,308 @@ assert hints["r"].__bound__ == hints["t"], "the bound resolves to the earlier pa print("ok") "#; -/// The polyfilled output imports `typing_extensions` for `TypeAliasType`, so an -/// interpreter without it can only run the rename half. -fn python_with_typing_extensions(python: &str) -> bool { - Command::new(python) - .args(["-c", "import typing_extensions"]) - .status() - .map(|s| s.success()) - .unwrap_or(false) +/// `some T` declares a type parameter named after its parameter, which python writes nowhere +/// in a type-parameter list, so it is a `TypeVar` at every version. the signature reads the +/// hole under the parameter's name and the body reads the parameter under it +const SOME_PARAMETERS: &str = r#" +def inc(n: some int) -> int: + return n + 1 + +def echo(n: some int) -> n: + return n + +def pair[T](x: T, n: some int) -> n: + def inner() -> None: + assert n == 3, "the body reads the parameter, not the type parameter" + inner() + return n + +assert inc(1) == 2, "the parameter list survives" +assert echo(2) == 2, "the return type names the hole" +assert pair("a", 3) == 3, "a hole beside a written type parameter" + +import typing +hints = typing.get_type_hints(echo) +assert hints["return"] == hints["n"], "the return type is the parameter's type parameter" + +print("ok") +"#; + +/// every place a type is evaluated where it is written, holding a union. `A | B` calls +/// `type.__or__`, which python only grew in 3.10, so below it each has to be spelled +/// `Union[A, B]` — whether the author wrote the union with `|`, `or` or `?`, and at any +/// depth in the type +const EVALUATED_UNIONS: &str = r#" +import typing +from typing import cast + +Optionals = list[int?] +Values = dict[str, int | None] +Either = list[int or str] +Pairs = list[(int, str | None)] +Handlers = list[(int | None) -> str] +Bound = typing.TypeVar("Bound", bound=int | str) + +class Items(list[int | str]): + pass + +def bounded[T: int | str](t: T) -> T: + return t + +class Holder[T: list[int?]]: + pass + +def annotated(a: list[int?], b: int or str) -> int?: + return None + +assert Optionals == list[typing.Optional[int]], "an optional inside a value subscript" +assert Values == dict[str, typing.Optional[int]], "a union inside a value subscript" +assert Either == list[typing.Union[int, str]], "a keyword union" +assert Pairs == list[tuple[int, typing.Optional[str]]], "a union in a tuple type" +assert Handlers == list[typing.Callable[[typing.Optional[int]], str]], "a union in an arrow" +assert Bound.__bound__ == typing.Union[int, str], "a typing construct's type argument" +assert Items.__orig_bases__ == (list[typing.Union[int, str]],), "a class base" +assert cast(int | str, 1) == 1 and cast(list[int?], [2]) == [2], "a cast target" +assert bounded(3) == 3, "a polyfilled bound" +assert Holder[list[int]]() is not None, "a polyfilled class bound" +hints = typing.get_type_hints(annotated) +assert hints["a"] == list[typing.Optional[int]], "an optional the lowering spelled" +assert hints["b"] == typing.Union[int, str], "a keyword union the lowering spelled" +assert isinstance(None, int?) and not isinstance("a", int?), "an optional as a classinfo" +assert isinstance(None, int? | str) and isinstance(None, (bytes, float?)), "nested in a classinfo" +assert isinstance("a", (int | str)?) and issubclass(bool, int?), "around a union, and in issubclass" + +print("ok") +"#; + +/// the evaluated unions that need `typing_extensions` below 3.13: an alias is a +/// `TypeAliasType`, and a type parameter with a default a `TypeVar` that takes one +const EVALUATED_UNIONS_EXTENSIONS: &str = r#" +import typing + +type Alias = list[int?] | str + +def defaulted[T = int | None]() -> None: + pass + +assert Alias.__value__ == typing.Union[list[typing.Optional[int]], str], "an alias value" +print("ok") +"#; + +/// a module that binds the names of the constructors the polyfill calls. its own +/// `from typing import TypeVar` takes no `default=` below 3.13, and its `Generic` is the +/// module's to bind +const OWN_CONSTRUCTORS: &str = r#" +from typing import TypeVar + +U = TypeVar("U") + +class Generic: + pass + +def defaulted[T = int](t: T) -> T: + return t + +class Box[T = str]: + pass + +class Deco[**P]: + pass + +assert defaulted(1) == 1, "a defaulted function" +assert Box.__parameters__[0].__default__ == str, "a defaulted class" +assert len(Deco.__parameters__) == 1, "a parameter specification" +assert U.__name__ == "U" and Generic.__module__ == "__main__", "the module's own names" +print("ok") +"#; + +/// a module that binds the names of the type variables the polyfill declares its type +/// parameters as. each declaration is written into the scope the generic stands in, so one +/// under a name the module binds would overwrite it +const OWN_TYPE_VARIABLES: &str = r#" +import typing +from typing import TypeVar + +_T = TypeVar("_T", bound=int) + +def legacy(x: _T) -> _T: + return x + +def modern[T](x: T) -> T: + return x + +class Holder: + _U = "a class attribute" + + def method[U](self, x: U) -> U: + return x + +if len("a") == 2: + def skipped[S](x: S) -> S: + return x + +def later[S](x: S) -> S: + return x + +assert _T.__bound__ === int, "the module's own type variable keeps its bound" +assert typing.get_type_hints(legacy)["x"] === _T, "and a function declared over it reads it" +assert typing.get_type_hints(modern)["x"] !== _T, "a type parameter is a type variable of its own" +assert Holder._U == "a class attribute", "a class body keeps its own name" +assert typing.get_type_hints(later)["x"].__bound__ === None, "a declaration in an `if` not taken" +print("ok") +"#; + +/// a default on a parameter specification or a variadic. python rejects a parameter without +/// a default after one with a default, so dropping either default fails the class at import +const DEFAULTED_PACKS: &str = r#" +class Box[T = str, **P = [int]]: + pass + +class Row[T = str, *Ts = *tuple[int, str]]: + pass + +class Shaped[T = str, P: (*: *, **: *) = (int, bytes)]: + pass + +def call[T = int, **P = [str]](x: T) -> T: + return x + +def defaults(cls: type) -> list[object]: + return [parameter.__default__ for parameter in getattr(cls, "__parameters__")] + +assert defaults(Box) == [str, [int]], "a parameter specification's default" +assert repr(defaults(Row)[1]) in ("typing_extensions.Unpack[tuple[int, str]]", "*tuple[int, str]"), "a variadic's" +assert defaults(Shaped) == [str, [int, bytes]], "a parameter list, as a list" +assert call(1) == 1, "a defaulted function" +print("ok") +"#; + +/// an interpreter found under one of the names the tests look for +struct Interpreter { + command: String, + version: PythonVersion, + typing_extensions: bool, +} + +impl std::fmt::Display for Interpreter { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{} ({})", self.command, self.version)?; + if self.typing_extensions { + write!(f, " with `typing_extensions`")?; + } + Ok(()) + } +} + +/// what an interpreter says about itself: its version, whether it has `typing_extensions`, and +/// the file it runs from, which tells two names for one interpreter apart +const PROBE: &str = "import sys, importlib.util; \ +print(sys.version_info[0], sys.version_info[1], \ +int(importlib.util.find_spec('typing_extensions') is not None), sys.executable)"; + +/// every interpreter this machine has under `$PYTHON`, `python3.N` or `python3`, oldest first, +/// each listed once however many of the names reach it +/// +/// the polyfill is only told apart from native syntax by an old interpreter, and its output +/// imports `typing_extensions` below 3.13, which an interpreter often does not have. `python3` +/// alone is whichever one the path finds first, so every name is asked +fn interpreters() -> Vec { + let mut candidates: Vec = std::env::var("PYTHON").into_iter().collect(); + candidates.extend((8..=15).map(|minor| format!("python3.{minor}"))); + candidates.push("python3".to_owned()); + + let mut seen = std::collections::HashSet::new(); + let mut found: Vec = candidates + .into_iter() + .filter_map(|command| { + let output = Command::new(&command) + .args(["-c", PROBE]) + .stderr(Stdio::null()) + .output() + .ok() + .filter(|output| output.status.success())?; + let stdout = String::from_utf8(output.stdout).ok()?; + let mut fields = stdout.trim_end().splitn(4, ' '); + let major = fields.next()?.parse().ok()?; + let minor = fields.next()?.parse().ok()?; + let typing_extensions = fields.next()? == "1"; + let executable = fields.next()?; + let executable = std::fs::canonicalize(executable) + .unwrap_or_else(|_| std::path::PathBuf::from(executable)); + seen.insert(executable).then(|| Interpreter { + command, + version: PythonVersion::from((major, minor)), + typing_extensions, + }) + }) + .collect(); + found.sort_by_key(|interpreter| interpreter.version); + found +} + +/// run `program`, transpiled for `target`, on the oldest interpreter found that runs what it is +/// transpiled for, and say which one ran it. `extensions` asks for one with +/// `typing_extensions`. none is a skip, said so +/// +/// a skip passes, so nothing on the test's line says it ran nothing: the nextest configuration +/// shows what this binary's passing tests print +#[expect( + clippy::print_stderr, + reason = "a skipped test must say why it skipped, or it reads as a pass" +)] +fn run_on_oldest(program: &str, target: PythonVersion, extensions: bool) { + let found = interpreters(); + let Some(interpreter) = found.iter().find(|interpreter| { + interpreter.version >= target && (interpreter.typing_extensions || !extensions) + }) else { + let needs = if extensions { + " with `typing_extensions`" + } else { + "" + }; + eprintln!("skipping: no interpreter of python {target} or later{needs} found"); + return; + }; + run_at(interpreter, program, target); +} + +/// the target the polyfill lowers for on `interpreter`: 3.10, the one a user gets without +/// asking, or the interpreter's own version when that is older +fn polyfill_target(interpreter: &Interpreter) -> PythonVersion { + interpreter.version.min(PythonVersion::PY310) } -fn run_polyfilled(python: &str, program: &str) { +/// run `program`, lowered by the polyfill, on the oldest interpreter found with +/// `typing_extensions`, which the polyfill's output imports below 3.13 +#[expect( + clippy::print_stderr, + reason = "a skipped test must say why it skipped, or it reads as a pass" +)] +fn run_polyfilled_with_extensions(program: &str) { + let found = interpreters(); + let Some(interpreter) = found.iter().find(|interpreter| { + interpreter.typing_extensions && interpreter.version >= PythonVersion::PY39 + }) else { + eprintln!( + "skipping: no interpreter of python 3.9 or later with `typing_extensions` found, \ + so the polyfill's output was not run" + ); + return; + }; + run_at(interpreter, program, polyfill_target(interpreter)); +} + +#[expect( + clippy::print_stderr, + reason = "a test that can skip says what it ran on, so a pass is told apart from a skip" +)] +fn run_at(interpreter: &Interpreter, program: &str, min_version: PythonVersion) { let config = Config { - min_version: PythonVersion::PY310, + min_version, ..Config::default() }; let transpiled = transpile(program, &config).expect("transpile should succeed"); - let output = Command::new(python) + let output = Command::new(&interpreter.command) .arg("-c") .arg(&transpiled) .output() @@ -128,58 +410,100 @@ fn run_polyfilled(python: &str, program: &str) { assert!( output.status.success(), - "polyfilled program failed on {python}:\n--- stdout ---\n{}\n--- stderr ---\n{}\n--- transpiled ---\n{transpiled}", + "program transpiled for {min_version} failed on {interpreter}:\n--- stdout ---\n{}\n--- stderr ---\n{}\n--- transpiled ---\n{transpiled}", String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr), ); assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); + eprintln!("ran, transpiled for {min_version}, on {interpreter}"); } #[test] -#[expect( - clippy::print_stderr, - reason = "a skipped test must say why it skipped, or it reads as a pass" -)] fn polyfilled_type_parameters_rename_everywhere() { - let Some(python) = python() else { - eprintln!("skipping polyfill runtime test: no `python3` interpreter found"); - return; - }; - run_polyfilled(&python, RENAMES); + run_on_oldest(RENAMES, PythonVersion::PY310, false); +} + +#[test] +fn polyfilled_aliases_and_bounds_run() { + run_polyfilled_with_extensions(ALIASES); } +#[test] +fn polyfilled_dependent_bounds_run() { + run_polyfilled_with_extensions(DEPENDENT_BOUNDS); +} + +#[test] +fn some_parameters_run_at_every_version() { + run_on_oldest(SOME_PARAMETERS, PythonVersion::PY310, false); + run_on_oldest(SOME_PARAMETERS, PythonVersion::PY313, false); +} + +/// an interpreter without `type.__or__` is the only one that can tell a union spelled for +/// it from one that is not, so each one found runs the unions transpiled for 3.9 #[test] #[expect( clippy::print_stderr, reason = "a skipped test must say why it skipped, or it reads as a pass" )] -fn polyfilled_aliases_and_bounds_run() { - let Some(python) = python() else { - eprintln!("skipping polyfill runtime test: no `python3` interpreter found"); - return; - }; - if !python_with_typing_extensions(&python) { - eprintln!("skipping polyfill alias runtime test: {python} has no `typing_extensions`"); - return; +fn evaluated_unions_run_below_python_310() { + let found = interpreters(); + let older: Vec<&Interpreter> = found + .iter() + .filter(|interpreter| { + (PythonVersion::PY39..PythonVersion::PY310).contains(&interpreter.version) + }) + .collect(); + if older.is_empty() { + eprintln!("skipping: no interpreter of python 3.9 found"); + } + for interpreter in older { + run_at(interpreter, EVALUATED_UNIONS, PythonVersion::PY39); + if interpreter.typing_extensions { + run_at( + interpreter, + EVALUATED_UNIONS_EXTENSIONS, + PythonVersion::PY39, + ); + } else { + eprintln!( + "skipping the aliases and defaults: {interpreter} has no `typing_extensions`" + ); + } } - run_polyfilled(&python, ALIASES); } +/// the `TypeVar` a default needs is `typing_extensions`' below 3.13, so every interpreter older +/// than that is one that can tell it from the module's own #[test] #[expect( clippy::print_stderr, reason = "a skipped test must say why it skipped, or it reads as a pass" )] -fn polyfilled_dependent_bounds_run() { - let Some(python) = python() else { - eprintln!("skipping polyfill runtime test: no `python3` interpreter found"); - return; - }; - if !python_with_typing_extensions(&python) { - eprintln!( - "skipping polyfill dependent-bound runtime test: {python} has no `typing_extensions`" - ); - return; +fn a_module_binding_the_constructors_runs() { + let found = interpreters(); + let older: Vec<&Interpreter> = found + .iter() + .filter(|interpreter| { + interpreter.typing_extensions + && (PythonVersion::PY39..PythonVersion::PY313).contains(&interpreter.version) + }) + .collect(); + if older.is_empty() { + eprintln!("skipping: no interpreter of python 3.9 to 3.12 with `typing_extensions` found"); } - run_polyfilled(&python, DEPENDENT_BOUNDS); + for interpreter in older { + run_at(interpreter, OWN_CONSTRUCTORS, PythonVersion::PY39); + } +} + +#[test] +fn a_module_binding_the_type_variables_runs() { + run_on_oldest(OWN_TYPE_VARIABLES, PythonVersion::PY39, false); +} + +#[test] +fn defaulted_parameter_specifications_and_variadics_run() { + run_polyfilled_with_extensions(DEFAULTED_PACKS); + run_on_oldest(DEFAULTED_PACKS, PythonVersion::PY313, false); } diff --git a/crates/ruff_python_parser/src/error.rs b/crates/ruff_python_parser/src/error.rs index 91a33e91ff..f8342fb865 100644 --- a/crates/ruff_python_parser/src/error.rs +++ b/crates/ruff_python_parser/src/error.rs @@ -993,6 +993,18 @@ pub enum UnsupportedSyntaxErrorKind { /// case +1: ... # error before 3.15 /// ``` UnaryPlusMatchPattern, + + /// basedpython: a destructuring pattern — `let P := v`, an `if let` / `elif let` + /// clause, or a pattern written as a `for` or `with` target or as a parameter. none has + /// a python spelling of its own, and each is written as a `match` statement, so it + /// needs a python that runs one + /// + /// ## Examples + /// + /// ```by + /// let Point(x, y) := origin # error before 3.8, where no `match` can be written + /// ``` + Destructuring, } impl Display for UnsupportedSyntaxError { @@ -1093,6 +1105,9 @@ impl Display for UnsupportedSyntaxError { UnsupportedSyntaxErrorKind::UnaryPlusMatchPattern => { "Unary '+' is not allowed in a literal pattern" } + UnsupportedSyntaxErrorKind::Destructuring => { + "Cannot use a destructuring pattern, which is written as a `match` statement," + } }; write!( @@ -1171,6 +1186,51 @@ impl UnsupportedSyntaxErrorKind { UnsupportedSyntaxErrorKind::UnaryPlusMatchPattern => { Change::Added(PythonVersion::PY315) } + UnsupportedSyntaxErrorKind::Destructuring => { + UnsupportedSyntaxErrorKind::Match.changed_version() + } + } + } + + /// basedpython: whether the lowering rewrites this syntax into python that `target_version` + /// parses, so a `.by` file may write it whatever python it targets + /// + /// the rest is written into the python as it stands, so it is checked against the target as + /// in a `.py` file. the transpiler re-parses its output at the target and refuses anything + /// left that the target cannot parse, so a kind wrongly listed here is still refused by the + /// transpile, just no longer reported by a checker first + pub fn is_lowered_by_basedpython(self, target_version: PythonVersion) -> bool { + match self { + // pep 695 and pep 696 lower to `TypeVar` declarations, pep 646 to `Unpack`, and a + // `lazy` import to a module the first attribute read imports + UnsupportedSyntaxErrorKind::TypeParameterList + | UnsupportedSyntaxErrorKind::TypeAliasStatement + | UnsupportedSyntaxErrorKind::TypeParamDefault + | UnsupportedSyntaxErrorKind::StarAnnotation + | UnsupportedSyntaxErrorKind::StarExpressionInIndex + | UnsupportedSyntaxErrorKind::LazyImportStatement => true, + // the `match` lowering writes each pattern as a test built of assignment + // expressions, in which a unary `+` is an ordinary operator. a destructuring is + // written as a `match`, so it goes wherever one goes + UnsupportedSyntaxErrorKind::Match + | UnsupportedSyntaxErrorKind::UnaryPlusMatchPattern + | UnsupportedSyntaxErrorKind::Destructuring => { + UnsupportedSyntaxErrorKind::Match.is_unsupported(target_version) + && UnsupportedSyntaxErrorKind::Walrus.is_supported(target_version) + } + UnsupportedSyntaxErrorKind::Walrus + | UnsupportedSyntaxErrorKind::ExceptStar + | UnsupportedSyntaxErrorKind::UnparenthesizedNamedExpr(_) + | UnsupportedSyntaxErrorKind::ParenthesizedKeywordArgumentName + | UnsupportedSyntaxErrorKind::StarTuple(_) + | UnsupportedSyntaxErrorKind::RelaxedDecorator(_) + | UnsupportedSyntaxErrorKind::PositionalOnlyParameter + | UnsupportedSyntaxErrorKind::Pep701FString(_) + | UnsupportedSyntaxErrorKind::ParenthesizedContextManager + | UnsupportedSyntaxErrorKind::UnpackingInComprehension(_) + | UnsupportedSyntaxErrorKind::UnparenthesizedUnpackInFor + | UnsupportedSyntaxErrorKind::UnparenthesizedExceptionTypes + | UnsupportedSyntaxErrorKind::TemplateStrings => false, } } diff --git a/crates/ruff_python_parser/src/parser/expression.rs b/crates/ruff_python_parser/src/parser/expression.rs index 5fb91a0057..42b006d541 100644 --- a/crates/ruff_python_parser/src/parser/expression.rs +++ b/crates/ruff_python_parser/src/parser/expression.rs @@ -1173,6 +1173,25 @@ impl<'src> Parser<'src> { node_index: AtomicNodeIndex::NONE, }) } + // basedpython: `some T` names a parameter's own type parameter, so it means + // something only at the top of a parameter's annotation, where the parameter + // parser takes it. anywhere else two adjacent names are never valid python, so + // `some` followed by a name is this keyword written where it cannot stand. + // parsing on from the name keeps one error rather than a cascade + TokenKind::Name + if self.options.is_basedpython + && self.src_text(self.current_token_range()) == "some" + && self.peek() == TokenKind::Name => + { + self.add_error( + ParseErrorType::OtherError( + "`some` is only allowed at the top of a parameter annotation".to_string(), + ), + self.current_token_range(), + ); + self.bump(TokenKind::Name); + return self.parse_atom(context); + } TokenKind::Name => { // basedpython `typeof ` keyword: when the current // identifier is `typeof` and the following token starts an diff --git a/crates/ruff_python_parser/src/parser/mod.rs b/crates/ruff_python_parser/src/parser/mod.rs index 72074c35cf..474489c525 100644 --- a/crates/ruff_python_parser/src/parser/mod.rs +++ b/crates/ruff_python_parser/src/parser/mod.rs @@ -754,9 +754,11 @@ impl<'src> Parser<'src> { /// Add an [`UnsupportedSyntaxError`] with the given [`UnsupportedSyntaxErrorKind`] and /// [`TextRange`] if its minimum version is less than [`Parser::target_version`]. fn add_unsupported_syntax_error(&mut self, kind: UnsupportedSyntaxErrorKind, range: TextRange) { - // basedpython transpiles modern syntax (PEP 695 type params, etc.) down to - // the runtime target version, so the source itself is not constrained - if self.options.is_basedpython { + // basedpython lowers some modern syntax (pep 695 type parameters and the like) to + // what the target parses, so the source is not held to the target there + if self.options.is_basedpython + && kind.is_lowered_by_basedpython(self.options.target_version) + { return; } if kind.is_unsupported(self.options.target_version) { diff --git a/crates/ruff_python_parser/src/parser/statement.rs b/crates/ruff_python_parser/src/parser/statement.rs index 41effc25ce..6410073993 100644 --- a/crates/ruff_python_parser/src/parser/statement.rs +++ b/crates/ruff_python_parser/src/parser/statement.rs @@ -4575,10 +4575,20 @@ impl<'src> Parser<'src> { "pattern-matching `if let` is not valid in .py files".to_string(), TextRange::new(let_range.start(), pattern.end()), ); + self.add_destructuring_syntax_error(TextRange::new(let_range.start(), pattern.end())); Some(pattern) } + /// basedpython: holds a destructuring pattern to the python the file targets, which + /// runs it as the `match` statement it is written as. a `.py` file is refused the form + /// outright, so it is not told the version as well + fn add_destructuring_syntax_error(&mut self, range: TextRange) { + if self.options.is_basedpython { + self.add_unsupported_syntax_error(UnsupportedSyntaxErrorKind::Destructuring, range); + } + } + /// basedpython: the synthetic binder holding the value `pattern` /// destructures, as an expression. /// @@ -4647,6 +4657,7 @@ impl<'src> Parser<'src> { "a destructuring `let` is not valid in .py files".to_string(), TextRange::new(start, pattern.end()), ); + self.add_destructuring_syntax_error(TextRange::new(start, pattern.end())); let value = self.parse_expression_list(ExpressionContext::default()); @@ -5087,6 +5098,7 @@ impl<'src> Parser<'src> { "a destructuring `for` target is not valid in .py files".to_string(), pattern.range(), ); + self.add_destructuring_syntax_error(pattern.range()); target.expr = self.destructure_binder(&pattern, ExprContext::Store); Some(Box::new(pattern)) } @@ -7027,6 +7039,7 @@ impl<'src> Parser<'src> { "a destructuring `with` target is not valid in .py files".to_string(), pattern.range(), ); + self.add_destructuring_syntax_error(pattern.range()); let binder = self.destructure_binder(&pattern, ExprContext::Store); return (binder, Some(Box::new(pattern))); } @@ -7944,6 +7957,7 @@ impl<'src> Parser<'src> { "a destructuring parameter is not valid in .py files".to_string(), pattern.range(), ); + self.add_destructuring_syntax_error(pattern.range()); self.destructure_binder_identifier(pattern) } None => self.parse_identifier(), @@ -7967,8 +7981,22 @@ impl<'src> Parser<'src> { self.error_if_not_basedpython( "`some` parameter annotations are not valid in .py files".to_string(), ); + // the annotation of `*args` / `**kwargs` types each element rather than + // the parameter, so the parameter's name could not name the type `some` + // opens. the annotation then reads as though `some` were not there + if matches!(allow_star_annotation, AllowStarAnnotation::No) { + is_some = true; + } else { + self.add_error( + ParseErrorType::OtherError( + "`some` is not allowed on a variadic parameter: its annotation \ + is the type of each element, not of the parameter" + .to_string(), + ), + self.current_token_range(), + ); + } self.bump(TokenKind::Name); - is_some = true; } if self.at_expr() { let parsed_expr = match allow_star_annotation { @@ -9048,12 +9076,11 @@ impl<'src> Parser<'src> { type_params: Option, parameters: &mut ast::Parameters, ) -> Option { + // `*args` / `**kwargs` never carry `some`, see `parse_parameter` let ast::Parameters { posonlyargs, args, - vararg, kwonlyargs, - kwarg, .. } = parameters; let holes: Vec = posonlyargs @@ -9061,8 +9088,6 @@ impl<'src> Parser<'src> { .chain(args.iter_mut()) .chain(kwonlyargs.iter_mut()) .map(|parameter| &mut parameter.parameter) - .chain(vararg.iter_mut().map(Box::as_mut)) - .chain(kwarg.iter_mut().map(Box::as_mut)) .filter(|parameter| parameter.is_some) .filter_map(|parameter| { // the annotation becomes a reference to the hole, which is what makes the diff --git a/crates/ruff_python_parser/src/parser/tests.rs b/crates/ruff_python_parser/src/parser/tests.rs index b12df92e0c..50ea86206f 100644 --- a/crates/ruff_python_parser/src/parser/tests.rs +++ b/crates/ruff_python_parser/src/parser/tests.rs @@ -1,5 +1,6 @@ use std::assert_matches; +use ruff_python_ast::comparable::ComparableStmt; use ruff_python_ast::helpers::{UseSiteVariance, use_site_variance_marker}; use ruff_python_ast::{ Expr, InterpolatedStringElement, IpyEscapeKind, ModModule, Number, Operator, Pattern, Stmt, @@ -3326,3 +3327,109 @@ fn basedpython_init_method_rejects_a_destructuring_parameter() { parsed.errors() ); } + +/// `some` is taken as a keyword only at the top of a parameter's annotation. written anywhere +/// else in front of a name it is reported once, and what follows it parses as though it were +/// not there +#[test] +fn a_misplaced_some_is_one_error() { + for (source, parsed_as) in [ + ( + "def f(n: int) -> some int: ...\n", + "def f(n: int) -> int: ...", + ), + ( + "def g(xs: list[some int]): ...\n", + "def g(xs: list[int]): ...", + ), + ("x: some int = 1\n", "x: int = 1"), + ("y = cast(some int?, 1)\n", "y = cast(int?, 1)"), + ] { + let parsed = parse_basedpython_module_with_errors(source); + let messages: Vec = parsed + .errors() + .iter() + .map(|error| error.error.to_string()) + .collect(); + assert_eq!( + messages, + ["`some` is only allowed at the top of a parameter annotation"], + "{source}" + ); + let expected = parse_basedpython_module(&format!("{parsed_as}\n")); + let comparable = |body: &[Stmt]| -> Vec { + body.iter() + .map(|stmt| format!("{:?}", ComparableStmt::from(stmt))) + .collect() + }; + assert_eq!( + comparable(&parsed.syntax().body), + comparable(&expected.syntax().body), + "{source}" + ); + } +} + +/// a variable named `some` followed by anything that continues an expression is that +/// variable, as it is in python +#[test] +fn a_variable_named_some_is_a_variable() { + for source in [ + "some(int)\n", + "some[int]\n", + "some - 1\n", + "some cast int\n", + "x = some\n", + "def f(some: int) -> None: ...\n", + ] { + let parsed = parse_basedpython_module_with_errors(source); + assert!( + parsed.errors().is_empty(), + "{source}: {:?}", + parsed.errors() + ); + } +} + +/// the annotation of `*args` / `**kwargs` is the type of each element, so the parameter's name +/// cannot name the type `some` would open. `some` there is reported once, and the annotation +/// parses as though it were not there +#[test] +fn some_on_a_variadic_parameter_is_one_error() { + for (source, parsed_as, expected_errors) in [ + ("def f(*xs: some int): ...\n", "def f(*xs: int): ...", 1), + ("def g(**kw: some int): ...\n", "def g(**kw: int): ...", 1), + ( + "def h(a: some int, *xs: some str, **kw: some bytes): ...\n", + "def h(a: some int, *xs: str, **kw: bytes): ...", + 2, + ), + ] { + let parsed = parse_basedpython_module_with_errors(source); + let messages: Vec = parsed + .errors() + .iter() + .map(|error| error.error.to_string()) + .collect(); + assert_eq!( + messages, + vec![ + "`some` is not allowed on a variadic parameter: its annotation is the type of \ + each element, not of the parameter"; + expected_errors + ], + "{source}" + ); + let expected = parse_basedpython_module(&format!("{parsed_as}\n")); + let comparable = |body: &[Stmt]| -> Vec { + body.iter() + .map(|stmt| format!("{:?}", ComparableStmt::from(stmt))) + .collect() + }; + assert_eq!( + comparable(&parsed.syntax().body), + comparable(&expected.syntax().body), + "{source}" + ); + } +} diff --git a/crates/ty/docs/cli.md b/crates/ty/docs/cli.md index 4d1ca9f79d..91f296c1f4 100644 --- a/crates/ty/docs/cli.md +++ b/crates/ty/docs/cli.md @@ -262,7 +262,7 @@ by run [OPTIONS] [MODULE] [ARGS]...

The interpreter is chosen exactly as it is without this, and the program is started as LAUNCHER <interpreter> <runner> <module> <args>: the launcher is handed the interpreter by run chose rather than having to name one, and is run once, for the program — never for the version probe, which is made against the interpreter itself.

--min-version version

minimum Python version the output must run on [default: the version of the interpreter that will run it]

--no-unique-loop-bindings

leave a closure made inside a loop sharing the loop's one binding, as python does, instead of binding the values of the iteration it was made in

-
--python, --venv path

The interpreter to run on, or the environment holding it.

+
--python, --venv path

The interpreter to run on, or the environment holding it. A bare name that is neither, like python3.14, is found on PATH.

Defaults to the project environment — the same one by check resolves imports against — then $PYTHON, then python3 on PATH.

--runtime-raises-checks

wrap every function with a raises clause in a runtime guard that fails when it raises something the clause does not include

--soundness spec

which runtime type-soundness checks to insert: default, all (adds the opt-in parameters entry checks), none, or a comma-separated subset of generic-calls, projections, iterations, assignments, returns, arguments, parameters

diff --git a/crates/ty/src/args.rs b/crates/ty/src/args.rs index 606ec53251..b5de2bc9d5 100644 --- a/crates/ty/src/args.rs +++ b/crates/ty/src/args.rs @@ -81,7 +81,8 @@ pub(crate) enum Command { /// [default: the version of the interpreter that will run it] #[arg(long, value_name = "VERSION")] min_version: Option, - /// The interpreter to run on, or the environment holding it. + /// The interpreter to run on, or the environment holding it. A bare name + /// that is neither, like `python3.14`, is found on `PATH`. /// /// Defaults to the project environment — the same one `by check` /// resolves imports against — then `$PYTHON`, then `python3` on `PATH`. diff --git a/crates/ty/src/by_commands.rs b/crates/ty/src/by_commands.rs index 903f446d2c..55628c3d0f 100644 --- a/crates/ty/src/by_commands.rs +++ b/crates/ty/src/by_commands.rs @@ -668,7 +668,9 @@ fn discover_interpreter( if let Some(flag) = flag { // a `--python` may name the interpreter itself or the environment it - // lives in, the same way `by check --python` does + // lives in, the same way `by check --python` does. failing both, it is + // run as it is written, which finds a bare name like `python3.14` on + // `PATH`, as `$PYTHON` and the `python3` fallback are found if flag.is_file() { return Ok(named(flag.display().to_string(), "`--python`")); } @@ -858,9 +860,11 @@ fn discovered_environment(root: &Path) -> Option { }) } -/// The interpreter inside the environment rooted at `path`, if there is one. +/// The interpreter inside the environment rooted at `path`, if there is one. a +/// relative `path` is taken from the working directory fn interpreter_in_environment(path: &Path, origin: SysPrefixPathOrigin) -> Option { - let sys_path = SystemPath::from_std_path(path)?; + let path = std::path::absolute(path).ok()?; + let sys_path = SystemPath::from_std_path(&path)?; let system = OsSystem::new(sys_path); let environment = PythonEnvironment::new(sys_path, origin, &system).ok()?; environment @@ -1778,7 +1782,7 @@ fn write_traceback_runtime( } const BY_RUNNER_SRC: &str = r#"# generated by `by run` — runs the target module with .by-aware tracebacks -import ast +import dis import hashlib import importlib.machinery import importlib.util @@ -1787,6 +1791,7 @@ import os import sys import traceback import types +import warnings from _by_sourcemap import DIGESTS, SOURCEMAP @@ -1798,6 +1803,9 @@ _BY_MAP = {os.path.realpath(py): (py, info) for py, info in SOURCEMAP.items()} # with the key of the entry each was transpiled under _BY_SOURCES = {os.path.realpath(info[0]): py for py, info in SOURCEMAP.items()} +# this file, whose frames are never the program's +_RUNNER = os.path.realpath(__file__) + # entries already checked against their digests, so each file is read at most # once no matter how many frames it contributes _CURRENT = set() @@ -1875,7 +1883,12 @@ def _rewrite(frames): ), None, ) - frames = frames[first:] if first is not None else frames + if first is not None: + # below the program's first frame, the runner's own are the loader it puts on + # every staged module, which a frame of the program never called + frames = [ + f for f in frames[first:] if os.path.realpath(f.filename) != _RUNNER + ] out = [] for f in frames: compiled_from = _BY_SOURCES.get(os.path.realpath(f.filename)) @@ -1910,6 +1923,21 @@ def _rewrite(frames): def _excepthook(etype, evalue, tb): frames = _rewrite(traceback.extract_tb(tb)) + if ( + isinstance(evalue, SyntaxError) + and evalue.filename is not None + and os.path.realpath(evalue.filename) in _BY_SOURCES + and all( + isinstance(frame, traceback.FrameSummary) + and os.path.realpath(frame.filename) == _RUNNER + for frame in frames + ) + ): + # the program's own module, which the runner could not compile: every frame + # is the runner's, so it is reported as python reports a script it cannot + # compile, by where the error is and nothing above it + sys.stderr.write("".join(traceback.format_exception_only(etype, evalue))) + return sys.stderr.write("Traceback (most recent call last):\n") # a frame `_rewrite` could only name is already text. the others are formatted a # run at a time, so python still folds a repeated one @@ -1934,18 +1962,118 @@ def _by_source_of(origin): return None if entry is None else entry[1][0] +def _varint(out, value): + while value >= 64: + out.append(0x40 | (value & 63)) + value >>= 6 + out.append(value) + + +def _svarint(out, value): + _varint(out, (-value << 1) | 1 if value < 0 else value << 1) + + +def _relined(code, filename, by_line, width): + """`code` and every code object it holds, named `filename`, with each location + moved to the line `by_line` gives, spanning the whole of that line. + + The line table's format is private to each python version: see `Objects/locations.md` + (3.11 on) and `Objects/lnotab_notes.txt` (3.10, and 3.9 before it) in cpython. + """ + consts = tuple( + _relined(const, filename, by_line, width) if isinstance(const, types.CodeType) else const + for const in code.co_consts + ) + first = by_line(code.co_firstlineno) + table = bytearray() + if sys.version_info >= (3, 11): + # an entry for each run of up to 8 code units at one line, in the long form, + # which holds any line and column, or the form for no location + lines = [None if line is None else by_line(line) for line, *_ in code.co_positions()] + previous = first + start = 0 + while start < len(lines): + line = lines[start] + end = start + 1 + while end < len(lines) and end - start < 8 and lines[end] == line: + end += 1 + if line is None: + table.append(0x80 | (15 << 3) | (end - start - 1)) + else: + table.append(0x80 | (14 << 3) | (end - start - 1)) + _svarint(table, line - previous) + _varint(table, 0) + _varint(table, 1) + _varint(table, width(line) + 1) + previous = line + start = end + return code.replace( + co_filename=filename, co_firstlineno=first, co_consts=consts, co_linetable=bytes(table) + ) + if sys.version_info >= (3, 10): + # a (bytecode length, line delta) pair for each range, -128 for no line + previous = first + for start, end, line in code.co_lines(): + size = end - start + delta = -128 + if line is not None: + line = by_line(line) + delta = line - previous + previous = line + while delta > 127: + table += bytes((0, 127)) + delta -= 127 + while delta < -127: + table += bytes((0, 256 - 127)) + delta += 127 + while size > 254: + table += bytes((254, delta & 255)) + delta = -128 if line is None else 0 + size -= 254 + table += bytes((size, delta & 255)) + return code.replace( + co_filename=filename, co_firstlineno=first, co_consts=consts, co_linetable=bytes(table) + ) + # a (bytecode offset delta, line delta) pair for each place the line changes + offset, previous = 0, first + for start, line in dis.findlinestarts(code): + line = by_line(line) + step, delta = start - offset, line - previous + while step > 255: + table += bytes((255, 0)) + step -= 255 + while delta > 127: + table += bytes((step, 127)) + step, delta = 0, delta - 127 + while delta < -128: + table += bytes((step, 128)) + step, delta = 0, delta + 128 + table += bytes((step, delta & 255)) + offset, previous = start, line + return code.replace( + co_filename=filename, co_firstlineno=first, co_consts=consts, co_lnotab=bytes(table) + ) + + def _by_code(origin): """The staged `.py` at `origin` compiled as the `.by` it was transpiled from. A program reads its own locations back — `traceback.format_exc()`, the file and line `warnings` attributes a warning to, `co_filename`, `inspect` — and - python answers each from the code object. So the code object is compiled under + python answers each from the code object. So the code object is named after the `.by` path, with every line the `.by` line it came from: the answer is then the author's wherever it is read, and the source line python shows beside it is read from the `.by` too. A line the transpiler wrote on its own — a runtime helper — has no `.by` line, and says so as line 0. Columns are the generated - file's, so every node spans the whole of its `.by` line instead: python marks - no part of a line a node covers, and reads what it can from the `.by` text. + file's, so every location spans the whole of its `.by` line instead: python + marks no part of a line a location covers, and reads what it can from the `.by` + text. + + The `.py` is compiled from its text, as python's own loader would, and only + then moved to the `.by`: python 3.9 to 3.11 refuse, at the interpreter's + recursion limit, a syntax tree nested deeper than the source they accept. A + warning the compiler raises is moved to its `.by` line the same way, and so is + the `SyntaxError` it raises for one the warning filters make an error. `None` when there is no `.by` to answer with, or it has changed since it was transpiled, which leaves the module to python's own loader. @@ -1957,7 +2085,7 @@ def _by_code(origin): if not _is_current(key, by_path): return None with open(origin, "rb") as handle: - tree = ast.parse(handle.read(), origin) + source = handle.read() with open(by_path, "rb") as handle: widths = [len(line.rstrip(b"\r\n")) for line in handle] @@ -1965,12 +2093,32 @@ def _by_code(origin): mapped = lines[line - 1] if line is not None and 0 < line <= len(lines) else None return 0 if mapped is None else mapped + 1 - for node in ast.walk(tree): - if "lineno" in node._attributes: - node.lineno = node.end_lineno = by_line(node.lineno) - node.col_offset = 0 - node.end_col_offset = widths[node.lineno - 1] if node.lineno else 0 - return compile(tree, by_path, "exec", dont_inherit=True) + def width(line): + return widths[line - 1] if 0 < line <= len(widths) else 0 + + with warnings.catch_warnings(record=True) as caught: + try: + code = compile(source, origin, "exec", dont_inherit=True) + except SyntaxError as error: + # a warning the filters turn into an error (`-W error`) is raised by the + # compiler as a `SyntaxError` at the line it would have warned about, so + # it is moved to the `.by` like the warning. its columns are the + # generated file's, which is why it points at no part of the line + line = by_line(error.lineno) if error.filename == origin else 0 + if line == 0: + raise + text = linecache.getline(by_path, line) or None + raise SyntaxError(error.msg, (by_path, line, None, text)) from None + for warning in caught: + if warning.filename == origin: + warnings.warn_explicit( + warning.message, warning.category, by_path, by_line(warning.lineno) + ) + else: + warnings.warn_explicit( + warning.message, warning.category, warning.filename, warning.lineno + ) + return _relined(code, by_path, by_line, width) class _ByLoader: diff --git a/crates/ty/tests/by_e2e.rs b/crates/ty/tests/by_e2e.rs index 9bf728f67d..d2e4bea2cb 100644 --- a/crates/ty/tests/by_e2e.rs +++ b/crates/ty/tests/by_e2e.rs @@ -1294,6 +1294,127 @@ fn run_checks_a_deeply_nested_expression() { assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "2000"); } +/// python 3.9 to 3.11 compile a syntax tree handed to `compile` under the +/// interpreter's recursion limit, and refuse one nested deeper than the source +/// they compile without complaint. `by run` names each module's code after its +/// `.by`, and a module nested this deep still runs on each of them +#[test] +fn run_compiles_a_deeply_nested_expression_on_each_interpreter() { + let dir = tempfile::tempdir().expect("tempdir"); + let terms = vec!["1"; 2000].join(" + "); + fs::write(dir.path().join("main.by"), format!("print({terms})\n")).unwrap(); + + let interpreters = ["python3", "python3.9", "python3.10", "python3.11"] + .into_iter() + .filter(|python| { + Command::new(python) + .args(["-c", ""]) + .status() + .is_ok_and(|status| status.success()) + }); + for python in interpreters { + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") + .args(["run", "--python", python, "main"]) + .current_dir(dir.path()) + .output() + .expect("failed to spawn by"); + + assert!( + output.status.success(), + "by run on {python} failed:\n{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!( + String::from_utf8_lossy(&output.stdout).trim(), + "2000", + "on {python}" + ); + } +} + +/// a warning python raises while compiling a module names the `.by` line it came +/// from, like one the program raises as it runs +#[test] +fn run_names_the_by_line_of_a_compile_time_warning() { + let dir = tempfile::tempdir().expect("tempdir"); + // the force-unwrap writes a runtime helper above the print, so the generated + // line is not the `.by` line + fs::write( + dir.path().join("main.by"), + "def f(a: int?) -> int:\n return a!\n\nprint(\"\\d\", f(1))\n", + ) + .unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") + // an invalid escape is a `DeprecationWarning` before 3.12, hidden by default + .env("PYTHONWARNINGS", "always") + .args(["run", "main"]) + .current_dir(dir.path()) + .output() + .expect("failed to spawn by"); + + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(output.status.success(), "by run failed:\n{stderr}"); + assert!( + stderr.contains("main.by:4: ") && stderr.contains("invalid escape sequence"), + "the warning should name the .by line of the string:\n{stderr}" + ); + assert!( + !stderr.contains(".py:"), + "the warning should not name the generated file:\n{stderr}" + ); +} + +/// a warning the filters make an error is raised by python's compiler as a `SyntaxError` +/// at the line it would have warned about, which names the `.by` line as the warning does. +/// in the module `by run` was asked to run, it is reported the way python reports a script +/// it cannot compile, with none of the runner's frames above it +#[test] +fn run_names_the_by_line_of_a_compile_time_warning_made_an_error() { + let dir = tempfile::tempdir().expect("tempdir"); + // the force-unwrap writes a runtime helper above the print, so the generated + // line is not the `.by` line + let program = "def f(a: int?) -> int:\n return a!\n\nprint(\"\\d\", f(1))\n"; + fs::write(dir.path().join("main.by"), program).unwrap(); + fs::write(dir.path().join("bad.by"), program).unwrap(); + fs::write( + dir.path().join("importer.by"), + "import bad\n\nprint(bad.f(2))\n", + ) + .unwrap(); + + for (module, by_file) in [("main", "main.by"), ("importer", "bad.by")] { + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") + // an invalid escape is a `DeprecationWarning` before 3.12 and a + // `SyntaxWarning` from it, and an error under this filter on both + .env("PYTHONWARNINGS", "error") + .args(["run", module]) + .current_dir(dir.path()) + .output() + .expect("failed to spawn by"); + + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + !output.status.success(), + "by run {module} succeeded:\n{stderr}" + ); + assert!( + stderr.contains(&format!("{by_file}\", line 4\n")) + && stderr.contains("SyntaxError") + && stderr.contains("invalid escape sequence"), + "the error should name the .by line of the string:\n{stderr}" + ); + let generated = by_file.replace(".by", ".py"); + assert!( + !stderr.contains(&generated) && !stderr.contains("_by_runner"), + "the error should name neither the generated file nor the runner:\n{stderr}" + ); + } +} + #[test] fn run_force_unwrap_yields_inner_value() { // `Some(x)` lowers to the `Optional(x)` wrapper; force-unwrapping it must @@ -6240,6 +6361,51 @@ fn run_falls_back_to_the_python_variable() { ); } +/// `--python` takes a bare name, like `$PYTHON` does, and finds it on `PATH`. it +/// was read as a path relative to the working directory and panicked there +#[test] +fn run_finds_a_bare_python_name_on_path() { + let dir = tempfile::tempdir().expect("tempdir"); + fs::write(dir.path().join("main.by"), "print(1)\n").unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") + .args(["run", "--python", "python3", "main"]) + .current_dir(dir.path()) + .output() + .expect("failed to spawn by"); + + assert!( + output.status.success(), + "by run failed:\n{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "1"); +} + +/// a relative `--python` that is neither an interpreter nor an environment is +/// reported as an interpreter that cannot be run, not a panic +#[test] +fn run_reports_a_relative_python_that_is_not_one() { + let dir = tempfile::tempdir().expect("tempdir"); + fs::write(dir.path().join("main.by"), "print(1)\n").unwrap(); + fs::create_dir(dir.path().join("empty")).unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") + .args(["run", "--python", "empty", "main"]) + .current_dir(dir.path()) + .output() + .expect("failed to spawn by"); + + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(!output.status.success(), "expected a refusal:\n{stderr}"); + assert!( + stderr.contains("could not run `empty`"), + "the message has to name the interpreter it could not run:\n{stderr}" + ); +} + /// a configured environment that cannot be resolved is what `by check` refuses /// outright. falling past it ran the program on a different python than the one /// it had just been checked against, and reported that as a version mismatch — diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_optional_type.md b/crates/ty_python_semantic/resources/mdtest/basedpython_optional_type.md index be868a3960..4fcf8b6357 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_optional_type.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_optional_type.md @@ -77,6 +77,31 @@ def f(x: str? | int) -> None: reveal_type(x) # revealed: str | None | int ``` +## an optional read as a value is the union + +where a program evaluates `T?` — a type alias, the classes `isinstance` is given — it is the union +object `T | None` is, so an alias of it is that union and `isinstance` narrows by it + +```by +Alias = int? +reveal_type(Alias) # revealed: + +def f(x: Alias, y: object) -> None: + reveal_type(x) # revealed: int | None + if isinstance(y, int?): + reveal_type(y) # revealed: int | None +``` + +inside a union or a tuple of classes it is the same union + +```by +def g(y: object) -> None: + if isinstance(y, int? | str): + reveal_type(y) # revealed: int | None | str + if isinstance(y, (bytes?, str)): + reveal_type(y) # revealed: bytes | None | str +``` + ## double optional is a distinct wrapped type a single `T?` is the lossless union `T | None`, but a nested optional cannot collapse that way (the @@ -371,6 +396,44 @@ def f(x: object??): sink(x) ``` +## a type test of an optional is not flagged + +a call that tests the type of an argument reads the optional rather than discarding it: its result +says whether the value was `None`, and narrows the value when the call is a condition. the argument +`isinstance` and `hasattr` test, and the one a `TypeIs` or `TypeGuard` return narrows, takes +`object` for exactly that reason + +```by +from typing import TypeGuard, TypeIs + +def is_int(v: object) -> TypeIs[int]: + return isinstance(v, int) + +def guard(v: object) -> TypeGuard[int]: + return isinstance(v, int) + +def f(x: int?, y: str?) -> None: + isinstance(x, int) + hasattr(x, "real") + callable(y) + is_int(x) + guard(x) +``` + +## the other arguments of a type test are flagged + +only the argument the test is about is read by it. an optional passed to another `object` parameter +of the same call is still widened + +```by +def is_str(tag: object, v: object) -> v is str: + return isinstance(v, str) + +def f(x: int?, y: str?): + # error: [optional-object-conversion] "Optional `str | None` is implicitly widened to `object`" + is_str(y, x) +``` + ## passing an optional to a parameter of equal depth stays silent the receiving parameter preserves the `None` arm, so nothing is lost: diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_reified_generics.md b/crates/ty_python_semantic/resources/mdtest/basedpython_reified_generics.md index 6d31b39c82..8e5149309a 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_reified_generics.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_reified_generics.md @@ -41,6 +41,20 @@ def f[T](x: T) -> T: reveal_type(f[int](1)) # revealed: int ``` +## a parameter of the same name is not a read of the type parameter + +A parameter is bound in the body's scope and the type parameter in the one around it, so the body +reads the parameter under that name. `n` here is used only in annotations, stays erased, and the +call needs no specialization even though nothing solves `n`: + +```by +def f[n](n: n | None = None) -> n | None: + reveal_type(n) # revealed: n@f | None + return n + +f() +``` + ## `reified` declares reification outright The `reified` modifier reifies a type parameter whether or not the body ever reads it as a value. diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_some_parameters.md b/crates/ty_python_semantic/resources/mdtest/basedpython_some_parameters.md new file mode 100644 index 0000000000..f0da4a36fe --- /dev/null +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_some_parameters.md @@ -0,0 +1,126 @@ +# basedpython: `some` parameters + +`some T` in a parameter's annotation gives the parameter a type parameter of its own, bounded by `T` +and named after the parameter. the rest of the signature names that type by the parameter's name, so +a function can hand back the exact type it was called with without declaring a type parameter list + +## the argument's own type comes back + +`echo` accepts any `str`, and returns the type of the argument it was given + +```by +def echo(s: some str) -> s: + return s + +class Name(str): + pass + +reveal_type(echo) # revealed: def echo(s: some str) -> s +reveal_type(echo("hi")) # revealed: "hi" +reveal_type(echo(Name("x"))) # revealed: Name +``` + +## the bound is checked at the call + +an argument outside the bound is rejected, as it is for a declared type parameter with that bound + +```by +def echo(s: some str) -> s: + return s + +echo(1) # error: [invalid-argument-type] +``` + +## the body reads the parameter's type + +inside the body, the parameter has the type parameter's type + +```by +def echo(s: some str) -> s: + reveal_type(s) # revealed: s@echo + return s +``` + +## each `some` parameter is a type parameter of its own + +two `some` parameters are solved independently, and a return type can name both + +```by +def pair(a: some int, b: some str) -> tuple[a, b]: + return (a, b) + +reveal_type(pair(1, "x")) # revealed: (1, "x") +``` + +## a union bound + +the bound is any type expression, a union among them + +```by +def either(v: some int | str) -> v: + return v + +reveal_type(either(1)) # revealed: 1 +reveal_type(either("a")) # revealed: "a" +``` + +## `some` is written at the top of a parameter annotation + +`some` names the parameter's own type parameter, so it is not a type of its own. written anywhere +else — a return type, inside another type, a variable's annotation — it is reported where it stands, +and the type after it reads as though `some` were not there + +```by +# error: [invalid-syntax] "`some` is only allowed at the top of a parameter annotation" +def f(n: int) -> some int: + return n + +# error: [invalid-syntax] "`some` is only allowed at the top of a parameter annotation" +def g(xs: list[some int]) -> None: + reveal_type(xs) # revealed: list[int] + +# error: [invalid-syntax] "`some` is only allowed at the top of a parameter annotation" +x: some int = 1 + +reveal_type(f(1)) # revealed: int +``` + +## `some` is not written on `*args` or `**kwargs` + +the annotation of a variadic parameter is the type of each element, while the parameter itself is a +tuple or a dict of them. the parameter's name cannot name both, so `some` is reported there, and the +annotation reads as though `some` were not there + +```by +# error: [invalid-syntax] "`some` is not allowed on a variadic parameter: its annotation is the type of each element, not of the parameter" +def first(*xs: some int) -> int: + reveal_type(xs) # revealed: (*: int) + return xs[0] + +# error: [invalid-syntax] "`some` is not allowed on a variadic parameter: its annotation is the type of each element, not of the parameter" +def named(**kw: some int) -> int: + reveal_type(kw) # revealed: dict[str, int] + return kw["a"] +``` + +## on a target without type parameter syntax + +the type parameter is declared the way the target python can declare one, which includes an optional +bound on a python without `X | None` + +```toml +[environment] +python-version = "3.9" +``` + +```by +def label(n: some int?) -> str: + return "none" if n is None else str(n) + +def echo(s: some str) -> s: + return s + +assert label(None) == "none" +assert label(3) == "3" +assert echo("hi") == "hi" +``` diff --git a/crates/ty_python_semantic/resources/mdtest/diagnostics/version_related_syntax_errors.md b/crates/ty_python_semantic/resources/mdtest/diagnostics/version_related_syntax_errors.md index 0a35fe4153..f99572055e 100644 --- a/crates/ty_python_semantic/resources/mdtest/diagnostics/version_related_syntax_errors.md +++ b/crates/ty_python_semantic/resources/mdtest/diagnostics/version_related_syntax_errors.md @@ -35,3 +35,141 @@ match 2: case 1: print("it's one") ``` + +## In a basedpython file + +A basedpython file is transpiled to python for the version it targets. Syntax the transpiler lowers +is fine at any version, while syntax it writes into the python as it stands has to be syntax that +version parses. + +### Syntax the python keeps + +A `/` is written into the python as it stands, so a target before 3.8 cannot run it: + +```toml +[environment] +python-version = "3.7" +``` + +```by +def f(a: int, /) -> None: # error: [invalid-syntax] "Cannot use positional-only parameter separator on Python 3.7 (syntax was added in Python 3.8)" + pass +``` + +### Syntax the transpiler lowers + +Type parameters are lowered to `TypeVar` declarations, and a `type` statement to a `TypeAliasType`: + +```toml +[environment] +python-version = "3.9" +``` + +```by +def first[T](items: list[T]) -> T: + return items[0] + +type Pair = tuple[int, int] + +assert first([1, 2]) == 1 +``` + +### A `match` statement + +A `match` statement is lowered to a chain of `if` statements whose tests bind what the patterns +capture, which takes the assignment expressions of python 3.8: + +```toml +[environment] +python-version = "3.8" +``` + +```by +def describe(x: int) -> str: + match x: + case 1: + return "one" + case _: + return "other" + +assert describe(1) == "one" +``` + +### A `match` statement before 3.8 + +Before 3.8, the `match` statement is left as it is: + +```toml +[environment] +python-version = "3.7" +``` + +```by +def describe(x: int) -> str: + match x: # error: [invalid-syntax] "Cannot use `match` statement on Python 3.7 (syntax was added in Python 3.10)" + case 1: + return "one" + case _: + return "other" +``` + +### A destructuring pattern + +A destructuring pattern — `let P := v`, an `if let` clause, or a pattern written as a `for` or +`with` target or as a parameter — is written as a `match` statement, so it runs wherever one does. +From 3.8 that is the lowered `match`: + +```toml +[environment] +python-version = "3.8" +``` + +```by +def total(pair: tuple[int, int] | None) -> int: + if let (a, b) := pair: + return a + b + return 0 + +def first(pair: tuple[int, int]) -> int: + let (a, _) := pair + return a + +assert total((1, 2)) == 3 +assert first((4, 5)) == 4 +``` + +### A destructuring pattern before 3.8 + +Before 3.8 no `match` can be written, so neither can a destructuring pattern: + +```toml +[environment] +python-version = "3.7" +``` + +```by +def total(pair: tuple[int, int] | None) -> int: + # error: [invalid-syntax] "Cannot use a destructuring pattern, which is written as a `match` statement, on Python 3.7 (syntax was added in Python 3.10)" + if let (a, b) := pair: + return a + b + return 0 + +def first(pair: tuple[int, int]) -> int: + # error: [invalid-syntax] "Cannot use a destructuring pattern, which is written as a `match` statement, on Python 3.7 (syntax was added in Python 3.10)" + let (a, _) := pair + return a + +class Point: + __match_args__ = ("x", "y") + + def __init__(self, x: int, y: int) -> None: + self.x = x + self.y = y + +def sum_all(points: list[Point]) -> int: + n = 0 + # error: [invalid-syntax] "Cannot use a destructuring pattern, which is written as a `match` statement, on Python 3.7 (syntax was added in Python 3.10)" + for Point(x, y) in points: + n += x + y + return n +``` diff --git a/crates/ty_python_semantic/resources/mdtest/narrow/isinstance.md b/crates/ty_python_semantic/resources/mdtest/narrow/isinstance.md index 5c50407756..bc3325ae81 100644 --- a/crates/ty_python_semantic/resources/mdtest/narrow/isinstance.md +++ b/crates/ty_python_semantic/resources/mdtest/narrow/isinstance.md @@ -278,6 +278,54 @@ def _(x: dict[int, str] | ListStrOrInt): reveal_type(x) # revealed: list[str] | int ``` +## `classinfo` is a union before Python 3.10 + +A union object is a class only from Python 3.10 on. Before that, `isinstance()` and `issubclass()` +reject one with a `TypeError`, whether it is spelled with `|` or with `typing.Union`, so we report +it: + +```toml +[environment] +python-version = "3.9" +``` + +```py +from typing import Union + +IntOrStr = Union[int, str] + +def _(x: object, y: type): + # error: [invalid-argument-type] "A union cannot be used with `isinstance()` before Python 3.10" + isinstance(x, IntOrStr) + # error: [invalid-argument-type] "A union cannot be used with `issubclass()` before Python 3.10" + issubclass(y, (bytes, Union[int, None])) +``` + +## `classinfo` is a union before Python 3.10, in basedpython + +A basedpython module is transpiled before it runs, and a union spelled in the call itself — with +`|`, or as an optional `T?` — is written as the tuple of classes it stands for, which `isinstance()` +accepts at every version. A union reached through a name is written as the `typing.Union` it is +everywhere else, and is rejected like one: + +```toml +[environment] +python-version = "3.9" +``` + +```by +IntOrStr = int | str + +def _(x: object, y: type): + isinstance(x, int | str) + isinstance(x, (bytes, int?)) + issubclass(y, int | None) + # error: [invalid-argument-type] "A union cannot be used with `isinstance()` before Python 3.10" + isinstance(x, IntOrStr) + # error: [invalid-argument-type] "A union cannot be used with `isinstance()` before Python 3.10" + isinstance(x, (bytes, IntOrStr)) +``` + ## `Optional` as `classinfo` ```py diff --git a/crates/ty_python_semantic/resources/mdtest/repeated_underscore.md b/crates/ty_python_semantic/resources/mdtest/repeated_underscore.md index 0d837d6ffe..cebf8d3892 100644 --- a/crates/ty_python_semantic/resources/mdtest/repeated_underscore.md +++ b/crates/ty_python_semantic/resources/mdtest/repeated_underscore.md @@ -338,6 +338,128 @@ reveal_type(Override.f) # revealed: def f(self, x: int = 1, y: str = "a") Override().f() ``` +## an override of a method whose `_`s are numbered + +A base that repeats `_` numbers its own, and an override takes those names and kinds as it takes any +other. The first is `_` in both, so a read of `_` in the override's body is its first parameter, as +it is in any definition that repeats `_`: + +```by +class Base: + def f(self, _: int, _: int) -> int: + return _ + +class Override(Base): + override def f(self, _: int, _: int) -> int: + return _ * 10 + +reveal_type(Override.f) # revealed: def f(self, _: int, _: int, /) -> int + +assert Override().f(1, 2) == 10 +``` + +A variadic `_` has no counterpart in the base, and is numbered after the names the others take: + +```by +class Variadic(Base): + override def f(self, _: int, _: int, *_: int) -> int: + return 20 + +reveal_type(Variadic.f) # revealed: def f(self, _: int, _: int, /, *_: int) -> int + +assert Variadic().f(1, 2, 3) == 20 +``` + +## an override of an override + +An override that repeats `_` has the names of the method it overrides, so an override of it takes +those same names in turn: + +```by +class Base: + def f(self, x: int, y: int) -> int: + return 1 + +class Middle(Base): + override def f(self, _: int, _: int) -> int: + return 2 + +class Override(Middle): + override def f(self, _: int, _: int) -> int: + return _ * 30 + +reveal_type(Override.f) # revealed: def f(self, x: int, y: int) -> int + +assert Override().f(x=1, y=2) == 30 +``` + +## a base parameter named `_` + +A base parameter that its author named `_` is reached by that keyword, so the override's `_` in that +position keeps the name rather than being numbered: + +```by +class Base: + def f(self, x: int, _: int) -> int: + return x + +class Override(Base): + override def f(self, _: int, _: int) -> int: + return _ + +reveal_type(Override.f) # revealed: def f(self, x: int, _: int) -> int + +assert Override().f(1, _=2) == 1 +``` + +## an override of a base in another module + +`base.by`: + +```by +class Base: + def f(self, x: int, y: int) -> int: + return 1 +``` + +`main.by`: + +```by +from base import Base + +class Override(Base): + override def f(self, _: int, _: int) -> int: + return _ * 40 + +reveal_type(Override.f) # revealed: def f(self, x: int, y: int) -> int + +assert Override().f(x=1, y=2) == 40 +``` + +## an override of a generic base + +The names come from the base whether or not the override specializes it: + +```by +class Base[T]: + def f(self, x: T, y: T) -> int: + return 1 + +class Specialized(Base[int]): + override def f(self, _: int, _: int) -> int: + return _ * 50 + +class Generic[T](Base[T]): + override def f(self, _: T, _: T) -> int: + return 60 + +reveal_type(Specialized.f) # revealed: def f(self, x: int, y: int) -> int +reveal_type(Generic[str]().f) # revealed: bound method Generic[str].f(x: str, y: str) -> int + +assert Specialized().f(x=1, y=2) == 50 +assert Generic[str]().f(x="a", y="b") == 60 +``` + ## an override that does not line up with its base is numbered Where the base has no parameter at a `_`'s position, none of the `_`s take a name from it. The diff --git a/crates/ty_python_semantic/src/place.rs b/crates/ty_python_semantic/src/place.rs index e9d4573ca6..988909202b 100644 --- a/crates/ty_python_semantic/src/place.rs +++ b/crates/ty_python_semantic/src/place.rs @@ -773,7 +773,7 @@ pub(crate) fn is_basedpython_implicit_typing_name(name: &str) -> bool { } /// The Python version in which `name` was added to the `typing` module, or -/// `None` if it has been available since before 3.10. +/// `None` if python 3.9 already has it. /// /// basedpython makes these names available regardless of the target Python /// version. This is the single source of truth for that set: `by_transforms`' @@ -783,6 +783,9 @@ pub(crate) fn is_basedpython_implicit_typing_name(name: &str) -> bool { /// (see `infer_import_from_definition`). pub fn basedpython_typing_added_in(name: &str) -> Option { match name { + // 3.10 + "ParamSpec" | "ParamSpecArgs" | "ParamSpecKwargs" | "Concatenate" | "TypeAlias" + | "TypeGuard" | "is_typeddict" => Some(PythonVersion::PY310), // 3.11 "Never" | "assert_never" diff --git a/crates/ty_python_semantic/src/reified.rs b/crates/ty_python_semantic/src/reified.rs index f385f17047..b4a4388024 100644 --- a/crates/ty_python_semantic/src/reified.rs +++ b/crates/ty_python_semantic/src/reified.rs @@ -64,6 +64,12 @@ pub fn reified_type_param_names( } let mut active: FxHashSet<&str> = candidates.iter().map(|(name, _)| name.as_str()).collect(); + // the body reads a parameter under its own name, which a type parameter of that name does + // not reach: the parameter is bound in the body's scope, the type parameter in the one + // around it + for parameter in &function.parameters { + active.remove(parameter.name().id.as_str()); + } shadow_bound_names(&function.body, &mut active); let param_typevars = param_annotation_typevars(&function.parameters, &active); let mut finder = ValueUseFinder { diff --git a/crates/ty_python_semantic/src/types/diagnostic.rs b/crates/ty_python_semantic/src/types/diagnostic.rs index e7d32a5cb3..c8caaff0fe 100644 --- a/crates/ty_python_semantic/src/types/diagnostic.rs +++ b/crates/ty_python_semantic/src/types/diagnostic.rs @@ -2796,6 +2796,10 @@ declare_lint! { /// the target narrows back to the optional type, so nothing is lost there. /// The diagnostic surfaces at each use of the value as `object` instead. /// + /// Nor is an argument whose type the call tests — the first argument of + /// `isinstance`, or the one a `TypeIs` return narrows. The test reads the + /// `None` rather than losing it. + /// /// ## Examples /// ```by /// def sink(o: object): ... diff --git a/crates/ty_python_semantic/src/types/function.rs b/crates/ty_python_semantic/src/types/function.rs index ae1cf9c8b7..16d0e2cc35 100644 --- a/crates/ty_python_semantic/src/types/function.rs +++ b/crates/ty_python_semantic/src/types/function.rs @@ -692,6 +692,7 @@ impl<'db> OverloadLiteral<'db> { Some(BaseParameter { name: parameter.name()?.clone(), positional_only: parameter.is_positional_only(), + underscore: parameter.is_repeated_underscore(), }) }) .collect() @@ -3205,6 +3206,16 @@ fn check_classinfo_in_isinstance<'db>( diagnostic .set_primary_annotation_message("This call will raise `TypeError` at runtime"); } + Type::KnownInstance(KnownInstanceType::UnionType(_)) + if context.program_environment().python_version(db) < PythonVersion::PY310 + && !spelled_as_classes_by_the_lowering( + context, + call_expression, + classinfo_expr, + ) => + { + report_union_classinfo_before_python_310(context, call_expression, function); + } Type::KnownInstance(KnownInstanceType::UnionType(_)) => { report_invalid_union_type_elements( db, @@ -3239,6 +3250,57 @@ fn check_classinfo_in_isinstance<'db>( } } +/// whether the union `classinfo_expr` evaluates to is spelled as a tuple of classes where it +/// stands, by the lowering a basedpython module goes through before it runs +/// +/// below 3.10 a union object is no class, and `isinstance` rejects one. the transpiler writes +/// a union spelled in the call itself as the tuple it stands for, so +/// `isinstance(x, int | str)` runs as `isinstance(x, (int, str,))`, and the same for an optional +/// `int?`, alone or inside a tuple the call spells. that is only done for a call to `isinstance` +/// or `issubclass` by those names, and a union reached any other way, through a name for one, +/// is the `typing.Union` the transpiler spells everywhere else +fn spelled_as_classes_by_the_lowering( + context: &InferContext<'_, '_>, + call_expression: &ast::ExprCall, + classinfo_expr: Option<&ast::Expr>, +) -> bool { + context.file().source_type(context.db()) == ast::PySourceType::BasedPython + && matches!( + call_expression.func.as_ref(), + ast::Expr::Name(name) if matches!(name.id.as_str(), "isinstance" | "issubclass") + ) + && matches!( + classinfo_expr, + Some( + ast::Expr::BinOp(ast::ExprBinOp { + op: ast::Operator::BitOr, + .. + }) | ast::Expr::UnaryOp(ast::ExprUnaryOp { + op: ast::UnaryOp::Optional, + .. + }) + ) + ) +} + +/// report a union object passed to `isinstance()` / `issubclass()` on a python older than +/// 3.10, where it is not a class and the call raises +fn report_union_classinfo_before_python_310( + context: &InferContext<'_, '_>, + call_expression: &ast::ExprCall, + function: KnownFunction, +) { + let Some(builder) = context.report_lint(&INVALID_ARGUMENT_TYPE, call_expression) else { + return; + }; + let function_name: &str = function.into(); + let mut diagnostic = builder.into_diagnostic(format_args!( + "A union cannot be used with `{function_name}()` before Python 3.10" + )); + diagnostic.set_primary_annotation_message("This call will raise `TypeError` at runtime"); + diagnostic.info("Pass a tuple of classes instead"); +} + /// Report an error if a `types.UnionType` instance passed to `isinstance()`/`issubclass()` /// contains elements that are not class objects. fn report_invalid_union_type_elements<'db>( diff --git a/crates/ty_python_semantic/src/types/infer/builder.rs b/crates/ty_python_semantic/src/types/infer/builder.rs index 86b1f58c02..53a63f33bd 100644 --- a/crates/ty_python_semantic/src/types/infer/builder.rs +++ b/crates/ty_python_semantic/src/types/infer/builder.rs @@ -134,7 +134,9 @@ use crate::types::infer::{ use crate::types::match_pattern::{ClassPatternPositionalResult, class_pattern_positional_result}; use crate::types::match_type::literal_pattern_type; use crate::types::narrow::NarrowingEvaluatorExtension; -use crate::types::narrow::{pattern_subject_type, pattern_success_types}; +use crate::types::narrow::{ + argument_tested_by_known_function, pattern_subject_type, pattern_success_types, +}; use crate::types::newtype::NewType; use crate::types::receivers; use crate::types::regex; @@ -178,10 +180,10 @@ use crate::types::{ Parameters, ProgramEnvironment, PropertyDeprecations, RestrictedType, SentinelInstance, Signature, SpecialFormType, SubclassOfType, Type, TypeAliasType, TypeAndQualifiers, TypeContext, TypeQualifiers, TypeVarBoundOrConstraints, TypeVarKind, TypeVarVariance, - TypedDictType, TypingModule, UnionAccumulator, UnionBuilder, UnionType, any_over_type, - binding_type, extract_fixed_length_iterable_element_types, infer_complete_scope_types, - infer_scope_types, is_discarded_dict_key_assignment, report_iteration_over_character, - todo_type, + TypedDictType, TypingModule, UnionAccumulator, UnionBuilder, UnionType, UnionTypeInstance, + any_over_type, binding_type, extract_fixed_length_iterable_element_types, + infer_complete_scope_types, infer_scope_types, is_discarded_dict_key_assignment, + report_iteration_over_character, todo_type, }; use crate::{AnalysisSettings, Db, DisplaySettings, FxIndexSet, FxOrderSet, SemanticModel}; use fluid::FluidTimeline; @@ -11282,6 +11284,20 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { else { return; }; + // a call that tests the type of an argument reads the optional rather than discarding + // it: its result says whether the value was `None`, and narrows it when the call is a + // condition. `isinstance(x, int)` and a function returning `TypeIs` both take `object` + // for that reason + let tested = match bindings.return_type(db, env) { + Type::TypeIs(_) | Type::TypeGuard(_) => { + typeguard::narrowed_argument(db, bindings, call).map(ast::ArgOrKeyword::value) + } + _ => bindings + .single_element() + .and_then(|binding| binding.callable_type.as_function_literal()) + .and_then(|function| function.known(db)) + .and_then(|function| argument_tested_by_known_function(function, call)), + }; for (argument, parameter_type) in arguments.iter().zip(parameter_types) { let Some(parameter_type) = parameter_type else { continue; @@ -11290,6 +11306,9 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { continue; } let value = argument.value(); + if tested.is_some_and(|tested| std::ptr::eq(tested, value)) { + continue; + } let argument_type = self.expression_type(value); if !is_optional_value(db, argument_type) { continue; @@ -15914,6 +15933,44 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { } } + // basedpython: `T?` read as a value — `isinstance(x, int?)`, `Alias = int?` — is + // the object the transpiler writes for it, `T | None`, so it is that union. an + // optional over an optional, or over a type variable, is instead the runtime + // `Optional` wrapper the type expression reading of `?` describes, which is left + // to the arm below + (ast::UnaryOp::Optional, _) + if self.is_basedpython_file() + && !matches!( + &*unary.operand, + ast::Expr::UnaryOp(operand) if operand.op == ast::UnaryOp::Optional + ) + && matches!( + operand_type, + Type::ClassLiteral(..) + | Type::SubclassOf(..) + | Type::GenericAlias(..) + | Type::SpecialForm(_) + | Type::KnownInstance(_) + ) + && !match operand_type { + Type::KnownInstance(KnownInstanceType::TypeVar(typevar)) => { + !typevar.is_self(db) + } + Type::SubclassOf(subclass_of) => subclass_of + .into_type_var() + .is_some_and(|typevar| !typevar.typevar(db).is_self(db)), + _ => false, + } => + { + UnionTypeInstance::from_value_expression_types( + db, + [operand_type, Type::none(db, env)], + self.scope(), + self.typevar_binding_context, + self.inference_flags(), + ) + } + // basedpython postfix `?` and `^` / `!` on non-optional operands. // These are wrapped-type surface syntax that the transpiler lowers // away; ty does not yet model the remaining unwrap/propagate cases diff --git a/crates/ty_python_semantic/src/types/infer/builder/typeguard.rs b/crates/ty_python_semantic/src/types/infer/builder/typeguard.rs index 576980ae6c..a0280d6497 100644 --- a/crates/ty_python_semantic/src/types/infer/builder/typeguard.rs +++ b/crates/ty_python_semantic/src/types/infer/builder/typeguard.rs @@ -15,25 +15,6 @@ pub(super) fn bind_type_guard_return_type<'db>( bindings: &Bindings<'db>, call: &ast::ExprCall, ) -> Type<'db> { - let arguments = &call.arguments; - - let narrowed_argument_index = || { - bindings - .single_element() - .and_then(|binding| { - binding - .signature_type - .as_function_literal() - .or_else(|| binding.callable_type.as_function_literal()) - .map(|function| { - usize::from( - function.has_implicit_receiver(db) && binding.bound_type.is_none(), - ) - }) - }) - .unwrap_or(0) - }; - // basedpython: the annotation names the place it narrows, which is not always an // argument — see [`crate::types::narrowing_guards`] let guard = || { @@ -67,72 +48,7 @@ pub(super) fn bind_type_guard_return_type<'db>( } } - // Use the call binding to find the argument that maps to the parameter the guard - // narrows. This supports keyword arguments without falling back to a later - // parameter when the target is defaulted. - let matched_narrowed_argument_index = bindings.single_element().and_then(|binding| { - let has_implicit_receiver = binding - .signature_type - .as_function_literal() - .or_else(|| binding.callable_type.as_function_literal()) - .is_some_and(|function| function.has_implicit_receiver(db)); - let bound_argument_offset = usize::from(binding.bound_type.is_some()); - let narrowed_parameter_index = |overload: &Binding<'db>| { - let guards = overload.signature.all_narrowing_guards(db); - match guards.first() { - Some(guard) => match guard_root(guard, overload.signature.parameters(), call) { - GuardRoot::Parameter(index) => Some(index), - GuardRoot::Receiver(_) | GuardRoot::Scope => None, - }, - None => Some(usize::from( - bound_argument_offset > 0 || has_implicit_receiver, - )), - } - }; - let narrowed_argument_index = |overload: &Binding<'db>| { - let narrowed_parameter_index = narrowed_parameter_index(overload)?; - overload - .argument_matches() - .iter() - .enumerate() - .skip(bound_argument_offset) - .find_map(|(argument_index, matched_argument)| { - matched_argument - .parameters - .iter() - .any(|parameter| parameter.index == narrowed_parameter_index) - .then_some(argument_index - bound_argument_offset) - }) - }; - let mut matching_overloads = binding.matching_overloads(); - let (_, first_overload) = matching_overloads.next()?; - let first_argument_index = narrowed_argument_index(first_overload); - - Some( - if matching_overloads - .all(|(_, overload)| narrowed_argument_index(overload) == first_argument_index) - { - first_argument_index - } else { - None - }, - ) - }); - - let argument = match matched_narrowed_argument_index { - Some(Some(argument_index)) => arguments.iter_source_order().nth(argument_index), - // The target parameter was omitted, so there is no expression to narrow. - Some(None) => return None, - // Preserve positional behavior when there isn't a unique callable binding whose - // parameter mapping we can use. - None => arguments - .args - .get(narrowed_argument_index()) - .map(ast::ArgOrKeyword::from), - }?; - if argument.is_variadic() { - return None; - } + let argument = narrowed_argument(db, bindings, call)?; match guard() { // the guard may name a member of the argument: `-> x.data is str` @@ -156,3 +72,98 @@ pub(super) fn bind_type_guard_return_type<'db>( _ => return_ty, } } + +/// the argument of `call` a `TypeIs` or `TypeGuard` its callee returns narrows: the one bound to +/// the parameter the guard names, or to the first parameter +pub(super) fn narrowed_argument<'a, 'db>( + db: &'db dyn Db, + bindings: &Bindings<'db>, + call: &'a ast::ExprCall, +) -> Option> { + let arguments = &call.arguments; + + let narrowed_argument_index = || { + bindings + .single_element() + .and_then(|binding| { + binding + .signature_type + .as_function_literal() + .or_else(|| binding.callable_type.as_function_literal()) + .map(|function| { + usize::from( + function.has_implicit_receiver(db) && binding.bound_type.is_none(), + ) + }) + }) + .unwrap_or(0) + }; + + // Use the call binding to find the argument that maps to the parameter the guard + // narrows. This supports keyword arguments without falling back to a later + // parameter when the target is defaulted. + let matched_narrowed_argument_index = bindings.single_element().and_then(|binding| { + let has_implicit_receiver = binding + .signature_type + .as_function_literal() + .or_else(|| binding.callable_type.as_function_literal()) + .is_some_and(|function| function.has_implicit_receiver(db)); + let bound_argument_offset = usize::from(binding.bound_type.is_some()); + let narrowed_parameter_index = |overload: &Binding<'db>| { + let guards = overload.signature.all_narrowing_guards(db); + match guards.first() { + Some(guard) => match guard_root(guard, overload.signature.parameters(), call) { + GuardRoot::Parameter(index) => Some(index), + GuardRoot::Receiver(_) | GuardRoot::Scope => None, + }, + None => Some(usize::from( + bound_argument_offset > 0 || has_implicit_receiver, + )), + } + }; + let narrowed_argument_index = |overload: &Binding<'db>| { + let narrowed_parameter_index = narrowed_parameter_index(overload)?; + overload + .argument_matches() + .iter() + .enumerate() + .skip(bound_argument_offset) + .find_map(|(argument_index, matched_argument)| { + matched_argument + .parameters + .iter() + .any(|parameter| parameter.index == narrowed_parameter_index) + .then_some(argument_index - bound_argument_offset) + }) + }; + let mut matching_overloads = binding.matching_overloads(); + let (_, first_overload) = matching_overloads.next()?; + let first_argument_index = narrowed_argument_index(first_overload); + + Some( + if matching_overloads + .all(|(_, overload)| narrowed_argument_index(overload) == first_argument_index) + { + first_argument_index + } else { + None + }, + ) + }); + + let argument = match matched_narrowed_argument_index { + Some(Some(argument_index)) => arguments.iter_source_order().nth(argument_index), + // The target parameter was omitted, so there is no expression to narrow. + Some(None) => return None, + // Preserve positional behavior when there isn't a unique callable binding whose + // parameter mapping we can use. + None => arguments + .args + .get(narrowed_argument_index()) + .map(ast::ArgOrKeyword::from), + }?; + if argument.is_variadic() { + return None; + } + Some(argument) +} diff --git a/crates/ty_python_semantic/src/types/narrow.rs b/crates/ty_python_semantic/src/types/narrow.rs index f91cc7a13e..170910c1b8 100644 --- a/crates/ty_python_semantic/src/types/narrow.rs +++ b/crates/ty_python_semantic/src/types/narrow.rs @@ -637,6 +637,22 @@ pub(crate) fn pattern_success_types<'db>( } } +/// the argument a call to `function` tests the type of, and narrows when the call is a condition: +/// the first of `isinstance(x, int)`, `issubclass(t, int)` and `hasattr(x, "a")` +pub(crate) fn argument_tested_by_known_function( + function: KnownFunction, + call: &ast::ExprCall, +) -> Option<&ast::Expr> { + if !call.arguments.keywords.is_empty() { + return None; + } + let [tested, _] = &*call.arguments.args else { + return None; + }; + (function == KnownFunction::HasAttr || function.into_classinfo_constraint_function().is_some()) + .then_some(tested) +} + /// Functions that can be used to narrow the type of a first argument using a "classinfo" second argument. /// /// A "classinfo" argument is either a class or a tuple of classes, or a tuple of tuples of classes @@ -4802,12 +4818,13 @@ impl<'db> NarrowingConstraintsBuilder<'db, '_> { None } } - Type::FunctionLiteral(function_type) if expr_call.arguments.keywords.is_empty() => { - let [first_arg, second_arg] = &*expr_call.arguments.args else { + Type::FunctionLiteral(function_type) => { + let function = function_type.known(db)?; + let first_arg = argument_tested_by_known_function(function, expr_call)?; + let [_, second_arg] = &*expr_call.arguments.args else { return None; }; let first_arg = PlaceExpr::try_from_expr(first_arg)?; - let function = function_type.known(db)?; let place = self.expect_place(&first_arg); if function == KnownFunction::HasAttr { diff --git a/crates/ty_python_semantic/src/types/repeated_underscore.rs b/crates/ty_python_semantic/src/types/repeated_underscore.rs index 0a9fab5a4c..0b5847e8f2 100644 --- a/crates/ty_python_semantic/src/types/repeated_underscore.rs +++ b/crates/ty_python_semantic/src/types/repeated_underscore.rs @@ -215,18 +215,21 @@ pub fn callable_parameter_slots( } /// a positional parameter of the method a definition overrides -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq, Hash, get_size2::GetSize)] pub struct BaseParameter { pub name: Name, pub positional_only: bool, + /// the parameter is one of the base's own repeated `_`s, so `name` is the lowering's + /// rather than one its author wrote + pub underscore: bool, } /// how the lowering writes a parameter list that repeats `_` #[derive(Clone, Debug, PartialEq, Eq, Hash, get_size2::GetSize, salsa::SalsaValue)] pub struct UnderscoreLowering { - /// the name each positional `_` takes from the overridden method, in declaration order. - /// `None` when they are numbered instead - inherited: Option>, + /// the parameter of the overridden method each positional `_` takes its name from, in + /// declaration order. `None` when they are numbered instead + inherited: Option>, /// how many of the leading positional parameters a call reaches by position alone positional_only: usize, } @@ -237,9 +240,8 @@ impl UnderscoreLowering { self.positional_only } - /// whether the positional `_`s take their names from the overridden method. no - /// parameter is called `_` in the python then, so a read of `_` in the body finds - /// nothing of the definition's own + /// whether the positional `_`s take their names from the overridden method. the first + /// of them is called `_` in the python then only where the base calls its parameter `_` pub fn is_inherited(&self) -> bool { self.inherited.is_some() } @@ -256,14 +258,14 @@ impl UnderscoreLowering { .iter_mut() .filter(|(slot, name)| slot.is_positional() && name == "_"); for ((_, name), base) in underscores.zip(inherited) { - name.clone_from(base); + name.clone_from(&base.name); } } let spelled: Vec<(ParameterSlot, &str)> = spelled .iter() .map(|(slot, name)| (*slot, name.as_str())) .collect(); - number_underscores(&spelled, !self.is_inherited(), written) + number_underscores(&spelled, self.is_inherited(), written) } } @@ -287,10 +289,17 @@ impl LoweredParameters { slots: &[(ParameterSlot, &str)], written: WrittenNames, ) -> Self { - // a name taken from an overridden method is a name the author can spell + // a name taken from an overridden method is a name the author can spell, unless the + // base has it from a repeated `_` of its own + let mut inherited = lowering.inherited.iter().flatten(); let underscores = slots .iter() - .map(|(_, spelled)| *spelled == "_" && !lowering.is_inherited()) + .map(|(slot, spelled)| { + *spelled == "_" + && (!lowering.is_inherited() + || !slot.is_positional() + || inherited.next().is_some_and(|base| base.underscore)) + }) .collect(); Self { names: lowering.names(slots, written), @@ -454,10 +463,15 @@ pub fn lower_repeated_underscores( { return Some(Err(UnderscoreRefusal::KeywordOnly { index })); } - if let Some((index, parameter)) = inherited - .iter() - .find(|(_, parameter)| reads_from_enclosing_scope(¶meter.name)) - { + // a name the list already gives a parameter is read from the definition itself, so a + // parameter taking it shadows nothing. only `_` can be one: an override of a method + // whose own `_`s are numbered takes `_` for the first of them + if let Some((index, parameter)) = inherited.iter().find(|(_, parameter)| { + !slots + .iter() + .any(|(_, spelled)| *spelled == parameter.name.as_str()) + && reads_from_enclosing_scope(¶meter.name) + }) { return Some(Err(UnderscoreRefusal::ShadowsEnclosingName { index: *index, name: parameter.name.clone(), @@ -477,7 +491,7 @@ pub fn lower_repeated_underscores( inherited: Some( inherited .into_iter() - .map(|(_, parameter)| parameter.name.clone()) + .map(|(_, parameter)| parameter.clone()) .collect(), ), positional_only, @@ -582,13 +596,15 @@ pub fn lowered_parameter_names( parameters: &[(ParameterSlot, &str)], written: WrittenNames, ) -> Vec { - number_underscores(parameters, true, written) + number_underscores(parameters, false, written) } -/// `parameters` with every `_` numbered, bar the first in numbering order when `keep_first` +/// `parameters` with every `_` numbered but the ones that keep the name: the first in +/// numbering order, or when the positional ones have taken the names of an overridden +/// method, each positional one still called `_`, which is a `_` the base calls `_` too fn number_underscores( parameters: &[(ParameterSlot, &str)], - keep_first: bool, + inherited: bool, written: WrittenNames, ) -> Vec { let taken: HashSet<&str> = parameters.iter().map(|(_, name)| *name).collect(); @@ -598,11 +614,15 @@ fn number_underscores( .iter() .map(|(_, name)| Name::from(*name)) .collect(); - let mut keep = keep_first; + let mut keep_first = !inherited; // candidates only ever count up, so one never repeats an earlier one let mut next = 2u32; for index in order { - if parameters[index].1 != "_" || std::mem::replace(&mut keep, false) { + let (slot, name) = parameters[index]; + if name != "_" + || (inherited && slot.is_positional()) + || std::mem::replace(&mut keep_first, false) + { continue; } let candidate = loop { @@ -638,6 +658,7 @@ mod tests { Some(BaseParameter { name: Name::from(*name), positional_only: *positional_only, + underscore: false, }) }) .collect() @@ -773,6 +794,37 @@ mod tests { assert!(!lowering.is_inherited()); } + /// an override of a method whose own `_`s are numbered takes `_` for the first of them, + /// and the body's `_` is already one of the list's parameters + #[test] + fn an_inherited_underscore_shadows_nothing() { + let slots = [(PositionalOrKeyword, "_"), (PositionalOrKeyword, "_")]; + let base: Vec<_> = base(&[("_", true), ("_2", true)]) + .into_iter() + .map(|parameter| { + parameter.map(|parameter| BaseParameter { + underscore: true, + ..parameter + }) + }) + .collect(); + let lowering = + lower_repeated_underscores(&slots, false, Some(&base), |name| name == "_", true) + .unwrap() + .unwrap(); + assert!(lowering.is_inherited()); + assert_eq!(lowering.positional_only(), 2); + let variadic = [ + (PositionalOrKeyword, "_"), + (PositionalOrKeyword, "_"), + (Variadic, "_"), + ]; + assert_eq!( + lowering.names(&variadic, WrittenNames::new("")), + [Name::from("_"), Name::from("_2"), Name::from("_3")] + ); + } + #[test] fn an_inherited_name_the_body_reads_is_refused() { let slots = [(PositionalOrKeyword, "_"), (PositionalOrKeyword, "_")]; diff --git a/docs/basedpython/development/how-transpilation-works.md b/docs/basedpython/development/how-transpilation-works.md index fef6a28959..3e639c6226 100644 --- a/docs/basedpython/development/how-transpilation-works.md +++ b/docs/basedpython/development/how-transpilation-works.md @@ -65,7 +65,13 @@ source (.by) version cannot parse. the first check asks whether the output is python at all; this one asks whether it is python the declared floor can run, so syntax no polyfill covers is a diagnostic instead of a - `SyntaxError` at import time in generated code + `SyntaxError` at import time in generated code. which kinds of + syntax a lowering removes is stated once, by + `UnsupportedSyntaxErrorKind::is_lowered_by_basedpython`; the parser + holds a `.by` file to its target for every other kind, so a checker + reports what this check would refuse. a basedpython form written as + syntax of its own is a kind in that list too: a destructuring + pattern is written as a `match`, and is held to where one runs ``` entry points in `crates/by_transforms/src/lib.rs`: diff --git a/docs/basedpython/development/sourcemaps.md b/docs/basedpython/development/sourcemaps.md index f8f464c6ed..7b4b2f98b9 100644 --- a/docs/basedpython/development/sourcemaps.md +++ b/docs/basedpython/development/sourcemaps.md @@ -97,9 +97,13 @@ rewritten since is a false one would compile the `.py`, so the shim compiles it instead, under the `.by` path and with each line the `.by` line it came from (a generated line is line 0). every location the running program reads back — a traceback it formats, the place a -warning is attributed to, `co_filename`, `inspect` — is then the author's. the -digests are checked then, and a mismatch leaves the module to python's loader. a -`.by` saved over during the run is caught when a frame is reported: the frame +warning is attributed to, `co_filename`, `inspect` — is then the author's, and so is +the line of a warning python raises while compiling it, and of the `SyntaxError` it raises +instead when the warning filters make that warning an error (`-W error`). the `.py` is compiled from its +text, as python's loader would, and its code objects are then moved to the `.by` lines: +python 3.9 to 3.11 refuse a syntax tree nested deeper than their recursion limit, even +one whose source they compile. the digests are checked then, and a mismatch leaves +the module to python's loader. a `.by` saved over during the run is caught when a frame is reported: the frame names the one generated line that `.by` line was written as, or no line when there was more than one diff --git a/docs/basedpython/features/index.md b/docs/basedpython/features/index.md index 806039e161..887aca9fd0 100644 --- a/docs/basedpython/features/index.md +++ b/docs/basedpython/features/index.md @@ -123,6 +123,7 @@ type parameters — their bounds, their variance, and what survives to runtime
- [generics](generics.md) +- [`some` parameters](some-parameters.md) - [type mappings](type-mappings.md) - [type parameter bound ranges](bound-ranges.md) - [bounds on a variadic pack](pack-bounds.md) diff --git a/docs/basedpython/features/polyfills.md b/docs/basedpython/features/polyfills.md index d35668adb2..075df20a32 100644 --- a/docs/basedpython/features/polyfills.md +++ b/docs/basedpython/features/polyfills.md @@ -111,6 +111,23 @@ ______________________________________________________________________ `TypeVar` with a `default=` argument requires Python 3.13+. basedpython imports `TypeVar` from `typing_extensions` instead (which supports `default=`). this applies when using PEP 695 generic syntax with a default (see the [generics polyfill](#generic-classes-and-functions-pep-695) below) +the same goes for `ParamSpec` and `TypeVarTuple`. a keyword argument can't be starred, so a variadic's default is passed unpacked with `Unpack`: + +```python +# basedpython source +class Row[T = str, *Ts = *tuple[int, str]]: ... +``` + +```python +# generated Python +_T = TypeVar("_T", default=str) # from typing_extensions +_Ts = TypeVarTuple("_Ts", default=Unpack[tuple[int, str]]) # from typing_extensions + +class Row(Generic[_T, Unpack[_Ts]]): ... +``` + +a module that binds `TypeVar` itself — `from typing import TypeVar` binds the one without `default=` — keeps its own, and the polyfill's is imported under a name the module does not spell, `TypeVar2`. the same goes for every name the polyfill calls: `Generic`, `ParamSpec`, `TypeVarTuple`, `Unpack` and `TypeAliasType` + the `[T = int]` header syntax is itself 3.13+: on a 3.12 target, a declaration with a defaulted type parameter is desugared by the generics polyfill while declarations without defaults keep the native syntax. a [reified](reified-generics.md) function can't be desugared, so a defaulted reified function on a 3.12 target is a transpile error ### `typing.TypeIs` (PEP 742) @@ -362,6 +379,9 @@ the lowering binds with assignment expressions, so it needs python 3.8. below th Alias = int | str isinstance(x, int | None) cast(int | str, value) +Rows = list[int | None] +class Items(list[int | str]): ... +def f[T: int | str](t: T) -> T: ... ``` ```python @@ -369,11 +389,23 @@ cast(int | str, value) Alias = Union[int, str] isinstance(x, (int, type(None),)) cast(Union[int, str], value) +Rows = list[Union[int, None]] +class Items(list[Union[int, str]]): ... +_T = TypeVar("_T", bound=Union[int, str]) +def f(t: _T) -> _T: ... ``` the two spellings are not interchangeable — `isinstance` takes a tuple of classes and rejects a `typing.Union` — so the classinfo argument of `isinstance` and `issubclass` becomes a tuple and everything else a `Union`. whether a `|` is a union at all is asked of the checker rather than guessed from the shape, so an ordinary bitwise or is left alone -[`T?`](wrapped-results.md) is spelled `Union[T, None]` on these targets for the same reason +a type the runtime evaluates is spelled this way at any depth: a type written as a value, a class base, a `cast` target, and a type parameter's bound, default or alias value, which the [generics polyfill](#generic-classes-and-functions-pep-695) turns into call arguments + +[`T?`](wrapped-results.md) and `A or B` are spelled `Union[T, None]` and `Union[A, B]` on these targets, in annotations too: a union basedpython spells itself is one `typing.get_type_hints` can evaluate. a `|` written in an annotation is kept as written. `T?` is a union where `isinstance` expects classes too, so there it is the tuple `(T, type(None),)`, alone or inside a union or tuple of classes + +a union given a name is spelled `Union[...]`, since it may be read as a type, so it cannot then be handed to `isinstance`: `Alias = int | str` followed by `isinstance(x, Alias)` raises `TypeError` below 3.10, and `by check` reports it. write the union, or the tuple, where `isinstance` is called + +### `typing.ParamSpec`, `Concatenate`, `TypeAlias`, `TypeGuard` (PEP 612, 613, 647) + +redirected to `typing_extensions`, as is the `ParamSpec` the [generics polyfill](#generic-classes-and-functions-pep-695) declares for a `**P` ______________________________________________________________________ @@ -389,7 +421,7 @@ python 3.12 introduced compact generic syntax. basedpython rewrites it using `ty | `def f[T](x: T) -> T: ...` | `def f(x: T) -> T: ...` | | `type Point = tuple[float, float]` | `Point: TypeAlias = tuple[float, float]` | -each type parameter becomes a module-level `TypeVar` with a mangled name (`_T`, `_K`, etc) +each type parameter becomes a `TypeVar` with a mangled name (`_T`, `_K`, etc), declared just above the generic, in the same block. the name is never one the module spells: in a module that binds `_T` itself, `T` is declared as `_T2`, so the module's own `_T` is left alone. a parameter declared again with different arguments, or in a different block — a method's type parameter, declared in its class body, and a function's of the same name further down — gets a numbered name of its own, `_T_1` ```python # python source @@ -488,6 +520,8 @@ it silently redirects to `typing_extensions` | `reveal_type`, `assert_type` | 3.11 | `typing_extensions` | | `override` | 3.12 | `typing_extensions` | | `TypeVar(default=...)` | 3.13 | `typing_extensions` | +| `ParamSpec(default=...)` | 3.13 | `typing_extensions` | +| `TypeVarTuple(default=...)` | 3.13 | `typing_extensions` | | `TypeIs` | 3.13 | `typing_extensions` | | `ReadOnly` | 3.13 | `typing_extensions` | | `deprecated` (warnings) | 3.13 | `typing_extensions` | diff --git a/docs/basedpython/features/repeated-underscore.md b/docs/basedpython/features/repeated-underscore.md index 551b88114b..e83b002ec7 100644 --- a/docs/basedpython/features/repeated-underscore.md +++ b/docs/basedpython/features/repeated-underscore.md @@ -122,6 +122,10 @@ a base parameter that is positional-only stays so, and a default the base gives a parameter comes with it, as it does for any override (see [inherited default values](inherited-defaults.md)) +a base that repeats `_` itself gives the override its own numbered names, the +first still `_`, and a base that is itself such an override gives the names it +took + where the base has no parameter at some `_`'s position, none of the `_`s take a name from it and they are numbered as above. that override is reported as an `invalid-method-override` anyway diff --git a/docs/basedpython/features/some-parameters.md b/docs/basedpython/features/some-parameters.md new file mode 100644 index 0000000000..7cecafea45 --- /dev/null +++ b/docs/basedpython/features/some-parameters.md @@ -0,0 +1,73 @@ +# `some` parameters + +`some T` in a parameter's annotation gives that parameter a type parameter of its own, bounded by +`T` and named after the parameter. the rest of the signature names that type by the parameter's +name: + +```by +def echo(s: some str) -> s: + return s + +echo("hi") # "hi" +``` + +transpiles to: + +```python +from typing import TypeVar +_s = TypeVar("_s", bound=str) +def echo(s: _s) -> _s: + return s +``` + +`echo` accepts any `str` and returns the type it was called with — a literal stays a literal, a +subclass stays the subclass. it is `def echo[S: str](s: S) -> S` without the type parameter list + +## what it means + +- each `some` parameter is a type parameter of its own, solved from its argument at the call: + + ```by + def pair(a: some int, b: some str) -> tuple[a, b]: + return (a, b) + + pair(1, "x") # (1, "x") + ``` + +- an argument outside the bound is an `invalid-argument-type` error, as it is for any bounded type + parameter + +- the bound is any type expression, a union or an optional among them: `n: some int?` + +- in the body, the parameter's name is the parameter. `return s` returns the argument; only the + signature reads `s` as a type + +- `some` is written at the top of a parameter's annotation. it is not a type of its own, and cannot + be nested inside one (`list[some int]`), written as a return type or on a variable. anywhere else + it is a syntax error: "`some` is only allowed at the top of a parameter annotation" + +- `some` is not written on `*args` or `**kwargs`. their annotation is the type of each element, while + the parameter is a tuple or a dict of them, so the parameter's name could not name the type. a + type parameter in a list says which is meant: `def first[T: int](*xs: T) -> T` + +## on older pythons + +the type parameter is always declared as a `TypeVar`, whatever the target — native type parameter +syntax has nowhere to put one the source never wrote in a list. a bound python cannot evaluate on +the target is spelled the way it can: `some int?` is `bound=Union[int, None]` below 3.10 + +## in inferred signatures + +an unannotated parameter's inferred type is shown in the same notation, since it is the same thing: a +type parameter bounded by what the body needs. see [sound types](sound-types.md) + +```by +def twice(n = 1): + return n * 2 + +reveal_type(twice) # def twice(n: some int = 1) -> n * 2 +``` + +## see also + +- [generics](generics.md) — declaring type parameters in a list diff --git a/ty.schema.json b/ty.schema.json index db22a28089..9c4066f38d 100644 --- a/ty.schema.json +++ b/ty.schema.json @@ -2093,7 +2093,7 @@ }, "optional-object-conversion": { "title": "detects an optional value implicitly widened to `object`", - "description": "## What it does\nChecks for an optional value passed where `object` is expected, silently\ndiscarding a layer of optionality.\n\n## Why is this bad?\n`object` swallows both the present value and `None`, so consuming an\noptional as `object` loses the information that the value could be\nabsent. It is almost always unintended: either the value should be\nunwrapped first with `!`, or the widening should be made explicit with\n`cast object`.\n\nAssigning an optional to a declared `object` variable is *not* flagged —\nthe target narrows back to the optional type, so nothing is lost there.\nThe diagnostic surfaces at each use of the value as `object` instead.\n\n## Examples\n```by\ndef sink(o: object): ...\n\ndef f(x: int?):\n sink(x) # warning: `int | None` widened to `object`\n sink(x!) # ok — unwrapped first\n sink(x cast object) # ok — explicit\n```", + "description": "## What it does\nChecks for an optional value passed where `object` is expected, silently\ndiscarding a layer of optionality.\n\n## Why is this bad?\n`object` swallows both the present value and `None`, so consuming an\noptional as `object` loses the information that the value could be\nabsent. It is almost always unintended: either the value should be\nunwrapped first with `!`, or the widening should be made explicit with\n`cast object`.\n\nAssigning an optional to a declared `object` variable is *not* flagged —\nthe target narrows back to the optional type, so nothing is lost there.\nThe diagnostic surfaces at each use of the value as `object` instead.\n\nNor is an argument whose type the call tests — the first argument of\n`isinstance`, or the one a `TypeIs` return narrows. The test reads the\n`None` rather than losing it.\n\n## Examples\n```by\ndef sink(o: object): ...\n\ndef f(x: int?):\n sink(x) # warning: `int | None` widened to `object`\n sink(x!) # ok — unwrapped first\n sink(x cast object) # ok — explicit\n```", "default": "warn", "oneOf": [ { diff --git a/zensical.toml b/zensical.toml index 807c233347..423102f51e 100644 --- a/zensical.toml +++ b/zensical.toml @@ -139,6 +139,7 @@ features = [ ] }, { "generics" = [ { "generics" = "features/generics.md" }, + { "some parameters" = "features/some-parameters.md" }, { "type mappings" = "features/type-mappings.md" }, { "type parameter bound ranges" = "features/bound-ranges.md" }, { "bounds on a variadic pack" = "features/pack-bounds.md" }, From d4676f00a1a0433c66e57d27f9142e161171b975 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Thu, 24 Sep 2026 07:17:58 +1000 Subject: [PATCH 13/16] transpiler: write every lowering-spelled name through one fresh-name facility typing names, builtins, runtime helper names and module-level private symbols are imported or declared under names the module does not bind; one decision names each polyfilled type parameter for the polyfill, inline protocols and anonymous named tuples; the isinstance classinfo decision is one query ty and the transpiler share; method type variables are declared at module scope; a data class gets its slots below 3.10; keyword-pack defaults follow PEP 696 --- .config/nextest.toml | 11 +- crates/by_transforms/src/lib.rs | 77 +- crates/by_transforms/src/runtime.rs | 1 + .../by_transforms/src/runtime/_by_runtime.py | 96 ++ .../src/transforms/annotation.rs | 58 +- .../src/transforms/anon_named_tuple.rs | 123 +- .../src/transforms/ast_driver.rs | 102 +- .../by_transforms/src/transforms/callable.rs | 98 +- .../src/transforms/checked_cast.rs | 34 +- .../src/transforms/context_params.rs | 10 +- .../src/transforms/decorated_binding.rs | 17 +- .../src/transforms/decorator_keyword.rs | 43 +- .../src/transforms/dynamic_keyword.rs | 28 +- crates/by_transforms/src/transforms/enums.rs | 72 +- .../by_transforms/src/transforms/extension.rs | 55 +- .../by_transforms/src/transforms/generics.rs | 1141 +++++++++++++---- .../src/transforms/grapheme_string.rs | 19 +- .../src/transforms/implicit_receiver.rs | 22 +- .../src/transforms/inferred_annotation.rs | 73 +- .../src/transforms/init_method.rs | 4 +- .../src/transforms/just_float.rs | 42 +- .../src/transforms/kw_subscript.rs | 29 +- .../src/transforms/literal_string.rs | 18 +- .../src/transforms/literal_types.rs | 39 +- .../src/transforms/main_function.rs | 8 +- .../src/transforms/match_polyfill.rs | 36 +- .../src/transforms/match_type.rs | 11 +- .../by_transforms/src/transforms/modifiers.rs | 247 +++- .../src/transforms/mutable_defaults.rs | 27 +- .../src/transforms/optional_type.rs | 34 +- .../by_transforms/src/transforms/overload.rs | 69 +- .../src/transforms/parametric_is.rs | 58 +- .../by_transforms/src/transforms/propagate.rs | 82 +- .../src/transforms/properties.rs | 46 +- .../src/transforms/protocol_type.rs | 121 +- .../src/transforms/repeated_underscore.rs | 454 ++++++- .../src/transforms/runtime_union.rs | 171 ++- .../by_transforms/src/transforms/sentinel.rs | 7 +- .../src/transforms/symbolic_type_op.rs | 208 ++- .../src/transforms/template_type.rs | 19 +- .../by_transforms/src/transforms/top_star.rs | 44 +- .../by_transforms/src/transforms/type_is.rs | 18 +- .../src/transforms/typed_dict_literal.rs | 82 +- .../src/transforms/typeof_keyword.rs | 7 +- crates/by_transforms/src/transforms/unpack.rs | 24 +- .../src/transforms/visibility_rename.rs | 18 +- crates/by_transforms/src/type_info.rs | 174 ++- .../tests/builtin_names_runtime.rs | 91 ++ .../by_transforms/tests/conversion_runtime.rs | 73 +- .../by_transforms/tests/data_class_runtime.rs | 92 ++ .../tests/fstring_debug_runtime.rs | 31 +- .../by_transforms/tests/grapheme_runtime.rs | 43 +- .../by_transforms/tests/interpreters/mod.rs | 116 ++ .../tests/lazy_import_runtime.rs | 58 +- .../by_transforms/tests/polyfill_runtime.rs | 282 ++-- .../tests/tstring_expression_runtime.rs | 36 +- .../by_transforms/tests/visibility_runtime.rs | 89 ++ .../mdtest/basedpython_keyword_variadic.md | 61 + .../resources/mdtest/narrow/isinstance.md | 23 + crates/ty_python_semantic/src/types.rs | 1 + .../src/types/classinfo_spelling.rs | 88 ++ .../ty_python_semantic/src/types/display.rs | 76 +- .../ty_python_semantic/src/types/function.rs | 31 +- .../src/types/ide_support.rs | 16 + .../src/types/infer/builder.rs | 2 +- .../src/types/infer/builder/typevar.rs | 43 +- .../src/types/repeated_underscore.rs | 10 +- .../development/how-transpilation-works.md | 40 + docs/basedpython/features/inline-protocol.md | 4 +- docs/basedpython/features/keyword-variadic.md | 20 +- docs/basedpython/features/modifiers.md | 11 +- docs/basedpython/features/polyfills.md | 6 +- 72 files changed, 4122 insertions(+), 1398 deletions(-) create mode 100644 crates/by_transforms/tests/builtin_names_runtime.rs create mode 100644 crates/by_transforms/tests/data_class_runtime.rs create mode 100644 crates/by_transforms/tests/interpreters/mod.rs create mode 100644 crates/ty_python_semantic/src/types/classinfo_spelling.rs diff --git a/.config/nextest.toml b/.config/nextest.toml index 94d5783739..0a9dd34bdc 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -7,12 +7,13 @@ serial = { max-threads = 1 } filter = 'binary(file_watching)' test-group = 'serial' -# the polyfill runtime tests run their output on an old interpreter with -# `typing_extensions`, which a machine often does not have. each says which -# interpreter it ran on, or that it found none and ran nothing, and a passing -# test's output is otherwise hidden, so a skip would read as a pass +# the runtime tests that need something of their interpreter — an old version +# with `typing_extensions`, a new one, the `regex` package — run on one found to +# have it, which a machine often does not. each says which interpreter it ran +# on, or that it found none and ran nothing, and a passing test's output is +# otherwise hidden, so a skip would read as a pass [[profile.default.overrides]] -filter = 'binary(polyfill_runtime)' +filter = 'binary(polyfill_runtime) | binary(grapheme_runtime) | binary(conversion_runtime) | binary(lazy_import_runtime) | binary(fstring_debug_runtime) | binary(tstring_expression_runtime) | binary(visibility_runtime) | binary(data_class_runtime) | binary(builtin_names_runtime)' success-output = "final" [profile.ci] diff --git a/crates/by_transforms/src/lib.rs b/crates/by_transforms/src/lib.rs index 65cd4239c2..37a9ef01da 100644 --- a/crates/by_transforms/src/lib.rs +++ b/crates/by_transforms/src/lib.rs @@ -20,6 +20,8 @@ pub mod entry_point { }; } +use transforms::repeated_underscore::{CodeNames, PrivateNames}; + use std::collections::{BTreeSet, HashSet}; use ruff_db::files::{File, system_path_to_file}; @@ -153,24 +155,24 @@ fn transpile_with_report( return Ok((source.to_owned(), RuntimeRequirements::default())); } - // the names the module spells as written, which a repeated `_` parameter is - // numbered around. read before any rewrite, as the native compiler reads them - let written_names = WrittenNames::new(source); - // one db over the original source, shared by the qualification phase below // and — as long as nothing rewrites the source — by phase 0's type-aware // passes, which would otherwise build an identical one of their own let (local_db, local_file) = make_in_memory_db(source); + let parsed = ruff_db::parsed::parsed_module( + &local_db, + ty_python_semantic::Db::program_file(&local_db, local_file).python_file(&local_db), + ) + .load(&local_db); // what the author wrote, before any rewrite, so phase 3 can tell a helper // call the transpiler emitted from a name the program reads itself - let author = AuthorNames::new( - ruff_db::parsed::parsed_module( - &local_db, - ty_python_semantic::Db::program_file(&local_db, local_file).python_file(&local_db), - ) - .load(&local_db) - .suite(), - ); + let author = AuthorNames::new(parsed.suite()); + // the names the module spells as written, which a repeated `_` parameter is + // numbered around and a lowering's typing name kept clear of. read before any + // rewrite, as the native compiler reads them + let code_names = CodeNames::of(parsed.suite()); + let written_names = WrittenNames::new(source).with_code(&code_names); + let as_written = source; // --- Erased-union reification: give a `list[int] | list[str]` parameter a // reified type parameter, while the source is still the one ty checks --- @@ -200,6 +202,13 @@ fn transpile_with_report( return Err(first.clone()); } let source = enum_lowered.output.as_ref(); + let private_names = PrivateNames::decide( + ty_python_semantic::private_symbols(&local_db, local_file) + .iter() + .map(ruff_python_ast::name::Name::as_str), + &[as_written, source], + ); + let written_names = written_names.with_private(&private_names); // --- Phase 0: AST rewrite passes --- let unchanged = !reified_changed @@ -352,10 +361,6 @@ pub fn transpile_typed_with_report( }; let source_ref = ruff_db::source::source_text(db, file); let original_source = source_ref.as_str(); - // the names the module spells as written, which a repeated `_` parameter is - // numbered around. read before any rewrite, as the native compiler reads them - let written_names = WrittenNames::new(original_source); - if config.is_python { let out = original_source.to_owned(); return Ok(( @@ -365,14 +370,17 @@ pub fn transpile_typed_with_report( )); } - let author = AuthorNames::new( - ruff_db::parsed::parsed_module( - db, - ty_python_semantic::Db::program_file(db, file).python_file(db), - ) - .load(db) - .suite(), - ); + let parsed = ruff_db::parsed::parsed_module( + db, + ty_python_semantic::Db::program_file(db, file).python_file(db), + ) + .load(db); + let author = AuthorNames::new(parsed.suite()); + // the names the module spells as written, which a repeated `_` parameter is + // numbered around and a lowering's typing name kept clear of. read before any + // rewrite, as the native compiler reads them + let code_names = CodeNames::of(parsed.suite()); + let written_names = WrittenNames::new(original_source).with_code(&code_names); // erased-union reification: give a `list[int] | list[str]` parameter a // reified type parameter, against the source ty checks. edits stay inside @@ -405,6 +413,13 @@ pub fn transpile_typed_with_report( return Err(first.clone().into()); } let working_source = enum_lowered.output.as_ref(); + let private_names = PrivateNames::decide( + ty_python_semantic::private_symbols(db, file) + .iter() + .map(ruff_python_ast::name::Name::as_str), + &[original_source, working_source], + ); + let written_names = written_names.with_private(&private_names); // two independent facts, deliberately kept apart: `enum_changed` says the // enum phase *renumbered lines*, so the final map must compose through its // line map (which is empty when it didn't fire, and would map every line to @@ -572,8 +587,13 @@ fn run_anon_named_tuple_cleanup( ty_python_semantic::Db::program_file(&db, file), ); - let mut anon = - transforms::anon_named_tuple::AnonNamedTuple::new(src, written, &model, config.clone()); + let mut anon = transforms::anon_named_tuple::AnonNamedTuple::new( + src, + written, + &model, + config.clone(), + module.suite(), + ); for stmt in module.suite() { anon.visit_stmt(stmt); } @@ -605,7 +625,10 @@ fn run_anon_named_tuple_cleanup( preamble.push('\n'); } }; - push_missing(&mut preamble, "from typing import NamedTuple"); + push_missing( + &mut preamble, + &written.import_from("typing", &["NamedTuple"]), + ); let (imports, helpers) = anon.callable.take_requirements(); for line in imports.into_iter().chain(runtime_entries(config, &helpers)) { push_missing(&mut preamble, line.trim_end_matches('\n')); diff --git a/crates/by_transforms/src/runtime.rs b/crates/by_transforms/src/runtime.rs index 962b648467..9ef9e45a3b 100644 --- a/crates/by_transforms/src/runtime.rs +++ b/crates/by_transforms/src/runtime.rs @@ -112,6 +112,7 @@ helpers! { MATCH_REST = "_by_match_rest", MATCH_ARGS = "_by_match_args", MATCH_ATTR = "_by_match_attr", + DATACLASS_SLOTS = "_by_dataclass_slots", } /// one definition of the runtime, with whatever finishes setting it up diff --git a/crates/by_transforms/src/runtime/_by_runtime.py b/crates/by_transforms/src/runtime/_by_runtime.py index 26d246f200..60be12911e 100644 --- a/crates/by_transforms/src/runtime/_by_runtime.py +++ b/crates/by_transforms/src/runtime/_by_runtime.py @@ -1437,3 +1437,99 @@ def _by_match_attr(subject, name): return getattr(subject, name) except AttributeError: return _by_match_miss + + +# --- slotted dataclasses below python 3.10 ----------------------------------- +# a `data class` is a slotted dataclass, and `dataclass(slots=True)` is new in +# 3.10. this is what that option does, to a class `dataclass` has already +# processed: a class cannot take `__slots__` once it exists, so an identical one +# is made with them, its fields' class-level defaults left out (`dataclass` has +# them already) and every zero-argument `super()` pointed at it + + +def _by_dataclass_getstate(self): + import dataclasses + return [getattr(self, f.name) for f in dataclasses.fields(self)] + + +def _by_dataclass_setstate(self, state): + import dataclasses + for f, value in zip(dataclasses.fields(self), state): + # a frozen dataclass refuses a plain assignment + object.__setattr__(self, f.name, value) + + +def _by_dataclass_slots_of(cls): + slots = cls.__dict__.get("__slots__") + if slots is None: + found = [] + if getattr(cls, "__weakrefoffset__", -1) != 0: + found.append("__weakref__") + if getattr(cls, "__dictoffset__", -1) != 0: + found.append("__dict__") + return found + if isinstance(slots, str): + return [slots] + if hasattr(slots, "__next__"): + raise TypeError(f"Slots of '{cls.__name__}' cannot be determined") + return list(slots) + + +def _by_dataclass_class_cell(f, old, new): + if f is None: + return False + try: + index = f.__code__.co_freevars.index("__class__") + except ValueError: + return False + cell = f.__closure__[index] + try: + contents = cell.cell_contents + except ValueError: + return False + if contents is old: + cell.cell_contents = new + return True + return False + + +def _by_dataclass_slots(cls): + import dataclasses, inspect, types + if "__slots__" in cls.__dict__: + raise TypeError(f"{cls.__name__} already specifies __slots__") + namespace = dict(cls.__dict__) + field_names = tuple(f.name for f in dataclasses.fields(cls)) + inherited = set() + for base in cls.__mro__[1:-1]: + inherited.update(_by_dataclass_slots_of(base)) + namespace["__slots__"] = tuple(name for name in field_names if name not in inherited) + for name in field_names: + namespace.pop(name, None) + namespace.pop("__dict__", None) + namespace.pop("__weakref__", None) + qualname = getattr(cls, "__qualname__", None) + new = type(cls)(cls.__name__, cls.__bases__, namespace) + if qualname is not None: + new.__qualname__ = qualname + if cls.__dataclass_params__.frozen: + if "__getstate__" not in namespace: + new.__getstate__ = _by_dataclass_getstate + if "__setstate__" not in namespace: + new.__setstate__ = _by_dataclass_setstate + for member in new.__dict__.values(): + # a static or class method keeps its function where 3.10 and later let + # `unwrap` find it + if isinstance(member, (staticmethod, classmethod)): + member = member.__func__ + member = inspect.unwrap(member) + if isinstance(member, types.FunctionType): + if _by_dataclass_class_cell(member, cls, new): + break + elif isinstance(member, property): + if ( + _by_dataclass_class_cell(member.fget, cls, new) + or _by_dataclass_class_cell(member.fset, cls, new) + or _by_dataclass_class_cell(member.fdel, cls, new) + ): + break + return new diff --git a/crates/by_transforms/src/transforms/annotation.rs b/crates/by_transforms/src/transforms/annotation.rs index 4f8621ae04..2eaac90f2a 100644 --- a/crates/by_transforms/src/transforms/annotation.rs +++ b/crates/by_transforms/src/transforms/annotation.rs @@ -14,16 +14,6 @@ use crate::transforms::type_expr_walker::{ }; use crate::type_info::TypeInfo; -/// The element type an unpacked tuple element wraps, whichever way the target -/// spells the unpack. -fn strip_unpack(element: &str) -> Option<&str> { - element.strip_prefix('*').or_else(|| { - element - .strip_prefix("Unpack[") - .and_then(|rest| rest.strip_suffix(']')) - }) -} - /// Whether a parameter-shape tuple element is a `*: *Args` variadic — one whose annotation is /// itself an unpack, and so names the whole run of fields rather than typing each one. fn is_unpacked_variadic(element: &Expr) -> bool { @@ -44,6 +34,7 @@ fn is_unpacked_variadic(element: &Expr) -> bool { /// subscripts on unresolved names are left alone. pub(crate) struct TupleLiteralType<'src> { source: &'src str, + written: WrittenNames<'src>, types: &'src dyn TypeInfo, min_version: PythonVersion, /// set when a lowering spelled an `Unpack`, so the pass can ask for the import @@ -74,6 +65,7 @@ impl<'src> TupleLiteralType<'src> { } Self { source, + written, types, min_version: config.min_version, needs_unpack_import: Cell::new(false), @@ -92,10 +84,21 @@ impl<'src> TupleLiteralType<'src> { format!("*{inner}") } else { self.needs_unpack_import.set(true); - format!("Unpack[{inner}]") + format!("{}[{inner}]", self.written.imported("typing", "Unpack")) } } + /// The element type an unpacked tuple element wraps, whichever way [`Self::unpack`] + /// spelled the unpack. + fn strip_unpack<'a>(&self, element: &'a str) -> Option<&'a str> { + element.strip_prefix('*').or_else(|| { + element + .strip_prefix(&self.written.imported("typing", "Unpack")) + .and_then(|rest| rest.strip_prefix('[')) + .and_then(|rest| rest.strip_suffix(']')) + }) + } + fn src(&self, range: ruff_text_size::TextRange) -> &str { &self.source[usize::from(range.start())..usize::from(range.end())] } @@ -129,7 +132,7 @@ impl<'src> TupleLiteralType<'src> { // annotation Expr::Tuple(t) if t.parenthesized => { if t.elts.is_empty() { - return Some("tuple[()]".to_owned()); + return Some(format!("{}[()]", self.written.builtin("tuple"))); } // `*: T` and a bare unpack `*A` share one AST shape, so which // one a `Starred` element is comes from the enclosing tuple: @@ -142,7 +145,7 @@ impl<'src> TupleLiteralType<'src> { .filter(|s| !s.is_empty()) .collect(); if lowered.is_empty() { - return Some("tuple[()]".to_owned()); + return Some(format!("{}[()]", self.written.builtin("tuple"))); } // pure variadic `(*: T)` → `tuple[T, ...]` directly // rather than the wrapped `tuple[*tuple[T, ...]]` form. @@ -152,11 +155,15 @@ impl<'src> TupleLiteralType<'src> { if parameter_shape && lowered.len() == 1 && !t.elts.first().is_some_and(is_unpacked_variadic) - && let Some(rest) = strip_unpack(&lowered[0]) + && let Some(rest) = self.strip_unpack(&lowered[0]) { return Some(rest.to_owned()); } - Some(format!("tuple[{}]", lowered.join(", "))) + Some(format!( + "{}[{}]", + self.written.builtin("tuple"), + lowered.join(", ") + )) } // `(int, str) * n` — an operation `symbolic_type_op` folded is the type it @@ -315,7 +322,10 @@ impl<'src> TupleLiteralType<'src> { let value_src = self .transform_annotation(&named.value) .unwrap_or_else(|| self.fallback_src(&named.value)); - return self.unpack(&format!("tuple[{value_src}, ...]")); + return self.unpack(&format!( + "{}[{value_src}, ...]", + self.written.builtin("tuple") + )); } self.transform_annotation(&named.value) .unwrap_or_else(|| self.fallback_src(&named.value)) @@ -330,7 +340,10 @@ impl<'src> TupleLiteralType<'src> { .transform_annotation(&s.value) .unwrap_or_else(|| self.fallback_src(&s.value)); if parameter_shape { - self.unpack(&format!("tuple[{value_src}, ...]")) + self.unpack(&format!( + "{}[{value_src}, ...]", + self.written.builtin("tuple") + )) } else { self.unpack(&value_src) } @@ -340,7 +353,14 @@ impl<'src> TupleLiteralType<'src> { // tuple's whole-expression edit subsumes the optional pass's edit _ => self .transform_annotation(elt) - .or_else(|| optional_type::rewrite_type_expr(self.source, elt, self.min_version)) + .or_else(|| { + optional_type::rewrite_type_expr( + self.source, + self.written, + elt, + self.min_version, + ) + }) .unwrap_or_else(|| self.fallback_src(elt)), } } @@ -428,7 +448,7 @@ impl TypeAwarePass for TupleLiteralTypePass<'_> { } if inner.needs_unpack_import.get() { ctx.required_imports - .push("from typing import Unpack".to_owned()); + .push(self.written.import_from("typing", &["Unpack"])); } // whatever the element lowerer spelled needs its own imports, and a // callable arrow among the elements needs its hoisted `Protocol` class. diff --git a/crates/by_transforms/src/transforms/anon_named_tuple.rs b/crates/by_transforms/src/transforms/anon_named_tuple.rs index a8abcfe1ae..44556c5699 100644 --- a/crates/by_transforms/src/transforms/anon_named_tuple.rs +++ b/crates/by_transforms/src/transforms/anon_named_tuple.rs @@ -39,6 +39,7 @@ use ruff_python_ast::{Expr, Stmt, StmtAnnAssign, StmtFunctionDef, StmtReturn}; use ruff_text_size::Ranged; use crate::transforms::callable::CallableSyntax; +use crate::transforms::generics::PolyfilledTypeParams; use crate::transforms::repeated_underscore::WrittenNames; use crate::type_info::TypeInfo; @@ -86,8 +87,9 @@ impl Shape { format!("_AnonNamedTuple_{truncated:08x}") } - fn class_def(&self, name: &str) -> String { - let mut out = format!("class {name}(NamedTuple):\n"); + /// the class declaring this shape as `name`, `typing.NamedTuple` written as `named_tuple` + fn class_def(&self, name: &str, named_tuple: &str) -> String { + let mut out = format!("class {name}({named_tuple}):\n"); for (field_name, field_type) in &self.fields { let _ = writeln!(out, " {field_name}: {field_type}"); } @@ -133,11 +135,9 @@ pub(crate) struct AnonNamedTuple<'src> { /// Set when at least one anonymous named tuple was seen, so the preamble /// emits the `NamedTuple` import. pub(crate) needs_import: bool, - /// Modules a field type names that the source never imported. A shape is - /// built through `&self`, so this collects behind a cell + /// The imports of the modules and `typing` names a field type reads that the source + /// never imported. A shape is built through `&self`, so this collects behind a cell type_only_imports: std::cell::RefCell>, - /// `typing` names a field type reads that the source never imported - typing_names: std::cell::RefCell>, /// Active function-scope return-annotation shape stack. Empty when not /// inside a function. The innermost (last) entry governs how a `return` /// statement inside the current function is coerced. @@ -150,9 +150,10 @@ pub(crate) struct AnonNamedTuple<'src> { /// the same module-level `_T = TypeVar("_T")` that the generics polyfill /// emits, instead of leaving an unbound `T`. /// - /// Skipped on Python 3.12+: PEP 695 native syntax doesn't rename, so the - /// stack stays empty. + /// A list kept as native PEP 695 syntax isn't renamed, so it pushes no frame. typevar_rename_stack: Vec>, + /// the name the generics polyfill declares each type parameter under + polyfilled: PolyfilledTypeParams, /// Names bound to an anonymous named tuple by an alias — `P = (a: int)` or /// `type P = (a: int)`. Coercion asks what the annotation *means*, not how /// it is spelled, so a plain tuple literal under `x: P` is wrapped exactly @@ -171,8 +172,10 @@ impl<'src> AnonNamedTuple<'src> { written: WrittenNames<'src>, types: &'src dyn TypeInfo, config: crate::Config, + stmts: &[Stmt], ) -> Self { Self { + polyfilled: PolyfilledTypeParams::decide(source, written, stmts, &config), source, written, types, @@ -184,7 +187,6 @@ impl<'src> AnonNamedTuple<'src> { range_to_value_render: Vec::new(), needs_import: false, type_only_imports: std::cell::RefCell::default(), - typing_names: std::cell::RefCell::default(), return_shape_stack: Vec::new(), typevar_rename_stack: Vec::new(), aliases: HashMap::new(), @@ -199,8 +201,9 @@ impl<'src> AnonNamedTuple<'src> { /// `NamedTuple` definitions are valid Python. pub(crate) fn class_defs(&self) -> String { let mut out = String::new(); + let named_tuple = self.written.imported("typing", "NamedTuple"); for (shape, name) in &self.shapes { - out.push_str(&shape.class_def(name)); + out.push_str(&shape.class_def(name, &named_tuple)); out.push('\n'); } out @@ -438,9 +441,10 @@ impl<'src> AnonNamedTuple<'src> { its own" )); }; - self.type_only_imports.borrow_mut().extend(d.modules); - self.typing_names.borrow_mut().extend(d.typing_names); - d.text + self.type_only_imports + .borrow_mut() + .extend(d.imports_in(self.written)); + d.text_in(self.written) }; fields.push((name, type_display)); } @@ -688,29 +692,13 @@ impl<'src> AnonNamedTuple<'src> { self.render_subbed(elt.range()) } - /// Build a typevar rename frame from a `[T, U, ...]` type-parameter list - /// and push it onto the stack. Returns whether a frame was pushed so the - /// caller knows to pop. No-op (returns false) when targeting Python 3.12+ - /// since native PEP 695 generics don't get renamed. + /// Push the typevar rename frame the generics polyfill gives a `[T, U, ...]` + /// type-parameter list. Returns whether a frame was pushed so the caller + /// knows to pop. No-op (returns false) for a list kept as native syntax fn push_typevar_scope_from(&mut self, tp: Option<&ruff_python_ast::TypeParams>) -> bool { - if self.config.min_version >= ruff_python_ast::PythonVersion::PY312 { - return false; - } - let Some(tp) = tp else { + let Some(frame) = tp.and_then(|tp| self.polyfilled.hoisted_renames(tp)) else { return false; }; - let mut frame = HashMap::new(); - for param in &tp.type_params { - let name = match param { - ruff_python_ast::TypeParam::TypeVar(tv) => tv.name.id.as_str(), - ruff_python_ast::TypeParam::TypeVarTuple(tvt) => tvt.name.id.as_str(), - ruff_python_ast::TypeParam::ParamSpec(ps) => ps.name.id.as_str(), - }; - frame.insert( - name.to_owned(), - super::generics::polyfilled_name(self.written, name), - ); - } if frame.is_empty() { return false; } @@ -927,7 +915,8 @@ impl super::ast_driver::TypeAwarePass for AnonNamedTuplePass<'_> { types: &dyn TypeInfo, ctx: &mut super::ast_driver::PassContext, ) { - let mut inner = AnonNamedTuple::new(self.source, self.written, types, self.config.clone()); + let mut inner = + AnonNamedTuple::new(self.source, self.written, types, self.config.clone(), stmts); for stmt in stmts { inner.visit_stmt(stmt); } @@ -936,7 +925,7 @@ impl super::ast_driver::TypeAwarePass for AnonNamedTuplePass<'_> { } if inner.needs_import { ctx.required_imports - .push("from typing import NamedTuple".to_owned()); + .push(self.written.import_from("typing", &["NamedTuple"])); // the field lowerer's own classes come first: a field type may name // one. they go in as a single entry so the sort below cannot part // them from the classes that reference them @@ -949,20 +938,7 @@ impl super::ast_driver::TypeAwarePass for AnonNamedTuplePass<'_> { let (imports, helpers) = inner.callable.take_requirements(); ctx.required_imports.extend(imports); ctx.runtime.extend(helpers); - ctx.type_only_imports.extend( - inner - .type_only_imports - .borrow() - .iter() - .map(|module| format!("import {module}")), - ); - ctx.type_only_imports.extend( - inner - .typing_names - .borrow() - .iter() - .map(|name| format!("from typing import {name}")), - ); + ctx.type_only_imports.extend(inner.type_only_imports.take()); for fix in std::mem::take(&mut inner.edits) { for edit in fix.edits() { let range = edit.range(); @@ -986,6 +962,28 @@ mod tests { ); } + /// kept as native syntax, `T` exists only in the scope of `A`, and the hoisted class is + /// built at import, ahead of `A`, so the field reads it off `A` as a forward reference + #[test] + fn a_native_type_parameter_is_read_off_its_generic() { + let out = transpile( + indoc! {" + class A[T]: + def m(self, p: (a: T, b: int)) -> int: + return p.b + "}, + &Config { + min_version: crate::PythonVersion::PY313, + ..Config::test_default() + }, + ) + .unwrap(); + assert!( + out.contains(" a: \"A.__type_params__[0]\"\n b: int\n"), + "got:\n{out}" + ); + } + /// the synthesized class body leaves the statement the field type was /// written in, so nothing else lowers it — `dynamic` reaching the output /// verbatim is a `NameError` at import, and a wrapped type is a syntax error @@ -1293,6 +1291,33 @@ mod tests { ); } + /// the hoisted class names the type variable the polyfill declared for the generic's `T`, + /// which an earlier generic declaring `T` differently leaves as `_T_1` + #[test] + fn field_type_uses_the_name_the_polyfill_declared() { + check( + indoc! {" + class A[T: int]: ... + class B[T]: + def get(self, t: T) -> (x: T, y: int): + return (t, 1) + "}, + indoc! {r#" + from typing import NamedTuple, TypeVar, Generic + class _AnonNamedTuple_3e0dae9f(NamedTuple): + x: "_T_1" + y: int + + _T = TypeVar("_T", bound=int) + class A(Generic[_T]): ... + _T_1 = TypeVar("_T_1") + class B(Generic[_T_1]): + def get(self, t: _T_1) -> _AnonNamedTuple_3e0dae9f: + return _AnonNamedTuple_3e0dae9f(t, 1) + "#}, + ); + } + #[test] fn anon_nt_in_protocol_body() { // Protocol attribute annotations use the anon-NT class name directly. diff --git a/crates/by_transforms/src/transforms/ast_driver.rs b/crates/by_transforms/src/transforms/ast_driver.rs index bb75c6fbb3..a35b374ceb 100644 --- a/crates/by_transforms/src/transforms/ast_driver.rs +++ b/crates/by_transforms/src/transforms/ast_driver.rs @@ -1246,7 +1246,8 @@ pub(crate) fn run_against_source<'a>( // to `LiteralString` rather than being erased with the other markers. it has // to be collected from the db's own (marker-bearing) parse, since the // blanked copy the passes walk no longer has a `literal` keyword in it - let literal_string_rewrites = literal_string::collect(parsed_handle.suite(), &semantic_model); + let literal_string_rewrites = + literal_string::collect(parsed_handle.suite(), &semantic_model, written); // a static resource import has no python spelling at all, so the document it // names is read now — from the db's own parse, which is the one the checker @@ -1256,13 +1257,14 @@ pub(crate) fn run_against_source<'a>( source_ref, ); - let symbolic_folds = symbolic_type_op::collect_symbolic_folds( + let mut symbolic_folds = symbolic_type_op::collect_symbolic_folds( parsed_handle.suite(), &semantic_model, + written, config.float_literals, + config.min_version, ); - let symbolic_needs_literal_import = symbolic_folds.needs_literal_import; - let symbolic_needs_any_import = symbolic_folds.needs_any_import; + let symbolic_imports = std::mem::take(&mut symbolic_folds.imports); ctx.claimed_type_op_ranges = symbolic_folds.claimed_ranges(); ctx.symbolic_substitutions = symbolic_folds.substitutions(); let symbolic_pass = symbolic_type_op::SymbolicTypeOp::new(symbolic_folds); @@ -1276,7 +1278,8 @@ pub(crate) fn run_against_source<'a>( &ctx.claimed_type_op_ranges, ); ctx.structural_typeof_ranges.clone_from(&typeof_skip); - let typeof_inner = typeof_keyword::TypeofFold::new(typeof_skip); + let typeof_inner = + typeof_keyword::TypeofFold::new(typeof_skip, written.imported("ty_extensions", "TypeOf")); let typeof_pass = VisitorPass { inner: &typeof_inner, changed_cell: typeof_inner.changed_cell(), @@ -1288,9 +1291,9 @@ pub(crate) fn run_against_source<'a>( let tuple_index_pass = tuple_index::TupleIndexPass::new(); - let decorated_binding_pass = decorated_binding::DecoratedBindingPass::new(source_ref); + let decorated_binding_pass = decorated_binding::DecoratedBindingPass::new(source_ref, written); - let sentinel_inner = sentinel::Sentinel::new(); + let sentinel_inner = sentinel::Sentinel::new(written.imported("typing_extensions", "Sentinel")); let sentinel_pass = VisitorPass { inner: &sentinel_inner, changed_cell: sentinel_inner.changed_cell(), @@ -1314,11 +1317,11 @@ pub(crate) fn run_against_source<'a>( }; let export_import_pass = export_import::ExportImport::new(source_ref); - let dynamic_keyword_pass = dynamic_keyword::DynamicKeywordPass::new(); + let dynamic_keyword_pass = dynamic_keyword::DynamicKeywordPass::new(written); let character_type_pass = character_type::CharacterTypePass::new(); - let grapheme_string_pass = grapheme_string::GraphemeStringPass::new(); - let type_is_pass = type_is::TypeIs::new(source_ref); - let top_star_pass = top_star::TopStar::new(); + let grapheme_string_pass = grapheme_string::GraphemeStringPass::new(written); + let type_is_pass = type_is::TypeIs::new(source_ref, written); + let top_star_pass = top_star::TopStar::new(written); let identity_swap_pass = identity_swap::IdentitySwap::new(source_ref); let compat_pass = compat::CompatRewrite::new(source_ref, config.clone()); let string_tag_pass = string_tag::StringTagPass::new(source_ref, config.clone()); @@ -1335,8 +1338,12 @@ pub(crate) fn run_against_source<'a>( config.inject_future_annotations, ); let init_method_pass = init_method::InitMethod::new(source_ref, written, config.clone()); - let properties_pass = - properties::PropertiesPass::new(source_ref, accessor_value_ranges, config.min_version); + let properties_pass = properties::PropertiesPass::new( + source_ref, + written, + accessor_value_ranges, + config.min_version, + ); let local_once_pass = local_once::LocalOncePass::new(source_ref); let raises_strip_pass = raises_clause::RaisesStripPass::new(source_ref); let return_value_use_pass = return_value_use::ReturnValueUsePass::new( @@ -1346,12 +1353,12 @@ pub(crate) fn run_against_source<'a>( let raises_guard_pass = raises_clause::RaisesGuardPass::new(source_ref, config.runtime_raises_checks); let type_fn_pass = type_fn::TypeFnPass::new(source_ref); - let match_type_pass = match_type::MatchTypePass::new(source_ref); - let modifiers_pass = modifiers::ModifiersPass::new(source_ref, config.is_stub); + let match_type_pass = match_type::MatchTypePass::new(source_ref, written); + let modifiers_pass = modifiers::ModifiersPass::new(source_ref, written, config); let main_function_pass = main_function::MainFunction::new(source_ref, written); let build_stamps_pass = build_stamps::BuildStampsPass::new(source_ref, config.stamps.clone()); let empty_declarations_pass = empty_declarations::EmptyDeclarations::new(); - let overload_pass = overload::Overload::new(source_ref, config.is_stub); + let overload_pass = overload::Overload::new(source_ref, written, config.is_stub); let decorator_keyword_pass = decorator_keyword::DecoratorKeyword::new( source_ref, written, @@ -1363,6 +1370,7 @@ pub(crate) fn run_against_source<'a>( ), ); let unpack_pass = unpack::UnpackSyntax::new( + written, config.clone(), unpack::collect_type_subscripts(parsed_handle.suite(), &semantic_model), ); @@ -1373,23 +1381,25 @@ pub(crate) fn run_against_source<'a>( parsed_handle.suite(), &semantic_model, ); - let just_float_pass = just_float::JustFloatPass::new(); + let just_float_pass = just_float::JustFloatPass::new(written); let float_const_pass = float_const::FloatConstPass::new(); - let kw_subscript_pass = kw_subscript::KwSubscriptPass::new(source_ref, config.min_version); + let kw_subscript_pass = + kw_subscript::KwSubscriptPass::new(source_ref, written, config.min_version); let generic_call_pass = generic_call::GenericCallStripPass::new(source_ref); let reified_generic_pass = reified_generic::ReifiedGenericPass::new(source_ref, config.min_version); let reified_class_pass = reified_class::ReifiedClassPass::new(source_ref, config.min_version); let type_reification_pass = type_reification::TypeReificationPass::new(config.min_version); - let visibility_rename_pass = visibility_rename::VisibilityRenamePass; - let parametric_is_pass = parametric_is::ParametricIsPass::new(source_ref); + let visibility_rename_pass = visibility_rename::VisibilityRenamePass { written }; + let parametric_is_pass = parametric_is::ParametricIsPass::new(source_ref, written); let implicit_typing_pass = implicit_typing::ImplicitTypingPass::new(); let inferred_annotation_pass = - inferred_annotation::InferredAnnotationPass::new(config.min_version); - let template_type_pass = template_type::TemplateTypePass; + inferred_annotation::InferredAnnotationPass::new(written, config.min_version); + let template_type_pass = template_type::TemplateTypePass { written }; let tuple_types_pass = annotation::TupleLiteralTypePass::new(source_ref, written, config.clone()); - let literal_types_pass = literal_types::LiteralTypePass::new(source_ref, config.float_literals); + let literal_types_pass = + literal_types::LiteralTypePass::new(source_ref, written, config.float_literals); let callable_pass = callable::CallableSyntaxPass::new(source_ref, written, config.clone()); let protocol_type_pass = protocol_type::ProtocolTypePass::new(source_ref, written, config.clone()); @@ -1397,26 +1407,27 @@ pub(crate) fn run_against_source<'a>( let force_unwrap_pass = force_unwrap::ForceUnwrapPass::new(source_ref); let flexible_keyword_pass = flexible_keyword::FlexibleKeywordPass; let some_ctor_pass = some_ctor::SomeCtorPass::new(); - let propagate_pass = propagate::PropagatePass::new(source_ref); + let propagate_pass = propagate::PropagatePass::new(source_ref, written); let none_chain_pass = none_chain::NoneChainPass::new(source_ref); - let optional_type_pass = optional_type::OptionalTypePass::new(source_ref, config.min_version); - let runtime_union_pass = runtime_union::RuntimeUnionPass::new(config.min_version); + let optional_type_pass = + optional_type::OptionalTypePass::new(source_ref, written, config.min_version); + let runtime_union_pass = runtime_union::RuntimeUnionPass::new(written, config.min_version); let generics_pass = generics::GenericPolyfillPass::new(source_ref, written, config.clone()); let soundness_pass = soundness::SoundnessPass::new(source_ref, written, config); - let checked_cast_pass = checked_cast::CheckedCastPass; + let checked_cast_pass = checked_cast::CheckedCastPass { written }; let module_api_pass = module_api::ModuleApiPass::new(source_ref); let trailing_lambda_pass = trailing_lambda::TrailingLambdaPass::new(source_ref); let if_let_pass = if_let::IfLetPass::new(source_ref); let class_pattern_star_pass = class_pattern_star::ClassPatternStarPass::new(source_ref); let destructure_pass = destructure::DestructurePass::new(source_ref); let statement_expression_pass = statement_expression::StatementExpressionPass::new(source_ref); - let context_params_pass = context_params::ContextParamsPass::new(source_ref); + let context_params_pass = context_params::ContextParamsPass::new(source_ref, written); let extension_block_pass = extension::ExtensionBlockPass::new(source_ref, written, config.is_stub); - let extension_call_pass = extension::ExtensionCallPass; + let extension_call_pass = extension::ExtensionCallPass { written }; let witness_dispatch_pass = conformance::WitnessDispatchPass; let conversion_pass = conversion::ConversionPass::new(source_ref); - let implicit_receiver_pass = implicit_receiver::ImplicitReceiverPass; + let implicit_receiver_pass = implicit_receiver::ImplicitReceiverPass { written }; let django_lookup_pass = django_lookup::DjangoLookupPass; let frameworks_pass = frameworks::FrameworksPass::new(source_ref); let variance_pass = decl_site_variance::VarianceStripPass::new(source_ref); @@ -1726,27 +1737,17 @@ pub(crate) fn run_against_source<'a>( // collect import requests the inner passes raised at the end of their run if typeof_inner.ever_changed() { ctx.required_imports - .push("from ty_extensions import TypeOf".to_owned()); - } - // symbolic folds that produced a `Literal[..]` need the import, unless the - // source already binds `Literal` - if symbolic_needs_literal_import && !literal_types::literal_already_imported(&semantic_model) { - ctx.required_imports - .push("from typing import Literal".to_owned()); - } - // symbolic folds that produced `Any` (e.g. `dynamic + 1`) need the import, - // unless the source already binds `Any` - if symbolic_needs_any_import && !semantic_model.is_bound_globally("Any") { - ctx.required_imports - .push("from typing import Any".to_owned()); + .push(written.import_from("ty_extensions", &["TypeOf"])); } + // symbolic folds that produced a `Literal[..]` or an `Any` (`dynamic + 1`) need the import + ctx.required_imports.extend(symbolic_imports); // typed lambdas are removed as source deletions so the statement around // them is never re-rendered (see `typed_lambda`); collect them here ctx.text_edits.extend(typed_lambda_inner.take_edits()); authorship.finished(Author::driver(typed_lambda_pass.name(), None), &ctx); if sentinel_inner.ever_changed() { ctx.required_imports - .push("from typing_extensions import Sentinel".to_owned()); + .push(written.import_from("typing_extensions", &["Sentinel"])); } // collapse the padding `blank` left behind. these are ordinary edits, so a @@ -1774,7 +1775,7 @@ pub(crate) fn run_against_source<'a>( authorship.finished(Author::driver("literal string", None), &ctx); if literal_string_rewrites.needs_import { ctx.required_imports - .push("from typing import LiteralString".to_owned()); + .push(written.import_from("typing", &["LiteralString"])); } // the last thing written into the tree: a soundness check made in a statement an @@ -1851,6 +1852,8 @@ pub(crate) fn run_against_source<'a>( // so only a checker ever reads the name, and a real import would give the // output an import edge — and a possible cycle — the source never had if !ctx.type_only_imports.is_empty() { + // the one typing name a lowering writes under its own name whatever the module + // binds: the block reads it where it binds it, ahead of anything the module runs let mut block = String::from("from typing import TYPE_CHECKING\nif TYPE_CHECKING:"); for line in std::mem::take(&mut ctx.type_only_imports) { block.push_str("\n "); @@ -2086,6 +2089,15 @@ pub(crate) fn run_against_source<'a>( for (start, end, repl) in edits { out.replace_range(start..end, repl.text()); } + // a builtin a lowering wrote where the module binds its name to something else is read + // under a name of the lowering's own, bound here, ahead of everything that reads it + let everything_written = std::iter::once(out.as_str()) + .chain(ctx.required_imports.iter().map(String::as_str)) + .chain(ctx.epilogue.iter().map(String::as_str)) + .collect::>() + .join("\n"); + let builtins = written.builtin_imports(&everything_written, config.min_version.minor); + ctx.required_imports.splice(0..0, builtins); // a name the module already imports from the same place goes on its line rather than on // one of ours. done here and not against `required_imports` alone, because the statement // it joins is the module's own diff --git a/crates/by_transforms/src/transforms/callable.rs b/crates/by_transforms/src/transforms/callable.rs index aa396feb9a..717aa44acd 100644 --- a/crates/by_transforms/src/transforms/callable.rs +++ b/crates/by_transforms/src/transforms/callable.rs @@ -169,7 +169,11 @@ impl<'src> CallableSyntax<'src> { arms.join(" | ") } else { self.needs_union_import = true; - format!("Union[{}]", arms.join(", ")) + format!( + "{}[{}]", + self.written.imported("typing", "Union"), + arms.join(", ") + ) } } @@ -202,24 +206,22 @@ impl<'src> CallableSyntax<'src> { /// nothing defines pub(crate) fn take_requirements(&mut self) -> (Vec, Vec) { let mut lines = Vec::new(); - for (needed, line) in [ - (self.needs_import, "from typing import Callable"), - ( - self.needs_concatenate_import, - "from typing import Concatenate", - ), - (self.needs_protocol_import, "from typing import Protocol"), + for (needed, module, name) in [ + (self.needs_import, "typing", "Callable"), + (self.needs_concatenate_import, "typing", "Concatenate"), + (self.needs_protocol_import, "typing", "Protocol"), ( self.needs_intersection_import, - "from ty_extensions import Intersection", + "ty_extensions", + "Intersection", ), - (self.needs_typeof_import, "from ty_extensions import TypeOf"), - (self.needs_not_import, "from ty_extensions import Not"), - (self.needs_annotated_import, "from typing import Annotated"), - (self.needs_union_import, "from typing import Union"), + (self.needs_typeof_import, "ty_extensions", "TypeOf"), + (self.needs_not_import, "ty_extensions", "Not"), + (self.needs_annotated_import, "typing", "Annotated"), + (self.needs_union_import, "typing", "Union"), ] { if needed { - lines.push(line.to_owned()); + lines.push(self.written.import_from(module, &[name])); } } lines.append(&mut self.extra_imports); @@ -274,8 +276,13 @@ impl<'src> CallableSyntax<'src> { return self.sweep_substitutions(expr.range()); } if let Some(types) = self.types - && let Some((text, imports)) = - rewrite_type_expr_with_imports(self.source, types, expr, self.float_literals) + && let Some((text, imports)) = rewrite_type_expr_with_imports( + self.source, + self.written, + types, + expr, + self.float_literals, + ) { self.extra_imports.extend(imports); return text; @@ -481,11 +488,14 @@ impl<'src> CallableSyntax<'src> { match self.unpacked_kwargs(&inner.value) { Some(UnpackedKwargsLowering::TypedDict) => { let ty = quote_forward_ref(&format!( - "Unpack[{}]", + "{}[{}]", + self.written.imported("typing", "Unpack"), self.rewrite_or_leaf(&inner.value) )); - self.extra_imports - .push("from typing import Unpack\n".to_owned()); + self.extra_imports.push(format!( + "{}\n", + self.written.import_from("typing", &["Unpack"]) + )); parts.push(format!("**kwargs: {ty}")); } Some(UnpackedKwargsLowering::Protocol(members)) => { @@ -549,7 +559,8 @@ impl<'src> CallableSyntax<'src> { self.protocol_shapes.insert(shape.clone(), name.clone()); let _ = writeln!( self.protocol_class_defs, - "class {name}(Protocol):\n def __call__({params}) -> {ret}: ...\n", + "class {name}({protocol}):\n def __call__({params}) -> {ret}: ...\n", + protocol = self.written.imported("typing", "Protocol"), params = shape.params, ret = shape.returns, ); @@ -582,12 +593,14 @@ impl<'src> CallableSyntax<'src> { .map(|receiver| self.rewrite_or_leaf(receiver)) .collect(); prefix_str.extend(prefix.iter().map(|a| self.rewrite_or_leaf(a))); + let callable = self.written.imported("typing", "Callable"); if prefix_str.is_empty() { - Some(format!("Callable[{ps}, {ret_str}]")) + Some(format!("{callable}[{ps}, {ret_str}]")) } else { self.needs_concatenate_import = true; Some(format!( - "Callable[Concatenate[{}, {ps}], {ret_str}]", + "{callable}[{}[{}, {ps}], {ret_str}]", + self.written.imported("typing", "Concatenate"), prefix_str.join(", ") )) } @@ -618,7 +631,10 @@ impl<'src> CallableSyntax<'src> { { self.needs_import = true; let ret_str = self.rewrite_or_leaf(returns); - Some(format!("Callable[..., {ret_str}]")) + Some(format!( + "{}[..., {ret_str}]", + self.written.imported("typing", "Callable") + )) } Expr::CallableType(ExprCallableType { @@ -635,7 +651,10 @@ impl<'src> CallableSyntax<'src> { rendered.extend(args.iter().map(|a| self.rewrite_or_leaf(a))); let args_str = rendered.join(", "); let ret_str = self.rewrite_or_leaf(returns); - Some(format!("Callable[[{args_str}], {ret_str}]")) + Some(format!( + "{}[[{args_str}], {ret_str}]", + self.written.imported("typing", "Callable") + )) } // intersection: `A & B`, `A and B`, `A & B and C` → `Intersection[…]`. @@ -646,7 +665,11 @@ impl<'src> CallableSyntax<'src> { let mut parts: Vec = Vec::new(); collect_intersect(expr, &mut parts); let rendered: Vec = parts.iter().map(|p| self.rewrite_or_leaf(p)).collect(); - Some(format!("Intersection[{}]", rendered.join(", "))) + Some(format!( + "{}[{}]", + self.written.imported("ty_extensions", "Intersection"), + rendered.join(", ") + )) } // keyword union: `A or B` → `A | B`. `|` and `or` flatten into one // chain so the rendered output carries no redundant parentheses @@ -672,7 +695,10 @@ impl<'src> CallableSyntax<'src> { Expr::UnaryOp(u) if matches!(u.op, UnaryOp::Not) => { self.needs_not_import = true; let inner = self.rewrite_or_leaf(&u.operand); - Some(format!("Not[{inner}]")) + Some(format!( + "{}[{inner}]", + self.written.imported("ty_extensions", "Not") + )) } // `T?` → `T | None` (and nested `T??` → `Optional[T | None]`), so the @@ -740,14 +766,21 @@ impl<'src> CallableSyntax<'src> { } metadata.reverse(); let inner = self.rewrite_or_leaf(inner); - Some(format!("Annotated[{inner}, {}]", metadata.join(", "))) + Some(format!( + "{}[{inner}, {}]", + self.written.imported("typing", "Annotated"), + metadata.join(", ") + )) } // `typeof X` → `TypeOf[X]` (parser tags such subscripts with `is_typeof`) Expr::Subscript(s) if s.is_typeof => { self.needs_typeof_import = true; let inner = self.rewrite_or_leaf(&s.slice); - Some(format!("TypeOf[{inner}]")) + Some(format!( + "{}[{inner}]", + self.written.imported("ty_extensions", "TypeOf") + )) } Expr::Subscript(s) => { @@ -824,7 +857,11 @@ impl<'src> CallableSyntax<'src> { .zip(t.elts.iter()) .map(|(r, e)| r.unwrap_or_else(|| self.src(e.range()).to_owned())) .collect(); - Some(format!("tuple[{}]", parts.join(", "))) + Some(format!( + "{}[{}]", + self.written.builtin("tuple"), + parts.join(", ") + )) } _ => None, @@ -925,7 +962,8 @@ pub(crate) fn lower_type_expr_full( } // no structural type-form — fall back to the per-leaf composer // (`float` → `JustFloat`, a literal → `Literal[…]`, `dynamic` → `Any`) - rewrite_type_expr_with_imports(source, types, expr, config.float_literals).map(|(text, _)| text) + rewrite_type_expr_with_imports(source, written, types, expr, config.float_literals) + .map(|(text, _)| text) } /// if `expr` is `Subscript(Name("__let__"|"__classvar__"|"__final__"), slice)`, diff --git a/crates/by_transforms/src/transforms/checked_cast.rs b/crates/by_transforms/src/transforms/checked_cast.rs index 1c5bb41dd8..3d10027ac8 100644 --- a/crates/by_transforms/src/transforms/checked_cast.rs +++ b/crates/by_transforms/src/transforms/checked_cast.rs @@ -51,6 +51,7 @@ use ruff_text_size::{Ranged, TextRange}; use super::ast_driver::{Fragment, PassContext, TypeAwarePass}; use super::parametric_is::{PredicateRuntime, ProbeStrictness, build_predicate}; +use super::repeated_underscore::WrittenNames; use crate::type_info::{CastCheck, SoundnessCheck, TypeInfo}; /// the lambda parameter a predicate-form cast binds its value to, so the value @@ -88,13 +89,14 @@ impl Helper { } } - fn open_call(self) -> &'static str { + /// the call this form opens, `typing.cast` written as `cast` + fn open_call(self, cast: &str) -> String { match self { - Self::Checked => "_checked_cast(", - Self::Try => "_try_cast(", - Self::CheckedPredicate => "_checked_cast_pred(", - Self::TryPredicate => "_try_cast_pred(", - Self::TypingCast => "cast(", + Self::Checked => "_checked_cast(".to_owned(), + Self::Try => "_try_cast(".to_owned(), + Self::CheckedPredicate => "_checked_cast_pred(".to_owned(), + Self::TryPredicate => "_try_cast_pred(".to_owned(), + Self::TypingCast => format!("{cast}("), } } @@ -114,6 +116,9 @@ impl Helper { struct CastLower<'a> { types: &'a dyn TypeInfo, + written: WrittenNames<'a>, + /// the name `typing.cast` is written under + cast: String, edits: Vec<(TextRange, Vec)>, used: BTreeSet, /// the runtime helpers the emitted predicates call @@ -121,9 +126,11 @@ struct CastLower<'a> { } impl<'a> CastLower<'a> { - fn new(types: &'a dyn TypeInfo) -> Self { + fn new(types: &'a dyn TypeInfo, written: WrittenNames<'a>) -> Self { Self { types, + written, + cast: written.imported("typing", "cast"), edits: Vec::new(), used: BTreeSet::new(), runtimes: BTreeSet::new(), @@ -148,6 +155,7 @@ impl<'a> CastLower<'a> { let value_ref = || Fragment::Lit(CAST_VALUE_PARAM.to_owned()); let (predicate, needs) = build_predicate( self.types, + self.written, &value_ref, value_arg, type_arg, @@ -197,7 +205,7 @@ impl<'a> CastLower<'a> { self.used.insert(helper); let mut fragments = vec![ - Fragment::Lit(helper.open_call().to_owned()), + Fragment::Lit(helper.open_call(&self.cast)), first, Fragment::Lit(", ".to_owned()), ]; @@ -238,11 +246,13 @@ impl<'ast> Visitor<'ast> for CastLower<'_> { } } -pub(crate) struct CheckedCastPass; +pub(crate) struct CheckedCastPass<'src> { + pub(crate) written: WrittenNames<'src>, +} -impl TypeAwarePass for CheckedCastPass { +impl TypeAwarePass for CheckedCastPass<'_> { fn run(&self, stmts: &[Stmt], types: &dyn TypeInfo, ctx: &mut PassContext) { - let mut inner = CastLower::new(types); + let mut inner = CastLower::new(types, self.written); for stmt in stmts { inner.visit_stmt(stmt); } @@ -261,7 +271,7 @@ impl TypeAwarePass for CheckedCastPass { } None => ctx .required_imports - .push("from typing import cast".to_owned()), + .push(self.written.import_from("typing", &["cast"])), } } ctx.template_edits.extend(inner.edits); diff --git a/crates/by_transforms/src/transforms/context_params.rs b/crates/by_transforms/src/transforms/context_params.rs index 5dc0d58001..01a862cbcf 100644 --- a/crates/by_transforms/src/transforms/context_params.rs +++ b/crates/by_transforms/src/transforms/context_params.rs @@ -49,15 +49,17 @@ use ruff_python_ast::{self as ast, Expr, Stmt}; use ruff_text_size::{Ranged, TextRange, TextSize}; use super::ast_driver::{PassContext, TypeAwarePass}; +use super::repeated_underscore::WrittenNames; use crate::type_info::TypeInfo; pub(crate) struct ContextParamsPass<'src> { source: &'src str, + written: WrittenNames<'src>, } impl<'src> ContextParamsPass<'src> { - pub(crate) fn new(source: &'src str) -> Self { - Self { source } + pub(crate) fn new(source: &'src str, written: WrittenNames<'src>) -> Self { + Self { source, written } } } @@ -69,6 +71,7 @@ impl TypeAwarePass for ContextParamsPass<'_> { fn run(&self, stmts: &[Stmt], types: &dyn TypeInfo, ctx: &mut PassContext) { let mut lowerer = ContextLowerer { source: self.source, + written: self.written, types, edits: Vec::new(), errors: Vec::new(), @@ -83,6 +86,7 @@ impl TypeAwarePass for ContextParamsPass<'_> { struct ContextLowerer<'src, 'ti> { source: &'src str, + written: WrittenNames<'src>, types: &'ti dyn TypeInfo, edits: Vec<(TextRange, String)>, errors: Vec, @@ -126,7 +130,7 @@ impl ContextLowerer<'_, '_> { }; let arguments = implicit .iter() - .map(|(parameter, variable)| format!("{parameter}={variable}")) + .map(|argument| argument.keyword(self.written)) .collect::>() .join(", "); let parens = call.arguments.range(); diff --git a/crates/by_transforms/src/transforms/decorated_binding.rs b/crates/by_transforms/src/transforms/decorated_binding.rs index 03212f906f..45f0f1d115 100644 --- a/crates/by_transforms/src/transforms/decorated_binding.rs +++ b/crates/by_transforms/src/transforms/decorated_binding.rs @@ -32,14 +32,16 @@ use ruff_python_ast::{Decorator, Expr, ModModule, Stmt}; use ruff_text_size::{Ranged, TextRange, TextSize}; use super::ast_driver::{AstPass, PassContext}; +use super::repeated_underscore::WrittenNames; pub(crate) struct DecoratedBindingPass<'src> { source: &'src str, + written: WrittenNames<'src>, } impl<'src> DecoratedBindingPass<'src> { - pub(crate) fn new(source: &'src str) -> Self { - Self { source } + pub(crate) fn new(source: &'src str, written: WrittenNames<'src>) -> Self { + Self { source, written } } } @@ -47,6 +49,7 @@ impl AstPass for DecoratedBindingPass<'_> { fn run(&self, module: &mut ModModule, ctx: &mut PassContext) { let mut inner = BindingVisitor { source: self.source, + annotated: self.written.imported("typing", "Annotated"), edits: Vec::new(), }; for stmt in &module.body { @@ -54,7 +57,7 @@ impl AstPass for DecoratedBindingPass<'_> { } if !inner.edits.is_empty() { ctx.required_imports - .push("from typing import Annotated".to_owned()); + .push(self.written.import_from("typing", &["Annotated"])); } ctx.text_edits.extend(inner.edits); } @@ -62,6 +65,8 @@ impl AstPass for DecoratedBindingPass<'_> { struct BindingVisitor<'src> { source: &'src str, + /// the name `typing.Annotated` is written under + annotated: String, /// `(range, replacement)` — the decorator-line erasures, and the two inserts /// that put the `Annotated` around the written type edits: Vec<(TextRange, String)>, @@ -93,8 +98,10 @@ impl BindingVisitor<'_> { // one that won, dropping the metadata. Nothing sits at a zero-width range, // so these two compose with whatever rewrites the type between them let range = written_type.range(); - self.edits - .push((TextRange::empty(range.start()), "Annotated[".to_owned())); + self.edits.push(( + TextRange::empty(range.start()), + format!("{}[", self.annotated), + )); self.edits.push(( TextRange::empty(range.end()), format!(", {}]", metadata.join(", ")), diff --git a/crates/by_transforms/src/transforms/decorator_keyword.rs b/crates/by_transforms/src/transforms/decorator_keyword.rs index 203f1112e6..d0a04a6562 100644 --- a/crates/by_transforms/src/transforms/decorator_keyword.rs +++ b/crates/by_transforms/src/transforms/decorator_keyword.rs @@ -158,11 +158,11 @@ impl AstPass for DecoratorKeyword<'_> { } if state.needs_callable { ctx.required_imports - .push("from typing import Callable".to_owned()); + .push(self.written.import_from("typing", &["Callable"])); } if state.needs_overload { ctx.required_imports - .push("from typing import overload".to_owned()); + .push(self.written.import_from("typing", &["overload"])); } // the emitted overloads are read and never run, so a name only they need is // imported where only a checker sees it @@ -257,6 +257,8 @@ impl State<'_> { } self.needs_overload = true; + let callable = self.written.imported("typing", "Callable"); + let overload = self.written.imported("typing", "overload"); let fn_name = func.name.as_str(); @@ -276,7 +278,7 @@ impl State<'_> { let decorated_type = |frags: &mut Vec| match fn_param.annotation() { Some(annotation) => frags.push(Fragment::Src(annotation.range())), // an unannotated decorated parameter accepts any callable - None => frags.push(Fragment::Lit("Callable[..., object]".to_owned())), + None => frags.push(Fragment::Lit(format!("{callable}[..., object]"))), }; // a declaration that wrote no return annotation still has a return type, and both // overloads state it: written as `object` the emitted python would be weaker than @@ -285,28 +287,19 @@ impl State<'_> { // says less than the truth rather than something else let inferred_return = self.return_types.by_function.get(&func.range()); if let Some(inferred) = inferred_return { - let mut imports = self.synthesized_imports.borrow_mut(); - imports.extend( - inferred - .modules - .iter() - .map(|module| format!("import {module}")), - ); - imports.extend( - inferred - .typing_names - .iter() - .map(|name| format!("from typing import {name}")), - ); + self.synthesized_imports + .borrow_mut() + .extend(inferred.imports_in(self.written)); } - let return_type = |frags: &mut Vec| match (&func.returns, inferred_return) { + let inferred_return = inferred_return.map(|inferred| inferred.text_in(self.written)); + let return_type = |frags: &mut Vec| match (&func.returns, &inferred_return) { (Some(returns), _) => frags.push(Fragment::Src(returns.range())), - (None, Some(inferred)) => frags.push(Fragment::Lit(inferred.text.clone())), - (None, None) => frags.push(Fragment::Lit("object".to_owned())), + (None, Some(inferred)) => frags.push(Fragment::Lit(inferred.clone())), + (None, None) => frags.push(Fragment::Lit(self.written.builtin("object"))), }; - if fn_param.annotation().is_none() { - self.needs_callable = true; - } + // the second overload's return type is a `Callable` whatever the decorated + // parameter declares + self.needs_callable = true; // the decorated parameter is declared as the source declares it, so applying the // decorator by keyword (`d(fn=f)`) is accepted here exactly where it is accepted // of the declaration @@ -336,7 +329,7 @@ impl State<'_> { // overload 1: applied to the function itself, with whatever options were given let mut header: Vec = Vec::new(); header.push(Fragment::Lit(format!( - "@overload\n{base_indent}def {fn_name}({decorated}: " + "@{overload}\n{base_indent}def {fn_name}({decorated}: " ))); decorated_type(&mut header); if !options.is_empty() { @@ -350,13 +343,13 @@ impl State<'_> { // overload 2: applied to the options alone, and what it hands back takes the // function header.push(Fragment::Lit(format!( - ": ...\n{base_indent}@overload\n{base_indent}def {fn_name}(" + ": ...\n{base_indent}@{overload}\n{base_indent}def {fn_name}(" ))); if !options.is_empty() { header.push(Fragment::Lit("*".to_owned())); declared_options(&mut header); } - header.push(Fragment::Lit(") -> Callable[[".to_owned())); + header.push(Fragment::Lit(format!(") -> {callable}[["))); decorated_type(&mut header); header.push(Fragment::Lit("], ".to_owned())); return_type(&mut header); diff --git a/crates/by_transforms/src/transforms/dynamic_keyword.rs b/crates/by_transforms/src/transforms/dynamic_keyword.rs index c73b84f281..07914459d7 100644 --- a/crates/by_transforms/src/transforms/dynamic_keyword.rs +++ b/crates/by_transforms/src/transforms/dynamic_keyword.rs @@ -23,6 +23,7 @@ use ruff_python_ast::{Expr, Stmt}; use ruff_text_size::Ranged; use crate::transforms::ast_driver::{PassContext, TypeAwarePass}; +use crate::transforms::repeated_underscore::WrittenNames; use crate::transforms::type_expr_walker::{ Recurse, TypeExprVisitor, TypePos, walk_one_type_expr, walk_type_positions, }; @@ -30,14 +31,17 @@ use crate::type_info::TypeInfo; pub(crate) struct DynamicKeyword<'src> { types: &'src dyn TypeInfo, + /// the name `typing.Any` is written under + any: String, pub(crate) edits: Vec, pub(crate) needs_any_import: bool, } impl<'src> DynamicKeyword<'src> { - pub(crate) fn new(types: &'src dyn TypeInfo) -> Self { + pub(crate) fn new(types: &'src dyn TypeInfo, written: WrittenNames) -> Self { Self { types, + any: written.imported("typing", "Any"), edits: Vec::new(), needs_any_import: false, } @@ -60,7 +64,7 @@ impl TypeExprVisitor for DynamicKeyword<'_> { { self.needs_any_import = true; self.edits.push(Fix::safe_edit(Edit::range_replacement( - "Any".to_owned(), + self.any.clone(), n.range(), ))); } @@ -68,27 +72,27 @@ impl TypeExprVisitor for DynamicKeyword<'_> { } } -pub(crate) struct DynamicKeywordPass; +pub(crate) struct DynamicKeywordPass<'src> { + written: WrittenNames<'src>, +} -impl DynamicKeywordPass { - pub(crate) fn new() -> Self { - Self +impl<'src> DynamicKeywordPass<'src> { + pub(crate) fn new(written: WrittenNames<'src>) -> Self { + Self { written } } } -impl TypeAwarePass for DynamicKeywordPass { +impl TypeAwarePass for DynamicKeywordPass<'_> { fn lowering(&self) -> Option { Some(super::ast_driver::Lowering::DynamicKeyword) } fn run(&self, stmts: &[Stmt], types: &dyn TypeInfo, ctx: &mut PassContext) { - let mut inner = DynamicKeyword::new(types); + let mut inner = DynamicKeyword::new(types, self.written); walk_type_positions(stmts, Some(types), &mut inner); - // skip the import when `Any` is already bound at module level (the user - // imported it themselves) — avoids a duplicate `from typing import Any` - if inner.needs_any_import && !types.is_bound_globally("Any") { + if inner.needs_any_import { ctx.required_imports - .push("from typing import Any".to_owned()); + .push(self.written.import_from("typing", &["Any"])); } for fix in inner.edits { for edit in fix.edits() { diff --git a/crates/by_transforms/src/transforms/enums.rs b/crates/by_transforms/src/transforms/enums.rs index c633be1db9..0e718e3d52 100644 --- a/crates/by_transforms/src/transforms/enums.rs +++ b/crates/by_transforms/src/transforms/enums.rs @@ -32,6 +32,7 @@ use ruff_python_ast::{Expr, PySourceType, PythonVersion, Stmt, StmtClassDef}; use ruff_python_parser::parse_unchecked_source; use ruff_text_size::{Ranged, TextLen, TextRange, TextSize}; +use super::repeated_underscore::{CodeNames, WrittenNames}; use super::source_util::{is_synthetic_decorator, line_indent, line_start}; use crate::Config; @@ -119,7 +120,8 @@ pub(crate) fn lower<'a>(source: &'a str, config: &Config) -> EnumLowering<'a> { }; } - let mut imports = ImportSet::default(); + let code = CodeNames::of(suite); + let mut imports = ImportSet::new(WrittenNames::new(source).with_code(&code)); let mut out = Out::default(); // a sealed hierarchy is mutually recursive (base methods reference variants @@ -362,12 +364,12 @@ fn emit_plain_enum( members: &[&Stmt], imports: &mut ImportSet, ) { - imports.add("enum", "Enum"); - imports.add("enum", "auto"); + let enum_ = imports.add("enum", "Enum"); + let auto = imports.add("enum", "auto"); - out.push_gen(&format!("{vis}class {name}(Enum):\n")); + out.push_gen(&format!("{vis}class {name}({enum_}):\n")); for variant in variants { - out.push_gen(&format!("{}{} = auto()\n", variant.indent, variant.name)); + out.push_gen(&format!("{}{} = {auto}()\n", variant.indent, variant.name)); } for member in members { emit_member(out, source, member); @@ -424,17 +426,17 @@ fn emit_sealed_hierarchy( // import annotations`), so naming a class defined further down costs // nothing at run time if !variants.is_empty() { - imports.add("typing", "ClassVar"); - } - for variant in variants { - let variant_class = variant_class_name(source, name, &variant.name); - let declared = match variant.kind { - // a payload variant is the class its call constructs - VariantKind::Tuple => format!("type[{variant_class}]"), - // a unit variant is the one instance of its class - VariantKind::Unit => variant_class, - }; - out.push_gen(&format!(" {}: ClassVar[{declared}]\n", variant.name)); + let class_var = imports.add("typing", "ClassVar"); + for variant in variants { + let variant_class = variant_class_name(source, name, &variant.name); + let declared = match variant.kind { + // a payload variant is the class its call constructs + VariantKind::Tuple => format!("{}[{variant_class}]", imports.builtin("type")), + // a unit variant is the one instance of its class + VariantKind::Unit => variant_class, + }; + out.push_gen(&format!(" {}: {class_var}[{declared}]\n", variant.name)); + } } // ordinary members (methods, classmethods, constants) — copied verbatim, // already indented under the enum in the source. they may refer to variants @@ -449,7 +451,7 @@ fn emit_sealed_hierarchy( // variant subclasses, emitted at module level and attached to the enum for variant in variants { out.push_gen("\n"); - emit_variant_class(out, source, name, variant, config); + emit_variant_class(out, source, name, variant, imports, config); } } @@ -475,6 +477,7 @@ fn emit_variant_class( source: &str, enum_name: &str, variant: &Variant, + imports: &mut ImportSet, config: &Config, ) { // a private module-level name holds the subclass; the public binding is the @@ -496,7 +499,10 @@ fn emit_variant_class( // fail outright) unless it reduces to that path. a `__reduce__` // returning a name makes both return the original object, which is // what the idiomatic `Enum` lowering of an all-unit enum already does - out.push_gen(" def __reduce__(self): return type(self).__qualname__\n"); + out.push_gen(&format!( + " def __reduce__(self): return {}(self).__qualname__\n", + imports.builtin("type") + )); if !config.is_stub { emit_variant_name_reset(out, enum_name, &variant.name, &mangled); out.push_gen(&format!("{enum_name}.{} = {mangled}()\n", variant.name)); @@ -511,8 +517,10 @@ fn emit_variant_class( } else { "" }; - out.push_gen("@final\n"); - out.push_gen(&format!("@dataclass(frozen=True{slots})\n")); + let final_ = imports.add("typing", "final"); + let dataclass = imports.add("dataclasses", "dataclass"); + out.push_gen(&format!("@{final_}\n")); + out.push_gen(&format!("@{dataclass}(frozen=True{slots})\n")); out.push_gen(&format!("class {mangled}({enum_name}):\n")); if variant.fields.is_empty() { // a zero-field payload variant (`A()`) is a valid @@ -666,24 +674,38 @@ impl Out { /// A small set of `from import ` requests, deduplicated and /// merged per module. -#[derive(Default)] -struct ImportSet { +struct ImportSet<'a> { + written: WrittenNames<'a>, modules: BTreeMap<&'static str, Vec<&'static str>>, } -impl ImportSet { - fn add(&mut self, module: &'static str, name: &'static str) { +impl<'a> ImportSet<'a> { + fn new(written: WrittenNames<'a>) -> Self { + Self { + written, + modules: BTreeMap::new(), + } + } + + /// ask for `name` from `module`, answering the name the lowering writes it under + fn add(&mut self, module: &'static str, name: &'static str) -> String { let names = self.modules.entry(module).or_default(); if !names.contains(&name) { names.push(name); } + self.written.imported(module, name) + } + + /// the name the lowering writes `name`, a builtin, under + fn builtin(&self, name: &str) -> String { + self.written.builtin(name) } fn render(&self) -> String { use std::fmt::Write as _; let mut out = String::new(); for (module, names) in &self.modules { - let _ = writeln!(out, "from {module} import {}", names.join(", ")); + let _ = writeln!(out, "{}", self.written.import_from(module, names)); } out } diff --git a/crates/by_transforms/src/transforms/extension.rs b/crates/by_transforms/src/transforms/extension.rs index d5cc6b8c49..ae736a5772 100644 --- a/crates/by_transforms/src/transforms/extension.rs +++ b/crates/by_transforms/src/transforms/extension.rs @@ -540,6 +540,8 @@ pub(super) fn arguments_span(arguments: &ast::Arguments) -> Option { pub(super) fn member_reference_fragments( info: &ty_python_semantic::ExtensionAttributeInfo, receiver: &[Fragment], + functools: &str, + written: WrittenNames, ) -> (Vec, bool) { let mut fragments = Vec::new(); let mut needs_functools = false; @@ -557,7 +559,7 @@ pub(super) fn member_reference_fragments( if info.receiver_is_class { fragments.extend_from_slice(receiver); } else { - fragments.push(Fragment::Lit("type(".to_owned())); + fragments.push(Fragment::Lit(format!("{}(", written.builtin("type")))); fragments.extend_from_slice(receiver); fragments.push(Fragment::Lit(")".to_owned())); } @@ -572,11 +574,11 @@ pub(super) fn member_reference_fragments( ExtensionMemberKind::Method | ExtensionMemberKind::ClassMethod => { needs_functools = true; fragments.push(Fragment::Lit(format!( - "functools.partial({}, ", + "{functools}.partial({}, ", info.function ))); if info.kind == ExtensionMemberKind::ClassMethod && !info.receiver_is_class { - fragments.push(Fragment::Lit("type(".to_owned())); + fragments.push(Fragment::Lit(format!("{}(", written.builtin("type")))); fragments.extend_from_slice(receiver); fragments.push(Fragment::Lit(")".to_owned())); } else { @@ -591,6 +593,9 @@ pub(super) fn member_reference_fragments( /// rewrites attribute accesses that ty resolved to extension members struct ExtensionCallLower<'a> { types: &'a dyn TypeInfo, + written: WrittenNames<'a>, + /// the name the `functools` module is imported under + functools: String, edits: Vec<(TextRange, Vec)>, imports: BTreeSet, needs_functools: bool, @@ -601,9 +606,11 @@ struct ExtensionCallLower<'a> { } impl<'a> ExtensionCallLower<'a> { - fn new(types: &'a dyn TypeInfo) -> Self { + fn new(types: &'a dyn TypeInfo, written: WrittenNames<'a>) -> Self { Self { types, + written, + functools: written.imported_module("functools"), edits: Vec::new(), imports: BTreeSet::new(), needs_functools: false, @@ -623,12 +630,13 @@ impl<'a> ExtensionCallLower<'a> { /// methods, the class object for a `class def` (via `type(…)` when the /// access went through an instance) fn receiver_fragments( + &self, info: &ty_python_semantic::ExtensionAttributeInfo, receiver: &Expr, fragments: &mut Vec, ) { if info.kind == ExtensionMemberKind::ClassMethod && !info.receiver_is_class { - fragments.push(Fragment::Lit("type(".to_owned())); + fragments.push(Fragment::Lit(format!("{}(", self.written.builtin("type")))); fragments.push(Fragment::Src(receiver.range())); fragments.push(Fragment::Lit(")".to_owned())); } else { @@ -666,7 +674,7 @@ impl<'ast> Visitor<'ast> for ExtensionCallLower<'_> { let receiver_counts = info.kind != ExtensionMemberKind::StaticMethod; let mut written = false; if receiver_counts { - Self::receiver_fragments(&info, &attr.value, &mut fragments); + self.receiver_fragments(&info, &attr.value, &mut fragments); written = true; } if let Some(span) = arguments_span(&call.arguments) { @@ -687,11 +695,11 @@ impl<'ast> Visitor<'ast> for ExtensionCallLower<'_> { Vec::new() } }; - for (parameter, variable) in implicit { + for argument in implicit { if written { fragments.push(Fragment::Lit(", ".to_owned())); } - fragments.push(Fragment::Lit(format!("{parameter}={variable}"))); + fragments.push(Fragment::Lit(argument.keyword(self.written))); written = true; } fragments.push(Fragment::Lit(")".to_owned())); @@ -727,7 +735,10 @@ impl<'ast> Visitor<'ast> for ExtensionCallLower<'_> { if info.receiver_is_class { fragments.push(Fragment::Src(attr.value.range())); } else { - fragments.push(Fragment::Lit("type(".to_owned())); + fragments.push(Fragment::Lit(format!( + "{}(", + self.written.builtin("type") + ))); fragments.push(Fragment::Src(attr.value.range())); fragments.push(Fragment::Lit(")".to_owned())); } @@ -741,10 +752,10 @@ impl<'ast> Visitor<'ast> for ExtensionCallLower<'_> { // receiver the way the bound method would have self.needs_functools = true; fragments.push(Fragment::Lit(format!( - "functools.partial({}, ", - info.function + "{}.partial({}, ", + self.functools, info.function ))); - Self::receiver_fragments(&info, &attr.value, &mut fragments); + self.receiver_fragments(&info, &attr.value, &mut fragments); fragments.push(Fragment::Lit(")".to_owned())); } } @@ -780,15 +791,18 @@ impl ExtensionCallLower<'_> { // dunder's result, and negates it for `not in` let wrapper = match expr { Expr::Compare(compare) => match compare.ops.as_ref() { - [ast::CmpOp::In] => Some("bool("), - [ast::CmpOp::NotIn] => Some("not bool("), + [ast::CmpOp::In] => Some(""), + [ast::CmpOp::NotIn] => Some("not "), _ => None, }, _ => None, }; let mut fragments = Vec::new(); if let Some(wrapper) = wrapper { - fragments.push(Fragment::Lit(wrapper.to_owned())); + fragments.push(Fragment::Lit(format!( + "{wrapper}{}(", + self.written.builtin("bool") + ))); } fragments.push(Fragment::Lit(format!("{}(", rewrite.info.function))); fragments.push(Fragment::Src(receiver.range())); @@ -819,11 +833,13 @@ fn operator_operands(expr: &Expr) -> Option<(&Expr, Option<&Expr>)> { } } -pub(crate) struct ExtensionCallPass; +pub(crate) struct ExtensionCallPass<'src> { + pub(crate) written: WrittenNames<'src>, +} -impl TypeAwarePass for ExtensionCallPass { +impl TypeAwarePass for ExtensionCallPass<'_> { fn run(&self, stmts: &[Stmt], types: &dyn TypeInfo, ctx: &mut PassContext) { - let mut inner = ExtensionCallLower::new(types); + let mut inner = ExtensionCallLower::new(types, self.written); for stmt in stmts { inner.visit_stmt(stmt); } @@ -832,7 +848,8 @@ impl TypeAwarePass for ExtensionCallPass { return; } if inner.needs_functools { - ctx.required_imports.push("import functools".to_owned()); + ctx.required_imports + .push(self.written.import_module("functools")); } ctx.required_imports.extend(inner.imports); ctx.template_edits.extend(inner.edits); diff --git a/crates/by_transforms/src/transforms/generics.rs b/crates/by_transforms/src/transforms/generics.rs index 7f49f9a1ed..cf77de1355 100644 --- a/crates/by_transforms/src/transforms/generics.rs +++ b/crates/by_transforms/src/transforms/generics.rs @@ -33,13 +33,9 @@ pub(crate) struct GenericPolyfill<'src> { /// dedup, a module with several generics over the same name produces /// repeated identical declarations (and an `F811 redefinition` warning). emitted_typevar_defs: std::collections::HashSet, - /// each name a `TypeVar` definition this pass writes binds, with the arguments it is - /// called with and the suite it is written into. a name is only ever bound in one suite: - /// a definition written into a class body is not visible to a function the module - /// declares later, and one written into an `if` does not run when the `if` does not - emitted_typevar_signatures: HashMap, - /// the suite the statement being visited stands in - suite: Suite, + /// how each type-parameter list in the module is lowered, and the name each polyfilled + /// parameter is declared under + decided: PolyfilledTypeParams, /// names of classes/functions whose first type parameter has a top-parameters /// bound (i.e. `class A[P: (*: *, **: *)]`). subscript sites for these /// targets get tuple slices rewritten to list form so paramspec @@ -94,100 +90,104 @@ pub(crate) struct ImportNeeds { unpack: bool, paramspec: bool, typealias_type: bool, - typevar_needs_ext: bool, // TypeVar(default=) on < 3.13 - /// a `default=` below 3.13, which `typing`'s own `ParamSpec` does not take - paramspec_needs_ext: bool, - /// a `default=` below 3.13, which `typing`'s own `TypeVarTuple` does not take - typevar_tuple_needs_ext: bool, } impl ImportNeeds { /// Build the import lines to prepend to the file. - fn into_lines(self, constructors: &Constructors) -> Vec { - let mut lines = Vec::new(); - - let mut typing_names: Vec = Vec::new(); - let mut ext_names: Vec = Vec::new(); - - if self.typevar { - let name = imported("TypeVar", &constructors.type_var); - if self.typevar_needs_ext { - ext_names.push(name); - } else { - typing_names.push(name); - } - } - if self.typevar_tuple { - let name = imported("TypeVarTuple", &constructors.type_var_tuple); - if self.typevar_tuple_needs_ext { - ext_names.push(name); - } else { - typing_names.push(name); + fn into_lines(self, constructors: &Constructors, written: WrittenNames) -> Vec { + // one line per module, in the order each is first asked for + let mut modules: Vec<(&'static str, Vec<&'static str>)> = Vec::new(); + for (needed, constructor) in [ + (self.typevar, &constructors.type_var), + (self.typevar_tuple, &constructors.type_var_tuple), + (self.unpack, &constructors.unpack), + (self.paramspec, &constructors.param_spec), + (self.generic, &constructors.generic), + (self.typealias_type, &constructors.type_alias_type), + ] { + if !needed { + continue; } - } - if self.unpack { - typing_names.push(imported("Unpack", &constructors.unpack)); - } - if self.paramspec { - let name = imported("ParamSpec", &constructors.param_spec); - if self.paramspec_needs_ext { - ext_names.push(name); - } else { - typing_names.push(name); + match modules + .iter_mut() + .find(|(module, _)| *module == constructor.module) + { + Some((_, names)) => names.push(constructor.name), + None => modules.push((constructor.module, vec![constructor.name])), } } - if self.generic { - typing_names.push(imported("Generic", &constructors.generic)); - } - if self.typealias_type { - ext_names.push(imported("TypeAliasType", &constructors.type_alias_type)); - } + // `typing` ahead of `typing_extensions`, as a reader expects + modules.sort_by_key(|(module, _)| *module != "typing"); + modules + .into_iter() + .map(|(module, names)| written.import_from(module, &names)) + .collect() + } +} - if !typing_names.is_empty() { - lines.push(format!("from typing import {}", typing_names.join(", "))); - } - if !ext_names.is_empty() { - lines.push(format!( - "from typing_extensions import {}", - ext_names.join(", ") - )); - } +/// a typing constructor the polyfill calls: the module it comes from, and the name it is +/// called by there. one the module binds is imported under a name it does not: the module's +/// own binding need not be the one the polyfill needs. `from typing import TypeVar` binds a +/// `TypeVar` that takes no `default=` below 3.13, and nothing stops a module binding `Generic` +/// to a class of its own +struct Constructor { + module: &'static str, + name: &'static str, + local: String, +} - lines +impl Constructor { + fn new(written: WrittenNames, module: &'static str, name: &'static str) -> Self { + Self { + module, + name, + local: written.imported(module, name), + } } } -/// `name` imported so that it is bound to `local` -fn imported(name: &str, local: &str) -> String { - if name == local { - name.to_owned() - } else { - format!("{name} as {local}") +impl std::fmt::Display for Constructor { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.local) } } -/// the names the typing constructors the polyfill writes are called by. one the module spells -/// is imported under a name it does not: the module's own binding need not be the one the -/// polyfill needs. `from typing import TypeVar` binds a `TypeVar` that takes no `default=` -/// below 3.13, and nothing stops a module binding `Generic` to a class of its own +/// the typing constructors the polyfill calls struct Constructors { - type_var: String, - generic: String, - type_var_tuple: String, - unpack: String, - param_spec: String, - type_alias_type: String, + type_var: Constructor, + generic: Constructor, + type_var_tuple: Constructor, + unpack: Constructor, + param_spec: Constructor, + type_alias_type: Constructor, } impl Constructors { - fn new(written: WrittenNames) -> Self { + /// `typing`'s own `TypeVar`, `ParamSpec` and `TypeVarTuple` take no `default=` below 3.13, + /// so a module that declares one with a default calls `typing_extensions`' + fn new(written: WrittenNames, config: &Config, decided: &PolyfilledTypeParams) -> Self { + let with_default = |declares_default: bool| { + if declares_default && config.min_version < PythonVersion::PY313 { + "typing_extensions" + } else { + "typing" + } + }; Self { - type_var: written.fresh("TypeVar"), - generic: written.fresh("Generic"), - type_var_tuple: written.fresh("TypeVarTuple"), - unpack: written.fresh("Unpack"), - param_spec: written.fresh("ParamSpec"), - type_alias_type: written.fresh("TypeAliasType"), + type_var: Constructor::new(written, with_default(decided.defaults.type_var), "TypeVar"), + generic: Constructor::new(written, "typing", "Generic"), + type_var_tuple: Constructor::new( + written, + with_default(decided.defaults.type_var_tuple), + "TypeVarTuple", + ), + unpack: Constructor::new(written, "typing", "Unpack"), + param_spec: Constructor::new( + written, + with_default(decided.defaults.param_spec), + "ParamSpec", + ), + type_alias_type: Constructor::new(written, "typing_extensions", "TypeAliasType"), } } } @@ -215,7 +215,9 @@ impl<'src> GenericPolyfill<'src> { config: Config, symbolic_substitutions: Vec<(TextRange, String)>, pending_edits: Vec<(TextRange, String)>, + decided: PolyfilledTypeParams, ) -> Self { + let constructors = Constructors::new(written, &config, &decided); Self { source, written, @@ -223,10 +225,9 @@ impl<'src> GenericPolyfill<'src> { config, edits: Vec::new(), needed_imports: ImportNeeds::default(), - constructors: Constructors::new(written), + constructors, emitted_typevar_defs: std::collections::HashSet::new(), - emitted_typevar_signatures: HashMap::new(), - suite: Suite::Module, + decided, parameters_targets: HashSet::new(), needed_imports_any: false, generic_class_renames: HashMap::new(), @@ -332,61 +333,25 @@ impl<'src> GenericPolyfill<'src> { /// pre-scan for `private type` aliases so a reference inside a later /// alias's value can be renamed as the value is re-rendered fn collect_private_aliases(&mut self, stmts: &[Stmt]) { - struct Collect<'a>(&'a mut HashMap); - impl<'ast> Visitor<'ast> for Collect<'_> { + struct Collect<'a, 'src>(&'a mut HashMap, WrittenNames<'src>); + impl<'ast> Visitor<'ast> for Collect<'_, '_> { fn visit_stmt(&mut self, stmt: &'ast Stmt) { if let Stmt::TypeAlias(alias) = stmt && alias.is_private && let Expr::Name(name) = alias.name.as_ref() { - self.0.insert(name.id.to_string(), format!("_{}", name.id)); + self.0 + .insert(name.id.to_string(), self.1.module_private(&name.id)); } ruff_python_ast::visitor::walk_stmt(self, stmt); } } - let mut collect = Collect(&mut self.private_aliases); + let mut collect = Collect(&mut self.private_aliases, self.written); for stmt in stmts { collect.visit_stmt(stmt); } } - /// whether the target can keep this parameter list as native syntax: - /// pep 695 lists need 3.12+, and a pep 696 default (`[T = int]`) bumps - /// the requirement to 3.13. a defaulted list on a 3.12 target polyfills - /// the declaration exactly like pre-3.12 code - fn supports_native_type_params(&self, params: &[TypeParam]) -> bool { - let required = if params.iter().any(|p| p.default().is_some()) { - PythonVersion::PY313 - } else { - PythonVersion::PY312 - }; - self.config.min_version >= required - } - - /// the name the `TypeVar` a parameter declared as `source_name` is bound to - /// - /// the first definition is named [`polyfilled_name`] (`T` → `_T`). a later one written - /// with the same arguments into the same suite reuses that name, so python sees one - /// object; any other takes a numeric suffix (`_T_1`, `_T_2`, …) the module does not spell, - /// so it shadows neither an earlier definition nor a binding of the module's own - fn unique_typevar_name(&mut self, source_name: &str, signature_args: &str) -> String { - let base = polyfilled_name(self.written, source_name); - let signature = (signature_args.to_owned(), self.suite); - let name = std::iter::once(base.clone()) - .chain((1u32..u32::MAX).map(|number| format!("{base}_{number}"))) - .find( - |candidate| match self.emitted_typevar_signatures.get(candidate) { - Some(existing) => *existing == signature, - None => !self.written.spells(candidate), - }, - ) - .unwrap_or(base); - self.emitted_typevar_signatures - .entry(name.clone()) - .or_insert(signature); - name - } - /// Skip `TypeVar` declarations already written into the suite being visited. a name is /// bound in one suite only, so a definition seen before was written into this one fn dedupe_defs(&mut self, defs: &[String], indent: &str) -> String { @@ -423,7 +388,11 @@ impl<'src> GenericPolyfill<'src> { if matches!(starred.value.as_ref(), Expr::Starred(_)) { return String::new(); } - return format!("*tuple[{}, ...]", self.src(named.value.range())); + return format!( + "*{}[{}, ...]", + self.written.builtin("tuple"), + self.src(named.value.range()) + ); } self.src(named.value.range()).to_owned() } @@ -431,7 +400,11 @@ impl<'src> GenericPolyfill<'src> { if matches!(s.value.as_ref(), Expr::Starred(_)) { return String::new(); } - format!("*tuple[{}, ...]", self.src(s.value.range())) + format!( + "*{}[{}, ...]", + self.written.builtin("tuple"), + self.src(s.value.range()) + ) } _ => self.src(elt.range()).to_owned(), } @@ -478,31 +451,41 @@ impl<'src> GenericPolyfill<'src> { } } - /// write the declaration of a `ParamSpec` a parameter named `name` lowers to, with - /// `default` as its `default=` argument, and answer the name it is bound to + /// write the declaration of a `ParamSpec` bound to `mangled`, with `default` as its + /// `default=` argument fn declare_param_spec( &mut self, - name: &str, + mangled: &str, default: Option, defs: &mut Vec, - ) -> String { + ) { let arguments = default - .map(|default| { - self.needed_imports.paramspec_needs_ext |= - self.config.min_version < PythonVersion::PY313; - format!(", default={default}") - }) + .map(|default| format!(", default={default}")) .unwrap_or_default(); - let mangled = self.unique_typevar_name(name, &format!("ParamSpec({arguments})")); defs.push(format!( "{mangled} = {}(\"{mangled}\"{arguments})", self.constructors.param_spec )); self.needed_imports.paramspec = true; - mangled } - fn process_type_params(&mut self, params: &[TypeParam]) -> ProcessedTypeParams { + /// lower `params`, each declared under the name at the same position in `names` + fn process_type_params( + &mut self, + params: &[TypeParam], + names: &[String], + qualified: &HashMap, + ) -> ProcessedTypeParams { + // a bound that reads a class's name through the class is written ahead of the class, + // so it is a string, which `TypeVar` keeps as a forward reference + let through_classes = |text: String| { + let read_through = apply_renames_to_rendered(&text, qualified); + if read_through == text { + text + } else { + super::source_util::python_string_literal(&read_through) + } + }; let mut generic_args: Vec = Vec::new(); let mut param_names: Vec = Vec::new(); let mut defs: Vec = Vec::new(); @@ -518,7 +501,8 @@ impl<'src> GenericPolyfill<'src> { visible.extend(enclosing.iter().map(|(k, v)| (k.clone(), v.clone()))); } - for param in params { + for (param, mangled) in params.iter().zip(names) { + let mangled = mangled.clone(); match param { TypeParam::TypeVar(tv) => { let name = tv.name.id.as_str(); @@ -532,7 +516,7 @@ impl<'src> GenericPolyfill<'src> { .default .as_deref() .map(|default| self.param_spec_default(default, &visible)); - let mangled = self.declare_param_spec(name, default, &mut defs); + self.declare_param_spec(&mangled, default, &mut defs); renames.insert(name.to_owned(), mangled.clone()); visible.insert(name.to_owned(), mangled.clone()); param_names.push(mangled.clone()); @@ -540,8 +524,6 @@ impl<'src> GenericPolyfill<'src> { continue; } - // build the non-name TypeVar arguments first so we can - // pick a unique mangled name based on the call signature let mut extra_args: Vec = Vec::new(); if let Some(bound) = &tv.bound { @@ -580,13 +562,13 @@ impl<'src> GenericPolyfill<'src> { .collect::>() .join(", "); if inner.is_empty() { - "tuple[()]".to_owned() + format!("{}[()]", self.written.builtin("tuple")) } else if t.elts.len() == 1 && let Some(rest) = inner.strip_prefix("*") { rest.to_owned() } else { - format!("tuple[{inner}]") + format!("{}[{inner}]", self.written.builtin("tuple")) } } else { lower_type_expr_full( @@ -602,16 +584,16 @@ impl<'src> GenericPolyfill<'src> { }; extra_args.push(format!( "bound={}", - apply_renames_to_rendered(&bound_src, &visible) + through_classes(apply_renames_to_rendered(&bound_src, &visible)) )); } } if let Some(default) = &tv.default { - if self.config.min_version < PythonVersion::PY313 { - self.needed_imports.typevar_needs_ext = true; - } - extra_args.push(format!("default={}", self.default_arg(default, &visible))); + extra_args.push(format!( + "default={}", + through_classes(self.default_arg(default, &visible)) + )); } // basedpython variance keywords: forward `out`/`in`/`in out` @@ -632,11 +614,6 @@ impl<'src> GenericPolyfill<'src> { None => {} } - // pick a unique mangled name based on the call signature - // so two classes that both declare `T` but with different - // bounds / variance / defaults don't shadow each other - let signature_args = extra_args.join(", "); - let mangled = self.unique_typevar_name(name, &signature_args); renames.insert(name.to_owned(), mangled.clone()); visible.insert(name.to_owned(), mangled.clone()); let mut args: Vec = vec![format!("\"{mangled}\"")]; @@ -670,20 +647,11 @@ impl<'src> GenericPolyfill<'src> { rendered } }); - let signature = default - .as_ref() - .map_or_else(String::new, |default| format!("default={default}")); - let mangled = - self.unique_typevar_name(name, &format!("TypeVarTuple({signature})")); renames.insert(name.to_owned(), mangled.clone()); visible.insert(name.to_owned(), mangled.clone()); - let arguments = if default.is_some() { - self.needed_imports.typevar_tuple_needs_ext |= - self.config.min_version < PythonVersion::PY313; - format!(", {signature}") - } else { - String::new() - }; + let arguments = default + .map(|default| format!(", default={default}")) + .unwrap_or_default(); defs.push(format!( "{mangled} = {}(\"{mangled}\"{arguments})", self.constructors.type_var_tuple @@ -708,7 +676,7 @@ impl<'src> GenericPolyfill<'src> { .default .as_deref() .map(|default| self.param_spec_default(default, &visible)); - let mangled = self.declare_param_spec(name, default, &mut defs); + self.declare_param_spec(&mangled, default, &mut defs); renames.insert(name.to_owned(), mangled.clone()); visible.insert(name.to_owned(), mangled.clone()); param_names.push(mangled.clone()); @@ -777,14 +745,14 @@ impl<'src> GenericPolyfill<'src> { .collect::>() .join(", "); let replacement = if inner.is_empty() { - "tuple[()]".to_owned() + format!("{}[()]", self.written.builtin("tuple")) } else if t.elts.len() == 1 && let Some(rest) = inner.strip_prefix("*") { // pure variadic `(*: T)` → `tuple[T, ...]` rest.to_owned() } else { - format!("tuple[{inner}]") + format!("{}[{inner}]", self.written.builtin("tuple")) }; self.edits.push(Fix::safe_edit(Edit::range_replacement( replacement, @@ -812,25 +780,24 @@ impl<'src> GenericPolyfill<'src> { // this pass, which owns the base list for a type-param class (modifiers // skips it to avoid two competing base-parens around the type params) let deferred_protocol = class.arguments.is_none() && self.has_protocol_marker(class); - // PEP 695 class type params are native syntax in 3.12+ (3.13+ with defaults) - if self.supports_native_type_params(&tp.type_params) { + let Some((names, anchor)) = self.decided.declared(tp) else { self.lower_type_param_bounds(&tp.type_params); if deferred_protocol { // keep the native `[T]`, append the base after it: `[T](Protocol)` self.edits.push(Fix::safe_edit(Edit::insertion( - "(Protocol)".to_owned(), + format!("({})", self.written.imported("typing", "Protocol")), tp.range().end(), ))); } return HashMap::new(); - } + }; let ProcessedTypeParams { generic_args, defs, renames: rename_map, .. - } = self.process_type_params(&tp.type_params); + } = self.process_type_params(&tp.type_params, &names, &self.decided.qualified(tp)); // record for module-level variant subclasses that reference these params self.generic_class_renames .insert(class.name.id.as_str().to_owned(), rename_map.clone()); @@ -874,7 +841,10 @@ impl<'src> GenericPolyfill<'src> { // the marker protocol's base goes in the same parens as `Generic`, // positional and before it: `(Protocol, Generic[_T])` self.edits.push(Fix::safe_edit(Edit::range_replacement( - format!("(Protocol, {generic_str})"), + format!( + "({}, {generic_str})", + self.written.imported("typing", "Protocol") + ), tp.range(), ))); } else { @@ -884,8 +854,8 @@ impl<'src> GenericPolyfill<'src> { ))); } - // Insert TypeVar definitions before the class. - let (line_start, indent) = self.line_start_of(class.range().start()); + // Insert TypeVar definitions before the statement they are declared ahead of. + let (line_start, indent) = self.line_start_of(anchor); let indent = indent.to_owned(); let prefix = self.dedupe_defs(&defs, &indent); if !prefix.is_empty() { @@ -932,33 +902,27 @@ impl<'src> GenericPolyfill<'src> { self.parameters_targets .insert(func.name.id.as_str().to_owned()); } - // basedpython: `some T` declares a type parameter the source writes nowhere in the - // list, so native syntax has no place for it, and it is always declared as a `TypeVar` - let is_hole = - |param: &TypeParam| matches!(param, TypeParam::TypeVar(tv) if tv.is_some_hole); - let has_hole = tp.type_params.iter().any(is_hole); - // PEP 695 function type params are native syntax in 3.12+ (3.13+ with defaults) - if !has_hole && self.supports_native_type_params(&tp.type_params) { + let Some((names, anchor)) = self.decided.declared(tp) else { self.lower_type_param_bounds(&tp.type_params); return HashMap::new(); - } + }; let ProcessedTypeParams { defs, renames: rename_map, .. - } = self.process_type_params(&tp.type_params); + } = self.process_type_params(&tp.type_params, &names, &self.decided.qualified(tp)); // Remove `[T, ...]` from the function signature. a list of holes alone is the // parser's, and there are no brackets in the source to remove: its range is the // parameter list's - if !tp.type_params.iter().all(is_hole) { + if !tp.type_params.iter().all(is_some_hole) { self.edits .push(Fix::safe_edit(Edit::range_deletion(tp.range()))); } - // Insert TypeVar definitions before the function. - let (line_start, indent) = self.line_start_of(func.range().start()); + // Insert TypeVar definitions before the statement they are declared ahead of. + let (line_start, indent) = self.line_start_of(anchor); let indent = indent.to_owned(); let prefix = self.dedupe_defs(&defs, &indent); if !prefix.is_empty() { @@ -1011,22 +975,23 @@ impl<'src> GenericPolyfill<'src> { fn process_type_alias(&mut self, alias: &StmtTypeAlias) { // `type Point = tuple[float, float]` // → `Point = TypeAliasType("Point", tuple[float, float])` - let params = alias - .type_params - .as_deref() - .map_or(&[][..], |tp| tp.type_params.as_slice()); - if self.supports_native_type_params(params) { - if let Some(tp) = &alias.type_params { - self.lower_type_param_bounds(&tp.type_params); - } - return; - } + let names = match &alias.type_params { + Some(tp) => match self.decided.declared(tp) { + Some((names, _)) => Some(names), + None => { + self.lower_type_param_bounds(&tp.type_params); + return; + } + }, + None if is_native(&self.config, &[]) => return, + None => None, + }; // this replacement subsumes `modifiers`' `private ` deletion and the // rename of the definition site, so the private name has to be applied // here instead let name_src = if alias.is_private { - format!("_{}", self.src(alias.name.range())) + self.written.module_private(self.src(alias.name.range())) } else { self.src(alias.name.range()).to_owned() }; @@ -1036,13 +1001,15 @@ impl<'src> GenericPolyfill<'src> { // also sit inside the subsumed value, so they are renamed here too let mut rename_map = self.private_aliases.clone(); - let (type_params_arg, defs) = if let Some(tp) = &alias.type_params { + let (type_params_arg, defs) = if let Some(tp) = &alias.type_params + && let Some(names) = names + { let ProcessedTypeParams { param_names, defs: type_defs, renames: tp_renames, .. - } = self.process_type_params(&tp.type_params); + } = self.process_type_params(&tp.type_params, &names, &self.decided.qualified(tp)); rename_map.extend(tp_renames); // `type_params=` wants each parameter object itself, so a variadic @@ -1067,8 +1034,9 @@ impl<'src> GenericPolyfill<'src> { let mut replacement = self.dedupe_defs(&defs, &indent); let _ = write!( replacement, - "{indent}{name_src} = {}(\"{name_src}\", object{type_params_arg})", - self.constructors.type_alias_type + "{indent}{name_src} = {}(\"{name_src}\", {}{type_params_arg})", + self.constructors.type_alias_type, + self.written.builtin("object") ); self.edits.push(Fix::safe_edit(Edit::range_replacement( replacement, @@ -1163,11 +1131,11 @@ impl GenericPolyfill<'_> { continue; } // `*name: T` - parts.push("Any".to_owned()); + parts.push(self.written.imported("typing", "Any")); self.needed_imports_any = true; } else { // `name: T` - parts.push("Any".to_owned()); + parts.push(self.written.imported("typing", "Any")); self.needed_imports_any = true; } } @@ -1177,7 +1145,7 @@ impl GenericPolyfill<'_> { continue; } // `*: T` - parts.push("Any".to_owned()); + parts.push(self.written.imported("typing", "Any")); self.needed_imports_any = true; } _ => { @@ -1250,15 +1218,6 @@ impl<'ast> Visitor<'ast> for GenericPolyfill<'_> { } } - fn visit_body(&mut self, body: &'ast [Stmt]) { - let outer = self.suite; - if let Some(first) = body.first() { - self.suite = Suite::Nested(first.start()); - } - walk_body(self, body); - self.suite = outer; - } - fn visit_expr(&mut self, expr: &'ast Expr) { if let Expr::Subscript(sub) = expr { self.rewrite_parameters_subscript(sub); @@ -1477,7 +1436,7 @@ fn rename_in_stmt(stmt: &Stmt, renames: &HashMap, edits: &mut Ve /// declared as: `_T` for `T`, spelled so that the module spells it nowhere. the definition is /// written into the scope the generic stands in, where a name the module binds would be /// overwritten by it -pub(crate) fn polyfilled_name(written: WrittenNames, name: &str) -> String { +fn polyfilled_name(written: WrittenNames, name: &str) -> String { if name.starts_with('_') { written.fresh(name) } else { @@ -1485,14 +1444,591 @@ pub(crate) fn polyfilled_name(written: WrittenNames, name: &str) -> String { } } +/// whether the target keeps the type-parameter list `params` as native syntax: pep 695 lists +/// need 3.12, and a pep 696 default (`[T = int]`) 3.13. a defaulted list on a 3.12 target is +/// polyfilled exactly as below 3.12 +fn is_native(config: &Config, params: &[TypeParam]) -> bool { + let required = if params.iter().any(|p| p.default().is_some()) { + PythonVersion::PY313 + } else { + PythonVersion::PY312 + }; + config.min_version >= required +} + +/// basedpython: `some T` declares a type parameter the source writes nowhere in the list, so +/// native syntax has no place for it, and it is always declared as a `TypeVar` +fn is_some_hole(param: &TypeParam) -> bool { + matches!(param, TypeParam::TypeVar(tv) if tv.is_some_hole) +} + +/// the polyfill's decision for every type-parameter list in a module: whether it is lowered, +/// and the name the `TypeVar`, `ParamSpec` or `TypeVarTuple` each parameter of a lowered list +/// is declared under +/// +/// a lowering that moves a type out of its generic — an inline protocol or an anonymous named +/// tuple hoisted to module scope — has to name the variable the polyfill declared, so it reads +/// this rather than working the name out again +pub(crate) struct PolyfilledTypeParams { + /// how each lowered list is declared, keyed on the list's range + lists: HashMap, + /// for each list kept as native syntax whose generic module scope reaches by a path of + /// names, the expression that reaches each parameter from module scope: + /// `B.__type_params__[0]` for the `T` of `class B[T]` + native: HashMap>, + /// which kinds of declaration a lowered list gives a default + defaults: Defaults, +} + +/// how the polyfill declares one type-parameter list +struct Declared { + /// the name each parameter is declared under, in order + names: Vec, + /// the statement the declarations are written ahead of, in its suite: the outermost `class` + /// or `def` around the generic that stands at module scope, or the generic itself when + /// something it declares reads a name only a scope inside that statement binds + /// + /// at module scope a declaration is a global, which is where everything that reads it + /// looks: `typing.get_type_hints` resolves a method's annotations in the module's + /// namespace, and a method's body cannot see its class's. one written in a class body + /// reaches neither + /// + /// a bound that reads a name of the class around the generic moves out with it, and + /// reads the name through the class + anchor: TextSize, + /// each class-level name a bound, constraint or default reads, by the path that reaches + /// it from where the declarations are written: `Inner` in `class C` is `C.Inner`. the + /// class does not exist yet where they are written, so what reads one of these is written + /// as a string, which `TypeVar` keeps as a forward reference + qualified: HashMap, +} + +/// which kinds of type parameter the polyfill declares with a default somewhere in a module +#[derive(Default, Clone, Copy)] +struct Defaults { + type_var: bool, + param_spec: bool, + type_var_tuple: bool, +} + +impl PolyfilledTypeParams { + /// decide every list in `stmts`, the module `source` parses to + pub(crate) fn decide( + source: &str, + written: WrittenNames, + stmts: &[Stmt], + config: &Config, + ) -> Self { + let mut decider = Decider { + source, + written, + config, + lists: HashMap::new(), + native: HashMap::new(), + defaults: Defaults::default(), + signatures: HashMap::new(), + suite: Suite::Module, + enclosing: Vec::new(), + scopes: Vec::new(), + local_type_params: Vec::new(), + module_statement: None, + }; + for stmt in stmts { + decider.visit_stmt(stmt); + } + Self { + lists: decider.lists, + native: decider.native, + defaults: decider.defaults, + } + } + + /// what each parameter of `type_params` is written as in a type hoisted to module scope + /// out of its generic — an inline protocol, an anonymous named tuple: the polyfill's name + /// for it, or for a list kept as native syntax, the parameter read off its generic. + /// `None` when the generic is not reached from module scope by a path of names — one + /// local to a function, or renamed or replaced by a decorator + pub(crate) fn hoisted_renames( + &self, + type_params: &ruff_python_ast::TypeParams, + ) -> Option> { + self.renames(type_params) + .or_else(|| self.native.get(&type_params.range()).cloned()) + } + + /// the name each of `type_params` is declared under, in order, and the start of the + /// statement the declarations are written ahead of, or `None` when the list is kept as + /// native syntax + fn declared( + &self, + type_params: &ruff_python_ast::TypeParams, + ) -> Option<(Vec, TextSize)> { + self.lists + .get(&type_params.range()) + .map(|declared| (declared.names.clone(), declared.anchor)) + } + + /// the class-level names the declarations of `type_params` read through their class + fn qualified(&self, type_params: &ruff_python_ast::TypeParams) -> HashMap { + self.lists + .get(&type_params.range()) + .map(|declared| declared.qualified.clone()) + .unwrap_or_default() + } + + /// what each parameter of `type_params` is renamed to, or `None` when the list is kept as + /// native syntax + pub(crate) fn renames( + &self, + type_params: &ruff_python_ast::TypeParams, + ) -> Option> { + let declared = self.lists.get(&type_params.range())?; + Some( + type_params + .iter() + .zip(&declared.names) + .map(|(param, name)| (param.name().id.to_string(), name.clone())) + .collect(), + ) + } +} + /// the statement list a definition the polyfill writes lands in, told apart by where its /// first statement starts -#[derive(Clone, Copy, PartialEq, Eq, Debug)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)] enum Suite { Module, Nested(TextSize), } +/// walks a module in the order the polyfill writes its definitions, naming each parameter +struct Decider<'a> { + source: &'a str, + written: WrittenNames<'a>, + config: &'a Config, + lists: HashMap, + /// see [`PolyfilledTypeParams::native`] + native: HashMap>, + defaults: Defaults, + /// each name a definition binds, with what the definition declares and the suite it is + /// written into. a name is only ever bound in one suite: a definition written into a class + /// body is not visible to a function the module declares later, and one written into an + /// `if` does not run when the `if` does not + signatures: HashMap, + /// the suite the statement being visited stands in + suite: Suite, + /// the renames of the lists enclosing the statement being visited, outermost first + enclosing: Vec>, + /// each scope around the statement being visited, outermost first: a class body, or a + /// function's parameters and body. the module is none of them + scopes: Vec, + /// the type parameters in scope declared inside one of `scopes` rather than at module + /// scope, by the names they are written with + local_type_params: Vec, + /// the start of the outermost statement around the one being visited that stands at + /// module scope, and the suite it stands in + module_statement: Option<(TextSize, Suite)>, +} + +impl Decider<'_> { + fn src(&self, expr: &Expr) -> &str { + &self.source[expr.range()] + } + + /// `expr` with the type parameters `visible` in scope renamed, which is what tells apart + /// two bounds that name different variables with the same spelling + fn spelled(&self, expr: &Expr, visible: &HashMap) -> String { + apply_renames_to_rendered(self.src(expr), visible) + } + + /// what `param` declares: its kind, bound or constraints, default and variance. two + /// parameters that declare the same thing in the same suite share one variable + fn signature(&self, param: &TypeParam, visible: &HashMap) -> String { + let default = param + .default() + .map(|default| format!(", default={}", self.spelled(default, visible))) + .unwrap_or_default(); + match param { + TypeParam::TypeVar(tv) => { + if tv.bound.as_deref().is_some_and(is_parameters_bound) { + return format!("ParamSpec({default})"); + } + let bound = match tv.bound.as_deref() { + Some(constraints) if tv.is_type_mapping => { + format!(", constraints={}", self.spelled(constraints, visible)) + } + Some(bound) => format!(", bound={}", self.spelled(bound, visible)), + None => String::new(), + }; + let variance = match tv.variance { + Some(ruff_python_ast::Variance::Covariant) => ", covariant=True", + Some(ruff_python_ast::Variance::Contravariant) => ", contravariant=True", + Some(ruff_python_ast::Variance::Invariant) | None => "", + }; + format!("TypeVar({bound}{default}{variance})") + } + TypeParam::TypeVarTuple(_) => format!("TypeVarTuple({default})"), + TypeParam::ParamSpec(_) => format!("ParamSpec({default})"), + } + } + + /// the name a parameter declared as `source_name` with `signature` is bound to + /// + /// the first definition is named [`polyfilled_name`] (`T` → `_T`). a later one declaring + /// the same thing in the same suite reuses that name, so python sees one object; any other + /// takes a numeric suffix (`_T_1`, `_T_2`, …) the module does not spell, so it shadows + /// neither an earlier definition nor a binding of the module's own + fn name(&mut self, source_name: &str, signature: String, suite: Suite) -> String { + let base = polyfilled_name(self.written, source_name); + let signature = (signature, suite); + let name = std::iter::once(base.clone()) + .chain((1u32..u32::MAX).map(|number| format!("{base}_{number}"))) + .find(|candidate| match self.signatures.get(candidate) { + Some(existing) => *existing == signature, + None => !self.written.taken(candidate), + }) + .unwrap_or(base); + self.signatures.entry(name.clone()).or_insert(signature); + name + } + + /// where the declarations of `type_params`, the list of the statement starting at `start`, + /// are written ahead of, and the suite that statement stands in + /// + /// at module scope, unless a bound, constraint or default reads a name a scope around the + /// generic binds — a class-level name, say, or a type parameter declared in such a scope — + /// which a module-scope declaration would read before it exists + fn anchor( + &self, + type_params: &ruff_python_ast::TypeParams, + start: TextSize, + ) -> (TextSize, Suite) { + let Some(module_statement) = self.module_statement else { + return (start, self.suite); + }; + let reads_a_local = type_params.iter().any(|param| { + let read = [ + match param { + TypeParam::TypeVar(tv) => tv.bound.as_deref(), + _ => None, + }, + param.default(), + ]; + read.into_iter().flatten().flat_map(names_read).any(|name| { + self.local_type_params.iter().any(|local| *local == name) + || self.scopes.iter().any(|scope| scope.names.contains(name)) + }) + }); + if reads_a_local { + (start, self.suite) + } else { + module_statement + } + } + + /// where the declarations of `type_params`, the list of a generic standing directly in a + /// class body, are written when a bound, constraint or default reads a name of that class + /// — and the path each such name is read through from there + /// + /// the class body is where those names are evaluated, and nothing but the class body + /// sees a declaration written there: not `typing.get_type_hints`, not the method's own + /// body, and not a function nested in it, whose annotations are evaluated when the method + /// runs. so the declarations go ahead of the outermost of the classes the generic stands + /// in, and read the class's names through the classes. a name any other scope binds, or + /// a type parameter declared in one, keeps them where they are + fn through_classes( + &self, + type_params: &ruff_python_ast::TypeParams, + ) -> Option<(TextSize, Suite, HashMap)> { + // the classes the generic stands in, the innermost last, with no function between + let first_class = self + .scopes + .iter() + .rposition(|scope| scope.class.is_none()) + .map_or(0, |function| function + 1); + let classes = &self.scopes[first_class..]; + let innermost = classes.last()?; + let path: Vec<&str> = classes + .iter() + .filter_map(|scope| scope.class.as_deref()) + .collect(); + let mut qualified = HashMap::new(); + for param in type_params { + let read = match param { + TypeParam::TypeVar(tv) if !tv.bound.as_deref().is_some_and(is_parameters_bound) => { + [tv.bound.as_deref(), param.default()] + } + // a `ParamSpec` or `TypeVarTuple` default is written without a way to read + // a class's name through the class + _ => [None, param.default()], + }; + let cannot_qualify = !matches!(param, TypeParam::TypeVar(tv) + if !tv.bound.as_deref().is_some_and(is_parameters_bound)); + for name in read.into_iter().flatten().flat_map(names_read) { + if self.local_type_params.iter().any(|local| *local == name) { + return None; + } + if innermost.names.contains(name) { + if cannot_qualify { + return None; + } + qualified.insert(name.to_owned(), format!("{}.{name}", path.join("."))); + } else if self.scopes.iter().any(|scope| scope.names.contains(name)) { + return None; + } + } + } + let outermost = classes.first()?; + (!qualified.is_empty()).then_some((outermost.start, outermost.suite, qualified)) + } + + /// record how a type hoisted out of `name`, a generic kept as native syntax, reads each of + /// its parameters `type_params`: through the classes around it, when there are only classes, + /// each bound under its own name + fn record_native( + &mut self, + type_params: &ruff_python_ast::TypeParams, + name: &ruff_python_ast::Identifier, + decorators: &[ruff_python_ast::Decorator], + ) { + if !keeps_its_name(self.source, decorators) + || !self + .scopes + .iter() + .all(|scope| scope.class.is_some() && scope.keeps_its_name) + { + return; + } + let path: Vec<&str> = self + .scopes + .iter() + .filter_map(|scope| scope.class.as_deref()) + .chain(std::iter::once(name.as_str())) + .collect(); + let path = path.join("."); + let reads = type_params + .iter() + .enumerate() + .map(|(index, param)| { + ( + param.name().id.to_string(), + format!("{path}.__type_params__[{index}]"), + ) + }) + .collect(); + self.native.insert(type_params.range(), reads); + } + + /// decide `type_params`, the list of the statement starting at `start`, answering what each + /// parameter is renamed to + fn decide( + &mut self, + type_params: &ruff_python_ast::TypeParams, + start: TextSize, + kept_native: bool, + ) -> HashMap { + if kept_native { + return HashMap::new(); + } + let (anchor, suite, qualified) = match self.through_classes(type_params) { + Some((anchor, suite, qualified)) => (anchor, suite, qualified), + None => { + let (anchor, suite) = self.anchor(type_params, start); + (anchor, suite, HashMap::new()) + } + }; + // declared inside a scope the module statement opens, where only that scope reads it + let local = self + .module_statement + .is_some_and(|(module_statement, _)| module_statement != anchor); + let mut visible: HashMap = self + .enclosing + .iter() + .flatten() + .map(|(k, v)| (k.clone(), v.clone())) + .chain(qualified.iter().map(|(k, v)| (k.clone(), v.clone()))) + .collect(); + let mut renames = HashMap::new(); + let mut names = Vec::new(); + for param in type_params { + if param.default().is_some() { + let kind = match param { + TypeParam::TypeVar(tv) + if tv.bound.as_deref().is_some_and(is_parameters_bound) => + { + &mut self.defaults.param_spec + } + TypeParam::TypeVar(_) => &mut self.defaults.type_var, + TypeParam::ParamSpec(_) => &mut self.defaults.param_spec, + TypeParam::TypeVarTuple(_) => &mut self.defaults.type_var_tuple, + }; + *kind = true; + } + let signature = self.signature(param, &visible); + let source_name = param.name().id.as_str(); + let name = self.name(source_name, signature, suite); + visible.insert(source_name.to_owned(), name.clone()); + renames.insert(source_name.to_owned(), name.clone()); + names.push(name); + } + if local { + self.local_type_params + .extend(type_params.iter().map(|param| param.name().id.to_string())); + } + self.lists.insert( + type_params.range(), + Declared { + names, + anchor, + qualified, + }, + ); + renames + } +} + +impl<'ast> Visitor<'ast> for Decider<'_> { + fn visit_stmt(&mut self, stmt: &'ast Stmt) { + let entered = matches!(stmt, Stmt::ClassDef(_) | Stmt::FunctionDef(_)) + && self.module_statement.is_none(); + if entered { + self.module_statement = Some((stmt.start(), self.suite)); + } + let local_type_params = self.local_type_params.len(); + let renames = match stmt { + Stmt::ClassDef(class) => class.type_params.as_deref().map(|tp| { + let native = is_native(self.config, &tp.type_params); + if native { + self.record_native(tp, &class.name, &class.decorator_list); + } + self.decide(tp, class.start(), native) + }), + // a `type def` is erased by its own pass, so none of its parameters is declared + Stmt::FunctionDef(func) if ruff_python_ast::helpers::is_type_def(func) => None, + Stmt::FunctionDef(func) => func.type_params.as_deref().map(|tp| { + let native = !tp.type_params.iter().any(is_some_hole) + && is_native(self.config, &tp.type_params); + if native { + self.record_native(tp, &func.name, &func.decorator_list); + } + let mut renames = self.decide(tp, func.start(), native); + // the body reads a parameter under its own name, which a type parameter of + // that name does not reach: the parameter is bound in the body's scope, the + // type parameter in the one around it + for parameter in &func.parameters { + renames.remove(parameter.name().as_str()); + } + renames + }), + // an alias writes its declarations into its own replacement + Stmt::TypeAlias(alias) => { + if let Some(tp) = alias.type_params.as_deref() { + let native = is_native(self.config, &tp.type_params); + let module_statement = self.module_statement.take(); + self.decide(tp, alias.start(), native); + self.module_statement = module_statement; + self.local_type_params.truncate(local_type_params); + } + return; + } + _ => None, + }; + // the scope the statement opens, which its body's declarations cannot be read in from + // module scope + let scope = match stmt { + Stmt::ClassDef(class) => Some(Scope { + names: class.body.iter().flat_map(bound_names).collect(), + class: Some(class.name.id.to_string()), + keeps_its_name: keeps_its_name(self.source, &class.decorator_list), + start: class.start(), + suite: self.suite, + }), + Stmt::FunctionDef(func) => Some(Scope { + names: func + .parameters + .iter() + .map(|parameter| parameter.name().to_string()) + .chain(func.body.iter().flat_map(bound_names)) + .collect(), + class: None, + keeps_its_name: false, + start: func.start(), + suite: self.suite, + }), + _ => None, + }; + let opens_scope = scope.is_some(); + self.scopes.extend(scope); + self.enclosing.push(renames.unwrap_or_default()); + walk_stmt(self, stmt); + self.enclosing.pop(); + if opens_scope { + self.scopes.pop(); + } + self.local_type_params.truncate(local_type_params); + if entered { + self.module_statement = None; + } + } + + fn visit_body(&mut self, body: &'ast [Stmt]) { + let outer = self.suite; + if let Some(first) = body.first() { + self.suite = Suite::Nested(first.start()); + } + walk_body(self, body); + self.suite = outer; + } +} + +/// whether a definition with `decorators` is bound under the name it is written with, and to +/// the definition itself: no decorator of the author's, which may return anything, and no +/// visibility keyword, which renames it +fn keeps_its_name(source: &str, decorators: &[ruff_python_ast::Decorator]) -> bool { + decorators.iter().all(|decorator| { + super::source_util::is_synthetic_decorator(source, decorator) + && !matches!( + &decorator.expression, + Expr::Name(name) + if matches!(name.id.as_str(), "private" | "protected" | "decorator_keyword") + ) + }) +} + +/// a scope around a statement the polyfill visits +struct Scope { + /// the names the scope binds + names: HashSet, + /// the class's name, when the scope is a class body + class: Option, + /// whether the class is bound under its own name, and to itself + keeps_its_name: bool, + /// where the statement opening the scope starts, and the suite it stands in + start: TextSize, + suite: Suite, +} + +/// the names `stmt` binds in the scope it stands in +fn bound_names(stmt: &Stmt) -> impl Iterator { + crate::runtime::bindings(stmt).into_keys() +} + +/// the names `expr` reads +fn names_read(expr: &Expr) -> Vec<&str> { + struct Reads<'a>(Vec<&'a str>); + impl<'a> Visitor<'a> for Reads<'a> { + fn visit_expr(&mut self, expr: &'a Expr) { + if let Expr::Name(name) = expr { + self.0.push(name.id.as_str()); + } + walk_expr(self, expr); + } + } + let mut reads = Reads(Vec::new()); + reads.visit_expr(expr); + reads.0 +} + pub(crate) struct GenericPolyfillPass<'src> { source: &'src str, written: WrittenNames<'src>, @@ -1543,6 +2079,7 @@ impl super::ast_driver::TypeAwarePass for GenericPolyfillPass<'_> { self.config.clone(), ctx.symbolic_substitutions.clone(), ctx.text_edits.clone(), + PolyfilledTypeParams::decide(self.source, self.written, stmts, &self.config), ); inner.collect_private_aliases(stmts); for stmt in stmts { @@ -1554,12 +2091,14 @@ impl super::ast_driver::TypeAwarePass for GenericPolyfillPass<'_> { ctx.text_edits .retain(|(range, _)| !superseded.contains(range)); let emits_any = inner.needed_imports_any; - for line in std::mem::take(&mut inner.needed_imports).into_lines(&inner.constructors) { + for line in + std::mem::take(&mut inner.needed_imports).into_lines(&inner.constructors, self.written) + { ctx.required_imports.push(line); } if emits_any { ctx.required_imports - .push("from typing import Any".to_owned()); + .push(self.written.import_from("typing", &["Any"])); } for (at, prefix) in std::mem::take(&mut inner.statement_prefixes) { ctx.statement_inserts @@ -1836,8 +2375,8 @@ mod tests { indoc! {" from typing import TypeVar, Generic _T = TypeVar(\"_T\") + _U = TypeVar(\"_U\", bound=_T) class Owner(Generic[_T]): - _U = TypeVar(\"_U\", bound=_T) def narrow(self, u: _U) -> None: ... "}, ); @@ -2146,8 +2685,8 @@ mod tests { ); } - /// every constructor the polyfill writes is called by a name the module does not spell, - /// whatever the module binds that name to + /// a constructor the polyfill writes is called by a name the module does not spell when the + /// module binds that name to something other than the constructor #[test] fn a_constructor_the_module_binds_is_imported_under_another_name() { check_at( @@ -2180,10 +2719,10 @@ mod tests { class A[_T]: ... "}, indoc! {" - from typing import TypeVar, TypeVar as TypeVar2, Generic + from typing import TypeVar, Generic _T = TypeVar(\"_T\", bound=int) def legacy(x: _T) -> _T: ... - _T2 = TypeVar2(\"_T2\") + _T2 = TypeVar(\"_T2\") def modern(x: _T2) -> _T2: ... class A(Generic[_T2]): ... "}, @@ -2191,9 +2730,10 @@ mod tests { ); } - /// a definition written into a class body or an `if` is not in scope where the module - /// declares a function later, so that function gets a definition of its own, under a name - /// that shadows neither + /// a definition written into an `if` does not run when the `if` does not, so a function the + /// module declares later gets a definition of its own, under a name that shadows neither. + /// a method's is written ahead of its class, at module scope, where everything that reads + /// it looks #[test] fn a_type_variable_is_bound_in_one_suite() { check_at( @@ -2207,15 +2747,135 @@ mod tests { "}, indoc! {" from typing import TypeVar + _T = TypeVar(\"_T\") class C: - _T = TypeVar(\"_T\") def m(self, x: _T) -> _T: ... if flag: _T_1 = TypeVar(\"_T_1\") def g(x: _T_1) -> _T_1: ... - _T_2 = TypeVar(\"_T_2\") - def f(x: _T_2) -> _T_2: ... - def h(x: _T_2) -> _T_2: ... + def f(x: _T) -> _T: ... + def h(x: _T) -> _T: ... + "}, + PythonVersion::PY311, + ); + } + + /// a function nested in another has its type variable declared at module scope too, ahead + /// of the statement at module scope around it + #[test] + fn a_nested_function_declares_its_type_variable_at_module_scope() { + check_at( + indoc! {" + @decorate + def outer() -> None: + def inner[T](x: T) -> T: ... + "}, + indoc! {" + from typing import TypeVar + _T = TypeVar(\"_T\") + @decorate + def outer() -> None: + def inner(x: _T) -> _T: ... + "}, + PythonVersion::PY311, + ); + } + + /// a bound that reads a class-level name is declared at module scope with the rest, and + /// reads the name through the class. the class does not exist yet where the declaration + /// is written, so the bound is a string, which `TypeVar` keeps as a forward reference. a + /// declaration in the class body would be out of reach of `typing.get_type_hints`, and of + /// a function nested in the method, whose annotations are evaluated when the method runs + #[test] + fn a_bound_reading_a_class_level_name_reads_it_through_the_class() { + check_at( + indoc! {" + class C: + class Inner: ... + def m[T: Inner, U: T](self, x: T, y: U) -> T: ... + def n[V](self, v: V) -> V: ... + "}, + indoc! {" + from typing import TypeVar + _T = TypeVar(\"_T\", bound=\"C.Inner\") + _U = TypeVar(\"_U\", bound=_T) + _V = TypeVar(\"_V\") + class C: + class Inner: ... + def m(self, x: _T, y: _U) -> _T: ... + def n(self, v: _V) -> _V: ... + "}, + PythonVersion::PY311, + ); + } + + /// in a class nested in another, the name is read through both + #[test] + fn a_bound_reading_a_nested_class_level_name_reads_it_through_every_class() { + check_at( + indoc! {" + class A: + class B: + class Inner: ... + def m[T: Inner](self, x: T) -> T: ... + "}, + indoc! {" + from typing import TypeVar + _T = TypeVar(\"_T\", bound=\"A.B.Inner\") + class A: + class B: + class Inner: ... + def m(self, x: _T) -> _T: ... + "}, + PythonVersion::PY311, + ); + } + + /// a class inside a function has its declarations written ahead of it in the function, + /// where the method's body sees them as it sees the function's other names + #[test] + fn a_bound_reading_a_class_level_name_in_a_function_is_declared_in_the_function() { + check_at( + indoc! {" + def f() -> object: + class C: + class Inner: ... + def m[T: Inner](self, x: T) -> T: ... + return C + "}, + indoc! {" + from typing import TypeVar + def f() -> object: + _T = TypeVar(\"_T\", bound=\"C.Inner\") + class C: + class Inner: ... + def m(self, x: _T) -> _T: ... + return C + "}, + PythonVersion::PY311, + ); + } + + /// a bound that reads a name of the function around the generic is declared where that + /// name is bound, as python would evaluate it + #[test] + fn a_bound_reading_a_function_local_name_is_declared_in_the_function() { + check_at( + indoc! {" + def f() -> object: + class Local: ... + def g[T: Local](x: T) -> T: + return x + return g + "}, + indoc! {" + from typing import TypeVar + def f() -> object: + class Local: ... + _T = TypeVar(\"_T\", bound=Local) + def g(x: _T) -> _T: + return x + return g "}, PythonVersion::PY311, ); @@ -2242,6 +2902,21 @@ mod tests { ); } + /// a runtime helper is bound at module scope beside the definition, and the module never + /// spells it, so a name the module leaves free can still be one + #[test] + fn a_type_variable_skips_a_runtime_helper_name() { + check_at( + "def f[force_unwrap](x: force_unwrap) -> force_unwrap: ...\n", + indoc! {" + from typing import TypeVar + _force_unwrap2 = TypeVar(\"_force_unwrap2\") + def f(x: _force_unwrap2) -> _force_unwrap2: ... + "}, + PythonVersion::PY311, + ); + } + #[test] fn type_alias_default_downleveled_on_312() { check_at( diff --git a/crates/by_transforms/src/transforms/grapheme_string.rs b/crates/by_transforms/src/transforms/grapheme_string.rs index 16fc7da2e7..768a2f7b53 100644 --- a/crates/by_transforms/src/transforms/grapheme_string.rs +++ b/crates/by_transforms/src/transforms/grapheme_string.rs @@ -51,6 +51,7 @@ use ruff_text_size::{Ranged, TextRange}; use super::ast_driver::{Fragment, PassContext, TypeAwarePass}; use super::coalesce::is_trivially_pure; +use super::repeated_underscore::WrittenNames; use crate::type_info::TypeInfo; #[expect( @@ -59,6 +60,7 @@ use crate::type_info::TypeInfo; )] struct GraphemeString<'src> { types: &'src dyn TypeInfo, + written: WrittenNames<'src>, /// attribute nodes that are the direct callee of a call — the grapheme /// properties are not callable there, so they pass through untouched callee_attrs: HashSet, @@ -131,7 +133,7 @@ impl<'ast> Visitor<'ast> for GraphemeString<'_> { self.template( expr, vec![ - Fragment::Lit("len(_by_graphemes(".to_owned()), + Fragment::Lit(format!("{}(_by_graphemes(", self.written.builtin("len"))), Fragment::Src(receiver), Fragment::Lit("))".to_owned()), ], @@ -171,7 +173,7 @@ impl<'ast> Visitor<'ast> for GraphemeString<'_> { "unicode_scalars" => self.template( expr, vec![ - Fragment::Lit("iter(".to_owned()), + Fragment::Lit(format!("{}(", self.written.builtin("iter"))), Fragment::Src(receiver), Fragment::Lit(")".to_owned()), ], @@ -279,18 +281,21 @@ impl GraphemeString<'_> { } } -pub(crate) struct GraphemeStringPass; +pub(crate) struct GraphemeStringPass<'src> { + written: WrittenNames<'src>, +} -impl GraphemeStringPass { - pub(crate) fn new() -> Self { - Self +impl<'src> GraphemeStringPass<'src> { + pub(crate) fn new(written: WrittenNames<'src>) -> Self { + Self { written } } } -impl TypeAwarePass for GraphemeStringPass { +impl TypeAwarePass for GraphemeStringPass<'_> { fn run(&self, stmts: &[Stmt], types: &dyn TypeInfo, ctx: &mut PassContext) { let mut inner = GraphemeString { types, + written: self.written, callee_attrs: HashSet::new(), needs_grapheme_helper: false, needs_character_class: false, diff --git a/crates/by_transforms/src/transforms/implicit_receiver.rs b/crates/by_transforms/src/transforms/implicit_receiver.rs index 9e7724eb8c..c748524b50 100644 --- a/crates/by_transforms/src/transforms/implicit_receiver.rs +++ b/crates/by_transforms/src/transforms/implicit_receiver.rs @@ -25,11 +25,15 @@ use ty_python_semantic::ImplicitReceiverReference; use super::ast_driver::{Fragment, PassContext, TypeAwarePass}; use super::extension::{arguments_span, spine_has_optional}; +use super::repeated_underscore::WrittenNames; use super::trailing_lambda::RECEIVER_PARAMETER; use crate::type_info::TypeInfo; struct ImplicitReceiverLower<'a> { types: &'a dyn TypeInfo, + written: WrittenNames<'a>, + /// the name the `functools` module is imported under + functools: String, edits: Vec<(TextRange, Vec)>, errors: Vec, /// attribute ranges already rewritten as part of an enclosing call, so the @@ -71,7 +75,10 @@ impl<'ast> Visitor<'ast> for ImplicitReceiverLower<'_> { self.edits.push(( attr.range(), vec![ - Fragment::Lit(format!("functools.partial({}, ", attr.attr)), + Fragment::Lit(format!( + "{}.partial({}, ", + self.functools, attr.attr + )), Fragment::Src(attr.value.range()), Fragment::Lit(")".to_owned()), ], @@ -120,6 +127,8 @@ impl<'ast> Visitor<'ast> for ImplicitReceiverLower<'_> { super::extension::member_reference_fragments( &info, &[Fragment::Lit(RECEIVER_PARAMETER.to_owned())], + &self.functools, + self.written, ); self.needs_functools |= needs_functools; fragments @@ -161,12 +170,16 @@ fn chain_error(name: &str, verb: &str) -> String { format!("implicit receiver `{name}` cannot be {verb} through an optional chain yet") } -pub(crate) struct ImplicitReceiverPass; +pub(crate) struct ImplicitReceiverPass<'src> { + pub(crate) written: WrittenNames<'src>, +} -impl TypeAwarePass for ImplicitReceiverPass { +impl TypeAwarePass for ImplicitReceiverPass<'_> { fn run(&self, stmts: &[Stmt], types: &dyn TypeInfo, ctx: &mut PassContext) { let mut inner = ImplicitReceiverLower { types, + written: self.written, + functools: self.written.imported_module("functools"), edits: Vec::new(), handled: Vec::new(), errors: Vec::new(), @@ -181,7 +194,8 @@ impl TypeAwarePass for ImplicitReceiverPass { return; } if inner.needs_functools { - ctx.required_imports.push("import functools".to_owned()); + ctx.required_imports + .push(self.written.import_module("functools")); } ctx.required_imports.extend(inner.imports); ctx.template_edits.extend(inner.edits); diff --git a/crates/by_transforms/src/transforms/inferred_annotation.rs b/crates/by_transforms/src/transforms/inferred_annotation.rs index 1626abf587..01a43310e1 100644 --- a/crates/by_transforms/src/transforms/inferred_annotation.rs +++ b/crates/by_transforms/src/transforms/inferred_annotation.rs @@ -30,19 +30,27 @@ use ruff_python_ast::{Expr, ExprName, Stmt, StmtClassDef}; use ruff_text_size::{Ranged, TextRange}; use crate::transforms::ast_driver::{PassContext, TypeAwarePass}; +use crate::transforms::repeated_underscore::WrittenNames; use crate::type_info::TypeInfo; -pub(crate) struct InferredAnnotationPass { +pub(crate) struct InferredAnnotationPass<'src> { + written: WrittenNames<'src>, min_version: ruff_python_ast::PythonVersion, } -impl InferredAnnotationPass { - pub(crate) fn new(min_version: ruff_python_ast::PythonVersion) -> Self { - Self { min_version } +impl<'src> InferredAnnotationPass<'src> { + pub(crate) fn new( + written: WrittenNames<'src>, + min_version: ruff_python_ast::PythonVersion, + ) -> Self { + Self { + written, + min_version, + } } } -impl TypeAwarePass for InferredAnnotationPass { +impl TypeAwarePass for InferredAnnotationPass<'_> { fn lowering(&self) -> Option { Some(super::ast_driver::Lowering::InferredAnnotation) } @@ -50,34 +58,27 @@ impl TypeAwarePass for InferredAnnotationPass { fn run(&self, stmts: &[Stmt], types: &dyn TypeInfo, ctx: &mut PassContext) { let mut state = State { types, + written: self.written, min_version: self.min_version, edits: Vec::new(), - modules: BTreeSet::new(), - typing_names: BTreeSet::new(), + imports: BTreeSet::new(), }; for stmt in stmts { state.visit_stmt(stmt); } ctx.text_edits.extend(state.edits); - ctx.type_only_imports - .extend(state.modules.into_iter().map(|m| format!("import {m}"))); - ctx.type_only_imports.extend( - state - .typing_names - .into_iter() - .map(|name| format!("from typing import {name}")), - ); + ctx.type_only_imports.extend(state.imports); } } struct State<'a> { types: &'a dyn TypeInfo, + written: WrittenNames<'a>, min_version: ruff_python_ast::PythonVersion, edits: Vec<(TextRange, String)>, - /// modules a synthesized annotation names but the source never imported - modules: BTreeSet, - /// `typing` names it reads that the source never imported either - typing_names: BTreeSet<&'static str>, + /// the modules and `typing` names a synthesized annotation reads but the source never + /// imported, as the import lines that bind them + imports: BTreeSet, } impl State<'_> { @@ -117,10 +118,11 @@ impl State<'_> { return; }; let pos = name.range().end(); - self.edits - .push((TextRange::new(pos, pos), format!(": {}", annotation.text))); - self.modules.extend(annotation.modules); - self.typing_names.extend(annotation.typing_names); + self.edits.push(( + TextRange::new(pos, pos), + format!(": {}", annotation.text_in(self.written)), + )); + self.imports.extend(annotation.imports_in(self.written)); } } @@ -395,6 +397,31 @@ mod tests { ); } + /// a module that binds a `typing` name to something of its own keeps it, and the + /// annotation reads `typing`'s under a name of its own + #[test] + fn a_typing_name_the_module_binds_otherwise_is_imported_fresh() { + check( + indoc! {" + Never = 4 + def boom(): + raise ValueError + class A: + n = boom() + "}, + indoc! {" + from typing import TYPE_CHECKING + if TYPE_CHECKING: + from typing_extensions import Never as Never2 + Never = 4 + def boom(): + raise ValueError + class A: + n: Never2 = boom() + "}, + ); + } + /// ty reports the modules a *type* lives in, which is not the same question as which /// ones its spelling names: the type spelled `None` is reported as living in `types`, /// and importing it bought the emitted file a `if TYPE_CHECKING: import types` block diff --git a/crates/by_transforms/src/transforms/init_method.rs b/crates/by_transforms/src/transforms/init_method.rs index 06bd908398..f5c6f6a24f 100644 --- a/crates/by_transforms/src/transforms/init_method.rs +++ b/crates/by_transforms/src/transforms/init_method.rs @@ -72,7 +72,7 @@ impl TypeAwarePass for InitMethod<'_> { } if state.needs_missing.into_inner() { ctx.required_imports - .extend(super::mutable_defaults::sentinel_definition(self.source)); + .extend(super::mutable_defaults::sentinel_definition(self.written)); } ctx.text_edits.extend(state.edits.into_inner()); ctx.template_edits.extend(state.templates.into_inner()); @@ -357,7 +357,7 @@ impl State<'_> { } = parameter_guards( func, self.written, - &super::mutable_defaults::sentinel_name(self.source), + &super::mutable_defaults::sentinel_name(self.written), self.types, self.config.is_stub, ); diff --git a/crates/by_transforms/src/transforms/just_float.rs b/crates/by_transforms/src/transforms/just_float.rs index 22e3179130..c354301352 100644 --- a/crates/by_transforms/src/transforms/just_float.rs +++ b/crates/by_transforms/src/transforms/just_float.rs @@ -25,6 +25,7 @@ use ruff_text_size::Ranged; use crate::config::FloatLiteralLowering; use crate::transforms::ast_driver::{PassContext, TypeAwarePass}; use crate::transforms::literal_types::LiteralType; +use crate::transforms::repeated_underscore::WrittenNames; use crate::transforms::type_expr_walker::{ Recurse, TypeExprVisitor, TypePos, walk_one_type_expr, walk_type_positions, }; @@ -32,15 +33,17 @@ use crate::type_info::TypeInfo; pub(crate) struct JustFloat<'src> { types: &'src dyn TypeInfo, + written: WrittenNames<'src>, edits: Vec, needs_float_alias: bool, needs_complex_alias: bool, } impl<'src> JustFloat<'src> { - fn new(types: &'src dyn TypeInfo) -> Self { + fn new(types: &'src dyn TypeInfo, written: WrittenNames<'src>) -> Self { Self { types, + written, edits: Vec::new(), needs_float_alias: false, needs_complex_alias: false, @@ -81,7 +84,7 @@ impl TypeExprVisitor for JustFloat<'_> { self.needs_complex_alias = true; } self.edits.push(Fix::safe_edit(Edit::range_replacement( - replacement.to_owned(), + self.written.imported("ty_extensions", replacement), n.range(), ))); } @@ -89,29 +92,31 @@ impl TypeExprVisitor for JustFloat<'_> { } } -pub(crate) struct JustFloatPass; +pub(crate) struct JustFloatPass<'src> { + written: WrittenNames<'src>, +} -impl JustFloatPass { - pub(crate) fn new() -> Self { - Self +impl<'src> JustFloatPass<'src> { + pub(crate) fn new(written: WrittenNames<'src>) -> Self { + Self { written } } } -impl TypeAwarePass for JustFloatPass { +impl TypeAwarePass for JustFloatPass<'_> { fn lowering(&self) -> Option { Some(super::ast_driver::Lowering::JustFloat) } fn run(&self, stmts: &[Stmt], types: &dyn TypeInfo, ctx: &mut PassContext) { - let mut inner = JustFloat::new(types); + let mut inner = JustFloat::new(types, self.written); walk_type_positions(stmts, Some(types), &mut inner); if inner.needs_float_alias { ctx.required_imports - .push("from ty_extensions import JustFloat".to_owned()); + .push(self.written.import_from("ty_extensions", &["JustFloat"])); } if inner.needs_complex_alias { ctx.required_imports - .push("from ty_extensions import JustComplex".to_owned()); + .push(self.written.import_from("ty_extensions", &["JustComplex"])); } for fix in inner.edits { for edit in fix.edits() { @@ -141,6 +146,7 @@ impl TypeAwarePass for JustFloatPass { /// otherwise be the only trace left — needs these so the lowered names resolve. pub(crate) fn rewrite_type_expr_with_imports( source: &str, + written: WrittenNames, types: &dyn TypeInfo, expr: &Expr, float_literals: FloatLiteralLowering, @@ -148,22 +154,22 @@ pub(crate) fn rewrite_type_expr_with_imports( let mut all_edits: Vec = Vec::new(); let mut imports: Vec = Vec::new(); - let mut lt = LiteralType::new(source, types, float_literals); + let mut lt = LiteralType::new(source, written, types, float_literals); lt.emit_type_edits(expr, true); if lt.needs_literal_import { - imports.push("from typing import Literal".to_owned()); + imports.push(written.import_from("typing", &["Literal"])); } for fix in lt.edits { all_edits.extend(fix.into_edits()); } - let mut jf = JustFloat::new(types); + let mut jf = JustFloat::new(types, written); jf.emit_in_type_expr(expr); if jf.needs_float_alias { - imports.push("from ty_extensions import JustFloat".to_owned()); + imports.push(written.import_from("ty_extensions", &["JustFloat"])); } if jf.needs_complex_alias { - imports.push("from ty_extensions import JustComplex".to_owned()); + imports.push(written.import_from("ty_extensions", &["JustComplex"])); } for fix in jf.edits { all_edits.extend(fix.into_edits()); @@ -172,10 +178,10 @@ pub(crate) fn rewrite_type_expr_with_imports( // `dynamic` → `Any` in the same composed rewrite, so a `type X = dynamic` // / `def f[T: dynamic]` polyfilled on Python < 3.12 doesn't leak the bare // keyword (an undefined name the final parse can't catch) - let mut dk = crate::transforms::dynamic_keyword::DynamicKeyword::new(types); + let mut dk = crate::transforms::dynamic_keyword::DynamicKeyword::new(types, written); dk.emit_in_type_expr(expr); - if dk.needs_any_import && !types.is_bound_globally("Any") { - imports.push("from typing import Any".to_owned()); + if dk.needs_any_import { + imports.push(written.import_from("typing", &["Any"])); } for fix in dk.edits { all_edits.extend(fix.into_edits()); diff --git a/crates/by_transforms/src/transforms/kw_subscript.rs b/crates/by_transforms/src/transforms/kw_subscript.rs index 1197f60a1e..02b050a065 100644 --- a/crates/by_transforms/src/transforms/kw_subscript.rs +++ b/crates/by_transforms/src/transforms/kw_subscript.rs @@ -12,19 +12,27 @@ use ruff_python_ast::{Expr, PythonVersion, Stmt}; use ruff_text_size::{Ranged, TextRange}; use crate::transforms::ast_driver::{PassContext, TypeAwarePass}; +use crate::transforms::repeated_underscore::WrittenNames; use crate::type_info::TypeInfo; pub(crate) struct KwSubscript<'src, T: TypeInfo + ?Sized> { source: &'src str, + written: WrittenNames<'src>, types: Option<&'src T>, min_version: PythonVersion, edits: Vec, } impl<'src, T: TypeInfo + ?Sized> KwSubscript<'src, T> { - fn new(source: &'src str, types: Option<&'src T>, min_version: PythonVersion) -> Self { + fn new( + source: &'src str, + written: WrittenNames<'src>, + types: Option<&'src T>, + min_version: PythonVersion, + ) -> Self { Self { source, + written, types, min_version, edits: Vec::new(), @@ -41,8 +49,13 @@ impl<'src, T: TypeInfo + ?Sized> KwSubscript<'src, T> { /// import for nested `??` is still raised by `OptionalTypePass`, which walks /// every expression independently fn value_src(&self, expr: &Expr) -> String { - crate::transforms::optional_type::rewrite_type_expr(self.source, expr, self.min_version) - .unwrap_or_else(|| self.src(expr.range()).to_owned()) + crate::transforms::optional_type::rewrite_type_expr( + self.source, + self.written, + expr, + self.min_version, + ) + .unwrap_or_else(|| self.src(expr.range()).to_owned()) } /// Lower a subscript of a class declaring a keyword-variadic pack. @@ -309,13 +322,19 @@ impl<'src, T: TypeInfo + ?Sized> KwSubscript<'src, T> { pub(crate) struct KwSubscriptPass<'src> { source: &'src str, + written: WrittenNames<'src>, min_version: PythonVersion, } impl<'src> KwSubscriptPass<'src> { - pub(crate) fn new(source: &'src str, min_version: PythonVersion) -> Self { + pub(crate) fn new( + source: &'src str, + written: WrittenNames<'src>, + min_version: PythonVersion, + ) -> Self { Self { source, + written, min_version, } } @@ -329,7 +348,7 @@ impl TypeAwarePass for KwSubscriptPass<'_> { fn run(&self, stmts: &[Stmt], types: &dyn TypeInfo, ctx: &mut PassContext) { let mut inner: KwSubscript<'_, dyn TypeInfo> = - KwSubscript::new(self.source, Some(types), self.min_version); + KwSubscript::new(self.source, self.written, Some(types), self.min_version); for stmt in stmts { inner.visit_stmt(stmt); } diff --git a/crates/by_transforms/src/transforms/literal_string.rs b/crates/by_transforms/src/transforms/literal_string.rs index 61f31aa4a0..369a7eb17f 100644 --- a/crates/by_transforms/src/transforms/literal_string.rs +++ b/crates/by_transforms/src/transforms/literal_string.rs @@ -24,6 +24,7 @@ use ruff_python_ast::helpers::{TypeModifier, type_modifier_marker}; use ruff_python_ast::{Expr, Stmt}; use ruff_text_size::{Ranged, TextRange}; +use crate::transforms::repeated_underscore::WrittenNames; use crate::transforms::type_expr_walker::{Recurse, TypeExprVisitor, TypePos, walk_type_positions}; use crate::type_info::TypeInfo; @@ -32,8 +33,9 @@ use crate::type_info::TypeInfo; pub(crate) struct LiteralStringRewrites { /// each marker's full source range — the keyword through the type it wraps ranges: Vec, - /// whether any rewrite fired *and* `LiteralString` is not already bound, so - /// the driver can add the import + /// the name `typing.LiteralString` is written under + literal_string: String, + /// whether any rewrite fired, so the driver can add the import pub(crate) needs_import: bool, } @@ -42,7 +44,7 @@ impl LiteralStringRewrites { pub(crate) fn edits(&self) -> impl Iterator + '_ { self.ranges .iter() - .map(|range| (*range, "LiteralString".to_owned())) + .map(|range| (*range, self.literal_string.clone())) } /// whether `range` falls inside a rewritten marker. The blanking pass emits @@ -58,16 +60,20 @@ impl LiteralStringRewrites { /// Collect every `literal str` marker in `stmts` that ty reduced to /// `LiteralString`. `stmts` must come from the same parse `types` answers for. -pub(crate) fn collect(stmts: &[Stmt], types: &dyn TypeInfo) -> LiteralStringRewrites { +pub(crate) fn collect( + stmts: &[Stmt], + types: &dyn TypeInfo, + written: WrittenNames, +) -> LiteralStringRewrites { let mut collector = Collector { types, ranges: Vec::new(), }; walk_type_positions(stmts, Some(types), &mut collector); - let needs_import = !collector.ranges.is_empty() && !types.is_bound_globally("LiteralString"); LiteralStringRewrites { + needs_import: !collector.ranges.is_empty(), ranges: collector.ranges, - needs_import, + literal_string: written.imported("typing", "LiteralString"), } } diff --git a/crates/by_transforms/src/transforms/literal_types.rs b/crates/by_transforms/src/transforms/literal_types.rs index a5e0957409..912015d39b 100644 --- a/crates/by_transforms/src/transforms/literal_types.rs +++ b/crates/by_transforms/src/transforms/literal_types.rs @@ -20,6 +20,7 @@ use ruff_text_size::{Ranged, TextRange, TextSize}; use crate::config::FloatLiteralLowering; use crate::transforms::ast_driver::{PassContext, TypeAwarePass}; +use crate::transforms::repeated_underscore::WrittenNames; use crate::transforms::type_expr_walker::{ Recurse, TypeExprVisitor, TypePos, walk_type_positions_skipping, }; @@ -27,6 +28,9 @@ use crate::type_info::{TypeInfo, trailing_name}; pub(crate) struct LiteralType<'src> { source: &'src str, + /// the name `typing.Literal` is written under + literal: String, + written: WrittenNames<'src>, types: &'src dyn TypeInfo, float_literals: FloatLiteralLowering, pub(crate) edits: Vec, @@ -36,11 +40,14 @@ pub(crate) struct LiteralType<'src> { impl<'src> LiteralType<'src> { pub(crate) fn new( source: &'src str, + written: WrittenNames<'src>, types: &'src dyn TypeInfo, float_literals: FloatLiteralLowering, ) -> Self { Self { source, + literal: written.imported("typing", "Literal"), + written, types, float_literals, edits: Vec::new(), @@ -81,14 +88,14 @@ impl<'src> LiteralType<'src> { if is_literal_expr(expr, self.float_literals) { self.needs_literal_import = true; self.edits.push(Fix::safe_edit(Edit::range_replacement( - format!("Literal[{}]", self.src(expr.range())), + format!("{}[{}]", self.literal, self.src(expr.range())), expr.range(), ))); return; } if let Some(nominal) = nominal_float_type(expr, self.float_literals) { self.edits.push(Fix::safe_edit(Edit::range_replacement( - nominal.to_owned(), + self.written.builtin(nominal), expr.range(), ))); return; @@ -170,7 +177,7 @@ impl<'src> LiteralType<'src> { let lit_str = std::mem::take(&mut group_list).join(", "); self.needs_literal_import = true; self.edits.push(Fix::safe_edit(Edit::range_replacement( - format!("Literal[{lit_str}]"), + format!("{}[{lit_str}]", self.literal), TextRange::new(start, group_end), ))); } @@ -212,13 +219,19 @@ impl<'src> LiteralType<'src> { pub(crate) struct LiteralTypePass<'src> { source: &'src str, + written: WrittenNames<'src>, float_literals: FloatLiteralLowering, } impl<'src> LiteralTypePass<'src> { - pub(crate) fn new(source: &'src str, float_literals: FloatLiteralLowering) -> Self { + pub(crate) fn new( + source: &'src str, + written: WrittenNames<'src>, + float_literals: FloatLiteralLowering, + ) -> Self { Self { source, + written, float_literals, } } @@ -230,11 +243,11 @@ impl TypeAwarePass for LiteralTypePass<'_> { } fn run(&self, stmts: &[Stmt], types: &dyn TypeInfo, ctx: &mut PassContext) { - let mut inner = LiteralType::new(self.source, types, self.float_literals); + let mut inner = LiteralType::new(self.source, self.written, types, self.float_literals); walk_type_positions_skipping(stmts, Some(types), &ctx.claimed_type_op_ranges, &mut inner); - if inner.needs_literal_import && !literal_already_imported(types) { + if inner.needs_literal_import { ctx.required_imports - .push("from typing import Literal".to_owned()); + .push(self.written.import_from("typing", &["Literal"])); } for fix in inner.edits { for edit in fix.edits() { @@ -289,10 +302,12 @@ fn is_literal_expr(expr: &Expr, float_literals: FloatLiteralLowering) -> bool { pub(crate) fn float_literal_spelling( expr: &Expr, text: &str, + written: WrittenNames, + literal: &str, float_literals: FloatLiteralLowering, ) -> Option { if let Some(nominal) = nominal_float_type(expr, float_literals) { - return Some(nominal.to_owned()); + return Some(written.builtin(nominal)); } let is_float = |number: &ruff_python_ast::Number| !matches!(number, ruff_python_ast::Number::Int(_)); @@ -306,7 +321,7 @@ pub(crate) fn float_literal_spelling( } _ => return None, }; - is_float(number).then(|| format!("Literal[{text}]")) + is_float(number).then(|| format!("{literal}[{text}]")) } /// the builtin a float or complex literal type is one of, when the project @@ -352,12 +367,6 @@ fn flatten_into<'a>(expr: &'a Expr, out: &mut Vec<&'a Expr>) { out.push(expr); } -/// Whether `Literal` is already bound at module level, so lib.rs can avoid -/// prepending a duplicate import. -pub(crate) fn literal_already_imported(types: &dyn TypeInfo) -> bool { - types.is_bound_globally("Literal") -} - #[cfg(test)] mod tests { use crate::config::FloatLiteralLowering; diff --git a/crates/by_transforms/src/transforms/main_function.rs b/crates/by_transforms/src/transforms/main_function.rs index 3b6ceab5d6..daa15e581a 100644 --- a/crates/by_transforms/src/transforms/main_function.rs +++ b/crates/by_transforms/src/transforms/main_function.rs @@ -73,8 +73,12 @@ impl AstPass for MainFunction<'_> { "main(*_by_args, **_by_kwargs)".to_owned() }; if entry.function.is_async { - ctx.epilogue.push(format!(" asyncio.run({call})")); - ctx.required_imports.push("import asyncio".to_owned()); + ctx.epilogue.push(format!( + " {}.run({call})", + self.written.imported_module("asyncio") + )); + ctx.required_imports + .push(self.written.import_module("asyncio")); } else { ctx.epilogue.push(format!(" {call}")); } diff --git a/crates/by_transforms/src/transforms/match_polyfill.rs b/crates/by_transforms/src/transforms/match_polyfill.rs index c7463d4428..3828e08e98 100644 --- a/crates/by_transforms/src/transforms/match_polyfill.rs +++ b/crates/by_transforms/src/transforms/match_polyfill.rs @@ -67,7 +67,7 @@ use ruff_python_ast::{ use ruff_python_trivia::{SimpleTokenKind, SimpleTokenizer}; use ruff_text_size::{Ranged, TextRange, TextSize}; -use super::repeated_underscore::WrittenNames; +use super::repeated_underscore::{CodeNames, WrittenNames}; use super::source_util::{preamble_offset, temporary_name}; use crate::Config; @@ -88,8 +88,13 @@ pub(crate) fn lower(source: String, config: &Config) -> String { } let parsed = ruff_python_parser::parse_unchecked_source(&source, PySourceType::Python); + // the builtins the tests call are read under names of their own where the module binds + // theirs, as every lowering's are ([`WrittenNames::builtin`]) + let code = CodeNames::of(parsed.suite()); + let written = WrittenNames::new(&source).with_code(&code); let mut lower = Lower { source: &source, + written, edits: Vec::new(), counter: 0, needs: Needs::default(), @@ -108,7 +113,12 @@ pub(crate) fn lower(source: String, config: &Config) -> String { .iter() .any(|(_, replacement)| replacement.contains(MISS)); let body = apply(&source, edits); - let preamble = crate::runtime_preamble(config, &helpers(&needs, sentinel)); + let mut preamble = String::new(); + for import in written.builtin_imports(&body, config.min_version.minor) { + preamble.push_str(&import); + preamble.push('\n'); + } + preamble.push_str(&crate::runtime_preamble(config, &helpers(&needs, sentinel))); let at = preamble_offset(&body); format!("{}{preamble}{}", &body[..at], &body[at..]) } @@ -149,6 +159,7 @@ struct Needs { struct Lower<'src> { source: &'src str, + written: WrittenNames<'src>, edits: Vec<(TextRange, String)>, /// monotonic across the file, so every temporary is distinct counter: usize, @@ -342,11 +353,19 @@ impl Lower<'_> { let star = sequence.patterns.iter().position(Pattern::is_match_star); let fixed = match star { None => { - parts.push(format!("len({subject}) == {}", sequence.patterns.len())); + parts.push(format!( + "{}({subject}) == {}", + self.written.builtin("len"), + sequence.patterns.len() + )); sequence.patterns.len() } Some(star) => { - parts.push(format!("len({subject}) >= {}", sequence.patterns.len() - 1)); + parts.push(format!( + "{}({subject}) >= {}", + self.written.builtin("len"), + sequence.patterns.len() - 1 + )); star } }; @@ -375,7 +394,11 @@ impl Lower<'_> { } else { format!("{subject}[{star}:-{after}]") }; - parts.push(format!("({} := list({slice})) is not {MISS}", name.id)); + parts.push(format!( + "({} := {}({slice})) is not {MISS}", + name.id, + self.written.builtin("list") + )); } } conjoin(parts) @@ -414,7 +437,8 @@ impl Lower<'_> { fn compile_class(&mut self, class: &PatternMatchClass, subject: &str) -> String { let class_name = self.fresh(); let mut parts = vec![format!( - "isinstance({subject}, ({class_name} := {}))", + "{}({subject}, ({class_name} := {}))", + self.written.builtin("isinstance"), self.expr_src(class.cls.range()) )]; diff --git a/crates/by_transforms/src/transforms/match_type.rs b/crates/by_transforms/src/transforms/match_type.rs index 23d29578e3..5a112deadf 100644 --- a/crates/by_transforms/src/transforms/match_type.rs +++ b/crates/by_transforms/src/transforms/match_type.rs @@ -30,14 +30,16 @@ use ruff_python_ast::{ModModule, Stmt, StmtTypeAlias, TypeParam}; use ruff_text_size::{Ranged, TextRange, TextSize}; use super::ast_driver::{AstPass, PassContext}; +use super::repeated_underscore::WrittenNames; pub(crate) struct MatchTypePass<'src> { source: &'src str, + written: WrittenNames<'src>, } impl<'src> MatchTypePass<'src> { - pub(crate) fn new(source: &'src str) -> Self { - Self { source } + pub(crate) fn new(source: &'src str, written: WrittenNames<'src>) -> Self { + Self { source, written } } } @@ -55,6 +57,7 @@ impl AstPass for MatchTypePass<'_> { fn run(&self, module: &mut ModModule, ctx: &mut PassContext) { let mut inner = MatchTypeLowering { source: self.source, + object: self.written.builtin("object"), edits: Vec::new(), }; for stmt in &module.body { @@ -66,6 +69,8 @@ impl AstPass for MatchTypePass<'_> { struct MatchTypeLowering<'src> { source: &'src str, + /// the name the builtin `object` is written under + object: String, edits: Vec<(TextRange, String)>, } @@ -76,7 +81,7 @@ impl<'ast> Visitor<'ast> for MatchTypeLowering<'_> { { self.edits.push(( match_body_range(alias, self.source), - " = object".to_string(), + format!(" = {}", self.object), )); // the case bodies are erased with the rest of the statement, so nothing // inside them is worth descending into diff --git a/crates/by_transforms/src/transforms/modifiers.rs b/crates/by_transforms/src/transforms/modifiers.rs index d83e30366e..d2b14e9c92 100644 --- a/crates/by_transforms/src/transforms/modifiers.rs +++ b/crates/by_transforms/src/transforms/modifiers.rs @@ -17,6 +17,7 @@ //! `class def` → `@classmethod` //! `data class` → `@dataclass(slots=True)` (from `dataclasses`) //! `frozen data class` → `@dataclass(frozen=True, slots=True)` +//! below 3.10 `slots=True` is the runtime's `@_by_dataclass_slots` instead //! `let x = 5` → `x: Final = 5` (from `typing`) //! `class a = 1` → `a: ClassVar = 1` (from `typing`) //! `newtype Foo = int` → `Foo = NewType("Foo", int)` (from `typing`) @@ -38,6 +39,8 @@ use ruff_python_stdlib::basedpython::visibility_rename; use ruff_text_size::{Ranged, TextRange, TextSize}; use super::ast_driver::{AstPass, PassContext}; +use super::repeated_underscore::WrittenNames; +use crate::{Config, PythonVersion}; /// basedpython: the visibility a modifier keyword records, read off the /// synthetic decorator the parser leaves in its place @@ -92,6 +95,7 @@ fn base_head_name(base: &Expr) -> Option<&str> { #[expect(clippy::struct_excessive_bools)] pub(crate) struct Modifiers<'src> { source: &'src str, + written: WrittenNames<'src>, edits: Vec, /// Needs `from typing import final` (decorator for classes/methods) needs_final: bool, @@ -105,6 +109,10 @@ pub(crate) struct Modifiers<'src> { needs_protocol: bool, needs_classvar: bool, needs_newtype: bool, + /// whether a `data class` is given its slots by the runtime helper, on a python whose + /// `dataclass` has no `slots` option + slots_by_helper: bool, + needs_dataclass_slots: bool, /// Names marked `export`/`public` at module level. Used to generate `__all__`. exports: Vec, /// Module-level names renamed by `private` (original → `_original`). @@ -131,9 +139,10 @@ pub(crate) struct Modifiers<'src> { } impl<'src> Modifiers<'src> { - fn new(source: &'src str) -> Self { + fn new(source: &'src str, written: WrittenNames<'src>) -> Self { Self { source, + written, edits: Vec::new(), needs_final: false, needs_final_annotation: false, @@ -144,6 +153,8 @@ impl<'src> Modifiers<'src> { needs_protocol: false, needs_classvar: false, needs_newtype: false, + slots_by_helper: false, + needs_dataclass_slots: false, exports: Vec::new(), private_renames: Vec::new(), sealed_classes: Vec::new(), @@ -225,7 +236,10 @@ impl<'src> Modifiers<'src> { // two insertions rather than a replacement, so the lowerings written // inside the default still apply to it where it now sits self.edits.push(Fix::safe_edit(Edit::insertion( - "field(default_factory=lambda: ".to_owned(), + format!( + "{}(default_factory=lambda: ", + self.written.imported("dataclasses", "field") + ), default.range().start(), ))); self.edits.push(Fix::safe_edit(Edit::insertion( @@ -272,22 +286,27 @@ impl<'src> Modifiers<'src> { "final" => { self.needs_final = true; self.edits.push(Fix::safe_edit(Edit::range_replacement( - format!("@final\n{indent}"), + format!("@{}\n{indent}", self.written.imported("typing", "final")), dec.range(), ))); } - "data_class" => { + "data_class" | "frozen_data_class" => { self.needs_dataclass = true; + let dataclass = self.written.imported("dataclasses", "dataclass"); + let frozen = name == "frozen_data_class"; + let decorators = if self.slots_by_helper { + self.needs_dataclass_slots = true; + let options = if frozen { "(frozen=True)" } else { "" }; + format!( + "@{}\n{indent}@{dataclass}{options}\n{indent}", + crate::runtime::DATACLASS_SLOTS.name() + ) + } else { + let frozen = if frozen { "frozen=True, " } else { "" }; + format!("@{dataclass}({frozen}slots=True)\n{indent}") + }; self.edits.push(Fix::safe_edit(Edit::range_replacement( - format!("@dataclass(slots=True)\n{indent}"), - dec.range(), - ))); - self.wrap_mutable_field_defaults(class); - } - "frozen_data_class" => { - self.needs_dataclass = true; - self.edits.push(Fix::safe_edit(Edit::range_replacement( - format!("@dataclass(frozen=True, slots=True)\n{indent}"), + decorators, dec.range(), ))); self.wrap_mutable_field_defaults(class); @@ -344,21 +363,24 @@ impl<'src> Modifiers<'src> { "abstract" => { self.needs_abstractmethod = true; self.edits.push(Fix::safe_edit(Edit::range_replacement( - format!("@abstractmethod\n{indent}"), + format!( + "@{}\n{indent}", + self.written.imported("abc", "abstractmethod") + ), dec.range(), ))); } "final" => { self.needs_final = true; self.edits.push(Fix::safe_edit(Edit::range_replacement( - format!("@final\n{indent}"), + format!("@{}\n{indent}", self.written.imported("typing", "final")), dec.range(), ))); } "override" => { self.needs_override = true; self.edits.push(Fix::safe_edit(Edit::range_replacement( - format!("@override\n{indent}"), + format!("@{}\n{indent}", self.written.imported("typing", "override")), dec.range(), ))); } @@ -369,13 +391,13 @@ impl<'src> Modifiers<'src> { } "static" => { self.edits.push(Fix::safe_edit(Edit::range_replacement( - format!("@staticmethod\n{indent}"), + format!("@{}\n{indent}", self.written.builtin("staticmethod")), dec.range(), ))); } "classmethod" => { self.edits.push(Fix::safe_edit(Edit::range_replacement( - format!("@classmethod\n{indent}"), + format!("@{}\n{indent}", self.written.builtin("classmethod")), dec.range(), ))); } @@ -412,7 +434,7 @@ impl<'src> Modifiers<'src> { fn rename_with_underscore(&mut self, range: TextRange) { let original = self.src(range).to_owned(); self.edits.push(Fix::safe_edit(Edit::range_replacement( - module_private_name(&original), + self.written.module_private(&original), range, ))); } @@ -490,7 +512,7 @@ impl<'src> Modifiers<'src> { // `private def` is, and for the same reason. its references are // renamed by `visibility_rename`, which asks ty where each one resolves self.private_renames.push(written.clone()); - module_private_name(&written) + self.written.module_private(&written) } else { written }; @@ -527,7 +549,7 @@ impl<'src> Modifiers<'src> { let pre_range = TextRange::new(stmt_start, slice.range().start()); self.needs_classvar = true; self.edits.push(Fix::safe_edit(Edit::range_replacement( - format!("{name}: ClassVar["), + format!("{name}: {}[", self.written.imported("typing", "ClassVar")), pre_range, ))); self.edits.push(Fix::safe_edit(Edit::insertion( @@ -543,7 +565,7 @@ impl<'src> Modifiers<'src> { let pre_range = TextRange::new(stmt_start, slice.range().start()); self.needs_final_annotation = true; self.edits.push(Fix::safe_edit(Edit::range_replacement( - format!("{name}: Final["), + format!("{name}: {}[", self.written.imported("typing", "Final")), pre_range, ))); self.edits.push(Fix::safe_edit(Edit::insertion( @@ -556,7 +578,7 @@ impl<'src> Modifiers<'src> { // read-only declaration self.needs_final_annotation = true; self.edits.push(Fix::safe_edit(Edit::range_replacement( - format!("{name}: Final"), + format!("{name}: {}", self.written.imported("typing", "Final")), TextRange::new(stmt_start, node.range().end()), ))); } @@ -572,7 +594,7 @@ impl<'src> Modifiers<'src> { id if is_let_marker_id(id) => { self.needs_final_annotation = true; self.edits.push(Fix::safe_edit(Edit::range_replacement( - format!("{name}: Final = "), + format!("{name}: {} = ", self.written.imported("typing", "Final")), prefix_range, ))); } @@ -595,7 +617,7 @@ impl<'src> Modifiers<'src> { id if is_classvar_marker_id(id) => { self.needs_classvar = true; self.edits.push(Fix::safe_edit(Edit::range_replacement( - format!("{name}: ClassVar = "), + format!("{name}: {} = ", self.written.imported("typing", "ClassVar")), prefix_range, ))); } @@ -603,7 +625,10 @@ impl<'src> Modifiers<'src> { let value_src = self.src(value_range).to_owned(); self.needs_newtype = true; self.edits.push(Fix::safe_edit(Edit::range_replacement( - format!("{name} = NewType(\"{name}\", {value_src})"), + format!( + "{name} = {}(\"{name}\", {value_src})", + self.written.imported("typing", "NewType") + ), TextRange::new(stmt_start, value_range.end()), ))); } @@ -619,7 +644,7 @@ impl<'src> Modifiers<'src> { TextRange::new(slice.range().end(), self.value_range(node, value).start()); self.needs_classvar = true; self.edits.push(Fix::safe_edit(Edit::range_replacement( - format!("{name}: ClassVar["), + format!("{name}: {}[", self.written.imported("typing", "ClassVar")), pre_range, ))); self.edits.push(Fix::safe_edit(Edit::range_replacement( @@ -655,7 +680,7 @@ impl<'src> Modifiers<'src> { } else { self.needs_final_annotation = true; self.edits.push(Fix::safe_edit(Edit::range_replacement( - format!("{name}: Final["), + format!("{name}: {}[", self.written.imported("typing", "Final")), pre_range, ))); self.edits.push(Fix::safe_edit(Edit::range_replacement( @@ -692,7 +717,8 @@ impl<'src> Modifiers<'src> { } fn insert_protocol_base(&mut self, class: &StmtClassDef) { - self.insert_base_class(class, "Protocol"); + let protocol = self.written.imported("typing", "Protocol"); + self.insert_base_class(class, &protocol); } } @@ -749,12 +775,20 @@ pub(crate) fn module_private_name(name: &str) -> String { pub(crate) struct ModifiersPass<'src> { source: &'src str, + written: WrittenNames<'src>, is_stub: bool, + /// whether the python the module targets has `dataclass(slots=True)` + dataclass_slots: bool, } impl<'src> ModifiersPass<'src> { - pub(crate) fn new(source: &'src str, is_stub: bool) -> Self { - Self { source, is_stub } + pub(crate) fn new(source: &'src str, written: WrittenNames<'src>, config: &Config) -> Self { + Self { + source, + written, + is_stub: config.is_stub, + dataclass_slots: config.min_version >= PythonVersion::PY310, + } } } @@ -770,7 +804,9 @@ impl AstPass for ModifiersPass<'_> { } fn run(&self, module: &mut ruff_python_ast::ModModule, ctx: &mut PassContext) { - let mut inner = Modifiers::new(self.source); + let mut inner = Modifiers::new(self.source, self.written); + // a stub is never run, so it keeps the option a checker reads + inner.slots_by_helper = !self.dataclass_slots && !self.is_stub; for stmt in &module.body { inner.visit_stmt(stmt); } @@ -785,42 +821,24 @@ impl AstPass for ModifiersPass<'_> { }; let class_bases = std::mem::take(&mut inner.class_bases); - // typing import grouping mirrors lib.rs's preamble logic - let mut typing_imports: Vec<&'static str> = Vec::new(); - if inner.needs_final { - typing_imports.push("final"); - } - if inner.needs_final_annotation { - typing_imports.push("Final"); + if inner.needs_dataclass_slots { + ctx.runtime.insert(crate::runtime::DATACLASS_SLOTS); } - if inner.needs_classvar { - typing_imports.push("ClassVar"); - } - if inner.needs_newtype { - typing_imports.push("NewType"); - } - if inner.needs_override { - typing_imports.push("override"); - } - for name in typing_imports { - ctx.required_imports - .push(format!("from typing import {name}")); - } - if inner.needs_abstractmethod { - ctx.required_imports - .push("from abc import abstractmethod".to_owned()); - } - if inner.needs_dataclass { - ctx.required_imports - .push("from dataclasses import dataclass".to_owned()); - } - if inner.needs_dataclass_field { - ctx.required_imports - .push("from dataclasses import field".to_owned()); - } - if inner.needs_protocol { - ctx.required_imports - .push("from typing import Protocol".to_owned()); + for (needed, module, name) in [ + (inner.needs_final, "typing", "final"), + (inner.needs_final_annotation, "typing", "Final"), + (inner.needs_classvar, "typing", "ClassVar"), + (inner.needs_newtype, "typing", "NewType"), + (inner.needs_override, "typing", "override"), + (inner.needs_abstractmethod, "abc", "abstractmethod"), + (inner.needs_dataclass, "dataclasses", "dataclass"), + (inner.needs_dataclass_field, "dataclasses", "field"), + (inner.needs_protocol, "typing", "Protocol"), + ] { + if needed { + ctx.required_imports + .push(self.written.import_from(module, &[name])); + } } for fix in inner.edits { @@ -1020,6 +1038,26 @@ mod tests { ); } + /// the class body binds `staticmethod` itself, so the decorator reads the builtin under + /// a name of its own + #[test] + fn static_def_in_a_class_that_binds_staticmethod() { + check( + indoc! {" + class A: + staticmethod = 3 + static def helper(): ... + "}, + indoc! {" + from builtins import staticmethod as staticmethod2 + class A: + staticmethod: int = 3 + @staticmethod2 + def helper(): ... + "}, + ); + } + #[test] fn class_def() { check( @@ -1110,6 +1148,35 @@ mod tests { ); } + /// `dataclass` has no `slots` option below 3.10, so the runtime's helper does what it + /// would, to the class `dataclass` returns + #[test] + fn data_class_below_python_310() { + let out = transpile( + indoc! {" + data class Point: + x: int + + final frozen data class Frozen: + x: int + "}, + &Config { + min_version: PythonVersion::PY39, + ..Config::test_default() + }, + ) + .unwrap(); + assert!( + out.contains("@_by_dataclass_slots\n@dataclass\nclass Point:") + && out.contains( + "@final\n@_by_dataclass_slots\n@dataclass(frozen=True)\nclass Frozen:" + ) + && out.contains("def _by_dataclass_slots(cls):") + && !out.contains("slots=True"), + "got:\n{out}" + ); + } + #[test] fn nested_modifiers_in_class() { // Default min_version is PY310; `typing.override` was added in PY312 @@ -1718,6 +1785,52 @@ mod tests { ); } + #[test] + fn private_type_alias_already_underscored_polyfilled() { + check( + indoc! {" + private type _Alias = int + + def f(x: _Alias) -> _Alias: + return x + "}, + indoc! {" + from typing_extensions import TypeAliasType + _Alias = TypeAliasType(\"_Alias\", int) + + def f(x: _Alias) -> _Alias: + return x + "}, + ); + } + + /// `_force_unwrap` is the runtime helper `x!` calls, and `_count` the module's own + #[test] + fn a_private_symbol_is_emitted_under_a_name_the_module_does_not_have() { + let out = transpile( + indoc! {" + _count = 0 + + private def force_unwrap(x: int) -> int: + return x + + private def count() -> int: + return force_unwrap(_count) + + def first(x: int | None) -> int: + return x! + "}, + &Config::test_default(), + ) + .unwrap(); + assert!( + out.contains("def _force_unwrap2(x: int) -> int:") + && out.contains("def _count2() -> int:\n return _force_unwrap2(_count)") + && out.contains("return _force_unwrap(x)"), + "got:\n{out}" + ); + } + #[test] fn private_generic_type_alias_polyfilled() { check( diff --git a/crates/by_transforms/src/transforms/mutable_defaults.rs b/crates/by_transforms/src/transforms/mutable_defaults.rs index 1be5ecb657..db5b12080f 100644 --- a/crates/by_transforms/src/transforms/mutable_defaults.rs +++ b/crates/by_transforms/src/transforms/mutable_defaults.rs @@ -72,6 +72,8 @@ pub(crate) enum Guard { name: String, sentinel: String, function: String, + /// the name the builtin `TypeError` is written under + type_error: String, }, /// bind `_` to the parameter that was the first `_`, for a body that reads it. a /// repeated `_` that takes its name from an overridden method leaves no parameter @@ -107,9 +109,10 @@ impl PrologueStatement for Guard { name, sentinel, function, + type_error, } => { frags.push(Fragment::Lit(format!( - "if {name} is {sentinel}:\n{base} raise TypeError(\"{function}() missing required argument: '{name}'\")" + "if {name} is {sentinel}:\n{base} raise {type_error}(\"{function}() missing required argument: '{name}'\")" ))); } Guard::Rebind { parameter } => { @@ -149,10 +152,10 @@ fn sentinel_edit(default: TextRange, sentinel: &str) -> (TextRange, Vec String { - WrittenNames::new(source).fresh("_MISSING") +pub(crate) fn sentinel_name(written: WrittenNames) -> String { + written.fresh("_MISSING") } /// The lines the output needs before it can use the sentinel: the sentinel itself, and the @@ -163,10 +166,15 @@ pub(crate) fn sentinel_name(source: &str) -> String { /// says so. Declaring it `Any` is how typeshed declares its own sentinels, and it costs the /// output nothing: the value never reaches the body, whose guard replaces it before anything /// reads it. -pub(crate) fn sentinel_definition(source: &str) -> [String; 2] { +pub(crate) fn sentinel_definition(written: WrittenNames) -> [String; 2] { [ - "from typing import Any".to_owned(), - format!("{}: Any = object()", sentinel_name(source)), + written.import_from("typing", &["Any"]), + format!( + "{}: {} = {}()", + sentinel_name(written), + written.imported("typing", "Any"), + written.builtin("object") + ), ] } @@ -258,6 +266,7 @@ pub(crate) fn parameter_guards( .to_string(), sentinel: sentinel.to_owned(), function: f.name.id.to_string(), + type_error: written_names.builtin("TypeError"), }); } None => {} @@ -370,7 +379,7 @@ impl MutableDefaults<'_> { } = parameter_guards( f, self.written, - &sentinel_name(self.source), + &sentinel_name(self.written), self.types, self.is_stub, ); @@ -457,7 +466,7 @@ impl TypeAwarePass for MutableDefaultsPass<'_> { } if inner.used { ctx.required_imports - .extend(sentinel_definition(self.source)); + .extend(sentinel_definition(self.written)); } ctx.template_edits.extend(inner.edits); ctx.relocating_edits.extend(inner.relocating); diff --git a/crates/by_transforms/src/transforms/optional_type.rs b/crates/by_transforms/src/transforms/optional_type.rs index 7d0da61513..2a915a797c 100644 --- a/crates/by_transforms/src/transforms/optional_type.rs +++ b/crates/by_transforms/src/transforms/optional_type.rs @@ -32,6 +32,7 @@ use ruff_python_ast::{Expr, PythonVersion, Stmt, UnaryOp}; use ruff_text_size::{Ranged, TextRange}; use super::ast_driver::{PassContext, TypeAwarePass}; +use super::repeated_underscore::WrittenNames; use crate::type_info::TypeInfo; /// Walks the source AST and emits narrow text edits that lower each optional in @@ -46,6 +47,10 @@ struct OptionalLower<'src> { needs_union: bool, /// whether the target can spell a union with `|` (python 3.10) native_union: bool, + /// the name `typing.Union` is written under + union: String, + /// the name the builtin `type` is written under + type_: String, /// the optionals standing where `isinstance` expects classes, which a target /// without `|` spells as a tuple of classes, see [`super::runtime_union`] classinfo: HashSet, @@ -58,12 +63,14 @@ struct OptionalLower<'src> { } impl<'src> OptionalLower<'src> { - fn new(source: &'src str, min_version: PythonVersion) -> Self { + fn new(source: &'src str, written: WrittenNames, min_version: PythonVersion) -> Self { Self { edits: Vec::new(), needs_runtime: false, needs_union: false, native_union: min_version >= PythonVersion::PY310, + union: written.imported("typing", "Union"), + type_: written.builtin("type"), classinfo: HashSet::new(), source, typevar_scopes: Vec::new(), @@ -147,14 +154,17 @@ impl<'ast> Visitor<'ast> for OptionalLower<'_> { prefix.push('('); } else if !self.native_union { self.needs_union = true; - prefix.push_str("Union["); + prefix.push_str(&self.union); + prefix.push('['); } if !prefix.is_empty() { self.edits.push((TextRange::empty(node.start()), prefix)); } let mut replacement = close_parens; if classinfo { - replacement.push_str(", type(None),)"); + replacement.push_str(", "); + replacement.push_str(&self.type_); + replacement.push_str("(None),)"); } else if self.native_union { replacement.push_str(" | None"); } else { @@ -174,13 +184,19 @@ impl<'ast> Visitor<'ast> for OptionalLower<'_> { pub(crate) struct OptionalTypePass<'src> { source: &'src str, + written: WrittenNames<'src>, min_version: PythonVersion, } impl<'src> OptionalTypePass<'src> { - pub(crate) fn new(source: &'src str, min_version: PythonVersion) -> Self { + pub(crate) fn new( + source: &'src str, + written: WrittenNames<'src>, + min_version: PythonVersion, + ) -> Self { Self { source, + written, min_version, } } @@ -195,10 +211,11 @@ impl<'src> OptionalTypePass<'src> { /// which independently walks every type position. fn collect_edits( source: &str, + written: WrittenNames, expr: &Expr, min_version: PythonVersion, ) -> Vec<(TextRange, String)> { - let mut lower = OptionalLower::new(source, min_version); + let mut lower = OptionalLower::new(source, written, min_version); lower.visit_expr(expr); lower.edits } @@ -208,10 +225,11 @@ fn collect_edits( /// kw-subscript) to lower a nested `T?` when they splice their element types. pub(crate) fn rewrite_type_expr( source: &str, + written: WrittenNames, expr: &Expr, min_version: PythonVersion, ) -> Option { - let mut edits = collect_edits(source, expr, min_version); + let mut edits = collect_edits(source, written, expr, min_version); if edits.is_empty() { return None; } @@ -237,7 +255,7 @@ impl TypeAwarePass for OptionalTypePass<'_> { } fn run(&self, stmts: &[Stmt], types: &dyn TypeInfo, ctx: &mut PassContext) { - let mut lower = OptionalLower::new(self.source, self.min_version); + let mut lower = OptionalLower::new(self.source, self.written, self.min_version); if !lower.native_union { lower.classinfo = super::runtime_union::classinfo_optionals(stmts, types); } @@ -249,7 +267,7 @@ impl TypeAwarePass for OptionalTypePass<'_> { } if lower.needs_union { ctx.required_imports - .push("from typing import Union".to_owned()); + .push(self.written.import_from("typing", &["Union"])); } ctx.text_edits.extend(lower.edits); } diff --git a/crates/by_transforms/src/transforms/overload.rs b/crates/by_transforms/src/transforms/overload.rs index 8a4c361702..429a5b7ee3 100644 --- a/crates/by_transforms/src/transforms/overload.rs +++ b/crates/by_transforms/src/transforms/overload.rs @@ -25,15 +25,21 @@ use ruff_python_ast::{Expr, ModModule, Stmt, StmtFunctionDef}; use ruff_text_size::{Ranged, TextRange, TextSize}; use super::ast_driver::{AstPass, PassContext}; +use super::repeated_underscore::WrittenNames; pub(crate) struct Overload<'src> { source: &'src str, + written: WrittenNames<'src>, is_stub: bool, } impl<'src> Overload<'src> { - pub(crate) fn new(source: &'src str, is_stub: bool) -> Self { - Self { source, is_stub } + pub(crate) fn new(source: &'src str, written: WrittenNames<'src>, is_stub: bool) -> Self { + Self { + source, + written, + is_stub, + } } } @@ -41,54 +47,31 @@ impl AstPass for Overload<'_> { fn run(&self, module: &mut ModModule, ctx: &mut PassContext) { let mut state = State { source: self.source, + overload: self.written.imported("typing", "overload"), + not_implemented: self.written.builtin("NotImplementedError"), is_stub: self.is_stub, edits: RefCell::new(Vec::new()), - first_emitted: None, + writes_overload: false, }; state.visit_body(&module.body); - if let Some(first_emitted) = state.first_emitted - && !imports_overload_before(&module.body, first_emitted) - { + if state.writes_overload { ctx.required_imports - .push("from typing import overload".to_owned()); + .push(self.written.import_from("typing", &["overload"])); } ctx.text_edits.extend(state.edits.into_inner()); } } -/// whether the module already binds `overload` at `pos`, through a top-level -/// `from typing import overload` of its own. -/// -/// the name is read where a decorator runs, which is when the `def` below it is -/// evaluated — so an import after that `def` binds the name too late and the output -/// still needs its own. an alias binds some other name, and an import nested in a -/// function or under `if TYPE_CHECKING:` is not in scope at module level at all, -/// so neither is one of these -fn imports_overload_before(body: &[Stmt], pos: TextSize) -> bool { - body.iter().any(|stmt| { - let Stmt::ImportFrom(import) = stmt else { - return false; - }; - import.level == 0 - && import - .module - .as_ref() - .is_some_and(|m| m.as_str() == "typing") - && import.range().end() <= pos - && import - .names - .iter() - .any(|alias| alias.name.as_str() == "overload" && alias.asname.is_none()) - }) -} - struct State<'src> { source: &'src str, + /// the name `typing.overload` is written under + overload: String, + /// the name the builtin `NotImplementedError` is written under + not_implemented: String, is_stub: bool, edits: RefCell>, - /// where the earliest `@overload` this pass writes goes, which is the position the - /// name has to be bound by. `None` when it writes none and needs no import - first_emitted: Option, + /// whether this pass writes an `@overload`, which needs the import + writes_overload: bool, } impl State<'_> { @@ -132,11 +115,11 @@ impl State<'_> { // the stub idiom and round-trips with the reverse pass. only a runtime // `.by` file needs the `raise NotImplementedError` body let body = if self.is_abstract(func) && !self.is_stub { - ": raise NotImplementedError" + format!(": raise {}", self.not_implemented) } else { - ": ..." + ": ...".to_owned() }; - self.push(TextRange::new(end, end), body.to_owned()); + self.push(TextRange::new(end, end), body); } fn is_stub_shaped(func: &StmtFunctionDef) -> bool { @@ -165,12 +148,10 @@ impl State<'_> { fn add_overload_stub(&mut self, func: &StmtFunctionDef) { if !Self::wears_overload(func) { let start = func.range().start(); - self.first_emitted = Some(match self.first_emitted { - Some(first) => first.min(start), - None => start, - }); + self.writes_overload = true; let indent = self.line_indent(start).to_owned(); - self.push(TextRange::new(start, start), format!("@overload\n{indent}")); + let decorator = format!("@{}\n{indent}", self.overload); + self.push(TextRange::new(start, start), decorator); } if func.body.is_empty() { let end = func.range().end(); diff --git a/crates/by_transforms/src/transforms/parametric_is.rs b/crates/by_transforms/src/transforms/parametric_is.rs index fed78e432b..0fc33740ec 100644 --- a/crates/by_transforms/src/transforms/parametric_is.rs +++ b/crates/by_transforms/src/transforms/parametric_is.rs @@ -58,6 +58,7 @@ use ruff_text_size::{Ranged, TextRange}; use ty_python_semantic::{ArgVariance, ParametricIsPlan, ProtocolMemberCheck, TargetSpelling}; use super::ast_driver::{Fragment, PassContext, TypeAwarePass}; +use super::repeated_underscore::WrittenNames; use crate::type_info::TypeInfo; /// probes a value's `__orig_class__` against a target alias — the runtime @@ -188,6 +189,7 @@ impl PredicateNeeds { /// `probe`, which decides what an unreified value means. fn arm_predicate( types: &dyn TypeInfo, + written: WrittenNames, value_ref: &dyn Fn() -> Fragment, value_expr: &Expr, arm: &Expr, @@ -198,14 +200,14 @@ fn arm_predicate( needs.all_true = false; needs.references_value = true; return vec![ - Fragment::Lit("isinstance(".to_owned()), + Fragment::Lit(format!("{}(", written.builtin("isinstance"))), value_ref(), Fragment::Lit(", ".to_owned()), Fragment::Src(arm.range()), Fragment::Lit(")".to_owned()), ]; }; - fragments_for_plan(types, plan, value_ref, arm, probe, needs) + fragments_for_plan(types, written, plan, value_ref, arm, probe, needs) } /// the fragments one already-classified plan lowers to. split out from @@ -213,6 +215,7 @@ fn arm_predicate( /// not spell separately — a PEP 695 alias names a whole union with one word fn fragments_for_plan( types: &dyn TypeInfo, + written: WrittenNames, plan: ParametricIsPlan, value_ref: &dyn Fn() -> Fragment, arm: &Expr, @@ -319,7 +322,7 @@ fn fragments_for_plan( ]; } vec![ - Fragment::Lit("isinstance(".to_owned()), + Fragment::Lit(format!("{}(", written.builtin("isinstance"))), value_ref(), Fragment::Lit(", ".to_owned()), Fragment::Src(arm.range()), @@ -329,7 +332,7 @@ fn fragments_for_plan( ParametricIsPlan::Isinstance(target) => { needs.references_value = true; let mut frags = vec![ - Fragment::Lit("isinstance(".to_owned()), + Fragment::Lit(format!("{}(", written.builtin("isinstance"))), value_ref(), Fragment::Lit(", ".to_owned()), ]; @@ -343,7 +346,7 @@ fn fragments_for_plan( ParametricIsPlan::IsCallable => { needs.references_value = true; vec![ - Fragment::Lit("callable(".to_owned()), + Fragment::Lit(format!("{}(", written.builtin("callable"))), value_ref(), Fragment::Lit(")".to_owned()), ] @@ -357,7 +360,7 @@ fn fragments_for_plan( ParametricIsPlan::Equality { class, value } => { needs.references_value = true; vec![ - Fragment::Lit("(type(".to_owned()), + Fragment::Lit(format!("({}(", written.builtin("type"))), value_ref(), Fragment::Lit(format!(") is {class} and ")), value_ref(), @@ -376,8 +379,15 @@ fn fragments_for_plan( } ParametricIsPlan::Subclass(target) => { needs.references_value = true; - let mut frags = vec![Fragment::Lit("(isinstance(".to_owned()), value_ref()]; - frags.push(Fragment::Lit(", type) and issubclass(".to_owned())); + let mut frags = vec![ + Fragment::Lit(format!("({}(", written.builtin("isinstance"))), + value_ref(), + ]; + frags.push(Fragment::Lit(format!( + ", {}) and {}(", + written.builtin("type"), + written.builtin("issubclass") + ))); frags.push(value_ref()); frags.push(Fragment::Lit(", ".to_owned())); frags.push(target_fragment(&target, arm)); @@ -406,6 +416,7 @@ fn fragments_for_plan( any_true |= matches!(arm_plan, ParametricIsPlan::Fold(true)); frags.extend(fragments_for_plan( types, + written, arm_plan.clone(), value_ref, arm, @@ -459,13 +470,16 @@ fn python_string_literal(text: &str) -> String { /// the arms of a union the source never spelled as one. pub(crate) fn build_predicate( types: &dyn TypeInfo, + written: WrittenNames, value_ref: &dyn Fn() -> Fragment, value_expr: &Expr, target: &Expr, probe: ProbeStrictness, ) -> (Vec, PredicateNeeds) { let mut needs = PredicateNeeds::new(); - let frags = arm_predicate(types, value_ref, value_expr, target, probe, &mut needs); + let frags = arm_predicate( + types, written, value_ref, value_expr, target, probe, &mut needs, + ); (frags, needs) } @@ -476,6 +490,7 @@ fn effect_free(expr: &Expr) -> bool { struct ParametricIs<'src, 'ti> { source: &'src str, + written: WrittenNames<'src>, types: &'ti dyn TypeInfo, edits: Vec<(TextRange, Vec)>, /// the runtime helpers the predicates emitted so far call @@ -510,12 +525,25 @@ impl ParametricIs<'_, '_> { mentions.set(mentions.get() + 1); Fragment::Src(lhs.range()) }; - let (frags, needs) = - build_predicate(self.types, &counting, lhs, rhs, ProbeStrictness::Strict); + let (frags, needs) = build_predicate( + self.types, + self.written, + &counting, + lhs, + rhs, + ProbeStrictness::Strict, + ); let via_lambda = mentions.get() > 1 && !effect_free(lhs); let (frags, needs) = if via_lambda { let param = || Fragment::Lit(UNION_VALUE_PARAM.to_owned()); - build_predicate(self.types, ¶m, lhs, rhs, ProbeStrictness::Strict) + build_predicate( + self.types, + self.written, + ¶m, + lhs, + rhs, + ProbeStrictness::Strict, + ) } else { (frags, needs) }; @@ -683,11 +711,12 @@ impl<'ast> Visitor<'ast> for ParametricIs<'_, '_> { pub(crate) struct ParametricIsPass<'src> { source: &'src str, + written: WrittenNames<'src>, } impl<'src> ParametricIsPass<'src> { - pub(crate) fn new(source: &'src str) -> Self { - Self { source } + pub(crate) fn new(source: &'src str, written: WrittenNames<'src>) -> Self { + Self { source, written } } } @@ -704,6 +733,7 @@ impl TypeAwarePass for ParametricIsPass<'_> { fn run(&self, stmts: &[Stmt], types: &dyn TypeInfo, ctx: &mut PassContext) { let mut inner = ParametricIs { source: self.source, + written: self.written, types, edits: Vec::new(), runtimes: BTreeSet::new(), diff --git a/crates/by_transforms/src/transforms/propagate.rs b/crates/by_transforms/src/transforms/propagate.rs index 81ce4aed9c..a4d98daeae 100644 --- a/crates/by_transforms/src/transforms/propagate.rs +++ b/crates/by_transforms/src/transforms/propagate.rs @@ -7,9 +7,9 @@ //! //! ```python //! # f()^.bar (f() : int?) becomes -//! _prop0 = f() -//! if _prop0 is None: return _prop0 -//! _prop0.bar +//! __by_prop_0__ = f() +//! if __by_prop_0__ is None: return __by_prop_0__ +//! __by_prop_0__.bar //! ``` //! //! A trivially-pure operand (a bare name) needs no temp: @@ -27,15 +27,21 @@ use ruff_python_ast::{Expr, Stmt, UnaryOp}; use ruff_text_size::{Ranged, TextRange, TextSize}; use super::ast_driver::{PassContext, TypeAwarePass}; +use super::repeated_underscore::WrittenNames; +use super::source_util::temporary_name; use crate::type_info::{AbsentTest, TypeInfo}; /// The guard condition that detects the "absent" value for a given operand /// `target` (a temp name or the operand source). `T?` tests `is None`; a /// result-like `T | E` tests `isinstance(_, BaseException)`. -fn absent_condition(test: AbsentTest, target: &str) -> String { +fn absent_condition(test: AbsentTest, target: &str, written: WrittenNames) -> String { match test { AbsentTest::Optional | AbsentTest::WrappedOptional => format!("{target} is None"), - AbsentTest::Result => format!("isinstance({target}, BaseException)"), + AbsentTest::Result => format!( + "{}({target}, {})", + written.builtin("isinstance"), + written.builtin("BaseException") + ), } } @@ -67,6 +73,7 @@ enum Scope { struct Propagate<'src> { source: &'src str, + written: WrittenNames<'src>, types: &'src dyn TypeInfo, edits: Vec<(TextRange, String)>, /// (start offset, indentation) of each enclosing statement; the top is the @@ -81,9 +88,10 @@ struct Propagate<'src> { } impl<'src> Propagate<'src> { - fn new(source: &'src str, types: &'src dyn TypeInfo) -> Self { + fn new(source: &'src str, written: WrittenNames<'src>, types: &'src dyn TypeInfo) -> Self { Self { source, + written, types, edits: Vec::new(), stmt_stack: Vec::new(), @@ -177,15 +185,15 @@ impl<'ast> Visitor<'ast> for Propagate<'_> { "" }; let (guard, value) = if is_trivially_pure(&unary.operand) { - let cond = absent_condition(absent, &operand_src); + let cond = absent_condition(absent, &operand_src, self.written); ( format!("if {cond}: return {operand_src}\n{indent}"), format!("{operand_src}{unwrap}"), ) } else { - let temp = format!("_prop{}", self.counter); + let temp = temporary_name(self.written, "prop", self.counter); self.counter += 1; - let cond = absent_condition(absent, &temp); + let cond = absent_condition(absent, &temp, self.written); ( format!("{temp} = {operand_src}\n{indent}if {cond}: return {temp}\n{indent}"), format!("{temp}{unwrap}"), @@ -204,17 +212,18 @@ impl<'ast> Visitor<'ast> for Propagate<'_> { pub(crate) struct PropagatePass<'src> { source: &'src str, + written: WrittenNames<'src>, } impl<'src> PropagatePass<'src> { - pub(crate) fn new(source: &'src str) -> Self { - Self { source } + pub(crate) fn new(source: &'src str, written: WrittenNames<'src>) -> Self { + Self { source, written } } } impl TypeAwarePass for PropagatePass<'_> { fn run(&self, stmts: &[Stmt], types: &dyn TypeInfo, ctx: &mut PassContext) { - let mut inner = Propagate::new(self.source, types); + let mut inner = Propagate::new(self.source, self.written, types); for stmt in stmts { inner.visit_stmt(stmt); } @@ -269,9 +278,9 @@ mod tests { return Optional(5) def f() -> int | None: - _prop0 = g() - if _prop0 is None: return _prop0 - x = _prop0.value + __by_prop_0__ = g() + if __by_prop_0__ is None: return __by_prop_0__ + x = __by_prop_0__.value return x "}, ); @@ -327,9 +336,9 @@ mod tests { return None def g() -> str: - _prop0 = f() - if _prop0 is None: return _prop0 - _prop0.__class__.__name__ + __by_prop_0__ = f() + if __by_prop_0__ is None: return __by_prop_0__ + __by_prop_0__.__class__.__name__ "}, ); } @@ -344,9 +353,9 @@ mod tests { "}, indoc! {" def g(a) -> int | None: - _prop0 = a() - if _prop0 is None: return _prop0 - y = _prop0 + __by_prop_0__ = a() + if __by_prop_0__ is None: return __by_prop_0__ + y = __by_prop_0__ return y "}, ); @@ -368,14 +377,39 @@ mod tests { def f() -> int | TypeError: ... def m() -> int | TypeError: - _prop0 = f() - if isinstance(_prop0, BaseException): return _prop0 - x = _prop0 + __by_prop_0__ = f() + if isinstance(__by_prop_0__, BaseException): return __by_prop_0__ + x = __by_prop_0__ return x "}, ); } + /// the guard reads the builtins through names of its own where the function binds + /// theirs, and its temporary is a name the function cannot bind + #[test] + fn propagate_result_keeps_clear_of_the_functions_names() { + check( + indoc! {" + def f() -> int | TypeError: ... + + def m(isinstance: int, _prop0: int) -> int | TypeError: + x = f()^ + return x + isinstance + _prop0 + "}, + indoc! {" + from builtins import isinstance as isinstance2 + def f() -> int | TypeError: ... + + def m(isinstance: int, _prop0: int) -> int | TypeError: + __by_prop_0__ = f() + if isinstance2(__by_prop_0__, BaseException): return __by_prop_0__ + x = __by_prop_0__ + return x + isinstance + _prop0 + "}, + ); + } + /// a trivially-pure result operand needs no temp; the guard tests the /// name directly #[test] diff --git a/crates/by_transforms/src/transforms/properties.rs b/crates/by_transforms/src/transforms/properties.rs index e184b9726a..81ddb8ff87 100644 --- a/crates/by_transforms/src/transforms/properties.rs +++ b/crates/by_transforms/src/transforms/properties.rs @@ -53,6 +53,7 @@ use ruff_python_stdlib::basedpython::visibility_rename; use ruff_text_size::{Ranged, TextRange, TextSize}; use super::ast_driver::{Fragment, PassContext, TypeAwarePass, render_stmt}; +use super::repeated_underscore::WrittenNames; use super::source_util::{line_indent, line_start}; use crate::type_info::TypeInfo; @@ -128,6 +129,7 @@ impl<'ast> Visitor<'ast> for ValueRangeCollector<'_> { pub(crate) struct PropertiesPass<'src> { source: &'src str, + written: WrittenNames<'src>, value_ranges: ValueRanges, min_version: ruff_python_ast::PythonVersion, } @@ -135,11 +137,13 @@ pub(crate) struct PropertiesPass<'src> { impl<'src> PropertiesPass<'src> { pub(crate) fn new( source: &'src str, + written: WrittenNames<'src>, value_ranges: ValueRanges, min_version: ruff_python_ast::PythonVersion, ) -> Self { Self { source, + written, value_ranges, min_version, } @@ -579,14 +583,8 @@ impl PropertiesPass<'_> { .flatten() .map(|synthesized| { ctx.type_only_imports - .extend(synthesized.modules.into_iter().map(|m| format!("import {m}"))); - ctx.type_only_imports.extend( - synthesized - .typing_names - .into_iter() - .map(|name| format!("from typing import {name}")), - ); - synthesized.text + .extend(synthesized.imports_in(self.written)); + synthesized.text_in(self.written) }); Some(Backing { annotation: None, @@ -606,20 +604,20 @@ impl PropertiesPass<'_> { let is_abstract = has("abstract"); let mut accessor_decorators = String::new(); - if is_abstract { - ctx.required_imports - .push("from abc import abstractmethod".to_owned()); - let _ = write!(accessor_decorators, "@abstractmethod\n{indent}"); - } - if has("override") { - ctx.required_imports - .push("from typing import override".to_owned()); - let _ = write!(accessor_decorators, "@override\n{indent}"); - } - if has("final") { - ctx.required_imports - .push("from typing import final".to_owned()); - let _ = write!(accessor_decorators, "@final\n{indent}"); + for (written_by, module, name) in [ + (is_abstract, "abc", "abstractmethod"), + (has("override"), "typing", "override"), + (has("final"), "typing", "final"), + ] { + if written_by { + ctx.required_imports + .push(self.written.import_from(module, &[name])); + let _ = write!( + accessor_decorators, + "@{}\n{indent}", + self.written.imported(module, name) + ); + } } let mut frags: Vec = Vec::new(); @@ -660,9 +658,9 @@ impl PropertiesPass<'_> { // rather than a `property` taking an instance let (decorator, receiver) = if is_static { ctx.runtime.insert(crate::runtime::STATIC_PROPERTY); - ("_by_static_property", "cls") + ("_by_static_property".to_owned(), "cls") } else { - ("property", "self") + (self.written.builtin("property"), "self") }; frags.push(Fragment::Lit(format!( "@{decorator}\n{indent}{accessor_decorators}def {emitted}({receiver})" diff --git a/crates/by_transforms/src/transforms/protocol_type.rs b/crates/by_transforms/src/transforms/protocol_type.rs index 146d93e577..52e6daabb4 100644 --- a/crates/by_transforms/src/transforms/protocol_type.rs +++ b/crates/by_transforms/src/transforms/protocol_type.rs @@ -65,8 +65,9 @@ impl Shape { format!("_Protocol_{truncated:08x}") } - fn class_def(&self, name: &str) -> String { - let mut out = format!("class {name}(Protocol):\n"); + /// the class declaring this shape as `name`, `typing.Protocol` written as `protocol` + fn class_def(&self, name: &str, protocol: &str) -> String { + let mut out = format!("class {name}({protocol}):\n"); if self.members.is_empty() { out.push_str(" pass\n"); return out; @@ -100,6 +101,7 @@ impl<'src> ProtocolTypePass<'src> { struct ProtocolTypeLowering<'src> { source: &'src str, + written: WrittenNames<'src>, /// One lowerer drives every member type in the run, so its imports and the /// `_Callable_*` classes it synthesizes can be collected at the end. The /// `callable` pass deliberately does not descend into an inline protocol, @@ -127,6 +129,7 @@ impl<'src> ProtocolTypeLowering<'src> { ) -> Self { Self { source, + written, callable: CallableSyntax::new(source, written, config) .with_types(types) .with_claimed_ranges(claimed), @@ -140,8 +143,9 @@ impl<'src> ProtocolTypeLowering<'src> { fn class_defs(&self) -> String { let mut out = String::new(); + let protocol = self.written.imported("typing", "Protocol"); for (shape, name) in &self.shapes { - out.push_str(&shape.class_def(name)); + out.push_str(&shape.class_def(name, &protocol)); out.push('\n'); } out @@ -335,35 +339,18 @@ impl<'ast> ruff_python_ast::visitor::Visitor<'ast> for ValueProtocolWalker<'_, ' } } -/// Collects the typevar renames the PEP 695 polyfill will apply, keyed by the +/// Collects the typevar renames the PEP 695 polyfill applies, keyed by the /// range of the generic scope that declares them. struct TypevarScopeWalker<'a> { - written: WrittenNames<'a>, - config: crate::Config, + polyfilled: &'a super::generics::PolyfilledTypeParams, scopes: &'a mut Vec<(TextRange, HashMap)>, } impl TypevarScopeWalker<'_> { fn record(&mut self, range: TextRange, type_params: Option<&ruff_python_ast::TypeParams>) { - // python 3.12+ keeps PEP 695 native, so nothing is renamed - if self.config.min_version >= ruff_python_ast::PythonVersion::PY312 { - return; - } - let Some(type_params) = type_params else { - return; - }; - let frame: HashMap = type_params - .type_params - .iter() - .map(|param| { - let name = param.name().id.as_str(); - ( - name.to_owned(), - super::generics::polyfilled_name(self.written, name), - ) - }) - .collect(); - if !frame.is_empty() { + if let Some(frame) = type_params.and_then(|tp| self.polyfilled.hoisted_renames(tp)) + && !frame.is_empty() + { self.scopes.push((range, frame)); } } @@ -393,10 +380,10 @@ fn lower<'src>( config: &crate::Config, ) -> ProtocolTypeLowering<'src> { let mut inner = ProtocolTypeLowering::new(source, written, types, claimed, config); + let polyfilled = super::generics::PolyfilledTypeParams::decide(source, written, stmts, config); { let mut walker = TypevarScopeWalker { - written, - config: config.clone(), + polyfilled: &polyfilled, scopes: &mut inner.typevar_scopes, }; for stmt in stmts { @@ -450,7 +437,7 @@ pub(crate) fn cleanup( preamble.push('\n'); } }; - push_missing(&mut preamble, "from typing import Protocol"); + push_missing(&mut preamble, &written.import_from("typing", &["Protocol"])); let (imports, helpers) = inner.callable.take_requirements(); for line in imports .into_iter() @@ -498,7 +485,7 @@ impl TypeAwarePass for ProtocolTypePass<'_> { if inner.needs_import { ctx.required_imports - .push("from typing import Protocol".to_owned()); + .push(self.written.import_from("typing", &["Protocol"])); // synthesized class defs are raw multi-line python source; push // them as a single non-`from` line so `merge_from_imports` leaves // them untouched and they land in the preamble verbatim. the @@ -707,6 +694,82 @@ mod tests { ); } + /// kept as native syntax, `T` exists only in the scope of `A`, so the hoisted class reads it + /// off `A`. a generic no path of names reaches from module scope — local to a function — + /// has nothing to read it through + #[test] + fn a_native_type_parameter_is_read_off_its_generic() { + let out = transpile( + indoc! {" + class A[T]: + def get(self) -> protocol(a: T; def m(self, x: T) -> T): ... + def put[U](self, p: protocol(u: U)) -> None: ... + def f[V](p: protocol(v: V)) -> None: ... + def g(): + def h[W](p: protocol(w: W)) -> None: ... + "}, + &Config { + min_version: crate::PythonVersion::PY313, + ..Config::test_default() + }, + ) + .unwrap(); + assert!( + out.contains(" a: \"A.__type_params__[0]\"\n") + && out.contains( + " def m(self, x: \"A.__type_params__[0]\") -> \"A.__type_params__[0]\": ...\n" + ) + && out.contains(" u: \"A.put.__type_params__[0]\"\n") + && out.contains(" v: \"f.__type_params__[0]\"\n") + && out.contains(" w: \"W\"\n"), + "got:\n{out}" + ); + } + + /// an earlier generic declaring `T` differently takes `_T`, so the member names the one + /// the polyfill declared for this generic's `T` + #[test] + fn typevar_member_uses_the_name_the_polyfill_declared() { + check( + indoc! {" + class A[T: int]: ... + class B[T]: + def get(self) -> protocol(a: T): ... + "}, + indoc! {r#" + from typing import Protocol, TypeVar, Generic + class _Protocol_fd816244(Protocol): + a: "_T_1" + _T = TypeVar("_T", bound=int) + class A(Generic[_T]): ... + _T_1 = TypeVar("_T_1") + class B(Generic[_T_1]): + def get(self) -> _Protocol_fd816244: ... + "#}, + ); + } + + /// a defaulted list is polyfilled on 3.12, which has native syntax for one without a + /// default + #[test] + fn typevar_member_of_a_defaulted_list_on_312() { + check_py312( + indoc! {" + class A[T = int]: + def get(self) -> protocol(a: T): ... + "}, + indoc! {r#" + from typing import Generic, Protocol + from typing_extensions import TypeVar + class _Protocol_ea950896(Protocol): + a: "_T" + _T = TypeVar("_T", default=int) + class A(Generic[_T]): + def get(self) -> _Protocol_ea950896: ... + "#}, + ); + } + #[test] fn method_parameter_spec_survives() { check( diff --git a/crates/by_transforms/src/transforms/repeated_underscore.rs b/crates/by_transforms/src/transforms/repeated_underscore.rs index 48d7234273..d7510ce996 100644 --- a/crates/by_transforms/src/transforms/repeated_underscore.rs +++ b/crates/by_transforms/src/transforms/repeated_underscore.rs @@ -136,6 +136,219 @@ fn repeats_underscore(parameters: &Parameters) -> bool { pub struct WrittenNames<'src> { names: decision::WrittenNames<'src>, lowerings: Option<&'src UnderscoreLowerings>, + /// every name the module's code binds or reads, which a typing name a lowering writes + /// has to stay clear of + code: Option<&'src CodeNames>, + /// the names the module's `private` symbols are emitted under, which no other name a + /// lowering writes may take + private: Option<&'src PrivateNames>, +} + +/// the name each module-level `private` symbol is emitted under +/// +/// `private` hides a symbol by giving it a leading underscore, `private def helper` becoming +/// `_helper`. when the module already has that name — its own `_helper`, a runtime helper's, +/// or one a lowering wrote before this — the renamed symbol would take it over, so it is +/// emitted under the next name nothing has, `_helper2`, `_helper3`, …: +/// +/// ```by +/// private def force_unwrap(x: int) -> int: # `_force_unwrap` is the helper `x!` calls +/// return x + 100 +/// ``` +/// +/// a name the author already wrote with a leading underscore is emitted as written +#[derive(Debug, Default)] +pub(crate) struct PrivateNames { + emitted: HashMap, + /// the names in `emitted` a symbol's own spelling does not account for, which every + /// name a lowering takes afterwards has to stay clear of + claimed: std::collections::HashSet, +} + +impl PrivateNames { + /// how each of `symbols`, the module-level names a module declares `private`, is emitted. + /// `sources` are the texts of the module the lowering reads — as written, and as the + /// passes that run before the name is decided left it + pub(crate) fn decide<'a>(symbols: impl IntoIterator, sources: &[&str]) -> Self { + let mut symbols: Vec<&str> = symbols.into_iter().collect(); + // the order a name is taken in decides who gets `_helper` and who `_helper2`, so it + // must not be the order of a hash set + symbols.sort_unstable(); + symbols.dedup(); + let mut private = Self::default(); + for symbol in symbols { + if symbol.starts_with('_') { + private.emitted.insert(symbol.to_owned(), symbol.to_owned()); + continue; + } + let stem = format!("_{symbol}"); + let emitted = decision::WrittenNames::new("").fresh_outside(&stem, |candidate| { + crate::runtime::defines(candidate) + || private.claimed.contains(candidate) + || sources + .iter() + .any(|source| decision::WrittenNames::new(source).spells(candidate)) + }); + private.claimed.insert(emitted.clone()); + private.emitted.insert(symbol.to_owned(), emitted); + } + private + } +} + +/// how a module's own code binds each name, in any scope, which says whether a name a lowering +/// writes reads what the lowering means by it +/// +/// a keyword the parser stands up as a name — the `final` of `final class`, the `cast` of +/// `x cast int` — binds nothing, and a read binds nothing either: a name only read is a builtin, +/// or a typing name basedpython imports implicitly, which is the lowering's own +#[derive(Debug, Default)] +pub(crate) struct CodeNames { + bindings: std::collections::HashMap>, + /// whether the module imports `*` from somewhere, which binds names it never spells + star_import: bool, +} + +/// one way a module binds a name +#[derive(Debug, Clone, PartialEq, Eq)] +enum Binding { + /// `from import `, which binds the name to what `module` exports as it + From(String), + /// `import `, which binds the name to the module of that name + Module, + /// anything else + Other, +} + +impl CodeNames { + /// the names `suite`, a module's statements as the author wrote them, binds + pub(crate) fn of(suite: &[Stmt]) -> Self { + use ruff_python_ast::visitor::source_order::{ + SourceOrderVisitor, walk_except_handler, walk_expr, walk_parameter, walk_pattern, + walk_stmt, walk_type_param, + }; + use ruff_python_ast::{ExceptHandler, Pattern, TypeParam}; + + #[derive(Default)] + struct Collect(CodeNames); + impl Collect { + fn bind(&mut self, name: &str, binding: Binding) { + self.0 + .bindings + .entry(name.to_owned()) + .or_default() + .push(binding); + } + } + impl<'a> SourceOrderVisitor<'a> for Collect { + fn visit_stmt(&mut self, stmt: &'a Stmt) { + match stmt { + Stmt::FunctionDef(function) => self.bind(&function.name, Binding::Other), + Stmt::ClassDef(class) => self.bind(&class.name, Binding::Other), + Stmt::Global(global) => { + for name in &global.names { + self.bind(name, Binding::Other); + } + } + Stmt::Nonlocal(nonlocal) => { + for name in &nonlocal.names { + self.bind(name, Binding::Other); + } + } + Stmt::Import(import) => { + for alias in &import.names { + match &alias.asname { + Some(asname) => self.bind(asname, Binding::Other), + None => { + let root = alias.name.split('.').next().unwrap_or_default(); + self.bind(root, Binding::Module); + } + } + } + } + Stmt::ImportFrom(import) => { + for alias in &import.names { + if &*alias.name == "*" { + self.0.star_import = true; + continue; + } + match (&alias.asname, &import.module) { + (None, Some(module)) if import.level == 0 => { + self.bind(&alias.name, Binding::From(module.to_string())); + } + (asname, _) => self + .bind(asname.as_ref().unwrap_or(&alias.name), Binding::Other), + } + } + } + _ => {} + } + walk_stmt(self, stmt); + } + + fn visit_expr(&mut self, expr: &'a Expr) { + match expr { + Expr::Name(name) if name.ctx.is_store() || name.ctx.is_del() => { + self.bind(&name.id, Binding::Other); + } + // the callee of `x cast int` is the keyword, not a name + Expr::Call(call) if call.cast_kind.is_some() => { + self.visit_arguments(&call.arguments); + return; + } + _ => {} + } + walk_expr(self, expr); + } + + fn visit_parameter(&mut self, parameter: &'a Parameter) { + self.bind(¶meter.name, Binding::Other); + walk_parameter(self, parameter); + } + + fn visit_except_handler(&mut self, handler: &'a ExceptHandler) { + let ExceptHandler::ExceptHandler(handler_node) = handler; + if let Some(name) = &handler_node.name { + self.bind(name, Binding::Other); + } + walk_except_handler(self, handler); + } + + fn visit_pattern(&mut self, pattern: &'a Pattern) { + let name = match pattern { + Pattern::MatchAs(pattern) => pattern.name.as_ref(), + Pattern::MatchStar(pattern) => pattern.name.as_ref(), + Pattern::MatchMapping(pattern) => pattern.rest.as_ref(), + _ => None, + }; + if let Some(name) = name { + self.bind(name, Binding::Other); + } + walk_pattern(self, pattern); + } + + fn visit_type_param(&mut self, type_param: &'a TypeParam) { + self.bind(type_param.name(), Binding::Other); + walk_type_param(self, type_param); + } + } + + let mut collect = Collect::default(); + for stmt in suite { + collect.visit_stmt(stmt); + } + collect.0 + } + + /// whether every binding of `name` is `binding`, so a lowering can read the module's own + /// `name` as the thing `binding` binds it to + fn only_binds(&self, name: &str, binding: &Binding) -> bool { + !self.star_import + && self + .bindings + .get(name) + .is_none_or(|bindings| bindings.iter().all(|bound| bound == binding)) + } } impl<'src> WrittenNames<'src> { @@ -144,6 +357,34 @@ impl<'src> WrittenNames<'src> { Self { names: decision::WrittenNames::new(source), lowerings: None, + code: None, + private: None, + } + } + + /// these names, with `code` the names the module's code binds or reads + pub(crate) fn with_code<'a>(self, code: &'a CodeNames) -> WrittenNames<'a> + where + 'src: 'a, + { + WrittenNames { + names: self.names, + lowerings: self.lowerings, + code: Some(code), + private: self.private, + } + } + + /// these names, with `private` the names the module's `private` symbols are emitted under + pub(crate) fn with_private<'a>(self, private: &'a PrivateNames) -> WrittenNames<'a> + where + 'src: 'a, + { + WrittenNames { + names: self.names, + lowerings: self.lowerings, + code: self.code, + private: Some(private), } } @@ -156,6 +397,8 @@ impl<'src> WrittenNames<'src> { WrittenNames { names: self.names, lowerings: Some(lowerings), + code: self.code, + private: self.private, } } @@ -165,13 +408,121 @@ impl<'src> WrittenNames<'src> { /// a binding the lowering writes under a name the source wrote would shadow it — the /// `inner` a `decorator def` writes standing where an option of that name is declared, /// which the dispatcher then passed the dispatcher itself for + /// + /// nor is it a runtime helper's, or one a `private` symbol is emitted under: those are + /// bound at module scope too, and the source never spells them pub(crate) fn fresh(self, stem: &str) -> String { - self.names.fresh(stem) + self.names + .fresh_outside(stem, |name| self.lowering_binds(name)) + } + + /// whether `name` is one a lowering cannot bind: the module spells it, a runtime + /// helper is bound under it, or a `private` symbol is emitted under it + pub(crate) fn taken(self, name: &str) -> bool { + self.names.spells(name) || self.lowering_binds(name) + } + + /// whether the module has `name` bound without spelling it: a runtime helper is bound + /// under it, or a `private` symbol is emitted under it + fn lowering_binds(self, name: &str) -> bool { + crate::runtime::defines(name) + || self + .private + .is_some_and(|private| private.claimed.contains(name)) + } + + /// the name `symbol`, a module-level name the module declares `private`, is emitted + /// under ([`PrivateNames`]). without the module's decision, it gains a leading + /// underscore unless it has one already + pub(crate) fn module_private(self, symbol: &str) -> String { + match self.private.and_then(|private| private.emitted.get(symbol)) { + Some(emitted) => emitted.clone(), + None => super::modifiers::module_private_name(symbol), + } } - /// whether the module spells `name` anywhere - pub(crate) fn spells(self, name: &str) -> bool { - self.names.spells(name) + /// the name a lowering writes `name`, which `module` exports, under — `typing`, + /// `typing_extensions`, `collections.abc` and the like + /// + /// a module that binds `name` itself may bind it to anything at all — `Union = …` — so the + /// lowering's is imported under a name the module does not spell ([`Self::import_from`]). + /// one that binds it only by importing it from `module` has it already, and one that never + /// binds it reads the lowering's. without the module's bindings, every name the module + /// spells counts as bound + pub(crate) fn imported(self, module: &str, name: &str) -> String { + self.name_for(name, &Binding::From(module.to_owned())) + } + + /// the name a lowering writes `name`, a builtin, under: the builtin's own unless the + /// module binds that name to something else, anywhere — a parameter named `type`, a + /// class attribute named `staticmethod` — and then a name of the lowering's own, which + /// [`Self::builtin_imports`] binds + pub(crate) fn builtin(self, name: &str) -> String { + self.imported("builtins", name) + } + + /// the import that binds each builtin `output`, python a lowering wrote, reads under a + /// name of the lowering's own ([`Self::builtin`]), on a python of `minor` + /// + /// such a name is spelled nowhere in the module, so only a lowering can have written it + pub(crate) fn builtin_imports(self, output: &str, minor: u8) -> Vec { + let written = decision::WrittenNames::new(output); + ruff_python_stdlib::builtins::python_builtins(minor, false) + .filter_map(|name| { + let local = self.builtin(name); + (local != name && written.spells(&local)) + .then(|| format!("from builtins import {name} as {local}")) + }) + .collect() + } + + /// the import that binds each of `names`, which `module` exports, to the name + /// [`Self::imported`] answers for it + pub(crate) fn import_from(self, module: &str, names: &[&str]) -> String { + let names: Vec = names + .iter() + .map(|name| { + let local = self.imported(module, name); + if local == *name { + local + } else { + format!("{name} as {local}") + } + }) + .collect(); + format!("from {module} import {}", names.join(", ")) + } + + /// the name a lowering reads `module`, a module it reads attributes of, under, as + /// [`Self::imported`] answers for an exported name + pub(crate) fn imported_module(self, module: &str) -> String { + self.name_for(module, &Binding::Module) + } + + /// the import that binds `module` to the name [`Self::imported_module`] answers for it + pub(crate) fn import_module(self, module: &str) -> String { + let local = self.imported_module(module); + if local == module { + format!("import {module}") + } else { + format!("import {module} as {local}") + } + } + + fn name_for(self, name: &str, binding: &Binding) -> String { + let free = self.code.map_or_else( + || !self.names.spells(name), + |code| code.only_binds(name, binding), + ); + if free && !self.lowering_binds(name) { + name.to_owned() + } else { + // the name itself is taken even when the module never spells it: a star import + // binds it unseen + self.names.fresh_outside(name, |candidate| { + candidate == name || self.lowering_binds(candidate) + }) + } } /// how `parameters` is lowered when it repeats `_`. without ty's answer, the list is @@ -518,6 +869,101 @@ mod tests { transpile(input, &Config::test_default()).unwrap_err() } + fn check_at(input: &str, expected: &str, min_version: crate::PythonVersion) { + let config = Config { + min_version, + ..Config::test_default() + }; + assert_eq!(transpile(input, &config).unwrap(), expected); + } + + /// a typing name a lowering writes is imported under a name the module does not spell + /// when the module binds that name to something of its own + #[test] + fn a_typing_name_the_module_binds_is_imported_under_another_name() { + check_at( + indoc! {" + Union = 1 + x = list[int?] + "}, + indoc! {" + from __future__ import annotations + from typing import Union as Union2 + Union = 1 + x = list[Union2[int, None]] + "}, + crate::PythonVersion::PY39, + ); + } + + /// a module that binds the name only by importing it from where the lowering imports it + /// has the lowering's already + #[test] + fn a_typing_name_the_module_imports_from_the_same_module_is_the_modules_own() { + check( + indoc! {" + from typing import Literal + x: 1 | 2 = 1 + "}, + indoc! {" + from typing import Literal + x: Literal[1, 2] = 1 + "}, + ); + } + + /// the same name exported by another module need not be the same object + #[test] + fn a_typing_name_the_module_imports_from_elsewhere_is_not() { + check( + indoc! {" + from typing_extensions import Literal + x: 1 | 2 = 1 + "}, + indoc! {" + from typing import Literal as Literal2 + from typing_extensions import Literal + x: Literal2[1, 2] = 1 + "}, + ); + } + + /// a star import binds names the module never spells + #[test] + fn a_star_import_takes_every_typing_name() { + check( + indoc! {" + from os.path import * + x: 1 | 2 = 1 + "}, + indoc! {" + from typing import Literal as Literal2 + from os.path import * + x: Literal2[1, 2] = 1 + "}, + ); + } + + /// a keyword the parser writes as a name binds nothing, and a parameter is a binding like + /// any other + #[test] + fn a_keyword_binds_no_typing_name() { + check( + indoc! {" + final class A: ... + def f(Literal: int) -> None: ... + x: 1 | 2 = 1 + "}, + indoc! {" + from typing import Literal as Literal2, final + @final + class A: ... + def f(Literal: int) -> None: ... + x: Literal2[1, 2] = 1 + "}, + ); + } + #[test] fn two_underscores() { check( diff --git a/crates/by_transforms/src/transforms/runtime_union.rs b/crates/by_transforms/src/transforms/runtime_union.rs index 02385561ee..e17f67f8e8 100644 --- a/crates/by_transforms/src/transforms/runtime_union.rs +++ b/crates/by_transforms/src/transforms/runtime_union.rs @@ -29,6 +29,10 @@ //! asks [`classinfo_optionals`] which of its optionals stand in the classinfo //! argument. //! +//! Which expressions of the argument are read as classes is decided once, in +//! `ty_python_semantic::types::classinfo_spelling`, which ty reads too: a union +//! spelled there is one it does not report as a union `isinstance` rejects. +//! //! Whether a `|` is a union at all is asked of the checker rather than guessed //! from the shape: `a | b` is overwhelmingly a bitwise or, and only the types of //! its operands tell the two apart. @@ -36,26 +40,32 @@ use std::collections::HashSet; use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; -use ruff_python_ast::{Expr, ExprCall, Operator, PythonVersion, Stmt, UnaryOp}; +use ruff_python_ast::{Expr, Operator, PythonVersion, Stmt, UnaryOp}; use ruff_text_size::{Ranged, TextRange}; +use ty_python_semantic::types::classinfo_spelling; use super::ast_driver::{Fragment, PassContext, TypeAwarePass}; +use super::repeated_underscore::WrittenNames; use crate::type_info::TypeInfo; /// the version `type.__or__` arrived in const MIN_VERSION: PythonVersion = PythonVersion::PY310; -pub(crate) struct RuntimeUnionPass { +pub(crate) struct RuntimeUnionPass<'src> { + written: WrittenNames<'src>, min_version: PythonVersion, } -impl RuntimeUnionPass { - pub(crate) fn new(min_version: PythonVersion) -> Self { - Self { min_version } +impl<'src> RuntimeUnionPass<'src> { + pub(crate) fn new(written: WrittenNames<'src>, min_version: PythonVersion) -> Self { + Self { + written, + min_version, + } } } -impl TypeAwarePass for RuntimeUnionPass { +impl TypeAwarePass for RuntimeUnionPass<'_> { fn lowering(&self) -> Option { Some(super::ast_driver::Lowering::RuntimeUnion) } @@ -70,13 +80,17 @@ impl TypeAwarePass for RuntimeUnionPass { if self.min_version >= MIN_VERSION { return; } - let mut lower = Lower::new(types); + let mut lower = Lower::new( + types, + self.written.imported("typing", "Union"), + self.written.builtin("type"), + ); for stmt in stmts { lower.visit_stmt(stmt); } if lower.needs_import { ctx.required_imports - .push("from typing import Union".to_owned()); + .push(self.written.import_from("typing", &["Union"])); } ctx.template_edits.extend(lower.edits); } @@ -85,7 +99,8 @@ impl TypeAwarePass for RuntimeUnionPass { /// The optionals in `stmts` that stand where `isinstance` / `issubclass` expects /// classes, and so are spelled as a tuple of classes below 3.10. pub(crate) fn classinfo_optionals(stmts: &[Stmt], types: &dyn TypeInfo) -> HashSet { - let mut lower = Lower::new(types); + // only the ranges are read, never the spelling + let mut lower = Lower::new(types, String::new(), String::new()); for stmt in stmts { lower.visit_stmt(stmt); } @@ -94,6 +109,10 @@ pub(crate) fn classinfo_optionals(stmts: &[Stmt], types: &dyn TypeInfo) -> HashS struct Lower<'a> { types: &'a dyn TypeInfo, + /// the name `typing.Union` is written under + union: String, + /// the name the builtin `type` is written under + type_: String, edits: Vec<(TextRange, Vec)>, needs_import: bool, /// every optional the classinfo walk reached, for the optional lowering @@ -101,9 +120,11 @@ struct Lower<'a> { } impl<'a> Lower<'a> { - fn new(types: &'a dyn TypeInfo) -> Self { + fn new(types: &'a dyn TypeInfo, union: String, type_: String) -> Self { Self { types, + union, + type_, edits: Vec::new(), needs_import: false, classinfo_optionals: HashSet::new(), @@ -122,7 +143,7 @@ impl<'ast> Visitor<'ast> for Lower<'_> { fn visit_expr(&mut self, expr: &'ast Expr) { if let Expr::Call(call) = expr - && let Some(classinfo) = self.classinfo_argument(call) + && let Some(classinfo) = self.types.classinfo_argument(call) { for (index, argument) in call.arguments.args.iter().enumerate() { if index == 1 { @@ -140,7 +161,7 @@ impl<'ast> Visitor<'ast> for Lower<'_> { if let Some(arms) = self.union_arms(expr) { self.needs_import = true; - let fragments = spell(&arms, Form::Union); + let fragments = spell(&arms, Form::Union(&self.union)); self.edits.push((expr.range(), fragments)); for arm in arms { self.visit_expr(arm); @@ -154,30 +175,15 @@ impl<'ast> Visitor<'ast> for Lower<'_> { /// How a union is spelled, which is decided by where it stands. #[derive(Clone, Copy, PartialEq, Eq)] -enum Form { - /// a tuple of classes, for `isinstance` / `issubclass` - ClassInfo, - /// `Union[...]`, for everywhere else - Union, +enum Form<'a> { + /// a tuple of classes, for `isinstance` / `issubclass`, the builtin `type` written + /// under the name given + ClassInfo(&'a str), + /// `Union[...]`, for everywhere else, `Union` written under the name given + Union(&'a str), } impl<'ast> Lower<'_> { - /// The second positional argument of a call to the real `isinstance` or - /// `issubclass`. A file that binds either name means something else by it, - /// and gets no special treatment. - fn classinfo_argument(&self, call: &'ast ExprCall) -> Option<&'ast Expr> { - let Expr::Name(name) = call.func.as_ref() else { - return None; - }; - if !matches!(name.id.as_str(), "isinstance" | "issubclass") { - return None; - } - if !self.types.is_unbound_at(name.id.as_str(), &call.func) { - return None; - } - call.arguments.args.get(1) - } - /// The arms of `expr`, when it is a union the runtime will evaluate. fn union_arms(&self, expr: &'ast Expr) -> Option> { let Expr::BinOp(binop) = expr else { @@ -186,42 +192,32 @@ impl<'ast> Lower<'_> { if binop.op != Operator::BitOr || !self.types.is_runtime_union(expr) { return None; } - let mut arms = Vec::new(); - collect_arms(expr, &mut arms); - Some(arms) + Some(classinfo_spelling::union_arms(expr)) } - /// Visit an expression standing where `isinstance` expects classes. A union - /// here becomes a tuple, and so does one nested inside a tuple or list the - /// argument already spells, or inside an optional; everything else is - /// ordinary value context. - fn visit_classinfo(&mut self, expr: &'ast Expr) { - if let Some(arms) = self.union_arms(expr) { - let fragments = spell(&arms, Form::ClassInfo); - self.edits.push((expr.range(), fragments)); - for arm in arms { - self.visit_classinfo(arm); - } - return; - } - match expr { - Expr::Tuple(tuple) => { - for element in &tuple.elts { - self.visit_classinfo(element); + /// Visit the argument `isinstance` reads as classes. Each union where it reads classes + /// becomes a tuple, and each optional there is spelled as one by the optional lowering; + /// everything else is ordinary value context. + fn visit_classinfo(&mut self, classinfo: &'ast Expr) { + let types = self.types; + let positions = + classinfo_spelling::class_positions(classinfo, &|expr| types.is_runtime_union(expr)); + for position in positions { + match position { + Expr::Tuple(_) | Expr::List(_) => {} + Expr::UnaryOp(optional) + if optional.op == UnaryOp::Optional && types.is_runtime_union(position) => + { + self.classinfo_optionals.insert(position.range()); } + _ => match self.union_arms(position) { + Some(arms) => { + let fragments = spell(&arms, Form::ClassInfo(&self.type_)); + self.edits.push((position.range(), fragments)); + } + None => self.visit_expr(position), + }, } - Expr::List(list) => { - for element in &list.elts { - self.visit_classinfo(element); - } - } - Expr::UnaryOp(optional) - if optional.op == UnaryOp::Optional && self.types.is_runtime_union(expr) => - { - self.classinfo_optionals.insert(expr.range()); - self.visit_classinfo(&optional.operand); - } - _ => self.visit_expr(expr), } } } @@ -230,19 +226,18 @@ impl<'ast> Lower<'_> { /// that a lowering inside it — an optional `T?`, a tuple type — still composes. fn spell(arms: &[&Expr], form: Form) -> Vec { let mut fragments = Vec::with_capacity(arms.len() * 2 + 2); - fragments.push(Fragment::Lit( - match form { - Form::ClassInfo => "(", - Form::Union => "Union[", - } - .to_owned(), - )); + fragments.push(Fragment::Lit(match form { + Form::ClassInfo(_) => "(".to_owned(), + Form::Union(union) => format!("{union}["), + })); for (index, arm) in arms.iter().enumerate() { if index > 0 { fragments.push(Fragment::Lit(", ".to_owned())); } - if form == Form::ClassInfo && arm.is_none_literal_expr() { - fragments.push(Fragment::Lit("type(None)".to_owned())); + if let Form::ClassInfo(type_) = form + && arm.is_none_literal_expr() + { + fragments.push(Fragment::Lit(format!("{type_}(None)"))); } else { fragments.push(Fragment::Src(arm.range())); } @@ -251,26 +246,14 @@ fn spell(arms: &[&Expr], form: Form) -> Vec { // than a parenthesized class fragments.push(Fragment::Lit( match form { - Form::ClassInfo => ",)", - Form::Union => "]", + Form::ClassInfo(_) => ",)", + Form::Union(_) => "]", } .to_owned(), )); fragments } -/// Flatten `a | b | c`, which parses as `(a | b) | c`, into its arms. -fn collect_arms<'ast>(expr: &'ast Expr, arms: &mut Vec<&'ast Expr>) { - if let Expr::BinOp(binop) = expr - && binop.op == Operator::BitOr - { - collect_arms(&binop.left, arms); - collect_arms(&binop.right, arms); - return; - } - arms.push(expr); -} - #[cfg(test)] mod tests { use crate::{Config, PythonVersion, transpile}; @@ -423,6 +406,18 @@ mod tests { ); } + /// the builtin reached by another name is the builtin all the same + #[test] + fn an_isinstance_by_another_name_takes_a_tuple() { + let out = lowered(indoc! {" + from builtins import isinstance as is_instance + + def f(x: object): + return is_instance(x, int | str) + "}); + assert!(out.contains("is_instance(x, (int, str,))"), "got:\n{out}"); + } + /// a target that has `type.__or__` keeps every union as written #[test] fn untouched_from_python_310() { diff --git a/crates/by_transforms/src/transforms/sentinel.rs b/crates/by_transforms/src/transforms/sentinel.rs index 54b6d9efdb..2f8cb8776e 100644 --- a/crates/by_transforms/src/transforms/sentinel.rs +++ b/crates/by_transforms/src/transforms/sentinel.rs @@ -20,13 +20,16 @@ use ruff_python_ast::{ use ruff_text_size::TextRange; pub(crate) struct Sentinel { + /// the name `typing_extensions.Sentinel` is written under + constructor: String, changed: Cell, ever_changed: Cell, } impl Sentinel { - pub(crate) fn new() -> Self { + pub(crate) fn new(constructor: String) -> Self { Self { + constructor, changed: Cell::new(false), ever_changed: Cell::new(false), } @@ -62,7 +65,7 @@ impl Transformer for Sentinel { func: Box::new(Expr::Name(ExprName { node_index: AtomicNodeIndex::NONE, range: TextRange::default(), - id: Name::from("Sentinel"), + id: Name::from(self.constructor.as_str()), ctx: ExprContext::Load, })), arguments: Arguments { diff --git a/crates/by_transforms/src/transforms/symbolic_type_op.rs b/crates/by_transforms/src/transforms/symbolic_type_op.rs index 1f41529159..506c5d241c 100644 --- a/crates/by_transforms/src/transforms/symbolic_type_op.rs +++ b/crates/by_transforms/src/transforms/symbolic_type_op.rs @@ -21,7 +21,7 @@ //! already carries the resolved type, so the result stays correct either way. use std::cell::RefCell; -use std::collections::HashMap; +use std::collections::{BTreeSet, HashMap}; use ruff_python_ast::visitor::transformer::{Transformer, walk_expr}; use ruff_python_ast::visitor::{self, Visitor}; @@ -31,9 +31,11 @@ use ruff_text_size::{Ranged, TextRange}; use super::ast_driver::{AstPass, PassContext}; use super::literal_types::float_literal_spelling; +use super::repeated_underscore::WrittenNames; use super::type_expr_walker::{ Recurse, TypeExprVisitor, TypePos, walk_one_type_expr, walk_type_positions, }; +use crate::PythonVersion; use crate::config::FloatLiteralLowering; use crate::type_info::TypeInfo; @@ -48,12 +50,9 @@ struct Fold { /// source range. pub(crate) struct SymbolicFolds { folds: HashMap, - /// whether any replacement references `typing.Literal`, so the driver can - /// add the import - pub(crate) needs_literal_import: bool, - /// whether any replacement is `Any` (e.g. `dynamic + 1` folds to `Any`), so - /// the driver can add `from typing import Any` - pub(crate) needs_any_import: bool, + /// the imports the replacements read: a `typing` name (`Literal`, the `Any` of + /// `dynamic + 1`), or the module a class is qualified with + pub(crate) imports: BTreeSet, } impl SymbolicFolds { @@ -82,29 +81,37 @@ impl SymbolicFolds { pub(crate) fn collect_symbolic_folds( stmts: &[Stmt], types: &dyn TypeInfo, + written: WrittenNames, float_literals: FloatLiteralLowering, + min_version: PythonVersion, ) -> SymbolicFolds { let mut collector = FoldCollector { types, + written, + literal: written.imported("typing", "Literal"), + any: written.imported("typing", "Any"), float_literals, + min_version, folds: HashMap::new(), - needs_literal_import: false, - needs_any_import: false, + imports: BTreeSet::new(), }; walk_type_positions(stmts, Some(types), &mut collector); SymbolicFolds { folds: collector.folds, - needs_literal_import: collector.needs_literal_import, - needs_any_import: collector.needs_any_import, + imports: collector.imports, } } struct FoldCollector<'a> { types: &'a dyn TypeInfo, + written: WrittenNames<'a>, + /// the names `typing.Literal` and `typing.Any` are written under + literal: String, + any: String, float_literals: FloatLiteralLowering, + min_version: PythonVersion, folds: HashMap, - needs_literal_import: bool, - needs_any_import: bool, + imports: BTreeSet, } impl TypeExprVisitor for FoldCollector<'_> { @@ -169,61 +176,62 @@ impl TypeExprVisitor for FoldCollector<'_> { if !foldable { return Recurse::Descend; } - let Some(rendered) = self.types.symbolic_type_fold(expr) else { - return Recurse::Descend; - }; - // the special float-literal types render as the bare names `inf` / - // `-inf` / `nan`, which have no python literal syntax — leave them for - // `float_const` to erase to `float` rather than folding to an undefined - // name (only `float.inf` etc. produce these, so this never shadows a - // real arithmetic fold) - if matches!(rendered.as_str(), "inf" | "-inf" | "nan") { - return Recurse::Descend; - } - // the rendered type must itself parse as a type expression, and `Unknown` - // is not a runtime name either - let (rendered, parsed) = match parse_expression(&rendered) { - Ok(parsed) if rendered != "Unknown" => (rendered, parsed), - // some forms *must* be replaced, because their source spelling names - // nothing at runtime: a `type def` application (its declaration is - // erased, so the name would dangle) and an attribute type (`T.a` is an - // attribute access on a `TypeVar` object). `Any` is the honest - // spelling when the resolved type has none — a deferred application, - // or a member python cannot write down such as a bound method or a - // callable, which ty renders in arrow form. anything else keeps its - // source so ty's own diagnostic stands. - // - // the widening is deliberately silent: it is the same trade every - // deferred operation already makes when it lowers to its reduced form - // (`Array[Dim + 1]` → `Array[int]`), the `.by` file keeps the precise - // type either way, and the transpiler has no warning channel — only - // hard errors, which would reject perfectly good source - _ => { - if !(self.types.is_type_fn_application(expr) || self.types.is_attribute_type(expr)) + // some forms *must* be replaced, because their source spelling names nothing at + // runtime: a `type def` application (its declaration is erased, so the name would + // dangle) and an attribute type (`T.a` is an attribute access on a `TypeVar` + // object). `Any` is the honest spelling when the resolved type has none the output + // can resolve — a deferred application, or a member python cannot write down such + // as a bound method or a callable, which ty renders in arrow form. anything else + // keeps its source so ty's own diagnostic stands. + // + // the widening is deliberately silent: it is the same trade every deferred + // operation already makes when it lowers to its reduced form (`Array[Dim + 1]` → + // `Array[int]`), the `.by` file keeps the precise type either way, and the + // transpiler has no warning channel — only hard errors, which would reject + // perfectly good source + let must_replace = + self.types.is_type_fn_application(expr) || self.types.is_attribute_type(expr); + let spelled = self + .types + .symbolic_type_fold(expr, self.min_version) + // the special float-literal types render as the bare names `inf` / `-inf` / + // `nan`, which have no python literal syntax — leave them for `float_const` + // to erase to `float` rather than folding to an undefined name (only + // `float.inf` etc. produce these, so this never shadows a real arithmetic + // fold) + .filter(|spelled| !matches!(spelled.text.as_str(), "inf" | "-inf" | "nan" | "Unknown")); + let rendered = match spelled { + Some(mut spelled) => { + // ty spells a float or complex literal type as the bare number, which + // python reads as no type at all. it is written the way the same literal + // written in the source is + if let Ok(parsed) = parse_expression(&spelled.text) + && let Some(floats) = spell_float_literals( + &spelled.text, + parsed.expr(), + self.written, + &self.literal, + self.float_literals, + ) { - return Recurse::Descend; + spelled.text = floats; } - let Ok(parsed) = parse_expression("Any") else { - return Recurse::Descend; - }; - ("Any".to_string(), parsed) + self.imports.extend(spelled.imports_in(self.written)); + spelled.text_in(self.written) } + None if must_replace => self.any.clone(), + None => return Recurse::Descend, + }; + let Ok(parsed) = parse_expression(&rendered) else { + return Recurse::Descend; }; - // ty spells a float or complex literal type as the bare number, which python reads - // as no type at all. it is written the way the same literal written in the source is - let (rendered, parsed) = - match spell_float_literals(&rendered, parsed.expr(), self.float_literals) { - Some(spelled) => match parse_expression(&spelled) { - Ok(parsed) => (spelled, parsed), - Err(_) => return Recurse::Descend, - }, - None => (rendered, parsed), - }; - if rendered.contains("Literal[") { - self.needs_literal_import = true; + if uses_name(parsed.expr(), &self.literal) { + self.imports + .insert(self.written.import_from("typing", &["Literal"])); } - if rendered == "Any" { - self.needs_any_import = true; + if uses_name(parsed.expr(), &self.any) { + self.imports + .insert(self.written.import_from("typing", &["Any"])); } let mut node = *parsed.into_syntax().body; super::rerender::forget_expr_ranges(&mut node); @@ -233,15 +241,38 @@ impl TypeExprVisitor for FoldCollector<'_> { } } +/// whether `expr` reads `name` +fn uses_name(expr: &Expr, name: &str) -> bool { + struct Uses<'a> { + name: &'a str, + found: bool, + } + impl<'b> Visitor<'b> for Uses<'_> { + fn visit_expr(&mut self, expr: &'b Expr) { + if matches!(expr, Expr::Name(candidate) if candidate.id.as_str() == self.name) { + self.found = true; + } + visitor::walk_expr(self, expr); + } + } + let mut uses = Uses { name, found: false }; + uses.visit_expr(expr); + uses.found +} + /// `rendered` with each float or complex literal type in `parsed` (its parse) spelled as /// [`float_literal_spelling`] spells a written one, or `None` when it holds none fn spell_float_literals( rendered: &str, parsed: &Expr, + written: WrittenNames, + literal: &str, float_literals: FloatLiteralLowering, ) -> Option { struct Spellings<'a> { rendered: &'a str, + written: WrittenNames<'a>, + literal: &'a str, float_literals: FloatLiteralLowering, edits: Vec<(TextRange, String)>, } @@ -254,7 +285,9 @@ fn spell_float_literals( return; } let text = &self.rendered[expr.range()]; - if let Some(spelling) = float_literal_spelling(expr, text, self.float_literals) { + if let Some(spelling) = + float_literal_spelling(expr, text, self.written, self.literal, self.float_literals) + { self.edits.push((expr.range(), spelling)); return; } @@ -263,6 +296,8 @@ fn spell_float_literals( } let mut spellings = Spellings { rendered, + written, + literal, float_literals, edits: Vec::new(), }; @@ -376,6 +411,51 @@ mod tests { ); } + /// a class of the module's own named like a `typing` special form is the class where the + /// fold names it, and `typing`'s `Literal` goes under a name of its own + #[test] + fn a_class_named_literal_is_the_class_in_a_fold() { + check( + indoc! {" + class Literal: ... + class Box: + a: Literal + def f[T: Box](y: T.a) -> T.a: + return y + c: 1 + 1 + "}, + indoc! {" + from typing import Literal as Literal2, TypeVar + class Literal: ... + class Box: + a: Literal + _T = TypeVar(\"_T\", bound=Box) + def f(y: Literal) -> Literal: + return y + c: Literal2[2] + "}, + ); + } + + /// a type that names both the class and the special form has no spelling that tells + /// them apart, so an attribute type, which has to be replaced, is written `Any` + #[test] + fn a_fold_naming_a_class_and_a_special_form_alike_is_any() { + let out = transpile( + indoc! {" + import typing + class Literal: ... + class Box: + a: Literal | typing.Literal[1] + def f[T: Box](y: T.a) -> T.a: + return y + "}, + &Config::test_default(), + ) + .unwrap(); + assert!(out.contains("def f(y: Any) -> Any:"), "got:\n{out}"); + } + /// a fold to a float literal type is written as the same literal written in the source /// is: `float` by default, since python has no spelling for the literal itself #[test] diff --git a/crates/by_transforms/src/transforms/template_type.rs b/crates/by_transforms/src/transforms/template_type.rs index 38fb28267b..07038f4ae3 100644 --- a/crates/by_transforms/src/transforms/template_type.rs +++ b/crates/by_transforms/src/transforms/template_type.rs @@ -34,24 +34,29 @@ use ruff_python_ast::{Expr, Stmt}; use ruff_text_size::{Ranged, TextRange}; use crate::transforms::ast_driver::{PassContext, TypeAwarePass}; +use crate::transforms::repeated_underscore::WrittenNames; use crate::transforms::type_expr_walker::{ Recurse, TypeExprVisitor, TypePos, walk_type_positions_skipping, }; use crate::type_info::TypeInfo; -pub(crate) struct TemplateTypePass; +pub(crate) struct TemplateTypePass<'src> { + pub(crate) written: WrittenNames<'src>, +} -impl TypeAwarePass for TemplateTypePass { +impl TypeAwarePass for TemplateTypePass<'_> { fn run(&self, stmts: &[Stmt], types: &dyn TypeInfo, ctx: &mut PassContext) { let mut inner = TemplateType { types, + literal: self.written.imported("typing", "Literal"), + str_: self.written.builtin("str"), edits: Vec::new(), needs_literal_import: false, }; walk_type_positions_skipping(stmts, Some(types), &ctx.claimed_type_op_ranges, &mut inner); if inner.needs_literal_import { ctx.required_imports - .push("from typing import Literal".to_owned()); + .push(self.written.import_from("typing", &["Literal"])); } ctx.text_edits.extend(inner.edits); } @@ -59,6 +64,10 @@ impl TypeAwarePass for TemplateTypePass { struct TemplateType<'src> { types: &'src dyn TypeInfo, + /// the name `typing.Literal` is written under + literal: String, + /// the name the builtin `str` is written under + str_: String, edits: Vec<(TextRange, String)>, needs_literal_import: bool, } @@ -76,9 +85,9 @@ impl TypeExprVisitor for TemplateType<'_> { .map(|value| render_string_literal(value)) .collect::>() .join(", "); - format!("Literal[{arms}]") + format!("{}[{arms}]", self.literal) } - _ => "str".to_owned(), + _ => self.str_.clone(), }; self.edits.push((expr.range(), replacement)); // the holes are type expressions of their own, but they are gone from diff --git a/crates/by_transforms/src/transforms/top_star.rs b/crates/by_transforms/src/transforms/top_star.rs index 30b33a9449..185c1cc7e4 100644 --- a/crates/by_transforms/src/transforms/top_star.rs +++ b/crates/by_transforms/src/transforms/top_star.rs @@ -19,19 +19,24 @@ use ruff_python_ast::{ use ruff_text_size::{Ranged, TextRange}; use super::ast_driver::{AstPass, PassContext, render_expr}; +use super::repeated_underscore::WrittenNames; use super::type_expr_walker::{Recurse, TypeExprVisitor, TypePos, walk_type_positions}; -pub(crate) struct TopStar; +pub(crate) struct TopStar<'src> { + written: WrittenNames<'src>, +} -impl TopStar { - pub(crate) fn new() -> Self { - Self +impl<'src> TopStar<'src> { + pub(crate) fn new(written: WrittenNames<'src>) -> Self { + Self { written } } } -impl AstPass for TopStar { +impl AstPass for TopStar<'_> { fn run(&self, module: &mut ModModule, ctx: &mut PassContext) { let mut state = State { + top: self.written.imported("ty_extensions", "Top"), + any: self.written.imported("typing", "Any"), edits: Vec::new(), needs_top: false, needs_any: false, @@ -41,16 +46,19 @@ impl AstPass for TopStar { ctx.text_edits.extend(state.edits); if state.needs_top { ctx.required_imports - .push("from ty_extensions import Top".to_owned()); + .push(self.written.import_from("ty_extensions", &["Top"])); } if state.needs_any { ctx.required_imports - .push("from typing import Any".to_owned()); + .push(self.written.import_from("typing", &["Any"])); } } } struct State { + /// the names `ty_extensions.Top` and `typing.Any` are written under + top: String, + any: String, edits: Vec<(TextRange, String)>, needs_top: bool, needs_any: bool, @@ -64,7 +72,7 @@ impl TypeExprVisitor for State { self.needs_top = true; self.needs_any = true; let mut new_slice = (*s.slice).clone(); - replace_markers_with_any(&mut new_slice); + replace_markers_with_any(&mut new_slice, &self.any); let inner = Expr::Subscript(ExprSubscript { node_index: AtomicNodeIndex::NONE, range: TextRange::default(), @@ -80,7 +88,7 @@ impl TypeExprVisitor for State { value: Box::new(Expr::Name(ExprName { node_index: AtomicNodeIndex::NONE, range: TextRange::default(), - id: Name::from("Top"), + id: Name::from(self.top.as_str()), ctx: ExprContext::Load, })), slice: Box::new(inner), @@ -99,11 +107,11 @@ impl TypeExprVisitor for State { } } -fn any_expr() -> Expr { +fn any_expr(any: &str) -> Expr { Expr::Name(ExprName { node_index: AtomicNodeIndex::NONE, range: TextRange::default(), - id: Name::from("Any"), + id: Name::from(any), ctx: ExprContext::Load, }) } @@ -111,26 +119,26 @@ fn any_expr() -> Expr { /// Replace every top-star marker reachable from `expr` with an `Any` name, /// stopping at nested `Subscript` boundaries so markers bind to their /// enclosing subscript only -fn replace_markers_with_any(expr: &mut Expr) { +fn replace_markers_with_any(expr: &mut Expr, any: &str) { if is_top_star_marker(expr) { - *expr = any_expr(); + *expr = any_expr(any); return; } match expr { Expr::Subscript(_) => {} Expr::BinOp(b) => { - replace_markers_with_any(&mut b.left); - replace_markers_with_any(&mut b.right); + replace_markers_with_any(&mut b.left, any); + replace_markers_with_any(&mut b.right, any); } Expr::Tuple(t) => { for elt in &mut t.elts { - replace_markers_with_any(elt); + replace_markers_with_any(elt, any); } } - Expr::UnaryOp(u) => replace_markers_with_any(&mut u.operand), + Expr::UnaryOp(u) => replace_markers_with_any(&mut u.operand, any), Expr::BoolOp(b) => { for v in &mut b.values { - replace_markers_with_any(v); + replace_markers_with_any(v, any); } } _ => {} diff --git a/crates/by_transforms/src/transforms/type_is.rs b/crates/by_transforms/src/transforms/type_is.rs index 25b2f9fbeb..dc4f348bb7 100644 --- a/crates/by_transforms/src/transforms/type_is.rs +++ b/crates/by_transforms/src/transforms/type_is.rs @@ -22,15 +22,17 @@ use ruff_python_ast::{ use ruff_text_size::{Ranged, TextRange, TextSize}; use super::ast_driver::{AstPass, PassContext, render_expr}; +use super::repeated_underscore::WrittenNames; use super::type_expr_walker::{Recurse, TypeExprVisitor, TypePos, walk_type_positions_skipping}; pub(crate) struct TypeIs<'src> { src: &'src str, + written: WrittenNames<'src>, } impl<'src> TypeIs<'src> { - pub(crate) fn new(src: &'src str) -> Self { - Self { src } + pub(crate) fn new(src: &'src str, written: WrittenNames<'src>) -> Self { + Self { src, written } } } @@ -43,6 +45,7 @@ impl AstPass for TypeIs<'_> { fn run(&self, module: &mut ModModule, ctx: &mut PassContext) { let mut state = State { + type_is: self.written.imported("typing", "TypeIs"), edits: Vec::new(), needs_import: false, }; @@ -53,6 +56,7 @@ impl AstPass for TypeIs<'_> { // returns, and their ranges are claimed so the `TypeIs[T]` rewrite below skips them let mut guards = ReturnGuards { src: self.src, + bool_: self.written.builtin("bool"), edits: Vec::new(), claimed: Vec::new(), }; @@ -67,12 +71,14 @@ impl AstPass for TypeIs<'_> { // typing.TypeIs landed in 3.13 (PEP 742). on older runtimes the // typing_redirect pass switches the import to typing_extensions ctx.required_imports - .push("from typing import TypeIs".to_owned()); + .push(self.written.import_from("typing", &["TypeIs"])); } } } struct State { + /// the name `typing.TypeIs` is written under + type_is: String, edits: Vec<(TextRange, String)>, needs_import: bool, } @@ -80,6 +86,8 @@ struct State { /// lowers the narrowing return annotations that have no `TypeIs` spelling struct ReturnGuards<'src> { src: &'src str, + /// the name the builtin `bool` is written under + bool_: String, edits: Vec<(TextRange, String)>, claimed: Vec, } @@ -120,7 +128,7 @@ impl ReturnGuards<'_> { .iter() .any(|parameter| parameter.name().id == *name); if !narrows_a_parameter { - self.edits.push((returns.range(), "bool".to_owned())); + self.edits.push((returns.range(), self.bool_.clone())); self.claimed.push(returns.range()); } } @@ -150,7 +158,7 @@ impl TypeExprVisitor for State { value: Box::new(Expr::Name(ExprName { node_index: AtomicNodeIndex::NONE, range: TextRange::default(), - id: Name::from("TypeIs"), + id: Name::from(self.type_is.as_str()), ctx: ExprContext::Load, })), slice: Box::new(target.clone()), diff --git a/crates/by_transforms/src/transforms/typed_dict_literal.rs b/crates/by_transforms/src/transforms/typed_dict_literal.rs index a6dd5ba1c1..37fe10159a 100644 --- a/crates/by_transforms/src/transforms/typed_dict_literal.rs +++ b/crates/by_transforms/src/transforms/typed_dict_literal.rs @@ -29,6 +29,7 @@ //! //! field types are emitted as forward references — see [`quote_type`] +use std::collections::BTreeSet; use std::collections::hash_map::DefaultHasher; use std::fmt::Write as _; use std::hash::{Hash, Hasher}; @@ -67,10 +68,12 @@ impl Shape { format!("_TypedDict_{truncated:08x}") } - fn class_def(&self, name: &str) -> String { + /// the class declaring this shape as `name`, `typing_extensions.TypedDict` written as + /// `typed_dict` + fn class_def(&self, name: &str, typed_dict: &str) -> String { let bases = match &self.extra_items { - Some(ty) => format!("TypedDict, extra_items={}", quote_type(ty)), - None => "TypedDict, closed=True".to_owned(), + Some(ty) => format!("{typed_dict}, extra_items={}", quote_type(ty)), + None => format!("{typed_dict}, closed=True"), }; let mut out = format!("class {name}({bases}):\n"); if self.fields.is_empty() { @@ -114,6 +117,7 @@ fn quote_type(type_source: &str) -> String { pub(crate) struct TypedDictLiteral<'src> { source: &'src str, + written: WrittenNames<'src>, edits: Vec, errors: Vec, /// Insertion-ordered map of shape → synthesized class name so emitted @@ -124,10 +128,9 @@ pub(crate) struct TypedDictLiteral<'src> { /// spans with their class names. range_to_class: Vec<(ruff_text_size::TextRange, String)>, needs_import: bool, - /// Whether any field rendering used basedpython literal-type promotion. - /// Lib's preamble step turns this into the `from typing import Literal` - /// import. - needs_literal_import: bool, + /// the `(module, name)` of each name a field type this pass writes itself reads: the + /// `Literal` of a promoted literal type, the `Intersection` of an intersection + field_imports: BTreeSet<(&'static str, &'static str)>, /// set when a nested `T??` field needs the runtime `Optional[...]` wrapper needs_optional_runtime: bool, /// the shared type-expression lowerer, which prints every field type this pass does @@ -145,21 +148,23 @@ impl<'src> TypedDictLiteral<'src> { ) -> Self { Self { source, + written, callable: CallableSyntax::new(source, written, config).with_types(types), edits: Vec::new(), errors: Vec::new(), shapes: IndexMap::new(), range_to_class: Vec::new(), needs_import: false, - needs_literal_import: false, + field_imports: BTreeSet::new(), needs_optional_runtime: false, } } fn class_defs(&self) -> String { let mut out = String::new(); + let typed_dict = self.written.imported("typing_extensions", "TypedDict"); for (shape, name) in &self.shapes { - out.push_str(&shape.class_def(name)); + out.push_str(&shape.class_def(name, &typed_dict)); out.push('\n'); } out @@ -212,7 +217,7 @@ impl<'src> TypedDictLiteral<'src> { // erases type arguments anyway, so the shape lowers to a // `TypedDict` that admits the pack's keys as extra items if matches!(&item.value, Expr::Name(_)) && extra_items.is_none() { - extra_items = Some("object".to_owned()); + extra_items = Some(self.written.builtin("object")); continue; } return None; @@ -277,11 +282,16 @@ impl<'src> TypedDictLiteral<'src> { let parts = flatten_bitand(expr); let rendered: Vec = parts.iter().map(|p| self.render_field_type(p)).collect(); - format!("Intersection[{}]", rendered.join(", ")) + self.field_imports.insert(("ty_extensions", "Intersection")); + format!( + "{}[{}]", + self.written.imported("ty_extensions", "Intersection"), + rendered.join(", ") + ) } Expr::Tuple(t) if t.parenthesized && !t.elts.is_empty() => { let parts: Vec = t.elts.iter().map(|e| self.render_field_type(e)).collect(); - format!("tuple[{}]", parts.join(", ")) + format!("{}[{}]", self.written.builtin("tuple"), parts.join(", ")) } // `T?` → `T | None` (nested `T??` → `Optional[T | None]`) so an // optional field type composes instead of leaking `?` @@ -306,8 +316,12 @@ impl<'src> TypedDictLiteral<'src> { ) } _ if is_literal_value(expr) => { - self.needs_literal_import = true; - format!("Literal[{}]", self.render_subbed(expr.range())) + self.field_imports.insert(("typing", "Literal")); + format!( + "{}[{}]", + self.written.imported("typing", "Literal"), + self.render_subbed(expr.range()) + ) } _ => self.callable.lower_type_expr(expr), } @@ -319,15 +333,16 @@ impl<'src> TypedDictLiteral<'src> { let parts = flatten_bitor(expr); let mut out_groups: Vec = Vec::new(); let mut pending_list: Vec = Vec::new(); + let literal = self.written.imported("typing", "Literal"); let flush = |pending: &mut Vec, out: &mut Vec| { if !pending.is_empty() { - out.push(format!("Literal[{}]", pending.join(", "))); + out.push(format!("{literal}[{}]", pending.join(", "))); pending.clear(); } }; for p in parts { if is_literal_value(p) { - self.needs_literal_import = true; + self.field_imports.insert(("typing", "Literal")); pending_list.push(self.render_subbed(p.range())); } else { flush(&mut pending_list, &mut out_groups); @@ -493,9 +508,9 @@ impl AstPass for TypedDictLiteralPass<'_> { // information the field lowerer asks is about those nodes, and the edits are keyed // on the source ranges both share walk_type_positions(self.suite, None, &mut inner); - if inner.needs_literal_import { + for (module, name) in &inner.field_imports { ctx.required_imports - .push("from typing import Literal".to_owned()); + .push(self.written.import_from(module, &[name])); } if inner.needs_optional_runtime { ctx.runtime.insert(crate::runtime::OPTIONAL); @@ -505,8 +520,10 @@ impl AstPass for TypedDictLiteralPass<'_> { // which the stdlib `typing.TypedDict` rejects on every released // python. source from `typing_extensions` so the generated class // body actually executes at runtime - ctx.required_imports - .push("from typing_extensions import TypedDict".to_owned()); + ctx.required_imports.push( + self.written + .import_from("typing_extensions", &["TypedDict"]), + ); // synthesized class defs are raw multi-line python source. push // them into required_imports as a single non-`from` line so // merge_from_imports leaves them untouched and they end up in @@ -532,9 +549,9 @@ impl AstPass for TypedDictLiteralPass<'_> { } } -/// AST-pass wrapper around [`TypedDictLiteral`] that surfaces its -/// `needs_literal_import` flag back to the driver so the literal-types -/// import is emitted alongside the synthesized class definitions +/// AST-pass wrapper around [`TypedDictLiteral`] that surfaces the imports its +/// field types read back to the driver, so they are emitted alongside the +/// synthesized class definitions pub(crate) struct TypedDictLiteralPass<'src> { source: &'src str, written: WrittenNames<'src>, @@ -595,6 +612,25 @@ mod tests { assert!(out.contains("a: _TypedDict_"), "got: {out}"); } + /// a field's type is read when the class's hints are, so the name it reads is imported + #[test] + fn an_intersection_field_imports_intersection() { + let out = transpile( + indoc! {" + class A: ... + class B: ... + a: {\"x\": A & B} + "}, + &Config::test_default(), + ) + .unwrap(); + assert!( + out.contains(" x: \"Intersection[A, B]\"\n") + && out.contains("from ty_extensions import Intersection"), + "got: {out}" + ); + } + #[test] fn extra_items_marker() { let out = transpile("b: {\"key\": int, **: str}\n", &Config::test_default()).unwrap(); diff --git a/crates/by_transforms/src/transforms/typeof_keyword.rs b/crates/by_transforms/src/transforms/typeof_keyword.rs index e94421de73..6c36a80372 100644 --- a/crates/by_transforms/src/transforms/typeof_keyword.rs +++ b/crates/by_transforms/src/transforms/typeof_keyword.rs @@ -28,6 +28,8 @@ use super::type_expr_walker::{Recurse, TypeExprVisitor, TypePos, walk_type_posit use crate::type_info::TypeInfo; pub(crate) struct TypeofFold { + /// the name `ty_extensions.TypeOf` is written under + type_of: String, changed: Cell, ever_changed: Cell, /// ranges of `typeof` nodes owned by the type-expression lowerer — skipped @@ -35,8 +37,9 @@ pub(crate) struct TypeofFold { } impl TypeofFold { - pub(crate) fn new(skip: Vec) -> Self { + pub(crate) fn new(skip: Vec, type_of: String) -> Self { Self { + type_of, changed: Cell::new(false), ever_changed: Cell::new(false), skip, @@ -65,7 +68,7 @@ impl Transformer for TypeofFold { *s.value = Expr::Name(ExprName { node_index: ruff_python_ast::AtomicNodeIndex::NONE, range: TextRange::default(), - id: Name::from("TypeOf"), + id: Name::from(self.type_of.as_str()), ctx: ExprContext::Load, }); self.changed.set(true); diff --git a/crates/by_transforms/src/transforms/unpack.rs b/crates/by_transforms/src/transforms/unpack.rs index 61be3afb9a..e44f91282f 100644 --- a/crates/by_transforms/src/transforms/unpack.rs +++ b/crates/by_transforms/src/transforms/unpack.rs @@ -24,18 +24,25 @@ use ruff_python_ast::{Expr, ModModule, Stmt}; use ruff_text_size::{Ranged, TextRange, TextSize}; use super::ast_driver::{AstPass, PassContext}; +use super::repeated_underscore::WrittenNames; use super::type_expr_walker::{Recurse, TypeExprVisitor, TypePos, walk_type_positions}; use crate::config::Config; use crate::type_info::TypeInfo; -pub(crate) struct UnpackSyntax { +pub(crate) struct UnpackSyntax<'src> { + written: WrittenNames<'src>, config: Config, type_subscripts: TypeSubscripts, } -impl UnpackSyntax { - pub(crate) fn new(config: Config, type_subscripts: TypeSubscripts) -> Self { +impl<'src> UnpackSyntax<'src> { + pub(crate) fn new( + written: WrittenNames<'src>, + config: Config, + type_subscripts: TypeSubscripts, + ) -> Self { Self { + written, config, type_subscripts, } @@ -63,7 +70,7 @@ impl TypeExprVisitor for TypeSubscripts { } } -impl AstPass for UnpackSyntax { +impl AstPass for UnpackSyntax<'_> { fn lowering(&self) -> Option { Some(super::ast_driver::Lowering::Unpack) } @@ -85,6 +92,7 @@ impl AstPass for UnpackSyntax { return; } let mut state = State { + unpack: format!("{}[", self.written.imported("typing", "Unpack")), edits: RefCell::new(Vec::new()), needs_import: false, lowered_varargs: pack.lowered_varargs, @@ -95,7 +103,7 @@ impl AstPass for UnpackSyntax { } if state.needs_import { ctx.required_imports - .push("from typing import Unpack".to_owned()); + .push(self.written.import_from("typing", &["Unpack"])); } ctx.text_edits.extend(state.edits.into_inner()); } @@ -178,6 +186,8 @@ fn star_token_range(unpack: TextRange) -> TextRange { } struct State<'a> { + /// `Unpack[`, with `typing.Unpack` written under the name the lowering imports it by + unpack: String, edits: RefCell>, needs_import: bool, lowered_varargs: Vec, @@ -201,7 +211,7 @@ impl State<'_> { let star_range = star_token_range(starred.range()); self.edits .borrow_mut() - .push((star_range, "Unpack[".to_owned())); + .push((star_range, self.unpack.clone())); let end = starred.range().end(); self.edits .borrow_mut() @@ -220,7 +230,7 @@ impl State<'_> { let star_range = star_token_range(ann.range()); self.edits .borrow_mut() - .push((star_range, "Unpack[".to_owned())); + .push((star_range, self.unpack.clone())); let end = ann.range().end(); self.edits .borrow_mut() diff --git a/crates/by_transforms/src/transforms/visibility_rename.rs b/crates/by_transforms/src/transforms/visibility_rename.rs index ffb3633cbd..ebb26b8bf8 100644 --- a/crates/by_transforms/src/transforms/visibility_rename.rs +++ b/crates/by_transforms/src/transforms/visibility_rename.rs @@ -41,13 +41,15 @@ use ruff_python_ast::{self as ast, Expr, ExprContext, Stmt}; use ruff_text_size::{Ranged, TextRange, TextSize}; use super::ast_driver::{PassContext, TypeAwarePass}; -use super::modifiers::module_private_name; +use super::repeated_underscore::WrittenNames; use super::source_util::header_end; use crate::type_info::TypeInfo; -pub(crate) struct VisibilityRenamePass; +pub(crate) struct VisibilityRenamePass<'src> { + pub(crate) written: WrittenNames<'src>, +} -impl TypeAwarePass for VisibilityRenamePass { +impl TypeAwarePass for VisibilityRenamePass<'_> { fn lowering(&self) -> Option { Some(super::ast_driver::Lowering::VisibilityRename) } @@ -64,6 +66,7 @@ impl TypeAwarePass for VisibilityRenamePass { declarations.visit_body(stmts); let mut renamer = Renamer { types, + written: self.written, edits: Vec::new(), header_end: TextSize::new(0), module_private, @@ -147,6 +150,7 @@ fn has_visibility_modifier(decorators: &[ast::Decorator]) -> bool { struct Renamer<'a> { types: &'a dyn TypeInfo, + written: WrittenNames<'a>, edits: Vec<(ruff_text_size::TextRange, String)>, /// the offset past the enclosing `def`'s header, or zero outside one header_end: TextSize, @@ -229,7 +233,7 @@ impl Renamer<'_> { fn rename_binding(&mut self, name: &ast::Identifier) { if self.binds_module(name.as_str()) { self.edits - .push((name.range, module_private_name(name.as_str()))); + .push((name.range, self.written.module_private(name.as_str()))); } } @@ -246,7 +250,7 @@ impl Renamer<'_> { if !self.binds_module(bound) { return; } - let renamed = module_private_name(bound); + let renamed = self.written.module_private(bound); match &alias.asname { Some(asname) => self.edits.push((asname.range, renamed)), // `import a.b` binds its top-level package, which no alias keeps: the @@ -311,7 +315,7 @@ impl<'ast> Visitor<'ast> for Renamer<'_> { for name in &global.names { if self.module_private.contains(name.as_str()) { self.edits - .push((name.range, module_private_name(name.as_str()))); + .push((name.range, self.written.module_private(name.as_str()))); } } } @@ -372,7 +376,7 @@ impl<'ast> Visitor<'ast> for Renamer<'_> { && self.types.resolves_to_module_scope(name) == Some(true) { self.edits - .push((name.range, module_private_name(name.id.as_str()))); + .push((name.range, self.written.module_private(name.id.as_str()))); } else if let Some(renamed) = self.types.class_body_member_name(name) { self.edits.push((name.range, renamed)); } diff --git a/crates/by_transforms/src/type_info.rs b/crates/by_transforms/src/type_info.rs index 0e050bb813..1734c973f9 100644 --- a/crates/by_transforms/src/type_info.rs +++ b/crates/by_transforms/src/type_info.rs @@ -1,5 +1,6 @@ //! Abstraction over type/binding information consumed by transforms. +use crate::transforms::repeated_underscore::WrittenNames; use crate::transforms::trailing_lambda::RECEIVER_PARAMETER; use ruff_python_ast::helpers::is_dotted_name; use ruff_python_ast::{ @@ -7,7 +8,6 @@ use ruff_python_ast::{ StmtFunctionDef, }; use ruff_python_parser::parse_expression; -use ruff_python_stdlib::basedpython::IMPLICIT_TYPING_NAMES; use ruff_text_size::TextRange; use ty_python_core::scope::ScopeKind; use ty_python_core::{global_scope, place_table, semantic_index}; @@ -66,6 +66,62 @@ pub(crate) struct SynthesizedType { pub(crate) typing_names: Vec<&'static str>, } +impl SynthesizedType { + /// the type as the module `written` is of writes it, each of [`Self::typing_names`] under + /// the name the lowering imports it by + pub(crate) fn text_in(&self, written: WrittenNames) -> String { + struct Roots<'a> { + typing_names: &'a [&'static str], + written: WrittenNames<'a>, + edits: Vec<(TextRange, String)>, + } + impl<'ast> ruff_python_ast::visitor::Visitor<'ast> for Roots<'_> { + fn visit_expr(&mut self, expr: &'ast Expr) { + if let Expr::Name(name) = expr + && self.typing_names.contains(&name.id.as_str()) + { + let local = self.written.imported("typing", name.id.as_str()); + if local != name.id.as_str() { + self.edits.push((name.range, local)); + } + } + ruff_python_ast::visitor::walk_expr(self, expr); + } + } + let Ok(parsed) = parse_expression(&self.text) else { + return self.text.clone(); + }; + let mut roots = Roots { + typing_names: &self.typing_names, + written, + edits: Vec::new(), + }; + ruff_python_ast::visitor::Visitor::visit_expr(&mut roots, parsed.expr()); + roots + .edits + .sort_by_key(|(range, _)| std::cmp::Reverse(range.start())); + let mut text = self.text.clone(); + for (range, local) in roots.edits { + text.replace_range(std::ops::Range::::from(range), &local); + } + text + } + + /// the imports [`Self::text_in`] reads: each module it qualifies a class with, and each + /// `typing` name + pub(crate) fn imports_in(&self, written: WrittenNames) -> Vec { + self.modules + .iter() + .map(|module| format!("import {module}")) + .chain( + self.typing_names + .iter() + .map(|name| written.import_from("typing", &[name])), + ) + .collect() + } +} + /// How an assignment inside a trailing-lambda block reaches an enclosing scope, /// so the block writes through instead of shadowing with a fresh local. #[derive(Clone, Copy, PartialEq, Eq, Debug)] @@ -76,6 +132,33 @@ pub(crate) enum CaptureKind { Nonlocal, } +/// a `context` argument a call is given implicitly +pub(crate) struct ImplicitContextArgument { + /// the parameter it fills + parameter: String, + variable: ContextVariable, +} + +/// the variable that fills an implicit `context` argument +enum ContextVariable { + /// a variable the lowered module spells as ty names it + Named(String), + /// a module-level `private` variable, which the lowering emits under another name + ModulePrivate(String), +} + +impl ImplicitContextArgument { + /// the `parameter=variable` keyword argument, the variable spelled as the lowered module + /// binds it + pub(crate) fn keyword(&self, written: crate::WrittenNames) -> String { + let variable = match &self.variable { + ContextVariable::Named(variable) => variable.clone(), + ContextVariable::ModulePrivate(variable) => written.module_private(variable), + }; + format!("{}={variable}", self.parameter) + } +} + pub(crate) trait TypeInfo { /// whether `X[…]` treats the slice as type arguments, where `X` is whatever /// is being subscripted — a bare name (`list[int]`) or a dotted one @@ -205,6 +288,11 @@ pub(crate) trait TypeInfo { /// target and nothing ever evaluates it fn is_runtime_union(&self, expr: &Expr) -> bool; + /// the argument a call to the builtin `isinstance` or `issubclass` reads as classes, when + /// `call` is one, whatever the name the callee is reached by — ty's answer, the one it + /// exempts a union the lowering spells there by + fn classinfo_argument<'a>(&self, call: &'a ExprCall) -> Option<&'a Expr>; + /// when `attribute` resolves to a basedpython `extension` member, the /// backing-function rewrite to apply (`xs.second()` → /// `_by_ext__list__second(xs)`). `None` for ordinary attributes — @@ -418,7 +506,11 @@ pub(crate) trait TypeInfo { /// kept precise. returns `None` when ty resolves no concrete type — e.g. an /// unsupported operation inferred as `Unknown` — so the caller leaves the /// source unchanged and ty's own diagnostic stands - fn symbolic_type_fold(&self, expr: &Expr) -> Option; + fn symbolic_type_fold( + &self, + expr: &Expr, + min_version: ruff_python_ast::PythonVersion, + ) -> Option; /// basedpython: the finite set of strings an f-string in a type position /// denotes, when it denotes one — `f"a{1 | 2}b"` is `["a1b", "a2b"]`. @@ -565,7 +657,10 @@ pub(crate) trait TypeInfo { /// which of those a read of `_` means is not decided, so no argument can be /// written for it. an error too when a parameter to fill is a `_` the callee /// repeats, since which of those a keyword `_` names is not decided either - fn implicit_context_arguments(&self, call: &ExprCall) -> Result, String>; + fn implicit_context_arguments( + &self, + call: &ExprCall, + ) -> Result, String>; /// whether `expr`'s inferred type is a string — a `str` / `Character` / /// `LiteralString` / string-literal / `str`-subclass instance. dynamic @@ -752,7 +847,7 @@ impl TypeInfo for SemanticModel<'_> { // `final A`, which has no python spelling at all, and the annotation a reader // would write for it is `A` let promoted = ty.promote(db, &env).promote_class_literals(db, &env); - spell_for_python(db, &env, self, promoted, min_version) + spell_for_python(db, &env, self, promoted, min_version, false) } fn is_return_value_marker(&self, name: &ExprName) -> bool { @@ -795,6 +890,10 @@ impl TypeInfo for SemanticModel<'_> { SemanticModel::denotes_plain_value(self, expr) } + fn classinfo_argument<'a>(&self, call: &'a ExprCall) -> Option<&'a Expr> { + ty_python_semantic::types::ide_support::classinfo_argument(self, call) + } + fn is_runtime_union(&self, expr: &Expr) -> bool { // `x = int | str` reads as the `types.UnionType` object it builds. in a // position that is *also* a type expression — a `cast` target, an @@ -1071,7 +1170,7 @@ impl TypeInfo for SemanticModel<'_> { let env = self.program_environment(); let ty = expr.inferred_type(self)?; let promoted = ty.promote(db, &env).promote_class_literals(db, &env); - spell_for_python(db, &env, self, promoted, min_version) + spell_for_python(db, &env, self, promoted, min_version, false) } fn template_literal_strings(&self, expr: &Expr) -> Option> { @@ -1079,7 +1178,11 @@ impl TypeInfo for SemanticModel<'_> { ty_python_semantic::finite_string_set(self.db(), ty) } - fn symbolic_type_fold(&self, expr: &Expr) -> Option { + fn symbolic_type_fold( + &self, + expr: &Expr, + min_version: ruff_python_ast::PythonVersion, + ) -> Option { let ty = expr.inferred_type(self)?; // fold concrete types and explicit `Any` (e.g. `dynamic + 1`, which ty // resolves to `Any`), but not the `Unknown` / Todo dynamics an @@ -1088,13 +1191,14 @@ impl TypeInfo for SemanticModel<'_> { if ty.is_dynamic() && !matches!(ty, Type::Dynamic(DynamicType::Any)) { return None; } - // display with the standard (non-basedpython) renderer so literals come - // out as `Literal[..]` rather than bare — the transpiler emits python - Some(strip_binding_context_suffix(&display_for_python( + spell_for_python( self.db(), &self.program_environment(), + self, ty, - ))) + min_version, + true, + ) } fn class_typevars(&self, expr: &Expr) -> Option)>> { @@ -1329,7 +1433,10 @@ impl TypeInfo for SemanticModel<'_> { SemanticModel::trailing_lambda_callee_is_once(self, callee) } - fn implicit_context_arguments(&self, call: &ExprCall) -> Result, String> { + fn implicit_context_arguments( + &self, + call: &ExprCall, + ) -> Result, String> { let Some(callee) = call.func.inferred_type(self) else { return Ok(Vec::new()); }; @@ -1354,17 +1461,15 @@ impl TypeInfo for SemanticModel<'_> { Ok(implicit .arguments .into_iter() - .map(|argument| { - let variable = if argument.is_block_receiver { - RECEIVER_PARAMETER.to_string() + .map(|argument| ImplicitContextArgument { + parameter: argument.parameter.to_string(), + variable: if argument.is_block_receiver { + ContextVariable::Named(RECEIVER_PARAMETER.to_string()) } else if argument.is_module_private { - // the binding is a module-level `private` variable, which the - // lowering emits under its underscored name - crate::transforms::modifiers::module_private_name(&argument.variable) + ContextVariable::ModulePrivate(argument.variable.to_string()) } else { - argument.variable.to_string() - }; - (argument.parameter.to_string(), variable) + ContextVariable::Named(argument.variable.to_string()) + }, }) .collect()) } @@ -1421,7 +1526,7 @@ impl TypeInfo for SemanticModel<'_> { // ``); `type[int]` is its promoted form, and the type ty // itself reads back off the undeclared attribute through an instance let promoted = ty.promote(db, &env).promote_class_literals(db, &env); - spell_for_python(db, &env, self, promoted, min_version) + spell_for_python(db, &env, self, promoted, min_version, false) } fn class_body_annotation_is_semantic(&self, class_def: &StmtClassDef) -> bool { @@ -1501,12 +1606,16 @@ fn display_for_python<'db>( /// the modules reported are only those the spelling actually names, because the reporting /// is per-type rather than per-name: ty names `types` for the type spelled `None`, which /// bought an emitted file a `if TYPE_CHECKING: import types` it never read. +/// +/// `literal` makes `Literal` a `typing` name like the implicit ones, for a caller that +/// writes the import it needs: a symbolic fold, whose answer is so often a literal type fn spell_for_python<'db>( db: &'db dyn Db, env: &ProgramEnvironment<'db>, model: &SemanticModel<'db>, ty: Type<'db>, min_version: ruff_python_ast::PythonVersion, + literal: bool, ) -> Option { let spelling = ty.source_spelling_in( db, @@ -1517,20 +1626,31 @@ fn spell_for_python<'db>( let text = strip_binding_context_suffix(&spelling.text); let read = names_read(&text)?; + let mut modules = spelling.modules; + modules.retain(|module| read.iter().any(|name| name == module)); + // a name the display writes that is neither a class nor a module is a `typing` special + // form's, whatever the module binds under it — `Literal` in `Literal[1]` — and is written + // under the name the lowering imports it by let typing_names: Vec<&'static str> = read .iter() - .filter(|name| !model.is_bound_globally(name)) - .filter_map(|name| IMPLICIT_TYPING_NAMES.iter().copied().find(|n| *n == name)) + .filter(|name| !spelling.bare_classes.contains(name) && !modules.contains(name)) + .filter_map(|name| typing_name(name, literal)) .collect(); - let mut modules = spelling.modules; - modules.retain(|module| read.iter().any(|name| name == module)); let spelled = SynthesizedType { text, modules, typing_names, }; - resolvable_in_output(model, min_version, &spelled, &read).then_some(spelled) + resolvable_in_output(model, min_version, &spelled, &spelling.bare_classes, &read) + .then_some(spelled) +} + +/// the `typing` name `name` is, when ty's display writes it for a special form rather than +/// for a class: one basedpython binds implicitly, or, when `literal`, `Literal` +fn typing_name(name: &str, literal: bool) -> Option<&'static str> { + ruff_python_stdlib::basedpython::implicit_typing_name(name) + .or_else(|| (literal && name == "Literal").then_some("Literal")) } /// Whether every name `spelled` reads will resolve in the emitted file. @@ -1544,11 +1664,13 @@ fn resolvable_in_output( model: &SemanticModel<'_>, min_version: ruff_python_ast::PythonVersion, spelled: &SynthesizedType, + bare_classes: &[String], read: &[String], ) -> bool { read.iter().all(|name| { spelled.modules.iter().any(|module| module == name) || spelled.typing_names.contains(&name.as_str()) + || bare_classes.contains(name) || model.is_bound_globally(name) || ruff_python_stdlib::builtins::is_python_builtin(name, min_version.minor, false) }) diff --git a/crates/by_transforms/tests/builtin_names_runtime.rs b/crates/by_transforms/tests/builtin_names_runtime.rs new file mode 100644 index 0000000000..ecea123963 --- /dev/null +++ b/crates/by_transforms/tests/builtin_names_runtime.rs @@ -0,0 +1,91 @@ +//! Runtime test for the builtins the lowerings write. +//! +//! A lowering that calls `isinstance` or decorates with `staticmethod` means the builtin, and +//! reads whatever the scope it lands in binds under that name. A module that binds the name +//! itself — a parameter named `isinstance`, a class attribute named `staticmethod` — has the +//! builtin read under a name of the lowering's own, imported from `builtins`. The transform +//! unit tests pin the spelling; this test runs it, below python 3.10, where `match` is +//! polyfilled too, and on the newest interpreter found. + +use std::process::Command; + +use by_transforms::{Config, PythonVersion, transpile}; + +mod interpreters; +use interpreters::Interpreter; + +const PROGRAM: &str = r#" +class A: + staticmethod = 3 + + static def m() -> int: + return 1 + +def f(isinstance: int, x: object) -> bool: + return x is int + +def g(type: int, x: object) -> bool: + return x is int | None + +class E(Exception): + pass + +def r() -> int | E: + return 3 + +def k(isinstance: int) -> int | E: + y = r()^ + return y + isinstance + +def n(len: int, xs: list[int]) -> int: + match xs: + case [a, b]: + return a + b + len + return 0 + +assert A.m() == 1 and A.staticmethod == 3 +assert f(1, 2) and not f(1, "s") +assert g(1, None) and g(1, 2) and not g(1, "s") +assert k(1) == 4 +assert n(10, [1, 2]) == 13 and n(10, [1]) == 0 +print("ok") +"#; + +fn run_at(interpreter: &Interpreter, target: PythonVersion) { + let config = Config { + min_version: target, + ..Config::default() + }; + let transpiled = transpile(PROGRAM, &config).expect("transpile should succeed"); + let output = Command::new(&interpreter.command) + .arg("-c") + .arg(&transpiled) + .output() + .expect("failed to spawn python"); + assert!( + output.status.success(), + "program transpiled for {target} failed on {interpreter}:\n--- stdout ---\n{}\n--- stderr ---\n{}\n--- transpiled ---\n{transpiled}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); + interpreters::ran(interpreter, target); +} + +#[test] +fn a_builtin_a_lowering_writes_is_the_builtin_below_python_310() { + if let Some(interpreter) = interpreters::oldest( + PythonVersion::PY39, + "import typing_extensions", + "with `typing_extensions`", + ) { + run_at(&interpreter, PythonVersion::PY39); + } +} + +#[test] +fn a_builtin_a_lowering_writes_is_the_builtin() { + if let Some(interpreter) = interpreters::interpreters().pop() { + run_at(&interpreter, interpreter.version); + } +} diff --git a/crates/by_transforms/tests/conversion_runtime.rs b/crates/by_transforms/tests/conversion_runtime.rs index 717bd1dcd0..dbccba3276 100644 --- a/crates/by_transforms/tests/conversion_runtime.rs +++ b/crates/by_transforms/tests/conversion_runtime.rs @@ -16,10 +16,12 @@ //! assignment, a plain assignment to a name declared elsewhere, an attribute //! assignment, a `return`, and element-wise inside a literal collection -use std::process::{Command, Stdio}; +use std::process::Command; use by_transforms::{Config, PythonVersion, transpile}; +mod interpreters; + /// basedpython whose module-level `assert`s exercise the three dunders end to end const PROGRAM: &str = r#" class Celsius: @@ -245,34 +247,19 @@ assert c["x"] == 1 print("ok") "#; -/// the first interpreter that accepts `probe` -fn python_supporting(probe: &str) -> Option { - let mut candidates = Vec::new(); - if let Ok(p) = std::env::var("PYTHON") { - candidates.push(p); - } - candidates.extend(["python3.13", "python3"].map(String::from)); - - candidates.into_iter().find(|py| { - Command::new(py) - .args(["-c", probe]) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status() - .map(|s| s.success()) - .unwrap_or(false) - }) -} - -/// transpile `program` and run it, asserting it exits cleanly -fn run_transpiled(python: &str, program: &str, min_version: PythonVersion) { +/// transpile `program` for `min_version` and run it on the oldest interpreter found of that +/// version or later that `probe` runs on, asserting it exits cleanly +fn run_transpiled(program: &str, min_version: PythonVersion, probe: &str, needs: &str) { + let Some(interpreter) = interpreters::oldest(min_version, probe, needs) else { + return; + }; let config = Config { min_version, ..Config::default() }; let transpiled = transpile(program, &config).expect("transpile should succeed"); - let output = Command::new(python) + let output = Command::new(&interpreter.command) .arg("-c") .arg(&transpiled) .output() @@ -280,47 +267,31 @@ fn run_transpiled(python: &str, program: &str, min_version: PythonVersion) { assert!( output.status.success(), - "transpiled conversion program failed on {python}:\n--- stdout ---\n{}\n--- stderr ---\n{}\n--- transpiled ---\n{transpiled}", + "transpiled conversion program failed on {interpreter}:\n--- stdout ---\n{}\n--- stderr ---\n{}\n--- transpiled ---\n{transpiled}", String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr), ); + interpreters::ran(&interpreter, min_version); } #[test] -#[expect( - clippy::print_stderr, - reason = "a skipped test must say why it skipped, or it reads as a pass" -)] fn conversions_run_correctly() { - let Some(python) = python_supporting("import sys; sys.exit(0)") else { - eprintln!("skipping conversion runtime test: no python interpreter found"); - return; - }; - run_transpiled(&python, PROGRAM, PythonVersion::PY313); + run_transpiled(PROGRAM, PythonVersion::PY313, "", ""); } #[test] -#[expect( - clippy::print_stderr, - reason = "a skipped test must say why it skipped, or it reads as a pass" -)] fn frozen_displays_run_correctly() { - let Some(python) = python_supporting("import sys; sys.exit(0)") else { - eprintln!("skipping frozen display runtime test: no python interpreter found"); - return; - }; - run_transpiled(&python, FROZEN_PROGRAM, PythonVersion::PY313); + run_transpiled(FROZEN_PROGRAM, PythonVersion::PY313, "", ""); } #[test] -#[expect( - clippy::print_stderr, - reason = "a skipped test must say why it skipped, or it reads as a pass" -)] +#[ignore = "`transpile` analyses a module at ty's newest python, 3.14, where `frozendict` is no \ + builtin, so no conversion is written; `by transpile` in a 3.15 project writes it"] fn frozendict_displays_run_correctly() { - let Some(python) = python_supporting("frozendict") else { - eprintln!("skipping frozendict runtime test: no interpreter with `frozendict` (3.15+)"); - return; - }; - run_transpiled(&python, FROZENDICT_PROGRAM, PythonVersion::PY315); + run_transpiled( + FROZENDICT_PROGRAM, + PythonVersion::PY315, + "frozendict", + "with `frozendict`", + ); } diff --git a/crates/by_transforms/tests/data_class_runtime.rs b/crates/by_transforms/tests/data_class_runtime.rs new file mode 100644 index 0000000000..c9cee1a92a --- /dev/null +++ b/crates/by_transforms/tests/data_class_runtime.rs @@ -0,0 +1,92 @@ +//! Runtime test for the slots a `data class` is given. +//! +//! A `data class` is a slotted dataclass. `dataclass(slots=True)` is new in python 3.10, so +//! below it the runtime's `_by_dataclass_slots` does what that option does. The transform +//! unit tests pin which of the two a target gets; this test runs both and checks they +//! behave alike. + +use std::process::Command; + +use by_transforms::{Config, PythonVersion, transpile}; + +mod interpreters; +use interpreters::Interpreter; + +const PROGRAM: &str = r#" +import pickle + +data class Pair: + a: int + b: list[int] = [] + + def total(self) -> int: + return self.a + len(self.b) + +frozen data class Point: + x: int + y: int = 0 + +data class Child(Pair): + c: int = 4 + + override def total(self) -> int: + return super().total() + self.c + +p = Pair(1) +assert getattr(Pair, "__slots__") == ("a", "b") and not hasattr(p, "__dict__") +try: + setattr(p, "extra", 1) +except AttributeError: + pass +else: + raise AssertionError("a slotted instance takes no attribute it does not declare") +p.b.append(2) +assert Pair(1).b == [], "a mutable default is still made per instance" +q = Point(1) +assert pickle.loads(pickle.dumps(q)) == q, "a frozen slotted instance round-trips" +assert getattr(Child, "__slots__") == ("c",), "a base's slots are not declared again" +assert Child(1, [1, 2]).total() == 7, "a zero-argument `super()` reaches the slotted class" +print("ok") +"#; + +fn run_at(interpreter: &Interpreter, target: PythonVersion) { + let config = Config { + min_version: target, + ..Config::default() + }; + let transpiled = transpile(PROGRAM, &config).expect("transpile should succeed"); + let output = Command::new(&interpreter.command) + .arg("-c") + .arg(&transpiled) + .output() + .expect("failed to spawn python"); + assert!( + output.status.success(), + "program transpiled for {target} failed on {interpreter}:\n--- stdout ---\n{}\n--- stderr ---\n{}\n--- transpiled ---\n{transpiled}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); + interpreters::ran(interpreter, target); +} + +#[test] +fn a_data_class_is_slotted_below_python_310() { + if let Some(interpreter) = interpreters::oldest( + PythonVersion::PY39, + "import typing_extensions", + "with `typing_extensions`", + ) { + run_at(&interpreter, PythonVersion::PY39); + } +} + +/// `dataclass(slots=True)` itself. the `override` the program writes is `typing`'s from 3.12, +/// and python's own `slots=True` points a zero-argument `super()` at the class it makes from +/// 3.13 on +#[test] +fn a_data_class_is_slotted_from_python_310() { + if let Some(interpreter) = interpreters::oldest(PythonVersion::PY313, "", "") { + run_at(&interpreter, PythonVersion::PY312); + } +} diff --git a/crates/by_transforms/tests/fstring_debug_runtime.rs b/crates/by_transforms/tests/fstring_debug_runtime.rs index db9cea3ded..da0d531034 100644 --- a/crates/by_transforms/tests/fstring_debug_runtime.rs +++ b/crates/by_transforms/tests/fstring_debug_runtime.rs @@ -16,7 +16,8 @@ use std::process::Command; use by_transforms::{Config, PythonVersion, transpile}; -mod common; +mod interpreters; +use interpreters::Interpreter; /// basedpython whose `=` fields all hold an expression that lowers const PROGRAM: &str = r#" @@ -65,8 +66,8 @@ show(3, 7, {"k": 5}) print("ok") "#; -fn run(python: &str, source: &str, what: &str) { - let output = Command::new(python) +fn run(python: &Interpreter, source: &str, what: &str) { + let output = Command::new(&python.command) .arg("-c") .arg(source) .output() @@ -80,29 +81,12 @@ fn run(python: &str, source: &str, what: &str) { } #[test] -#[expect( - clippy::print_stderr, - reason = "a skipped test must say why it skipped, or it reads as a pass" -)] fn a_debug_field_prints_the_source_the_author_wrote() { - let Some(python) = common::python() else { - eprintln!("skipping f-string `=` field runtime test: no interpreter found"); + // a replacement field may nest the string's own quote from 3.12 on, and hold a backslash + // from the same release. the program is transpiled for 3.13 + let Some(python) = interpreters::oldest(PythonVersion::PY313, "", "") else { return; }; - // a replacement field may nest the string's own quote from 3.12 on, and hold a - // backslash from the same release - let probe = Command::new(&python) - .args([ - "-c", - "import sys; sys.exit(0 if sys.version_info >= (3, 12) else 1)", - ]) - .status() - .map(|status| status.success()) - .unwrap_or(false); - if !probe { - eprintln!("skipping f-string `=` field runtime test: needs python 3.12 or later"); - return; - } run(&python, PYTHON_REFERENCE, "the plain-python reference"); @@ -112,4 +96,5 @@ fn a_debug_field_prints_the_source_the_author_wrote() { }; let transpiled = transpile(PROGRAM, &config).expect("transpile should succeed"); run(&python, &transpiled, "the transpiled program"); + interpreters::ran(&python, PythonVersion::PY313); } diff --git a/crates/by_transforms/tests/grapheme_runtime.rs b/crates/by_transforms/tests/grapheme_runtime.rs index b7d41ee894..91534a0f5d 100644 --- a/crates/by_transforms/tests/grapheme_runtime.rs +++ b/crates/by_transforms/tests/grapheme_runtime.rs @@ -11,13 +11,15 @@ //! //! The grapheme helpers require the third-party `regex` package (the only //! widely available UAX #29 engine). If no interpreter with `regex` importable -//! can be found, the test skips rather than fails — it documents a runtime -//! dependency, it doesn't police the CI image. +//! can be found, the test skips rather than fails, and says so — it documents a +//! runtime dependency, it doesn't police the CI image. use std::process::Command; use by_transforms::{Config, PythonVersion, transpile}; +mod interpreters; + /// basedpython source whose module-level `assert`s exercise the grapheme /// surface end to end. every string here is a single grapheme cluster made of /// multiple code points, so a code-point implementation would give wrong @@ -71,35 +73,13 @@ assert type("a") is str, "a bare literal is still a plain str" print("ok") "#; -/// Locate an interpreter with `regex` importable: `$PYTHON` first, then a short -/// list of common names. Returns `None` (test skips) when none qualifies. -fn python_with_regex() -> Option { - let mut candidates = Vec::new(); - if let Ok(p) = std::env::var("PYTHON") { - candidates.push(p); - } - candidates.extend(["python3.13", "python3", "python"].map(String::from)); - - candidates.into_iter().find(|py| { - Command::new(py) - .args(["-c", "import regex"]) - .status() - .map(|s| s.success()) - .unwrap_or(false) - }) -} - #[test] -#[expect( - clippy::print_stderr, - reason = "a skipped test must say why it skipped, or it reads as a pass" -)] fn grapheme_surface_runs_correctly() { - let Some(python) = python_with_regex() else { - eprintln!( - "skipping grapheme runtime test: no interpreter with `regex` found \ - (set PYTHON to one with `pip install regex`)" - ); + let Some(interpreter) = interpreters::oldest( + PythonVersion::PY313, + "import regex", + "with `regex` (`pip install regex` into one, or set `PYTHON` to one that has it)", + ) else { return; }; @@ -109,7 +89,7 @@ fn grapheme_surface_runs_correctly() { }; let transpiled = transpile(PROGRAM, &config).expect("transpile should succeed"); - let output = Command::new(&python) + let output = Command::new(&interpreter.command) .arg("-c") .arg(&transpiled) .output() @@ -117,9 +97,10 @@ fn grapheme_surface_runs_correctly() { assert!( output.status.success(), - "transpiled grapheme program failed on {python}:\n--- stdout ---\n{}\n--- stderr ---\n{}\n--- transpiled ---\n{transpiled}", + "transpiled grapheme program failed on {interpreter}:\n--- stdout ---\n{}\n--- stderr ---\n{}\n--- transpiled ---\n{transpiled}", String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr), ); assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); + interpreters::ran(&interpreter, PythonVersion::PY313); } diff --git a/crates/by_transforms/tests/interpreters/mod.rs b/crates/by_transforms/tests/interpreters/mod.rs new file mode 100644 index 0000000000..0ae3db04e7 --- /dev/null +++ b/crates/by_transforms/tests/interpreters/mod.rs @@ -0,0 +1,116 @@ +//! The interpreters the runtime tests run transpiled python on, and how a test says which one +//! ran it. +//! +//! A test that needs something of its interpreter — a version, a module — skips when none here +//! has it, and a skip passes. Nothing on a passing test's line tells the two apart, so every test +//! built on this says which interpreter it ran on, or why it ran nothing, and the nextest +//! configuration shows what these binaries print when they pass. + +use std::process::{Command, Stdio}; + +use by_transforms::PythonVersion; + +/// an interpreter found under one of the names the tests look for +pub(crate) struct Interpreter { + pub(crate) command: String, + pub(crate) version: PythonVersion, + pub(crate) typing_extensions: bool, +} + +impl std::fmt::Display for Interpreter { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{} ({})", self.command, self.version)?; + if self.typing_extensions { + write!(f, " with `typing_extensions`")?; + } + Ok(()) + } +} + +impl Interpreter { + /// whether `probe`, python source, runs to completion here + fn runs(&self, probe: &str) -> bool { + Command::new(&self.command) + .args(["-c", probe]) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .is_ok_and(|status| status.success()) + } +} + +/// what an interpreter says about itself: its version, whether it has `typing_extensions`, and +/// the file it runs from, which tells two names for one interpreter apart +const PROBE: &str = "import sys, importlib.util; \ +print(sys.version_info[0], sys.version_info[1], \ +int(importlib.util.find_spec('typing_extensions') is not None), sys.executable)"; + +/// every interpreter this machine has under `$PYTHON`, `python3.N` or `python3`, oldest first, +/// each listed once however many of the names reach it +/// +/// `python3` alone is whichever one the path finds first, which need not be one a test can use, +/// so every name is asked +pub(crate) fn interpreters() -> Vec { + let mut candidates: Vec = std::env::var("PYTHON").into_iter().collect(); + candidates.extend((8..=15).map(|minor| format!("python3.{minor}"))); + candidates.push("python3".to_owned()); + + let mut seen = std::collections::HashSet::new(); + let mut found: Vec = candidates + .into_iter() + .filter_map(|command| { + let output = Command::new(&command) + .args(["-c", PROBE]) + .stderr(Stdio::null()) + .output() + .ok() + .filter(|output| output.status.success())?; + let stdout = String::from_utf8(output.stdout).ok()?; + let mut fields = stdout.trim_end().splitn(4, ' '); + let major = fields.next()?.parse().ok()?; + let minor = fields.next()?.parse().ok()?; + let typing_extensions = fields.next()? == "1"; + let executable = fields.next()?; + let executable = std::fs::canonicalize(executable) + .unwrap_or_else(|_| std::path::PathBuf::from(executable)); + seen.insert(executable).then(|| Interpreter { + command, + version: PythonVersion::from((major, minor)), + typing_extensions, + }) + }) + .collect(); + found.sort_by_key(|interpreter| interpreter.version); + found +} + +/// the oldest interpreter of `at_least` or later on which `probe`, python source, runs. none is a +/// skip, said so, with `needs` saying what else the test needed of it when it needed more than a +/// version +#[expect( + clippy::print_stderr, + reason = "a skipped test must say why it skipped, or it reads as a pass" +)] +pub(crate) fn oldest(at_least: PythonVersion, probe: &str, needs: &str) -> Option { + let found = interpreters() + .into_iter() + .find(|interpreter| interpreter.version >= at_least && interpreter.runs(probe)); + if found.is_none() { + let needs = if needs.is_empty() { + String::new() + } else { + format!(" {needs}") + }; + eprintln!("skipping: no interpreter of python {at_least} or later{needs} found"); + } + found +} + +/// say that `interpreter` ran the test, on python transpiled for `target` +#[expect( + clippy::print_stderr, + reason = "a test that can skip says what it ran on, so a pass is told apart from a skip" +)] +pub(crate) fn ran(interpreter: &Interpreter, target: PythonVersion) { + eprintln!("ran, transpiled for {target}, on {interpreter}"); +} diff --git a/crates/by_transforms/tests/lazy_import_runtime.rs b/crates/by_transforms/tests/lazy_import_runtime.rs index 14aadef40e..64a531be45 100644 --- a/crates/by_transforms/tests/lazy_import_runtime.rs +++ b/crates/by_transforms/tests/lazy_import_runtime.rs @@ -18,17 +18,15 @@ //! resolved eagerly would pass every forwarding assertion while quietly //! destroying the feature. -#![expect( - clippy::print_stderr, - reason = "a skipped test must say why it skipped, or it reads as a pass" -)] - use std::fs; use std::path::{Path, PathBuf}; use std::process::Command; use by_transforms::{Config, transpile}; +mod interpreters; +use interpreters::Interpreter; + /// module whose values the importer exercises. every binding is a plain value /// (not a callable), which is exactly the case the proxy used to break const VALUES_MODULE: &str = r#" @@ -256,31 +254,10 @@ assert typing.get_type_hints(half)["value"] === fractions.Fraction, "the string print("ok") "#; -/// an interpreter to run the transpiled output on. `$PYTHON` first, then the -/// usual names; `None` (test skips) when none is found -fn python() -> Option { - let mut candidates = Vec::new(); - if let Ok(p) = std::env::var("PYTHON") { - candidates.push(p); - } - candidates.extend(["python3", "python"].map(String::from)); - - candidates.into_iter().find(|py| { - Command::new(py) - .args(["-c", ""]) - .status() - .map(|s| s.success()) - .unwrap_or(false) - }) -} - -/// whether `python` runs what the transpiler targets by default, 3.10 — a -/// dataclass's `KW_ONLY` among it -fn runs_the_default_target(python: &str) -> bool { - Command::new(python) - .args(["-c", "import sys; sys.exit(sys.version_info < (3, 10))"]) - .status() - .is_ok_and(|status| status.success()) +/// the interpreter to run the transpiled output on: the oldest found that runs what the +/// transpiler targets by default, 3.10 — a dataclass's `KW_ONLY` among it. none is a skip, said so +fn python() -> Option { + interpreters::oldest(Config::default().min_version, "", "") } /// transpile `sources` (`module stem` → basedpython source) into a fresh @@ -299,8 +276,8 @@ fn build_case(case: &str, sources: &[(&str, &str)]) -> PathBuf { } /// run `main.py` in `dir`, asserting it exits cleanly and prints `ok` -fn run_main(python: &str, dir: &Path) { - let output = Command::new(python) +fn run_main(python: &Interpreter, dir: &Path) { + let output = Command::new(&python.command) .arg("main.py") .current_dir(dir) .output() @@ -313,12 +290,12 @@ fn run_main(python: &str, dir: &Path) { String::from_utf8_lossy(&output.stderr), ); assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); + interpreters::ran(python, Config::default().min_version); } #[test] fn proxy_forwards_value_operations() { let Some(python) = python() else { - eprintln!("skipping lazy-import runtime test: no python interpreter found"); return; }; let dir = build_case( @@ -331,7 +308,6 @@ fn proxy_forwards_value_operations() { #[test] fn character_identity_survives_an_import() { let Some(python) = python() else { - eprintln!("skipping lazy-import runtime test: no python interpreter found"); return; }; let dir = build_case( @@ -344,7 +320,6 @@ fn character_identity_survives_an_import() { #[test] fn import_stays_lazy_until_first_use() { let Some(python) = python() else { - eprintln!("skipping lazy-import runtime test: no python interpreter found"); return; }; let dir = build_case( @@ -357,7 +332,6 @@ fn import_stays_lazy_until_first_use() { #[test] fn a_name_is_claimed_only_once_the_module_is_lazy() { let Some(python) = python() else { - eprintln!("skipping lazy-import runtime test: no python interpreter found"); return; }; let dir = build_case( @@ -370,7 +344,6 @@ fn a_name_is_claimed_only_once_the_module_is_lazy() { #[test] fn an_unlazifiable_name_in_a_multi_name_import_stays_bound() { let Some(python) = python() else { - eprintln!("skipping lazy-import runtime test: no python interpreter found"); return; }; let dir = build_case("lazy_mixed", &[("main", MIXED_MAIN)]); @@ -380,7 +353,6 @@ fn an_unlazifiable_name_in_a_multi_name_import_stays_bound() { #[test] fn a_submodule_resolves_through_the_proxy() { let Some(python) = python() else { - eprintln!("skipping lazy-import runtime test: no python interpreter found"); return; }; let dir = build_case("lazy_submodule", &[("main", SUBMODULE_MAIN)]); @@ -390,13 +362,8 @@ fn a_submodule_resolves_through_the_proxy() { #[test] fn an_annotation_reads_the_imported_object() { let Some(python) = python() else { - eprintln!("skipping lazy-import runtime test: no python interpreter found"); return; }; - if !runs_the_default_target(&python) { - eprintln!("skipping lazy-import runtime test: {python} is older than python 3.10"); - return; - } let dir = build_case("lazy_annotations", &[("main", ANNOTATIONS_MAIN)]); run_main(&python, &dir); } @@ -404,13 +371,8 @@ fn an_annotation_reads_the_imported_object() { #[test] fn a_string_annotation_resolves_to_the_imported_object() { let Some(python) = python() else { - eprintln!("skipping lazy-import runtime test: no python interpreter found"); return; }; - if !runs_the_default_target(&python) { - eprintln!("skipping lazy-import runtime test: {python} is older than python 3.10"); - return; - } let dir = build_case( "lazy_future_annotations", &[("main", FUTURE_ANNOTATIONS_MAIN)], diff --git a/crates/by_transforms/tests/polyfill_runtime.rs b/crates/by_transforms/tests/polyfill_runtime.rs index 3283944f25..349a41cdcd 100644 --- a/crates/by_transforms/tests/polyfill_runtime.rs +++ b/crates/by_transforms/tests/polyfill_runtime.rs @@ -16,10 +16,13 @@ //! Asserting on the lowered *text* is what let all three through, so these //! tests execute it instead. -use std::process::{Command, Stdio}; +use std::process::Command; use by_transforms::{Config, PythonVersion, transpile}; +mod interpreters; +use interpreters::Interpreter; + /// exercises the polyfill's typevar rename across every position that renders /// replacement text rather than patching source bytes const RENAMES: &str = r#" @@ -171,6 +174,9 @@ assert hints["b"] == typing.Union[int, str], "a keyword union the lowering spell assert isinstance(None, int?) and not isinstance("a", int?), "an optional as a classinfo" assert isinstance(None, int? | str) and isinstance(None, (bytes, float?)), "nested in a classinfo" assert isinstance("a", (int | str)?) and issubclass(bool, int?), "around a union, and in issubclass" +assert isinstance(1, (bytes, (float, int | str))), "in a tuple nested in the classinfo" +from builtins import isinstance as is_instance +assert is_instance(1, int | str), "an isinstance reached by another name" print("ok") "#; @@ -277,93 +283,152 @@ assert call(1) == 1, "a defaulted function" print("ok") "#; -/// an interpreter found under one of the names the tests look for -struct Interpreter { - command: String, - version: PythonVersion, - typing_extensions: bool, -} +/// a method's type variable, and a nested function's, is declared at module scope, where +/// `typing.get_type_hints` resolves the annotations that name it: in the function's module +/// namespace, which a class body or an enclosing function's locals are not part of +const METHOD_TYPE_VARIABLES: &str = r#" +import typing -impl std::fmt::Display for Interpreter { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{} ({})", self.command, self.version)?; - if self.typing_extensions { - write!(f, " with `typing_extensions`")?; - } - Ok(()) - } -} +class Box: + def put[T](self, x: T) -> T: + return x -/// what an interpreter says about itself: its version, whether it has `typing_extensions`, and -/// the file it runs from, which tells two names for one interpreter apart -const PROBE: &str = "import sys, importlib.util; \ -print(sys.version_info[0], sys.version_info[1], \ -int(importlib.util.find_spec('typing_extensions') is not None), sys.executable)"; - -/// every interpreter this machine has under `$PYTHON`, `python3.N` or `python3`, oldest first, -/// each listed once however many of the names reach it -/// -/// the polyfill is only told apart from native syntax by an old interpreter, and its output -/// imports `typing_extensions` below 3.13, which an interpreter often does not have. `python3` -/// alone is whichever one the path finds first, so every name is asked -fn interpreters() -> Vec { - let mut candidates: Vec = std::env::var("PYTHON").into_iter().collect(); - candidates.extend((8..=15).map(|minor| format!("python3.{minor}"))); - candidates.push("python3".to_owned()); - - let mut seen = std::collections::HashSet::new(); - let mut found: Vec = candidates - .into_iter() - .filter_map(|command| { - let output = Command::new(&command) - .args(["-c", PROBE]) - .stderr(Stdio::null()) - .output() - .ok() - .filter(|output| output.status.success())?; - let stdout = String::from_utf8(output.stdout).ok()?; - let mut fields = stdout.trim_end().splitn(4, ' '); - let major = fields.next()?.parse().ok()?; - let minor = fields.next()?.parse().ok()?; - let typing_extensions = fields.next()? == "1"; - let executable = fields.next()?; - let executable = std::fs::canonicalize(executable) - .unwrap_or_else(|_| std::path::PathBuf::from(executable)); - seen.insert(executable).then(|| Interpreter { - command, - version: PythonVersion::from((major, minor)), - typing_extensions, - }) - }) - .collect(); - found.sort_by_key(|interpreter| interpreter.version); - found -} +def outer(): + def inner[U](y: U) -> U: + return y + return inner + +hints = typing.get_type_hints(Box.put) +assert hints["x"] === hints["return"], "a method's annotations resolve" +nested = typing.get_type_hints(outer()) +assert nested["y"] === nested["return"], "and a nested function's" +assert Box().put(1) == 1 +print("ok") +"#; + +/// a method whose bound reads a name of its class. the declaration reads it through the class, +/// so it is at module scope with the others: `typing.get_type_hints` finds it, and so does a +/// function nested in the method, whose annotations are evaluated when the method runs on a +/// target that does not defer them +const CLASS_LEVEL_BOUNDS: &str = r#" +import typing + +class C: + class Inner: + pass + + def m[T: Inner, U: T](self, x: T, y: U) -> T: + def nested[V](v: V, t: T) -> V: + return v + return nested(x, y) + +hints = typing.get_type_hints(C.m) +assert hints["x"] === hints["return"] and hints["y"].__bound__ === hints["x"] +assert typing.get_type_hints(C.m)["x"].__bound__.__forward_arg__ == "C.Inner" +inner = C.Inner() +assert C().m(inner, inner) === inner +print("ok") +"#; + +/// a type moved out of its generic to module scope — an inline protocol, an anonymous named +/// tuple — names the type variable the polyfill declared for the generic's parameter. an earlier +/// generic declaring a parameter of the same name differently takes the first name, so a guess +/// at it reads that one instead. where the type parameters stay native syntax, the parameter +/// exists only in its generic's scope, so the hoisted type reads it off the generic +const HOISTED_TYPES: &str = r#" +import typing + +class Bounded[T: int]: + pass + +class Holder[T]: + def takes(self, p: protocol(a: T)) -> None: + pass + + def gives(self, t: T) -> (x: T, y: int): + return (t, 1) + +parameter = getattr(Holder, "__parameters__")[0] +hoisted = typing.get_type_hints(Holder.takes)["p"] +assert typing.get_type_hints(hoisted)["a"] === parameter, "an inline protocol names its generic's" +named = typing.get_type_hints(Holder.gives)["return"] +assert typing.get_type_hints(named)["x"] === parameter, "and so does an anonymous named tuple" +print("ok") +"#; + +/// a module that binds, as values of its own, the names of the typing constructs the lowerings +/// write. each lowering reads the construct it means under a name the module does not spell, +/// and the module keeps its own +const OWN_TYPING_NAMES: &str = r#" +import typing + +Union = Callable = Literal = Any = Protocol = NamedTuple = "mine" +cast = overload = Final = ClassVar = final = override = "mine" +abstractmethod = Annotated = TypeIs = NewType = TypeVar = Generic = "mine" + +class Base: + def name(self) -> str: + return "base" + +final class Derived(Base): + override def name(self) -> str: + return "derived" + +class Counter: + class var count: int = 0 + +protocol Named: + def name(self) -> str: ... + +let limit: int = 3 + +def is_int(x: object) -> x is int: + return type(x) === int + +def first[T](xs: list[T]) -> T: + return xs[0] + +def call(f: (int) -> str) -> str: + return f(1) + +def pick(x: "a" | "b") -> dynamic: + return x + +def opt(x: int?) -> list[int?]: + return [x] + +def shape(p: protocol(a: int)) -> (x: int, y: str): + return (p.a, "s") + +newtype UserId = int + +class HasA: + a: int = 3 + +assert Derived().name() == "derived" and Counter.count == 0 and limit == 3 +assert first([1]) == 1 and call(str) == "1" and pick("a") == "a" and opt(None) == [None] +assert shape(HasA()).x == 3 and UserId(3) == 3 and is_int(1) and (5 cast! int) == 5 +assert typing.get_type_hints(opt)["x"] == typing.Optional[int], "a union the lowering spelled" +assert typing.get_type_hints(pick)["x"] == typing.Literal["a", "b"], "a literal type" +assert typing.get_type_hints(call)["f"] == typing.Callable[[int], str], "an arrow" +assert Union == Callable == Literal == Any == Protocol == NamedTuple == "mine" +assert cast == overload == Final == ClassVar == final == override == "mine" +assert abstractmethod == Annotated == TypeIs == NewType == TypeVar == Generic == "mine" +print("ok") +"#; /// run `program`, transpiled for `target`, on the oldest interpreter found that runs what it is /// transpiled for, and say which one ran it. `extensions` asks for one with /// `typing_extensions`. none is a skip, said so -/// -/// a skip passes, so nothing on the test's line says it ran nothing: the nextest configuration -/// shows what this binary's passing tests print -#[expect( - clippy::print_stderr, - reason = "a skipped test must say why it skipped, or it reads as a pass" -)] fn run_on_oldest(program: &str, target: PythonVersion, extensions: bool) { - let found = interpreters(); - let Some(interpreter) = found.iter().find(|interpreter| { - interpreter.version >= target && (interpreter.typing_extensions || !extensions) - }) else { - let needs = if extensions { - " with `typing_extensions`" - } else { - "" - }; - eprintln!("skipping: no interpreter of python {target} or later{needs} found"); - return; + let (probe, needs) = if extensions { + ("import typing_extensions", "with `typing_extensions`") + } else { + ("", "") }; - run_at(interpreter, program, target); + if let Some(interpreter) = interpreters::oldest(target, probe, needs) { + run_at(&interpreter, program, target); + } } /// the target the polyfill lowers for on `interpreter`: 3.10, the one a user gets without @@ -374,28 +439,16 @@ fn polyfill_target(interpreter: &Interpreter) -> PythonVersion { /// run `program`, lowered by the polyfill, on the oldest interpreter found with /// `typing_extensions`, which the polyfill's output imports below 3.13 -#[expect( - clippy::print_stderr, - reason = "a skipped test must say why it skipped, or it reads as a pass" -)] fn run_polyfilled_with_extensions(program: &str) { - let found = interpreters(); - let Some(interpreter) = found.iter().find(|interpreter| { - interpreter.typing_extensions && interpreter.version >= PythonVersion::PY39 - }) else { - eprintln!( - "skipping: no interpreter of python 3.9 or later with `typing_extensions` found, \ - so the polyfill's output was not run" - ); - return; - }; - run_at(interpreter, program, polyfill_target(interpreter)); + if let Some(interpreter) = interpreters::oldest( + PythonVersion::PY39, + "import typing_extensions", + "with `typing_extensions`", + ) { + run_at(&interpreter, program, polyfill_target(&interpreter)); + } } -#[expect( - clippy::print_stderr, - reason = "a test that can skip says what it ran on, so a pass is told apart from a skip" -)] fn run_at(interpreter: &Interpreter, program: &str, min_version: PythonVersion) { let config = Config { min_version, @@ -415,7 +468,7 @@ fn run_at(interpreter: &Interpreter, program: &str, min_version: PythonVersion) String::from_utf8_lossy(&output.stderr), ); assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); - eprintln!("ran, transpiled for {min_version}, on {interpreter}"); + interpreters::ran(interpreter, min_version); } #[test] @@ -447,7 +500,7 @@ fn some_parameters_run_at_every_version() { reason = "a skipped test must say why it skipped, or it reads as a pass" )] fn evaluated_unions_run_below_python_310() { - let found = interpreters(); + let found = interpreters::interpreters(); let older: Vec<&Interpreter> = found .iter() .filter(|interpreter| { @@ -481,7 +534,7 @@ fn evaluated_unions_run_below_python_310() { reason = "a skipped test must say why it skipped, or it reads as a pass" )] fn a_module_binding_the_constructors_runs() { - let found = interpreters(); + let found = interpreters::interpreters(); let older: Vec<&Interpreter> = found .iter() .filter(|interpreter| { @@ -507,3 +560,28 @@ fn defaulted_parameter_specifications_and_variadics_run() { run_polyfilled_with_extensions(DEFAULTED_PACKS); run_on_oldest(DEFAULTED_PACKS, PythonVersion::PY313, false); } + +#[test] +fn a_hoisted_type_names_its_generics_type_variable() { + run_on_oldest(HOISTED_TYPES, PythonVersion::PY39, false); + run_on_oldest(HOISTED_TYPES, PythonVersion::PY313, false); +} + +#[test] +fn a_module_binding_the_typing_names_runs() { + run_on_oldest(OWN_TYPING_NAMES, PythonVersion::PY39, true); + run_on_oldest(OWN_TYPING_NAMES, PythonVersion::PY313, false); +} + +#[test] +fn a_method_type_variable_resolves_in_type_hints() { + run_on_oldest(METHOD_TYPE_VARIABLES, PythonVersion::PY39, false); +} + +/// 3.10 and 3.11 evaluate annotations when the function is defined, which a nested function +/// is each time the method runs +#[test] +fn a_bound_reading_a_class_level_name_resolves() { + run_on_oldest(CLASS_LEVEL_BOUNDS, PythonVersion::PY39, false); + run_on_oldest(CLASS_LEVEL_BOUNDS, PythonVersion::PY310, false); +} diff --git a/crates/by_transforms/tests/tstring_expression_runtime.rs b/crates/by_transforms/tests/tstring_expression_runtime.rs index 6baa95bc30..f3051a3043 100644 --- a/crates/by_transforms/tests/tstring_expression_runtime.rs +++ b/crates/by_transforms/tests/tstring_expression_runtime.rs @@ -18,6 +18,9 @@ use std::process::Command; use by_transforms::{Config, PythonVersion, transpile}; +mod interpreters; +use interpreters::Interpreter; + /// basedpython whose t-string fields hold expressions that lower const PROGRAM: &str = r#" from string.templatelib import Template @@ -82,28 +85,8 @@ show(3, 7, {"k": 5}) print("ok") "#; -/// An interpreter that has t-strings. `$PYTHON` is taken only when it is one, because -/// the suite's usual interpreter is older and would skip every assertion here silently. -fn python_314() -> Option { - let mut candidates = Vec::new(); - if let Ok(python) = std::env::var("PYTHON") { - candidates.push(python); - } - candidates.extend(["python3.14", "python3"].map(String::from)); - candidates.into_iter().find(|python| { - Command::new(python) - .args([ - "-c", - "import sys; sys.exit(0 if sys.version_info >= (3, 14) else 1)", - ]) - .status() - .map(|status| status.success()) - .unwrap_or(false) - }) -} - -fn run(python: &str, source: &str, what: &str) { - let output = Command::new(python) +fn run(python: &Interpreter, source: &str, what: &str) { + let output = Command::new(&python.command) .arg("-c") .arg(source) .output() @@ -117,13 +100,9 @@ fn run(python: &str, source: &str, what: &str) { } #[test] -#[expect( - clippy::print_stderr, - reason = "a skipped test must say why it skipped, or it reads as a pass" -)] fn a_field_reports_the_source_the_author_wrote() { - let Some(python) = python_314() else { - eprintln!("skipping t-string `expression` runtime test: needs python 3.14 or later"); + // an interpreter that has t-strings + let Some(python) = interpreters::oldest(PythonVersion::PY314, "", "") else { return; }; @@ -135,4 +114,5 @@ fn a_field_reports_the_source_the_author_wrote() { }; let transpiled = transpile(PROGRAM, &config).expect("transpile should succeed"); run(&python, &transpiled, "the transpiled program"); + interpreters::ran(&python, PythonVersion::PY314); } diff --git a/crates/by_transforms/tests/visibility_runtime.rs b/crates/by_transforms/tests/visibility_runtime.rs index 8fbb651f8d..91699ac668 100644 --- a/crates/by_transforms/tests/visibility_runtime.rs +++ b/crates/by_transforms/tests/visibility_runtime.rs @@ -20,6 +20,8 @@ use by_transforms::{Config, PythonVersion, transpile}; mod common; use common::python; +mod interpreters; + /// Every visibility shape that reaches `__all__`, in both keyword orders. const PROGRAM: &str = r#" export private def helper() -> int: @@ -380,3 +382,90 @@ fn a_renamed_name_is_read_back_wherever_python_looks_it_up() { ); assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); } + +/// module-level `private` symbols whose underscored name the module already has: a runtime +/// helper's (`_force_unwrap`, which `x!` calls), the author's own (`_shadowed`, `_theme`), and +/// ones a lowering takes for itself (the `_T` a polyfilled type parameter is declared under, the +/// `_MISSING` a mutable default is replaced by). each renamed symbol has to go somewhere else, +/// and every place that names it has to follow +const TAKEN_NAMES: &str = r#" +_shadowed = 7 +_theme = "light" + +private def force_unwrap(x: int) -> int: + return x + 100 + +private def shadowed() -> int: + return 1 + +private type T = int + +def ident[T](x: T) -> T: + return x + +def holder(v: T): + pass + +def count(xs: list[int] = []) -> int: + return len(xs) + +private let MISSING: int = 5 + +private context let theme = "dark" + +def paint(context theme: str) -> str: + return theme + +def first(x: int | None) -> int: + return x! + +def calls() -> tuple[int, int, int, int, int, str]: + return (first(1), force_unwrap(1), shadowed(), count(), MISSING, paint()) +"#; + +const TAKEN_NAMES_IMPORTER: &str = r#" +import importlib, typing +m = importlib.import_module("emitted") +assert m.calls() == (1, 101, 1, 0, 5, "dark"), m.calls() +assert m._shadowed == 7 and m._theme == "light", "the author's own names keep their values" +assert typing.get_type_hints(m.holder)["v"].__value__ is int, "`T` is still the private alias" +assert m.ident(2) == 2 +print("ok") +"#; + +#[test] +fn a_renamed_symbol_never_takes_a_name_the_module_already_has() { + let Some(interpreter) = interpreters::oldest( + PythonVersion::PY39, + "import typing_extensions", + "with `typing_extensions`", + ) else { + return; + }; + // the oldest target the interpreter runs, where the type parameter is polyfilled + let target = interpreter.version.min(PythonVersion::PY311); + let config = Config { + min_version: target, + ..Config::default() + }; + let transpiled = transpile(TAKEN_NAMES, &config).expect("transpile should succeed"); + + let dir = tempfile::tempdir().expect("failed to create a temp dir"); + std::fs::write(dir.path().join("emitted.py"), &transpiled).expect("failed to write the module"); + + let output = Command::new(&interpreter.command) + .arg("-c") + .arg(TAKEN_NAMES_IMPORTER) + .current_dir(dir.path()) + .output() + .expect("failed to spawn python"); + + assert!( + output.status.success(), + "importing the transpiled module failed on {interpreter}:\n--- stdout ---\n{}\n--- stderr ---\n{}\n--- transpiled ---\n{transpiled}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); + interpreters::ran(&interpreter, target); +} diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_keyword_variadic.md b/crates/ty_python_semantic/resources/mdtest/basedpython_keyword_variadic.md index 8324b7d55a..3e8d3d6eb4 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_keyword_variadic.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_keyword_variadic.md @@ -339,6 +339,67 @@ info: - as part of a type parameter list when defining a generic class info: - or as part of an argument list when specializing a generic class ``` +## a pack defaults to another pack + +a pack's default may name an earlier pack of its list, whose fields it takes when nothing +specializes it — as a `ParamSpec`'s default may name an earlier `ParamSpec`. python sees both packs +as `ParamSpec`s + +```toml +[environment] +python-version = "3.13" +``` + +```by +class A[**P, **Q = P]: + def second(self) -> (**Q) -> None: + raise NotImplementedError + +def f(a: A[foo=int]): + reveal_type(a.second()) # revealed: (*, foo: int) -> None +``` + +## a pack's default cannot spell fields + +a pack's fields are named, and none of the forms a `ParamSpec`'s default may spell — a list of +types, `...` — names them. nor does the empty tuple, so a pack's default can only name another pack + +```toml +[environment] +python-version = "3.13" +``` + +```by +# error: [invalid-type-variable-default] "The default of a keyword-variadic pack must name another pack" +class A[**P = [int]]: ... + +# error: [invalid-type-variable-default] "The default of a keyword-variadic pack must name another pack" +class B[**P = ...]: ... + +# error: [invalid-type-variable-default] "The default of a keyword-variadic pack must name another pack" +class C[**P = ()]: ... +``` + +## a class with two packs + +every keyword argument names a field of a class's first pack, so no specialization spells a second +one by keyword. each pack of such a class reads back as the parameter list it unpacks to, which +keeps the two apart + +```toml +[environment] +python-version = "3.13" +``` + +```by +class A[**P, **Q = P]: + def second(self) -> (**Q) -> None: + raise NotImplementedError + +def f(a: A[foo=int]): + reveal_type(a) # revealed: A[(*, foo: int), (*, foo: int)] +``` + ## `**kwargs: T` in an arrow is still a catch-all an annotated `**name: T` is an ordinary keyword-variadic parameter, not a pack unpacking diff --git a/crates/ty_python_semantic/resources/mdtest/narrow/isinstance.md b/crates/ty_python_semantic/resources/mdtest/narrow/isinstance.md index bc3325ae81..72f7084a93 100644 --- a/crates/ty_python_semantic/resources/mdtest/narrow/isinstance.md +++ b/crates/ty_python_semantic/resources/mdtest/narrow/isinstance.md @@ -326,6 +326,29 @@ def _(x: object, y: type): isinstance(x, (bytes, IntOrStr)) ``` +`isinstance()` also accepts a tuple nested inside the tuple of classes, and a union inside one is +written as a tuple too, however deep it stands. So is an optional around a union, and an optional +among a union's arms: + +```by +def _(x: object): + isinstance(x, (bytes, (float, int | str))) + isinstance(x, (int | str)?) + isinstance(x, int | str?) + # error: [invalid-argument-type] "A union cannot be used with `isinstance()` before Python 3.10" + isinstance(x, (bytes, (float, IntOrStr))) +``` + +What decides is the function called, not the name it is called by: `isinstance()` reached through +another name is written the same way: + +```by +from builtins import isinstance as is_instance + +def _(x: object): + is_instance(x, int | str) +``` + ## `Optional` as `classinfo` ```py diff --git a/crates/ty_python_semantic/src/types.rs b/crates/ty_python_semantic/src/types.rs index c72682cec6..003b005187 100644 --- a/crates/ty_python_semantic/src/types.rs +++ b/crates/ty_python_semantic/src/types.rs @@ -180,6 +180,7 @@ mod callable; pub mod character; mod class; mod class_base; +pub mod classinfo_spelling; pub(crate) mod composition; pub(crate) mod conformance; mod constraints; diff --git a/crates/ty_python_semantic/src/types/classinfo_spelling.rs b/crates/ty_python_semantic/src/types/classinfo_spelling.rs new file mode 100644 index 0000000000..f0be31f949 --- /dev/null +++ b/crates/ty_python_semantic/src/types/classinfo_spelling.rs @@ -0,0 +1,88 @@ +//! basedpython: where a union is written as the tuple of classes it stands for +//! +//! below 3.10 a union object is no class, and `isinstance` rejects one. the lowering writes a +//! union spelled in a call to the builtin `isinstance` or `issubclass` as the tuple it stands +//! for, so `isinstance(x, int | str)` runs as `isinstance(x, (int, str,))` — and the same for +//! an optional `int?`, and for either inside a tuple the call spells. a union reached any other +//! way, through a name for one, is the `typing.Union` the lowering writes everywhere else +//! +//! which calls and which expressions in them are read as classes is decided here once, for the +//! transpiler's rewrite and for ty's report of a union `isinstance` would reject alike. the call +//! is one whose callee is the builtin, whatever the name it is reached by: ty knows the callee +//! as the function it checks the call against, and the transpiler asks +//! [`ide_support::classinfo_argument`](crate::types::ide_support::classinfo_argument) + +use ruff_python_ast as ast; +use ruff_text_size::Ranged; + +/// the argument `isinstance` and `issubclass` read as classes, when `call` is a call to one of +/// them +pub(crate) fn classinfo_argument(call: &ast::ExprCall) -> Option<&ast::Expr> { + call.arguments.args.get(1) +} + +/// each expression the lowering reads as classes, `classinfo` among them: every element of a +/// tuple or list it spells, and every arm of a union and operand of an optional that +/// `is_union` says builds a union object. a `|` between two numbers is an ordinary operator, +/// and nothing inside one is read as classes. outermost first +pub fn class_positions<'a>( + classinfo: &'a ast::Expr, + is_union: &dyn Fn(&ast::Expr) -> bool, +) -> Vec<&'a ast::Expr> { + let mut positions = Vec::new(); + let mut pending = vec![classinfo]; + while let Some(expr) = pending.pop() { + positions.push(expr); + let inner: Vec<&ast::Expr> = match expr { + ast::Expr::Tuple(tuple) => tuple.elts.iter().collect(), + ast::Expr::List(list) => list.elts.iter().collect(), + ast::Expr::BinOp(binop) if binop.op == ast::Operator::BitOr && is_union(expr) => { + union_arms(expr) + } + ast::Expr::UnaryOp(optional) + if optional.op == ast::UnaryOp::Optional && is_union(expr) => + { + vec![optional.operand.as_ref()] + } + _ => Vec::new(), + }; + pending.extend(inner.into_iter().rev()); + } + positions +} + +/// the arms of `a | b | c`, which parses as `(a | b) | c` +pub fn union_arms(expr: &ast::Expr) -> Vec<&ast::Expr> { + let mut arms = Vec::new(); + let mut pending = vec![expr]; + while let Some(expr) = pending.pop() { + match expr { + ast::Expr::BinOp(binop) if binop.op == ast::Operator::BitOr => { + pending.push(&binop.right); + pending.push(&binop.left); + } + _ => arms.push(expr), + } + } + arms +} + +/// whether the lowering writes `expr`, a union object `call` — a call to the builtin +/// `isinstance` or `issubclass` — passes, as a tuple of classes: `expr` is a `|` or a `?` +/// standing where the call reads classes +/// +/// ty asks this of an expression whose type is already a union object, reached from the +/// argument through the tuples it spells, so a `|` or `?` on the way is one too +pub(crate) fn spelled_as_classes(call: &ast::ExprCall, expr: &ast::Expr) -> bool { + let builds_union = |expr: &ast::Expr| match expr { + ast::Expr::BinOp(binop) => binop.op == ast::Operator::BitOr, + ast::Expr::UnaryOp(optional) => optional.op == ast::UnaryOp::Optional, + _ => false, + }; + builds_union(expr) + && classinfo_argument(call).is_some_and(|classinfo| { + class_positions(classinfo, &builds_union) + .iter() + .any(|position| position.range() == expr.range()) + }) +} diff --git a/crates/ty_python_semantic/src/types/display.rs b/crates/ty_python_semantic/src/types/display.rs index 3e600a1ceb..2a9fd34a90 100644 --- a/crates/ty_python_semantic/src/types/display.rs +++ b/crates/ty_python_semantic/src/types/display.rs @@ -976,11 +976,17 @@ pub struct SourceSpelling { pub text: String, /// modules the spelling needs, each as an `import ` target pub modules: Vec, + /// the names of the classes [`Self::text`] writes bare, each read nowhere in it but as + /// that class + pub bare_classes: Vec, } /// Collects every class a type names, so each can be spelled for a particular file. struct ClassCollector<'a, 'db> { env: &'a ProgramEnvironment<'db>, + /// whether the display writes a symbolic operation as the type it reduces to, whose + /// classes are then the ones written + reduce_symbolic_operations: bool, visited_types: RefCell>>, classes: RefCell>>, } @@ -1013,6 +1019,11 @@ impl<'db> TypeVisitor<'db> for ClassCollector<'_, 'db> { return self.visit_type(db, Type::from(class)); } Type::TypeVar(_) => return, + Type::Deferred(deferred) + if self.reduce_symbolic_operations || !deferred.is_checked(db) => + { + return self.visit_type(db, deferred.reduced(db, self.env)); + } _ => {} } @@ -1045,6 +1056,7 @@ impl<'db> Type<'db> { ) -> Option { let collector = ClassCollector { env, + reduce_symbolic_operations: settings.reduce_symbolic_operations, visited_types: RefCell::default(), classes: RefCell::default(), }; @@ -1080,6 +1092,41 @@ impl<'db> Type<'db> { } } + let mut bare: Vec<&str> = classes + .iter() + .map(|class| class.name(db).as_str()) + .filter(|name| !qualified.contains_key(name)) + .collect(); + bare.sort_unstable(); + bare.dedup(); + + // the display writes other things bare too — a type variable, a `typing` special + // form — and one of them may share a class's name: a module's own `class Literal` + // beside the `Literal[1]` of a literal type. the text is written again with every + // bare class qualified, and a name that text still reads bare is not the class + // alone, so no spelling reads as this type + if !bare.is_empty() { + let mut every_class_qualified = qualified.clone(); + for name in &bare { + every_class_qualified.insert(name, QualificationLevel::ModuleName); + } + let text = self + .display_with( + db, + env, + DisplaySettings { + qualified: Rc::new(every_class_qualified), + ..settings.clone() + }, + ) + .to_string(); + let read = names_read_in(&text); + if bare.iter().any(|name| read.contains(*name)) { + return None; + } + } + let bare_classes = bare.into_iter().map(str::to_owned).collect(); + let settings = DisplaySettings { qualified: Rc::new(qualified), ..settings @@ -1087,10 +1134,30 @@ impl<'db> Type<'db> { Some(SourceSpelling { text: self.display_with(db, env, settings).to_string(), modules: modules.into_iter().collect(), + bare_classes, }) } } +/// the names `text`, a type expression, reads at its roots — `decimal` in `decimal.Decimal`, +/// `Literal` in `Literal[1]`. empty when it is no expression +fn names_read_in(text: &str) -> FxHashSet { + struct Roots(FxHashSet); + impl<'ast> ast::visitor::Visitor<'ast> for Roots { + fn visit_expr(&mut self, expr: &'ast ast::Expr) { + if let ast::Expr::Name(name) = expr { + self.0.insert(name.id.to_string()); + } + ast::visitor::walk_expr(self, expr); + } + } + let mut roots = Roots(FxHashSet::default()); + if let Ok(parsed) = ruff_python_parser::parse_expression(text) { + ast::visitor::Visitor::visit_expr(&mut roots, parsed.expr()); + } + roots.0 +} + /// Whether `name` reaches `class` from `file`'s module scope — a class the file defines or /// imports, or a builtin, all of which a bare name spells correctly there. fn file_binds_class<'db>( @@ -3140,6 +3207,10 @@ impl<'db> DisplaySpecialization<'_, 'db> { .variables(db) .collect::>(); let types = self.specialization.types(db); + let keyword_packs = variables + .iter() + .filter(|typevar| typevar.is_keyword_variadic(db)) + .count(); let mut wrote_any = false; for (typevar, ty) in variables.iter().zip(types) { if typevar.is_typevartuple(db) { @@ -3187,8 +3258,11 @@ impl<'db> DisplaySpecialization<'_, 'db> { } // basedpython: a keyword-variadic pack reads back the way it is written — - // `A[foo=int, bar=str]` — rather than as the parameter list that stores it + // `A[foo=int, bar=str]` — rather than as the parameter list that stores it. a class + // with two packs has no such spelling, since every keyword names a field of the + // first, so each reads back as its parameter list instead, which keeps them apart if typevar.is_keyword_variadic(self.db) + && keyword_packs == 1 && let Some(fields) = ty.keyword_pack_fields(self.db) { if fields.is_empty() { diff --git a/crates/ty_python_semantic/src/types/function.rs b/crates/ty_python_semantic/src/types/function.rs index 16d0e2cc35..a5badf47ac 100644 --- a/crates/ty_python_semantic/src/types/function.rs +++ b/crates/ty_python_semantic/src/types/function.rs @@ -3251,36 +3251,17 @@ fn check_classinfo_in_isinstance<'db>( } /// whether the union `classinfo_expr` evaluates to is spelled as a tuple of classes where it -/// stands, by the lowering a basedpython module goes through before it runs -/// -/// below 3.10 a union object is no class, and `isinstance` rejects one. the transpiler writes -/// a union spelled in the call itself as the tuple it stands for, so -/// `isinstance(x, int | str)` runs as `isinstance(x, (int, str,))`, and the same for an optional -/// `int?`, alone or inside a tuple the call spells. that is only done for a call to `isinstance` -/// or `issubclass` by those names, and a union reached any other way, through a name for one, -/// is the `typing.Union` the transpiler spells everywhere else +/// stands, by the lowering a basedpython module goes through before it runs. see +/// [`classinfo_spelling`](crate::types::classinfo_spelling) fn spelled_as_classes_by_the_lowering( context: &InferContext<'_, '_>, call_expression: &ast::ExprCall, classinfo_expr: Option<&ast::Expr>, ) -> bool { context.file().source_type(context.db()) == ast::PySourceType::BasedPython - && matches!( - call_expression.func.as_ref(), - ast::Expr::Name(name) if matches!(name.id.as_str(), "isinstance" | "issubclass") - ) - && matches!( - classinfo_expr, - Some( - ast::Expr::BinOp(ast::ExprBinOp { - op: ast::Operator::BitOr, - .. - }) | ast::Expr::UnaryOp(ast::ExprUnaryOp { - op: ast::UnaryOp::Optional, - .. - }) - ) - ) + && classinfo_expr.is_some_and(|classinfo_expr| { + crate::types::classinfo_spelling::spelled_as_classes(call_expression, classinfo_expr) + }) } /// report a union object passed to `isinstance()` / `issubclass()` on a python older than @@ -4508,7 +4489,7 @@ impl KnownFunction { call_expression, self, *second_argument, - call_expression.arguments.args.get(1), + crate::types::classinfo_spelling::classinfo_argument(call_expression), ); if self == KnownFunction::IsInstance { diff --git a/crates/ty_python_semantic/src/types/ide_support.rs b/crates/ty_python_semantic/src/types/ide_support.rs index 6d207fcd6a..6c5afb02f4 100644 --- a/crates/ty_python_semantic/src/types/ide_support.rs +++ b/crates/ty_python_semantic/src/types/ide_support.rs @@ -3241,6 +3241,22 @@ fn declared_parameter<'s, 'db>( .find(|candidate| candidate.source_parameter_index() == Some(position)) } +/// basedpython: the argument a call to the builtin `isinstance` or `issubclass` reads as +/// classes, when `call` is one — whatever the name the callee is reached by. see +/// [`classinfo_spelling`](crate::types::classinfo_spelling) +pub fn classinfo_argument<'a>( + model: &SemanticModel<'_>, + call: &'a ast::ExprCall, +) -> Option<&'a ast::Expr> { + let Type::FunctionLiteral(function) = call.func.inferred_type(model)? else { + return None; + }; + function + .known(model.db()) + .and_then(KnownFunction::into_classinfo_constraint_function)?; + crate::types::classinfo_spelling::classinfo_argument(call) +} + /// basedpython: how the lowering writes `function`'s parameters when they repeat `_` — the /// answer ty's signature of the definition is built from. `None` when they do not pub fn repeated_underscore_lowering( diff --git a/crates/ty_python_semantic/src/types/infer/builder.rs b/crates/ty_python_semantic/src/types/infer/builder.rs index 53a63f33bd..23a9b3e5d9 100644 --- a/crates/ty_python_semantic/src/types/infer/builder.rs +++ b/crates/ty_python_semantic/src/types/infer/builder.rs @@ -4927,7 +4927,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { // typevars are instead validated by `check_legacy_typevar_defaults` // (for functions) and `report_invalid_typevar_default_reference` // (for classes). - self.infer_paramspec_default(&default.value, None); + self.infer_paramspec_default(&default.value, None, TypeVarKind::LegacyParamSpec); } else { let default_ty = self.infer_type_expression(&default.value); let bound_or_constraints_node = arguments diff --git a/crates/ty_python_semantic/src/types/infer/builder/typevar.rs b/crates/ty_python_semantic/src/types/infer/builder/typevar.rs index 638503c04e..1651bf832c 100644 --- a/crates/ty_python_semantic/src/types/infer/builder/typevar.rs +++ b/crates/ty_python_semantic/src/types/infer/builder/typevar.rs @@ -953,7 +953,8 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { } } if let Some(default) = default.as_deref() { - self.infer_paramspec_default(default, Some(&name.id)); + let kind = TypeVarKind::double_starred_type_param(self.source_type()); + self.infer_paramspec_default(default, Some(&name.id), kind); } self.deferred_state = previous_deferred_state; } @@ -988,16 +989,19 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { bound_ty } + /// `kind` is what the parameter the default is for declares: a `ParamSpec`, or a basedpython + /// keyword-variadic pack pub(super) fn infer_paramspec_default( &mut self, default_expr: &ast::Expr, paramspec_name: Option<&str>, + kind: TypeVarKind, ) { let previously_allowed_paramspec = self .context .inference_flags .replace(InferenceFlags::ALLOW_PARAMSPEC_TYPE_EXPR, true); - self.infer_paramspec_default_impl(default_expr, paramspec_name); + self.infer_paramspec_default_impl(default_expr, paramspec_name, kind); self.context.inference_flags.set( InferenceFlags::ALLOW_PARAMSPEC_TYPE_EXPR, previously_allowed_paramspec, @@ -1008,14 +1012,19 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { &mut self, default_expr: &ast::Expr, paramspec_name: Option<&str>, + kind: TypeVarKind, ) { let db = self.db(); match default_expr { + // a `ParamSpec`'s default may spell a parameter list of its own. a keyword-variadic + // pack's may only name another pack: its fields are named, and neither form names them ast::Expr::EllipsisLiteral(ellipsis) => { let ty = self.infer_ellipsis_literal_expression(ellipsis); self.store_expression_type(default_expr, ty); - return; + if !kind.is_keyword_variadic() { + return; + } } ast::Expr::List(ast::ExprList { elts, .. }) => { let previously_allowed_paramspec = self @@ -1034,7 +1043,9 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { // use a heterogeneous tuple type to represent the list of types instead. let ty = Type::heterogeneous_tuple(db, self.program_environment(), types); self.store_expression_type(default_expr, ty); - return; + if !kind.is_keyword_variadic() { + return; + } } ast::Expr::Name(_) => { let ty = self.infer_type_expression(default_expr); @@ -1043,20 +1054,36 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { { return; } - let is_paramspec = match ty { + // another parameter of the same kind: a `ParamSpec` defaults to a `ParamSpec`, as + // PEP 696 has it, and a keyword-variadic pack to a pack. python sees both as a + // `ParamSpec`, and a pack's default is one there too + let same_kind = match ty { + Type::TypeVar(typevar) if kind.is_keyword_variadic() => { + typevar.is_keyword_variadic(db) + } Type::TypeVar(typevar) => typevar.is_paramspec(db), Type::KnownInstance(known_instance) => { - known_instance.class(db) == KnownClass::ParamSpec + !kind.is_keyword_variadic() + && known_instance.class(db) == KnownClass::ParamSpec } _ => false, }; - if is_paramspec { + if same_kind { return; } } _ => {} } - if let Some(builder) = self.context.report_lint(&INVALID_PARAMSPEC, default_expr) { + if kind.is_keyword_variadic() { + if let Some(builder) = self + .context + .report_lint(&INVALID_TYPE_VARIABLE_DEFAULT, default_expr) + { + builder.into_diagnostic( + "The default of a keyword-variadic pack must name another pack", + ); + } + } else if let Some(builder) = self.context.report_lint(&INVALID_PARAMSPEC, default_expr) { builder.into_diagnostic( "The default value to `ParamSpec` must be either \ a list of types, `ParamSpec`, or `...`", diff --git a/crates/ty_python_semantic/src/types/repeated_underscore.rs b/crates/ty_python_semantic/src/types/repeated_underscore.rs index 0b5847e8f2..8b13468ace 100644 --- a/crates/ty_python_semantic/src/types/repeated_underscore.rs +++ b/crates/ty_python_semantic/src/types/repeated_underscore.rs @@ -54,7 +54,13 @@ impl<'src> WrittenNames<'src> { /// `inner` a `decorator def` writes standing where an option of that name is declared, /// which the dispatcher then passed the dispatcher itself for pub fn fresh(self, stem: &str) -> String { - if !self.spells(stem) { + self.fresh_outside(stem, |_| false) + } + + /// [`Self::fresh`], also passing over every name `taken` answers for + pub fn fresh_outside(self, stem: &str, taken: impl Fn(&str) -> bool) -> String { + let free = |name: &str| !self.spells(name) && !taken(name); + if free(stem) { return stem.to_owned(); } // candidates only ever count up, so one never repeats an earlier one. a module @@ -62,7 +68,7 @@ impl<'src> WrittenNames<'src> { // of them is free long before the numbers run out (2u32..u32::MAX) .map(|number| format!("{stem}{number}")) - .find(|candidate| !self.spells(candidate)) + .find(|candidate| free(candidate)) .unwrap_or_else(|| stem.to_owned()) } diff --git a/docs/basedpython/development/how-transpilation-works.md b/docs/basedpython/development/how-transpilation-works.md index 3e639c6226..327a23f2ed 100644 --- a/docs/basedpython/development/how-transpilation-works.md +++ b/docs/basedpython/development/how-transpilation-works.md @@ -121,6 +121,46 @@ reformatting it rewrites transpiled output, and the isort rule would put a `from __future__ import annotations` at the top of every module that gets a helper pasted into it +## names the lowerings write + +a lowering writes names of its own into the module: a temporary, a `TypeVar` the +generics polyfill declares, and the typing constructs it spells a type with — +`Union`, `Callable`, `Literal`, `final`, `functools.partial` and the rest. each is +chosen through `WrittenNames` (`transforms/repeated_underscore.rs`), so the names +cannot collide with the module's own: + +- a binding the lowering writes (`fresh`) takes a name the module spells nowhere, + and none a runtime helper is bound under +- a typing name (`imported`, `import_from`) keeps its own spelling unless the + module binds that name to something else, and is imported under a fresh name + when it does. a module that binds it only by importing the very same thing — + `from typing import Literal` beside a lowered `1 | 2` — reads the lowering's + already. a star import binds names unseen, so a module with one gets a fresh + name for every typing name +- a builtin (`builtin`) is the same: `isinstance`, `staticmethod`, `type` keep + their names unless the module binds them anywhere — a parameter named `type` + counts — and are then imported from `builtins` under a fresh name, which the + driver binds once it sees the output read it +- a module-level `private` symbol is emitted under `_name` unless the module, a + runtime helper or another lowering already has that name, and then under a + fresh one (`PrivateNames`). every other fresh name stays clear of the ones it + chose + +```by +Union = "mine" +x = list[int?] +``` + +```python +# generated python, for 3.9 +from typing import Union as Union2 +Union = "mine" +x = list[Union2[int, None]] +``` + +a keyword the parser stands up as a name — the `final` of `final class` — binds +nothing, so a module that writes one still gets `@final` + ## the phase-0 database three pre-passes run before phase 0 and can rewrite the source: erased-union diff --git a/docs/basedpython/features/inline-protocol.md b/docs/basedpython/features/inline-protocol.md index 6ebc19b438..b91005eb06 100644 --- a/docs/basedpython/features/inline-protocol.md +++ b/docs/basedpython/features/inline-protocol.md @@ -123,4 +123,6 @@ contributes nothing to the generated class a member naming a type variable is rewritten to the mangled name the [generics](generics.md) polyfill gives it below python 3.12, since the hoisted -class sits outside the scope that declared it +class sits outside the scope that declared it. where the type parameters stay +native syntax, the member reads the parameter off its generic instead — +`"A.__type_params__[0]"` for the `T` of `class A[T]` diff --git a/docs/basedpython/features/keyword-variadic.md b/docs/basedpython/features/keyword-variadic.md index 053f1fb67c..7bdd16b16f 100644 --- a/docs/basedpython/features/keyword-variadic.md +++ b/docs/basedpython/features/keyword-variadic.md @@ -77,6 +77,23 @@ this is why a pack's context can't also use the [keyword-subscript](kw-subscript.md) form for binding typevars by name: with a pack in scope, `A[foo=int]` always names a field +## defaults + +a pack's default names an earlier pack of the same list, whose fields it takes when nothing +specializes it, as a `ParamSpec`'s default names an earlier `ParamSpec`: + +```by +class A[**P, **Q = P]: + def second(self) -> (**Q) -> None: ... + +def f(a: A[foo=int]): + reveal_type(a.second()) # (*, foo: int) -> None +``` + +a list of types, `...` or `()` is an error there: none of them names fields. every keyword names a +field of the first pack, so a class with two packs has no keyword spelling for its specialization, +and each pack reads back as the parameter list it unpacks to — `A[(*, foo: int), (*, foo: int)]` + ## scope the keyword-pack reading is confined to `.by`. `.byi` is the interop surface with python's typing @@ -147,7 +164,8 @@ before specialization the pack shows as a pending splice — `{"tag": int, **Kwa pack a variance there is what surfaces the adoption bug above, so the two have to be fixed together. spelling the pack in an [arrow](callable.md) as well restores invariance - a pack cannot yet be *forwarded* as a type argument (`A[**Kwargs]` inside another signature) -- a pack cannot have a default (`class D[**Kwargs = ()]`) +- a pack's default cannot spell fields of its own (`class D[**Kwargs = ()]`), only name another + pack - a generic context containing a pack skips literal promotion for its other type variables, so `class Two[T, **Kwargs]` infers `T` as `Literal[...]`. this is pre-existing ty behaviour, shared with `ParamSpec` diff --git a/docs/basedpython/features/modifiers.md b/docs/basedpython/features/modifiers.md index 84492ccf10..63f63b7ac7 100644 --- a/docs/basedpython/features/modifiers.md +++ b/docs/basedpython/features/modifiers.md @@ -46,6 +46,9 @@ ORIGIN: Final = Point(0, 0) | `protocol Foo` | `class Foo(Protocol)` (base added) | | `sealed class Foo` | `class Foo` + `Foo.__sealed_members__ = (...)` | +`dataclass(slots=True)` is new in python 3.10. below it, a `data class` is +decorated with a runtime helper that makes it slotted the way that option would + `abstract` is a marker for the type checker; it has no runtime decorator. `open` is the inverse of `final` — a marker that the class is intended to be subclassed. neither emits a runtime artefact @@ -200,9 +203,11 @@ member is not a module export a module-level `private` strips the keyword and gives the symbol a leading underscore at the definition site *and* every same-module call site. a name that already has one keeps it — a second would make it a `__name`, which python -name-mangles wherever a class body reads it. it is excluded from `__all__` even -when no `export`/`public` declarations exist, and another module that imports it -is reported by `private-import` +name-mangles wherever a class body reads it. when the module already has the +underscored name — its own `_helper`, or one the lowering binds — the symbol is +emitted as `_helper2` instead, so it never takes over a name the module has. it +is excluded from `__all__` even when no `export`/`public` declarations exist, and +another module that imports it is reported by `private-import` a module-level `private` works on a variable exactly as it does on a function, class or type alias — `private count: int = 0` is emitted as `_count`, and so is diff --git a/docs/basedpython/features/polyfills.md b/docs/basedpython/features/polyfills.md index 075df20a32..66ad8a5b65 100644 --- a/docs/basedpython/features/polyfills.md +++ b/docs/basedpython/features/polyfills.md @@ -126,7 +126,7 @@ _Ts = TypeVarTuple("_Ts", default=Unpack[tuple[int, str]]) # from typing_extens class Row(Generic[_T, Unpack[_Ts]]): ... ``` -a module that binds `TypeVar` itself — `from typing import TypeVar` binds the one without `default=` — keeps its own, and the polyfill's is imported under a name the module does not spell, `TypeVar2`. the same goes for every name the polyfill calls: `Generic`, `ParamSpec`, `TypeVarTuple`, `Unpack` and `TypeAliasType` +a module that binds `TypeVar` itself to something other than the polyfill's — `from typing import TypeVar` binds the one without `default=` — keeps its own, and the polyfill's is imported under a name the module does not spell, `TypeVar2`. the same goes for every name the polyfill calls, `Generic`, `ParamSpec`, `TypeVarTuple`, `Unpack` and `TypeAliasType`, and for every typing name any other lowering writes the `[T = int]` header syntax is itself 3.13+: on a 3.12 target, a declaration with a defaulted type parameter is desugared by the generics polyfill while declarations without defaults keep the native syntax. a [reified](reified-generics.md) function can't be desugared, so a defaulted reified function on a 3.12 target is a transpile error @@ -395,7 +395,7 @@ _T = TypeVar("_T", bound=Union[int, str]) def f(t: _T) -> _T: ... ``` -the two spellings are not interchangeable — `isinstance` takes a tuple of classes and rejects a `typing.Union` — so the classinfo argument of `isinstance` and `issubclass` becomes a tuple and everything else a `Union`. whether a `|` is a union at all is asked of the checker rather than guessed from the shape, so an ordinary bitwise or is left alone +the two spellings are not interchangeable — `isinstance` takes a tuple of classes and rejects a `typing.Union` — so the classinfo argument of `isinstance` and `issubclass`, under whatever name they are called, becomes a tuple and everything else a `Union`. whether a `|` is a union at all is asked of the checker rather than guessed from the shape, so an ordinary bitwise or is left alone a type the runtime evaluates is spelled this way at any depth: a type written as a value, a class base, a `cast` target, and a type parameter's bound, default or alias value, which the [generics polyfill](#generic-classes-and-functions-pep-695) turns into call arguments @@ -421,7 +421,7 @@ python 3.12 introduced compact generic syntax. basedpython rewrites it using `ty | `def f[T](x: T) -> T: ...` | `def f(x: T) -> T: ...` | | `type Point = tuple[float, float]` | `Point: TypeAlias = tuple[float, float]` | -each type parameter becomes a `TypeVar` with a mangled name (`_T`, `_K`, etc), declared just above the generic, in the same block. the name is never one the module spells: in a module that binds `_T` itself, `T` is declared as `_T2`, so the module's own `_T` is left alone. a parameter declared again with different arguments, or in a different block — a method's type parameter, declared in its class body, and a function's of the same name further down — gets a numbered name of its own, `_T_1` +each type parameter becomes a `TypeVar` with a mangled name (`_T`, `_K`, etc), declared at module scope, just above the generic or above the class or function at module scope it is written inside. a method's type parameter is declared above its class, so `typing.get_type_hints` finds it in the module, where it resolves the method's annotations. a bound or default that reads a name of the class the generic stands in reads it through the class, as a string — `T: Inner` in `class C` is declared `TypeVar("_T", bound="C.Inner")` — since the class does not exist yet where the declaration is written. one that reads a name of a function around the generic stays in that function, where the name is in scope. the name is never one the module spells: in a module that binds `_T` itself, `T` is declared as `_T2`, so the module's own `_T` is left alone. a parameter declared again with different arguments, or in a different block — inside an `if`, say, and a function's of the same name further down — gets a numbered name of its own, `_T_1` ```python # python source From aaff3282c3f8791d69a59f0fd704309bbe4c8f0b Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Thu, 24 Sep 2026 10:40:59 +1000 Subject: [PATCH 14/16] transpiler: report visibility collisions, type `?` after a type test, relate guards by the parameter they narrow, and quote annotations once a protected or private rename onto a name the class hierarchy stores is invalid-visibility; `x is int?` parses the marker as part of the type and `?` over a value types as the `| None` it evaluates; a return guard's target is a type position, `-> a is not T` is refused, and callable types may return a guard, related by which parameter each narrows. a forward reference is not quoted again under the future import, synthesized annotations import with the module's other imports, imported backing functions and conversion targets are bound under per-module names, synthesized classes are named once per module by shape, and a private class-body type alias is a private member. a symbolic fold inside a tuple or callable type is written by the pass that replaces it, and a fold may name a type parameter bound where it is written --- .config/nextest.toml | 2 +- crates/by_transforms/src/lib.rs | 387 ++++++++++-- crates/by_transforms/src/runtime.rs | 253 +++++++- .../by_transforms/src/runtime/global_reads.rs | 260 ++++++++ .../src/transforms/annotation.rs | 2 +- .../src/transforms/anon_named_tuple.rs | 81 ++- .../src/transforms/ast_driver.rs | 138 +++-- .../src/transforms/auto_quote.rs | 60 +- .../by_transforms/src/transforms/callable.rs | 114 +++- .../src/transforms/conformance.rs | 18 +- .../src/transforms/conversion.rs | 21 +- .../src/transforms/decorator_keyword.rs | 25 +- .../by_transforms/src/transforms/extension.rs | 81 ++- .../by_transforms/src/transforms/generics.rs | 85 ++- .../src/transforms/implicit_receiver.rs | 5 +- .../src/transforms/inferred_annotation.rs | 82 ++- .../src/transforms/lazy_import.rs | 28 +- .../src/transforms/main_function.rs | 98 ++- .../src/transforms/match_polyfill.rs | 53 +- .../by_transforms/src/transforms/modifiers.rs | 171 +++++- .../src/transforms/parametric_is.rs | 86 ++- .../src/transforms/properties.rs | 6 +- .../src/transforms/protocol_type.rs | 18 +- .../src/transforms/reified_generic.rs | 7 +- .../src/transforms/repeated_underscore.rs | 238 +++++++- .../src/transforms/source_util.rs | 53 +- .../src/transforms/string_tag.rs | 8 +- .../src/transforms/type_expr_walker.rs | 31 +- .../by_transforms/src/transforms/type_is.rs | 275 ++++++--- .../src/transforms/typed_dict_literal.rs | 14 +- .../src/transforms/typing_redirect.rs | 10 +- .../src/transforms/unique_loop_bindings.rs | 7 +- .../src/transforms/visibility_rename.rs | 79 ++- crates/by_transforms/src/type_info.rs | 72 ++- .../tests/builtin_names_runtime.rs | 45 +- .../by_transforms/tests/conversion_runtime.rs | 12 +- .../by_transforms/tests/data_class_runtime.rs | 39 +- .../tests/forward_reference_runtime.rs | 72 +++ .../imported_conversion_class_runtime.rs | 117 ++++ .../tests/imported_extension_runtime.rs | 147 +++++ .../tests/inferred_annotation_runtime.rs | 205 +++++++ .../tests/lazy_import_runtime.rs | 33 + .../fixtures/pyflakes/F821_basedpython.by | 8 + crates/ruff_linter/src/checkers/ast/mod.rs | 11 +- ...s__undefined-name_F821_basedpython.by.snap | 216 +++---- crates/ruff_python_ast/src/helpers.rs | 40 +- .../resources/test/fixtures/ruff/callable.by | 5 + .../test/fixtures/ruff/wrapped_postfix.by | 18 + .../src/expression/binary_like.rs | 38 +- .../tests/snapshots/format@callable.by.snap | 10 + .../snapshots/format@wrapped_postfix.by.snap | 44 ++ .../src/parser/expression.rs | 113 +++- crates/ruff_python_parser/src/parser/tests.rs | 102 ++++ crates/ty/docs/rules.md | 565 +++++++++++------- crates/ty_python_core/src/builder.rs | 28 +- .../mdtest/basedpython_identity_narrow.md | 38 ++ .../basedpython_implicit_object_repr.md | 23 + .../mdtest/basedpython_optional_type.md | 50 ++ .../mdtest/basedpython_type_guards.md | 233 ++++++++ .../mdtest/basedpython_visibility.md | 218 +++++++ .../resources/mdtest/repeated_underscore.md | 37 +- ...owing\342\200\246_(831b7fe0c360003b).snap" | 65 ++ crates/ty_python_semantic/src/lib.rs | 5 +- .../ty_python_semantic/src/semantic_model.rs | 32 +- crates/ty_python_semantic/src/types.rs | 5 +- .../src/types/class/static_literal.rs | 21 +- .../src/types/class_base.rs | 1 + .../src/types/conformance.rs | 31 +- .../src/types/conversions.rs | 90 ++- .../src/types/diagnostic.rs | 5 + .../ty_python_semantic/src/types/display.rs | 6 + .../src/types/extensions.rs | 174 +++++- .../ty_python_semantic/src/types/function.rs | 2 + .../src/types/infer/builder.rs | 118 +++- .../types/infer/builder/binary_expressions.rs | 6 +- .../types/infer/builder/type_expression.rs | 97 ++- .../src/types/known_instance.rs | 27 +- .../ty_python_semantic/src/types/overrides.rs | 181 ++++++ .../src/types/reified_infer.rs | 5 +- .../src/types/relation_error.rs | 8 + .../src/types/repeated_underscore.rs | 93 ++- .../src/types/signatures.rs | 195 +++++- .../src/types/visibility.rs | 86 ++- .../src/server/api/requests/entry_point.rs | 8 +- docs/basedpython/features/conversions.md | 7 +- docs/basedpython/features/extensions.md | 9 +- .../features/forward-references.md | 5 +- docs/basedpython/features/identity-swap.md | 7 + docs/basedpython/features/modifiers.md | 42 +- .../features/repeated-underscore.md | 10 +- docs/basedpython/features/type-is.md | 37 ++ docs/basedpython/features/wrapped-results.md | 33 +- ty.schema.json | 2 +- 93 files changed, 5842 insertions(+), 906 deletions(-) create mode 100644 crates/by_transforms/src/runtime/global_reads.rs create mode 100644 crates/by_transforms/tests/imported_conversion_class_runtime.rs create mode 100644 crates/by_transforms/tests/imported_extension_runtime.rs create mode 100644 crates/by_transforms/tests/inferred_annotation_runtime.rs create mode 100644 "crates/ty_python_semantic/resources/mdtest/snapshots/basedpython_type_gua\342\200\246_-_basedpython___narrowi\342\200\246_-_a_callable_narrowing\342\200\246_(831b7fe0c360003b).snap" diff --git a/.config/nextest.toml b/.config/nextest.toml index 0a9dd34bdc..223bee2865 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -13,7 +13,7 @@ test-group = 'serial' # on, or that it found none and ran nothing, and a passing test's output is # otherwise hidden, so a skip would read as a pass [[profile.default.overrides]] -filter = 'binary(polyfill_runtime) | binary(grapheme_runtime) | binary(conversion_runtime) | binary(lazy_import_runtime) | binary(fstring_debug_runtime) | binary(tstring_expression_runtime) | binary(visibility_runtime) | binary(data_class_runtime) | binary(builtin_names_runtime)' +filter = 'binary(polyfill_runtime) | binary(grapheme_runtime) | binary(conversion_runtime) | binary(lazy_import_runtime) | binary(fstring_debug_runtime) | binary(tstring_expression_runtime) | binary(visibility_runtime) | binary(data_class_runtime) | binary(builtin_names_runtime) | binary(inferred_annotation_runtime) | binary(forward_reference_runtime) | binary(imported_extension_runtime) | binary(imported_conversion_class_runtime)' success-output = "final" [profile.ci] diff --git a/crates/by_transforms/src/lib.rs b/crates/by_transforms/src/lib.rs index 37a9ef01da..99984318bf 100644 --- a/crates/by_transforms/src/lib.rs +++ b/crates/by_transforms/src/lib.rs @@ -16,11 +16,12 @@ pub use transforms::soundness::{SoundnessSites, soundness_sites}; /// guard and its argument parser are generated from. pub mod entry_point { pub use crate::transforms::main_function::{ - Choice, CliSpelling, EntryParameter, EntryPoint, ParameterKind, entry_point, main_guards, + Choice, CliSpelling, Converter, EntryParameter, EntryPoint, ParameterKind, entry_point, + main_guards, }; } -use transforms::repeated_underscore::{CodeNames, PrivateNames}; +use transforms::repeated_underscore::{CodeNames, PrivateNames, SynthesizedClasses}; use std::collections::{BTreeSet, HashSet}; @@ -34,15 +35,22 @@ use ruff_text_size::{Ranged, TextRange, TextSize}; use salsa::Setter as _; use ty_project::{ProjectMetadata, TestDb}; -/// Creates a single-file in-memory database for transpilation. +/// Creates a single-file in-memory database for transpilation, analysing at ty's newest +/// python. /// /// The source is registered at `/input.by`. pub(crate) fn make_in_memory_db(source: &str) -> (TestDb, File) { + make_in_memory_db_at(source, PythonVersion::latest_ty()) +} + +/// [`make_in_memory_db`], analysing the source at `python_version` +pub(crate) fn make_in_memory_db_at(source: &str, python_version: PythonVersion) -> (TestDb, File) { let mut db = TestDb::new(ProjectMetadata::new( ruff_python_ast::name::Name::new_static(""), SystemPathBuf::from("/"), )); - db.init_program().expect("program init failed"); + db.init_program_with_python_version(python_version) + .expect("program init failed"); db.write_file("/input.by", source) .expect("write file failed"); let file = system_path_to_file(&db, "/input.by").expect("file not in db"); @@ -141,15 +149,31 @@ fn run_erased_union_phase<'a>( /// Transpile `.by` source text to python without a project db (single-file: /// type-aware passes see only this file). Used for stdin input and tests; the /// file-backed [`transpile_typed`] resolves cross-module types. +/// +/// the module is analysed at ty's newest python, whatever `config` targets. a +/// project analyses at its own version, which can differ in what the analysis +/// finds — a builtin a newer python adds, say — so [`transpile_analysed_at`] +/// takes one pub fn transpile(source: &str, config: &Config) -> Result { - transpile_with_report(source, config).map(|(output, _)| output) + transpile_analysed_at(source, config, PythonVersion::latest_ty()) } -/// Like [`transpile`], and also reports what the emitted python needs installed -/// to run. +/// [`transpile`], analysing the module at `python_version`, as `by transpile` +/// analyses a project's modules at the project's version +pub fn transpile_analysed_at( + source: &str, + config: &Config, + python_version: PythonVersion, +) -> Result { + transpile_with_report(source, config, python_version).map(|(output, _)| output) +} + +/// Like [`transpile_analysed_at`], and also reports what the emitted python +/// needs installed to run. fn transpile_with_report( source: &str, config: &Config, + python_version: PythonVersion, ) -> Result<(String, RuntimeRequirements), String> { if config.is_python { return Ok((source.to_owned(), RuntimeRequirements::default())); @@ -158,7 +182,7 @@ fn transpile_with_report( // one db over the original source, shared by the qualification phase below // and — as long as nothing rewrites the source — by phase 0's type-aware // passes, which would otherwise build an identical one of their own - let (local_db, local_file) = make_in_memory_db(source); + let (local_db, local_file) = make_in_memory_db_at(source, python_version); let parsed = ruff_db::parsed::parsed_module( &local_db, ty_python_semantic::Db::program_file(&local_db, local_file).python_file(&local_db), @@ -182,7 +206,7 @@ fn transpile_with_report( // that rewrite edits signatures, so the qualification phase below needs a // db over what it is actually editing rather than the pre-rewrite source - let rebuilt = reified_changed.then(|| make_in_memory_db(source)); + let rebuilt = reified_changed.then(|| make_in_memory_db_at(source, python_version)); let (qualify_db, qualify_file) = match &rebuilt { Some((db, file)) => (db as &dyn ty_python_semantic::Db, *file), None => (&local_db as &dyn ty_python_semantic::Db, local_file), @@ -207,8 +231,12 @@ fn transpile_with_report( .iter() .map(ruff_python_ast::name::Name::as_str), &[as_written, source], + ty_python_semantic::extension_backing_functions(&local_db, local_file), ); - let written_names = written_names.with_private(&private_names); + let synthesized_classes = SynthesizedClasses::default(); + let written_names = written_names + .with_private(&private_names) + .with_synthesized(&synthesized_classes); // --- Phase 0: AST rewrite passes --- let unchanged = !reified_changed @@ -218,7 +246,11 @@ fn transpile_with_report( source, written_names, config, - unchanged.then_some((&local_db as &dyn ty_python_semantic::Db, local_file)), + if unchanged { + transforms::ast_driver::SemanticSource::Project(&local_db, local_file) + } else { + transforms::ast_driver::SemanticSource::Fresh(python_version) + }, ); if let Some(first) = ast_errors.first() { return Err(first.clone()); @@ -226,7 +258,7 @@ fn transpile_with_report( let source = source.as_ref(); // --- Phase 1: basedpython lowering --- - let (db, file) = make_in_memory_db(source); + let (db, file) = make_in_memory_db_at(source, python_version); let source_ref = ruff_db::source::source_text(&db, file); let src = source_ref.as_str(); let module = ruff_db::parsed::parsed_module( @@ -389,8 +421,10 @@ pub fn transpile_typed_with_report( let reified_changed = matches!(reified, std::borrow::Cow::Owned(_)); // that rewrite edits signatures, so qualification needs a db over what it - // is actually editing rather than the project file - let rebuilt = reified_changed.then(|| make_in_memory_db(reified.as_ref())); + // is actually editing rather than the project file. that db, and any other + // this module is analysed in on its own, analyses at the project's version + let project_version = db.python_version_with_source(file).version; + let rebuilt = reified_changed.then(|| make_in_memory_db_at(reified.as_ref(), project_version)); let (qualify_db, qualify_file) = match &rebuilt { Some((rebuilt_db, rebuilt_file)) => { (rebuilt_db as &dyn ty_python_semantic::Db, *rebuilt_file) @@ -418,8 +452,12 @@ pub fn transpile_typed_with_report( .iter() .map(ruff_python_ast::name::Name::as_str), &[original_source, working_source], + ty_python_semantic::extension_backing_functions(db, file), ); - let written_names = written_names.with_private(&private_names); + let synthesized_classes = SynthesizedClasses::default(); + let written_names = written_names + .with_private(&private_names) + .with_synthesized(&synthesized_classes); // two independent facts, deliberately kept apart: `enum_changed` says the // enum phase *renumbered lines*, so the final map must compose through its // line map (which is empty when it didn't fire, and would map every line to @@ -442,9 +480,11 @@ pub fn transpile_typed_with_report( None }; let project = match &overlaid { - Some(overlaid) => Some((&*overlaid.db, overlaid.file)), - None if source_changed => None, - None => Some((db, file)), + Some(overlaid) => { + transforms::ast_driver::SemanticSource::Project(&*overlaid.db, overlaid.file) + } + None if source_changed => transforms::ast_driver::SemanticSource::Fresh(project_version), + None => transforms::ast_driver::SemanticSource::Project(db, file), }; // which imports must stay eager, computed against the *project* db: a // single-file db cannot resolve the modules that declare the conformances @@ -630,7 +670,10 @@ fn run_anon_named_tuple_cleanup( &written.import_from("typing", &["NamedTuple"]), ); let (imports, helpers) = anon.callable.take_requirements(); - for line in imports.into_iter().chain(runtime_entries(config, &helpers)) { + for line in imports + .into_iter() + .chain(runtime_entries(config, &helpers, &source)) + { push_missing(&mut preamble, line.trim_end_matches('\n')); } for defs in [anon.callable.class_defs().to_owned(), anon.class_defs()] { @@ -788,7 +831,7 @@ fn run_lazy_import_phase( None => body, }; } - let preamble = runtime_preamble(config, &helpers); + let preamble = runtime_preamble(config, &helpers, &body); if preamble.is_empty() { body } else { @@ -796,29 +839,40 @@ fn run_lazy_import_phase( } } -/// The lines that give a module the runtime helpers it calls: an import of the -/// module a build wrote them to, or the definitions themselves when this +/// The lines that give `module`, python, the runtime helpers it calls: an import of +/// the module a build wrote them to, or the definitions themselves when this /// transpile has nowhere to write one. /// /// Both come out of `_by_runtime.py`, so the pasted-in form and the imported one /// are the same code either way. -fn runtime_preamble(config: &Config, helpers: &[runtime::Helper]) -> String { - runtime_entries(config, helpers).concat() +fn runtime_preamble(config: &Config, helpers: &[runtime::Helper], module: &str) -> String { + runtime_entries(config, helpers, module).concat() } /// The same lines, one per entry and each newline-terminated, for a caller that -/// prepends them one at a time. -fn runtime_entries(config: &Config, helpers: &[runtime::Helper]) -> Vec { +/// prepends them one at a time. Pasted in, the definitions read a builtin `module` +/// binds for itself under a name of their own, imported ahead of them. +fn runtime_entries(config: &Config, helpers: &[runtime::Helper], module: &str) -> Vec { if helpers.is_empty() { return Vec::new(); } - match config.runtime_module.as_deref() { - Some(module) => vec![format!( + if let Some(runtime_module) = config.runtime_module.as_deref() { + return vec![format!( "{}\n", - runtime::import_line(module, helpers.iter().copied()) - )], - None => runtime::inline(helpers.iter().copied()), - } + runtime::import_line(runtime_module, helpers.iter().copied()) + )]; + } + let parsed = + ruff_python_parser::parse_unchecked_source(module, ruff_python_ast::PySourceType::Python); + let code = transforms::repeated_underscore::CodeNames::of(parsed.suite()); + let written = transforms::repeated_underscore::WrittenNames::new(module).with_code(&code); + let definitions = runtime::inline(helpers.iter().copied(), parsed.suite(), written); + written + .builtin_imports(&definitions.concat(), config.min_version.minor) + .into_iter() + .map(|import| format!("{import}\n")) + .chain(definitions) + .collect() } /// Version polyfill phase: rewrite syntax the target python cannot parse into @@ -878,9 +932,31 @@ fn splice_preamble(body: &str, preamble: &str) -> String { /// since the unified parser accepts those — a leftover flag in the output /// means a transform passed responsibility for invalid Python down the chain /// and we abort here. +/// whether the lowering preamble writes `from __future__ import annotations` +/// over a module: asked for, or on a target below 3.10, whose runtime cannot +/// evaluate the `X | Y` annotations the optional lowering itself produces — +/// unless the module is a stub or spells the import itself +pub(crate) fn writes_future_annotations(stmts: &[Stmt], config: &Config) -> bool { + (config.inject_future_annotations || config.min_version < PythonVersion::PY310) + && !config.is_stub + && !has_future_annotations(stmts) +} + +/// whether python evaluates the output's annotations as their definitions run: +/// not in a stub, not on a version that defers them (3.14), and not under a +/// `from __future__ import annotations` the module spells or the preamble writes. +/// every pass that quotes an annotation asks this, so a forward reference is +/// quoted exactly when nothing else defers it +pub(crate) fn annotations_evaluated(stmts: &[Stmt], config: &Config) -> bool { + !config.is_stub + && !config.min_version.defers_annotations() + && !has_future_annotations(stmts) + && !writes_future_annotations(stmts, config) +} + /// True when the user source already imports `annotations` from /// `__future__`, so the lowering doesn't emit a duplicate -fn has_future_annotations(stmts: &[Stmt]) -> bool { +pub(crate) fn has_future_annotations(stmts: &[Stmt]) -> bool { stmts.iter().any(|s| { let Stmt::ImportFrom(node) = s else { return false; @@ -1097,8 +1173,8 @@ impl AuthorNames { std::collections::HashMap::new(); for stmt in suite { // the range is the binding's own, so a refusal points at the `x` of `x = 1` - // rather than at the whole statement - for (name, range) in runtime::bindings(stmt) { + // rather than at the whole statement. a function's `global` binds here too + for (name, range) in runtime::module_bindings(stmt) { binds_at_module_scope .entry(name) .and_modify(|(count, _)| *count += 1) @@ -1136,7 +1212,7 @@ impl AuthorNames { fn verify_no_helper_name_clash(suite: &[Stmt], author: &AuthorNames) -> Result<(), TranspileError> { let mut provided: std::collections::HashMap = std::collections::HashMap::new(); for stmt in suite { - for name in runtime::bindings(stmt).into_keys() { + for name in runtime::module_bindings(stmt).into_keys() { if runtime::defines(&name) { *provided.entry(name).or_insert(0) += 1; } @@ -1434,12 +1510,7 @@ fn run_lowering_phase(source: &str, stmts: &[Stmt], config: &Config) -> Lowering // BOM, just a character python refuses to tokenize let (bom, source) = source.split_at(usize::from(source.bom_start_offset())); output.push_str(bom); - // below 3.10 the runtime cannot evaluate pep 604 `X | Y` annotations — - // which the optional lowering itself produces — so annotation evaluation - // must always be deferred on those targets - let needs_lazy_annotations = - config.inject_future_annotations || config.min_version < PythonVersion::PY310; - if needs_lazy_annotations && !config.is_stub && !has_future_annotations(stmts) { + if writes_future_annotations(stmts, config) { // a string is the module's `__doc__` only while it is the first statement, // so the import goes after it rather than over it — a module whose // annotations had to be deferred used to read back no docstring at all. @@ -2420,11 +2491,11 @@ mod cross_file { ]); let out = transpile_file(&project, "/main.by", &Config::test_default()); assert!( - out.contains("from ui.geometry import Dp as _by_conv__Dp"), + out.contains("from ui.geometry import Dp as _by_conv__ui_dgeometry__Dp"), "the declaring module is spelled through the imported package, got:\n{out}" ); assert!( - out.contains("print(pad(_by_conv__Dp.__of__(8)))"), + out.contains("print(pad(_by_conv__ui_dgeometry__Dp.__of__(8)))"), "the literal converts through the alias, got:\n{out}" ); } @@ -2446,10 +2517,13 @@ mod cross_file { ]); let out = transpile_file(&project, "/ui/widgets.by", &Config::test_default()); assert!( - out.contains("from .geometry import Dp as _by_conv__Dp"), + out.contains("from .geometry import Dp as _by_conv___dgeometry__Dp"), + "got:\n{out}" + ); + assert!( + out.contains("_by_conv___dgeometry__Dp.__of__(8)"), "got:\n{out}" ); - assert!(out.contains("_by_conv__Dp.__of__(8)"), "got:\n{out}"); } /// `f[int](1)` must lower to `f(1)` only because ty resolves the imported /// `f` to a generic *function* (constructor calls like `Foo[int](1)` keep @@ -2608,6 +2682,33 @@ mod cross_file { ); } + /// an importing module calls a backing function by the name the declaring module declares + /// it under, which is past what the declaring module spells + #[test] + fn an_imported_backing_function_is_called_by_its_declared_name() { + let project = project_db(&[ + ( + "/ext.by", + "def _by_ext__list__second() -> None: ...\n\nextension list:\n def second(self) -> Element:\n return self[1]\n", + ), + ( + "/main.by", + "import ext\n\nxs = [1, 2, 3]\nprint(xs.second())\n", + ), + ]); + let out = transpile_file(&project, "/main.by", &Config::test_default()); + assert!( + out.contains("from ext import _by_ext2__list__second") + && out.contains("print(_by_ext2__list__second(xs))"), + "got:\n{out}" + ); + let ext_out = transpile_file(&project, "/ext.by", &Config::test_default()); + assert!( + ext_out.contains("def _by_ext2__list__second(self):"), + "got:\n{ext_out}" + ); + } + /// Locate an interpreter for the cross-module runtime checks. fn python() -> Option { let mut candidates = Vec::new(); @@ -2800,18 +2901,22 @@ mod cross_file { ]); let out = transpile_file(&project, "/main.by", &Config::test_default()); assert!( - out.contains("return _by_witness(value, _by_conv__Show, \"show\")()"), + out.contains("return _by_witness(value, _by_conv__iface__Show, \"show\")()"), "the requirement should dispatch through the table, got:\n{out}" ); // the conformance is not this file's to register - assert!(!out.contains("_by_conform(_by_conv__Show,"), "got:\n{out}"); + assert!( + !out.contains("_by_conform(_by_conv__iface__Show,"), + "got:\n{out}" + ); // the declaring module emits both halves, naming the backing function it // lowered itself let adapters_out = transpile_file(&project, "/adapters.by", &Config::test_default()); assert!( - adapters_out - .contains("_by_conform(_by_conv__Show, str, {\"show\": _by_ext__str__show})"), + adapters_out.contains( + "_by_conform(_by_conv__iface__Show, str, {\"show\": _by_ext__str__show})" + ), "the declaring module should register the table, got:\n{adapters_out}" ); } @@ -2837,7 +2942,9 @@ mod cross_file { "the default's backing function must be imported, got:\n{out}" ); assert!( - out.contains("_by_conform(_by_conv__Show, str, {\"show\": _by_ext__Show__show})"), + out.contains( + "_by_conform(_by_conv__iface__Show, str, {\"show\": _by_ext__Show__show})" + ), "got:\n{out}" ); } @@ -2863,11 +2970,13 @@ mod cross_file { ]); let out = transpile_file(&project, "/main.by", &Config::test_default()); assert!( - out.contains("from temps import Celsius, report, Fahrenheit as _by_conv__Fahrenheit"), + out.contains( + "from temps import Celsius, report, Fahrenheit as _by_conv__temps__Fahrenheit" + ), "the target class should be imported under its alias, got:\n{out}" ); assert!( - out.contains("report(_by_conv__Fahrenheit.__from__(Celsius()))"), + out.contains("report(_by_conv__temps__Fahrenheit.__from__(Celsius()))"), "argument should be converted, got:\n{out}" ); } @@ -2893,7 +3002,7 @@ mod cross_file { ]); let out = transpile_file(&project, "/main.by", &Config::test_default()); assert!( - out.contains("report(_by_conv__Fahrenheit.__from__(Celsius()))"), + out.contains("report(_by_conv__temps__Fahrenheit.__from__(Celsius()))"), "argument should be converted through the alias, got:\n{out}" ); assert!( @@ -2923,15 +3032,118 @@ mod cross_file { ]); let out = transpile_file(&project, "/main.by", &Config::test_default()); assert!( - out.contains("from temps import Celsius, report, Fahrenheit as _by_conv__Fahrenheit"), + out.contains( + "from temps import Celsius, report, Fahrenheit as _by_conv__temps__Fahrenheit" + ), "the alias keeps the local class intact, got:\n{out}" ); assert!( - out.contains("report(_by_conv__Fahrenheit.__from__(Celsius()))"), + out.contains("report(_by_conv__temps__Fahrenheit.__from__(Celsius()))"), "the conversion must not go through the local class, got:\n{out}" ); } + /// two classes of one name from two modules are two conversion targets, so they are + /// imported under two aliases: under one, the second import would rebind the first and + /// both conversions would go through whichever class was imported last + #[test] + fn same_named_targets_from_two_modules_get_two_aliases() { + let length = |factor: u32| { + format!( + "frozen data class Length:\n value: float\n\n \ + class def __of__(cls, value: int) -> Length:\n \ + return Length(float(value * {factor}))\n" + ) + }; + let metric = length(1000); + let imperial = length(12); + let project = project_db(&[ + ("/metric.by", metric.as_str()), + ("/imperial.by", imperial.as_str()), + ( + "/main.by", + "import metric\nimport imperial\n\n\ + def m(amount: metric.Length) -> float:\n return amount.value\n\n\ + def i(amount: imperial.Length) -> float:\n return amount.value\n\n\ + print(m(2), i(2))\n", + ), + ]); + let out = transpile_file(&project, "/main.by", &Config::test_default()); + assert!( + out.contains("from metric import Length as _by_conv__metric__Length"), + "got:\n{out}" + ); + assert!( + out.contains("from imperial import Length as _by_conv__imperial__Length"), + "got:\n{out}" + ); + assert!( + out.contains( + "print(m(_by_conv__metric__Length.__of__(2)), i(_by_conv__imperial__Length.__of__(2)))" + ), + "got:\n{out}" + ); + } + + /// a module and a class name are written into an alias so that no two imports share one: + /// a `_` in the module is `_u` and a `.` is `_d`, so `a_b.c` and `a.b_c` stay apart + #[test] + fn a_module_path_is_written_into_its_alias_unambiguously() { + let length = "frozen data class Length:\n value: float\n\n \ + class def __of__(cls, value: int) -> Length:\n \ + return Length(float(value))\n"; + let project = project_db(&[ + ("/a_b/__init__.by", ""), + ("/a_b/c.by", length), + ("/a/__init__.by", ""), + ("/a/b_c.by", length), + ( + "/main.by", + "import a_b.c\nimport a.b_c\n\n\ + def f(x: a_b.c.Length, y: a.b_c.Length) -> float:\n return x.value + y.value\n\n\ + print(f(1, 2))\n", + ), + ]); + let out = transpile_file(&project, "/main.by", &Config::test_default()); + assert!( + out.contains("from a_b.c import Length as _by_conv__a_ub_dc__Length"), + "got:\n{out}" + ); + assert!( + out.contains("from a.b_c import Length as _by_conv__a_db_uc__Length"), + "got:\n{out}" + ); + } + + /// a file that spells a name beginning with the alias prefix gets another prefix, so no + /// alias can rebind a name of the file's own, nor be rebound by it + #[test] + fn a_file_spelling_the_alias_prefix_gets_another() { + let project = project_db(&[ + ( + "/temps.by", + "class Celsius:\n degrees: float = 0.0\n\n\ + class Fahrenheit:\n degrees: float = 0.0\n\n \ + @classmethod\n def __from__(cls, value: Celsius) -> Self:\n return cls()\n\n\ + def report(t: Fahrenheit) -> None: ...\n", + ), + ( + "/main.by", + "from temps import Celsius, report\n\n\ + _by_conv__temps__Fahrenheit = 1\n\nreport(Celsius())\n", + ), + ]); + let out = transpile_file(&project, "/main.by", &Config::test_default()); + assert!( + out.contains("report(_by_conv2__temps__Fahrenheit.__from__(Celsius()))"), + "got:\n{out}" + ); + assert!( + out.contains("_by_conv__temps__Fahrenheit = 1"), + "got:\n{out}" + ); + } + /// the line map must point a runtime statement in the generated python back /// to the line it came from in the `.by` source — the basis of `by run`'s /// traceback rewriting. exercised through a lazy import (large generated @@ -3167,6 +3379,25 @@ mod runtime_helper_check { assert!(error.contains("`_soundness_check`"), "{error}"); } + /// a function that declares a helper's name `global` binds it at module scope, as a + /// module-level assignment would + #[test] + fn a_global_binding_in_a_function_is_reported() { + let error = clash( + concat!( + "def rebind():\n", + " if True:\n", + " global _force_unwrap\n", + " _force_unwrap = lambda v: 99\n", + "\n", + "def f(y: int?) -> int:\n", + " return y!\n", + ), + &Config::test_default(), + ); + assert!(error.contains("`_force_unwrap`"), "{error}"); + } + /// a binding in a nested scope shadows the helper for that scope alone, so it is reported /// only when the lowering emits a read of the name inside it — here the checked return /// value, which lands one line below the author's own local @@ -3485,3 +3716,47 @@ mod runtime_helper_check { .unwrap(); } } + +#[cfg(test)] +mod synthesized_class_names { + use super::{Config, transpile}; + + /// the name of the class `out` declares under `prefix`, a name a lowering synthesized + fn synthesized(out: &str, prefix: &str) -> String { + let at = out + .find(&format!("class {prefix}")) + .unwrap_or_else(|| panic!("no `{prefix}` class in:\n{out}")) + + "class ".len(); + out[at..] + .chars() + .take_while(|c| c.is_alphanumeric() || *c == '_') + .collect() + } + + /// a class a lowering synthesizes for a shape is named after the shape, and a module + /// that already spells that name has it passed over, as every name a lowering binds + /// is. otherwise the module's own definition and the synthesized one would be one + /// name, and whichever came last would be read for both + #[test] + fn a_synthesized_class_name_the_module_spells_is_passed_over() { + for (source, prefix) in [ + ( + "def f() -> (a: int, b: str):\n return (a=1, b=\"x\")\n", + "_AnonNamedTuple_", + ), + ("def f(d: {\"a\": int}): ...\n", "_TypedDict_"), + ("def f(p: protocol(a: int)): ...\n", "_Protocol_"), + ("def f(c: (a: int) -> str): ...\n", "_Callable_"), + ] { + let config = Config::test_default(); + let name = synthesized(&transpile(source, &config).unwrap(), prefix); + let with_own = format!("{source}class {name}: ...\n"); + let out = transpile(&with_own, &config).unwrap(); + assert_eq!( + synthesized(&out, prefix), + format!("{name}2"), + "the module's own `{name}` is passed over:\n{out}" + ); + } + } +} diff --git a/crates/by_transforms/src/runtime.rs b/crates/by_transforms/src/runtime.rs index 9ef9e45a3b..df1d8397af 100644 --- a/crates/by_transforms/src/runtime.rs +++ b/crates/by_transforms/src/runtime.rs @@ -5,7 +5,9 @@ //! file beside the modules it emits and each module imports the names it calls. a //! transpile with nowhere to write it (`by transpile `, the language //! server's `by/transpile`) pastes the definitions in instead. both are slices of -//! the one text, so the two renderings cannot drift apart +//! the one text, so the two renderings cannot drift apart. the one difference is a +//! builtin the module binds for itself, which a pasted definition would otherwise +//! read from the module's globals: it reads it under a name of its own (`inline`) //! //! # naming a helper //! @@ -115,6 +117,8 @@ helpers! { DATACLASS_SLOTS = "_by_dataclass_slots", } +mod global_reads; + /// one definition of the runtime, with whatever finishes setting it up struct Definition { /// its source as the file spells it, newline-terminated like any other @@ -125,6 +129,10 @@ struct Definition { order: usize, /// the other definitions its body reads needs: Vec, + /// each read of a name the runtime does not define, which python looks up in the + /// globals of the module the definition runs in before the builtins, by its range in + /// [`Self::source`] + builtin_reads: Vec<(TextRange, String)>, } /// every definition, indexed by each name it binds @@ -174,6 +182,11 @@ fn build_index() -> BTreeMap { .collect(); needs.sort(); needs.dedup(); + let builtin_reads: Vec<(TextRange, String)> = global_reads::global_reads(&suite[at..=last]) + .into_iter() + .filter(|(_, name)| !defined.contains(name)) + .map(|(range, name)| (range - span.start(), name)) + .collect(); // `every_name_has_one_definition` keeps a name from being bound twice, // so first-wins here never decides anything for name in names.into_keys() { @@ -181,6 +194,7 @@ fn build_index() -> BTreeMap { source: source.clone(), order, needs: needs.clone(), + builtin_reads: builtin_reads.clone(), }); } order += 1; @@ -201,13 +215,68 @@ fn record(names: &mut Bindings, name: impl Into, range: TextRange) { names.entry(name.into()).or_insert(range); } -/// the names a top-level statement binds at module scope +/// the names a statement binds in the scope it stands in pub(crate) fn bindings(stmt: &Stmt) -> Bindings { let mut names = Bindings::new(); bind(stmt, &mut names); names } +/// the names a top-level statement binds at module scope: the ones it binds itself +/// ([`bindings`]), and the ones a function or class inside it, at any depth, declares +/// `global` and binds +pub(crate) fn module_bindings(stmt: &Stmt) -> Bindings { + let mut names = bindings(stmt); + global_bindings(stmt, &mut names); + names +} + +/// the names the scopes `stmt` opens, and the scopes inside those, declare `global` and bind +fn global_bindings(stmt: &Stmt, names: &mut Bindings) { + let body = match stmt { + Stmt::FunctionDef(def) => &def.body, + Stmt::ClassDef(def) => &def.body, + _ => { + let mut blocks = Vec::new(); + push_blocks(stmt, &mut blocks); + for block in blocks { + for stmt in block { + global_bindings(stmt, names); + } + } + return; + } + }; + let mut declared = BTreeSet::new(); + global_declarations(body, &mut declared); + let mut bound = Bindings::new(); + for stmt in body { + bind(stmt, &mut bound); + global_bindings(stmt, names); + } + for (name, range) in bound { + if declared.contains(&name) { + record(names, name, range); + } + } +} + +/// the names `body`'s own statements declare `global`, not descending into the scopes they +/// open +fn global_declarations(body: &[Stmt], out: &mut BTreeSet) { + for stmt in body { + if let Stmt::Global(node) = stmt { + out.extend(node.names.iter().map(ToString::to_string)); + } else { + let mut blocks = Vec::new(); + push_blocks(stmt, &mut blocks); + for block in blocks { + global_declarations(block, out); + } + } + } +} + /// a block that runs at module scope (an `if`, a `try`, a loop, a `case`) binds what the /// statements inside it bind. a function or class body is a scope of its own /// @@ -989,12 +1058,65 @@ pub(crate) fn defines(name: &str) -> bool { index().contains_key(name) } -/// the definitions `helpers` need, as preamble entries for a module that has -/// nowhere to import them from -pub(crate) fn inline(helpers: impl IntoIterator) -> Vec { - closure(helpers) +/// the definitions `helpers` need, as preamble entries for a module that has nowhere to +/// import them from, whose top-level statements are `module` and whose names `written` +/// holds +/// +/// a definition pasted into a module reads the builtins through the module's globals, +/// so a builtin the module binds at its top level for itself is read under the name a +/// lowering reads it by ([`WrittenNames::builtin`]), which the caller imports from +/// `builtins` ([`WrittenNames::builtin_imports`]). a star import may bind any of them, +/// so it counts as binding each +/// +/// [`WrittenNames::builtin`]: crate::transforms::repeated_underscore::WrittenNames::builtin +/// [`WrittenNames::builtin_imports`]: crate::transforms::repeated_underscore::WrittenNames::builtin_imports +pub(crate) fn inline( + helpers: impl IntoIterator, + module: &[Stmt], + written: crate::transforms::repeated_underscore::WrittenNames, +) -> Vec { + let definitions = closure(helpers); + let read: BTreeSet<&str> = definitions + .iter() + .flat_map(|definition| &definition.builtin_reads) + .map(|(_, name)| name.as_str()) + .collect(); + let bound = if read.is_empty() { + Bindings::new() + } else { + let mut bound = Bindings::new(); + for stmt in module { + for (name, range) in module_bindings(stmt) { + record(&mut bound, name, range); + } + } + bound + }; + let renamed: BTreeMap<&str, String> = read .into_iter() - .map(|definition| definition.source.clone()) + .filter(|name| bound.contains_key(*name) || bound.contains_key("*")) + .map(|name| (name, written.builtin(name))) + .filter(|(name, local)| name != local) + .collect(); + + definitions + .into_iter() + .map(|definition| { + let mut source = definition.source.clone(); + let mut reads: Vec<&(TextRange, String)> = definition + .builtin_reads + .iter() + .filter(|(_, name)| renamed.contains_key(name.as_str())) + .collect(); + reads.sort_by_key(|(range, _)| std::cmp::Reverse(range.start())); + for (range, name) in reads { + source.replace_range( + std::ops::Range::::from(*range), + &renamed[name.as_str()], + ); + } + source + }) .collect() } @@ -1076,7 +1198,12 @@ mod tests { /// the class it patches, or the proxy is left with no operators at all #[test] fn set_up_code_travels_with_its_definition() { - let pasted = inline([LAZY_ATTR]).concat(); + let pasted = inline( + [LAZY_ATTR], + &[], + crate::transforms::repeated_underscore::WrittenNames::new(""), + ) + .concat(); assert!(pasted.contains("class _LazyAttr:"), "{pasted}"); assert!( pasted.contains("_by_forward_operators(_LazyAttr)"), @@ -1091,7 +1218,12 @@ mod tests { /// runs on 3.9 — so each carries the suppression the decorator would have been #[test] fn the_proxies_overrides_do_not_report_in_the_emitted_python() { - let pasted = inline([LAZY_ATTR]).concat(); + let pasted = inline( + [LAZY_ATTR], + &[], + crate::transforms::repeated_underscore::WrittenNames::new(""), + ) + .concat(); for member in ["__class__", "__setattr__", "__delattr__"] { let line = pasted .lines() @@ -1103,4 +1235,107 @@ mod tests { ); } } + + /// the reads a pasted definition renames are the names python itself looks up in the + /// module's globals: its compiler's symbol table answers the same question, so the two + /// are held to agree over the whole runtime. with no python to ask, there is nothing + /// to compare against + #[test] + fn the_global_reads_are_the_ones_python_finds() { + const SYMTABLE: &str = r#" +import symtable, sys +table = symtable.symtable(sys.stdin.read(), "_by_runtime.py", "exec") +defined = {s.get_name() for s in table.get_symbols() if s.is_assigned() or s.is_imported() or s.is_namespace()} +found = set() +def walk(scope): + for symbol in scope.get_symbols(): + global_read = symbol.is_global() and not symbol.is_declared_global() if scope.get_type() != "module" else True + if global_read and symbol.is_referenced() and symbol.get_name() not in defined: + found.add(symbol.get_name()) + for child in scope.get_children(): + walk(child) +walk(table) +print(" ".join(sorted(found))) +"#; + let Ok(mut child) = std::process::Command::new("python3") + .args(["-c", SYMTABLE]) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .spawn() + else { + return; + }; + if let Some(mut stdin) = child.stdin.take() { + std::io::Write::write_all(&mut stdin, SOURCE.as_bytes()).expect("write the runtime"); + } + let output = child.wait_with_output().expect("run python"); + assert!(output.status.success(), "{output:?}"); + let python: BTreeSet = String::from_utf8_lossy(&output.stdout) + .split_whitespace() + .map(str::to_owned) + .collect(); + let ours: BTreeSet = index() + .values() + .flat_map(|definition| &definition.builtin_reads) + .map(|(_, name)| name.clone()) + .collect(); + assert_eq!(ours, python); + } + + /// a module that binds a builtin at its top level for itself has the pasted definitions + /// read that builtin under the name a lowering would, and a module that binds it only + /// in a function of its own leaves them reading it as written + #[test] + fn a_pasted_definition_reads_a_builtin_the_module_binds_under_its_own_name() { + let module = "def getattr(o, n): return 42\ndef f(format): return format\n"; + let parsed = ruff_python_parser::parse_unchecked_source( + module, + ruff_python_ast::PySourceType::Python, + ); + let code = crate::transforms::repeated_underscore::CodeNames::of(parsed.suite()); + let written = + crate::transforms::repeated_underscore::WrittenNames::new(module).with_code(&code); + let pasted = inline([LAZY_ATTR], parsed.suite(), written).concat(); + assert!( + pasted.contains("v = getattr2(m, self._by_attr)") && !pasted.contains("getattr(m,"), + "{pasted}" + ); + assert!( + pasted.contains("lambda s, f: format(s._by_resolve(), f)"), + "{pasted}" + ); + } + + /// a function or class that declares a builtin's name `global` and binds it rebinds the + /// module's, at any depth, which the pasted definitions would then read + #[test] + fn a_pasted_definition_reads_a_builtin_a_global_declaration_rebinds_under_its_own_name() { + let module = concat!( + "def outer():\n", + " class C:\n", + " def rebind(self):\n", + " global getattr\n", + " for getattr in [len]:\n", + " pass\n", + "def local():\n", + " format = 1\n", + ); + let parsed = ruff_python_parser::parse_unchecked_source( + module, + ruff_python_ast::PySourceType::Python, + ); + let code = crate::transforms::repeated_underscore::CodeNames::of(parsed.suite()); + let written = + crate::transforms::repeated_underscore::WrittenNames::new(module).with_code(&code); + let pasted = inline([LAZY_ATTR], parsed.suite(), written).concat(); + assert!( + pasted.contains("v = getattr2(m, self._by_attr)") && !pasted.contains("getattr(m,"), + "{pasted}" + ); + // a local binding leaves the module's alone + assert!( + pasted.contains("lambda s, f: format(s._by_resolve(), f)"), + "{pasted}" + ); + } } diff --git a/crates/by_transforms/src/runtime/global_reads.rs b/crates/by_transforms/src/runtime/global_reads.rs new file mode 100644 index 0000000000..5a02171744 --- /dev/null +++ b/crates/by_transforms/src/runtime/global_reads.rs @@ -0,0 +1,260 @@ +//! the names a runtime definition reads from the globals of the module it runs in +//! +//! pasted into a module, a helper's free names are looked up in that module's globals +//! before the builtins, so a module that binds `getattr` at its top level hands the +//! helper its own `getattr`. this finds each such read, by python's scoping rules: a name +//! a function, lambda or comprehension binds is its own, and a class body's names are +//! seen by the class body alone, never by a function inside it + +use std::collections::HashSet; + +use ruff_python_ast::visitor::source_order::{SourceOrderVisitor, walk_expr, walk_stmt}; +use ruff_python_ast::{self as ast, Expr, Stmt}; +use ruff_text_size::{Ranged, TextRange}; + +use super::{Bindings, bind, bind_target}; + +#[derive(Clone, Copy, PartialEq, Eq)] +enum ScopeKind { + Module, + Class, + Function, +} + +struct Scope { + kind: ScopeKind, + /// the names the scope binds for itself + locals: HashSet, +} + +/// every read, in `stmts`, of a name that resolves to the module's globals, with its range +pub(super) fn global_reads(stmts: &[Stmt]) -> Vec<(TextRange, String)> { + let mut reads = Reads { + scopes: vec![Scope { + kind: ScopeKind::Module, + locals: HashSet::default(), + }], + found: Vec::new(), + }; + for stmt in stmts { + reads.visit_stmt(stmt); + } + reads.found +} + +struct Reads { + scopes: Vec, + found: Vec<(TextRange, String)>, +} + +impl Reads { + /// whether `name`, read in the innermost scope, is bound by it or by a function + /// around it. a class body's names are its own and reach no scope inside it + fn is_local(&self, name: &str) -> bool { + let Some((innermost, outer)) = self.scopes.split_last() else { + return false; + }; + if innermost.kind != ScopeKind::Module && innermost.locals.contains(name) { + return true; + } + outer + .iter() + .any(|scope| scope.kind == ScopeKind::Function && scope.locals.contains(name)) + } + + fn in_scope(&mut self, kind: ScopeKind, locals: HashSet, run: impl FnOnce(&mut Self)) { + self.scopes.push(Scope { kind, locals }); + run(self); + self.scopes.pop(); + } + + fn parameters(&mut self, parameters: &ast::Parameters) { + for parameter in parameters { + if let Some(default) = parameter.default() { + self.visit_expr(default); + } + if let Some(annotation) = parameter.annotation() { + self.visit_expr(annotation); + } + } + } +} + +impl<'a> SourceOrderVisitor<'a> for Reads { + fn visit_stmt(&mut self, stmt: &'a Stmt) { + match stmt { + Stmt::FunctionDef(function) => { + // the decorators, defaults and annotations are evaluated where the `def` is + for decorator in &function.decorator_list { + self.visit_expr(&decorator.expression); + } + self.parameters(&function.parameters); + if let Some(returns) = &function.returns { + self.visit_expr(returns); + } + let mut locals: HashSet = function + .parameters + .iter() + .map(|parameter| parameter.name().to_string()) + .collect(); + locals.extend(scope_bindings(&function.body)); + for name in declared_global(&function.body) { + locals.remove(&name); + } + self.in_scope(ScopeKind::Function, locals, |reads| { + for stmt in &function.body { + reads.visit_stmt(stmt); + } + }); + } + Stmt::ClassDef(class) => { + for decorator in &class.decorator_list { + self.visit_expr(&decorator.expression); + } + if let Some(arguments) = &class.arguments { + self.visit_arguments(arguments); + } + let locals = scope_bindings(&class.body); + self.in_scope(ScopeKind::Class, locals, |reads| { + for stmt in &class.body { + reads.visit_stmt(stmt); + } + }); + } + _ => walk_stmt(self, stmt), + } + } + + fn visit_expr(&mut self, expr: &'a Expr) { + match expr { + Expr::Name(name) if name.ctx.is_load() => { + if !self.is_local(name.id.as_str()) { + self.found.push((name.range(), name.id.to_string())); + } + } + Expr::Lambda(lambda) => { + let mut locals = HashSet::default(); + if let Some(parameters) = &lambda.parameters { + self.parameters(parameters); + locals.extend( + parameters + .iter() + .map(|parameter| parameter.name().to_string()), + ); + } + locals.extend(walrus_targets(&lambda.body)); + self.in_scope(ScopeKind::Function, locals, |reads| { + reads.visit_expr(&lambda.body); + }); + } + Expr::ListComp(ast::ExprListComp { + elt, generators, .. + }) + | Expr::SetComp(ast::ExprSetComp { + elt, generators, .. + }) + | Expr::Generator(ast::ExprGenerator { + elt, generators, .. + }) => self.comprehension(generators, |reads| reads.visit_expr(elt)), + Expr::DictComp(ast::ExprDictComp { + key, + value, + generators, + .. + }) => self.comprehension(generators, |reads| { + if let Some(key) = key { + reads.visit_expr(key); + } + reads.visit_expr(value); + }), + _ => walk_expr(self, expr), + } + } +} + +impl Reads { + /// a comprehension is a scope of its own, binding its targets. its first iterable is + /// evaluated in the scope around it + fn comprehension( + &mut self, + generators: &[ast::Comprehension], + element: impl FnOnce(&mut Self), + ) { + let Some((first, _)) = generators.split_first() else { + return; + }; + self.visit_expr(&first.iter); + let mut targets = Bindings::new(); + for generator in generators { + bind_target(&generator.target, &mut targets); + } + let locals = targets.into_keys().collect(); + self.in_scope(ScopeKind::Function, locals, |reads| { + for (index, generator) in generators.iter().enumerate() { + if index > 0 { + reads.visit_expr(&generator.iter); + } + for condition in &generator.ifs { + reads.visit_expr(condition); + } + } + element(reads); + }); + } +} + +/// the names the statements of one scope bind for it +fn scope_bindings(body: &[Stmt]) -> HashSet { + let mut names = Bindings::new(); + for stmt in body { + bind(stmt, &mut names); + } + names.into_keys().collect() +} + +/// the names a function body declares `global`, wherever in the body it does so +fn declared_global(body: &[Stmt]) -> Vec { + #[derive(Default)] + struct Globals(Vec); + impl<'a> SourceOrderVisitor<'a> for Globals { + fn visit_stmt(&mut self, stmt: &'a Stmt) { + match stmt { + Stmt::Global(global) => { + self.0.extend(global.names.iter().map(ToString::to_string)); + } + // a nested scope's declarations are its own + Stmt::FunctionDef(_) | Stmt::ClassDef(_) => {} + _ => walk_stmt(self, stmt), + } + } + fn visit_expr(&mut self, _: &'a Expr) {} + } + let mut globals = Globals::default(); + for stmt in body { + globals.visit_stmt(stmt); + } + globals.0 +} + +/// the names a `:=` binds in `expr`, outside any lambda nested in it +fn walrus_targets(expr: &Expr) -> Vec { + #[derive(Default)] + struct Targets(Vec); + impl<'a> SourceOrderVisitor<'a> for Targets { + fn visit_expr(&mut self, expr: &'a Expr) { + match expr { + Expr::Named(named) => { + if let Expr::Name(name) = named.target.as_ref() { + self.0.push(name.id.to_string()); + } + } + Expr::Lambda(_) => return, + _ => {} + } + walk_expr(self, expr); + } + } + let mut targets = Targets::default(); + targets.visit_expr(expr); + targets.0 +} diff --git a/crates/by_transforms/src/transforms/annotation.rs b/crates/by_transforms/src/transforms/annotation.rs index 2eaac90f2a..fe19648202 100644 --- a/crates/by_transforms/src/transforms/annotation.rs +++ b/crates/by_transforms/src/transforms/annotation.rs @@ -424,7 +424,7 @@ impl TypeAwarePass for TupleLiteralTypePass<'_> { } /// each element of a tuple type it replaces is printed by the shared type-expression - /// lowerer + /// lowerer, and an operation `symbolic_type_op` folded inside one is written as the fold fn subsumes(&self) -> &'static [super::ast_driver::Lowering] { super::callable::TYPE_EXPRESSION } diff --git a/crates/by_transforms/src/transforms/anon_named_tuple.rs b/crates/by_transforms/src/transforms/anon_named_tuple.rs index 44556c5699..7320570491 100644 --- a/crates/by_transforms/src/transforms/anon_named_tuple.rs +++ b/crates/by_transforms/src/transforms/anon_named_tuple.rs @@ -29,9 +29,8 @@ //! ``` use std::collections::HashMap; -use std::collections::hash_map::DefaultHasher; use std::fmt::Write as _; -use std::hash::{Hash, Hasher}; +use std::hash::Hash; use ruff_diagnostics::{Edit, Fix}; use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; @@ -77,14 +76,9 @@ struct Shape { } impl Shape { - fn class_name(&self) -> String { - let mut hasher = DefaultHasher::new(); - self.hash(&mut hasher); - // 8 hex chars from 64-bit hash is enough to avoid collisions in a - // typical module while staying readable - #[expect(clippy::cast_possible_truncation)] - let truncated = hasher.finish() as u32; - format!("_AnonNamedTuple_{truncated:08x}") + /// the name of the class declaring this shape, one the module has no other use of + fn class_name(&self, written: WrittenNames) -> String { + written.synthesized_class("_AnonNamedTuple_", self) } /// the class declaring this shape as `name`, `typing.NamedTuple` written as `named_tuple` @@ -135,9 +129,13 @@ pub(crate) struct AnonNamedTuple<'src> { /// Set when at least one anonymous named tuple was seen, so the preamble /// emits the `NamedTuple` import. pub(crate) needs_import: bool, - /// The imports of the modules and `typing` names a field type reads that the source - /// never imported. A shape is built through `&self`, so this collects behind a cell + /// The modules a field type qualifies a class with that the source never imported under + /// their own names. A shape is built through `&self`, so this collects behind a cell type_only_imports: std::cell::RefCell>, + /// The imports of the `typing` names a field type reads, collected the same way + typing_imports: std::cell::RefCell>, + /// whether python evaluates the synthesized class's field annotations as it is made + annotations_evaluated: bool, /// Active function-scope return-annotation shape stack. Empty when not /// inside a function. The innermost (last) entry governs how a `return` /// statement inside the current function is coerced. @@ -187,6 +185,8 @@ impl<'src> AnonNamedTuple<'src> { range_to_value_render: Vec::new(), needs_import: false, type_only_imports: std::cell::RefCell::default(), + typing_imports: std::cell::RefCell::default(), + annotations_evaluated: false, return_shape_stack: Vec::new(), typevar_rename_stack: Vec::new(), aliases: HashMap::new(), @@ -443,8 +443,11 @@ impl<'src> AnonNamedTuple<'src> { }; self.type_only_imports .borrow_mut() - .extend(d.imports_in(self.written)); - d.text_in(self.written) + .extend(d.modules.iter().cloned()); + self.typing_imports + .borrow_mut() + .extend(d.typing_imports_in(self.written)); + d.annotation_in(self.written, self.annotations_evaluated) }; fields.push((name, type_display)); } @@ -464,7 +467,7 @@ impl<'src> AnonNamedTuple<'src> { let name = if let Some(existing) = self.shapes.get(&shape) { existing.clone() } else { - let n = shape.class_name(); + let n = shape.class_name(self.written); self.shapes.insert(shape, n.clone()); n }; @@ -655,7 +658,7 @@ impl<'src> AnonNamedTuple<'src> { let name = if let Some(existing) = self.shapes.get(&shape) { existing.clone() } else { - let n = shape.class_name(); + let n = shape.class_name(self.written); self.shapes.insert(shape, n.clone()); n }; @@ -917,6 +920,7 @@ impl super::ast_driver::TypeAwarePass for AnonNamedTuplePass<'_> { ) { let mut inner = AnonNamedTuple::new(self.source, self.written, types, self.config.clone(), stmts); + inner.annotations_evaluated = ctx.annotations_evaluated; for stmt in stmts { inner.visit_stmt(stmt); } @@ -939,6 +943,7 @@ impl super::ast_driver::TypeAwarePass for AnonNamedTuplePass<'_> { ctx.required_imports.extend(imports); ctx.runtime.extend(helpers); ctx.type_only_imports.extend(inner.type_only_imports.take()); + ctx.required_imports.extend(inner.typing_imports.take()); for fix in std::mem::take(&mut inner.edits) { for edit in fix.edits() { let range = edit.range(); @@ -952,9 +957,53 @@ impl super::ast_driver::TypeAwarePass for AnonNamedTuplePass<'_> { #[cfg(test)] mod tests { use crate::python_passthrough::unchanged; + use crate::transforms::repeated_underscore::SynthesizedClasses; use crate::{Config, transpile}; use indoc::indoc; + /// two one-field shapes, `(: int)` and `(: int)`, whose names hash alike + fn colliding_field_names() -> (String, String) { + let mut seen = std::collections::HashMap::new(); + for index in 0u32.. { + let name = format!("f{index}"); + let shape = super::Shape { + fields: vec![(name.clone(), "int".to_owned())], + }; + if let Some(earlier) = seen.insert(SynthesizedClasses::hash_of(&shape), name.clone()) { + return (earlier, name); + } + } + unreachable!("the hashes of four billion shapes cannot all differ") + } + + /// a class is named after 32 bits of its shape's hash, which two shapes can share. the + /// second is numbered apart, rather than declared under the first's name, where python + /// would read whichever class came last for both + #[test] + fn two_shapes_hashing_alike_are_named_apart() { + let (first, second) = colliding_field_names(); + let out = transpile( + &format!( + "def a() -> ({first}: int):\n return ({first}=1)\n\n\ + def b() -> ({second}: int):\n return ({second}=2)\n" + ), + &Config::test_default(), + ) + .unwrap(); + let hash = SynthesizedClasses::hash_of(&super::Shape { + fields: vec![(first.clone(), "int".to_owned())], + }); + let name = format!("_AnonNamedTuple_{hash:08x}"); + for expected in [ + format!("class {name}(NamedTuple):\n {first}: int\n"), + format!("class {name}2(NamedTuple):\n {second}: int\n"), + format!("def a() -> {name}:\n return {name}(1)\n"), + format!("def b() -> {name}2:\n return {name}2(2)\n"), + ] { + assert!(out.contains(&expected), "expected `{expected}` in:\n{out}"); + } + } + fn check(input: &str, expected: &str) { assert_eq!( transpile(input, &Config::test_default()).unwrap(), diff --git a/crates/by_transforms/src/transforms/ast_driver.rs b/crates/by_transforms/src/transforms/ast_driver.rs index a35b374ceb..880588644f 100644 --- a/crates/by_transforms/src/transforms/ast_driver.rs +++ b/crates/by_transforms/src/transforms/ast_driver.rs @@ -64,6 +64,15 @@ enum SemDb<'p> { Local(ty_project::TestDb, ruff_db::files::File), } +/// what [`run_against_source`]'s type-aware passes analyse the module in +#[derive(Clone, Copy)] +pub(crate) enum SemanticSource<'p> { + /// the caller's db, which holds the source being lowered + Project(&'p dyn ty_python_semantic::Db, ruff_db::files::File), + /// a single-file db of its own, analysing the source at this python + Fresh(ruff_python_ast::PythonVersion), +} + /// One fragment of a [`PassContext::template_edits`] replacement: literal text, /// or a passthrough span of original source. Passthrough spans are materialized /// with any sibling edits inside them applied, so a wide rewrite (e.g. @@ -103,6 +112,11 @@ pub(crate) struct PassContext { /// surrounding context contains basedpython markers a sibling pass /// hasn't lowered yet pub(crate) text_edits: Vec<(TextRange, String)>, + /// Indices into [`text_edits`](Self::text_edits) of the edits a later pass withdrew, + /// because a replacement of its own re-renders what they wrote. They stay in the list, + /// whose order is how the driver knows which pass wrote each edit, and are left out + /// of the output + pub(crate) withdrawn_text_edits: Vec, /// Structured sub-statement edits whose replacement is a [`Fragment`] list. /// Unlike `text_edits` (whose plain string wins over anything nested inside /// it — which is a transpile error unless the pass writes that lowering itself, @@ -141,13 +155,20 @@ pub(crate) struct PassContext { /// Lines to append AFTER the spliced body (e.g. modifiers' auto- /// generated `__all__ = [...]`). Driver emits each as its own line pub(crate) epilogue: Vec, - /// Import lines a *synthesized* type expression needs that the source never - /// wrote (`import decimal` for an inferred `decimal.Decimal` annotation). - /// The driver emits these under `if TYPE_CHECKING:` — the output always - /// carries `from __future__ import annotations`, so the name is only ever - /// read by a checker, and a runtime import here would add an import edge the - /// source does not have + /// The modules a *synthesized* annotation qualifies a class with that the source + /// never imported under their own names (`decimal` for an inferred + /// `decimal.Decimal` annotation). The annotation is written as a string + /// ([`SynthesizedType::annotation_in`]), so the module is read only by a checker + /// and by `typing.get_type_hints` + /// + /// [`SynthesizedType::annotation_in`]: crate::type_info::SynthesizedType::annotation_in pub(crate) type_only_imports: BTreeSet, + /// The names the extension backing functions are declared under, which the driver + /// hoists to the module top once lowering is done + pub(crate) extension_backing_functions: BTreeSet, + /// Whether python evaluates an annotation as its definition runs + /// ([`crate::annotations_evaluated`]) + pub(crate) annotations_evaluated: bool, /// Source ranges of operations that `symbolic_type_op` resolved up front /// (e.g. `1 + 1` → `Literal[2]`). Type-aware passes skip these via /// [`walk_type_positions_skipping`](super::type_expr_walker::walk_type_positions_skipping) @@ -1112,11 +1133,11 @@ fn merge_from_imports(lines: Vec) -> (Vec, Vec) { let mut groups: indexmap::IndexMap> = indexmap::IndexMap::new(); let mut other: Vec = Vec::new(); for line in lines { - // an entry that spans lines is not an import line but a block — the - // `if TYPE_CHECKING:` one, which opens with a `from typing import`. merged - // as though the whole block were a name list, the block's own lines become - // a "name" and every later `from typing import` lands after them, inside - // the block: `if TYPE_CHECKING:\n import types, overload` + // an entry that spans lines is not an import line but a block, such as a + // class a lowering synthesized. one that opens with a `from ... import` + // would otherwise be merged as though the whole block were a name list, its + // own lines becoming a "name" that every later import of that module lands + // after, inside the block if !line.contains('\n') && let Some(rest) = line.strip_prefix("from ") && let Some((module, names)) = rest.split_once(" import ") @@ -1147,8 +1168,8 @@ fn merge_from_imports(lines: Vec) -> (Vec, Vec) { /// statements back into the source text. Returns a borrowed `Cow` when /// nothing changed. /// -/// `project`, when `Some`, supplies the real project db + file so type-aware -/// passes resolve cross-module imports (e.g. an imported generic function for +/// `project`, when [`SemanticSource::Project`], supplies the real project db + file so +/// type-aware passes resolve cross-module imports (e.g. an imported generic function for /// `generic_call`). The chosen db owns the parse the type-aware passes query: /// `inferred_type` does AST node-identity lookups, so the model and the walked /// suite must come from one db @@ -1156,7 +1177,7 @@ pub(crate) fn run_against_source<'a>( source: &'a str, written: repeated_underscore::WrittenNames, config: &Config, - project: Option<(&dyn ty_python_semantic::Db, ruff_db::files::File)>, + project: SemanticSource<'_>, ) -> (Cow<'a, str>, Vec, Vec>) { // blank the keyword-prefix type markers — use-site variance and the // `literal`/`final` type modifiers — out up front; downstream passes @@ -1171,9 +1192,9 @@ pub(crate) fn run_against_source<'a>( // that's sound precisely because blanking preserves byte positions: the // db's parse and the blanked parse below agree on every node's range let sem = match project { - Some((pdb, pfile)) => SemDb::Project(pdb, pfile), - None => { - let (db, file) = crate::make_in_memory_db(source); + SemanticSource::Project(pdb, pfile) => SemDb::Project(pdb, pfile), + SemanticSource::Fresh(python_version) => { + let (db, file) = crate::make_in_memory_db_at(source, python_version); SemDb::Local(db, file) } }; @@ -1226,7 +1247,10 @@ pub(crate) fn run_against_source<'a>( .iter() .map(|s| (usize::from(s.range().start()), usize::from(s.range().end()))) .collect(); - let mut ctx = PassContext::default(); + let mut ctx = PassContext { + annotations_evaluated: crate::annotations_evaluated(&module.body, config), + ..PassContext::default() + }; // how each parameter list that repeats `_` is lowered is ty's answer, which every pass // that names a parameter reads through `written` — the ones walking a tree of their own @@ -1332,11 +1356,7 @@ pub(crate) fn run_against_source<'a>( mutable_defaults::MutableDefaultsPass::new(source_ref, written, config.is_stub); let unique_loop_bindings_pass = unique_loop_bindings::UniqueLoopBindingsPass::new(source_ref, config.unique_loop_bindings); - let auto_quote_pass = auto_quote::AutoQuote::new( - source_ref, - config.min_version, - config.inject_future_annotations, - ); + let auto_quote_pass = auto_quote::AutoQuote::new(source_ref); let init_method_pass = init_method::InitMethod::new(source_ref, written, config.clone()); let properties_pass = properties::PropertiesPass::new( source_ref, @@ -1846,20 +1866,29 @@ pub(crate) fn run_against_source<'a>( config.min_version >= ruff_python_ast::PythonVersion::PY314, ); - // a synthesized annotation may name a class the source never imported. the - // import goes under `if TYPE_CHECKING:` as one block: every such annotation - // is a string at runtime (the lowering always emits the `__future__` import), - // so only a checker ever reads the name, and a real import would give the - // output an import edge — and a possible cycle — the source never had - if !ctx.type_only_imports.is_empty() { - // the one typing name a lowering writes under its own name whatever the module - // binds: the block reads it where it binds it, ahead of anything the module runs - let mut block = String::from("from typing import TYPE_CHECKING\nif TYPE_CHECKING:"); - for line in std::mem::take(&mut ctx.type_only_imports) { - block.push_str("\n "); - block.push_str(&line); + // a synthesized annotation may name a class from a module the source never imports as it + // runs — one it imports only for a checker, to keep out of an import cycle. the annotation + // is a string, so nothing evaluates it but `typing.get_type_hints`. where imports are + // deferred, the module is imported with the others, and runs only when that asks, after + // every module is initialized. where they are not, an import would run it as this module + // starts, so it is imported for a checker alone, and `get_type_hints` cannot resolve it + let modules = std::mem::take(&mut ctx.type_only_imports); + if !modules.is_empty() { + let imports = modules.iter().map(|module| format!("import {module}")); + if config.lazy_imports || config.is_stub { + ctx.required_imports.extend(imports); + } else { + let mut block = format!( + "{}\nif {}:", + written.import_from("typing", &["TYPE_CHECKING"]), + written.imported("typing", "TYPE_CHECKING") + ); + for import in imports { + block.push_str("\n "); + block.push_str(&import); + } + ctx.required_imports.push(block); } - ctx.required_imports.push(block); } ctx.required_imports.sort(); @@ -1882,7 +1911,11 @@ pub(crate) fn run_against_source<'a>( Some(module) => ctx .required_imports .push(crate::runtime::import_line(module, helpers)), - None => ctx.required_imports.extend(crate::runtime::inline(helpers)), + None => ctx.required_imports.extend(crate::runtime::inline( + helpers, + parsed_handle.suite(), + written, + )), } } ctx.required_imports.extend(definitions); @@ -2022,6 +2055,8 @@ pub(crate) fn run_against_source<'a>( // template edit instead *materializes* nested edits within its `Src` // passthrough spans, so wide rewrites compose with inner lowerings // who wrote each edit, in the order they are chained below + let withdrawn: std::collections::HashSet = + ctx.withdrawn_text_edits.iter().copied().collect(); let origins: Vec> = [ (EditList::Text, ctx.text_edits.len()), (EditList::Template, ctx.template_edits.len()), @@ -2030,13 +2065,16 @@ pub(crate) fn run_against_source<'a>( ] .into_iter() .flat_map(|(list, len)| (0..len).map(move |index| (list, index))) + .filter(|&(list, index)| !(matches!(list, EditList::Text) && withdrawn.contains(&index))) .map(|(list, index)| authorship.author_of(list, index).into_iter().collect()) .chain(rerendered_by) .collect(); let sub_edits: Vec<(usize, usize, SubPatch)> = ctx .text_edits .into_iter() - .map(|(r, s)| { + .enumerate() + .filter(|(index, _)| !withdrawn.contains(index)) + .map(|(_, (r, s))| { ( usize::from(r.start()), usize::from(r.end()), @@ -2162,7 +2200,12 @@ pub(crate) fn run_against_source<'a>( // their source ranges for sibling-pass composition); hoist them to the // module top now that lowering is done, so a member called before its // block's position still resolves - let (out, table) = extension::hoist_backing_functions(out, table, preamble_end); + let (out, table) = extension::hoist_backing_functions( + out, + table, + preamble_end, + &ctx.extension_backing_functions, + ); (Cow::Owned(out), ctx.errors, table) } @@ -2364,7 +2407,7 @@ mod driver_tests { src, repeated_underscore::WrittenNames::new(src), &Config::test_default(), - None, + SemanticSource::Fresh(ruff_python_ast::PythonVersion::latest_ty()), ); assert!(!out.contains("??"), "still has ??: {out}"); } @@ -2444,6 +2487,16 @@ mod subsumed_lowerings { /// the declared pairs shown written somewhere other than a case here, by the declaring /// pass's name: the test of the pass's own that shows it, or why it holds const SHOWN_ELSEWHERE: &[(&str, Lowering, &str)] = &[ + ( + "TupleLiteralTypePass", + Lowering::SymbolicTypeOp, + "nested_in_subscript", + ), + ( + "CallableSyntaxPass", + Lowering::SymbolicTypeOp, + "tuple_repetition_unpacked_in_tuple_type", + ), ( "GenericPolyfillPass", Lowering::MatchType, @@ -2551,6 +2604,11 @@ mod subsumed_lowerings { Lowering::LiteralType, "protocol_int_literal_argument", ), + ( + "ParametricIsPass", + Lowering::OptionalType, + "an_optional_target_tests_for_none_too", + ), // the runtime-union lowering runs only below 3.10, where the cases above never go ( "CallableSyntaxPass", diff --git a/crates/by_transforms/src/transforms/auto_quote.rs b/crates/by_transforms/src/transforms/auto_quote.rs index 5608faf06f..f909b96dfa 100644 --- a/crates/by_transforms/src/transforms/auto_quote.rs +++ b/crates/by_transforms/src/transforms/auto_quote.rs @@ -17,9 +17,11 @@ //! not by the point the annotation runs — defined further down, the class the //! annotation sits in, or imported only under `if TYPE_CHECKING:` //! -//! annotation quoting is skipped when annotations are not evaluated eagerly: -//! on python >= 3.14 they are deferred natively (PEP 649), and a user-written -//! or opt-in `from __future__ import annotations` defers every one. a class +//! annotation quoting is skipped when annotations are not evaluated eagerly +//! ([`crate::annotations_evaluated`]): on python >= 3.14 they are deferred +//! natively (PEP 649), and a `from __future__ import annotations` — written by +//! the user, opted into, or written for a target below 3.10 — defers every one, +//! where a quote would reach `__annotations__` as a string inside a string. a class //! *base*, and a value-position subscript in a class body (`list[A]()`), //! evaluates while the class is being built on every version, so a //! self-reference there is always quoted. a direct base (`class A(A):`) is @@ -33,7 +35,7 @@ //! cannot be a string, so there only the self-reference itself is quoted use ruff_python_ast::visitor::{Visitor, walk_expr}; -use ruff_python_ast::{AnyParameterRef, Expr, ExprName, PythonVersion, Stmt, StmtClassDef}; +use ruff_python_ast::{AnyParameterRef, Expr, ExprName, Stmt, StmtClassDef}; use ruff_text_size::{Ranged, TextRange}; use super::ast_driver::{Fragment, PassContext, TypeAwarePass}; @@ -42,17 +44,11 @@ use crate::type_info::TypeInfo; pub(crate) struct AutoQuote<'src> { source: &'src str, - min_version: PythonVersion, - inject_future: bool, } impl<'src> AutoQuote<'src> { - pub(crate) fn new(source: &'src str, min_version: PythonVersion, inject_future: bool) -> Self { - Self { - source, - min_version, - inject_future, - } + pub(crate) fn new(source: &'src str) -> Self { + Self { source } } } @@ -64,13 +60,10 @@ impl TypeAwarePass for AutoQuote<'_> { } fn run(&self, stmts: &[Stmt], types: &dyn TypeInfo, ctx: &mut PassContext) { - let quote_annotations = !(self.min_version.defers_annotations() - || self.inject_future - || has_future_annotations(stmts)); let mut walk = Walk { source: self.source, types, - quote_annotations, + quote_annotations: ctx.annotations_evaluated, edits: Vec::new(), }; walk.block(stmts, Scope::Module); @@ -78,14 +71,6 @@ impl TypeAwarePass for AutoQuote<'_> { } } -fn has_future_annotations(stmts: &[Stmt]) -> bool { - stmts.iter().any(|s| { - matches!(s, Stmt::ImportFrom(node) - if node.module.as_deref() == Some("__future__") - && node.names.iter().any(|a| a.name.as_str() == "annotations")) - }) -} - /// the kind of scope a block of statements runs in #[derive(Clone, Copy, PartialEq, Eq)] enum Scope { @@ -768,6 +753,33 @@ mod tests { ); } + /// below 3.10 the output defers every annotation with the future import, and a + /// quote there would reach `__annotations__` as a string inside a string. a class + /// base is still evaluated as the class is built, so its self-reference is quoted + #[test] + fn not_quoted_where_the_future_import_is_written() { + let config = Config { + min_version: PythonVersion::PY39, + ..Config::test_default() + }; + let out = transpile_with( + "class A(list[A]):\n item: A\n def f(self, other: A) -> A: ...\n", + &config, + ); + assert!( + out.starts_with("from __future__ import annotations\n"), + "should write the future import, got: {out}" + ); + assert!( + out.contains("item: A\n") && out.contains("def f(self, other: A) -> A:"), + "should leave annotations bare under the written future import, got: {out}" + ); + assert!( + out.contains("class A(list[\"A\"]):"), + "a base self-ref must stay quoted, got: {out}" + ); + } + /// nothing in a stub is evaluated, and a checker reads a forward reference in /// one without quotes #[test] diff --git a/crates/by_transforms/src/transforms/callable.rs b/crates/by_transforms/src/transforms/callable.rs index 717aa44acd..38236d4930 100644 --- a/crates/by_transforms/src/transforms/callable.rs +++ b/crates/by_transforms/src/transforms/callable.rs @@ -19,12 +19,14 @@ //! and the annotation becomes `_Callable_` use std::collections::HashMap; -use std::collections::hash_map::DefaultHasher; use std::fmt::Write as _; -use std::hash::{Hash, Hasher}; +use std::hash::Hash; use ruff_diagnostics::{Edit, Fix}; -use ruff_python_ast::helpers::{is_classvar_marker_id, is_final_marker_id, is_let_marker_id}; +use ruff_python_ast::helpers::{ + ReturnGuardForm, callable_parameter_label, is_classvar_marker_id, is_final_marker_id, + is_let_marker_id, predicate_guard, +}; use ruff_python_ast::{Expr, ExprCallableType, Operator, PythonVersion, Stmt, UnaryOp}; use ruff_text_size::{Ranged, TextRange}; @@ -37,7 +39,8 @@ use crate::config::{Config, FloatLiteralLowering}; use crate::type_info::{TypeInfo, UnpackedKwargsLowering}; /// the lowerings [`CallableSyntax::lower_type_expr`] writes itself, wherever it prints a type -/// expression. a pass that prints one through it in place of the source writes these too +/// expression, a symbolic fold among them when it is handed the fold as a substitution. a +/// pass that prints one through it in place of the source writes these too pub(crate) const TYPE_EXPRESSION: &[Lowering] = &[ Lowering::Callable, Lowering::OptionalType, @@ -45,6 +48,7 @@ pub(crate) const TYPE_EXPRESSION: &[Lowering] = &[ Lowering::JustFloat, Lowering::DynamicKeyword, Lowering::FloatConst, + Lowering::SymbolicTypeOp, ]; #[expect( @@ -82,6 +86,7 @@ pub(crate) struct CallableSyntax<'src> { needs_typeof_import: bool, needs_not_import: bool, needs_annotated_import: bool, + needs_type_is_import: bool, needs_optional_runtime: bool, /// shape → synthesized class name. used to dedupe identical /// non-denotable callable shapes @@ -129,6 +134,7 @@ impl<'src> CallableSyntax<'src> { needs_typeof_import: false, needs_not_import: false, needs_annotated_import: false, + needs_type_is_import: false, needs_optional_runtime: false, protocol_shapes: HashMap::new(), protocol_class_defs: String::new(), @@ -218,6 +224,7 @@ impl<'src> CallableSyntax<'src> { (self.needs_typeof_import, "ty_extensions", "TypeOf"), (self.needs_not_import, "ty_extensions", "Not"), (self.needs_annotated_import, "typing", "Annotated"), + (self.needs_type_is_import, "typing", "TypeIs"), (self.needs_union_import, "typing", "Union"), ] { if needed { @@ -263,6 +270,10 @@ impl<'src> CallableSyntax<'src> { /// `JustFloat` import. Falls back to verbatim source when no leaf rewrite /// applies (or no type info is available). fn lower_leaf(&mut self, expr: &Expr) -> String { + // a claimed fold this lowerer was handed, inside a wider edit it makes + if let Some(name) = self.substitution_for(expr.range()) { + return name.to_owned(); + } // a leaf that *contains* an already-lowered subtree is rendered by // sweeping the substitutions over its source instead: the per-leaf // composers below would re-render the subtree from source and leak the @@ -317,6 +328,34 @@ impl<'src> CallableSyntax<'src> { out } + /// the return type of `ct` as python spells it. a narrowing predicate, + /// `(x: object) -> (x is int)`, is `TypeIs[int]` where it names one of the callable's + /// labelled parameters, and otherwise the `bool` the callable returns, as it is for a + /// `def`: `TypeIs` has no spelling for a member or for a place outside the parameters + pub(crate) fn render_return(&mut self, ct: &ExprCallableType) -> String { + let Some(guard) = predicate_guard(&ct.returns) else { + return self.rewrite_or_leaf(&ct.returns); + }; + let ReturnGuardForm::Predicate { ty } = guard.form else { + return self.rewrite_or_leaf(&ct.returns); + }; + let (name, members) = guard.place_parts(); + let names_a_parameter = members.is_empty() + && ct + .args + .iter() + .any(|argument| callable_parameter_label(argument) == Some(name)); + if !names_a_parameter { + return self.written.builtin("bool"); + } + self.needs_type_is_import = true; + format!( + "{}[{}]", + self.written.imported("typing", "TypeIs"), + self.rewrite_or_leaf(ty) + ) + } + /// `rewrite` the expression if it is a callable structural form this pass /// owns (callable arrow, `&` / `not` / `?`, subscript, …); otherwise lower /// it as a leaf so per-leaf rewrites inside it still apply. @@ -551,11 +590,7 @@ impl<'src> CallableSyntax<'src> { if let Some(name) = self.protocol_shapes.get(&shape) { return name.clone(); } - let mut hasher = DefaultHasher::new(); - shape.hash(&mut hasher); - #[expect(clippy::cast_possible_truncation)] - let truncated = hasher.finish() as u32; - let name = format!("_Callable_{truncated:08x}"); + let name = self.written.synthesized_class("_Callable_", &shape); self.protocol_shapes.insert(shape.clone(), name.clone()); let _ = writeln!( self.protocol_class_defs, @@ -570,7 +605,8 @@ impl<'src> CallableSyntax<'src> { fn rewrite(&mut self, expr: &Expr) -> Option { // a symbolic-fold-claimed sub-expression is opaque: the fold emits its // own edit over this exact range, so re-rendering here (and clobbering - // it with a wider edit) must not happen + // it with a wider edit) must not happen. a wider edit this lowerer makes + // anyway still writes it as the fold, through `lower_leaf` if self.claimed_ranges.contains(&expr.range()) { return None; } @@ -583,7 +619,7 @@ impl<'src> CallableSyntax<'src> { Expr::CallableType(ct) if self.paramspec_tail(ct).is_some() => { self.needs_import = true; let (prefix, paramspec) = self.paramspec_tail(ct)?; - let ret_str = self.rewrite_or_leaf(&ct.returns); + let ret_str = self.render_return(ct); let ps = self.rewrite_or_leaf(paramspec); // an implicit receiver is the callable's leading positional // parameter, so it joins the `Concatenate` prefix @@ -616,7 +652,7 @@ impl<'src> CallableSyntax<'src> { ) }); let params = self.render_protocol_params(ct, false, "self", implicit_receiver); - let returns = quote_forward_ref(&self.rewrite_or_leaf(&ct.returns)); + let returns = quote_forward_ref(&self.render_return(ct)); let shape = ProtocolShape { params, returns }; Some(self.class_name_for(shape)) } @@ -626,31 +662,25 @@ impl<'src> CallableSyntax<'src> { // a receiver in front of a gradual parameter list is absorbed by it: // `Callable[..., R]` already accepts the receiver-first call, and // `Concatenate[T, ...]` is not spellable on every supported version - Expr::CallableType(ExprCallableType { args, returns, .. }) - if matches!(args.as_slice(), [Expr::EllipsisLiteral(_)]) => - { + Expr::CallableType(ct) if matches!(ct.args.as_slice(), [Expr::EllipsisLiteral(_)]) => { self.needs_import = true; - let ret_str = self.rewrite_or_leaf(returns); + let ret_str = self.render_return(ct); Some(format!( "{}[..., {ret_str}]", self.written.imported("typing", "Callable") )) } - Expr::CallableType(ExprCallableType { - receiver, - args, - returns, - .. - }) => { + Expr::CallableType(ct) => { self.needs_import = true; - let mut rendered: Vec = receiver + let mut rendered: Vec = ct + .receiver .iter() .map(|receiver| self.rewrite_or_leaf(receiver)) .collect(); - rendered.extend(args.iter().map(|a| self.rewrite_or_leaf(a))); + rendered.extend(ct.args.iter().map(|a| self.rewrite_or_leaf(a))); let args_str = rendered.join(", "); - let ret_str = self.rewrite_or_leaf(returns); + let ret_str = self.render_return(ct); Some(format!( "{}[[{args_str}], {ret_str}]", self.written.imported("typing", "Callable") @@ -1105,8 +1135,9 @@ impl TypeAwarePass for CallableSyntaxPass<'_> { } /// a type expression it replaces is printed by the shared lowerer, which writes every - /// leaf inside it too, an inline protocol as the name of the class it hoists to, and a - /// union the runtime evaluates in the spelling the target can run + /// leaf inside it too, an inline protocol as the name of the class it hoists to, a + /// union the runtime evaluates in the spelling the target can run, and an operation + /// `symbolic_type_op` folded as the fold fn subsumes(&self) -> &'static [Lowering] { &[ Lowering::Callable, @@ -1117,6 +1148,7 @@ impl TypeAwarePass for CallableSyntaxPass<'_> { Lowering::FloatConst, Lowering::ProtocolType, Lowering::RuntimeUnion, + Lowering::SymbolicTypeOp, ] } @@ -1129,6 +1161,11 @@ impl TypeAwarePass for CallableSyntaxPass<'_> { .with_types(types) .with_claimed_ranges(&claimed) .with_owned_typeof(&structural_typeof); + // a fold inside a callable type this pass replaces is written as the fold; one + // it does not replace keeps its own edit + for (range, rendered) in &ctx.symbolic_substitutions { + inner.add_substitution(*range, rendered.clone()); + } crate::transforms::type_expr_walker::walk_type_positions_skipping( stmts, Some(types), @@ -1192,6 +1229,29 @@ mod tests { assert_eq!(transpile(input, &config).unwrap(), expected); } + /// a callable type returns a narrowing predicate as a `def` does: `TypeIs` where it names + /// one of the callable's parameters, and the `bool` the callable returns where it names a + /// place `TypeIs` has no spelling for. neither is a type test of its own + #[test] + fn a_predicate_a_callable_type_returns_is_its_return_type() { + let out = transpile( + indoc! {" + def f(check: (first: object, second: object) -> (second is int)): ... + def g(check: (object) -> (a is int)): ... + "}, + &Config::test_default(), + ) + .unwrap(); + for expected in [ + "from typing_extensions import TypeIs\n", + "def __call__(self, first: \"object\", second: \"object\") -> \"TypeIs[int]\": ...", + "def g(check: Callable[[object], bool]): ...", + ] { + assert!(out.contains(expected), "expected `{expected}` in:\n{out}"); + } + assert!(!out.contains("isinstance"), "got:\n{out}"); + } + /// a type written where the runtime evaluates it is spelled with `Union` below 3.10, /// at any depth, whether the union was written with `|`, `or` or `?` #[test] diff --git a/crates/by_transforms/src/transforms/conformance.rs b/crates/by_transforms/src/transforms/conformance.rs index ed5cd9a5ac..8582faf5a4 100644 --- a/crates/by_transforms/src/transforms/conformance.rs +++ b/crates/by_transforms/src/transforms/conformance.rs @@ -31,7 +31,6 @@ use std::collections::BTreeSet; use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; use ruff_python_ast::{Expr, Stmt}; use ruff_text_size::{Ranged, TextRange}; -use ty_python_semantic::ConversionImport; use super::ast_driver::{Fragment, PassContext, TypeAwarePass}; use super::extension::spine_has_optional; @@ -50,19 +49,6 @@ pub(crate) const WITNESS_HELPERS: &[crate::runtime::Helper] = &[ crate::runtime::WITNESS_GET, ]; -/// the `from import as ` a cross-module interface -/// spelling needs -fn import_line(import: &ConversionImport) -> String { - if import.alias == import.name { - format!("from {} import {}", import.module, import.name) - } else { - format!( - "from {} import {} as {}", - import.module, import.name, import.alias - ) - } -} - /// the `_by_conform(...)` registrations a conformance extension emits, appended /// to its block's lowering so they run once the backing functions and the /// interface are both bound. @@ -85,7 +71,7 @@ pub(crate) fn registration_fragments( let mut first = !had_members; for registration in registrations { if let Some(import) = ®istration.import { - ctx.required_imports.push(import_line(import)); + ctx.required_imports.push(import.statement()); } ctx.required_imports.extend(registration.imports); if !first { @@ -143,7 +129,7 @@ impl<'ast> Visitor<'ast> for WitnessDispatchLower<'_> { )); } else { if let Some(import) = &dispatch.import { - self.imports.insert(import_line(import)); + self.imports.insert(import.statement()); } let dispatcher = match dispatch.kind { ty_python_semantic::WitnessKind::Property => "_by_witness_get(", diff --git a/crates/by_transforms/src/transforms/conversion.rs b/crates/by_transforms/src/transforms/conversion.rs index 122ae80f80..fa51a154e5 100644 --- a/crates/by_transforms/src/transforms/conversion.rs +++ b/crates/by_transforms/src/transforms/conversion.rs @@ -26,7 +26,9 @@ use ruff_python_ast::visitor::Visitor; use ruff_python_ast::{self as ast, Expr, Stmt}; use ruff_text_size::{Ranged, TextRange, TextSize}; -use ty_python_semantic::{ConversionInfo, ConversionRuntime, PreludeDunderReceiver}; +use ty_python_semantic::{ + ConversionImport, ConversionInfo, ConversionRuntime, PreludeDunderReceiver, +}; use super::ast_driver::{Fragment, PassContext, TypeAwarePass}; use crate::type_info::TypeInfo; @@ -149,21 +151,8 @@ impl TypeAwarePass for ConversionPass<'_> { } None => {} } - for import in imports { - // always aliased: the class's own name may already mean - // something else here, and an import that rebinds it — or that - // this file's own class then shadows — would send the call to - // the wrong object at runtime - if import.alias == import.name { - ctx.required_imports - .push(format!("from {} import {}", import.module, import.name)); - } else { - ctx.required_imports.push(format!( - "from {} import {} as {}", - import.module, import.name, import.alias - )); - } - } + ctx.required_imports + .extend(imports.iter().map(ConversionImport::statement)); // python binds a class name when its statement executes, so a // conversion that runs at import time cannot precede the class it // converts through. inside a function body the name is resolved at diff --git a/crates/by_transforms/src/transforms/decorator_keyword.rs b/crates/by_transforms/src/transforms/decorator_keyword.rs index d0a04a6562..8b550a498f 100644 --- a/crates/by_transforms/src/transforms/decorator_keyword.rs +++ b/crates/by_transforms/src/transforms/decorator_keyword.rs @@ -150,7 +150,9 @@ impl AstPass for DecoratorKeyword<'_> { errors: RefCell::new(Vec::new()), needs_callable: false, needs_overload: false, + synthesized_modules: RefCell::new(Vec::new()), synthesized_imports: RefCell::new(Vec::new()), + annotations_evaluated: ctx.annotations_evaluated, class_depth: 0, }; for stmt in &module.body { @@ -164,15 +166,19 @@ impl AstPass for DecoratorKeyword<'_> { ctx.required_imports .push(self.written.import_from("typing", &["overload"])); } - // the emitted overloads are read and never run, so a name only they need is - // imported where only a checker sees it ctx.type_only_imports + .extend(state.synthesized_modules.into_inner()); + ctx.required_imports .extend(state.synthesized_imports.into_inner()); ctx.template_edits.extend(state.templates.into_inner()); ctx.errors.extend(state.errors.into_inner()); } } +#[expect( + clippy::struct_excessive_bools, + reason = "independent facts about the module and the imports the visit needs" +)] struct State<'src> { source: &'src str, written: WrittenNames<'src>, @@ -182,8 +188,13 @@ struct State<'src> { errors: RefCell>, needs_callable: bool, needs_overload: bool, - /// the imports a written-out return type names that the source never wrote + /// the modules a written-out return type qualifies a class with that the source never + /// imported under their own names + synthesized_modules: RefCell>, + /// the imports of the `typing` names a written-out return type reads synthesized_imports: RefCell>, + /// whether python evaluates an overload's annotations as it is defined + annotations_evaluated: bool, class_depth: u32, } @@ -287,11 +298,15 @@ impl State<'_> { // says less than the truth rather than something else let inferred_return = self.return_types.by_function.get(&func.range()); if let Some(inferred) = inferred_return { + self.synthesized_modules + .borrow_mut() + .extend(inferred.modules.iter().cloned()); self.synthesized_imports .borrow_mut() - .extend(inferred.imports_in(self.written)); + .extend(inferred.typing_imports_in(self.written)); } - let inferred_return = inferred_return.map(|inferred| inferred.text_in(self.written)); + let inferred_return = inferred_return + .map(|inferred| inferred.annotation_in(self.written, self.annotations_evaluated)); let return_type = |frags: &mut Vec| match (&func.returns, &inferred_return) { (Some(returns), _) => frags.push(Fragment::Src(returns.range())), (None, Some(inferred)) => frags.push(Fragment::Lit(inferred.clone())), diff --git a/crates/by_transforms/src/transforms/extension.rs b/crates/by_transforms/src/transforms/extension.rs index ae736a5772..0da20d51ac 100644 --- a/crates/by_transforms/src/transforms/extension.rs +++ b/crates/by_transforms/src/transforms/extension.rs @@ -286,10 +286,13 @@ impl<'a> ExtensionBlockPass<'a> { fragments.push(Fragment::Lit("\n".to_owned())); } - fragments.push(Fragment::Lit(format!( - "def {}(", - backing_name(target, ordinal, func.name.as_str()) - ))); + // every call of the member was rewritten to the name ty gives it, so the + // backing function is declared under that name too + let name = types + .extension_backing_function(class, func.name.as_str()) + .unwrap_or_else(|| backing_name(target, ordinal, func.name.as_str())); + fragments.push(Fragment::Lit(format!("def {name}("))); + ctx.extension_backing_functions.insert(name); parameter_fragments(&func.parameters, self.written, &mut fragments); fragments.push(Fragment::Lit(")".to_owned())); @@ -392,9 +395,11 @@ impl TypeAwarePass for ExtensionBlockPass<'_> { /// compose their edits inside them. but a member may be *called* before that /// position — a plain top-level call resolves left-to-right, so the `def` must /// precede every use. this runs on the finished output text: it re-parses, -/// finds the top-level `_by_ext…` functions, and moves their line blocks to -/// just after the preamble and the leading imports, permuting the line table -/// identically so source maps stay aligned +/// finds the top-level functions named in `backing`, the names the block +/// lowering declared them under — each one the module spells nowhere, so none +/// is the module's own — and moves their line blocks to just after the +/// preamble and the leading imports, permuting the line table identically so +/// source maps stay aligned /// /// `preamble_end` is one past the last byte of the generated preamble. the /// defs have to stay below it: the preamble carries the runtime helpers a @@ -404,9 +409,9 @@ pub(crate) fn hoist_backing_functions( out: String, table: Vec>, preamble_end: usize, + backing: &BTreeSet, ) -> (String, Vec>) { - // cheap early-out: no backing functions were emitted - if !out.contains(EXTENSION_MARKER) { + if backing.is_empty() { return (out, table); } @@ -436,10 +441,11 @@ pub(crate) fn hoist_backing_functions( } // byte ranges of the top-level backing functions (decorators included) + let backing_names = backing; let mut backing: Vec<(usize, usize)> = Vec::new(); for stmt in module { if let Stmt::FunctionDef(func) = stmt - && func.name.starts_with("_by_ext") + && backing_names.contains(func.name.as_str()) { let start = func .decorator_list @@ -620,9 +626,8 @@ impl<'a> ExtensionCallLower<'a> { } fn note_import(&mut self, info: &ty_python_semantic::ExtensionAttributeInfo) { - if let Some(module) = &info.import_from { - self.imports - .insert(format!("from {module} import {}", info.function)); + if let Some(import) = &info.import { + self.imports.insert(import.statement()); } } @@ -1022,6 +1027,56 @@ mod tests { ); } + /// a backing function is named past what the module spells, as every name a lowering + /// binds is: the module's own `_by_ext__list__second` keeps its name, and the member moves + /// on to the next ordinal, which the reverse transform reads past + #[test] + fn a_backing_function_name_the_module_spells_is_passed_over() { + let out = check( + "def _by_ext__list__second(xs: list[int]) -> str:\n return \"mine\"\n\nextension list:\n def second(self) -> Element:\n return self[1]\n\nprint(_by_ext__list__second([1]))\nprint([1, 2].second())\n", + ); + assert!( + out.contains("def _by_ext2__list__second(self):"), + "got:\n{out}" + ); + assert!( + out.contains("print(_by_ext__list__second([1]))") + && out.contains("print(_by_ext2__list__second([1, 2]))"), + "got:\n{out}" + ); + } + + /// a `private` symbol is emitted under a name with a leading underscore, which is past what + /// the module spells too, so it is kept clear of the backing functions' names + #[test] + fn a_private_symbol_is_kept_clear_of_a_backing_function() { + let out = check( + "private def by_ext__list__second() -> str:\n return \"private\"\n\nextension list:\n def second(self) -> Element:\n return self[1]\n\nprint(by_ext__list__second(), [1, 2].second())\n", + ); + assert!( + out.contains("def _by_ext__list__second(self):") + && out.contains("def _by_ext__list__second2() -> str:"), + "got:\n{out}" + ); + assert!( + out.contains("print(_by_ext__list__second2(), _by_ext__list__second([1, 2]))"), + "got:\n{out}" + ); + } + + /// only the backing functions the block lowering declared move to the module top. a + /// function of the module's own whose name happens to start the same way stays where it + /// was written, under the decorator it reads + #[test] + fn a_function_of_the_modules_own_is_not_hoisted() { + let out = check( + "def my_deco(f):\n return f\n\n@my_deco\ndef _by_extract() -> int:\n return 1\n\nextension list:\n def second(self) -> Element:\n return self[1]\n\nprint(_by_extract(), [1, 2].second())\n", + ); + let deco = out.find("def my_deco").expect("the decorator"); + let own = out.find("@my_deco").expect("the decorated function"); + assert!(deco < own, "got:\n{out}"); + } + #[test] fn unapplied_method_reference_becomes_partial() { let out = check( diff --git a/crates/by_transforms/src/transforms/generics.rs b/crates/by_transforms/src/transforms/generics.rs index cf77de1355..594375bc84 100644 --- a/crates/by_transforms/src/transforms/generics.rs +++ b/crates/by_transforms/src/transforms/generics.rs @@ -330,20 +330,25 @@ impl<'src> GenericPolyfill<'src> { } } - /// pre-scan for `private type` aliases so a reference inside a later - /// alias's value can be renamed as the value is re-rendered + /// pre-scan for the module's `private type` aliases so a reference inside a + /// later alias's value can be renamed as the value is re-rendered. like the + /// module's other private symbols, they are the ones declared outside any class + /// or function body fn collect_private_aliases(&mut self, stmts: &[Stmt]) { struct Collect<'a, 'src>(&'a mut HashMap, WrittenNames<'src>); impl<'ast> Visitor<'ast> for Collect<'_, '_> { fn visit_stmt(&mut self, stmt: &'ast Stmt) { - if let Stmt::TypeAlias(alias) = stmt - && alias.is_private - && let Expr::Name(name) = alias.name.as_ref() - { - self.0 - .insert(name.id.to_string(), self.1.module_private(&name.id)); + match stmt { + Stmt::TypeAlias(alias) + if alias.is_private + && let Expr::Name(name) = alias.name.as_ref() => + { + self.0 + .insert(name.id.to_string(), self.1.module_private(&name.id)); + } + Stmt::ClassDef(_) | Stmt::FunctionDef(_) => {} + _ => ruff_python_ast::visitor::walk_stmt(self, stmt), } - ruff_python_ast::visitor::walk_stmt(self, stmt); } } let mut collect = Collect(&mut self.private_aliases, self.written); @@ -989,8 +994,16 @@ impl<'src> GenericPolyfill<'src> { // this replacement subsumes `modifiers`' `private ` deletion and the // rename of the definition site, so the private name has to be applied - // here instead - let name_src = if alias.is_private { + // here instead: a module's own symbol as the module renames it, and a + // class's own member as the class body spells it. any other alias keeps + // its name, as the native `type` statement of a class body does + let class_member_name = match alias.name.as_ref() { + Expr::Name(name) => self.types.class_body_member_name(name), + _ => None, + }; + let name_src = if let Some(renamed) = class_member_name { + renamed + } else if alias.is_private && self.enclosing_renames.is_empty() { self.written.module_private(self.src(alias.name.range())) } else { self.src(alias.name.range()).to_owned() @@ -998,8 +1011,10 @@ impl<'src> GenericPolyfill<'src> { let raw_value_src = self.src(alias.value.range()).to_owned(); // references to a `private type` alias declared elsewhere in the module - // also sit inside the subsumed value, so they are renamed here too + // also sit inside the subsumed value, so they are renamed here too, and so + // is a reference to a restricted member of the class the alias is in let mut rename_map = self.private_aliases.clone(); + rename_map.extend(class_member_references(&alias.value, self.types)); let (type_params_arg, defs) = if let Some(tp) = &alias.type_params && let Some(names) = names @@ -1039,7 +1054,7 @@ impl<'src> GenericPolyfill<'src> { self.written.builtin("object") ); self.edits.push(Fix::safe_edit(Edit::range_replacement( - replacement, + at_statement_start(&replacement, &indent), alias.range(), ))); return; @@ -1092,12 +1107,19 @@ impl<'src> GenericPolyfill<'src> { ); self.edits.push(Fix::safe_edit(Edit::range_replacement( - replacement, + at_statement_start(&replacement, &indent), alias.range(), ))); } } +/// `lines`, each written with the statement's `indent`, as a replacement for a statement: +/// its range starts after the indent the source already wrote, so the first line leaves +/// its own out +fn at_statement_start(lines: &str, indent: &str) -> String { + lines.strip_prefix(indent).unwrap_or(lines).to_owned() +} + impl GenericPolyfill<'_> { /// Rewrites a tuple slice of a parameters-typed subscript to a list. /// `A[(int, str)]` → `A[[int, str]]` so the runtime `ParamSpec` accepts @@ -2088,8 +2110,14 @@ impl super::ast_driver::TypeAwarePass for GenericPolyfillPass<'_> { // an edit we re-rendered with the typevar rename applied has to lose its // original, or the un-renamed text can still win the overlap race let superseded = std::mem::take(&mut inner.superseded); - ctx.text_edits - .retain(|(range, _)| !superseded.contains(range)); + let withdrawn: Vec = ctx + .text_edits + .iter() + .enumerate() + .filter(|(_, (range, _))| superseded.contains(range)) + .map(|(index, _)| index) + .collect(); + ctx.withdrawn_text_edits.extend(withdrawn); let emits_any = inner.needed_imports_any; for line in std::mem::take(&mut inner.needed_imports).into_lines(&inner.constructors, self.written) @@ -2114,6 +2142,31 @@ impl super::ast_driver::TypeAwarePass for GenericPolyfillPass<'_> { } } +/// each name `value`, the value of a type alias, reads that is a restricted member of the +/// class the alias is declared in, with the spelling a read of it takes +fn class_member_references(value: &Expr, types: &dyn TypeInfo) -> HashMap { + struct Collect<'a> { + types: &'a dyn TypeInfo, + found: HashMap, + } + impl<'ast> Visitor<'ast> for Collect<'_> { + fn visit_expr(&mut self, expr: &'ast Expr) { + if let Expr::Name(name) = expr + && let Some(renamed) = self.types.class_body_member_name(name) + { + self.found.insert(name.id.to_string(), renamed); + } + walk_expr(self, expr); + } + } + let mut collect = Collect { + types, + found: HashMap::new(), + }; + collect.visit_expr(value); + collect.found +} + #[cfg(test)] mod tests { use crate::{Config, transpile}; diff --git a/crates/by_transforms/src/transforms/implicit_receiver.rs b/crates/by_transforms/src/transforms/implicit_receiver.rs index c748524b50..d917e00c6b 100644 --- a/crates/by_transforms/src/transforms/implicit_receiver.rs +++ b/crates/by_transforms/src/transforms/implicit_receiver.rs @@ -119,9 +119,8 @@ impl<'ast> Visitor<'ast> for ImplicitReceiverLower<'_> { // to read off the receiver — the reference is its // backing function bound to the block's receiver ImplicitReceiverReference::ExtensionMember(info) => { - if let Some(module) = &info.import_from { - self.imports - .insert(format!("from {module} import {}", info.function)); + if let Some(import) = &info.import { + self.imports.insert(import.statement()); } let (fragments, needs_functools) = super::extension::member_reference_fragments( diff --git a/crates/by_transforms/src/transforms/inferred_annotation.rs b/crates/by_transforms/src/transforms/inferred_annotation.rs index 01a43310e1..5fc3e0c64b 100644 --- a/crates/by_transforms/src/transforms/inferred_annotation.rs +++ b/crates/by_transforms/src/transforms/inferred_annotation.rs @@ -60,14 +60,17 @@ impl TypeAwarePass for InferredAnnotationPass<'_> { types, written: self.written, min_version: self.min_version, + evaluated: ctx.annotations_evaluated, edits: Vec::new(), + modules: BTreeSet::new(), imports: BTreeSet::new(), }; for stmt in stmts { state.visit_stmt(stmt); } ctx.text_edits.extend(state.edits); - ctx.type_only_imports.extend(state.imports); + ctx.type_only_imports.extend(state.modules); + ctx.required_imports.extend(state.imports); } } @@ -75,9 +78,13 @@ struct State<'a> { types: &'a dyn TypeInfo, written: WrittenNames<'a>, min_version: ruff_python_ast::PythonVersion, + /// whether python evaluates a class body's annotations as the class is made + evaluated: bool, edits: Vec<(TextRange, String)>, - /// the modules and `typing` names a synthesized annotation reads but the source never - /// imported, as the import lines that bind them + /// the modules a synthesized annotation qualifies a class with that the source never + /// imported under their own names + modules: BTreeSet, + /// the imports of the `typing` names a synthesized annotation reads imports: BTreeSet, } @@ -120,9 +127,14 @@ impl State<'_> { let pos = name.range().end(); self.edits.push(( TextRange::new(pos, pos), - format!(": {}", annotation.text_in(self.written)), + format!( + ": {}", + annotation.annotation_in(self.written, self.evaluated) + ), )); - self.imports.extend(annotation.imports_in(self.written)); + self.modules.extend(annotation.modules.iter().cloned()); + self.imports + .extend(annotation.typing_imports_in(self.written)); } } @@ -308,8 +320,10 @@ mod tests { #[test] fn a_class_the_file_does_not_bind_is_qualified() { // `Decimal` is bound here as `Dec`, so the bare name ty displays would - // be unresolved in the output — the spelling names the module instead, - // and the module is imported for the checker that reads it + // be unresolved in the output — the spelling names the module instead. + // the module may be one the source imports only for a checker, so the + // annotation is a string the class body does not evaluate, and with + // imports not deferred the module is imported for a checker alone check( indoc! {" from decimal import Decimal as Dec @@ -322,11 +336,50 @@ mod tests { import decimal from decimal import Decimal as Dec class A: - d: decimal.Decimal = Dec(1) + d: \"decimal.Decimal\" = Dec(1) "}, ); } + /// where imports are deferred, the module is imported with the others, so it runs only + /// when `typing.get_type_hints` reads the annotation + #[test] + fn a_module_a_synthesized_annotation_names_is_deferred_with_the_other_imports() { + let out = transpile( + indoc! {" + from decimal import Decimal as Dec + class A: + d = Dec(1) + "}, + &Config::default(), + ) + .unwrap(); + assert!(out.contains("decimal = _lazy_module(\"decimal\")"), "{out}"); + assert!(out.contains("d: \"decimal.Decimal\" = Dec(1)"), "{out}"); + assert!(!out.contains("TYPE_CHECKING"), "{out}"); + } + + /// below 3.10 the output defers every annotation, and from 3.14 python does, so the + /// annotation is written as it is + #[test] + fn a_deferred_synthesized_annotation_is_not_a_string() { + for min_version in [crate::PythonVersion::PY39, crate::PythonVersion::PY314] { + let out = transpile( + indoc! {" + from decimal import Decimal as Dec + class A: + d = Dec(1) + "}, + &Config { + min_version, + ..Config::test_default() + }, + ) + .unwrap(); + assert!(out.contains("d: decimal.Decimal = Dec(1)"), "{out}"); + } + } + /// and the block stays a block. it opens with a `from typing import TYPE_CHECKING`, /// and merging the imports by module treated the whole thing as one such line — every /// `typing` name that sorts after `TYPE_CHECKING` was then appended to the block's last @@ -349,7 +402,7 @@ mod tests { @final class A: - d: decimal.Decimal = Dec(1) + d: \"decimal.Decimal\" = Dec(1) "}, ); } @@ -386,9 +439,7 @@ mod tests { "}, // the import redirect retargets `Never` for the 3.10 default indoc! {" - from typing import TYPE_CHECKING - if TYPE_CHECKING: - from typing_extensions import Never + from typing_extensions import Never def boom(): raise ValueError class A: @@ -410,9 +461,7 @@ mod tests { n = boom() "}, indoc! {" - from typing import TYPE_CHECKING - if TYPE_CHECKING: - from typing_extensions import Never as Never2 + from typing_extensions import Never as Never2 Never = 4 def boom(): raise ValueError @@ -424,8 +473,7 @@ mod tests { /// ty reports the modules a *type* lives in, which is not the same question as which /// ones its spelling names: the type spelled `None` is reported as living in `types`, - /// and importing it bought the emitted file a `if TYPE_CHECKING: import types` block - /// that nothing in it read + /// and importing it bought the emitted file an `import types` that nothing in it read #[test] fn a_module_the_spelling_does_not_name_is_not_imported() { check( diff --git a/crates/by_transforms/src/transforms/lazy_import.rs b/crates/by_transforms/src/transforms/lazy_import.rs index e9ff18b49a..7b81e1c7f9 100644 --- a/crates/by_transforms/src/transforms/lazy_import.rs +++ b/crates/by_transforms/src/transforms/lazy_import.rs @@ -18,7 +18,8 @@ //! - relative imports (`from .pkg import x`) //! - `import a.b` without an alias (binds the top package, which //! `LazyLoader` does not register) -//! - bootstrap modules (`sys`, `importlib*`) — the helpers depend on them +//! - bootstrap modules (`sys`, `importlib*`) — the helpers depend on them — and +//! `builtins`, which is loaded before any module runs //! //! A multi-name `import a, b` mixing the two is split, keeping a plain import //! for the names that stay eager. @@ -154,8 +155,12 @@ impl<'src> LazyImport<'src> { self.eager.iter().any(|module| module == name) } + /// a module the polyfill leaves eager. the helpers run on `sys` and `importlib`, and + /// `builtins` is loaded before any module runs, so deferring it defers nothing — while + /// its proxy would stand where a lowering reads a builtin under a name of its own, and + /// fail an identity test such as `type(x) is str2` fn is_bootstrap(name: &str) -> bool { - matches!(name, "sys" | "importlib") || name.starts_with("importlib.") + matches!(name, "sys" | "importlib" | "builtins") || name.starts_with("importlib.") } fn process_import(&mut self, node: &StmtImport) { @@ -957,6 +962,25 @@ mod tests { assert_eq!(out, "import sys\n"); } + /// a builtin read under a name of its own is the builtin itself, which an identity test + /// against it — the class guard of a literal's type test — needs + #[test] + fn polyfill_builtins_import_unchanged() { + let out = transpile( + "class str:\n pass\n\ndef f(a: object) -> bool:\n return a is \"x\"\n", + &Config { + lazy_imports: true, + ..Config::test_default() + }, + ) + .unwrap(); + assert!( + out.starts_with("from builtins import str as str2\n") + && out.contains("return (type(a) is str2 and a == \"x\")"), + "{out}" + ); + } + #[test] fn polyfill_multi_name_splits_around_bootstrap() { // the rewrite replaces the whole statement, so `sys` has to come back diff --git a/crates/by_transforms/src/transforms/main_function.rs b/crates/by_transforms/src/transforms/main_function.rs index daa15e581a..5897f35110 100644 --- a/crates/by_transforms/src/transforms/main_function.rs +++ b/crates/by_transforms/src/transforms/main_function.rs @@ -54,7 +54,10 @@ impl AstPass for MainFunction<'_> { } ctx.epilogue.push("if __name__ == \"__main__\":".to_owned()); - let spec: Vec = entry.exposed().map(EntryParameter::spec_entry).collect(); + let spec: Vec = entry + .exposed() + .map(|parameter| parameter.spec_entry(self.written)) + .collect(); let extra = entry.extra_arguments; let call = if spec.is_empty() && extra.is_none() { "main()".to_owned() @@ -66,7 +69,7 @@ impl AstPass for MainFunction<'_> { ctx.epilogue.push(format!(" {entry},")); } let close = match extra { - Some(converter) => format!(" ], {converter})"), + Some(converter) => format!(" ], {})", converter.spelled(self.written)), None => " ])".to_owned(), }; ctx.epilogue.push(close); @@ -105,7 +108,7 @@ pub struct EntryPoint<'a> { pub parameters: Vec>, /// the converter for the arguments the interface does not claim, when a /// leading `*rest` asks for them — see `extra_arguments_converter` - pub extra_arguments: Option<&'static str>, + pub extra_arguments: Option, } impl EntryPoint<'_> { @@ -216,10 +219,15 @@ impl EntryParameter<'_> { /// the `(name, converter, kind, required, choices)` tuple `_by_main_args` /// consumes; only an exposed parameter has one - fn spec_entry(&self) -> String { + fn spec_entry(&self, written: WrittenNames) -> String { let (converter, choices) = match &self.spelling { - Some(spelling) => (spelling.converter.unwrap_or("None"), &spelling.choices), - None => ("None", &None), + Some(spelling) => ( + spelling + .converter + .map_or_else(|| "None".to_owned(), |converter| converter.spelled(written)), + &spelling.choices, + ), + None => ("None".to_owned(), &None), }; let required = if self.is_required() { "True" } else { "False" }; let name = self.name(); @@ -244,7 +252,7 @@ impl EntryParameter<'_> { pub struct CliSpelling { /// the callable that converts the argument, as the source names it; `None` /// for a `--name` / `--no-name` flag pair, which takes no value - pub converter: Option<&'static str>, + pub converter: Option, /// the values the annotation admits, when it is a literal union — argparse /// rejects anything else before `main` runs pub choices: Option>, @@ -333,12 +341,38 @@ fn parameters<'a>(params: &'a Parameters, written: WrittenNames) -> Vec &'static str { + match self { + Converter::Written(name) | Converter::Builtin(name) => name, + } + } + + fn spelled(self, written: WrittenNames) -> String { + match self { + Converter::Written(name) => name.to_owned(), + Converter::Builtin(name) => written.builtin(name), + } + } +} + /// The command-line spelling of an annotation — its converter and, for a /// literal union, the values it admits — or `None` when it has none. /// @@ -350,17 +384,17 @@ fn cli_type(annotation: &Expr) -> Option<(CliType, Option>)> { Expr::Name(name) => { let ty = match name.id.as_str() { "bool" => CliType::Flag, - "str" => CliType::Value("str"), - "int" => CliType::Value("int"), - "float" => CliType::Value("float"), - "Path" => CliType::Value("Path"), + "str" => CliType::Value(Converter::Written("str")), + "int" => CliType::Value(Converter::Written("int")), + "float" => CliType::Value(Converter::Written("float")), + "Path" => CliType::Value(Converter::Written("Path")), _ => return None, }; Some((ty, None)) } Expr::Attribute(attr) => (attr.attr.as_str() == "Path" && matches!(&*attr.value, Expr::Name(name) if name.id.as_str() == "pathlib")) - .then_some((CliType::Value("pathlib.Path"), None)), + .then_some((CliType::Value(Converter::Written("pathlib.Path")), None)), // `T?` — an absent argument is what the `None` stands for, so the // spelling is `T`'s Expr::UnaryOp(unary) if matches!(unary.op, ast::UnaryOp::Optional) => { @@ -393,7 +427,10 @@ fn cli_type(annotation: &Expr) -> Option<(CliType, Option>)> { // `T | None` — the `None` is what leaving the argument out means (Some(spelling), None) => Some(spelling), // every operand a literal of one kind: the values it admits - (None, Some(converter)) => Some((CliType::Value(converter), Some(literals))), + (None, Some(converter)) => Some(( + CliType::Value(Converter::Builtin(converter)), + Some(literals), + )), // a named type beside a literal (`int | "a"`) admits neither the // type's values nor the literal's, so nothing on the command // line could satisfy both. nothing but `None`s says nothing @@ -412,7 +449,10 @@ fn cli_type(annotation: &Expr) -> Option<(CliType, Option>)> { } literals.push(rendered); } - Some((CliType::Value(converter?), Some(literals))) + Some(( + CliType::Value(Converter::Builtin(converter?)), + Some(literals), + )) } _ => None, } @@ -431,13 +471,13 @@ fn cli_type(annotation: &Expr) -> Option<(CliType, Option>)> { /// vararg's annotation converts them, exactly as a declared parameter's does. /// An annotation with no command-line spelling has nothing to convert with, and /// the vararg goes back to being one nothing fills. -fn extra_arguments_converter(params: &Parameters) -> Option<&'static str> { +fn extra_arguments_converter(params: &Parameters) -> Option { let vararg = params.vararg.as_ref()?; if !(params.posonlyargs.is_empty() && params.args.is_empty()) { return None; } match vararg.annotation.as_deref() { - None => Some("str"), + None => Some(Converter::Builtin("str")), Some(annotation) => match cli_type(annotation) { Some((CliType::Value(converter), None)) => Some(converter), // a flag is not a value, and a literal union's `choices` have no @@ -879,6 +919,28 @@ mod tests { ); } + /// the converter for a literal's values is the lowering's choice, not a name the + /// source wrote, so it is the builtin even where the module binds the name itself + #[test] + fn a_literal_union_converts_with_the_builtin_the_module_shadows() { + let out = transpile( + indoc! {" + def str(o: object) -> int: + return 7 + + def main(mode: \"fast\" | \"slow\" = \"fast\"): + pass + "}, + &Config::test_default(), + ) + .unwrap(); + assert!( + out.starts_with("from builtins import str as str2\n") + && out.contains("(\"mode\", str2, \"any\", False, (\"fast\", \"slow\",)),"), + "got:\n{out}" + ); + } + #[test] fn an_optional_literal_union_keeps_its_values() { let out = guard("def main(mode: (\"fast\" | \"slow\")? = None):\n pass\n"); diff --git a/crates/by_transforms/src/transforms/match_polyfill.rs b/crates/by_transforms/src/transforms/match_polyfill.rs index 3828e08e98..980369ecf6 100644 --- a/crates/by_transforms/src/transforms/match_polyfill.rs +++ b/crates/by_transforms/src/transforms/match_polyfill.rs @@ -113,14 +113,19 @@ pub(crate) fn lower(source: String, config: &Config) -> String { .iter() .any(|(_, replacement)| replacement.contains(MISS)); let body = apply(&source, edits); - let mut preamble = String::new(); + let mut imports = String::new(); for import in written.builtin_imports(&body, config.min_version.minor) { - preamble.push_str(&import); - preamble.push('\n'); + imports.push_str(&import); + imports.push('\n'); } - preamble.push_str(&crate::runtime_preamble(config, &helpers(&needs, sentinel))); + // the runtime definitions the tests call are decided against the module with those + // imports in it, so a builtin they read is read under the same name, and spliced after + // them let at = preamble_offset(&body); - format!("{}{preamble}{}", &body[..at], &body[at..]) + let body = format!("{}{imports}{}", &body[..at], &body[at..]); + let runtime = crate::runtime_preamble(config, &helpers(&needs, sentinel), &body); + let at = preamble_offset(&body); + format!("{}{runtime}{}", &body[..at], &body[at..]) } /// Apply disjoint replacements, ascending by start. @@ -826,4 +831,42 @@ mod tests { "got:\n{out}" ); } + + /// a builtin the module binds is read under one name of the lowering's own, whichever + /// lowering reads it: the type test before this phase, and the tests and the runtime + /// definitions this one writes, all read the one `isinstance2`, imported once, ahead of + /// the definitions that read it + #[test] + fn a_builtin_is_read_under_one_name_by_every_lowering() { + let out = lowered(indoc! {r#" + def len(a: object) -> int: + return 99 + + def isinstance(a: object, b: object) -> str: + return "mine" + + def f(x: object): + print(x is int) + match x: + case [a, b]: + print(a, b) + case int(): + print("int") + "#}); + assert!( + !out.contains("isinstance3") && !out.contains("len3"), + "got:\n{out}" + ); + for import in [ + "from builtins import isinstance as isinstance2\n", + "from builtins import len as len2\n", + ] { + assert_eq!(out.matches(import).count(), 1, "`{import}` once in:\n{out}"); + let definitions = out.find("def _by_match_seq").expect("the sequence test"); + assert!( + out.find(import) < Some(definitions), + "imported first:\n{out}" + ); + } + } } diff --git a/crates/by_transforms/src/transforms/modifiers.rs b/crates/by_transforms/src/transforms/modifiers.rs index d2b14e9c92..79efa8428d 100644 --- a/crates/by_transforms/src/transforms/modifiers.rs +++ b/crates/by_transforms/src/transforms/modifiers.rs @@ -31,7 +31,7 @@ use ruff_python_ast::helpers::{ DeclarationMarker, DeclarationMarkerKind, MemberVisibility, declaration_marker_visibility, is_classvar_annot_marker_id, is_classvar_marker_id, is_final_marker_id, is_let_marker_id, }; -use ruff_python_ast::visitor::{Visitor, walk_stmt}; +use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; use ruff_python_ast::{ self as ast, Expr, Stmt, StmtAnnAssign, StmtClassDef, StmtFunctionDef, StmtTypeAlias, }; @@ -112,6 +112,10 @@ pub(crate) struct Modifiers<'src> { /// whether a `data class` is given its slots by the runtime helper, on a python whose /// `dataclass` has no `slots` option slots_by_helper: bool, + /// whether a `data class` whose methods reach the class through a `__class__` cell is + /// given its slots by the runtime helper, on a python whose `dataclass(slots=True)` + /// leaves that cell pointing at the class it replaced + slots_by_helper_for_class_cell: bool, needs_dataclass_slots: bool, /// Names marked `export`/`public` at module level. Used to generate `__all__`. exports: Vec, @@ -154,6 +158,7 @@ impl<'src> Modifiers<'src> { needs_classvar: false, needs_newtype: false, slots_by_helper: false, + slots_by_helper_for_class_cell: false, needs_dataclass_slots: false, exports: Vec::new(), private_renames: Vec::new(), @@ -294,7 +299,9 @@ impl<'src> Modifiers<'src> { self.needs_dataclass = true; let dataclass = self.written.imported("dataclasses", "dataclass"); let frozen = name == "frozen_data_class"; - let decorators = if self.slots_by_helper { + let decorators = if self.slots_by_helper + || (self.slots_by_helper_for_class_cell && reads_class_cell(&class.body)) + { self.needs_dataclass_slots = true; let options = if frozen { "(frozen=True)" } else { "" }; format!( @@ -463,12 +470,11 @@ impl<'src> Modifiers<'src> { /// `private type X = V` → `type _X = V`. the modifier keyword is not a /// synthetic decorator here (a type alias has no decorator list), it is a - /// flag on the node, so the prefix is erased from the source directly. the - /// definition site is renamed by [`NameRenamer`] along with every reference. + /// flag on the node, so the prefix is erased from the source directly. a + /// module's alias is renamed, at the definition site and every reference, + /// by the visibility rename, which reads the module's private symbols /// - /// unlike a `private` method, a nested alias gets the same `_` prefix rather - /// than `__` name-mangling: an alias is a type, not a member, and mangling - /// would not survive the `TypeAliasType` polyfill's whole-statement rewrite + /// an alias in a class body is not among them, so it keeps its name fn process_type_alias(&mut self, alias: &StmtTypeAlias) { if !alias.is_private { return; @@ -773,12 +779,37 @@ pub(crate) fn module_private_name(name: &str) -> String { format!("_{name}") } +/// whether any function in a class body can reach the class through a `__class__` +/// cell. python gives a class that cell when a function in its body names `super` or +/// `__class__`, so a body naming neither has none. a name anywhere in the body counts, +/// a nested class's included, which can only count a class that has no cell +fn reads_class_cell(body: &[Stmt]) -> bool { + struct Finder(bool); + impl<'a> Visitor<'a> for Finder { + fn visit_expr(&mut self, expr: &'a Expr) { + if let Expr::Name(name) = expr + && matches!(name.id.as_str(), "super" | "__class__") + { + self.0 = true; + } + walk_expr(self, expr); + } + } + let mut finder = Finder(false); + finder.visit_body(body); + finder.0 +} + pub(crate) struct ModifiersPass<'src> { source: &'src str, written: WrittenNames<'src>, is_stub: bool, /// whether the python the module targets has `dataclass(slots=True)` dataclass_slots: bool, + /// whether the `dataclass(slots=True)` of the python the module targets re-points + /// the `__class__` cell of the class's methods at the class it makes, which 3.13 + /// began doing. before it, zero-argument `super()` in a method raises a `TypeError` + dataclass_slots_keep_class_cell: bool, } impl<'src> ModifiersPass<'src> { @@ -788,6 +819,7 @@ impl<'src> ModifiersPass<'src> { written, is_stub: config.is_stub, dataclass_slots: config.min_version >= PythonVersion::PY310, + dataclass_slots_keep_class_cell: config.min_version >= PythonVersion::PY313, } } } @@ -807,6 +839,8 @@ impl AstPass for ModifiersPass<'_> { let mut inner = Modifiers::new(self.source, self.written); // a stub is never run, so it keeps the option a checker reads inner.slots_by_helper = !self.dataclass_slots && !self.is_stub; + inner.slots_by_helper_for_class_cell = + !self.dataclass_slots_keep_class_cell && !self.is_stub; for stmt in &module.body { inner.visit_stmt(stmt); } @@ -1177,6 +1211,65 @@ mod tests { ); } + /// below 3.13 `dataclass(slots=True)` leaves the `__class__` cell of the class's + /// methods pointing at the class it replaced, so a class whose methods name `super` or + /// `__class__` is given its slots by the helper, which re-points the cell + #[test] + fn data_class_reaching_its_class_cell_below_python_313() { + let source = indoc! {" + class Base: + def f(self) -> int: + return 1 + + data class Plain: + x: int + + data class Calls(Base): + x: int + + override def f(self) -> int: + return super().f() + 1 + + frozen data class Names: + x: int + + def me(self) -> type: + return __class__ + "}; + for (target, helper) in [ + (PythonVersion::PY310, true), + (PythonVersion::PY312, true), + (PythonVersion::PY313, false), + ] { + let out = transpile( + source, + &Config { + min_version: target, + ..Config::test_default() + }, + ) + .unwrap(); + assert!( + out.contains("@dataclass(slots=True)\nclass Plain:"), + "{target}: a class without the cell keeps the option:\n{out}" + ); + let calls = if helper { + "@_by_dataclass_slots\n@dataclass\nclass Calls(Base):" + } else { + "@dataclass(slots=True)\nclass Calls(Base):" + }; + let names = if helper { + "@_by_dataclass_slots\n@dataclass(frozen=True)\nclass Names:" + } else { + "@dataclass(frozen=True, slots=True)\nclass Names:" + }; + assert!( + out.contains(calls) && out.contains(names), + "{target}: got:\n{out}" + ); + } + } + #[test] fn nested_modifiers_in_class() { // Default min_version is PY310; `typing.override` was added in PY312 @@ -1804,6 +1897,70 @@ mod tests { ); } + /// below 3.10 the value's union is spelled `Union[...]`, an edit of the value that the + /// polyfill's replacement carries along with the `private ` deletion + #[test] + fn private_type_alias_with_a_union_polyfilled_below_310() { + check_at( + indoc! {" + private type Alias = int | str + + def f(x: Alias) -> Alias: + return x + "}, + indoc! {" + from __future__ import annotations + from typing import Union + from typing_extensions import TypeAliasType + _Alias = TypeAliasType(\"_Alias\", Union[int, str]) + + def f(x: _Alias) -> _Alias: + return x + "}, + PythonVersion::PY39, + ); + } + + /// a class body's alias is polyfilled at the class body's indentation, and a + /// `private` one is a private member there, as it is in the native `type` statement: + /// declared as `__W`, and read as the mangled `_A__W` + #[test] + fn class_body_type_alias_polyfilled() { + let source = indoc! {" + class A: + private type W = int + type V = list[W] + + def f(self, w: W) -> V: + return [w] + "}; + check_at( + source, + indoc! {" + from typing_extensions import TypeAliasType + class A: + __W = TypeAliasType(\"__W\", int) + V = TypeAliasType(\"V\", list[_A__W]) + + def f(self, w: _A__W) -> V: + return [w] + "}, + PythonVersion::PY310, + ); + check_at( + source, + indoc! {" + class A: + type __W = int + type V = list[_A__W] + + def f(self, w: _A__W) -> V: + return [w] + "}, + PythonVersion::PY312, + ); + } + /// `_force_unwrap` is the runtime helper `x!` calls, and `_count` the module's own #[test] fn a_private_symbol_is_emitted_under_a_name_the_module_does_not_have() { diff --git a/crates/by_transforms/src/transforms/parametric_is.rs b/crates/by_transforms/src/transforms/parametric_is.rs index 0fc33740ec..847bb19895 100644 --- a/crates/by_transforms/src/transforms/parametric_is.rs +++ b/crates/by_transforms/src/transforms/parametric_is.rs @@ -51,6 +51,7 @@ use std::collections::BTreeSet; use std::fmt::Write as _; +use ruff_python_ast::helpers::{negated_narrowing_predicate, predicate_guard}; use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; use ruff_python_ast::{self as ast, CmpOp, Expr, PySourceType, Stmt}; use ruff_python_trivia::{SimpleTokenKind, SimpleTokenizer}; @@ -362,7 +363,7 @@ fn fragments_for_plan( vec![ Fragment::Lit(format!("({}(", written.builtin("type"))), value_ref(), - Fragment::Lit(format!(") is {class} and ")), + Fragment::Lit(format!(") is {} and ", written.builtin(&class))), value_ref(), Fragment::Lit(" == ".to_owned()), target_fragment(&value, arm), @@ -702,6 +703,20 @@ impl<'ast> Visitor<'ast> for ParametricIs<'_, '_> { } fn visit_expr(&mut self, expr: &'ast Expr) { + // the `x is int` a callable type returns, `(x: object) -> (x is int)`, is the + // narrowing predicate the callable lowering writes as `TypeIs`, not a type test + if let Expr::CallableType(callable) = expr + && (predicate_guard(&callable.returns).is_some() + || negated_narrowing_predicate(&callable.returns).is_some()) + { + if let Some(receiver) = &callable.receiver { + self.visit_expr(receiver); + } + for argument in &callable.args { + self.visit_expr(argument); + } + return; + } if let Expr::Compare(compare) = expr { self.process_compare(compare); } @@ -725,9 +740,13 @@ impl TypeAwarePass for ParametricIsPass<'_> { Some(super::ast_driver::Lowering::ParametricIs) } - /// the type it tests against is printed as the runtime probe it builds, literals included + /// the type it tests against is printed as the runtime probe it builds, literals and + /// the `None` of an optional included fn subsumes(&self) -> &'static [super::ast_driver::Lowering] { - &[super::ast_driver::Lowering::LiteralType] + &[ + super::ast_driver::Lowering::LiteralType, + super::ast_driver::Lowering::OptionalType, + ] } fn run(&self, stmts: &[Stmt], types: &dyn TypeInfo, ctx: &mut PassContext) { @@ -1334,6 +1353,67 @@ mod tests { ); } + /// a literal's class guard names the builtin class the value has, which a module that + /// binds the class's name for itself has read under a name of its own + #[test] + fn a_literal_guard_reads_the_builtin_class_the_module_shadows() { + let out = out(indoc! {" + class str: + pass + + def f(a: object) -> bool: + return a is \"x\" + "}); + assert!( + out.contains("from builtins import str as str2") + && out.contains("return (type(a) is str2 and a == \"x\")"), + "{out}" + ); + } + + #[test] + fn an_optional_target_tests_for_none_too() { + // `int?` is the union `int | None`, so it lowers as that union does — its + // `?` is part of the target, not an operator over the comparison + let out = out(indoc! {" + def f(a: object) -> bool: + return a is int? + + def g(a: object) -> bool: + return a is not int? + "}); + assert!( + out.contains("return (isinstance(a, int) or a is None)"), + "the optional's None is tested too: {out}" + ); + assert!( + out.contains("return not (isinstance(a, int) or a is None)"), + "`is not` negates the whole union: {out}" + ); + } + + #[test] + fn an_optional_target_needs_no_union_operator_below_310() { + let out = transpile( + indoc! {" + class A: ... + class B: ... + + def f(a: object) -> bool: + return a is (A | B)? + "}, + &Config { + min_version: PythonVersion::PY39, + ..Config::test_default() + }, + ) + .unwrap(); + assert!( + out.contains("return (isinstance(a, A) or isinstance(a, B) or a is None)"), + "each arm is tested on its own: {out}" + ); + } + #[test] fn union_effectful_lhs_binds_once_via_lambda() { // an effectful lhs must be evaluated exactly once across the arms, so it diff --git a/crates/by_transforms/src/transforms/properties.rs b/crates/by_transforms/src/transforms/properties.rs index 81ddb8ff87..fba7a558ec 100644 --- a/crates/by_transforms/src/transforms/properties.rs +++ b/crates/by_transforms/src/transforms/properties.rs @@ -583,8 +583,10 @@ impl PropertiesPass<'_> { .flatten() .map(|synthesized| { ctx.type_only_imports - .extend(synthesized.imports_in(self.written)); - synthesized.text_in(self.written) + .extend(synthesized.modules.iter().cloned()); + ctx.required_imports + .extend(synthesized.typing_imports_in(self.written)); + synthesized.annotation_in(self.written, ctx.annotations_evaluated) }); Some(Backing { annotation: None, diff --git a/crates/by_transforms/src/transforms/protocol_type.rs b/crates/by_transforms/src/transforms/protocol_type.rs index 52e6daabb4..3fd6f53ddc 100644 --- a/crates/by_transforms/src/transforms/protocol_type.rs +++ b/crates/by_transforms/src/transforms/protocol_type.rs @@ -28,9 +28,8 @@ //! ahead of everything the module defines, so an unquoted annotation naming a //! later class would be evaluated at class-body time and `NameError` -use std::collections::hash_map::DefaultHasher; use std::fmt::Write as _; -use std::hash::{Hash, Hasher}; +use std::hash::Hash; use std::collections::HashMap; @@ -57,12 +56,9 @@ struct Shape { } impl Shape { - fn class_name(&self) -> String { - let mut hasher = DefaultHasher::new(); - self.hash(&mut hasher); - #[expect(clippy::cast_possible_truncation)] - let truncated = hasher.finish() as u32; - format!("_Protocol_{truncated:08x}") + /// the name of the class declaring this shape, one the module has no other use of + fn class_name(&self, written: WrittenNames) -> String { + written.synthesized_class("_Protocol_", self) } /// the class declaring this shape as `name`, `typing.Protocol` written as `protocol` @@ -192,7 +188,7 @@ impl<'src> ProtocolTypeLowering<'src> { // a protocol method's receiver is its `self` parameter, never an implicit one None, ); - let returns = self.callable.lower_type_expr(&signature.returns); + let returns = self.callable.render_return(signature); let line = format!( "def {name}({params}) -> {returns}: ...", name = method.name.id, @@ -245,7 +241,7 @@ impl<'src> ProtocolTypeLowering<'src> { let name = if let Some(existing) = self.shapes.get(&shape) { existing.clone() } else { - let name = shape.class_name(); + let name = shape.class_name(self.written); self.shapes.insert(shape, name.clone()); name }; @@ -441,7 +437,7 @@ pub(crate) fn cleanup( let (imports, helpers) = inner.callable.take_requirements(); for line in imports .into_iter() - .chain(crate::runtime_entries(config, &helpers)) + .chain(crate::runtime_entries(config, &helpers, source)) { push_missing(&mut preamble, line.trim_end_matches('\n')); } diff --git a/crates/by_transforms/src/transforms/reified_generic.rs b/crates/by_transforms/src/transforms/reified_generic.rs index 5f143e4dd6..ade4a6c1a3 100644 --- a/crates/by_transforms/src/transforms/reified_generic.rs +++ b/crates/by_transforms/src/transforms/reified_generic.rs @@ -258,7 +258,12 @@ mod tests { // reading `T` in the body is what makes the function reified. the // wrapper's own source is `_by_runtime.py`'s to specify, so the expected // preamble is read from there rather than repeated here - let mut preamble = crate::runtime::inline([crate::runtime::GENERIC]).join("\n"); + let mut preamble = crate::runtime::inline( + [crate::runtime::GENERIC], + &[], + crate::transforms::repeated_underscore::WrittenNames::new(""), + ) + .join("\n"); preamble.push('\n'); check_at( indoc! {" diff --git a/crates/by_transforms/src/transforms/repeated_underscore.rs b/crates/by_transforms/src/transforms/repeated_underscore.rs index d7510ce996..12d3c15a66 100644 --- a/crates/by_transforms/src/transforms/repeated_underscore.rs +++ b/crates/by_transforms/src/transforms/repeated_underscore.rs @@ -142,6 +142,74 @@ pub struct WrittenNames<'src> { /// the names the module's `private` symbols are emitted under, which no other name a /// lowering writes may take private: Option<&'src PrivateNames>, + /// the names of the classes the module's lowerings synthesize for a shape + synthesized: Option<&'src SynthesizedClasses>, +} + +/// the name of each class a lowering synthesizes for a shape — an anonymous named tuple, a +/// typed dict literal, an inline protocol, a callable type python has no spelling for — +/// decided once for the module, however many lowerings meet the shape and in whatever order +/// +/// a class is named after a 32-bit hash of its shape, `_TypedDict_1a2b3c4d`, and two shapes +/// can hash alike. the first of them takes the name and each later one the next name nothing +/// has, `_TypedDict_1a2b3c4d2`, so two shapes are never declared under one name, which would +/// leave whichever class came last read for both +#[derive(Default)] +pub(crate) struct SynthesizedClasses { + /// each shape named so far, by the prefix and hash its name was made from + named: RefCell>, +} + +/// the shapes named after one prefix and hash, each with its name +type NamedShapes = Vec<(Box, String)>; + +impl std::fmt::Debug for SynthesizedClasses { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let named = self.named.borrow(); + f.debug_set() + .entries(named.values().flatten().map(|(_, name)| name)) + .finish() + } +} + +impl SynthesizedClasses { + /// the 32 bits of `shape`'s hash a synthesized class is named after + pub(crate) fn hash_of(shape: &impl std::hash::Hash) -> u32 { + use std::hash::Hasher; + + let mut hasher = std::hash::DefaultHasher::new(); + shape.hash(&mut hasher); + // the low half: eight hex digits keep the name readable, and a collision is + // numbered apart + let [a, b, c, d, ..] = hasher.finish().to_le_bytes(); + u32::from_le_bytes([a, b, c, d]) + } + + /// the name of the class declaring `shape`, its hash spelled after `prefix`, which + /// `written` has no other use of + fn name( + &self, + written: WrittenNames, + prefix: &'static str, + shape: &S, + hash: u32, + ) -> String { + let mut named = self.named.borrow_mut(); + let bucket = named.entry((prefix, hash)).or_default(); + if let Some((_, name)) = bucket + .iter() + .find(|(named, _)| named.downcast_ref::() == Some(shape)) + { + return name.clone(); + } + // a numbered name is longer than every name made from a hash alone, so only the + // names in this bucket can be one it would take + let name = written.fresh_outside(&format!("{prefix}{hash:08x}"), |candidate| { + bucket.iter().any(|(_, name)| name == candidate) + }); + bucket.push((Box::new(shape.clone()), name.clone())); + name + } } /// the name each module-level `private` symbol is emitted under @@ -168,8 +236,15 @@ pub(crate) struct PrivateNames { impl PrivateNames { /// how each of `symbols`, the module-level names a module declares `private`, is emitted. /// `sources` are the texts of the module the lowering reads — as written, and as the - /// passes that run before the name is decided left it - pub(crate) fn decide<'a>(symbols: impl IntoIterator, sources: &[&str]) -> Self { + /// passes that run before the name is decided left it. `reserved` are names the lowering + /// binds at module level that were decided before these, which the module spells nowhere + /// either: the backing functions of its extensions + pub(crate) fn decide<'a>( + symbols: impl IntoIterator, + sources: &[&str], + reserved: impl IntoIterator, + ) -> Self { + let reserved: std::collections::HashSet<&str> = reserved.into_iter().collect(); let mut symbols: Vec<&str> = symbols.into_iter().collect(); // the order a name is taken in decides who gets `_helper` and who `_helper2`, so it // must not be the order of a hash set @@ -184,6 +259,7 @@ impl PrivateNames { let stem = format!("_{symbol}"); let emitted = decision::WrittenNames::new("").fresh_outside(&stem, |candidate| { crate::runtime::defines(candidate) + || reserved.contains(candidate) || private.claimed.contains(candidate) || sources .iter() @@ -207,6 +283,13 @@ pub(crate) struct CodeNames { bindings: std::collections::HashMap>, /// whether the module imports `*` from somewhere, which binds names it never spells star_import: bool, + /// each builtin the module's top level imports under a name of its own, with the names + /// it is imported under — `from builtins import str as str2` + builtin_aliases: std::collections::HashMap>, + /// the names of those imported by the imports a preamble is spliced after, which + /// whatever is spliced there can read as it is defined + /// ([`super::source_util::preamble_offset`]) + leading_builtin_aliases: std::collections::HashSet, } /// one way a module binds a name @@ -214,6 +297,9 @@ pub(crate) struct CodeNames { enum Binding { /// `from import `, which binds the name to what `module` exports as it From(String), + /// `from import as `, which binds `other` to what `module` + /// exports as `name` + FromAs { module: String, name: String }, /// `import `, which binds the name to the module of that name Module, /// anything else @@ -276,6 +362,13 @@ impl CodeNames { (None, Some(module)) if import.level == 0 => { self.bind(&alias.name, Binding::From(module.to_string())); } + (Some(asname), Some(module)) if import.level == 0 => self.bind( + asname, + Binding::FromAs { + module: module.to_string(), + name: alias.name.to_string(), + }, + ), (asname, _) => self .bind(asname.as_ref().unwrap_or(&alias.name), Binding::Other), } @@ -334,12 +427,50 @@ impl CodeNames { } let mut collect = Collect::default(); - for stmt in suite { + // the statements before the first that is neither a docstring nor a leading import + let mut leading = true; + for (index, stmt) in suite.iter().enumerate() { collect.visit_stmt(stmt); + let docstring = index == 0 + && matches!(stmt, Stmt::Expr(expr) if expr.value.is_string_literal_expr()); + leading &= docstring || super::source_util::is_leading_import(stmt); + // an import in a nested scope binds its name there alone + if let Stmt::ImportFrom(import) = stmt + && import.level == 0 + && import.module.as_deref() == Some("builtins") + { + for alias in &import.names { + if let Some(asname) = &alias.asname { + collect + .0 + .builtin_aliases + .entry(alias.name.to_string()) + .or_default() + .push(asname.to_string()); + if leading { + collect.0.leading_builtin_aliases.insert(asname.to_string()); + } + } + } + } } collect.0 } + /// a name the module's top level imports the builtin `name` under, and binds to nothing + /// else anywhere — the first of them, when there are several + fn builtin_alias(&self, name: &str) -> Option<&str> { + let binding = Binding::FromAs { + module: "builtins".to_owned(), + name: name.to_owned(), + }; + self.builtin_aliases + .get(name)? + .iter() + .find(|alias| self.only_binds(alias, &binding)) + .map(String::as_str) + } + /// whether every binding of `name` is `binding`, so a lowering can read the module's own /// `name` as the thing `binding` binds it to fn only_binds(&self, name: &str, binding: &Binding) -> bool { @@ -359,6 +490,22 @@ impl<'src> WrittenNames<'src> { lowerings: None, code: None, private: None, + synthesized: None, + } + } + + /// these names, with `synthesized` the names of the classes the module's lowerings + /// synthesize for a shape + pub(crate) fn with_synthesized<'a>( + self, + synthesized: &'a SynthesizedClasses, + ) -> WrittenNames<'a> + where + 'src: 'a, + { + WrittenNames { + synthesized: Some(synthesized), + ..self } } @@ -372,6 +519,7 @@ impl<'src> WrittenNames<'src> { lowerings: self.lowerings, code: Some(code), private: self.private, + synthesized: self.synthesized, } } @@ -385,6 +533,7 @@ impl<'src> WrittenNames<'src> { lowerings: self.lowerings, code: self.code, private: Some(private), + synthesized: self.synthesized, } } @@ -399,6 +548,7 @@ impl<'src> WrittenNames<'src> { lowerings: Some(lowerings), code: self.code, private: self.private, + synthesized: self.synthesized, } } @@ -416,6 +566,27 @@ impl<'src> WrittenNames<'src> { .fresh_outside(stem, |name| self.lowering_binds(name)) } + /// [`Self::fresh`], also passing over every name `taken` answers for + fn fresh_outside(self, stem: &str, taken: impl Fn(&str) -> bool) -> String { + self.names + .fresh_outside(stem, |name| self.lowering_binds(name) || taken(name)) + } + + /// the name of the class a lowering synthesizes for `shape`, named after its hash + /// behind `prefix` ([`SynthesizedClasses`]). without the module's decision, two shapes + /// hashing alike share a name + pub(crate) fn synthesized_class( + self, + prefix: &'static str, + shape: &S, + ) -> String { + let hash = SynthesizedClasses::hash_of(shape); + match self.synthesized { + Some(synthesized) => synthesized.name(self, prefix, shape, hash), + None => self.fresh(&format!("{prefix}{hash:08x}")), + } + } + /// whether `name` is one a lowering cannot bind: the module spells it, a runtime /// helper is bound under it, or a `private` symbol is emitted under it pub(crate) fn taken(self, name: &str) -> bool { @@ -457,20 +628,37 @@ impl<'src> WrittenNames<'src> { /// module binds that name to something else, anywhere — a parameter named `type`, a /// class attribute named `staticmethod` — and then a name of the lowering's own, which /// [`Self::builtin_imports`] binds + /// + /// a module that already imports the builtin under a name of its own has it read under + /// that name. a lowering run over the output of an earlier one finds the earlier one's + /// import there, so every lowering of a module reads a builtin under one name pub(crate) fn builtin(self, name: &str) -> String { + if let Some(code) = self.code + && !code.only_binds(name, &Binding::From("builtins".to_owned())) + && let Some(alias) = code.builtin_alias(name) + && !self.lowering_binds(alias) + { + return alias.to_owned(); + } self.imported("builtins", name) } /// the import that binds each builtin `output`, python a lowering wrote, reads under a /// name of the lowering's own ([`Self::builtin`]), on a python of `minor` /// - /// such a name is spelled nowhere in the module, so only a lowering can have written it + /// such a name is spelled nowhere in the module, so only a lowering can have written it. + /// one the module imports among the imports a preamble is spliced after has its import + /// already, ahead of anything a lowering adds. imported anywhere else, it is imported + /// again, since a definition spliced ahead of that import may read it as it is defined pub(crate) fn builtin_imports(self, output: &str, minor: u8) -> Vec { let written = decision::WrittenNames::new(output); ruff_python_stdlib::builtins::python_builtins(minor, false) .filter_map(|name| { let local = self.builtin(name); - (local != name && written.spells(&local)) + let imported = self + .code + .is_some_and(|code| code.leading_builtin_aliases.contains(&local)); + (local != name && !imported && written.spells(&local)) .then(|| format!("from builtins import {name} as {local}")) }) .collect() @@ -858,9 +1046,49 @@ impl Transformer for Lowerer<'_, '_> { mod tests { use indoc::indoc; + use super::{SynthesizedClasses, WrittenNames}; use crate::python_passthrough::unchanged; use crate::{Config, transpile}; + /// a shape every other one of its kind hashes alike with + #[derive(Clone, PartialEq)] + struct Collides(&'static str); + + impl std::hash::Hash for Collides { + fn hash(&self, _: &mut H) {} + } + + /// a shape keeps the name it was first given, and one hashing alike with it takes the + /// next name the module has no use of, whichever lowering asks + #[test] + fn shapes_hashing_alike_are_named_apart_once_for_the_module() { + let stem = format!("_S_{:08x}", SynthesizedClasses::hash_of(&Collides(""))); + // the module spells the name the second shape would otherwise take + let source = format!("{stem}2 = 1\n"); + let synthesized = SynthesizedClasses::default(); + let one = WrittenNames::new(&source).with_synthesized(&synthesized); + let other = WrittenNames::new(&source).with_synthesized(&synthesized); + assert_eq!(one.synthesized_class("_S_", &Collides("a")), stem); + assert_eq!( + other.synthesized_class("_S_", &Collides("b")), + format!("{stem}3") + ); + assert_eq!(other.synthesized_class("_S_", &Collides("a")), stem); + assert_eq!( + one.synthesized_class("_S_", &Collides("c")), + format!("{stem}4") + ); + assert_eq!( + one.synthesized_class("_S_", &Collides("b")), + format!("{stem}3") + ); + // another kind of class is named apart by its prefix alone + assert_eq!( + one.synthesized_class("_T_", &Collides("b")), + format!("_T_{:08x}", SynthesizedClasses::hash_of(&Collides(""))) + ); + } + fn check(input: &str, expected: &str) { assert_eq!(transpile(input, &Config::test_default()).unwrap(), expected); } diff --git a/crates/by_transforms/src/transforms/source_util.rs b/crates/by_transforms/src/transforms/source_util.rs index 6e3f36456a..95baf7453a 100644 --- a/crates/by_transforms/src/transforms/source_util.rs +++ b/crates/by_transforms/src/transforms/source_util.rs @@ -60,29 +60,36 @@ pub(crate) fn docstring_end(text: &str) -> usize { /// The byte offset generated lines must be spliced at: past the BOM, the module /// docstring, and any `from __future__ import` — each of which is only valid -/// where it already is. +/// where it already is — and past the imports from `builtins` that follow them, +/// which bind builtins under the names a lowering reads them by, so that what is +/// spliced here can read them as it is defined. pub(crate) fn preamble_offset(text: &str) -> usize { - let mut at = docstring_end(text); + let start = docstring_end(text); let parsed = ruff_python_parser::parse_unchecked_source( - &text[at..], + &text[start..], ruff_python_ast::PySourceType::Python, ); + let mut at = start; for stmt in parsed.suite() { - let Stmt::ImportFrom(import) = stmt else { - break; - }; - if import - .module - .as_ref() - .is_none_or(|m| m.as_str() != "__future__") - { + if !is_leading_import(stmt) { break; } - at += line_end(&text[at..], usize::from(import.range().end())); + at = start + line_end(&text[start..], usize::from(stmt.range().end())); } at } +/// whether `stmt` is one of the imports a preamble is spliced after +/// ([`preamble_offset`]): one from `__future__`, or one from `builtins` +pub(crate) fn is_leading_import(stmt: &Stmt) -> bool { + matches!( + stmt, + Stmt::ImportFrom(import) + if import.level == 0 + && matches!(import.module.as_deref(), Some("__future__" | "builtins")) + ) +} + /// Render `value` as a python string literal. /// /// A conservative escaper that always emits a double-quoted form, so the result @@ -702,3 +709,25 @@ pub(crate) fn header_end(f: &StmtFunctionDef) -> TextSize { .map_or(TextSize::new(0), |r| r.range().end()), ) } + +#[cfg(test)] +mod tests { + use super::preamble_offset; + + /// a preamble goes after the imports binding a builtin under the name a lowering reads + /// it by, too, so what it defines can read that name as it is defined + /// + /// every statement's range is measured from where the parse began, so each import's + /// line end is too. measured from the one before it, the offset overshot the second + /// import, and a preamble landed in the middle of the module + #[test] + fn a_preamble_goes_after_every_leading_import() { + for text in [ + "from __future__ import annotations\nfrom __future__ import generator_stop\nx = 1\n", + "from __future__ import annotations\nfrom builtins import str as str2\nx = 1\n", + "\"\"\"doc\"\"\"\nfrom __future__ import annotations\nfrom __future__ import generator_stop\nx = 1\n", + ] { + assert_eq!(&text[preamble_offset(text)..], "x = 1\n", "{text}"); + } + } +} diff --git a/crates/by_transforms/src/transforms/string_tag.rs b/crates/by_transforms/src/transforms/string_tag.rs index 66627f0918..486d2783df 100644 --- a/crates/by_transforms/src/transforms/string_tag.rs +++ b/crates/by_transforms/src/transforms/string_tag.rs @@ -241,9 +241,11 @@ mod tests { fn polyfilled(input: &str, expected_body: &str) { let out = transpile(input, &Config::test_default()).unwrap(); let mut body = out.as_str(); - for definition in - crate::runtime::inline([crate::runtime::TEMPLATE, crate::runtime::INTERPOLATION]) - { + for definition in crate::runtime::inline( + [crate::runtime::TEMPLATE, crate::runtime::INTERPOLATION], + &[], + crate::transforms::repeated_underscore::WrittenNames::new(input), + ) { body = body .strip_prefix(definition.as_str()) .and_then(|rest| rest.strip_prefix('\n')) diff --git a/crates/by_transforms/src/transforms/type_expr_walker.rs b/crates/by_transforms/src/transforms/type_expr_walker.rs index adf7466f29..fe904f472d 100644 --- a/crates/by_transforms/src/transforms/type_expr_walker.rs +++ b/crates/by_transforms/src/transforms/type_expr_walker.rs @@ -37,14 +37,28 @@ //! `NamedTuple` and `TypedDict`. which arguments those are is the type checker's //! answer rather than a list kept here, so a form ty learns to check as a type is //! one this lowers without further change +//! 14. the type a narrowing predicate `a is T` written as a type tests for — the `T` of +//! a return guard `def f(a) -> a is T` use ruff_python_ast::helpers::declaration_annotation_type; use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; -use ruff_python_ast::{Expr, Operator, Parameters, Stmt, TypeParam, UnaryOp}; +use ruff_python_ast::{CmpOp, Expr, ExprCompare, Operator, Parameters, Stmt, TypeParam, UnaryOp}; use ruff_text_size::{Ranged, TextRange}; +use super::type_is::replaced_guard; use crate::type_info::{TypeInfo, trailing_name}; +/// `a is T` written as a type, the narrowing predicate of a return guard, with its `T` +fn narrowing_predicate(expr: &Expr) -> Option<(&ExprCompare, &Expr)> { + let Expr::Compare(compare) = expr else { + return None; + }; + match (&*compare.ops, &*compare.comparators, compare.left.as_ref()) { + ([CmpOp::Is], [target], Expr::Name(_)) => Some((compare, target)), + _ => None, + } +} + /// the kind of type position currently being visited. lets visitors /// distinguish (e.g.) a syntactic annotation from an interior subtree #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -85,6 +99,11 @@ pub(crate) enum RootKind { pub(crate) trait TypeExprVisitor { fn visit(&mut self, expr: &Expr, pos: TypePos) -> Recurse; + /// told of each narrowing predicate `a is T` standing where a type is written, the + /// return guard `def f(a) -> a is T`. its `T` is then visited in the predicate's place, + /// as the type it is + fn visit_predicate(&mut self, _predicate: &ExprCompare) {} + /// told ahead of each root the walk visits what kind of root it is, which holds /// for everything nested inside it fn enter_root(&mut self, _kind: RootKind) {} @@ -152,6 +171,11 @@ impl TypePosWalker<'_> { if self.claimed.iter().any(|c| c.contains_range(expr.range())) { return; } + if let Some((predicate, target)) = narrowing_predicate(expr) { + self.visitor.visit_predicate(predicate); + self.visit_type_expr(target, pos); + return; + } if self.visitor.visit(expr, pos) == Recurse::Stop { return; } @@ -346,9 +370,10 @@ impl<'ast> Visitor<'ast> for TypePosWalker<'_> { } Stmt::FunctionDef(f) => { self.visit_parameters(&f.parameters); - // `-> asserts x` names a place to narrow; it is not a type position at all + // a guard `TypeIs` cannot spell is replaced whole by what the function + // returns, so nothing of it is a type if let Some(ret) = &f.returns - && !f.is_asserts_return + && replaced_guard(f).is_none() { self.visit_root(ret, RootKind::Annotation); } diff --git a/crates/by_transforms/src/transforms/type_is.rs b/crates/by_transforms/src/transforms/type_is.rs index dc4f348bb7..4707b83a47 100644 --- a/crates/by_transforms/src/transforms/type_is.rs +++ b/crates/by_transforms/src/transforms/type_is.rs @@ -11,19 +11,18 @@ //! traversal is delegated to [`type_expr_walker`] (with `types = None`): the //! `a is T` written in a *return guard* is what this rewrites, and the same //! pair written in a body is a type test that [`parametric_is`] lowers. this -//! pass claims the guard first, so the two never rewrite the same span - -use ruff_python_ast::helpers::{ReturnGuardForm, return_guards}; -use ruff_python_ast::visitor::{Visitor, walk_stmt}; -use ruff_python_ast::{ - AtomicNodeIndex, CmpOp, Expr, ExprContext, ExprName, ExprSubscript, ModModule, Stmt, - StmtFunctionDef, name::Name, -}; +//! pass claims the guard first, so the two never rewrite the same span. the +//! walker hands every other pass the `T` in the guard's place, as the type it +//! is, so `TypeIs[...]` passes it through with everything they lower inside it + +use ruff_python_ast::helpers::{ReturnGuardForm, negated_narrowing_predicate, return_guards}; +use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; +use ruff_python_ast::{Expr, ExprCompare, ModModule, Stmt, StmtFunctionDef}; use ruff_text_size::{Ranged, TextRange, TextSize}; -use super::ast_driver::{AstPass, PassContext, render_expr}; +use super::ast_driver::{AstPass, Fragment, PassContext}; use super::repeated_underscore::WrittenNames; -use super::type_expr_walker::{Recurse, TypeExprVisitor, TypePos, walk_type_positions_skipping}; +use super::type_expr_walker::{Recurse, TypeExprVisitor, TypePos, walk_type_positions}; pub(crate) struct TypeIs<'src> { src: &'src str, @@ -46,27 +45,28 @@ impl AstPass for TypeIs<'_> { fn run(&self, module: &mut ModModule, ctx: &mut PassContext) { let mut state = State { type_is: self.written.imported("typing", "TypeIs"), - edits: Vec::new(), + templates: Vec::new(), needs_import: false, }; let body: &[Stmt] = &module.body; // narrowing annotations that name a place python can't: an assertion guard, and a // predicate on something other than a parameter. both lower to what the function - // returns, and their ranges are claimed so the `TypeIs[T]` rewrite below skips them + // returns, and the type walk below skips them let mut guards = ReturnGuards { src: self.src, bool_: self.written.builtin("bool"), edits: Vec::new(), - claimed: Vec::new(), + errors: Vec::new(), }; for stmt in body { guards.visit_stmt(stmt); } - state.edits.append(&mut guards.edits); + ctx.text_edits.append(&mut guards.edits); + ctx.errors.append(&mut guards.errors); - walk_type_positions_skipping(body, None, &guards.claimed, &mut state); - ctx.text_edits.extend(state.edits); + walk_type_positions(body, None, &mut state); + ctx.template_edits.extend(state.templates); if state.needs_import { // typing.TypeIs landed in 3.13 (PEP 742). on older runtimes the // typing_redirect pass switches the import to typing_extensions @@ -79,17 +79,55 @@ impl AstPass for TypeIs<'_> { struct State { /// the name `typing.TypeIs` is written under type_is: String, - edits: Vec<(TextRange, String)>, + /// `TypeIs[T]` over each predicate, passing `T`'s own source through so the + /// lowerings inside it apply + templates: Vec<(TextRange, Vec)>, needs_import: bool, } +/// a narrowing return annotation with no `TypeIs` spelling, which is lowered to what the +/// function returns +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ReplacedGuard { + /// `def f(x) -> asserts x` raises when the assertion doesn't hold, and returns `None` + /// when it does + Asserts, + /// `def f() -> a is int` narrows a place rather than an argument, and + /// `-> self.data is str` narrows a member of one. `TypeIs` can only name a bare + /// parameter, so these lower to the `bool` the function returns + Predicate, +} + +/// how `function`'s return annotation is replaced whole, when it is a guard `TypeIs` cannot +/// spell. nothing of such an annotation is a type in the output +pub(crate) fn replaced_guard(function: &StmtFunctionDef) -> Option { + function.returns.as_ref()?; + if function.is_asserts_return { + return Some(ReplacedGuard::Asserts); + } + let guards = return_guards(function)?; + let [guard] = guards.as_slice() else { + return None; + }; + if !matches!(guard.form, ReturnGuardForm::Predicate { .. }) { + return None; + } + let (name, members) = guard.place_parts(); + let narrows_a_parameter = members.is_empty() + && function + .parameters + .iter() + .any(|parameter| parameter.name().id == *name); + (!narrows_a_parameter).then_some(ReplacedGuard::Predicate) +} + /// lowers the narrowing return annotations that have no `TypeIs` spelling struct ReturnGuards<'src> { src: &'src str, /// the name the builtin `bool` is written under bool_: String, edits: Vec<(TextRange, String)>, - claimed: Vec, + errors: Vec, } impl ReturnGuards<'_> { @@ -97,40 +135,33 @@ impl ReturnGuards<'_> { let Some(returns) = function.returns.as_deref() else { return; }; - - // `def f(x) -> asserts x` raises when the assertion doesn't hold, and returns - // `None` when it does. the keyword is not part of `returns`, so the edit starts - // at the keyword itself - if function.is_asserts_return { - let keyword_start = self.src[..usize::from(returns.range().start())] - .rfind("asserts") - .map(|offset| TextSize::try_from(offset).expect("offset fits u32")) - .unwrap_or_else(|| returns.range().start()); - self.edits.push(( - TextRange::new(keyword_start, returns.range().end()), - "None".to_owned(), + // nothing narrows to everything but a type, so there is no annotation to lower it to + if !function.is_asserts_return && negated_narrowing_predicate(returns).is_some() { + self.errors.push(format!( + "`{}` of `{}` cannot be lowered: a narrowing predicate cannot be negated, \ + since `TypeIs` cannot narrow to everything but a type. write the predicate \ + as `is` and negate the call where it is used", + &self.src[returns.range()], + function.name, )); - self.claimed.push(returns.range()); return; } - - // `def f() -> a is int` narrows a place rather than an argument, and - // `-> self.data is str` narrows a member of one. `TypeIs` can only name a bare - // parameter, so anything else lowers to the `bool` the function returns - if let Some(guards) = return_guards(function) - && let [guard] = guards.as_slice() - && matches!(guard.form, ReturnGuardForm::Predicate { .. }) - { - let (name, members) = guard.place_parts(); - let narrows_a_parameter = members.is_empty() - && function - .parameters - .iter() - .any(|parameter| parameter.name().id == *name); - if !narrows_a_parameter { + match replaced_guard(function) { + // the keyword is not part of `returns`, so the edit starts at the keyword itself + Some(ReplacedGuard::Asserts) => { + let keyword_start = self.src[..usize::from(returns.range().start())] + .rfind("asserts") + .map(|offset| TextSize::try_from(offset).expect("offset fits u32")) + .unwrap_or_else(|| returns.range().start()); + self.edits.push(( + TextRange::new(keyword_start, returns.range().end()), + "None".to_owned(), + )); + } + Some(ReplacedGuard::Predicate) => { self.edits.push((returns.range(), self.bool_.clone())); - self.claimed.push(returns.range()); } + None => {} } } } @@ -142,35 +173,41 @@ impl<'ast> Visitor<'ast> for ReturnGuards<'_> { } walk_stmt(self, stmt); } -} -impl TypeExprVisitor for State { - fn visit(&mut self, expr: &Expr, _pos: TypePos) -> Recurse { - if let Expr::Compare(c) = expr - && c.ops.len() == 1 - && matches!(c.ops[0], CmpOp::Is) - && matches!(c.left.as_ref(), Expr::Name(_)) - && let Some(target) = c.comparators.first() + fn visit_expr(&mut self, expr: &'ast Expr) { + if let Expr::CallableType(callable) = expr + && negated_narrowing_predicate(&callable.returns).is_some() { - let new_node = Expr::Subscript(ExprSubscript { - node_index: AtomicNodeIndex::NONE, - range: TextRange::default(), - value: Box::new(Expr::Name(ExprName { - node_index: AtomicNodeIndex::NONE, - range: TextRange::default(), - id: Name::from(self.type_is.as_str()), - ctx: ExprContext::Load, - })), - slice: Box::new(target.clone()), - ctx: ExprContext::Load, - is_typeof: false, - is_type_decoration: false, - }); - self.needs_import = true; - self.edits.push((expr.range(), render_expr(&new_node))); + self.errors.push(format!( + "`{}` cannot be lowered: a narrowing predicate cannot be negated, since \ + `TypeIs` cannot narrow to everything but a type. write the predicate as `is` \ + and negate the call where it is used", + &self.src[callable.range()], + )); } + walk_expr(self, expr); + } +} + +impl TypeExprVisitor for State { + fn visit(&mut self, _expr: &Expr, _pos: TypePos) -> Recurse { Recurse::Descend } + + fn visit_predicate(&mut self, predicate: &ExprCompare) { + let [target] = &*predicate.comparators else { + return; + }; + self.needs_import = true; + self.templates.push(( + predicate.range(), + vec![ + Fragment::Lit(format!("{}[", self.type_is)), + Fragment::Src(target.range()), + Fragment::Lit("]".to_owned()), + ], + )); + } } #[cfg(test)] @@ -186,6 +223,42 @@ mod tests { ); } + /// `TypeIs` narrows to the type it names, and nothing in python's typing narrows to + /// everything but one, so a negated predicate is refused rather than lowered to the + /// `not isinstance(...)` a negated type test in a body is. the assertion form negates + #[test] + fn a_negated_predicate_is_refused() { + let error = transpile( + "def not_int(a: object) -> a is not int:\n return not isinstance(a, int)\n", + &Config::test_default(), + ) + .unwrap_err(); + assert!( + error.contains("a narrowing predicate cannot be negated"), + "got: {error}" + ); + let out = transpile( + "def check(a: object) -> asserts a is not int:\n assert not isinstance(a, int)\n", + &Config::test_default(), + ) + .unwrap(); + assert!(out.contains("-> None:"), "got: {out}"); + } + + /// a callable type's predicate cannot be negated either + #[test] + fn a_negated_predicate_a_callable_type_returns_is_refused() { + let error = transpile( + "def f(check: (x: object) -> (x is not int)): ...\n", + &Config::test_default(), + ) + .unwrap_err(); + assert!( + error.contains("a narrowing predicate cannot be negated"), + "got: {error}" + ); + } + fn check_py312(input: &str, expected: &str) { let config = Config { min_version: PythonVersion::PY312, @@ -208,6 +281,66 @@ mod tests { ); } + #[test] + fn lowerings_inside_the_target_apply() { + // the target keeps its own source, so what is lowered inside it, like an + // optional's `?`, is lowered here too + check_py312( + "def f(a) -> a is int?: ...\n", + indoc! {" + from typing_extensions import TypeIs + def f(a) -> TypeIs[int | None]: ... + "}, + ); + let below_310 = Config { + min_version: PythonVersion::PY39, + ..Config::test_default() + }; + let out = transpile("def f(a) -> a is int?: ...\n", &below_310).unwrap(); + assert!( + out.contains("def f(a) -> TypeIs[Union[int, None]]: ..."), + "the optional is spelled without `|`: {out}" + ); + } + + #[test] + fn every_type_form_in_the_target_is_lowered() { + // the target is lowered as the type it is, as it would be written in any annotation + check_py312( + indoc! {" + def f(a) -> a is (int) -> str: ... + def g(a) -> a is (int, str): ... + def h(a) -> a is \"x\": ... + def k(a) -> a is list[(int) -> str]?: ... + "}, + indoc! {" + from typing import Callable, Literal + from typing_extensions import TypeIs + def f(a) -> TypeIs[Callable[[int], str]]: ... + def g(a) -> TypeIs[tuple[int, str]]: ... + def h(a) -> TypeIs[Literal[\"x\"]]: ... + def k(a) -> TypeIs[list[Callable[[int], str]] | None]: ... + "}, + ); + } + + #[test] + fn a_guard_lowered_to_what_it_returns_holds_no_type() { + // the whole annotation is replaced, so nothing inside it is lowered + check( + indoc! {" + a = 1 + def f() -> a is (int) -> str: + return True + "}, + indoc! {" + a = 1 + def f() -> bool: + return True + "}, + ); + } + #[test] fn other_param_name() { check( diff --git a/crates/by_transforms/src/transforms/typed_dict_literal.rs b/crates/by_transforms/src/transforms/typed_dict_literal.rs index 37fe10159a..8e8dae60d4 100644 --- a/crates/by_transforms/src/transforms/typed_dict_literal.rs +++ b/crates/by_transforms/src/transforms/typed_dict_literal.rs @@ -30,9 +30,8 @@ //! field types are emitted as forward references — see [`quote_type`] use std::collections::BTreeSet; -use std::collections::hash_map::DefaultHasher; use std::fmt::Write as _; -use std::hash::{Hash, Hasher}; +use std::hash::Hash; use indexmap::IndexMap; use ruff_diagnostics::{Edit, Fix}; @@ -60,12 +59,9 @@ struct Shape { } impl Shape { - fn class_name(&self) -> String { - let mut hasher = DefaultHasher::new(); - self.hash(&mut hasher); - #[expect(clippy::cast_possible_truncation)] - let truncated = hasher.finish() as u32; - format!("_TypedDict_{truncated:08x}") + /// the name of the class declaring this shape, one the module has no other use of + fn class_name(&self, written: WrittenNames) -> String { + written.synthesized_class("_TypedDict_", self) } /// the class declaring this shape as `name`, `typing_extensions.TypedDict` written as @@ -357,7 +353,7 @@ impl<'src> TypedDictLiteral<'src> { let name = if let Some(existing) = self.shapes.get(&shape) { existing.clone() } else { - let n = shape.class_name(); + let n = shape.class_name(self.written); self.shapes.insert(shape, n.clone()); n }; diff --git a/crates/by_transforms/src/transforms/typing_redirect.rs b/crates/by_transforms/src/transforms/typing_redirect.rs index 563fd4bd7b..3ca128e4c1 100644 --- a/crates/by_transforms/src/transforms/typing_redirect.rs +++ b/crates/by_transforms/src/transforms/typing_redirect.rs @@ -228,13 +228,19 @@ mod tests { #[test] fn a_stub_needs_nothing_installed() { let source = "from typing import Self\n"; - let (_, needed) = crate::transpile_with_report(source, &Config::test_default()).unwrap(); + let (_, needed) = crate::transpile_with_report( + source, + &Config::test_default(), + crate::PythonVersion::latest_ty(), + ) + .unwrap(); assert_eq!(needed.specifiers(), ["typing_extensions>=4.12"]); let stub = Config { is_stub: true, ..Config::test_default() }; - let (out, needed) = crate::transpile_with_report(source, &stub).unwrap(); + let (out, needed) = + crate::transpile_with_report(source, &stub, crate::PythonVersion::latest_ty()).unwrap(); assert_eq!(out, "from typing_extensions import Self\n"); assert!(needed.specifiers().is_empty()); } diff --git a/crates/by_transforms/src/transforms/unique_loop_bindings.rs b/crates/by_transforms/src/transforms/unique_loop_bindings.rs index 480ff0bc1a..2249b480e3 100644 --- a/crates/by_transforms/src/transforms/unique_loop_bindings.rs +++ b/crates/by_transforms/src/transforms/unique_loop_bindings.rs @@ -579,7 +579,12 @@ mod tests { /// the preamble holds is still asserted fn check_body(input: &str, expected: &str) { let out = transpile(input, &Config::test_default()).unwrap(); - let mut runtime = crate::runtime::inline([crate::runtime::LOOP_BIND]).join("\n"); + let mut runtime = crate::runtime::inline( + [crate::runtime::LOOP_BIND], + &[], + crate::transforms::repeated_underscore::WrittenNames::new(input), + ) + .join("\n"); runtime.push('\n'); let body = out.replacen(&runtime, "", 1); assert_eq!(body, crate::python_passthrough::lazify_expected(expected)); diff --git a/crates/by_transforms/src/transforms/visibility_rename.rs b/crates/by_transforms/src/transforms/visibility_rename.rs index ebb26b8bf8..bcc7e7a27b 100644 --- a/crates/by_transforms/src/transforms/visibility_rename.rs +++ b/crates/by_transforms/src/transforms/visibility_rename.rs @@ -516,6 +516,77 @@ mod tests { assert!(out.contains("Registry._Registry__made += 1"), "got:\n{out}"); } + /// a private type alias in a class body is declared under the name python mangles, and + /// read by the mangled name itself, which an annotation python keeps as a string + /// still resolves by + #[test] + fn a_private_class_type_alias_is_declared_mangled() { + let source = indoc! {" + class Shape: + private type Size = int | str + type Sizes = list[Size] + + width: Size = 1 + + def area(self) -> Size: + return self.width + + def scaled[T](self, by: T) -> Size: + return self.width + "}; + let native = transpile( + source, + &Config { + min_version: crate::PythonVersion::PY312, + ..Config::test_default() + }, + ) + .unwrap(); + assert!( + native.contains(" type __Size = int | str\n"), + "got:\n{native}" + ); + assert!( + native.contains(" type Sizes = list[_Shape__Size]\n"), + "got:\n{native}" + ); + assert!( + native.contains(" width: _Shape__Size = 1\n"), + "got:\n{native}" + ); + assert!( + native.contains(" def area(self) -> _Shape__Size:\n"), + "got:\n{native}" + ); + // a generic method's signature is its own scope, which reads the class's names + assert!( + native.contains(" def scaled[T](self, by: T) -> _Shape__Size:\n"), + "got:\n{native}" + ); + + // below 3.12 the statement is a `TypeAliasType` call, which names the alias too + let polyfilled = transpile( + source, + &Config { + min_version: crate::PythonVersion::PY310, + ..Config::test_default() + }, + ) + .unwrap(); + assert!( + polyfilled.contains(" __Size = TypeAliasType(\"__Size\", int | str)\n"), + "got:\n{polyfilled}" + ); + assert!( + polyfilled.contains(" Sizes = TypeAliasType(\"Sizes\", list[_Shape__Size])\n"), + "got:\n{polyfilled}" + ); + assert!( + polyfilled.contains(" width: _Shape__Size = 1\n"), + "got:\n{polyfilled}" + ); + } + #[test] fn a_module_level_private_variable_is_renamed_where_it_is_the_symbol() { let out = out(indoc! {" @@ -553,7 +624,8 @@ mod tests { fn a_member_is_renamed_wherever_the_class_body_declares_it() { // a declaration inside a nested block, a decorated method and a nested // class are members like any other, and a bare name in the class body - // reads the declaration under the spelling python mangles there + // reads the declaration under its mangled name, as an annotation python + // keeps as a string has to let out = out(indoc! {" class Shape: if True: @@ -575,7 +647,10 @@ mod tests { return Shape.Inner() "}); assert!(out.contains(" __sides: int = 3"), "got:\n{out}"); - assert!(out.contains("doubled: int = __scale * 2"), "got:\n{out}"); + assert!( + out.contains("doubled: int = _Shape__scale * 2"), + "got:\n{out}" + ); assert!(out.contains("class __Inner:"), "got:\n{out}"); assert!(out.contains(" def __make() -> int:"), "got:\n{out}"); assert!( diff --git a/crates/by_transforms/src/type_info.rs b/crates/by_transforms/src/type_info.rs index 1734c973f9..8039be8611 100644 --- a/crates/by_transforms/src/type_info.rs +++ b/crates/by_transforms/src/type_info.rs @@ -107,6 +107,33 @@ impl SynthesizedType { text } + /// the type written as an annotation, as the module `written` is of writes it. + /// `evaluated` is whether python evaluates the annotation as its definition runs + /// ([`PassContext::annotations_evaluated`]) + /// + /// a module it qualifies a class with may be one the source never imports as it runs, only + /// for a checker, and evaluated as the definition runs the annotation would import that + /// module there, into whatever cycle the source kept it out of. such an annotation is + /// written as a string, so only `typing.get_type_hints` evaluates it + /// + /// [`PassContext::annotations_evaluated`]: crate::transforms::ast_driver::PassContext::annotations_evaluated + pub(crate) fn annotation_in(&self, written: WrittenNames, evaluated: bool) -> String { + let text = self.text_in(written); + if evaluated && !self.modules.is_empty() { + crate::transforms::source_util::string_repr(&text) + } else { + text + } + } + + /// the imports of each `typing` name [`Self::text_in`] reads + pub(crate) fn typing_imports_in(&self, written: WrittenNames) -> Vec { + self.typing_names + .iter() + .map(|name| written.import_from("typing", &[name])) + .collect() + } + /// the imports [`Self::text_in`] reads: each module it qualifies a class with, and each /// `typing` name pub(crate) fn imports_in(&self, written: WrittenNames) -> Vec { @@ -726,6 +753,11 @@ pub(crate) trait TypeInfo { /// member). the inferred-annotation transform must leave these alone fn class_body_annotation_is_semantic(&self, class_def: &StmtClassDef) -> bool; + /// basedpython: the name of the backing function `member` of the extension `class_def` + /// declares lowers to, the name every call of it is rewritten to call. `None` when + /// `class_def` is not an extension + fn extension_backing_function(&self, class_def: &StmtClassDef, member: &str) -> Option; + /// whether `expr` is a field-specifier call — `pydantic.Field(...)`, /// `dataclasses.field(...)`, `attrs.field(...)`, and the like. the checker /// models these as assignable to the field's declared type, but their @@ -847,7 +879,7 @@ impl TypeInfo for SemanticModel<'_> { // `final A`, which has no python spelling at all, and the annotation a reader // would write for it is `A` let promoted = ty.promote(db, &env).promote_class_literals(db, &env); - spell_for_python(db, &env, self, promoted, min_version, false) + spell_for_python(db, &env, self, promoted, min_version, false, None) } fn is_return_value_marker(&self, name: &ExprName) -> bool { @@ -1170,7 +1202,7 @@ impl TypeInfo for SemanticModel<'_> { let env = self.program_environment(); let ty = expr.inferred_type(self)?; let promoted = ty.promote(db, &env).promote_class_literals(db, &env); - spell_for_python(db, &env, self, promoted, min_version, false) + spell_for_python(db, &env, self, promoted, min_version, false, None) } fn template_literal_strings(&self, expr: &Expr) -> Option> { @@ -1191,6 +1223,8 @@ impl TypeInfo for SemanticModel<'_> { if ty.is_dynamic() && !matches!(ty, Type::Dynamic(DynamicType::Any)) { return None; } + // the fold is written where `expr` stands, so a name bound there — a type + // parameter of the definition it annotates — is one the output binds spell_for_python( self.db(), &self.program_environment(), @@ -1198,6 +1232,7 @@ impl TypeInfo for SemanticModel<'_> { ty, min_version, true, + Some(expr), ) } @@ -1526,7 +1561,7 @@ impl TypeInfo for SemanticModel<'_> { // ``); `type[int]` is its promoted form, and the type ty // itself reads back off the undeclared attribute through an instance let promoted = ty.promote(db, &env).promote_class_literals(db, &env); - spell_for_python(db, &env, self, promoted, min_version, false) + spell_for_python(db, &env, self, promoted, min_version, false, None) } fn class_body_annotation_is_semantic(&self, class_def: &StmtClassDef) -> bool { @@ -1538,6 +1573,11 @@ impl TypeInfo for SemanticModel<'_> { }) } + fn extension_backing_function(&self, class_def: &StmtClassDef, member: &str) -> Option { + let class = class_def.inferred_type(self)?.as_class_literal()?; + ty_python_semantic::extension_backing_function(self.db(), class, member) + } + fn static_resource(&self, path: &str, binding: &str) -> Result { let file = ty_python_semantic::resolve_static_resource(self.db(), self.file(), path) .map_err(|error| error.to_string())?; @@ -1608,7 +1648,9 @@ fn display_for_python<'db>( /// bought an emitted file a `if TYPE_CHECKING: import types` it never read. /// /// `literal` makes `Literal` a `typing` name like the implicit ones, for a caller that -/// writes the import it needs: a symbolic fold, whose answer is so often a literal type +/// writes the import it needs: a symbolic fold, whose answer is so often a literal type. +/// `written_at` is where the spelling is written, when that is an expression of the source: +/// a name bound in scope there resolves too fn spell_for_python<'db>( db: &'db dyn Db, env: &ProgramEnvironment<'db>, @@ -1616,6 +1658,7 @@ fn spell_for_python<'db>( ty: Type<'db>, min_version: ruff_python_ast::PythonVersion, literal: bool, + written_at: Option<&Expr>, ) -> Option { let spelling = ty.source_spelling_in( db, @@ -1642,8 +1685,15 @@ fn spell_for_python<'db>( modules, typing_names, }; - resolvable_in_output(model, min_version, &spelled, &spelling.bare_classes, &read) - .then_some(spelled) + resolvable_in_output( + model, + min_version, + &spelled, + &spelling.bare_classes, + &read, + written_at, + ) + .then_some(spelled) } /// the `typing` name `name` is, when ty's display writes it for a special form rather than @@ -1657,21 +1707,25 @@ fn typing_name(name: &str, literal: bool) -> Option<&'static str> { /// /// [`spell_for_python`] accounts for two kinds of name it has to import — a qualified /// class's module, and a `typing` name basedpython binds implicitly. What is left is fine -/// only if the output binds it anyway: a builtin, or a name the source's own global scope -/// binds, which the emitted file keeps. Anything else would reach the output unresolved, -/// so the spelling is refused rather than written with a name that is not there. +/// only if the output binds it anyway: a builtin, a name the source's own global scope +/// binds, which the emitted file keeps, or — when the spelling is written in place of a source +/// expression — a name bound in scope there, such as the type parameter of a generic +/// signature it annotates. Anything else would reach the output unresolved, so the spelling +/// is refused rather than written with a name that is not there. fn resolvable_in_output( model: &SemanticModel<'_>, min_version: ruff_python_ast::PythonVersion, spelled: &SynthesizedType, bare_classes: &[String], read: &[String], + written_at: Option<&Expr>, ) -> bool { read.iter().all(|name| { spelled.modules.iter().any(|module| module == name) || spelled.typing_names.contains(&name.as_str()) || bare_classes.contains(name) || model.is_bound_globally(name) + || written_at.is_some_and(|anchor| !model.is_unbound_at(name, anchor)) || ruff_python_stdlib::builtins::is_python_builtin(name, min_version.minor, false) }) } diff --git a/crates/by_transforms/tests/builtin_names_runtime.rs b/crates/by_transforms/tests/builtin_names_runtime.rs index ecea123963..fee87a59dd 100644 --- a/crates/by_transforms/tests/builtin_names_runtime.rs +++ b/crates/by_transforms/tests/builtin_names_runtime.rs @@ -51,12 +51,55 @@ assert n(10, [1, 2]) == 13 and n(10, [1]) == 0 print("ok") "#; +/// a module that binds a builtin at its top level, read by a type test phase 0 lowers and by +/// the `match` polyfill and its runtime definitions after it — under one name, imported +/// ahead of the definitions +const MODULE_BINDING: &str = r#" +def isinstance(a: object, b: object) -> str: + return "mine" + +def f(x: object) -> str: + if x is str: + return "str" + match x: + case [a, b]: + return "pair" + case int(): + return "int" + return "other" + +assert f("s") == "str" and f([1, 2]) == "pair" and f(3) == "int" and f(None) == "other" +assert isinstance(1, int) == "mine" +print("ok") +"#; + +/// a function that declares a builtin `global` and assigns it rebinds the module's, which the +/// definitions pasted into the module read through its globals +const GLOBAL_REBINDING: &str = r#" +def rebind(): + global isinstance + isinstance = lambda a, b: True + +def f(y: int?) -> int: + return y! + +rebind() +assert f(2) == 2 +print("ok") +"#; + fn run_at(interpreter: &Interpreter, target: PythonVersion) { + run_program_at(PROGRAM, interpreter, target); + run_program_at(MODULE_BINDING, interpreter, target); + run_program_at(GLOBAL_REBINDING, interpreter, target); +} + +fn run_program_at(program: &str, interpreter: &Interpreter, target: PythonVersion) { let config = Config { min_version: target, ..Config::default() }; - let transpiled = transpile(PROGRAM, &config).expect("transpile should succeed"); + let transpiled = transpile(program, &config).expect("transpile should succeed"); let output = Command::new(&interpreter.command) .arg("-c") .arg(&transpiled) diff --git a/crates/by_transforms/tests/conversion_runtime.rs b/crates/by_transforms/tests/conversion_runtime.rs index dbccba3276..a714e52bc8 100644 --- a/crates/by_transforms/tests/conversion_runtime.rs +++ b/crates/by_transforms/tests/conversion_runtime.rs @@ -18,7 +18,7 @@ use std::process::Command; -use by_transforms::{Config, PythonVersion, transpile}; +use by_transforms::{Config, PythonVersion, transpile_analysed_at}; mod interpreters; @@ -247,8 +247,9 @@ assert c["x"] == 1 print("ok") "#; -/// transpile `program` for `min_version` and run it on the oldest interpreter found of that -/// version or later that `probe` runs on, asserting it exits cleanly +/// transpile `program` for `min_version`, analysed at that version as a project requiring it +/// is, and run it on the oldest interpreter found of that version or later that `probe` runs +/// on, asserting it exits cleanly fn run_transpiled(program: &str, min_version: PythonVersion, probe: &str, needs: &str) { let Some(interpreter) = interpreters::oldest(min_version, probe, needs) else { return; @@ -257,7 +258,8 @@ fn run_transpiled(program: &str, min_version: PythonVersion, probe: &str, needs: min_version, ..Config::default() }; - let transpiled = transpile(program, &config).expect("transpile should succeed"); + let transpiled = + transpile_analysed_at(program, &config, min_version).expect("transpile should succeed"); let output = Command::new(&interpreter.command) .arg("-c") @@ -285,8 +287,6 @@ fn frozen_displays_run_correctly() { } #[test] -#[ignore = "`transpile` analyses a module at ty's newest python, 3.14, where `frozendict` is no \ - builtin, so no conversion is written; `by transpile` in a 3.15 project writes it"] fn frozendict_displays_run_correctly() { run_transpiled( FROZENDICT_PROGRAM, diff --git a/crates/by_transforms/tests/data_class_runtime.rs b/crates/by_transforms/tests/data_class_runtime.rs index c9cee1a92a..e8f2ba2928 100644 --- a/crates/by_transforms/tests/data_class_runtime.rs +++ b/crates/by_transforms/tests/data_class_runtime.rs @@ -1,9 +1,11 @@ //! Runtime test for the slots a `data class` is given. //! //! A `data class` is a slotted dataclass. `dataclass(slots=True)` is new in python 3.10, so -//! below it the runtime's `_by_dataclass_slots` does what that option does. The transform -//! unit tests pin which of the two a target gets; this test runs both and checks they -//! behave alike. +//! below it the runtime's `_by_dataclass_slots` does what that option does. Below 3.13 the +//! option leaves a method's zero-argument `super()` pointing at the class it replaced, so +//! there a class whose methods can reach it that way is given the helper too. The +//! transform unit tests pin which of the two a target gets; this test runs both and checks +//! they behave alike. use std::process::Command; @@ -81,6 +83,37 @@ fn a_data_class_is_slotted_below_python_310() { } } +/// on 3.10 to 3.12, a class without a zero-argument `super()` keeps `dataclass(slots=True)`, +/// and the one with it is given the helper. the `override` the program writes is `typing`'s +/// from 3.12, so an older interpreter needs `typing_extensions` +#[test] +#[expect( + clippy::print_stderr, + reason = "a skipped test must say why it skipped, or it reads as a pass" +)] +fn a_data_class_reaches_its_slotted_self_through_super_below_python_313() { + let found = interpreters::interpreters() + .into_iter() + .find(|interpreter| { + interpreter.version >= PythonVersion::PY310 + && interpreter.version < PythonVersion::PY313 + && (interpreter.typing_extensions || interpreter.version >= PythonVersion::PY312) + }); + let Some(interpreter) = found else { + eprintln!( + "skipping: no interpreter of python 3.10 to 3.12 found, with `typing_extensions` \ + below 3.12" + ); + return; + }; + let target = if interpreter.typing_extensions { + PythonVersion::PY310 + } else { + PythonVersion::PY312 + }; + run_at(&interpreter, target); +} + /// `dataclass(slots=True)` itself. the `override` the program writes is `typing`'s from 3.12, /// and python's own `slots=True` points a zero-argument `super()` at the class it makes from /// 3.13 on diff --git a/crates/by_transforms/tests/forward_reference_runtime.rs b/crates/by_transforms/tests/forward_reference_runtime.rs index 64eacaec0f..90cf287857 100644 --- a/crates/by_transforms/tests/forward_reference_runtime.rs +++ b/crates/by_transforms/tests/forward_reference_runtime.rs @@ -8,6 +8,10 @@ //! //! Needs an interpreter older than 3.14. From 3.14 annotations are deferred natively, so there //! is nothing to observe, and the test skips rather than pass without checking anything. +//! +//! Below 3.10 the output defers every annotation with `from __future__ import annotations` +//! instead, and there a quote would reach `__annotations__` as a string inside a string, which +//! a library reading the annotation as text takes for something else. use std::process::Command; @@ -16,6 +20,8 @@ use by_transforms::{Config, PythonVersion, transpile}; mod common; use common::python; +mod interpreters; + /// every place an annotation runs with the definition, naming something bound after it const PROGRAM: &str = r#" from typing import TYPE_CHECKING @@ -95,3 +101,69 @@ fn a_forward_reference_does_not_raise_at_import() { ); assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); } + +/// a forward reference in a module transpiled for 3.9, which defers every annotation with +/// `from __future__ import annotations`. quoted as well, `ClassVar[Later]` reached +/// `__annotations__` as `"'ClassVar[Later]'"`, which `dataclasses` does not read as a class +/// variable, so it became a field the constructor required +const DEFERRED_BY_THE_FUTURE_IMPORT: &str = r#" +from dataclasses import dataclass, fields +from typing import ClassVar, TypedDict, get_type_hints + + +def make(x: Later) -> Later: + return x + + +class Holder: + item: Later + + +@dataclass +class Record: + shared: ClassVar[Later] + item: Later + + +class Shape(TypedDict): + item: Later + + +class Later: ... + + +assert make.__annotations__ == {"x": "Later", "return": "Later"}, make.__annotations__ +assert Holder.__annotations__ == {"item": "Later"}, Holder.__annotations__ +assert [field.name for field in fields(Record)] == ["item"], fields(Record) +for target, name in ((make, "x"), (Holder, "item"), (Record, "item"), (Shape, "item")): + assert get_type_hints(target)[name] == Later, get_type_hints(target) +print("ok") +"#; + +#[test] +fn a_forward_reference_under_the_future_import_is_not_quoted() { + let target = PythonVersion::PY39; + let Some(interpreter) = interpreters::oldest(target, "", "") else { + return; + }; + let config = Config { + min_version: target, + ..Config::default() + }; + let transpiled = + transpile(DEFERRED_BY_THE_FUTURE_IMPORT, &config).expect("transpile should succeed"); + let output = Command::new(&interpreter.command) + .arg("-c") + .arg(&transpiled) + .output() + .expect("failed to spawn python"); + + assert!( + output.status.success(), + "program transpiled for {target} failed on {interpreter}:\n--- stdout ---\n{}\n--- stderr ---\n{}\n--- transpiled ---\n{transpiled}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); + interpreters::ran(&interpreter, target); +} diff --git a/crates/by_transforms/tests/imported_conversion_class_runtime.rs b/crates/by_transforms/tests/imported_conversion_class_runtime.rs new file mode 100644 index 0000000000..fda4d77211 --- /dev/null +++ b/crates/by_transforms/tests/imported_conversion_class_runtime.rs @@ -0,0 +1,117 @@ +//! Runtime test for the names a module imports the target class of a cross-module conversion +//! under. +//! +//! A conversion to a class declared in another module imports that class under an alias, so +//! that it never rebinds a name the module means something else by. Two classes of one name +//! from two modules are two targets, and a module can spell a name the alias would otherwise +//! take; either way a shared alias would send a conversion through the wrong class, or through +//! something that is not a class at all, with the checker satisfied. +//! +//! The lowering needs cross-module type information, so the project is built through a typed db +//! rather than the single-file `transpile`. + +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use by_transforms::{Config, PythonVersion, transpile_typed}; +use ruff_db::files::system_path_to_file; +use ruff_db::system::{DbWithWritableSystem, SystemPathBuf}; +use ty_project::{ProjectMetadata, TestDb}; + +mod interpreters; + +const METRIC: &str = r#" +frozen data class Length: + value: float + + class def __of__(cls, value: int) -> Length: + return Length(float(value * 1000)) +"#; + +const IMPERIAL: &str = r#" +frozen data class Length: + value: float + + class def __of__(cls, value: int) -> Length: + return Length(float(value * 12)) +"#; + +/// converts to both `Length`s, and spells the name the metric one's alias would begin with +const MAIN: &str = r#" +import metric +import imperial + +_by_conv__metric__Length = "spelled" + +def in_millimetres(amount: metric.Length) -> float: + return amount.value + +def in_inches(amount: imperial.Length) -> float: + return amount.value + +assert in_millimetres(2) == 2000.0, "the metric conversion" +assert in_inches(2) == 24.0, "the imperial conversion" +assert _by_conv__metric__Length == "spelled" +print("ok") +"#; + +const FILES: &[(&str, &str)] = &[ + ("/metric.by", METRIC), + ("/imperial.by", IMPERIAL), + ("/main.by", MAIN), +]; + +/// transpile every file of the project for `target` into a fresh directory under the cargo +/// temp dir +fn build(target: PythonVersion) -> PathBuf { + let dir = Path::new(env!("CARGO_TARGET_TMPDIR")).join("imported_conversion_class"); + // a stale directory from an earlier run would mask a transpile failure + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).expect("create case dir"); + + let mut db = TestDb::new(ProjectMetadata::new( + ruff_python_ast::name::Name::new_static(""), + SystemPathBuf::from("/"), + )); + db.init_program().expect("program init failed"); + for (path, source) in FILES { + db.write_file(path, source).expect("write file failed"); + } + let config = Config { + min_version: target, + ..Config::default() + }; + for (path, _) in FILES { + let file = system_path_to_file(&db, path).expect("file not in db"); + let transpiled = transpile_typed(&db, file, &config, None) + .unwrap_or_else(|error| panic!("transpile of {path} should succeed: {error}")); + let relative = path.trim_start_matches('/').replace(".by", ".py"); + fs::write(dir.join(relative), transpiled).expect("write module"); + } + dir +} + +#[test] +fn a_conversion_target_is_imported_clear_of_every_other_name() { + let target = PythonVersion::PY310; + let Some(interpreter) = interpreters::oldest(target, "", "") else { + return; + }; + let dir = build(target); + let output = Command::new(&interpreter.command) + .arg("main.py") + .current_dir(&dir) + .output() + .expect("failed to spawn python"); + + assert!( + output.status.success(), + "transpiled program failed on {interpreter}:\n--- stdout ---\n{}\n--- stderr ---\n{}\n--- main.py ---\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + fs::read_to_string(dir.join("main.py")).unwrap_or_default(), + ); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); + interpreters::ran(&interpreter, target); +} diff --git a/crates/by_transforms/tests/imported_extension_runtime.rs b/crates/by_transforms/tests/imported_extension_runtime.rs new file mode 100644 index 0000000000..4ef16c1fec --- /dev/null +++ b/crates/by_transforms/tests/imported_extension_runtime.rs @@ -0,0 +1,147 @@ +//! Runtime test for the names a module binds the backing functions of an imported extension +//! under. +//! +//! An extension member lowers to a module-level backing function, `_by_ext__Foo__bar`, which a +//! module using the member imports from the module declaring the extension. The declaring module +//! names its backing functions clear of what it spells itself, and knows nothing of the modules +//! importing them: one of those may spell the same name, declare a backing function of its own +//! under it — an extension of a different class that happens to be called `Foo` — or emit a +//! `private` symbol under it. Imported under its own name, the function is then shadowed, or +//! shadows the importing module's binding, and a call reaches the wrong function with the +//! checker satisfied. Every place a lowering names an imported backing function is exercised: a +//! member call, a member reached through a trailing lambda's implicit receiver, a conversion an +//! extension supplies, and a conformance's witness table. +//! +//! The lowering needs cross-module type information, so the project is built through a typed db +//! rather than the single-file `transpile`. + +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use by_transforms::{Config, PythonVersion, transpile_typed}; +use ruff_db::files::system_path_to_file; +use ruff_db::system::{DbWithWritableSystem, SystemPathBuf}; +use ty_project::{ProjectMetadata, TestDb}; + +mod interpreters; + +const EXTENSIONS: &str = r#" +class Foo: + def __init__(self, tag: str): + self.tag = tag + +extension Foo: + def bar(self) -> str: + return "a.bar:" + self.tag + + class def __of__(cls, value: str) -> Foo: + return Foo("a:" + value) + +protocol Show: + def show(self) -> str + +extension Show: + def show(self) -> str: + return "a.show" +"#; + +/// spells the name of each backing function it imports, declares a backing function of its +/// own under one of them, and emits a `private` symbol under another +const MAIN: &str = r#" +import extensions +from extensions import Show + +_by_ext__Foo____of__ = "spelled" +_by_ext__Show__show = "spelled" + +class Foo: ... + +extension Foo: + def bar(self) -> str: + return "main.bar" + +private def by_ext__Foo__bar2() -> str: + return "private" + +class Widget: ... + +extension Widget(Show): ... + +def render(value: Show) -> str: + return value.show() + +def apply(fn: extensions.Foo.() -> None): + fn(extensions.Foo("block")) + +seen: str = "" +apply: + seen = bar() + +converted: extensions.Foo = "x" + +assert extensions.Foo("call").bar() == "a.bar:call", "a member call" +assert Foo().bar() == "main.bar", "the module's own member of the same name" +assert seen == "a.bar:block", "a member reached through an implicit receiver" +assert converted.tag == "a:x", "a conversion an imported extension supplies" +assert render(Widget()) == "a.show", "a witness an imported extension supplies" +assert (_by_ext__Foo____of__, _by_ext__Show__show) == ("spelled", "spelled") +assert by_ext__Foo__bar2() == "private" +print("ok") +"#; + +const FILES: &[(&str, &str)] = &[("/extensions.by", EXTENSIONS), ("/main.by", MAIN)]; + +/// transpile every file of the project for `target` into a fresh directory under the cargo +/// temp dir +fn build(target: PythonVersion) -> PathBuf { + let dir = Path::new(env!("CARGO_TARGET_TMPDIR")).join("imported_extension"); + // a stale directory from an earlier run would mask a transpile failure + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).expect("create case dir"); + + let mut db = TestDb::new(ProjectMetadata::new( + ruff_python_ast::name::Name::new_static(""), + SystemPathBuf::from("/"), + )); + db.init_program().expect("program init failed"); + for (path, source) in FILES { + db.write_file(path, source).expect("write file failed"); + } + let config = Config { + min_version: target, + ..Config::default() + }; + for (path, _) in FILES { + let file = system_path_to_file(&db, path).expect("file not in db"); + let transpiled = transpile_typed(&db, file, &config, None) + .unwrap_or_else(|error| panic!("transpile of {path} should succeed: {error}")); + let relative = path.trim_start_matches('/').replace(".by", ".py"); + fs::write(dir.join(relative), transpiled).expect("write module"); + } + dir +} + +#[test] +fn an_imported_backing_function_is_bound_clear_of_the_modules_own_names() { + let target = PythonVersion::PY310; + let Some(interpreter) = interpreters::oldest(target, "", "") else { + return; + }; + let dir = build(target); + let output = Command::new(&interpreter.command) + .arg("main.py") + .current_dir(&dir) + .output() + .expect("failed to spawn python"); + + assert!( + output.status.success(), + "transpiled program failed on {interpreter}:\n--- stdout ---\n{}\n--- stderr ---\n{}\n--- main.py ---\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + fs::read_to_string(dir.join("main.py")).unwrap_or_default(), + ); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); + interpreters::ran(&interpreter, target); +} diff --git a/crates/by_transforms/tests/inferred_annotation_runtime.rs b/crates/by_transforms/tests/inferred_annotation_runtime.rs new file mode 100644 index 0000000000..b6cf2e695f --- /dev/null +++ b/crates/by_transforms/tests/inferred_annotation_runtime.rs @@ -0,0 +1,205 @@ +//! Runtime test for the imports an inferred annotation needs. +//! +//! An annotation the lowering writes can name a class the source never imported under that +//! name — `decimal.Decimal` for a value built by a `Decimal` imported as `Dec`. The module may +//! be one the source imports only for a checker, to keep it out of an import cycle, so where +//! python evaluates a class body's annotations as the class is made — from 3.10 until 3.14, +//! without `from __future__ import annotations` — the annotation is written as a string, and +//! only `typing.get_type_hints` evaluates it. Where the module's imports are deferred, the +//! module the annotation names is imported with them, and runs only when `get_type_hints` +//! asks; where they are not, it is imported for a checker alone. + +use std::fs; +use std::path::Path; +use std::process::Command; + +use by_transforms::{Config, PythonVersion, transpile, transpile_typed}; +use ruff_db::files::system_path_to_file; +use ruff_db::system::{DbWithWritableSystem, SystemPathBuf}; +use ty_project::{ProjectMetadata, TestDb}; + +mod interpreters; +use interpreters::Interpreter; + +const PROGRAM: &str = r#" +import typing +from decimal import Decimal as Dec + +class A: + d = Dec(1) + +assert typing.get_type_hints(A)["d"] === type(A.d) +print("ok") +"#; + +/// with imports not deferred, the module is imported for a checker alone, so the class is +/// made and the annotation stays the string it is written as +const PROGRAM_EAGER: &str = r#" +from decimal import Decimal as Dec + +class A: + d = Dec(1) + +assert A.__annotations__["d"] == "decimal.Decimal" +print("ok") +"#; + +fn run_at( + interpreter: &Interpreter, + target: PythonVersion, + lazy_imports: bool, + program: &str, + annotation: &str, +) { + let config = Config { + min_version: target, + lazy_imports, + ..Config::default() + }; + let transpiled = transpile(program, &config).expect("transpile should succeed"); + assert!( + transpiled.contains(&format!("d: {annotation} = Dec(1)")), + "the annotation names the module:\n{transpiled}" + ); + let output = Command::new(&interpreter.command) + .arg("-c") + .arg(&transpiled) + .output() + .expect("failed to spawn python"); + assert!( + output.status.success(), + "program transpiled for {target} failed on {interpreter}:\n--- stdout ---\n{}\n--- stderr ---\n{}\n--- transpiled ---\n{transpiled}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); + interpreters::ran(interpreter, target); +} + +/// from 3.10 nothing defers the annotations, so the class body would evaluate this one as the +/// class is made, were it not a string +#[test] +fn an_inferred_annotation_names_an_imported_module() { + if let Some(interpreter) = interpreters::oldest(PythonVersion::PY310, "", "") { + run_at( + &interpreter, + PythonVersion::PY310, + true, + PROGRAM, + "\"decimal.Decimal\"", + ); + run_at( + &interpreter, + PythonVersion::PY310, + false, + PROGRAM_EAGER, + "\"decimal.Decimal\"", + ); + } +} + +/// below 3.10 the annotations are deferred, and `typing.get_type_hints` still evaluates them +#[test] +fn an_inferred_annotation_resolves_when_deferred() { + if let Some(interpreter) = interpreters::oldest( + PythonVersion::PY39, + "import typing_extensions", + "with `typing_extensions`", + ) { + run_at( + &interpreter, + PythonVersion::PY39, + true, + PROGRAM, + "decimal.Decimal", + ); + } +} + +/// `x_mod` imports `m`, and so `b` imports it for a checker alone. `m`'s class holds a value +/// `b` declares an `x_mod.Thing`, and the annotation inferred for it names `x_mod`: evaluated as +/// the class is made, it would import `x_mod` while `m` is half made, and `x_mod`'s +/// `from m import C` would find no `C` +const CYCLE: &[(&str, &str)] = &[ + ( + "/main.by", + "import m\n\nassert m.C.x.val() == 7\nprint(\"ok\")\n", + ), + ("/m.by", "from b import make\n\nclass C:\n x = make()\n"), + ( + "/b.by", + r#"from typing import TYPE_CHECKING +if TYPE_CHECKING: + import x_mod + +class Impl: + def val(self) -> int: + return 7 + +def make() -> x_mod.Thing: + return Impl() +"#, + ), + ( + "/x_mod.by", + r#"from typing import Protocol +from m import C + +class Thing(Protocol): + def val(self) -> int: ... + +def use(c: C): ... +"#, + ), +]; + +/// the module an inferred annotation names is not imported into a cycle the source kept it +/// out of, on a python that evaluates the annotation as the class is made +#[test] +fn an_inferred_annotation_adds_no_import_cycle() { + let Some(interpreter) = interpreters::oldest(PythonVersion::PY310, "", "") else { + return; + }; + let mut db = TestDb::new(ProjectMetadata::new( + ruff_python_ast::name::Name::new_static(""), + SystemPathBuf::from("/"), + )); + db.init_program().expect("program init failed"); + for (path, source) in CYCLE { + db.write_file(path, source).expect("write file failed"); + } + for lazy_imports in [true, false] { + let dir = Path::new(env!("CARGO_TARGET_TMPDIR")) + .join(format!("inferred_annotation_cycle_{lazy_imports}")); + // a stale directory from an earlier run would mask a transpile failure + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).expect("create case dir"); + let config = Config { + min_version: PythonVersion::PY310, + lazy_imports, + ..Config::default() + }; + for (path, _) in CYCLE { + let file = system_path_to_file(&db, path).expect("file not in db"); + let transpiled = transpile_typed(&db, file, &config, None) + .unwrap_or_else(|error| panic!("transpile of {path} should succeed: {error}")); + let relative = path.trim_start_matches('/').replace(".by", ".py"); + fs::write(dir.join(relative), transpiled).expect("write module"); + } + let m = fs::read_to_string(dir.join("m.py")).expect("read m"); + assert!(m.contains("x: \"x_mod.Thing\" = make()"), "{m}"); + let output = Command::new(&interpreter.command) + .arg("main.py") + .current_dir(&dir) + .output() + .expect("failed to spawn python"); + assert!( + output.status.success(), + "program failed on {interpreter}, lazy imports {lazy_imports}:\n--- stdout ---\n{}\n--- stderr ---\n{}\n--- m ---\n{m}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); + interpreters::ran(&interpreter, PythonVersion::PY310); + } +} diff --git a/crates/by_transforms/tests/lazy_import_runtime.rs b/crates/by_transforms/tests/lazy_import_runtime.rs index 64a531be45..3d817b983b 100644 --- a/crates/by_transforms/tests/lazy_import_runtime.rs +++ b/crates/by_transforms/tests/lazy_import_runtime.rs @@ -254,6 +254,30 @@ assert typing.get_type_hints(half)["value"] === fractions.Fraction, "the string print("ok") "#; +/// a module that binds names of builtins the proxy reads when it resolves, formats and +/// iterates. the proxy is pasted into the module and reads the builtins through its +/// globals, so it reads them under names of its own +const BUILTIN_NAMES_MAIN: &str = r#" +from decimal import Decimal +from math import pi +from string import digits + +def getattr(o: object, name: str) -> int: + return 42 + +def format(value: object, spec: str) -> str: + return "shadowed" + +def iter(o: object) -> int: + return 0 + +assert Decimal(1) + 1 == Decimal(2), "resolving the proxy calls `getattr`" +assert f"{pi:.2f}" == "3.14", "formatting the proxy calls `format`" +assert [c for c in digits][:2] == ["0", "1"], "iterating the proxy calls `iter`" +assert getattr(1, "x") == 42 and format(1, "") == "shadowed" and iter(1) == 0 +print("ok") +"#; + /// the interpreter to run the transpiled output on: the oldest found that runs what the /// transpiler targets by default, 3.10 — a dataclass's `KW_ONLY` among it. none is a skip, said so fn python() -> Option { @@ -379,3 +403,12 @@ fn a_string_annotation_resolves_to_the_imported_object() { ); run_main(&python, &dir); } + +#[test] +fn the_proxy_reads_builtins_the_module_binds_for_itself() { + let Some(python) = python() else { + return; + }; + let dir = build_case("lazy_builtin_names", &[("main", BUILTIN_NAMES_MAIN)]); + run_main(&python, &dir); +} diff --git a/crates/ruff_linter/resources/test/fixtures/pyflakes/F821_basedpython.by b/crates/ruff_linter/resources/test/fixtures/pyflakes/F821_basedpython.by index 5d56abd268..752610faf6 100644 --- a/crates/ruff_linter/resources/test/fixtures/pyflakes/F821_basedpython.by +++ b/crates/ruff_linter/resources/test/fixtures/pyflakes/F821_basedpython.by @@ -41,6 +41,14 @@ def check_empty(x: str | None) -> asserts not x: def guard() -> undefined_place is int: return True +# a negated predicate is refused by the checker, and names its place the same way: +# only the type it tests against is a reference +def not_guard(x: object) -> x is not int: + return True + +def not_guard_undefined(x: object) -> x is not UndefinedNegatedType: + return True + # the type in `asserts x is T` is a real type reference — flagged when undefined def check_type(x: object) -> asserts x is UndefinedGuardType: raise ValueError diff --git a/crates/ruff_linter/src/checkers/ast/mod.rs b/crates/ruff_linter/src/checkers/ast/mod.rs index 87868f759a..eca5b18b1e 100644 --- a/crates/ruff_linter/src/checkers/ast/mod.rs +++ b/crates/ruff_linter/src/checkers/ast/mod.rs @@ -33,7 +33,8 @@ use ruff_db::diagnostic::{Annotation, Diagnostic, DiagnosticTag, IntoDiagnosticM use ruff_diagnostics::{Applicability, Fix, IsolationLevel}; use ruff_notebook::{CellOffsets, NotebookIndex}; use ruff_python_ast::helpers::{ - ReturnGuardForm, collect_import_from_member, is_docstring_stmt, return_guards, to_module_path, + ReturnGuardForm, collect_import_from_member, is_docstring_stmt, negated_narrowing_predicate, + return_guards, to_module_path, }; use ruff_python_ast::identifier::Identifier; use ruff_python_ast::name::QualifiedName; @@ -1430,6 +1431,14 @@ impl<'a> Visitor<'a> for Checker<'a> { ReturnGuardForm::Asserts { .. } => None, }) .collect(), + // a negated predicate, `-> x is not T`, is refused by the checker, and + // names its place the same way + (None, Some(returns)) + if self.source_type.is_basedpython() + && let Some(negated) = negated_narrowing_predicate(returns) => + { + negated.comparators.iter().collect() + } (None, Some(returns)) => vec![returns], (None, None) => Vec::new(), }; diff --git a/crates/ruff_linter/src/rules/pyflakes/snapshots/ruff_linter__rules__pyflakes__tests__undefined-name_F821_basedpython.by.snap b/crates/ruff_linter/src/rules/pyflakes/snapshots/ruff_linter__rules__pyflakes__tests__undefined-name_F821_basedpython.by.snap index 306869c9ac..95b6bd46f6 100644 --- a/crates/ruff_linter/src/rules/pyflakes/snapshots/ruff_linter__rules__pyflakes__tests__undefined-name_F821_basedpython.by.snap +++ b/crates/ruff_linter/src/rules/pyflakes/snapshots/ruff_linter__rules__pyflakes__tests__undefined-name_F821_basedpython.by.snap @@ -42,202 +42,212 @@ F821 Undefined name `UndefinedType` 29 | # `-> asserts x` names the place a call narrows, not a type: the name is the | +F821 Undefined name `UndefinedNegatedType` + --> F821_basedpython.by:49:48 + | +47 | return True +48 | +49 | def not_guard_undefined(x: object) -> x is not UndefinedNegatedType: + | ^^^^^^^^^^^^^^^^^^^^ +50 | return True + | + F821 Undefined name `UndefinedGuardType` - --> F821_basedpython.by:45:43 + --> F821_basedpython.by:53:43 | -44 | # the type in `asserts x is T` is a real type reference — flagged when undefined -45 | def check_type(x: object) -> asserts x is UndefinedGuardType: +52 | # the type in `asserts x is T` is a real type reference — flagged when undefined +53 | def check_type(x: object) -> asserts x is UndefinedGuardType: | ^^^^^^^^^^^^^^^^^^ -46 | raise ValueError +54 | raise ValueError | F821 Undefined name `UndefinedMember` - --> F821_basedpython.by:62:27 + --> F821_basedpython.by:70:27 | -60 | Defined = int -61 | good_protocol: protocol(a: Defined; def m(self, n: Defined) -> Defined) -62 | bad_protocol: protocol(a: UndefinedMember; def m(self) -> UndefinedReturn) +68 | Defined = int +69 | good_protocol: protocol(a: Defined; def m(self, n: Defined) -> Defined) +70 | bad_protocol: protocol(a: UndefinedMember; def m(self) -> UndefinedReturn) | ^^^^^^^^^^^^^^^ -63 | -64 | # a match type's `case` captures are type variables, and its bodies are type +71 | +72 | # a match type's `case` captures are type variables, and its bodies are type | F821 Undefined name `UndefinedReturn` - --> F821_basedpython.by:62:59 + --> F821_basedpython.by:70:59 | -60 | Defined = int -61 | good_protocol: protocol(a: Defined; def m(self, n: Defined) -> Defined) -62 | bad_protocol: protocol(a: UndefinedMember; def m(self) -> UndefinedReturn) +68 | Defined = int +69 | good_protocol: protocol(a: Defined; def m(self, n: Defined) -> Defined) +70 | bad_protocol: protocol(a: UndefinedMember; def m(self) -> UndefinedReturn) | ^^^^^^^^^^^^^^^ -63 | -64 | # a match type's `case` captures are type variables, and its bodies are type +71 | +72 | # a match type's `case` captures are type variables, and its bodies are type | F821 Undefined name `UndefinedBound` - --> F821_basedpython.by:73:20 + --> F821_basedpython.by:81:20 | -71 | (NDTuple[T, *Rest],) * Dim -72 | -73 | type BadBound[*Ts: UndefinedBound] = tuple[*Ts] +79 | (NDTuple[T, *Rest],) * Dim +80 | +81 | type BadBound[*Ts: UndefinedBound] = tuple[*Ts] | ^^^^^^^^^^^^^^ -74 | -75 | # a pack's bound is a type expression wherever it appears, so names inside the +82 | +83 | # a pack's bound is a type expression wherever it appears, so names inside the | F821 Undefined name `UndefinedElement` - --> F821_basedpython.by:77:29 + --> F821_basedpython.by:85:29 | -75 | # a pack's bound is a type expression wherever it appears, so names inside the -76 | # starred whole-pack forms and a keyword-variadic pack's bound resolve too -77 | type WholeTuple[*Ts: *(int, UndefinedElement)] = tuple[*Ts] +83 | # a pack's bound is a type expression wherever it appears, so names inside the +84 | # starred whole-pack forms and a keyword-variadic pack's bound resolve too +85 | type WholeTuple[*Ts: *(int, UndefinedElement)] = tuple[*Ts] | ^^^^^^^^^^^^^^^^ -78 | type EveryField[**Kwargs: UndefinedFieldBound] = int -79 | type WholeShape[**Kwargs: **{"a": UndefinedField}] = int +86 | type EveryField[**Kwargs: UndefinedFieldBound] = int +87 | type WholeShape[**Kwargs: **{"a": UndefinedField}] = int | F821 Undefined name `UndefinedFieldBound` - --> F821_basedpython.by:78:27 + --> F821_basedpython.by:86:27 | -76 | # starred whole-pack forms and a keyword-variadic pack's bound resolve too -77 | type WholeTuple[*Ts: *(int, UndefinedElement)] = tuple[*Ts] -78 | type EveryField[**Kwargs: UndefinedFieldBound] = int +84 | # starred whole-pack forms and a keyword-variadic pack's bound resolve too +85 | type WholeTuple[*Ts: *(int, UndefinedElement)] = tuple[*Ts] +86 | type EveryField[**Kwargs: UndefinedFieldBound] = int | ^^^^^^^^^^^^^^^^^^^ -79 | type WholeShape[**Kwargs: **{"a": UndefinedField}] = int -80 | type BoundsAreFine[*Ts: *(int, str), **Kwargs: **{"a": int}] = int +87 | type WholeShape[**Kwargs: **{"a": UndefinedField}] = int +88 | type BoundsAreFine[*Ts: *(int, str), **Kwargs: **{"a": int}] = int | F821 Undefined name `UndefinedField` - --> F821_basedpython.by:79:35 + --> F821_basedpython.by:87:35 | -77 | type WholeTuple[*Ts: *(int, UndefinedElement)] = tuple[*Ts] -78 | type EveryField[**Kwargs: UndefinedFieldBound] = int -79 | type WholeShape[**Kwargs: **{"a": UndefinedField}] = int +85 | type WholeTuple[*Ts: *(int, UndefinedElement)] = tuple[*Ts] +86 | type EveryField[**Kwargs: UndefinedFieldBound] = int +87 | type WholeShape[**Kwargs: **{"a": UndefinedField}] = int | ^^^^^^^^^^^^^^ -80 | type BoundsAreFine[*Ts: *(int, str), **Kwargs: **{"a": int}] = int +88 | type BoundsAreFine[*Ts: *(int, str), **Kwargs: **{"a": int}] = int | F821 Undefined name `Protocol` - --> F821_basedpython.by:88:15 + --> F821_basedpython.by:96:15 | -86 | implicit_value = Optional -87 | # a `typing` member basedpython does *not* auto-import is still flagged -88 | not_implicit: Protocol +94 | implicit_value = Optional +95 | # a `typing` member basedpython does *not* auto-import is still flagged +96 | not_implicit: Protocol | ^^^^^^^^ -89 | -90 | # `Some` is the present-case optional constructor, lowered to the injected +97 | +98 | # `Some` is the present-case optional constructor, lowered to the injected | F821 Undefined name `Character` - --> F821_basedpython.by:99:7 + --> F821_basedpython.by:107:7 | - 97 | overlapping_arg: Overlapping[int] - 98 | # in value position they are ordinary names — still flagged when unbound - 99 | print(Character, Overlapping) +105 | overlapping_arg: Overlapping[int] +106 | # in value position they are ordinary names — still flagged when unbound +107 | print(Character, Overlapping) | ^^^^^^^^^ -100 | -101 | # the return-value markers are decorators, implicitly available where a value is +108 | +109 | # the return-value markers are decorators, implicitly available where a value is | F821 Undefined name `Overlapping` - --> F821_basedpython.by:99:18 + --> F821_basedpython.by:107:18 | - 97 | overlapping_arg: Overlapping[int] - 98 | # in value position they are ordinary names — still flagged when unbound - 99 | print(Character, Overlapping) +105 | overlapping_arg: Overlapping[int] +106 | # in value position they are ordinary names — still flagged when unbound +107 | print(Character, Overlapping) | ^^^^^^^^^^^ -100 | -101 | # the return-value markers are decorators, implicitly available where a value is +108 | +109 | # the return-value markers are decorators, implicitly available where a value is | F821 Undefined name `ignorable_return_value` - --> F821_basedpython.by:114:17 + --> F821_basedpython.by:122:17 | -113 | # in a type position they are ordinary names — flagged when unbound -114 | marker_as_type: ignorable_return_value # F821 +121 | # in a type position they are ordinary names — flagged when unbound +122 | marker_as_type: ignorable_return_value # F821 | ^^^^^^^^^^^^^^^^^^^^^^ -115 | -116 | # known gap: an enum member reached by context-sensitive resolution is an +123 | +124 | # known gap: an enum member reached by context-sensitive resolution is an | F821 Undefined name `Bleu` - --> F821_basedpython.by:140:19 + --> F821_basedpython.by:148:19 | -139 | chosen: Colour = Green -140 | missing: Colour = Bleu # F821 +147 | chosen: Colour = Green +148 | missing: Colour = Bleu # F821 | ^^^^ F821 Undefined name `a_receiver_member` - --> F821_basedpython.by:162:1 + --> F821_basedpython.by:170:1 | -161 | # but the deferral stops at the block: the same name outside one is reported -162 | a_receiver_member("x") # F821 +169 | # but the deferral stops at the block: the same name outside one is reported +170 | a_receiver_member("x") # F821 | ^^^^^^^^^^^^^^^^^ F821 Undefined name `UndefinedPattern` - --> F821_basedpython.by:193:12 + --> F821_basedpython.by:201:12 | -191 | # an unbound name inside a pattern is still reported -192 | def undefined_in_pattern(value: object) -> None: -193 | if let UndefinedPattern(captured) := value: # F821 +199 | # an unbound name inside a pattern is still reported +200 | def undefined_in_pattern(value: object) -> None: +201 | if let UndefinedPattern(captured) := value: # F821 | ^^^^^^^^^^^^^^^^ -194 | print(captured) +202 | print(captured) | F821 Undefined name `UndefinedInterface` - --> F821_basedpython.by:217:32 + --> F821_basedpython.by:225:32 | -215 | # the extended type is looked up rather than resolved, so an undefined one is a -216 | # known gap here — `by check` reports it as an unresolved type -217 | extension UndefinedImplemented(UndefinedInterface): # F821 +223 | # the extended type is looked up rather than resolved, so an undefined one is a +224 | # known gap here — `by check` reports it as an unresolved type +225 | extension UndefinedImplemented(UndefinedInterface): # F821 | ^^^^^^^^^^^^^^^^^^ -218 | pass +226 | pass | F821 Undefined name `bare_name` - --> F821_basedpython.by:237:21 + --> F821_basedpython.by:245:21 | -236 | # but the suppression is only for that shape, so each of these is still reported -237 | Book.objects.filter(bare_name) # F821 +244 | # but the suppression is only for that shape, so each of these is still reported +245 | Book.objects.filter(bare_name) # F821 | ^^^^^^^^^ -238 | Book.objects.annotate(not_a_lookup_method.x == 1) # F821 -239 | Book.objects.filter(data[undefined_index] == 1) # F821 +246 | Book.objects.annotate(not_a_lookup_method.x == 1) # F821 +247 | Book.objects.filter(data[undefined_index] == 1) # F821 | F821 Undefined name `not_a_lookup_method` - --> F821_basedpython.by:238:23 + --> F821_basedpython.by:246:23 | -236 | # but the suppression is only for that shape, so each of these is still reported -237 | Book.objects.filter(bare_name) # F821 -238 | Book.objects.annotate(not_a_lookup_method.x == 1) # F821 +244 | # but the suppression is only for that shape, so each of these is still reported +245 | Book.objects.filter(bare_name) # F821 +246 | Book.objects.annotate(not_a_lookup_method.x == 1) # F821 | ^^^^^^^^^^^^^^^^^^^ -239 | Book.objects.filter(data[undefined_index] == 1) # F821 -240 | Book.objects.filter(no_lookup_for_ne != 1) # F821 +247 | Book.objects.filter(data[undefined_index] == 1) # F821 +248 | Book.objects.filter(no_lookup_for_ne != 1) # F821 | F821 Undefined name `undefined_index` - --> F821_basedpython.by:239:26 + --> F821_basedpython.by:247:26 | -237 | Book.objects.filter(bare_name) # F821 -238 | Book.objects.annotate(not_a_lookup_method.x == 1) # F821 -239 | Book.objects.filter(data[undefined_index] == 1) # F821 +245 | Book.objects.filter(bare_name) # F821 +246 | Book.objects.annotate(not_a_lookup_method.x == 1) # F821 +247 | Book.objects.filter(data[undefined_index] == 1) # F821 | ^^^^^^^^^^^^^^^ -240 | Book.objects.filter(no_lookup_for_ne != 1) # F821 +248 | Book.objects.filter(no_lookup_for_ne != 1) # F821 | F821 Undefined name `no_lookup_for_ne` - --> F821_basedpython.by:240:21 + --> F821_basedpython.by:248:21 | -238 | Book.objects.annotate(not_a_lookup_method.x == 1) # F821 -239 | Book.objects.filter(data[undefined_index] == 1) # F821 -240 | Book.objects.filter(no_lookup_for_ne != 1) # F821 +246 | Book.objects.annotate(not_a_lookup_method.x == 1) # F821 +247 | Book.objects.filter(data[undefined_index] == 1) # F821 +248 | Book.objects.filter(no_lookup_for_ne != 1) # F821 | ^^^^^^^^^^^^^^^^ F821 Undefined name `UndefinedRaised` - --> F821_basedpython.by:245:31 + --> F821_basedpython.by:253:31 | -243 | # a `raises` clause is an ordinary type expression, so an undefined name written -244 | # there is a real reference and is reported -245 | def raises_undefined() raises UndefinedRaised: # F821 +251 | # a `raises` clause is an ordinary type expression, so an undefined name written +252 | # there is a real reference and is reported +253 | def raises_undefined() raises UndefinedRaised: # F821 | ^^^^^^^^^^^^^^^ -246 | raise ValueError +254 | raise ValueError | diff --git a/crates/ruff_python_ast/src/helpers.rs b/crates/ruff_python_ast/src/helpers.rs index 72eba98b4f..9c8c6bff97 100644 --- a/crates/ruff_python_ast/src/helpers.rs +++ b/crates/ruff_python_ast/src/helpers.rs @@ -1237,6 +1237,25 @@ pub fn return_guards(function: &ast::StmtFunctionDef) -> Option Option<&Name> { + let Expr::Named(named) = argument else { + return None; + }; + let mut target = named.target.as_ref(); + while let Expr::Starred(starred) = target { + target = &starred.value; + } + target.as_name_expr().map(|name| &name.id) +} + +/// basedpython: the narrowing predicate a return type spells, `place is T` — a function's +/// return annotation, or the return of a callable type, `(x: object) -> (x is int)`. +pub fn predicate_guard(returns: &Expr) -> Option> { let Expr::Compare(compare) = returns else { return None; }; @@ -1246,10 +1265,27 @@ pub fn return_guards(function: &ast::StmtFunctionDef) -> Option a is not T`. +/// +/// A predicate lowers to `TypeIs[T]`, which narrows its argument to `T` where the call returns +/// true and away from `T` where it returns false. Python's typing has no form that narrows the +/// other way round, so a negated predicate has no meaning to lower to, and the checker and the +/// transpiler both refuse one. `-> asserts a is not T` is a different annotation, and allowed. +pub fn negated_narrowing_predicate(annotation: &Expr) -> Option<&ast::ExprCompare> { + let Expr::Compare(compare) = annotation else { + return None; + }; + (matches!(&*compare.ops, [ast::CmpOp::IsNot]) + && compare.comparators.len() == 1 + && is_guard_place(&compare.left)) + .then_some(compare) } /// One term of an `asserts` annotation: a place, a negated place, or a place tested with `is`. diff --git a/crates/ruff_python_formatter/resources/test/fixtures/ruff/callable.by b/crates/ruff_python_formatter/resources/test/fixtures/ruff/callable.by index 4503c436f3..d9db1d4b65 100644 --- a/crates/ruff_python_formatter/resources/test/fixtures/ruff/callable.by +++ b/crates/ruff_python_formatter/resources/test/fixtures/ruff/callable.by @@ -37,3 +37,8 @@ type Commented = ( # own-line between parameters str, ) -> bool + + +# a callable type's return binds tighter than a comparison, so the narrowing +# predicate it returns keeps its parentheses +def narrows(check: (first: object, second: object) -> (second is int)): ... diff --git a/crates/ruff_python_formatter/resources/test/fixtures/ruff/wrapped_postfix.by b/crates/ruff_python_formatter/resources/test/fixtures/ruff/wrapped_postfix.by index eb58237a4a..5ab10b01aa 100644 --- a/crates/ruff_python_formatter/resources/test/fixtures/ruff/wrapped_postfix.by +++ b/crates/ruff_python_formatter/resources/test/fixtures/ruff/wrapped_postfix.by @@ -12,6 +12,24 @@ o: dict[str, int?] p: ((int) -> str)? q: (int) -> str? +# a type test's type takes the marker and the result form after it, while a +# marker after a parenthesized comparison or a value comparison is over the +# whole comparison +r = x is int? +s = x is int ? str +t = x is not int ? str | bytes +u = (x is int) ? str +v = (x is int)? +w = x == y? +z = x == (y?) +if some_long_value_name is SomeRatherLongTypeName ? SomeRatherLongErrorName | AnotherErrorName: + pass +if (some_long_value_name is SomeRatherLongTypeName) ? SomeRatherLongErrorName | AnotherErrorName: + pass +# a type test broken over lines keeps its type and the result form after it on +# one line, since both are the type the value is tested against +long_result = some_long_value_name is not SomeRatherLongTypeName ? SomeRatherLongErrorName | AnotherErrorName + def f(x: int?) -> None: ... diff --git a/crates/ruff_python_formatter/src/expression/binary_like.rs b/crates/ruff_python_formatter/src/expression/binary_like.rs index 77738c02aa..c7f42f7c89 100644 --- a/crates/ruff_python_formatter/src/expression/binary_like.rs +++ b/crates/ruff_python_formatter/src/expression/binary_like.rs @@ -51,6 +51,7 @@ impl<'a> BinaryLike<'a> { expression: &compare.left, leading_comments, }, + OperatorPrecedence::Comparator, comments, trivia, parts, @@ -76,7 +77,13 @@ impl<'a> BinaryLike<'a> { trailing_comments: &[], })); - rec(Operand::Middle { expression }, comments, trivia, parts); + rec( + Operand::Middle { expression }, + OperatorPrecedence::Comparator, + comments, + trivia, + parts, + ); } parts.push(OperandOrOperator::Operator(Operator { @@ -92,6 +99,7 @@ impl<'a> BinaryLike<'a> { expression: last_expression, trailing_comments, }, + OperatorPrecedence::Comparator, comments, trivia, parts, @@ -115,6 +123,7 @@ impl<'a> BinaryLike<'a> { expression: left, leading_comments, }, + OperatorPrecedence::BooleanOperation, comments, trivia, parts, @@ -127,7 +136,13 @@ impl<'a> BinaryLike<'a> { if let Some((right, middle)) = rest.split_last() { for expression in middle { - rec(Operand::Middle { expression }, comments, trivia, parts); + rec( + Operand::Middle { expression }, + OperatorPrecedence::BooleanOperation, + comments, + trivia, + parts, + ); parts.push(OperandOrOperator::Operator(Operator { symbol: OperatorSymbol::Bool(bool_expression.op), trailing_comments: &[], @@ -139,6 +154,7 @@ impl<'a> BinaryLike<'a> { expression: right, trailing_comments, }, + OperatorPrecedence::BooleanOperation, comments, trivia, parts, @@ -160,6 +176,7 @@ impl<'a> BinaryLike<'a> { leading_comments, expression: &binary.left, }, + OperatorPrecedence::from(binary.op), comments, trivia, parts, @@ -175,19 +192,36 @@ impl<'a> BinaryLike<'a> { expression: binary.right.as_ref(), trailing_comments, }, + OperatorPrecedence::from(binary.op), comments, trivia, parts, ); } + /// `operand` of an operator of `precedence`, flattened into `parts` when it is itself an + /// unparenthesized operation. an operand that binds looser than the operator it belongs + /// to is kept whole: python has no such operand without parentheses, but in basedpython + /// a type test's type takes a result form, `x is T ? E` being `x is (T ? E)`, and + /// flattening would break the line at the `?` as if it were over the whole comparison fn rec<'a>( operand: Operand<'a>, + precedence: OperatorPrecedence, comments: &'a Comments, trivia: &TriviaRanges, parts: &mut SmallVec<[OperandOrOperator<'a>; 8]>, ) { let expression = operand.expression(); + let binds_looser = match expression { + Expr::BinOp(binary) => OperatorPrecedence::from(binary.op) > precedence, + Expr::Compare(_) => OperatorPrecedence::Comparator > precedence, + Expr::BoolOp(_) => OperatorPrecedence::BooleanOperation > precedence, + _ => false, + }; + if binds_looser { + parts.push(OperandOrOperator::Operand(operand)); + return; + } match expression { Expr::BinOp(binary) if !trivia.parenthesized().contains(expression.range()) => { let leading_comments = operand diff --git a/crates/ruff_python_formatter/tests/snapshots/format@callable.by.snap b/crates/ruff_python_formatter/tests/snapshots/format@callable.by.snap index 5f3861ec8a..a77c81a5c4 100644 --- a/crates/ruff_python_formatter/tests/snapshots/format@callable.by.snap +++ b/crates/ruff_python_formatter/tests/snapshots/format@callable.by.snap @@ -43,6 +43,11 @@ type Commented = ( # own-line between parameters str, ) -> bool + + +# a callable type's return binds tighter than a comparison, so the narrowing +# predicate it returns keeps its parentheses +def narrows(check: (first: object, second: object) -> (second is int)): ... ``` ## Output @@ -86,4 +91,9 @@ type Commented = ( # own-line between parameters str, ) -> bool + + +# a callable type's return binds tighter than a comparison, so the narrowing +# predicate it returns keeps its parentheses +def narrows(check: (first: object, second: object) -> (second is int)): ... ``` diff --git a/crates/ruff_python_formatter/tests/snapshots/format@wrapped_postfix.by.snap b/crates/ruff_python_formatter/tests/snapshots/format@wrapped_postfix.by.snap index 4899aa631e..72e61ca6ac 100644 --- a/crates/ruff_python_formatter/tests/snapshots/format@wrapped_postfix.by.snap +++ b/crates/ruff_python_formatter/tests/snapshots/format@wrapped_postfix.by.snap @@ -18,6 +18,24 @@ o: dict[str, int?] p: ((int) -> str)? q: (int) -> str? +# a type test's type takes the marker and the result form after it, while a +# marker after a parenthesized comparison or a value comparison is over the +# whole comparison +r = x is int? +s = x is int ? str +t = x is not int ? str | bytes +u = (x is int) ? str +v = (x is int)? +w = x == y? +z = x == (y?) +if some_long_value_name is SomeRatherLongTypeName ? SomeRatherLongErrorName | AnotherErrorName: + pass +if (some_long_value_name is SomeRatherLongTypeName) ? SomeRatherLongErrorName | AnotherErrorName: + pass +# a type test broken over lines keeps its type and the result form after it on +# one line, since both are the type the value is tested against +long_result = some_long_value_name is not SomeRatherLongTypeName ? SomeRatherLongErrorName | AnotherErrorName + def f(x: int?) -> None: ... @@ -50,6 +68,32 @@ o: dict[str, int?] p: ((int) -> str)? q: (int) -> str? +# a type test's type takes the marker and the result form after it, while a +# marker after a parenthesized comparison or a value comparison is over the +# whole comparison +r = x is int? +s = x is int ? str +t = x is not int ? str | bytes +u = (x is int) ? str +v = (x is int)? +w = x == y? +z = x == (y?) +if ( + some_long_value_name + is SomeRatherLongTypeName ? SomeRatherLongErrorName | AnotherErrorName +): + pass +if ( + some_long_value_name is SomeRatherLongTypeName +) ? SomeRatherLongErrorName | AnotherErrorName: + pass +# a type test broken over lines keeps its type and the result form after it on +# one line, since both are the type the value is tested against +long_result = ( + some_long_value_name + is not SomeRatherLongTypeName ? SomeRatherLongErrorName | AnotherErrorName +) + def f(x: int?) -> None: ... diff --git a/crates/ruff_python_parser/src/parser/expression.rs b/crates/ruff_python_parser/src/parser/expression.rs index 42b006d541..a47d9cbc9e 100644 --- a/crates/ruff_python_parser/src/parser/expression.rs +++ b/crates/ruff_python_parser/src/parser/expression.rs @@ -2356,6 +2356,7 @@ impl<'src> Parser<'src> { } } record_identity(&mut identity_ops, 0, self.at_identity_operator(op)); + let mut at_type_test = self.at_type_test(op); self.bump_cmp_op(op); let comparators_snapshot = self.expr_scratch.snapshot(); @@ -2366,13 +2367,15 @@ impl<'src> Parser<'src> { loop { progress.assert_progressing(self); - let comparator = self - .parse_binary_expression_or_higher( - OperatorPrecedence::ComparisonsMembershipIdentity, - context, - ) - .expr; - self.expr_scratch.push(comparator); + let comparator_start = self.node_start(); + let mut comparator = self.parse_binary_expression_or_higher( + OperatorPrecedence::ComparisonsMembershipIdentity, + context, + ); + if at_type_test { + comparator = self.parse_type_test_optional(comparator, comparator_start, context); + } + self.expr_scratch.push(comparator.expr); let next_token = self.current_token_kind(); if matches!(next_token, TokenKind::In) && context.is_in_excluded() { @@ -2390,6 +2393,7 @@ impl<'src> Parser<'src> { operators.len(), self.at_identity_operator(next_op), ); + at_type_test = self.at_type_test(next_op); self.bump_cmp_op(next_op); operators.push(next_op); } @@ -2426,6 +2430,101 @@ impl<'src> Parser<'src> { } } + /// basedpython: whether the comparison operator the parser is positioned at + /// is a type test — `is` or `is not` written as keywords, whose right-hand side + /// is a type + fn at_type_test(&self, op: CmpOp) -> bool { + self.options.is_basedpython + && matches!(op, CmpOp::Is | CmpOp::IsNot) + && !self.at_identity_operator(op) + } + + /// basedpython: the `?` and `??` optional markers written after a type test's + /// type, and the rest of the type after them. they belong to the type, so + /// `x is int?` tests for `int | None`, and `x is str? | int` for what the + /// annotation `str? | int` means + /// + /// everywhere else `?` binds looser than a comparison — it takes a whole union + /// as its operand — so without this the marker would wrap the comparison + /// instead, as `(x is int)?`, which is not a type at all. the same goes for a + /// `?` followed by an expression, the result form `T ? E`: `x is int ? E` tests + /// for `int ? E`. its error type ends where the type test's type would, so + /// `x is int ? E and y` is still a type test joined to `y`. a `??` followed by + /// an expression is the none-coalescing operator, and is left to the caller + fn parse_type_test_optional( + &mut self, + mut target: ParsedExpr, + start: TextSize, + context: ExpressionContext, + ) -> ParsedExpr { + loop { + if self.at(TokenKind::Question) + && self.options.mode != Mode::Ipython + && (EXPR_SET.contains(self.peek()) + || self.peek().is_soft_keyword() + || self.peek() == TokenKind::At) + { + self.bump(TokenKind::Question); + let error = self.parse_binary_expression_or_higher( + OperatorPrecedence::ComparisonsMembershipIdentity, + context, + ); + target = ParsedExpr { + expr: Expr::BinOp(ast::ExprBinOp { + left: Box::new(target.expr), + op: ast::Operator::Result, + right: Box::new(error.expr), + range: self.node_range(start), + node_index: AtomicNodeIndex::NONE, + }), + is_parenthesized: false, + parameter_borrow: ParameterBorrow::None, + }; + continue; + } + let layers = match self.current_token_kind() { + TokenKind::Question + if !(EXPR_SET.contains(self.peek()) + || self.peek().is_soft_keyword() + || self.peek() == TokenKind::At) => + { + 1 + } + TokenKind::DoubleQuestion + if !(EXPR_SET.contains(self.peek()) + || self.peek().is_soft_keyword() + || starts_statement_expression(self.peek())) => + { + 2 + } + _ => return target, + }; + if self.options.mode == Mode::Ipython { + return target; + } + self.bump(self.current_token_kind()); + let mut wrapped = target.expr; + for _ in 0..layers { + wrapped = Expr::UnaryOp(ast::ExprUnaryOp { + op: ast::UnaryOp::Optional, + operand: Box::new(wrapped), + range: self.node_range(start), + node_index: AtomicNodeIndex::NONE, + }); + } + target = self.parse_binary_expression_or_higher_recursive( + ParsedExpr { + expr: wrapped, + is_parenthesized: false, + parameter_borrow: ParameterBorrow::None, + }, + OperatorPrecedence::ComparisonsMembershipIdentity, + context, + start, + ); + } + } + /// basedpython: whether the comparison operator the parser is positioned at /// was written `===` / `!==` — python's identity comparison, which the lexer /// emits as its own token and which shares a [`CmpOp`] with the `is` keyword. diff --git a/crates/ruff_python_parser/src/parser/tests.rs b/crates/ruff_python_parser/src/parser/tests.rs index 50ea86206f..0e7230a1bd 100644 --- a/crates/ruff_python_parser/src/parser/tests.rs +++ b/crates/ruff_python_parser/src/parser/tests.rs @@ -107,6 +107,108 @@ fn a_trailing_type_test_still_chains() { ); } +/// the single expression statement of `source`, parsed as basedpython +fn single_expression(source: &str) -> Expr { + let parsed = parse_basedpython_module(source); + let [Stmt::Expr(statement)] = parsed.syntax().body.as_slice() else { + panic!("expected a single expression statement: {source}"); + }; + *statement.value.clone() +} + +/// how many optional markers wrap `expr`, and what they wrap +fn peel_optionals(expr: &Expr) -> (usize, &Expr) { + let mut depth = 0; + let mut inner = expr; + while let Expr::UnaryOp(unary) = inner + && unary.op == UnaryOp::Optional + { + depth += 1; + inner = &unary.operand; + } + (depth, inner) +} + +#[test] +fn a_type_tests_optional_marker_belongs_to_its_type() { + // `?` binds looser than a comparison, which would make `x is int?` the + // optional of a comparison. the type test's type takes the marker instead + for (source, depth) in [ + ("x is int?\n", 1), + ("x is not int?\n", 1), + ("x is (A | B)?\n", 1), + ("x is A | B?\n", 1), + ("x is int??\n", 2), + ] { + let Expr::Compare(compare) = single_expression(source) else { + panic!("expected a comparison: {source}"); + }; + let (found, inner) = peel_optionals(&compare.comparators[0]); + assert_eq!(found, depth, "{source}"); + assert!(!inner.is_unary_op_expr(), "{source}"); + } + + // the type goes on after the marker, as it would in an annotation + let Expr::Compare(compare) = single_expression("x is str? | int\n") else { + panic!("expected a comparison"); + }; + let Expr::BinOp(union) = &compare.comparators[0] else { + panic!("expected a union target"); + }; + assert_eq!(union.op, Operator::BitOr); + assert_eq!(peel_optionals(&union.left).0, 1); +} + +#[test] +fn a_type_tests_result_form_belongs_to_its_type() { + // `?` followed by an error type is the result form, and a type test's type + // takes it as it takes the optional marker + for source in ["x is int ? str\n", "x is not int ? str | bytes\n"] { + let Expr::Compare(compare) = single_expression(source) else { + panic!("expected a comparison: {source}"); + }; + let Expr::BinOp(result) = &compare.comparators[0] else { + panic!("expected a result target: {source}"); + }; + assert_eq!(result.op, Operator::Result, "{source}"); + assert!(result.left.is_name_expr(), "{source}"); + } + + // an optional marker after the error type is over the whole result, as in an + // annotation + let Expr::Compare(compare) = single_expression("x is int ? str?\n") else { + panic!("expected a comparison"); + }; + let (depth, inner) = peel_optionals(&compare.comparators[0]); + assert_eq!(depth, 1); + assert!(matches!(inner, Expr::BinOp(result) if result.op == Operator::Result)); + + // the error type ends where the type test's type would, so a boolean operator + // after it still joins the whole type test + let Expr::BoolOp(and) = single_expression("x is int ? str and y\n") else { + panic!("expected a boolean operation"); + }; + assert!(and.values[0].is_compare_expr()); +} + +#[test] +fn only_a_type_test_takes_a_trailing_optional_marker() { + // an identity comparison and an equality compare values, so the marker keeps + // wrapping the whole comparison, as it always did + for source in ["x === y?\n", "x == y?\n"] { + let expr = single_expression(source); + let (depth, inner) = peel_optionals(&expr); + assert_eq!(depth, 1, "{source}"); + assert!(inner.is_compare_expr(), "{source}"); + } + // a `??` with a right operand is none-coalescing, not a double optional + let Expr::BinOp(coalesce) = single_expression("x is int ?? y\n") else { + panic!("expected none-coalescing"); + }; + assert_eq!(coalesce.op, Operator::Coalesce); + assert!(coalesce.left.is_compare_expr()); +} + /// The keyword-argument names of the single call statement in `source`, each /// with the way it was spelled. fn keyword_names(source: &str) -> Vec<(String, ruff_python_ast::KeywordKey)> { diff --git a/crates/ty/docs/rules.md b/crates/ty/docs/rules.md index 84eb6ec21e..08fe076dc3 100644 --- a/crates/ty/docs/rules.md +++ b/crates/ty/docs/rules.md @@ -8,7 +8,7 @@ Default level: error · Added in 0.0.64 · Related issues · -View source +View source @@ -44,7 +44,7 @@ class Base(ABC): Default level: error · Added in 0.0.13 · Related issues · -View source +View source @@ -90,7 +90,7 @@ class Derived(Base): # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.61 · Related issues · -View source +View source @@ -123,7 +123,7 @@ f(1, b=s1) # ok — explicit Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.39 · Related issues · -View source +View source @@ -159,7 +159,7 @@ report(Celsius()) # error: two conversions apply Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.3 · Related issues · -View source +View source @@ -194,7 +194,7 @@ extension list: Default level: warn · Added in 0.0.1-alpha.20 · Related issues · -View source +View source @@ -258,7 +258,7 @@ class SubProto(BaseProto, Protocol): Default level: error · Added in 0.0.14 · Related issues · -View source +View source @@ -340,7 +340,7 @@ value = unknown # ty: ignore[unresolved-reference] Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.61 · Related issues · -View source +View source @@ -383,7 +383,7 @@ a4 = True + 1 # ok — a boolean used as a boolean Default level: error · Added in 0.0.16 · Related issues · -View source +View source @@ -435,7 +435,7 @@ Foo.method() # error Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -463,7 +463,7 @@ Calling a non-callable object will raise a `TypeError` at runtime. Default level: error · Added in 0.0.7 · Related issues · -View source +View source @@ -498,7 +498,7 @@ def f(x: object): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.40 · Related issues · -View source +View source @@ -546,7 +546,7 @@ def main(): Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.40 · Related issues · -View source +View source @@ -593,7 +593,7 @@ def Fixed(show: bool): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -626,7 +626,7 @@ a = 1 # error Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -660,7 +660,7 @@ class C(A, B): ... # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.40 · Related issues · -View source +View source @@ -706,7 +706,7 @@ def App(done: bool) -> int: Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -740,7 +740,7 @@ class B(A): ... # error Default level: error · Added in 0.0.1-alpha.29 · Related issues · -View source +View source @@ -785,7 +785,7 @@ type Tree = int | list[Tree] # valid recursive alias Default level: error · Added in 0.0.15 · Related issues · -View source +View source @@ -821,7 +821,7 @@ class Example: Default level: warn · Added in 0.0.1-alpha.16 · Related issues · -View source +View source @@ -860,7 +860,7 @@ old_func() # error: [deprecated] Default level: ignore · Added in 0.0.78 · Related issues · -View source +View source @@ -1032,7 +1032,7 @@ soundness checks from their type checker, and it may have false positives in som Default level: error · Level under ty-compatible: ignore · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1064,7 +1064,7 @@ This rule is currently disabled by default because of the number of false positi Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1095,7 +1095,7 @@ class B(A, A): ... # error Default level: error · Added in 0.0.1-alpha.12 · Related issues · -View source +View source @@ -1135,7 +1135,7 @@ class A: # error Default level: ignore · Added in 0.0.73 · Related issues · -View source +View source @@ -1246,7 +1246,7 @@ Python code. Default level: error · Added in 0.0.14 · Related issues · -View source +View source @@ -1295,7 +1295,7 @@ def bar() -> str: # error: [empty-body] Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.61 · Related issues · -View source +View source @@ -1356,7 +1356,7 @@ def h(x: object): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.3 · Related issues · -View source +View source @@ -1458,7 +1458,7 @@ def foo() -> "intt\b": ... # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1497,7 +1497,7 @@ def f(local fn: () -> None): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1538,7 +1538,7 @@ for x in [1, 2, 3]: Default level: warn · Added in 0.0.50 · Related issues · -View source +View source @@ -1578,7 +1578,7 @@ def g(value: ~A) -> None: ... # error: [experimental-syntax] Default level: error · Added in 0.0.20 · Related issues · -View source +View source @@ -1612,7 +1612,7 @@ def my_function() -> int: Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.40 · Related issues · -View source +View source @@ -1647,7 +1647,7 @@ let a = 1 Default level: error · Added in 0.0.15 · Related issues · -View source +View source @@ -1764,7 +1764,7 @@ def test() -> "Literal[5]": Default level: ignore · basedpython only, so absent under ty-compatible · Added in 0.0.72 · Related issues · -View source +View source @@ -1817,7 +1817,7 @@ unpacking — is not a declaration, and is never reported. Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.68 · Related issues · -View source +View source @@ -1892,7 +1892,7 @@ print(Labelled) # warning: prints `` Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.80 · Related issues · -View source +View source @@ -1931,7 +1931,7 @@ def audit(account: Account) -> int: Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1967,7 +1967,7 @@ class C(A, B): ... # error Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1996,7 +1996,7 @@ t[3] # error Default level: warn · Added in 0.0.1-alpha.33 · Related issues · -View source +View source @@ -2033,7 +2033,7 @@ class MyClass: ... Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.79 · Related issues · -View source +View source @@ -2069,7 +2069,7 @@ class Point: Default level: error · Added in 0.0.1-alpha.12 · Related issues · -View source +View source @@ -2164,7 +2164,7 @@ will produce instances with an atypical memory layout. Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -2196,7 +2196,7 @@ func("foo") # error: [invalid-argument-type] Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -2226,7 +2226,7 @@ a: int = "" # error Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -2307,7 +2307,7 @@ box.value = 1 # okay Default level: error · Added in 0.0.33 · Related issues · -View source +View source @@ -2352,7 +2352,7 @@ class Sub(Base): Default level: error · Added in 0.0.1-alpha.19 · Related issues · -View source +View source @@ -2394,7 +2394,7 @@ asyncio.run(main()) Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -2421,7 +2421,7 @@ class A(42): ... # error: [invalid-base] Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.79 · Related issues · -View source +View source @@ -2459,7 +2459,7 @@ build: # error: `build` is an experimental feature, and is off Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.5 · Related issues · -View source +View source @@ -2496,7 +2496,7 @@ extension str(A): # error: `str` does not answer every member of `A` Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -2525,7 +2525,7 @@ with 1: # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.39 · Related issues · -View source +View source @@ -2558,7 +2558,7 @@ class Fahrenheit: Default level: error · Added in 0.0.12 · Related issues · -View source +View source @@ -2611,7 +2611,7 @@ See: Default level: error · Added in 0.0.13 · Related issues · -View source +View source @@ -2647,7 +2647,7 @@ class A: Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -2673,13 +2673,84 @@ a: str # error [assignable to]: https://typing.python.org/en/latest/spec/glossary.html#term-assignable +## `invalid-decorator-def` + + +Default level: error · basedpython only, so absent under ty-compatible · +Added in 0.0.82 · +Related issues · +View source + + + +**What it does** + + +Checks for a `decorator def` whose parameters, or whose position, leave it with no decorator to +expand into. + +**Why is this bad?** + + +A `decorator def` is lowered to a pair of overloads and a runtime dispatcher, built from a first +parameter that receives the decorated function and options that each carry a default. A declaration +that does not have that shape has nothing to build from, and the lowering refuses it. Reported here, +the refusal arrives while the file is being checked rather than when it is transpiled. + +**Examples** + + +The first parameter is the one a decoration hands the function to, so there has to be one: + +```by +# error: [invalid-decorator-def] +decorator def logged() -> int: + return 1 +``` + +Every shape the decorator is applied in supplies that function, so the parameter never falls back to +a default: + +```by +def nothing() -> None: ... + +# error: [invalid-decorator-def] +decorator def retry(fn: (...) -> object = nothing) -> int: + return 1 +``` + +The options are what `@traced` decorates without, so each of them needs a default: + +```by +# error: [invalid-decorator-def] +decorator def traced(fn: (...) -> object, label: str) -> int: + return 1 +``` + +`decorator def` is module-scope only. A decorator written in a class body is a plain `def` that +returns a callable: + +```by +class Registry: + # error: [invalid-decorator-def] + decorator def register(fn: (...) -> object) -> int: + return 1 +``` + +The form the lowering accepts is a decorated parameter followed by options that each have a default: + +```by +decorator def traced(fn: (...) -> object, label: str = "") -> int: + return len(label) +``` + ## `invalid-enum-member-annotation` Default level: warn · Added in 0.0.20 · Related issues · -View source +View source @@ -2735,7 +2806,7 @@ class Pet(Enum): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -2800,7 +2871,7 @@ This rule corresponds to Ruff's Default level: error · Added in 0.0.1-alpha.28 · Related issues · -View source +View source @@ -2854,7 +2925,7 @@ class D(A): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.3 · Related issues · -View source +View source @@ -2887,7 +2958,7 @@ extension list[T: int]: # error: `list` declares no type parameter `T` Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.4 · Related issues · -View source +View source @@ -2916,7 +2987,7 @@ Author.objects.filter(name__startswith=1) # error: lookup wants `str` Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.36 · Related issues · -View source +View source @@ -2952,7 +3023,7 @@ def test_user(user: int) -> None: # error: fixture provides `str` Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.68 · Related issues · -View source +View source @@ -3000,7 +3071,7 @@ f"{'name':>10}" # ok Default level: error · Added in 0.0.1-alpha.35 · Related issues · -View source +View source @@ -3050,7 +3121,7 @@ class NonFrozenChild(FrozenBase): # error Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -3105,7 +3176,7 @@ class E(Generic[V]): Default level: error · Added in 0.0.12 · Related issues · -View source +View source @@ -3200,7 +3271,7 @@ a = 20 / 0 # type: ignore Default level: error · Added in 0.0.1-alpha.17 · Related issues · -View source +View source @@ -3247,7 +3318,7 @@ carol = Person(name="Carol", aeg=25) # typo! Default level: warn · Added in 0.0.15 · Related issues · -View source +View source @@ -3308,7 +3379,7 @@ def f(x, y, /): # Python 3.8+ syntax Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -3348,7 +3419,7 @@ def f(t: TypeVar("U")): ... # ty: ignore[invalid-type-form] Default level: error · Added in 0.0.18 · Related issues · -View source +View source @@ -3399,7 +3470,7 @@ match object(): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -3433,7 +3504,7 @@ class B(metaclass=42): ... # error Default level: error · Added in 0.0.1-alpha.20 · Related issues · -View source +View source @@ -3550,7 +3621,7 @@ Correct use of `@override` is enforced by ty's [`invalid-explicit-override`](#in Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.72 · Related issues · -View source +View source @@ -3593,7 +3664,7 @@ implements Backend # error: `Backend` is not a protocol Default level: error · Added in 0.0.72 · Related issues · -View source +View source @@ -3631,7 +3702,7 @@ from module import missing # error Default level: error · Added in 0.0.1-alpha.19 · Related issues · -View source +View source @@ -3696,7 +3767,7 @@ TypeError: typing.ClassVar[int] is not valid as type argument Default level: warn · Added in 0.0.31 · Related issues · -View source +View source @@ -3742,7 +3813,7 @@ admin[0] # "Alice" Default level: error · Added in 0.0.1-alpha.27 · Related issues · -View source +View source @@ -3780,7 +3851,7 @@ Baz = NewType("Baz", int | str) # error Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -3837,7 +3908,7 @@ def foo(x: int) -> int: ... Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.80 · Related issues · -View source +View source @@ -3875,7 +3946,7 @@ B().f() # 1, not 2 Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -3903,7 +3974,7 @@ def f(a: int = ""): ... # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.36 · Related issues · -View source +View source @@ -3936,7 +4007,7 @@ def test_add(a: int, b: int) -> None: Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -3972,7 +4043,7 @@ P2 = ParamSpec() # error Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -4024,7 +4095,7 @@ Declare the type variable with `TypeVar("T", covariant=True)` instead. Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -4095,7 +4166,7 @@ def g(): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.37 · Related issues · -View source +View source @@ -4130,7 +4201,7 @@ def g[T](value: T) raises T: # error: `T@g` is not always an exception Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.36 · Related issues · -View source +View source @@ -4163,7 +4234,7 @@ if m := re.match("(a)(b)", "ab"): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.62 · Related issues · -View source +View source @@ -4189,13 +4260,84 @@ runtime value that never arrives. type Alias[reified T] = list[T] # error: an alias's parameters are erased ``` +## `invalid-repeated-underscore` + + +Default level: error · basedpython only, so absent under ty-compatible · +Added in 0.0.83 · +Related issues · +View source + + + +**What it does** + + +Checks for a parameter list that repeats `_` in a shape the lowering has no python spelling for. + +**Why is this bad?** + + +Python refuses two parameters of one name, so every `_` after the first is given a name of its own. +That name is the lowering's rather than the author's, so a call reaches the parameter by position +alone: it is positional-only. Where that cannot be written, or would change a parameter the author +named, the lowering refuses the definition. Reported here, the refusal arrives while the file is +being checked rather than when it is transpiled. + +**Examples** + + +Python's `/` makes every parameter before it positional-only, so a named parameter ahead of a +repeated `_` would stop being reachable by keyword: + +```by +# error: [invalid-repeated-underscore] +def pair(a: int, _: int, b: int, _: int) -> int: + return a + b +``` + +Writing the `/` says so, and is accepted: + +```by +def explicit(a: int, _: int, b: int, _: int, /) -> int: + return a + b +``` + +A repeated `_` after `*` is keyword-only, and only a keyword reaches it: + +```by +# error: [invalid-repeated-underscore] +def keyword(_: int, *, _: int) -> None: ... +``` + +Python before 3.8 has no `/` at all, so a module targeting it cannot make a repeated `_` +positional-only, and every definition that needs the `/` is refused there. + +A method that overrides one takes the names the overridden method gives those positions. A name the +body reads from an enclosing scope would then read the parameter instead. Refused, the override +keeps no names a caller of the base could pass by keyword, which is reported too: + +```by +x = 1 + +class Base: + def f(self, x: int, y: int) -> int: + return x + y + +class Override(Base): + # error: [invalid-repeated-underscore] + # error: [invalid-method-override] + override def f(self, _: int, _: int) -> int: + return x +``` + ## `invalid-return-type` Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -4341,7 +4483,7 @@ def detail(request, pk: int): ... # ok Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.79 · Related issues · -View source +View source @@ -4380,7 +4522,7 @@ import "data/missing.json" as missing Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -4488,7 +4630,7 @@ class C: ... Default level: error · Added in 0.0.10 · Related issues · -View source +View source @@ -4539,7 +4681,7 @@ class MyClass: Default level: error · Added in 0.0.1-alpha.6 · Related issues · -View source +View source @@ -4585,7 +4727,7 @@ InvalidAlias = TypeAliasType("InvalidAlias", list[T], type_params=(list[T],)) # Default level: error · Added in 0.0.1-alpha.29 · Related issues · -View source +View source @@ -4651,7 +4793,7 @@ Bar[int] # error Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -4684,7 +4826,7 @@ TYPE_CHECKING = "" # error Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -4719,7 +4861,7 @@ b: Annotated[int] # error Default level: error · Added in 0.0.1-alpha.11 · Related issues · -View source +View source @@ -4776,7 +4918,7 @@ class C: Default level: error · Added in 0.0.15 · Related issues · -View source +View source @@ -4842,7 +4984,7 @@ class Owner[U]: Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -4899,7 +5041,7 @@ V = TypeVar("V", list[int], int) # valid constrained Type Default level: error · Added in 0.0.16 · Related issues · -View source +View source @@ -4941,7 +5083,7 @@ U = TypeVar("U", int, str, default=bytes) # error: [invalid-type-variable-defau Default level: error · Added in 0.0.28 · Related issues · -View source +View source @@ -4977,7 +5119,7 @@ class Child(Base): Default level: error · Added in 0.0.14 · Related issues · -View source +View source @@ -5018,7 +5160,7 @@ def f(options: dict[str, object]): Default level: error · Added in 0.0.9 · Related issues · -View source +View source @@ -5052,7 +5194,7 @@ class Foo(TypedDict): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.62 · Related issues · -View source +View source @@ -5093,7 +5235,7 @@ type Alias[out T] = list[T] # error: `list` is invariant Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.80 · Related issues · -View source +View source @@ -5111,6 +5253,11 @@ a `private` abstract method can never be overridden, and a dotted import binds a package no rename can keep. The declaration would read as restricted while doing something else. +The rename can also land on a name the class, a base class or a subclass +already stores another member under: `protected def m` is stored as `_m`, +so beside a `def _m` one of the two replaces the other at runtime while +the checker still sees both. + **Example** @@ -5128,7 +5275,7 @@ class Point: Default level: error · Added in 0.0.25 · Related issues · -View source +View source @@ -5162,7 +5309,7 @@ def gen() -> Iterator[int]: Default level: error · Added in 0.0.14 · Related issues · -View source +View source @@ -5228,7 +5375,7 @@ def h(arg2: type): Default level: error · Added in 0.0.15 · Related issues · -View source +View source @@ -5277,7 +5424,7 @@ def g(arg: object): Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.36 · Related issues · -View source +View source @@ -5308,7 +5455,7 @@ def f(s: str): Default level: warn · Added in 0.0.30 · Related issues · -View source +View source @@ -5350,7 +5497,7 @@ Movie = TypedDict("Film", {"title": str}) # error: [mismatched-type-name] Default level: warn · Added in 0.0.1-alpha.39 · Related issues · -View source +View source @@ -5418,7 +5565,7 @@ and nothing is reported. Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -5449,7 +5596,7 @@ func() # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.61 · Related issues · -View source +View source @@ -5457,12 +5604,15 @@ Added in 0.0.61 object, context b: str = "d") -> object: ... + +@deco # error: `b` cannot be filled at a decoration +def g(): ... ``` ## `missing-direct-dependency` @@ -5480,7 +5635,7 @@ f(1) # ok — `s` is passed implicitly Default level: ignore · Preview (since 0.0.76) · Related issues · -View source +View source @@ -5569,7 +5724,7 @@ Add `urllib3` to `project.dependencies` if your code imports it directly. Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.4 · Related issues · -View source +View source @@ -5597,7 +5752,7 @@ from django.db import models # warning: install `django-stubs` for precise type Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.81 · Related issues · -View source +View source @@ -5649,7 +5804,7 @@ def ignore(event: str): ... Default level: error · Level under ty-compatible: ignore · Added in 0.0.41 · Related issues · -View source +View source @@ -5710,7 +5865,7 @@ class ExplicitChild(Parent): Default level: error · Added in 0.0.75 · Related issues · -View source +View source @@ -5797,7 +5952,7 @@ class Item: Default level: error · Level under ty-compatible: ignore · Added in 0.0.45 · Related issues · -View source +View source @@ -5835,7 +5990,7 @@ def handle(m: re.Match[str]) -> str: Default level: error · Added in 0.0.1-alpha.20 · Related issues · -View source +View source @@ -5874,7 +6029,7 @@ alice["age"] # KeyError Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.40 · Related issues · -View source +View source @@ -5921,7 +6076,7 @@ def App(): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -5954,7 +6109,7 @@ def f(a: int | None): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -5992,7 +6147,7 @@ func("string") # error: [no-matching-overload] Default level: error · Added in 0.0.30 · Related issues · -View source +View source @@ -6029,7 +6184,7 @@ class Sub(Super): ... # error: [non-callable-init-subclass] Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.39 · Related issues · -View source +View source @@ -6060,7 +6215,7 @@ def f(x: int | str) -> int: Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.61 · Related issues · -View source +View source @@ -6089,7 +6244,7 @@ def f(a: object): Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.62 · Related issues · -View source +View source @@ -6133,7 +6288,7 @@ def g(o: object, shape: Shape): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -6162,7 +6317,7 @@ for i in 34: # error Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -6190,7 +6345,7 @@ Subscripting an object that does not support it will raise a `TypeError` at runt Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -6219,7 +6374,7 @@ def f(once done: () -> None): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -6247,7 +6402,7 @@ def f(once done: () -> None): Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.61 · Related issues · -View source +View source @@ -6268,6 +6423,10 @@ Assigning an optional to a declared `object` variable is *not* flagged — the target narrows back to the optional type, so nothing is lost there. The diagnostic surfaces at each use of the value as `object` instead. +Nor is an argument whose type the call tests — the first argument of +`isinstance`, or the one a `TypeIs` return narrows. The test reads the +`None` rather than losing it. + **Examples** ```by @@ -6285,7 +6444,7 @@ def f(x: int?): Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.62 · Related issues · -View source +View source @@ -6340,7 +6499,7 @@ def g(name: str | None): Default level: error · Added in 0.0.1-alpha.29 · Related issues · -View source +View source @@ -6377,7 +6536,7 @@ class B(A): Default level: error · Added in 0.0.16 · Related issues · -View source +View source @@ -6414,7 +6573,7 @@ class B(A): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.38 · Related issues · -View source +View source @@ -6457,7 +6616,7 @@ def main(): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -6488,7 +6647,7 @@ f(1, x=2) # error Default level: error · Added in 0.0.1-alpha.22 · Related issues · -View source +View source @@ -6519,7 +6678,7 @@ f(x=1) # error Default level: error · Level under ty-compatible: ignore · Added in 0.0.1-alpha.22 · Related issues · -View source +View source @@ -6557,7 +6716,7 @@ A.c # error Default level: warn · Added in 0.0.1-alpha.22 · Related issues · -View source +View source @@ -6595,7 +6754,7 @@ A()[0] # error Default level: error · Level under ty-compatible: ignore · Added in 0.0.1-alpha.22 · Related issues · -View source +View source @@ -6639,7 +6798,7 @@ from module import a # error Default level: warn · Added in 0.0.23 · Related issues · -View source +View source @@ -6671,7 +6830,7 @@ html.parser # error Default level: error · Level under ty-compatible: ignore · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -6707,7 +6866,7 @@ print(x) # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.79 · Related issues · -View source +View source @@ -6746,7 +6905,7 @@ Id("x") # error: `Id`'s constructor is private Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.80 · Related issues · -View source +View source @@ -6777,7 +6936,7 @@ __all__ = ["helper"] # error: `helper` is private Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -6809,7 +6968,7 @@ from helpers import Key # error: `Key` is private to `helpers` Default level: warn · Added in 0.0.60 · Related issues · -View source +View source @@ -6884,7 +7043,7 @@ def test() -> "int": Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.62 · Related issues · -View source +View source @@ -6926,7 +7085,7 @@ def g(a: bool | None): Default level: warn · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -6961,7 +7120,7 @@ cast(int, f()) # error Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.62 · Related issues · -View source +View source @@ -7015,7 +7174,7 @@ if sys.version_info >= (3, 12): # ok — artificially constant Default level: warn · Added in 0.0.18 · Related issues · -View source +View source @@ -7053,7 +7212,7 @@ class C: Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.62 · Related issues · -View source +View source @@ -7108,7 +7267,7 @@ class Sub(Base): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.62 · Related issues · -View source +View source @@ -7157,7 +7316,7 @@ def f(value: int | str) -> int: Default level: error · Added in 0.0.71 · Related issues · -View source +View source @@ -7221,7 +7380,7 @@ def g(values: tuple[int, ...]) -> None: Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.3 · Related issues · -View source +View source @@ -7254,7 +7413,7 @@ class C: Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.72 · Related issues · -View source +View source @@ -7290,7 +7449,7 @@ class C[T]: Default level: warn · Added in 0.0.71 · Related issues · -View source +View source @@ -7333,7 +7492,7 @@ def build(t: Tag) -> None: Default level: error · Added in 0.0.20 · Related issues · -View source +View source @@ -7377,7 +7536,7 @@ class Outer[T]: Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.40 · Related issues · -View source +View source @@ -7426,7 +7585,7 @@ def Observed(): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.40 · Related issues · -View source +View source @@ -7470,7 +7629,7 @@ def Counter(): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -7504,7 +7663,7 @@ static_assert(int(2.0 * 3.0) == 6) # error Default level: warn · Added in 0.0.39 · Related issues · -View source +View source @@ -7556,7 +7715,7 @@ limitation. Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -7590,7 +7749,7 @@ class B(A): ... # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -7622,7 +7781,7 @@ class Circle(Shape): ... # error: `Shape` is sealed in another workspace Default level: error · Added in 0.0.1-alpha.30 · Related issues · -View source +View source @@ -7731,7 +7890,7 @@ class Book: Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -7761,7 +7920,7 @@ f("foo") # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -7800,7 +7959,7 @@ def find(items: list[int]) -> int: Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -7847,7 +8006,7 @@ g: Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -7882,7 +8041,7 @@ f: # error: the block returns `None`, not `str` Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -7921,7 +8080,7 @@ def _(x: int): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.4 · Related issues · -View source +View source @@ -7952,7 +8111,7 @@ class User(BaseModel): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -8011,7 +8170,7 @@ class A: Default level: error · Added in 0.0.20 · Related issues · -View source +View source @@ -8084,7 +8243,7 @@ the project registers with `@register.simple_block_tag`. Default level: warn · Added in 0.0.1-alpha.39 · Related issues · -View source +View source @@ -8150,7 +8309,7 @@ what the projects depending on it read. Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.37 · Related issues · -View source +View source @@ -8179,7 +8338,7 @@ def f() raises TypeError: Default level: warn · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -8208,7 +8367,7 @@ reveal_type(1) # revealed: Literal[1] Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.40 · Related issues · -View source +View source @@ -8246,7 +8405,7 @@ def check(x: int | None) -> asserts x: Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.37 · Related issues · -View source +View source @@ -8276,7 +8435,7 @@ def main(): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -8307,7 +8466,7 @@ f(x=1, y=2) # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.36 · Related issues · -View source +View source @@ -8518,7 +8677,7 @@ page does not render at all. Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.72 · Related issues · -View source +View source @@ -8552,7 +8711,7 @@ implements Backend # error: this module does not answer `Backend` Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.40 · Related issues · -View source +View source @@ -8611,7 +8770,7 @@ def Held(): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -8644,7 +8803,7 @@ A().foo # error Default level: warn · Added in 0.0.1-alpha.15 · Related issues · -View source +View source @@ -8719,7 +8878,7 @@ def g(): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -8747,7 +8906,7 @@ import foo # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -8776,7 +8935,7 @@ def check(value: int | None) -> asserts values: # error: `values` is nothing Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -8894,7 +9053,7 @@ is one whose template set cannot be established. Default level: ignore · Added in 0.0.73 · Related issues · -View source +View source @@ -9022,7 +9181,7 @@ Python code. Default level: error · Added in 0.0.71 · Related issues · -View source +View source @@ -9063,7 +9222,7 @@ def f(a: object, b: int, c: Any): Default level: ignore · Added in 0.0.70 · Related issues · -View source +View source @@ -9208,7 +9367,7 @@ Python code. Default level: ignore · Added in 0.0.70 · Related issues · -View source +View source @@ -9355,7 +9514,7 @@ generator boundaries. Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.3 · Related issues · -View source +View source @@ -9405,7 +9564,7 @@ A() # error: nothing says which specialization this is Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.40 · Related issues · -View source +View source @@ -9452,7 +9611,7 @@ def Observed(items: StateList[int]): ... # ok: an observable handle Default level: warn · Added in 0.0.1-alpha.7 · Related issues · -View source +View source @@ -9498,7 +9657,7 @@ class D(C): ... # error: [unsupported-base] Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -9547,7 +9706,7 @@ b1 < b2 < b1 # error Default level: warn · Level under ty-compatible: ignore · Added in 0.0.12 · Related issues · -View source +View source @@ -9592,7 +9751,7 @@ def factory(base: type[Base]) -> type: Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -9624,7 +9783,7 @@ A() + A() # error Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.79 · Related issues · -View source +View source @@ -9667,7 +9826,7 @@ reveal_type(project.root) # revealed: "." Default level: warn · Added in 0.0.21 · Related issues · -View source +View source @@ -9746,7 +9905,7 @@ to `false` to prevent this rule from reporting unused `type: ignore` comments. Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.71 · Related issues · -View source +View source @@ -9832,7 +9991,7 @@ to `false`. Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.81 · Related issues · -View source +View source @@ -9877,7 +10036,7 @@ parse("1") Default level: warn · Added in 0.0.1-alpha.22 · Related issues · -View source +View source @@ -9956,7 +10115,7 @@ def foo(x: int | str) -> int | str: Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source diff --git a/crates/ty_python_core/src/builder.rs b/crates/ty_python_core/src/builder.rs index a7bdec45a0..47a533630e 100644 --- a/crates/ty_python_core/src/builder.rs +++ b/crates/ty_python_core/src/builder.rs @@ -4,8 +4,9 @@ use std::sync::Arc; use except_handlers::{ExceptionContextStackManager, ExceptionHandlers}; use itertools::Itertools; use ruff_python_ast::helpers::{ - BindingKeyword, Truthiness, any_over_expr, binding_keyword, is_dotted_name, - last_bound_parameter, parameter_modifiers, return_guards, statement_expression_values, + BindingKeyword, Truthiness, any_over_expr, binding_keyword, callable_parameter_label, + is_dotted_name, last_bound_parameter, parameter_modifiers, predicate_guard, return_guards, + statement_expression_values, }; use rustc_hash::{FxHashMap, FxHashSet}; @@ -8119,8 +8120,27 @@ impl<'ast> Visitor<'ast> for GuardTargetCollector<'_> { walk_stmt(self, stmt); } - fn visit_expr(&mut self, _expr: &'ast ast::Expr) { - // a guard is declared on a statement, and recovered from the `return`s of one + /// a callable type narrows what its return predicate names too, `(x: object) -> (x is int)`: + /// a labelled parameter's argument, or a place in the calling scope + fn visit_expr(&mut self, expr: &'ast ast::Expr) { + if let ast::Expr::CallableType(callable) = expr + && let Some(guard) = predicate_guard(&callable.returns) + { + let (name, members) = guard.place_parts(); + let members: Box<[Name]> = members.into_iter().cloned().collect(); + if callable + .args + .iter() + .any(|argument| callable_parameter_label(argument) == Some(name)) + { + if !members.is_empty() { + self.targets.member_chains.push(members); + } + } else { + self.targets.scope_places.push((name.clone(), members)); + } + } + walk_expr(self, expr); } } diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_identity_narrow.md b/crates/ty_python_semantic/resources/mdtest/basedpython_identity_narrow.md index b14f1a8a1f..575ce102d3 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_identity_narrow.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_identity_narrow.md @@ -199,6 +199,44 @@ def g(x: int): ... ``` +## An optional target holds for `None` too + +The `?` belongs to the target type, so `x is int?` tests for `int | None`, and `is not` for neither. +Elsewhere `?` binds looser than a comparison, but an optional marker on a whole comparison would not +be a type. + +```by +def f(x: int | str | None): + if x is int?: + reveal_type(x) # revealed: int | None + else: + reveal_type(x) # revealed: str + if x is not int?: + reveal_type(x) # revealed: str + +class A: ... +class B: ... + +def g(x: A | B | int | None): + if x is (A | B)?: + reveal_type(x) # revealed: A | B | None + else: + reveal_type(x) # revealed: int & not A & not B +``` + +## A result target is the target's type + +The result form `T ? E` after a type test is part of the target too, as it would be in an +annotation, so `x is int ? ValueError` tests for the type `int ? ValueError`. Result types are not +supported yet, so the target is rejected wherever it is written. + +```by +def f(x: object): + # error: [invalid-type-form] "Invalid binary operator `?` in type annotation" + if x is int ? ValueError: + ... +``` + ## Identity comparisons are left alone `===` and `!==` keep Python identity semantics, where an always-`False` comparison is already typed diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_implicit_object_repr.md b/crates/ty_python_semantic/resources/mdtest/basedpython_implicit_object_repr.md index 504873a88c..60149c3a48 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_implicit_object_repr.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_implicit_object_repr.md @@ -420,6 +420,29 @@ class Point: print(Point(1)) ``` +## a `data class` gets one the same way + +the keyword stands for `@dataclass(slots=True)`, which writes the `__repr__` just as the plain +decorator does, and so does a decorator the options were applied to ahead of the class. + +```by +from dataclasses import dataclass + +data class Point: + x: int + +frozen data class Frozen: + x: int + +slotted = dataclass(slots=True) + +@slotted +class Slotted: + x: int + +print(Point(1), Frozen(1), Slotted(1)) +``` + ## a dataclass that turned its `__repr__` off is reported ```py diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_optional_type.md b/crates/ty_python_semantic/resources/mdtest/basedpython_optional_type.md index 4fcf8b6357..1beb304eff 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_optional_type.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_optional_type.md @@ -102,6 +102,33 @@ def g(y: object) -> None: reveal_type(y) # revealed: bytes | None | str ``` +## an optional read as a value of something other than a type + +the value is still the union with `None` the program evaluates, so a value without a `| None` is +rejected. `?` binds looser than a comparison, which makes a comparison a common way to write one: +`x == y?` marks the whole comparison, and a `bool` has no `| None` + +```by +def f(x: object, y: int): + # error: [unsupported-operator] "Unary operator `?` is not supported for object of type `bool`" + (x is int)? + # error: [unsupported-operator] "Unary operator `?` is not supported for object of type `bool`" + x == y? + # error: [unsupported-operator] "Unary operator `?` is not supported for object of type `int`" + y? +``` + +a value that does support `| None` gives what its `__or__` returns + +```by +class Pattern: + def __or__(self, other: None) -> str: + return "pattern" + +def g(p: Pattern): + reveal_type(p?) # revealed: str +``` + ## double optional is a distinct wrapped type a single `T?` is the lossless union `T | None`, but a nested optional cannot collapse that way (the @@ -148,6 +175,29 @@ def bad[T](t: T) -> T?: return t # error: [invalid-return-type] ``` +## a wrapped optional read as a value is the wrapper class + +where a program evaluates a wrapped optional — an alias of `int??`, a `T?` in a function body — it +gets the runtime class the wrapped values are instances of, the same class whatever it wraps. in a +type expression, an alias of it is the wrapped optional it was written as + +```by +Alias = int?? +reveal_type(Alias) # revealed: + +def f(x: Alias, y: Alias | str) -> None: + reveal_type(x) # revealed: int?? + reveal_type(y) # revealed: int?? | str +``` + +a type parameter read as a value is a `type[T]`, which is not a type expression, so its wrapped +optional is still the wrapper class but denotes nothing more precise than `Unknown??` + +```by +def g[T](t: T) -> None: + reveal_type(T?) # revealed: +``` + ## `Self?` is a plain union `Self` stands for the enclosing class, which can never itself be an optional, so there is no inner diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_type_guards.md b/crates/ty_python_semantic/resources/mdtest/basedpython_type_guards.md index d5210c923e..53a1fa41fb 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_type_guards.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_type_guards.md @@ -798,6 +798,239 @@ def f(a: object): reveal_type(ok) # revealed: TypeIs[str @ a] ``` +## a callable type narrows what its predicate names + +A callable type returns a predicate the way a `def` does, parenthesized: its return binds tighter +than a comparison. Calling a value of the type narrows the argument matched to the parameter the +predicate names, and a `def` whose predicate names the same parameter is such a callable. + +```by +def f(check: (first: object, second: object) -> (second is int), a: int | str, b: int | str): + if check(a, b): + reveal_type(a) # revealed: int | str + reveal_type(b) # revealed: int + else: + reveal_type(b) # revealed: str + +def is_int(first: object, second: object) -> second is int: + return isinstance(second, int) + +f(is_int, 1, "b") +``` + +## a callable type's predicate can name a place + +A name that is not one of the callable's parameters is a place, narrowed where the call is written, +as it is for a `def`. + +```by +def src() -> int | None: + return 1 + +a = src() + +def f(check: (x: object) -> (a is int)): + if check(1): + reveal_type(a) # revealed: int +``` + +## a callable narrowing another parameter is not such a callable + + + +A caller narrows the argument the expected callable type names, so a function that tests a different +one would have it narrow an argument nothing was checked about. The two have to name the same +parameter, not just the same type. + +```by +def want_second(check: (first: object, second: object) -> (second is int)): ... + +def is_int_first(first: object, second: object) -> first is int: + return isinstance(first, int) + +def is_int_second(first: object, second: object) -> second is int: + return isinstance(second, int) + +want_second(is_int_second) +# error: [invalid-argument-type] +want_second(is_int_first) +``` + +A callable type is held to the same rule, in both directions. + +```by +def want_first(check: (first: object, second: object) -> (first is int)): ... + +def f(second_checker: (first: object, second: object) -> (second is int)): + want_second(second_checker) + # error: [invalid-argument-type] + want_first(second_checker) +``` + +## a parameter a caller passes by keyword is matched by name + +A keyword-only parameter is reached only by its name, so the function has to narrow the parameter of +that name, wherever it is declared. + +```by +def want_key(check: (*, key: object, other: object) -> (key is int)): ... + +def by_key(*, other: object, key: object) -> key is int: + return isinstance(key, int) + +def by_other(*, other: object, key: object) -> other is int: + return isinstance(other, int) + +want_key(by_key) +# error: [invalid-argument-type] +want_key(by_other) +``` + +## a `TypeIs` that names no parameter narrows the first + +PEP 742 narrows the first positional parameter, so that is the parameter such a function is matched +on. + +```by +from typing_extensions import TypeIs + +def want_first(check: (first: object, second: object) -> (first is int)): ... +def want_second(check: (first: object, second: object) -> (second is int)): ... + +def is_int(first: object, second: object) -> TypeIs[int]: + return isinstance(first, int) + +want_first(is_int) +# error: [invalid-argument-type] +want_second(is_int) +``` + +## a `Callable` narrows the first argument + +A `Callable` has no parameter names, so what it narrows is the first argument a caller passes. + +```by +from typing import Callable +from typing_extensions import TypeIs + +def want_first_argument(check: Callable[[object, object], TypeIs[int]]): ... + +def is_int_first(first: object, second: object) -> first is int: + return isinstance(first, int) + +def is_int_second(first: object, second: object) -> second is int: + return isinstance(second, int) + +want_first_argument(is_int_first) +# error: [invalid-argument-type] +want_first_argument(is_int_second) +``` + +A method's `TypeIs` narrows the parameter after `self`, which a caller of the unbound method passes +second. + +```by +class Checker: + def is_int(self, value: object) -> TypeIs[int]: + return isinstance(value, int) + +def want_checker_first(check: Callable[[Checker, object], TypeIs[int]]): ... +def want_value_first(check: Callable[[object], TypeIs[int]]): ... + +# error: [invalid-argument-type] +want_checker_first(Checker.is_int) +want_value_first(Checker().is_int) +``` + +## a function stored on a class narrows its receiver + +A plain function read off an instance is bound to it, so the parameter its `TypeIs` narrows is the +receiver, not the first argument the call writes. + +```py +from typing_extensions import TypeIs + +def is_int(first: object, second: object) -> TypeIs[int]: + return isinstance(first, int) + +class K: + check = is_int + +def f(v: int | str): + if K().check(v): + reveal_type(v) # revealed: int | str +``` + +## a callable naming a place narrows that place + +```by +def src() -> int | None: + return 1 + +a = src() +b = src() + +def on_a() -> a is int: + return a is not None + +def on_b() -> b is int: + return b is not None + +def want_a(check: () -> (a is int)): ... + +want_a(on_a) +# error: [invalid-argument-type] +want_a(on_b) +``` + +## an override narrows what the base narrows + +A call through the base narrows the argument the base names, so an override has to test the same +one. + +```by +from typing import override + +class Base: + def check(self, first: object, second: object) -> first is int: + return isinstance(first, int) + +class Sub(Base): + @override + # error: [invalid-method-override] + def check(self, first: object, second: object) -> second is int: + return isinstance(second, int) +``` + +## a callable type's predicate is parenthesized + +Unparenthesized, `is` takes the whole callable type. + +```by +# error: [invalid-type-form] "`is` takes the whole callable type here: a callable type's return binds tighter than a comparison" +def f(check: (x: object) -> x is int): ... +``` + +## a predicate cannot be negated + +A predicate is PEP 742 narrowing, which narrows to the type it names where the call is truthy. +Nothing in Python's typing narrows to everything but a type there, so `-> x is not T` has no meaning +to give it. `asserts x is not T` is allowed: an assertion removes a type once the call returns, and +has no truthy side. + +```by +# error: [invalid-type-form] "A narrowing predicate cannot be negated: `is not` would narrow to everything but the type, which `TypeIs` cannot express" +def not_str(x: object) -> x is not str: + return not isinstance(x, str) +``` + +Nor can a callable type's. + +```by +# error: [invalid-type-form] "A narrowing predicate cannot be negated: `is not` would narrow to everything but the type, which `TypeIs` cannot express" +def f(check: (x: object) -> (x is not str)): ... +``` + ## `asserts` must name a place ```by diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_visibility.md b/crates/ty_python_semantic/resources/mdtest/basedpython_visibility.md index 6c44356f21..cede3eae5d 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_visibility.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_visibility.md @@ -533,6 +533,171 @@ class B(A): return len(n) ``` +## a renamed member cannot take a name its class already uses + +`protected` stores `m` as `_m`, which is where the class's own `_m` is stored too. the checker sees +two members, but at runtime the later definition replaces the earlier one, so `use` would call the +protected method. + +```by +class A: + def _m(self) -> int: + return 1 + + # error: [invalid-visibility] "`m` cannot be `protected` here" + protected def m(self) -> int: + return 2 + + def use(self) -> int: + return self._m() # error: [used-underscore-name] +``` + +## a class attribute, an instance attribute or a property takes the name too + +any member stored under the renamed name collides with it, whatever kind of member it is. + +```by +class Counter: + _count = 0 + + # error: [invalid-visibility] "`count` cannot be `protected` here" + protected def count(self) -> int: + return 1 + +class Label: + # error: [invalid-visibility] "`text` cannot be `protected` here" + protected text: str = "" + + def reset(self): + self._text = 0 + +class Size: + def _width(self) -> int: + return 1 + + # error: [invalid-visibility] "`width` cannot be `protected` here" + protected var width: int + get() = 2 +``` + +## a renamed member cannot take a name a base class uses + +the subclass's `_m` would override the base's own `_m`, so the base's `use` would call it. + +```by +class A: + def _m(self) -> int: + return 1 + + def use(self) -> int: + return self._m() # error: [used-underscore-name] + +class B(A): + # error: [invalid-visibility] "`m` cannot be `protected` here" + protected def m(self) -> int: + return 2 +``` + +## a subclass cannot take the name a base's renamed member is stored under + +the other way round, a subclass's `_m` overrides the base's protected `m` at runtime, although to +the checker it is a new member that does not have to match `m`'s signature. + +```by +class A: + protected def m(self) -> int: + return 1 + +class B(A): + # error: [invalid-visibility] "`_m` replaces `A`'s `protected` member `m` at runtime" + def _m(self) -> str: + return "x" +``` + +an attribute a subclass only assigns in a method has no declaration of its own in the class body, so +the collision is reported on the class. + +```by +# error: [invalid-visibility] "`_m` replaces `A`'s `protected` member `m` at runtime" +class C(A): + def __init__(self): + self._m = 0 +``` + +## a private member cannot take the name python mangles another to + +python stores a name written with two leading underscores under the mangled name `_A__m`, which is +also where `private` stores `m`. + +```by +class A: + # error: [invalid-visibility] "`m` cannot be `private` here" + private def m(self) -> int: + return 1 + + def __m(self) -> int: + return 2 + +class B: + # error: [invalid-visibility] "`m` cannot be `private` here" + private def m(self) -> int: + return 1 + + def _B__m(self) -> int: + return 2 +``` + +## two private members collide when their classes share a name + +a private member is mangled with the name of the class that declares it, so a subclass named like +its base stores its private `m` where the base stores its own, and replaces it. + +`named_base.by`: + +```by +class A: + private def m(self) -> int: + return 1 + + def use(self) -> int: + return self.m() +``` + +`named_subclass.by`: + +```by +import named_base + +class A(named_base.A): + # error: [invalid-visibility] "`m` cannot be `private` here" + private def m(self) -> int: + return 2 +``` + +## redeclaring a member under its own name does not collide + +a protected override is stored where the member it overrides is, which is what an override is for. a +private member of a differently named subclass is stored under a name of its own. + +```by +class A: + protected def m(self) -> int: + return 1 + + private def p(self) -> int: + return 1 + +class B(A): + protected override def m(self) -> int: + return 2 + + private def p(self) -> int: + return 2 + + def _p(self) -> int: + return 3 +``` + ## `protected` is only a modifier on a class member outside a class body there is nothing for the "and its subclasses" half of `protected` to mean. @@ -725,6 +890,59 @@ class A: A.Inner # error: [inaccessible-member] ``` +## a private type alias in a class body + +A `type` alias declared `private` in a class body is a private member: the class's own body and its +methods' signatures may name it, and neither a subclass nor any other code may. + +```by +class Shape: + private type Size = int | float + + width: Size = 1 + + def area(self) -> Size: + return self.width + +class Square(Shape): + def side(self) -> Shape.Size: # error: [inaccessible-member] + return 1 + +def measure(shape: Shape) -> Shape.Size: # error: [inaccessible-member] + return 1 +``` + +## a private type alias is emitted under its mangled name + +The lowering declares the alias as `__Size`, which python mangles to `_Shape__Size`, and spells +every reference to it that way, so an annotation that names it resolves wherever python evaluates +it. + +```by +from typing import get_type_hints + +class Shape: + private type Size = int | float + type Sizes = list[Size] + + width: Size = 1 + widths: Sizes = [1] + +hints = get_type_hints(Shape) +assert hints["width"].__value__ == int | float +assert hints["widths"].__value__.__args__ == (hints["width"],) +assert "Size" not in vars(Shape) +``` + +## a private type alias cannot take a name its class stores + +```by +class A: + # error: [invalid-visibility] "`Size` cannot be `private` here" + private type Size = int + __Size: str = "a" +``` + ## a protected member reached through a union Every class a union can be declares the member the same way, so it has one name to be reached by. diff --git a/crates/ty_python_semantic/resources/mdtest/repeated_underscore.md b/crates/ty_python_semantic/resources/mdtest/repeated_underscore.md index cebf8d3892..9b2f576c73 100644 --- a/crates/ty_python_semantic/resources/mdtest/repeated_underscore.md +++ b/crates/ty_python_semantic/resources/mdtest/repeated_underscore.md @@ -340,9 +340,10 @@ Override().f() ## an override of a method whose `_`s are numbered -A base that repeats `_` numbers its own, and an override takes those names and kinds as it takes any -other. The first is `_` in both, so a read of `_` in the override's body is its first parameter, as -it is in any definition that repeats `_`: +A base that repeats `_` numbers its own. The first is `_` in the base, and the override's `_` in +that position takes that name as it takes any other, so a read of `_` in the override's body is its +first parameter, as it is in any definition that repeats `_`. The numbered ones are positional-only, +so no call passes one by name, and the override numbers its own there: ```by class Base: @@ -370,6 +371,36 @@ reveal_type(Variadic.f) # revealed: def f(self, _: int, _: int, /, *_: int) -> assert Variadic().f(1, 2, 3) == 20 ``` +## a name the base numbered is the override's to number + +The name the base's lowering gives a `_` is no name a caller passes, so the override does not take +it. Its own `_` there is numbered clear of the names its module spells, which include every name its +body reads: + +`base.by`: + +```by +class Base: + def f(self, _: int, _: int) -> int: + return 1 +``` + +`main.by`: + +```by +from base import Base + +_2 = 5 + +class Override(Base): + override def f(self, _: int, _: int) -> int: + return _2 # error: [used-underscore-name] + +reveal_type(Override.f) # revealed: def f(self, _: int, _: int, /) -> int + +assert Override().f(1, 2) == 5 +``` + ## an override of an override An override that repeats `_` has the names of the method it overrides, so an override of it takes diff --git "a/crates/ty_python_semantic/resources/mdtest/snapshots/basedpython_type_gua\342\200\246_-_basedpython___narrowi\342\200\246_-_a_callable_narrowing\342\200\246_(831b7fe0c360003b).snap" "b/crates/ty_python_semantic/resources/mdtest/snapshots/basedpython_type_gua\342\200\246_-_basedpython___narrowi\342\200\246_-_a_callable_narrowing\342\200\246_(831b7fe0c360003b).snap" new file mode 100644 index 0000000000..dd3b8f7ab4 --- /dev/null +++ "b/crates/ty_python_semantic/resources/mdtest/snapshots/basedpython_type_gua\342\200\246_-_basedpython___narrowi\342\200\246_-_a_callable_narrowing\342\200\246_(831b7fe0c360003b).snap" @@ -0,0 +1,65 @@ +--- +source: crates/mdtest/src/lib.rs +expression: snapshot +--- + +--- +mdtest name: basedpython_type_guards.md - basedpython: narrowing return annotations - a callable narrowing another parameter is not such a callable +mdtest path: crates/ty_python_semantic/resources/mdtest/basedpython_type_guards.md +--- + +# Source files + +## mdtest_snippet.by + +``` + 1 | def want_second(check: (first: object, second: object) -> (second is int)): ... + 2 | + 3 | def is_int_first(first: object, second: object) -> first is int: + 4 | return isinstance(first, int) + 5 | + 6 | def is_int_second(first: object, second: object) -> second is int: + 7 | return isinstance(second, int) + 8 | + 9 | want_second(is_int_second) +10 | # error: [invalid-argument-type] +11 | want_second(is_int_first) +12 | def want_first(check: (first: object, second: object) -> (first is int)): ... +13 | +14 | def f(second_checker: (first: object, second: object) -> (second is int)): +15 | want_second(second_checker) +16 | # error: [invalid-argument-type] +17 | want_first(second_checker) +``` + +# Diagnostics + +``` +error[invalid-argument-type]: Argument to function `want_second` is incorrect + --> src/mdtest_snippet.by:11:13 + | +11 | want_second(is_int_first) + | ^^^^^^^^^^^^ Expected `(first: object, second: object) -> TypeIs[int]`, found `def is_int_first(first: object, second: object) -> TypeIs[int]` +info: the return type narrows `first`, where `second` is expected +info: Function defined here + --> src/mdtest_snippet.by:1:5 + | +1 | def want_second(check: (first: object, second: object) -> (second is int)): ... + | ^^^^^^^^^^^ --------------------------------------------------------- Parameter declared here + +``` + +``` +error[invalid-argument-type]: Argument to function `want_first` is incorrect + --> src/mdtest_snippet.by:17:16 + | +17 | want_first(second_checker) + | ^^^^^^^^^^^^^^ Expected `(first: object, second: object) -> TypeIs[int]`, found `(first: object, second: object) -> TypeIs[int]` +info: the return type narrows `second`, where `first` is expected +info: Function defined here + --> src/mdtest_snippet.by:12:5 + | +12 | def want_first(check: (first: object, second: object) -> (first is int)): ... + | ^^^^^^^^^^ -------------------------------------------------------- Parameter declared here + +``` diff --git a/crates/ty_python_semantic/src/lib.rs b/crates/ty_python_semantic/src/lib.rs index 1215ab58f8..6351935f88 100644 --- a/crates/ty_python_semantic/src/lib.rs +++ b/crates/ty_python_semantic/src/lib.rs @@ -51,7 +51,10 @@ pub use types::conformance::{ConformanceRegistration, WitnessDispatch, WitnessKi pub use types::conversions::{ ConversionImport, ConversionInfo, ConversionRuntime, DISCARD_ADAPTER, }; -pub use types::extensions::{ExtensionAttributeInfo, ExtensionMemberKind}; +pub use types::extensions::{ + ExtensionAttributeInfo, ExtensionMemberKind, extension_backing_function, + extension_backing_functions, +}; pub use types::ide_support::{ ImplementationsFinder, ImportAliasResolution, OverridableMember, ResolvedDefinition, TypeHierarchyClass, contains_identifier, definitions_for_attribute, definitions_for_bin_op, diff --git a/crates/ty_python_semantic/src/semantic_model.rs b/crates/ty_python_semantic/src/semantic_model.rs index 0ca4100f4a..0bc8ef0341 100644 --- a/crates/ty_python_semantic/src/semantic_model.rs +++ b/crates/ty_python_semantic/src/semantic_model.rs @@ -303,8 +303,8 @@ impl<'db> SemanticModel<'db> { let mut imports = Vec::new(); let mut entries = Vec::new(); for entry in table { - if let Some(module) = &entry.import_from { - imports.push(format!("from {module} import {}", entry.function)); + if let Some(import) = &entry.import { + imports.push(import.statement()); } entries.push((entry.member, entry.function)); } @@ -652,28 +652,16 @@ impl<'db> SemanticModel<'db> { }) { return None; } - let extension_file = resolution.extension.file(db); - let import_from = if extension_file == self.file.file(db) { - None - } else { - // spelled the way this file already imports the module: ty's absolute - // module name can be one the interpreter cannot resolve (a file under a - // directory that is not an importable package), and a relative import has - // no absolute spelling at all - Some(crate::types::conversions::imported_module_spelling( - db, - self.file.file(db), - extension_file, - )?) - }; + let (function, import) = crate::types::extensions::backing_function_reference( + db, + self.file.file(db), + resolution.extension, + member, + )?; Some(crate::types::extensions::ExtensionAttributeInfo { - function: crate::types::extensions::backing_function_name( - db, - resolution.extension, - member, - ), + function, kind: resolution.kind, - import_from, + import, receiver_is_class, }) } diff --git a/crates/ty_python_semantic/src/types.rs b/crates/ty_python_semantic/src/types.rs index 003b005187..684584e4f2 100644 --- a/crates/ty_python_semantic/src/types.rs +++ b/crates/ty_python_semantic/src/types.rs @@ -9357,6 +9357,7 @@ impl<'db> Type<'db> { KnownInstanceType::WrappedOptional(ty) => { Ok(Type::KnownInstance(KnownInstanceType::WrappedOptional(*ty))) } + KnownInstanceType::WrappedOptionalClass(ty) => Ok(ty.inner(db)), KnownInstanceType::TypeGenericAlias(instance) => { // When `type[…]` appears in a value position (e.g. in an implicit type alias), // we infer its argument as a type expression. This ensures that we can emit @@ -11021,7 +11022,9 @@ impl<'db> Type<'db> { ); } } - KnownInstanceType::Annotated(ty) | KnownInstanceType::WrappedOptional(ty) => { + KnownInstanceType::Annotated(ty) + | KnownInstanceType::WrappedOptional(ty) + | KnownInstanceType::WrappedOptionalClass(ty) => { ty.inner(db).find_legacy_typevars_impl( db, env, diff --git a/crates/ty_python_semantic/src/types/class/static_literal.rs b/crates/ty_python_semantic/src/types/class/static_literal.rs index 81315e6197..b653a36a2b 100644 --- a/crates/ty_python_semantic/src/types/class/static_literal.rs +++ b/crates/ty_python_semantic/src/types/class/static_literal.rs @@ -989,9 +989,13 @@ impl<'db> StaticClassLiteral<'db> { .as_call_expr() .map_or(&decorator.expression, |call| &call.func); - definition_expression_type(db, class_definition, decorator_callable) - .as_function_literal() - .is_some_and(|function| function.is_known(db, KnownFunction::Dataclass)) + match definition_expression_type(db, class_definition, decorator_callable) { + Type::FunctionLiteral(function) => function.is_known(db, KnownFunction::Dataclass), + // `dataclass(...)` already applied to its options, as basedpython's `data + // class` keyword is + Type::DataclassDecorator(_) => true, + _ => false, + } }) } @@ -4589,6 +4593,17 @@ impl<'a> StatementVisitor<'a> for MemberVisibilityCollector { _ => {} } } + ast::Stmt::TypeAlias(alias) => { + if !self.in_method + && alias.is_private + && let ast::Expr::Name(name) = alias.name.as_ref() + { + self.members.insert( + name.id.clone(), + ruff_python_ast::helpers::MemberVisibility::Private, + ); + } + } // a compound statement's bodies are still the class's body _ => walk_stmt(self, stmt), } diff --git a/crates/ty_python_semantic/src/types/class_base.rs b/crates/ty_python_semantic/src/types/class_base.rs index c244873788..6c10ab98fb 100644 --- a/crates/ty_python_semantic/src/types/class_base.rs +++ b/crates/ty_python_semantic/src/types/class_base.rs @@ -270,6 +270,7 @@ impl<'db> ClassBase<'db> { | KnownInstanceType::UnionType(_) | KnownInstanceType::Literal(_) | KnownInstanceType::WrappedOptional(_) + | KnownInstanceType::WrappedOptionalClass(_) | KnownInstanceType::LiteralStringAlias(_) | KnownInstanceType::NamedTupleSpec(_) | KnownInstanceType::Sentinel(_) diff --git a/crates/ty_python_semantic/src/types/conformance.rs b/crates/ty_python_semantic/src/types/conformance.rs index 5e178d29d3..f4be840436 100644 --- a/crates/ty_python_semantic/src/types/conformance.rs +++ b/crates/ty_python_semantic/src/types/conformance.rs @@ -45,7 +45,7 @@ use crate::types::class::{ClassLiteral, ClassType, StaticClassLiteral}; use crate::types::context::InferContext; use crate::types::diagnostic::INVALID_CONFORMANCE; use crate::types::extensions::{ - applicable_extensions, backing_function_name, extended_class, extension_applies, + applicable_extensions, backing_function_reference, extended_class, extension_applies, extensions_in_module, own_member, }; use ty_module_resolver::ImportingFile; @@ -257,11 +257,12 @@ pub(crate) fn is_conformable<'db>(db: &'db dyn Db, interface: ClassType<'db>) -> pub(crate) struct WitnessEntry { /// the requirement being satisfied pub(crate) member: String, - /// the module-level backing function that implements it + /// the name the module registering the table refers to the backing function that + /// implements it by pub(crate) function: String, - /// the module to import that function from, when it comes from an extension - /// declared elsewhere - pub(crate) import_from: Option, + /// the import that binds the function under that name, when it comes from an + /// extension declared elsewhere + pub(crate) import: Option, } /// the witness table one conformance registers at runtime: every requirement an @@ -288,20 +289,16 @@ pub(crate) fn witness_table<'db>( let Some(supplier) = supplier else { continue; }; - let import_from = if supplier.file(db) == from_file { - None - } else { - match super::conversions::imported_module_spelling(db, from_file, supplier.file(db)) { - Some(module) => Some(module), - // the extension is not reachable from here; the checker has - // already declined to make the conformance applicable - None => continue, - } + // the extension is not reachable from here; the checker has already declined to + // make the conformance applicable + let Some((function, import)) = backing_function_reference(db, from_file, supplier, name) + else { + continue; }; entries.push(WitnessEntry { member: name.to_owned(), - function: backing_function_name(db, supplier, name), - import_from, + function, + import, }); } entries @@ -365,7 +362,7 @@ pub struct ConformanceRegistration { /// `(requirement, the backing function that answers it)`, for every /// requirement an extension supplies pub entries: Vec<(String, String)>, - /// the `from import ` lines those functions need + /// the `from import ` statements those functions need pub imports: Vec, } diff --git a/crates/ty_python_semantic/src/types/conversions.rs b/crates/ty_python_semantic/src/types/conversions.rs index 2bf17f4fc3..6bc670d61c 100644 --- a/crates/ty_python_semantic/src/types/conversions.rs +++ b/crates/ty_python_semantic/src/types/conversions.rs @@ -28,6 +28,7 @@ //! picking one silently would make the output depend on a rule nobody reads. use ruff_db::files::File; +use ruff_db::source::source_text; use ruff_python_ast as ast; use ruff_text_size::{Ranged, TextRange}; use ty_module_resolver::{ModuleName, file_to_module, resolve_module}; @@ -42,6 +43,7 @@ use crate::types::context::InferContext; use crate::types::diagnostic::{AMBIGUOUS_CONVERSION, INVALID_CONVERSION}; use crate::types::extensions::{self, ExtensionMemberKind, ExtensionMemberResolution}; use crate::types::function::FunctionType; +use crate::types::repeated_underscore::WrittenNames; use crate::types::signatures::Parameters; use crate::types::{MemberLookupPolicy, Type, TypeContext}; use ty_module_resolver::ImportingFile; @@ -1044,6 +1046,20 @@ pub struct ConversionImport { pub alias: String, } +impl ConversionImport { + /// the `from import ` statement, `as ` when the two differ + pub fn statement(&self) -> String { + if self.alias == self.name { + format!("from {} import {}", self.module, self.name) + } else { + format!( + "from {} import {} as {}", + self.module, self.name, self.alias + ) + } + } +} + /// the name of the adapter [`ConversionRuntime::DiscardReturn`] defines, which /// is what the emitted `prefix` spells. The definition itself lives with the /// transpiler's other injected helpers, and is built from this @@ -1223,23 +1239,16 @@ fn backing_call_info<'db>( extension: StaticClassLiteral<'db>, dunder: &str, ) -> ConversionInfo { - let function = extensions::backing_function_name(db, extension, dunder); - let extension_file = extension.file(db); - let mut imports = Vec::new(); - if extension_file != from_file { - let Some(module) = imported_module_spelling(db, from_file, extension_file) else { - return ConversionInfo::Rejected( - "the conversion this value needs comes from an `extension` in a module this \ - file does not import; import it, or convert the value explicitly" - .to_owned(), - ); - }; - imports.push(ConversionImport { - module, - name: function.clone(), - alias: function.clone(), - }); - } + let Some((function, function_import)) = + extensions::backing_function_reference(db, from_file, extension, dunder) + else { + return ConversionInfo::Rejected( + "the conversion this value needs comes from an `extension` in a module this \ + file does not import; import it, or convert the value explicitly" + .to_owned(), + ); + }; + let mut imports: Vec = function_import.into_iter().collect(); let (receiver, class_import) = match class_reference(db, env, from_file, model, anchor, class) { Ok(spelling) => spelling, Err(reason) => return ConversionInfo::Rejected(reason), @@ -1267,9 +1276,48 @@ fn backing_call_info<'db>( /// is not safe: the binding may be conditional (`if TYPE_CHECKING:`) or come /// after the site, and the end-of-scope type a symbol lookup reports says /// nothing about either. One leading underscore, not two, so a reference inside -/// a class body is not python name-mangled -fn conversion_alias(name: &str) -> String { - format!("_by_conv__{name}") +/// a class body is not python name-mangled. +/// +/// the module is part of the alias, so two classes of one name imported from two +/// modules are two bindings: `metric.Length` is `_by_conv__metric__Length` and +/// `imperial.Length` is `_by_conv__imperial__Length`. a `_` in the module is +/// written `_u` and a `.` `_d`, which leaves the first `__` after the prefix +/// marking where the class name starts, so no two imports share an alias. and no +/// name the file spells begins with the prefix ([`conversion_alias_prefix`]), so +/// none of them is an alias either +fn conversion_alias(db: &dyn Db, from_file: File, module: &str, name: &str) -> String { + let mut alias = conversion_alias_prefix(db, from_file).clone(); + for character in module.chars() { + match character { + '_' => alias.push_str("_u"), + '.' => alias.push_str("_d"), + character => alias.push(character), + } + } + alias.push_str("__"); + alias.push_str(name); + alias +} + +/// the first of `_by_conv__`, `_by_conv2__`, `_by_conv3__`, … that no name `file` +/// spells begins with — see [`conversion_alias`] +#[salsa::tracked(returns(ref), heap_size = ruff_memory_usage::heap_size)] +fn conversion_alias_prefix(db: &dyn Db, file: File) -> String { + let source = source_text(db, file); + let written = WrittenNames::new(source.as_str()); + let prefix = |number: u32| { + if number == 1 { + "_by_conv__".to_owned() + } else { + format!("_by_conv{number}__") + } + }; + // a module spells finitely many names, so one of the candidates is free long + // before the numbers run out + (1u32..u32::MAX) + .map(prefix) + .find(|candidate| !written.spells_a_name_beginning(candidate)) + .unwrap_or_else(|| prefix(1)) } /// how the conversion site spells `class`: its own name when this file declares @@ -1318,7 +1366,7 @@ pub(crate) fn class_reference<'db>( module this file does not import; import it, or convert the value explicitly" ) })?; - let alias = conversion_alias(&name); + let alias = conversion_alias(db, from_file, &module, &name); Ok(( alias.clone(), Some(ConversionImport { diff --git a/crates/ty_python_semantic/src/types/diagnostic.rs b/crates/ty_python_semantic/src/types/diagnostic.rs index c8caaff0fe..27330e1130 100644 --- a/crates/ty_python_semantic/src/types/diagnostic.rs +++ b/crates/ty_python_semantic/src/types/diagnostic.rs @@ -1243,6 +1243,11 @@ declare_lint! { /// binds a package no rename can keep. The declaration would read as /// restricted while doing something else. /// + /// The rename can also land on a name the class, a base class or a subclass + /// already stores another member under: `protected def m` is stored as `_m`, + /// so beside a `def _m` one of the two replaces the other at runtime while + /// the checker still sees both. + /// /// ## Example /// /// ```by diff --git a/crates/ty_python_semantic/src/types/display.rs b/crates/ty_python_semantic/src/types/display.rs index 2a9fd34a90..9393444065 100644 --- a/crates/ty_python_semantic/src/types/display.rs +++ b/crates/ty_python_semantic/src/types/display.rs @@ -4913,6 +4913,12 @@ impl<'db> FmtDetailed<'db> for DisplayKnownInstanceRepr<'_, 'db> { inner.inner(db).display(db, self.env).fmt_detailed(f)?; f.write_str(", ]'>") } + KnownInstanceType::WrappedOptionalClass(wrapped) => { + f.set_invalid_type_annotation(); + f.write_str("") + } KnownInstanceType::WrappedOptional(inner) => { // render a nested optional in surface `?` notation: count the // wrapper layers, then the innermost `base | None` union adds diff --git a/crates/ty_python_semantic/src/types/extensions.rs b/crates/ty_python_semantic/src/types/extensions.rs index 532965efc1..86a9dbc902 100644 --- a/crates/ty_python_semantic/src/types/extensions.rs +++ b/crates/ty_python_semantic/src/types/extensions.rs @@ -24,7 +24,10 @@ //! [`ClassDef`]: ruff_python_ast::StmtClassDef use ruff_db::files::File; +use ruff_db::source::source_text; use ruff_python_ast as ast; +use ruff_python_ast::name::Name; +use rustc_hash::{FxHashMap, FxHashSet}; use ty_module_resolver::{ModuleName, resolve_module}; use ty_python_core::{global_scope, place_table, semantic_index}; @@ -36,10 +39,11 @@ use crate::types::class::{ClassLiteral, ClassType, KnownClass, StaticClassLitera use crate::types::class_base::ClassBase; use crate::types::conformance; use crate::types::context::InferContext; -use crate::types::conversions::CONVERSION_DUNDERS; +use crate::types::conversions::{CONVERSION_DUNDERS, ConversionImport, imported_module_spelling}; use crate::types::diagnostic::INVALID_EXTENSION; use crate::types::generics::Specialization; use crate::types::member::class_member; +use crate::types::repeated_underscore::WrittenNames; use crate::types::typevar::{BoundTypeVarInstance, TypeVarBoundOrConstraints}; use crate::types::{MemberLookupPolicy, Type}; use ty_module_resolver::ImportingFile; @@ -321,12 +325,12 @@ pub enum ExtensionMemberKind { /// extension member #[derive(Debug, Clone, PartialEq, Eq)] pub struct ExtensionAttributeInfo { - /// the module-level backing function the member lowers to + /// the name the module being transpiled refers to the member's backing function by pub function: String, pub kind: ExtensionMemberKind, - /// the module to import the backing function from, when the extension is + /// the import that binds the backing function under that name, when the extension is /// declared in a module other than the one being transpiled - pub import_from: Option, + pub import: Option, /// whether the receiver is the class object itself (`str.parse(…)`) rather /// than an instance — decides what a `class def` member receives pub receiver_is_class: bool, @@ -335,8 +339,7 @@ pub struct ExtensionAttributeInfo { /// the backing-function name an extension member lowers to: /// `_by_ext__list__second`. when a module declares more than one extension /// of the same target name, later ones carry an ordinal (`_by_ext2__…`) so -/// their members do not collide. the transpiler's block lowering computes the -/// same name from the extension file's AST alone. +/// their members do not collide ([`backing_function_names`]). /// /// exactly one leading underscore: python private-name-mangles any `__name` /// reference inside a class body, so a two-underscore name would break an @@ -346,12 +349,26 @@ pub(crate) fn backing_function_name<'db>( extension: StaticClassLiteral<'db>, member: &str, ) -> String { + if let Some((_, _, name)) = backing_function_names(db, extension.file(db)) + .iter() + .find(|(candidate, name, _)| *candidate == extension && name == member) + { + return name.clone(); + } + // a member the extension's body does not bind has no backing function, and is named + // as the first candidate for it would be let target = extension.name(db); let ordinal = extensions_in_module(db, extension.file(db)) .iter() .filter(|candidate| candidate.name(db) == target) .position(|candidate| *candidate == extension) .unwrap_or(0); + backing_function_candidate(target, ordinal, member) +} + +/// the name a member's backing function is given when the extension is the `ordinal`th of +/// the module's extensions of `target` +fn backing_function_candidate(target: &str, ordinal: usize, member: &str) -> String { if ordinal == 0 { format!("_by_ext__{target}__{member}") } else { @@ -359,6 +376,151 @@ pub(crate) fn backing_function_name<'db>( } } +/// the name of the backing function each member of each extension `file` declares lowers +/// to, in the order the module declares them +/// +/// each is the first candidate ([`backing_function_candidate`]) from the extension's own +/// ordinal up that the module spells nowhere and that no backing function before it has +/// taken, so the module's own `_by_ext__list__second` keeps its name and the member is +/// `_by_ext2__list__second`. the reverse transform reads past the ordinal, so it still finds +/// the target and the member +#[salsa::tracked(returns(deref), heap_size = ruff_memory_usage::heap_size)] +fn backing_function_names( + db: &dyn Db, + file: File, +) -> Box<[(StaticClassLiteral<'_>, Name, String)]> { + let extensions = extensions_in_module(db, file); + if extensions.is_empty() { + return Box::default(); + } + let source = source_text(db, file); + let written = WrittenNames::new(source.as_str()); + let mut ordinals: FxHashMap<&Name, usize> = FxHashMap::default(); + let mut taken: FxHashSet = FxHashSet::default(); + let mut names = Vec::new(); + for &extension in extensions { + let target = extension.name(db); + let ordinal = *ordinals + .entry(target) + .and_modify(|ordinal| *ordinal += 1) + .or_insert(0); + for symbol in place_table(db, extension.body_scope(db)).symbols() { + if !symbol.is_bound() { + continue; + } + let member = symbol.name(); + // candidates only ever count up, and the module spells and the functions before + // this one take finitely many of them, so one is free long before the end + let name = (ordinal..usize::MAX) + .map(|candidate| backing_function_candidate(target, candidate, member)) + .find(|candidate| !written.spells(candidate) && !taken.contains(candidate)) + .unwrap_or_else(|| backing_function_candidate(target, ordinal, member)); + taken.insert(name.clone()); + names.push((extension, member.clone(), name)); + } + } + names.into_boxed_slice() +} + +/// the name `file` binds the backing function of each member of each extension it imports +/// under, in the order [`applicable_extensions`] lists the extensions +/// +/// the declaring module names its backing functions clear of what it spells, and nothing +/// else: two modules extending classes of the same name with members of the same name both +/// declare `_by_ext__Foo__bar`, and the module importing one may declare the other, or spell +/// the name itself. so each imported function is bound under the first of its own name, +/// `…2`, `…3`, … that the module spells nowhere, that none of its own backing functions is +/// declared under, and that no function imported before it has taken +#[salsa::tracked(returns(deref), heap_size = ruff_memory_usage::heap_size)] +fn imported_backing_function_names( + db: &dyn Db, + file: File, +) -> Box<[(StaticClassLiteral<'_>, Name, String)]> { + let source = source_text(db, file); + let written = WrittenNames::new(source.as_str()); + let mut taken: FxHashSet = backing_function_names(db, file) + .iter() + .map(|(_, _, name)| name.clone()) + .collect(); + let mut aliases: Vec<(StaticClassLiteral<'_>, Name, String)> = Vec::new(); + for &extension in applicable_extensions(db, file) { + let declaring = extension.file(db); + if declaring == file { + continue; + } + for (candidate, member, name) in backing_function_names(db, declaring) { + if *candidate != extension { + continue; + } + let alias = written.fresh_outside(name, |alias| taken.contains(alias)); + taken.insert(alias.clone()); + aliases.push((extension, member.clone(), alias)); + } + } + aliases.into_boxed_slice() +} + +/// how `from_file` refers to the backing function of `member` of `extension`: by the name it +/// is declared under when `from_file` declares the extension, and otherwise by the name +/// `from_file` imports it under ([`imported_backing_function_names`]), with the import that +/// binds it. `None` when `from_file` has no import spelling for the declaring module +pub(crate) fn backing_function_reference<'db>( + db: &'db dyn Db, + from_file: File, + extension: StaticClassLiteral<'db>, + member: &str, +) -> Option<(String, Option)> { + let name = backing_function_name(db, extension, member); + let declaring = extension.file(db); + if declaring == from_file { + return Some((name, None)); + } + // spelled the way this file already imports the module: ty's absolute module name can be + // one the interpreter cannot resolve (a file under a directory that is not an importable + // package), and a relative import has no absolute spelling at all + let module = imported_module_spelling(db, from_file, declaring)?; + // a member the extension's body does not bind has no backing function to import, and + // keeps the name it would have been declared under + let alias = imported_backing_function_names(db, from_file) + .iter() + .find(|(candidate, candidate_member, _)| { + *candidate == extension && candidate_member == member + }) + .map_or_else(|| name.clone(), |(_, _, alias)| alias.clone()); + Some(( + alias.clone(), + Some(ConversionImport { + module, + name, + alias, + }), + )) +} + +/// every name `file` binds a backing function under once lowered: the ones its extensions +/// declare, then the ones it imports from the modules declaring the others +pub fn extension_backing_functions(db: &dyn Db, file: File) -> impl Iterator { + backing_function_names(db, file) + .iter() + .chain(imported_backing_function_names(db, file)) + .map(|(_, _, name)| name.as_str()) +} + +/// the name of the backing function `member` of `extension` lowers to, or `None` when +/// `extension` is not an extension +pub fn extension_backing_function<'db>( + db: &'db dyn Db, + extension: ClassLiteral<'db>, + member: &str, +) -> Option { + let ClassLiteral::Static(extension) = extension else { + return None; + }; + extension + .is_extension(db) + .then(|| backing_function_name(db, extension, member)) +} + /// a successful extension-member resolution #[derive(Debug, Clone, Copy)] pub(crate) struct ExtensionMemberResolution<'db> { diff --git a/crates/ty_python_semantic/src/types/function.rs b/crates/ty_python_semantic/src/types/function.rs index a5badf47ac..26842520b2 100644 --- a/crates/ty_python_semantic/src/types/function.rs +++ b/crates/ty_python_semantic/src/types/function.rs @@ -1463,6 +1463,8 @@ impl<'db> OverloadLiteral<'db> { }); } + raw_signature.name_implicitly_narrowed_parameter(self.has_implicit_receiver(db)); + raw_signature } diff --git a/crates/ty_python_semantic/src/types/infer/builder.rs b/crates/ty_python_semantic/src/types/infer/builder.rs index 23a9b3e5d9..e3dba52428 100644 --- a/crates/ty_python_semantic/src/types/infer/builder.rs +++ b/crates/ty_python_semantic/src/types/infer/builder.rs @@ -15702,6 +15702,34 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { } } + /// a `?` outside a type expression over a value that has no `| None` + fn report_unsupported_optional_marker( + &self, + unary: &ast::ExprUnaryOp, + operand_type: Type<'db>, + ) { + let db = self.db(); + let env = self.program_environment(); + let Some(builder) = self.context.report_lint(&UNSUPPORTED_OPERATOR, unary) else { + return; + }; + let mut diagnostic = builder.into_diagnostic(format_args!( + "Unary operator `?` is not supported for object of type `{}`", + operand_type.display(db, env), + )); + diagnostic.info("outside a type expression, `T?` is evaluated as `T | None`"); + // `x == T?` reads as a comparison with an optional type, but the marker takes the + // whole comparison. a parenthesized one was written that way on purpose + if let ast::Expr::Compare(compare) = &*unary.operand + && compare.start() == unary.start() + { + diagnostic.info( + "`?` binds looser than a comparison, so it marks the whole comparison; \ + parenthesize the type to compare with an optional, as in `x == (T?)`", + ); + } + } + fn report_unsupported_unary_operator( &self, unary: &ast::ExprUnaryOp, @@ -15822,6 +15850,20 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { ) -> Type<'db> { let db = self.db(); let env = self.program_environment(); + // an optional over an optional, or over a type variable other than `Self`, is the + // runtime `Optional` wrapper rather than a union + let is_wrapped_optional = move |unary: &ast::ExprUnaryOp, operand_type: Type<'db>| { + matches!( + &*unary.operand, + ast::Expr::UnaryOp(operand) if operand.op == ast::UnaryOp::Optional + ) || match operand_type { + Type::KnownInstance(KnownInstanceType::TypeVar(typevar)) => !typevar.is_self(db), + Type::SubclassOf(subclass_of) => subclass_of + .into_type_var() + .is_some_and(|typevar| !typevar.typevar(db).is_self(db)), + _ => false, + } + }; let fallback_unary_expression_type = || { let unary_dunder_method = match op { ast::UnaryOp::Invert => "__invert__", @@ -15940,10 +15982,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { // to the arm below (ast::UnaryOp::Optional, _) if self.is_basedpython_file() - && !matches!( - &*unary.operand, - ast::Expr::UnaryOp(operand) if operand.op == ast::UnaryOp::Optional - ) + && !is_wrapped_optional(unary, operand_type) && matches!( operand_type, Type::ClassLiteral(..) @@ -15951,16 +15990,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { | Type::GenericAlias(..) | Type::SpecialForm(_) | Type::KnownInstance(_) - ) - && !match operand_type { - Type::KnownInstance(KnownInstanceType::TypeVar(typevar)) => { - !typevar.is_self(db) - } - Type::SubclassOf(subclass_of) => subclass_of - .into_type_var() - .is_some_and(|typevar| !typevar.typevar(db).is_self(db)), - _ => false, - } => + ) => { UnionTypeInstance::from_value_expression_types( db, @@ -15971,9 +16001,63 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { ) } - // basedpython postfix `?` and `^` / `!` on non-optional operands. - // These are wrapped-type surface syntax that the transpiler lowers - // away; ty does not yet model the remaining unwrap/propagate cases + // basedpython: over any other value the `?` is still the `| None` the transpiler + // writes, which a value that is not a type rarely supports — `(x is int)?` is + // `bool | None`, and raises where it runs + (ast::UnaryOp::Optional, _) + if self.is_basedpython_file() && !is_wrapped_optional(unary, operand_type) => + { + let mut state = BinaryInferenceState::default(); + let union = self.infer_binary_expression_type( + unary.into(), + operand_type, + Type::none(db, env), + ast::Operator::BitOr, + TypeContext::default(), + &mut state, + ); + self.report_deprecated_functions(unary, state.deprecated_functions); + union.unwrap_or_else(|| { + self.report_unsupported_optional_marker(unary, operand_type); + Type::unknown() + }) + } + + // basedpython: an optional over an optional, or over a type variable other than + // `Self`, is the wrapped optional in a type expression, and the transpiler writes + // it as the runtime wrapper class subscripted, which evaluates to the class itself + // whatever it wraps. the operand is a type form the program evaluates too, so one + // that is not a valid type expression still leaves the class, wrapping what the + // type expression falls back to + (ast::UnaryOp::Optional, _) if self.is_basedpython_file() => { + let inner = operand_type + .in_type_expression( + db, + self.scope(), + self.typevar_binding_context, + self.inference_flags(), + ) + .unwrap_or_else(|error| error.fallback_type); + let operand_is_optional = matches!( + &*unary.operand, + ast::Expr::UnaryOp(operand) if operand.op == ast::UnaryOp::Optional + ); + let inner = if operand_is_optional { + inner + } else { + UnionType::from_elements_leave_aliases(db, env, [inner, Type::none(db, env)]) + }; + Type::KnownInstance(KnownInstanceType::WrappedOptionalClass(InternedType::new( + db, + Type::KnownInstance(KnownInstanceType::WrappedOptional(InternedType::new( + db, inner, + ))), + ))) + } + + // basedpython `^` / `!` on non-optional operands. These are wrapped-type + // surface syntax that the transpiler lowers away; ty does not yet model the + // remaining unwrap/propagate cases (ast::UnaryOp::Optional | ast::UnaryOp::Propagate | ast::UnaryOp::Force, _) => { todo_type!("basedpython wrapped-type operator") } diff --git a/crates/ty_python_semantic/src/types/infer/builder/binary_expressions.rs b/crates/ty_python_semantic/src/types/infer/builder/binary_expressions.rs index d8b421eecf..ab08b6ba36 100644 --- a/crates/ty_python_semantic/src/types/infer/builder/binary_expressions.rs +++ b/crates/ty_python_semantic/src/types/infer/builder/binary_expressions.rs @@ -850,7 +850,8 @@ impl<'db> TypeInferenceBuilder<'db, '_> { | KnownInstanceType::Callable(_) | KnownInstanceType::TypeVar(_) | KnownInstanceType::TypeAliasType(_) - | KnownInstanceType::NewType(_), + | KnownInstanceType::NewType(_) + | KnownInstanceType::WrappedOptionalClass(_), ), Type::ClassLiteral(..) | Type::SubclassOf(..) @@ -864,7 +865,8 @@ impl<'db> TypeInferenceBuilder<'db, '_> { | KnownInstanceType::Callable(_) | KnownInstanceType::TypeVar(_) | KnownInstanceType::TypeAliasType(_) - | KnownInstanceType::NewType(_), + | KnownInstanceType::NewType(_) + | KnownInstanceType::WrappedOptionalClass(_), ), ast::Operator::BitOr, ) => { diff --git a/crates/ty_python_semantic/src/types/infer/builder/type_expression.rs b/crates/ty_python_semantic/src/types/infer/builder/type_expression.rs index ab7b819e5c..fd456868c4 100644 --- a/crates/ty_python_semantic/src/types/infer/builder/type_expression.rs +++ b/crates/ty_python_semantic/src/types/infer/builder/type_expression.rs @@ -30,7 +30,7 @@ use crate::types::repeated_underscore::{ LoweredParameters, ParameterSlot, WrittenNames, callable_parameter_slots, lower_repeated_underscores, }; -use crate::types::signatures::{ConcatenateTail, Signature}; +use crate::types::signatures::{ConcatenateTail, NarrowingGuard, NarrowingGuardKind, Signature}; use crate::types::special_form::{AliasSpec, LegacyStdlibAlias}; use crate::types::string_annotation::parse_string_annotation; use crate::types::template::{Promotable, TemplateLiteralType, TemplatePart}; @@ -1637,6 +1637,53 @@ impl<'db> TypeInferenceBuilder<'db, '_> { } return crate::types::TypeIsType::from_type_expression(self.db(), narrowed); } + // basedpython: `a is not T` reads like the predicate negated, but nothing in + // python's typing narrows to everything but a type. the place is a label, as + // in the predicate, so only the type is inferred + if self.is_basedpython_file() + && let Some(negated) = + ruff_python_ast::helpers::negated_narrowing_predicate(expression) + { + for comparator in &negated.comparators { + self.infer_type_expression(comparator); + } + if let Some(builder) = self.context.report_lint(&INVALID_TYPE_FORM, expression) + { + let mut diagnostic = builder.into_diagnostic( + "A narrowing predicate cannot be negated: `is not` would narrow to \ + everything but the type, which `TypeIs` cannot express", + ); + diagnostic.info( + "write the predicate as `is` and negate the call where it is used", + ); + } + return Type::unknown(); + } + // basedpython: a callable type's return binds tighter than a comparison, as it + // does than `|`, so `(x: object) -> x is int` tests the callable type `(x: + // object) -> x` rather than narrowing `x`. its return names a parameter, which + // is no type, so only the type tested for is inferred + if self.is_basedpython_file() + && matches!(&*compare.ops, [ast::CmpOp::Is | ast::CmpOp::IsNot]) + && let ast::Expr::CallableType(callable) = compare.left.as_ref() + && is_narrowing_predicate_place(&callable.returns) + { + for comparator in &compare.comparators { + self.infer_type_expression(comparator); + } + if let Some(builder) = self.context.report_lint(&INVALID_TYPE_FORM, expression) + { + let mut diagnostic = builder.into_diagnostic( + "`is` takes the whole callable type here: a callable type's return \ + binds tighter than a comparison", + ); + diagnostic.info( + "parenthesize the narrowing predicate the callable returns, as in \ + `(x: object) -> (x is int)`", + ); + } + return Type::unknown(); + } // basedpython: a rich comparison in a type expression folds like it // does on values (`1 < 2` → `Literal[True]`), and is kept symbolic // while an operand still mentions a type parameter so `I < 2` can @@ -1918,7 +1965,9 @@ impl<'db> TypeInferenceBuilder<'db, '_> { } None => Parameters::gradual_form(), }; - return Type::single_callable(db, Signature::new(parameters, return_type)); + let signature = self + .with_callable_narrowing_guard(callable, Signature::new(parameters, return_type)); + return Type::single_callable(db, signature); } // basedpython: a trailing bare `**P` unpacks a parameter pack — @@ -1972,7 +2021,11 @@ impl<'db> TypeInferenceBuilder<'db, '_> { .map(|tail| Parameters::concatenate(db, prefix_params, tail)) .unwrap_or_else(Parameters::unknown) }; - return Type::single_callable(db, Signature::new(parameters, return_type)); + let signature = self.with_callable_narrowing_guard( + callable, + Signature::new(parameters, return_type), + ); + return Type::single_callable(db, signature); } } @@ -1999,7 +2052,9 @@ impl<'db> TypeInferenceBuilder<'db, '_> { let previous = self .inference_flags() .replace(InferenceFlags::CHECK_UNBOUND_TYPEVARS, false); - let result = Type::single_callable(db, Signature::new(parameters, return_type)); + let signature = + self.with_callable_narrowing_guard(callable, Signature::new(parameters, return_type)); + let result = Type::single_callable(db, signature); self.inference_flags() .set(InferenceFlags::CHECK_UNBOUND_TYPEVARS, previous); result @@ -2010,6 +2065,37 @@ impl<'db> TypeInferenceBuilder<'db, '_> { /// follows, so a call is checked against the `__call__` or method the transpiler /// declares. a shape the lowering refuses is reported, and its parameters are left /// as written + /// basedpython: `signature`, the signature of `callable`, narrowing what its return + /// predicate names — `(x: object) -> (x is int)` — the way a `def` whose return annotation + /// is that predicate narrows it: the argument matched to a parameter of that name, and + /// otherwise a place of that name in the calling scope + fn with_callable_narrowing_guard( + &self, + callable: &ast::ExprCallableType, + mut signature: Signature<'db>, + ) -> Signature<'db> { + if !self.is_basedpython_file() || !matches!(signature.return_ty, Type::TypeIs(_)) { + return signature; + } + let Some(guard) = ruff_python_ast::helpers::predicate_guard(&callable.returns) else { + return signature; + }; + let (name, members) = guard.place_parts(); + let root_is_first_parameter = signature + .parameters() + .iter() + .next() + .and_then(Parameter::name) + .is_some_and(|first| first == name); + signature.narrowing_guards = Box::new([NarrowingGuard { + name: name.clone(), + members: members.into_iter().cloned().collect(), + root_is_first_parameter, + kind: NarrowingGuardKind::Predicate, + }]); + signature + } + fn lower_callable_repeated_underscores( &mut self, callable: &ast::ExprCallableType, @@ -3385,7 +3471,8 @@ impl<'db> TypeInferenceBuilder<'db, '_> { } Type::unknown() } - KnownInstanceType::WrappedOptional(_) => { + KnownInstanceType::WrappedOptional(_) + | KnownInstanceType::WrappedOptionalClass(_) => { if !self.in_string_annotation() { self.infer_expression(slice, TypeContext::default()); } diff --git a/crates/ty_python_semantic/src/types/known_instance.rs b/crates/ty_python_semantic/src/types/known_instance.rs index 398ef4cb3c..403d63e0d3 100644 --- a/crates/ty_python_semantic/src/types/known_instance.rs +++ b/crates/ty_python_semantic/src/types/known_instance.rs @@ -141,6 +141,13 @@ pub enum KnownInstanceType<'db> { /// operator peels one layer. WrappedOptional(InternedType<'db>), + /// basedpython: the runtime class a wrapped optional evaluates to outside a type + /// expression — `int??` read as a value, as an alias or `isinstance`'s classes. python + /// gets the one wrapper class whatever the layers wrap, so the value is a class, and the + /// wrapped optional it was written as (a [`Self::WrappedOptional`] type) is what it + /// denotes in a type expression + WrappedOptionalClass(InternedType<'db>), + /// An instance of `typing.GenericAlias` representing a `type[...]` expression. TypeGenericAlias(InternedType<'db>), @@ -217,6 +224,7 @@ pub(super) fn walk_known_instance_type<'db, V: visitor::TypeVisitor<'db> + ?Size | KnownInstanceType::Annotated(ty) | KnownInstanceType::TypeGenericAlias(ty) | KnownInstanceType::WrappedOptional(ty) + | KnownInstanceType::WrappedOptionalClass(ty) | KnownInstanceType::LiteralStringAlias(ty) => { visitor.visit_type(db, ty.inner(db)); } @@ -290,6 +298,9 @@ impl<'db> KnownInstanceType<'db> { Self::WrappedOptional(ty) => ty .recursive_type_normalized_impl(db, env, div, true) .map(Self::WrappedOptional), + Self::WrappedOptionalClass(ty) => ty + .recursive_type_normalized_impl(db, env, div, true) + .map(Self::WrappedOptionalClass), Self::TypeGenericAlias(ty) => ty .recursive_type_normalized_impl(db, env, div, true) .map(Self::TypeGenericAlias), @@ -349,6 +360,7 @@ impl<'db> KnownInstanceType<'db> { // a wrapped optional has no dedicated runtime class yet (the // `Option` prelude is still being settled), so fall back to `object` Self::WrappedOptional(_) => KnownClass::Object, + Self::WrappedOptionalClass(_) => KnownClass::Type, Self::LiteralStringAlias(_) => KnownClass::Str, Self::NewType(_) => KnownClass::NewType, Self::Sentinel(_) => KnownClass::Sentinel, @@ -388,9 +400,10 @@ impl<'db> KnownInstanceType<'db> { match self { Self::TypeAliasType(alias) => Some(Type::TypeAlias(alias)), Self::UnionType(instance) => instance.union_type(db).as_ref().ok().copied(), - Self::Literal(ty) | Self::Annotated(ty) | Self::LiteralStringAlias(ty) => { - Some(ty.inner(db)) - } + Self::Literal(ty) + | Self::Annotated(ty) + | Self::LiteralStringAlias(ty) + | Self::WrappedOptionalClass(ty) => Some(ty.inner(db)), Self::TypeGenericAlias(instance) => Some(instance.inner(db).to_meta_type(db, env)), Self::Callable(callable) => Some(Type::Callable(callable)), Self::NewType(newtype) => Some(Type::NewTypeInstance(newtype)), @@ -414,6 +427,7 @@ impl<'db> KnownInstanceType<'db> { | Self::LiteralStringAlias(_) | Self::NewType(_) | Self::Sentinel(_) + | Self::WrappedOptionalClass(_) ) } @@ -487,6 +501,13 @@ impl<'db> KnownInstanceType<'db> { .apply_type_mapping_impl(db, env, type_mapping, tcx, visitor), ))) } + KnownInstanceType::WrappedOptionalClass(ty) => { + Type::KnownInstance(KnownInstanceType::WrappedOptionalClass(InternedType::new( + db, + ty.inner(db) + .apply_type_mapping_impl(db, env, type_mapping, tcx, visitor), + ))) + } KnownInstanceType::Callable(callable_type) => { Type::KnownInstance(KnownInstanceType::Callable( callable_type.apply_type_mapping_impl(db, type_mapping, tcx, visitor), diff --git a/crates/ty_python_semantic/src/types/overrides.rs b/crates/ty_python_semantic/src/types/overrides.rs index 0b5ff15687..1cb093a5de 100644 --- a/crates/ty_python_semantic/src/types/overrides.rs +++ b/crates/ty_python_semantic/src/types/overrides.rs @@ -12,6 +12,7 @@ use ruff_db::{ use ruff_python_ast::helpers::{MemberVisibility, is_let_marker_id}; use ruff_python_ast::{self as ast, PythonVersion, name::Name}; use ruff_python_stdlib::identifiers::is_mangled_private; +use ruff_text_size::Ranged; use rustc_hash::FxHashSet; use crate::{ @@ -76,6 +77,8 @@ pub(super) fn check_class<'db>( inconsistent_generic_bases: bool, ) { let db = context.db(); + check_emitted_name_collisions(context, class); + let configuration = OverrideRulesConfig::from(context); if configuration.no_rules_enabled() { return; @@ -1684,6 +1687,184 @@ fn check_member_visibility<'db>( )); } +/// basedpython: reports a member of `class` that is stored under the same attribute +/// name as a different member of `class` or of a class in its MRO, because a +/// visibility keyword renamed one of them onto the other's name +/// +/// ```by +/// class A: +/// def _m(self) -> int: ... +/// protected def m(self) -> int: ... # stored as `_m`, replacing the method above +/// ``` +/// +/// the two are different members to the checker, but at runtime whichever the class +/// body or the MRO reaches first replaces the other. a member redeclared under its +/// own written name is an override, not a collision, unless it is `private`: a +/// private member belongs to its declaring class alone, so two classes of the same +/// name that each declare one still store both under one mangled name +fn check_emitted_name_collisions<'db>( + context: &InferContext<'db, '_>, + class: StaticClassLiteral<'db>, +) { + let db = context.db(); + let env = &context.program_environment(); + let mro: Vec> = class + .identity_specialization(db) + .iter_mro(db) + .filter_map(ClassBase::into_class) + .collect(); + let is_own = |candidate: ClassType<'db>| candidate.class_literal(db).as_static() == Some(class); + let mut reported: FxHashSet = FxHashSet::default(); + + for &declaring in &mro { + let Some(declaring_literal) = declaring.class_literal(db).as_static() else { + continue; + }; + let mut restricted: Vec<(&Name, MemberVisibility)> = declaring_literal + .member_visibilities(db) + .iter() + .filter(|(_, visibility)| **visibility != MemberVisibility::Public) + .map(|(name, visibility)| (name, *visibility)) + .collect(); + restricted.sort_unstable_by_key(|(name, _)| *name); + + for (written, visibility) in restricted { + let Some(stored) = crate::types::visibility::emitted_member_name( + db, + visibility, + declaring_literal, + written, + ) else { + continue; + }; + for &other in &mro { + if !is_own(declaring) && !is_own(other) { + continue; + } + for candidate in names_stored_as(db, other, &stored) { + let same_declaration = other == declaring && candidate == written.as_str(); + if same_declaration + || !defines_member(db, env, other, &candidate) + || stored_member_name(db, other, &candidate) != stored + { + continue; + } + let other_visibility = own_member_visibility(db, other, &candidate); + // the same member redeclared: an override, checked as one elsewhere + if candidate == written.as_str() + && visibility != MemberVisibility::Private + && other_visibility != MemberVisibility::Private + { + continue; + } + let (own_name, message, info) = if is_own(declaring) { + ( + written.clone(), + format!( + "`{written}` cannot be `{keyword}` here", + keyword = visibility.keyword() + ), + format!( + "`{keyword}` stores it as `{stored}`, where {other} `{candidate}` is \ + stored too, so one replaces the other at runtime", + keyword = visibility.keyword(), + other = if is_own(other) { + "this class's".to_owned() + } else { + format!("the base class `{}`'s", other.name(db)) + }, + ), + ) + } else { + ( + Name::new(&candidate), + format!( + "`{candidate}` replaces `{declaring}`'s `{keyword}` member \ + `{written}` at runtime", + declaring = declaring.name(db), + keyword = visibility.keyword(), + ), + format!( + "`{keyword}` stores `{declaring}.{written}` as `{stored}`, and \ + `{candidate}` here is stored under that name too", + declaring = declaring.name(db), + keyword = visibility.keyword(), + ), + ) + }; + if !reported.insert(own_name.clone()) { + continue; + } + let scope = class.body_scope(db); + let range = place_table(db, scope) + .symbol_id(&own_name) + .and_then(|symbol| symbol_definition(db, scope, symbol)) + .map_or_else( + || class.header_range(db), + |definition| definition.focus_range(db, context.module()).range(), + ); + let Some(builder) = + context.report_lint(&crate::types::diagnostic::INVALID_VISIBILITY, range) + else { + continue; + }; + let mut diagnostic = builder.into_diagnostic(message); + diagnostic.info(info); + } + } + } + } +} + +/// basedpython: the names a member of `class` could be written with to be stored +/// under `stored` — the name itself, the name `protected` prefixes to it, and the +/// two spellings python and `private` mangle to it in `class` +fn names_stored_as(db: &dyn Db, class: ClassType<'_>, stored: &str) -> Vec { + let mut names = vec![stored.to_owned()]; + if let Some(unprefixed) = stored.strip_prefix('_') { + names.push(unprefixed.to_owned()); + } + let mangle_prefix = crate::types::visibility::mangled_private_name(class.name(db), ""); + if let Some(member) = stored.strip_prefix(mangle_prefix.as_str()) { + names.push(format!("__{member}")); + names.push(member.to_owned()); + } + names.sort_unstable(); + names.dedup(); + names +} + +/// basedpython: whether `class`'s own body or its instances define `name` +fn defines_member<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + class: ClassType<'db>, + name: &str, +) -> bool { + !class.own_class_member(db, env, None, name).is_undefined() + || !class.own_instance_member(db, env, name).is_undefined() +} + +/// basedpython: the attribute `class`'s own member written `name` is stored +/// under at runtime — renamed by its visibility keyword, and otherwise mangled by +/// python where the name starts with two underscores +fn stored_member_name(db: &dyn Db, class: ClassType<'_>, name: &str) -> String { + let visibility = own_member_visibility(db, class, name); + if visibility != MemberVisibility::Public + && let Some(literal) = class.class_literal(db).as_static() + && let Some(renamed) = + crate::types::visibility::emitted_member_name(db, visibility, literal, name) + { + return renamed; + } + match name.strip_prefix("__") { + Some(member) if is_mangled_private(name) => { + crate::types::visibility::mangled_private_name(class.name(db), member) + } + _ => name.to_owned(), + } +} + pub(crate) fn is_constructor_like_method(name: &str) -> bool { matches!( name, diff --git a/crates/ty_python_semantic/src/types/reified_infer.rs b/crates/ty_python_semantic/src/types/reified_infer.rs index d6d1d79bda..64aa8f0f17 100644 --- a/crates/ty_python_semantic/src/types/reified_infer.rs +++ b/crates/ty_python_semantic/src/types/reified_infer.rs @@ -733,6 +733,8 @@ pub enum ParametricIsPlan { /// redundant: python's `1 == True` would otherwise let a `bool` satisfy /// `Literal[1]` Equality { + /// the builtin class the value has, by its name in `builtins`, which the lowering + /// reads under a name of its own where the module binds that one class: String, /// the value to compare against. the source's own literal where it /// wrote one — which is already spelled correctly for wherever it sits, @@ -1372,7 +1374,8 @@ fn literal_target_plan<'db>( target_node: Option<&ast::Expr>, ) -> ParametricIsPlan { let unspellable = ParametricIsPlan::ErasedTarget(ErasedTargetReason::Unspellable); - // the guard names a builtin, whose spelling is fixed and always in scope + // the guard names a builtin, and the value is the source's own literal where it wrote + // one let written = target_node .is_some_and(|node| node.is_literal_expr() || matches!(node, ast::Expr::UnaryOp(_))); let equality = |class: &str, value: String| ParametricIsPlan::Equality { diff --git a/crates/ty_python_semantic/src/types/relation_error.rs b/crates/ty_python_semantic/src/types/relation_error.rs index 989a56f5d1..3184fc182d 100644 --- a/crates/ty_python_semantic/src/types/relation_error.rs +++ b/crates/ty_python_semantic/src/types/relation_error.rs @@ -173,6 +173,11 @@ pub(crate) enum ErrorContext<'db> { source: Type<'db>, target: Type<'db>, }, + /// basedpython: both return a `TypeIs` / `TypeGuard`, about different arguments + NarrowsAnotherArgument { + source: String, + target: String, + }, IncompatibleParameterTypes { source: Type<'db>, target: Type<'db>, @@ -540,6 +545,9 @@ impl<'db> ErrorContext<'db> { relation = relation.description(), target = target.display(db, env), ), + Self::NarrowsAnotherArgument { source, target } => { + format!("the return type narrows {source}, where {target} is expected") + } Self::IncompatibleParameterTypes { source, target, diff --git a/crates/ty_python_semantic/src/types/repeated_underscore.rs b/crates/ty_python_semantic/src/types/repeated_underscore.rs index 8b13468ace..3d96b9d0ff 100644 --- a/crates/ty_python_semantic/src/types/repeated_underscore.rs +++ b/crates/ty_python_semantic/src/types/repeated_underscore.rs @@ -4,7 +4,8 @@ //! first a name of its own. those names are the lowering's rather than the author's, so no //! call reaches such a parameter by keyword: it is positional-only in the python. a method //! that overrides one is the exception, and takes the names the overridden method gives -//! those positions, so a call written against the base keeps working on the override +//! those positions, so a call written against the base keeps working on the override — bar +//! a name the base's own lowering numbered a `_` with, which no call passes either //! //! what each parameter is called in the python, and how many of them a call reaches by //! position alone, is decided here once — for ty's signature of the definition and for the @@ -72,6 +73,16 @@ impl<'src> WrittenNames<'src> { .unwrap_or_else(|| stem.to_owned()) } + /// whether a name spelled anywhere in the module begins with `prefix` + pub fn spells_a_name_beginning(self, prefix: &str) -> bool { + self.source.match_indices(prefix).any(|(start, _)| { + !self.source[..start] + .chars() + .next_back() + .is_some_and(is_identifier_continuation) + }) + } + /// whether `name` is spelled anywhere in the module, as a whole run of identifier /// characters rather than as a part of a longer one pub fn spells(self, name: &str) -> bool { @@ -230,6 +241,16 @@ pub struct BaseParameter { pub underscore: bool, } +impl BaseParameter { + /// whether an override's `_` in this position has to take this parameter's name. a name + /// the base's lowering numbered a `_` with is no name a call can pass: it is + /// positional-only, so the override numbers its own `_` there, clear of the names its + /// module spells — among them every name its body reads + fn imposes_name(&self) -> bool { + !(self.underscore && self.positional_only && self.name != "_") + } +} + /// how the lowering writes a parameter list that repeats `_` #[derive(Clone, Debug, PartialEq, Eq, Hash, get_size2::GetSize, salsa::SalsaValue)] pub struct UnderscoreLowering { @@ -259,19 +280,25 @@ impl UnderscoreLowering { .iter() .map(|(slot, name)| (*slot, Name::from(*name))) .collect(); - if let Some(inherited) = &self.inherited { + let kept = self.inherited.as_ref().map(|inherited| { + let mut kept = vec![false; spelled.len()]; let underscores = spelled .iter_mut() - .filter(|(slot, name)| slot.is_positional() && name == "_"); - for ((_, name), base) in underscores.zip(inherited) { - name.clone_from(&base.name); + .zip(&mut kept) + .filter(|((slot, name), _)| slot.is_positional() && name == "_"); + for (((_, name), kept), base) in underscores.zip(inherited) { + if base.imposes_name() { + name.clone_from(&base.name); + *kept = true; + } } - } + kept + }); let spelled: Vec<(ParameterSlot, &str)> = spelled .iter() .map(|(slot, name)| (*slot, name.as_str())) .collect(); - number_underscores(&spelled, self.is_inherited(), written) + number_underscores(&spelled, kept.as_deref(), written) } } @@ -473,9 +500,10 @@ pub fn lower_repeated_underscores( // parameter taking it shadows nothing. only `_` can be one: an override of a method // whose own `_`s are numbered takes `_` for the first of them if let Some((index, parameter)) = inherited.iter().find(|(_, parameter)| { - !slots - .iter() - .any(|(_, spelled)| *spelled == parameter.name.as_str()) + parameter.imposes_name() + && !slots + .iter() + .any(|(_, spelled)| *spelled == parameter.name.as_str()) && reads_from_enclosing_scope(¶meter.name) }) { return Some(Err(UnderscoreRefusal::ShadowsEnclosingName { @@ -536,7 +564,8 @@ pub fn lower_repeated_underscores( /// the parameter of `base` each of `positional_underscores` stands in for, by position /// -/// `None` unless every one of them has a counterpart with a name the definition does not +/// `None` unless every one of them has a counterpart, and every counterpart whose name the +/// `_` has to take ([`BaseParameter::imposes_name`]) has a name the definition does not /// already give a parameter of its own. an override that lines up with its base in part /// does not line up with it, and is reported as an invalid override fn inherited_names<'a>( @@ -557,7 +586,9 @@ fn inherited_names<'a>( .map(|&index| { base.get(index)? .as_ref() - .filter(|parameter| !own.contains(parameter.name.as_str())) + .filter(|parameter| { + !parameter.imposes_name() || !own.contains(parameter.name.as_str()) + }) .map(|parameter| (index, parameter)) }) .collect() @@ -602,15 +633,15 @@ pub fn lowered_parameter_names( parameters: &[(ParameterSlot, &str)], written: WrittenNames, ) -> Vec { - number_underscores(parameters, false, written) + number_underscores(parameters, None, written) } /// `parameters` with every `_` numbered but the ones that keep the name: the first in /// numbering order, or when the positional ones have taken the names of an overridden -/// method, each positional one still called `_`, which is a `_` the base calls `_` too +/// method, the ones `kept` says took one, which for a `_` is a `_` the base calls `_` too fn number_underscores( parameters: &[(ParameterSlot, &str)], - inherited: bool, + kept: Option<&[bool]>, written: WrittenNames, ) -> Vec { let taken: HashSet<&str> = parameters.iter().map(|(_, name)| *name).collect(); @@ -620,13 +651,13 @@ fn number_underscores( .iter() .map(|(_, name)| Name::from(*name)) .collect(); - let mut keep_first = !inherited; + let mut keep_first = kept.is_none(); // candidates only ever count up, so one never repeats an earlier one let mut next = 2u32; for index in order { - let (slot, name) = parameters[index]; + let name = parameters[index].1; if name != "_" - || (inherited && slot.is_positional()) + || kept.is_some_and(|kept| kept.get(index).copied().unwrap_or(false)) || std::mem::replace(&mut keep_first, false) { continue; @@ -831,6 +862,32 @@ mod tests { ); } + /// a name the base's lowering numbered a `_` with is positional-only, so no call passes + /// it, and the override numbers its own `_` there, past the `_2` its body reads + #[test] + fn a_numbered_base_name_is_numbered_again() { + let slots = [(PositionalOrKeyword, "_"), (PositionalOrKeyword, "_")]; + let base: Vec<_> = base(&[("_", true), ("_2", true)]) + .into_iter() + .map(|parameter| { + parameter.map(|parameter| BaseParameter { + underscore: true, + ..parameter + }) + }) + .collect(); + let lowering = + lower_repeated_underscores(&slots, false, Some(&base), |name| name == "_2", true) + .unwrap() + .unwrap(); + assert!(lowering.is_inherited()); + assert_eq!(lowering.positional_only(), 2); + assert_eq!( + lowering.names(&slots, WrittenNames::new("return _2")), + [Name::from("_"), Name::from("_3")] + ); + } + #[test] fn an_inherited_name_the_body_reads_is_refused() { let slots = [(PositionalOrKeyword, "_"), (PositionalOrKeyword, "_")]; diff --git a/crates/ty_python_semantic/src/types/signatures.rs b/crates/ty_python_semantic/src/types/signatures.rs index 540e6400d2..d00bc6d94d 100644 --- a/crates/ty_python_semantic/src/types/signatures.rs +++ b/crates/ty_python_semantic/src/types/signatures.rs @@ -829,6 +829,118 @@ impl<'db> NarrowingGuard<'db> { } } +/// basedpython: what a call narrows through a `TypeIs` / `TypeGuard` return, described by how +/// the caller reaches it rather than by name, so that two signatures can be compared: a +/// `def f(first, second) -> first is int` narrows a different argument than a +/// `(first, second) -> (second is int)` does, whatever its return type says +#[derive(Debug, PartialEq, Eq)] +enum NarrowedArgument<'a> { + /// the argument a call passes for a parameter + Parameter { + /// where a positional argument lands on it, if one can + position: Option, + /// the keyword that reaches it, if one can + keyword: Option<&'a Name>, + /// whether it is the `**kwargs` that catches an unmatched keyword + keyword_variadic: bool, + members: &'a [Name], + }, + /// the receiver a bound method was called on, which is not among its parameters + Receiver { members: &'a [Name] }, + /// a place in the calling scope, which is not an argument at all + Place { name: &'a Name, members: &'a [Name] }, + /// a signature with no parameter to narrow + Nothing, +} + +impl<'a> NarrowedArgument<'a> { + fn of_parameter(index: usize, parameter: &'a Parameter<'_>, members: &'a [Name]) -> Self { + Self::Parameter { + position: (parameter.is_positional() || parameter.is_variadic()).then_some(index), + keyword: parameter.keyword_name(), + keyword_variadic: parameter.is_keyword_variadic(), + members, + } + } + + /// whether a call made through a signature narrowing `self` narrows what `source`, the + /// signature actually called, tests: every way a caller can reach the argument `self` names + /// has to reach the one `source` names + fn is_narrowed_by(&self, source: &NarrowedArgument<'a>) -> bool { + match (self, source) { + (Self::Nothing, _) => true, + ( + Self::Parameter { + position, + keyword, + keyword_variadic, + members, + }, + Self::Parameter { + position: source_position, + keyword: source_keyword, + keyword_variadic: source_keyword_variadic, + members: source_members, + }, + ) => { + position.is_none_or(|position| *source_position == Some(position)) + && keyword.is_none_or(|keyword| *source_keyword == Some(keyword)) + && (!keyword_variadic || *source_keyword_variadic) + && members == source_members + } + ( + Self::Receiver { members }, + Self::Receiver { + members: source_members, + }, + ) => members == source_members, + ( + Self::Place { name, members }, + Self::Place { + name: source_name, + members: source_members, + }, + ) => name == source_name && members == source_members, + _ => false, + } + } +} + +impl fmt::Display for NarrowedArgument<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let path = |root: &str, members: &[Name]| { + std::iter::once(root) + .chain(members.iter().map(Name::as_str)) + .join(".") + }; + match self { + Self::Parameter { + keyword: Some(keyword), + members, + .. + } => write!(f, "`{}`", path(keyword, members)), + Self::Parameter { + position: Some(position), + members, + .. + } => { + write!(f, "the argument in position {}", position + 1)?; + if !members.is_empty() { + write!(f, " at `{}`", path("", members))?; + } + Ok(()) + } + Self::Parameter { members, .. } => write!(f, "`{}`", path("**kwargs", members)), + Self::Receiver { members: [] } => f.write_str("the receiver"), + Self::Receiver { members } => { + write!(f, "the receiver at `{}`", path("", members)) + } + Self::Place { name, members } => write!(f, "the place `{}`", path(name, members)), + Self::Nothing => f.write_str("nothing"), + } + } +} + /// Whether one callable signature's parameters are compatible with another's. pub(crate) enum ParameterConsistency<'db> { /// The parameters are compatible. @@ -1086,6 +1198,70 @@ impl<'db> Signature<'db> { } } + /// basedpython: names the parameter a `TypeIs` / `TypeGuard` return narrows when the + /// annotation names none: PEP 742's first positional parameter, after `self` / `cls` when + /// `skips_receiver` + /// + /// it is recorded as the guard `-> x is T` would have written, so a bound call, an unbound + /// call and a comparison with another signature all resolve it by name — once the receiver + /// is bound away, its position no longer says which parameter it was + pub(crate) fn name_implicitly_narrowed_parameter(&mut self, skips_receiver: bool) { + if !self.narrowing_guards.is_empty() + || !matches!(self.return_ty, Type::TypeIs(_) | Type::TypeGuard(_)) + { + return; + } + let index = usize::from(skips_receiver); + let Some(name) = self + .parameters + .positional() + .nth(index) + .and_then(Parameter::name) + else { + return; + }; + self.narrowing_guards = Box::new([NarrowingGuard { + name: name.clone(), + members: Box::default(), + root_is_first_parameter: index == 0, + kind: NarrowingGuardKind::Predicate, + }]); + } + + /// basedpython: what a call narrows through this signature's `TypeIs` / `TypeGuard` + /// return, or `None` for any other return. a guard names it; without one, a call narrows + /// whatever it passes first, as the call site resolves it + fn narrowed_argument(&self) -> Option> { + if !matches!(self.return_ty, Type::TypeIs(_) | Type::TypeGuard(_)) { + return None; + } + let Some(guard) = self.narrowing_guards.first() else { + return Some(match self.parameters.iter().next() { + Some(parameter) => NarrowedArgument::of_parameter(0, parameter, &[]), + None => NarrowedArgument::Nothing, + }); + }; + Some( + match self + .parameters + .iter() + .enumerate() + .find(|(_, parameter)| parameter.name() == Some(&guard.name)) + { + Some((index, parameter)) => { + NarrowedArgument::of_parameter(index, parameter, &guard.members) + } + None if guard.root_is_first_parameter => NarrowedArgument::Receiver { + members: &guard.members, + }, + None => NarrowedArgument::Place { + name: &guard.name, + members: &guard.members, + }, + }, + ) + } + /// Returns the binding referenced by a direct `P.args` or `P.kwargs` variadic parameter. /// /// Returns `None` if this signature has no `ParamSpec` component parameters, or if none of @@ -3526,7 +3702,7 @@ impl<'c, 'db> TypeRelationChecker<'_, 'c, 'db> { } else { self.check_type_pair(db, source.return_ty, target.return_ty) }; - let return_type_checks = !result + let mut return_type_checks = !result .intersect(db, self.constraints, return_type_constraints) .is_never_satisfied(db, env); if let Some(context) = self.report_context() @@ -3538,6 +3714,23 @@ impl<'c, 'db> TypeRelationChecker<'_, 'c, 'db> { }); } + // a `TypeIs` / `TypeGuard` is about one argument, so a caller narrowing the one `target` + // names relies on `source` testing that same one + if return_type_checks + && let Some(target_narrows) = target.narrowed_argument() + && let Some(source_narrows) = source.narrowed_argument() + && !target_narrows.is_narrowed_by(&source_narrows) + { + if let Some(context) = self.report_context() { + context.push(ErrorContext::NarrowsAnotherArgument { + source: source_narrows.to_string(), + target: target_narrows.to_string(), + }); + } + result.intersect(db, self.constraints, self.never()); + return_type_checks = false; + } + // basedpython: a variadic parameter that takes whole repetitions of several types, // `*args: *((int, str) * int)`, cannot be written out as a list of parameters, so the two // signatures are compared through the tuples of positional arguments they accept. diff --git a/crates/ty_python_semantic/src/types/visibility.rs b/crates/ty_python_semantic/src/types/visibility.rs index dfe5dc858d..ac5e5ee2c7 100644 --- a/crates/ty_python_semantic/src/types/visibility.rs +++ b/crates/ty_python_semantic/src/types/visibility.rs @@ -34,10 +34,11 @@ use ruff_text_size::Ranged; use rustc_hash::FxHashSet; use ruff_python_ast::statement_visitor::{StatementVisitor, walk_stmt}; -use ty_python_core::ProgramFile; use ty_python_core::ast_ids::HasScopedUseId; use ty_python_core::definition::DefinitionState; +use ty_python_core::place::{PlaceExpr, PlaceExprRef}; use ty_python_core::scope::{ScopeId, ScopeKind}; +use ty_python_core::{EnclosingSnapshotResult, ProgramFile}; use crate::Db; @@ -254,9 +255,12 @@ fn class_scope_binding<'db>( /// after `private x = 1`, `alias = helper`, `@size.setter` over a protected /// property. `None` for any other name /// -/// the class body spells the member as it is declared (`__x`, `_x`): python -/// mangles a class body's names lexically, so `__x` there reaches the same -/// attribute the declaration made +/// a binding spells the member as it is declared (`__x`, `_x`): python mangles a +/// class body's names lexically, so `__x` there makes the attribute the +/// declaration made. a read is spelled as it is from anywhere (`_A__x`): an +/// annotation python keeps as a string (`x: __W` under `from __future__ import +/// annotations`) is evaluated later, against the class's namespace, and the +/// string was never mangled pub(crate) fn class_body_member_name<'db>( db: &'db dyn Db, file: ProgramFile<'db>, @@ -264,16 +268,74 @@ pub(crate) fn class_body_member_name<'db>( ) -> Option { let index = crate::semantic_index(db, file); let file_scope = index.try_expression_scope_id(&ast::ExprRef::from(reference))?; - let class_node = index.scope(file_scope).node().as_class()?; - if reference.ctx.is_load() - && class_scope_binding(db, file_scope.to_scope_id(db, file), reference) - == ClassBinding::Unbound - { - return None; - } + let class_scope = match index.scope(file_scope).kind() { + ScopeKind::Class => { + if reference.ctx.is_load() + && class_scope_binding(db, file_scope.to_scope_id(db, file), reference) + == ClassBinding::Unbound + { + return None; + } + file_scope + } + // an annotation scope reads the names of the class body it is in: the value of a + // type alias declared there, `type V = list[W]`, and the signature of a generic + // method, `def m[T](self) -> W` + ScopeKind::TypeParams | ScopeKind::TypeAlias if reference.ctx.is_load() => { + let mut scope = file_scope; + let mut lazy = false; + let class_scope = loop { + lazy |= index.scope(scope).kind() == ScopeKind::TypeAlias; + // a type parameter of that name is the scope's own + if index + .place_table(scope) + .symbol_by_name(&reference.id) + .is_some_and(ty_python_core::symbol::Symbol::is_bound) + { + return None; + } + let parent = index.scope(scope).parent()?; + match index.scope(parent).kind() { + ScopeKind::TypeParams | ScopeKind::TypeAlias => scope = parent, + ScopeKind::Class => break parent, + _ => return None, + } + }; + // what the class has bound of the name by the time the annotation scope reads it: + // as of the definition for an eager scope, and once the body has run for a lazy one + let defined = |binding: ty_python_core::BindingWithConstraints<'_, '_>| { + matches!(binding.binding, DefinitionState::Defined(_)) + }; + let bound = if lazy { + let symbol = index.place_table(class_scope).symbol_id(&reference.id)?; + index + .use_def_map(class_scope) + .end_of_scope_symbol_bindings(symbol) + .any(defined) + } else { + let place = PlaceExpr::try_from_expr(reference)?; + match index.enclosing_snapshot(class_scope, PlaceExprRef::from(&place), file_scope) + { + EnclosingSnapshotResult::FoundBindings(mut bindings) => bindings.any(defined), + EnclosingSnapshotResult::FoundConstraint(_) + | EnclosingSnapshotResult::NotFound + | EnclosingSnapshotResult::NoLongerInEagerContext => false, + } + }; + if !bound { + return None; + } + class_scope + } + _ => return None, + }; + let class_node = index.scope(class_scope).node().as_class()?; let definition = index.expect_single_definition(class_node); let class = super::infer::original_class_type(db, definition)?.as_static()?; let visibility = *class.member_visibilities(db).get(reference.id.as_str())?; + if reference.ctx.is_load() { + return emitted_member_name(db, visibility, class, reference.id.as_str()); + } ruff_python_stdlib::basedpython::visibility_rename( reference.id.as_str(), visibility.name_prefix(), @@ -688,7 +750,7 @@ fn declared_function<'db>( /// /// The rule is python's: leading underscores are stripped from the class name, /// and a class named only with underscores mangles nothing. -fn mangled_private_name(class: &str, member: &str) -> String { +pub(crate) fn mangled_private_name(class: &str, member: &str) -> String { let class = class.trim_start_matches('_'); if class.is_empty() { return format!("__{member}"); diff --git a/crates/ty_server/src/server/api/requests/entry_point.rs b/crates/ty_server/src/server/api/requests/entry_point.rs index 08db32b9ab..1ebc3ed20a 100644 --- a/crates/ty_server/src/server/api/requests/entry_point.rs +++ b/crates/ty_server/src/server/api/requests/entry_point.rs @@ -184,7 +184,9 @@ impl BackgroundRequestHandler for EntryPointHandler { .as_deref() .map(|default| text(default.range())), cli: param.spelling.as_ref().map(|spelling| CliParameter { - converter: spelling.converter.map(str::to_owned), + converter: spelling + .converter + .map(|converter| converter.name().to_owned()), choices: spelling.choices.as_ref().map(|choices| { choices.iter().map(|choice| choice.value.clone()).collect() }), @@ -193,7 +195,9 @@ impl BackgroundRequestHandler for EntryPointHandler { }), }) .collect(), - extra_arguments: entry.extra_arguments.map(str::to_owned), + extra_arguments: entry + .extra_arguments + .map(|converter| converter.name().to_owned()), docstring: entry.docstring().map(str::to_owned), }) }); diff --git a/docs/basedpython/features/conversions.md b/docs/basedpython/features/conversions.md index e40c914a37..049f165aa2 100644 --- a/docs/basedpython/features/conversions.md +++ b/docs/basedpython/features/conversions.md @@ -374,13 +374,16 @@ a conversion in the same module as its target spells it by name. one that reaches across modules imports it under a mangled alias, always: ```python -from temperatures import Fahrenheit as _by_conv__Fahrenheit +from temperatures import Fahrenheit as _by_conv__temperatures__Fahrenheit ``` the alias is not decoration. importing the bare name would rebind whatever this file already means by it — and a same-named class of this file's own would then shadow the import, sending the call to the wrong object at runtime. a name the -user never writes cannot do either. +user never writes cannot do either. the module is part of the alias, so two +classes of one name from two modules are two bindings, and a file that happens +to spell a name beginning with `_by_conv__` has its aliases start `_by_conv2__` +instead. for the same reason, a conversion whose target is shadowed by a binding between it and the module is rejected rather than emitted: diff --git a/docs/basedpython/features/extensions.md b/docs/basedpython/features/extensions.md index 72b7251199..60fddbfad8 100644 --- a/docs/basedpython/features/extensions.md +++ b/docs/basedpython/features/extensions.md @@ -363,7 +363,9 @@ double-underscore backing name would break an extension call written in one when a module declares more than one extension of the same target, later ones mangle with an ordinal (`_by_ext2__list__…`) so their members don't -collide — conditional extensions of the same method name coexist this way +collide — conditional extensions of the same method name coexist this way. +a module that already has a `_by_ext__list__second` of its own keeps it, and +the member takes the next ordinal instead call sites are rewritten by the type checker. ty already knows the receiver's type and which extensions are in scope, so `xs.second()` resolves to the @@ -414,6 +416,11 @@ _by_ext__str__dedented(greeting) so the surface stays `import textwrap`, and only the functions actually used are imported — the implicit-import convenience costs nothing at runtime +a module that already has the name — its own extension of another class called +`str` with a `dedented` member, or a variable spelled that way — keeps it, and +the import is bound under the next name it does not have +(`from textwrap import _by_ext__str__dedented as _by_ext__str__dedented2`) + ### conformance, lowered a conformance is the one part that needs a runtime: `str` cannot be diff --git a/docs/basedpython/features/forward-references.md b/docs/basedpython/features/forward-references.md index 1787f74ea0..42f7995221 100644 --- a/docs/basedpython/features/forward-references.md +++ b/docs/basedpython/features/forward-references.md @@ -78,8 +78,9 @@ evaluated eagerly. it is skipped when: - the target is python 3.14 or newer — annotations are deferred natively (PEP 649), so the bare name resolves lazily - the file already defers every annotation through - `from __future__ import annotations`, whether you wrote it yourself or - opted into the blanket injection + `from __future__ import annotations`, whether you wrote it yourself, + opted into the blanket injection, or target a version below 3.10, which + always gets it a subscript base evaluates eagerly on every target, so its self-reference is quoted either way diff --git a/docs/basedpython/features/identity-swap.md b/docs/basedpython/features/identity-swap.md index f04b625307..d2916267a7 100644 --- a/docs/basedpython/features/identity-swap.md +++ b/docs/basedpython/features/identity-swap.md @@ -54,12 +54,19 @@ from typing import Literal def f(v: object): if v is int | str: ... # a union + if v is int?: ... # an optional: `int` or `None` if v is list: ... # a class, arguments and all if v is Literal[1, 2]: ... # a set of values if v is type[int]: ... # a class object if v is f"item-{int}": ... # a string pattern ``` +the `?` of an optional belongs to the type, so `v is int?` tests for `int | None`, +and so does the error type of a result, so `v is int ? ValueError` tests for +`int ? ValueError`. anywhere else `?` binds looser than a comparison, so +`v == t?` marks the whole comparison — write `v == (t?)` to compare with an +optional type + and anything an annotation rejects, a test rejects with the same message — `v is os` names a module, which is not a type in either place diff --git a/docs/basedpython/features/modifiers.md b/docs/basedpython/features/modifiers.md index 63f63b7ac7..23319c7198 100644 --- a/docs/basedpython/features/modifiers.md +++ b/docs/basedpython/features/modifiers.md @@ -47,7 +47,11 @@ ORIGIN: Final = Point(0, 0) | `sealed class Foo` | `class Foo` + `Foo.__sealed_members__ = (...)` | `dataclass(slots=True)` is new in python 3.10. below it, a `data class` is -decorated with a runtime helper that makes it slotted the way that option would +decorated with a runtime helper that makes it slotted the way that option would. +the option makes a new class, and before 3.13 it leaves a zero-argument `super()` +in the class's methods pointing at the class it replaced, where it raises a +`TypeError`. so below 3.13 a `data class` whose body names `super` or `__class__` +is given the helper too `abstract` is a marker for the type checker; it has no runtime decorator. `open` is the inverse of `final` — a marker that the class is intended to be @@ -293,6 +297,25 @@ class Counter: return cls.made + cls.LIMIT ``` +a `type` alias declared in a class body takes `private` too, and is a private +member like any other: the class's own body and its methods' signatures may name +it, and nothing else may, a subclass included + +```by +class Shape: + private type Size = int | float + + width: Size = 1 + + def area(self) -> Size: + return self.width ** 2 + +def measure(shape: Shape) -> Shape.Size: ... # error: `Size` is private to `Shape` +``` + +it is emitted as `type __Size`, and every reference to it as the mangled +`_Shape__Size`, so an annotation python keeps as a string still names it + ### visibility and inheritance a visibility keyword decides the name the member is emitted under, so a member @@ -339,6 +362,23 @@ class Derived(Base): a `protected` member keeps one name across the hierarchy, so it overrides like any other member and is checked like one +the emitted name can already belong to another member — the class's own, a base +class's or a subclass's. at runtime one of the two would replace the other, while +both still looked distinct, so that is `invalid-visibility`: + +```by +class A: + def _m(self) -> int: + return 1 + + protected def m(self) -> int: # error: `m` cannot be `protected` here: `_m` is taken + return 2 +``` + +the same holds for `private` beside a name python mangles to the same attribute +(`def __m` in `A` is stored as `_A__m`), and for two `private` members of +classes that share a name + ### names a member is looked up by a visibility keyword renames every place a member is named, not only attribute diff --git a/docs/basedpython/features/repeated-underscore.md b/docs/basedpython/features/repeated-underscore.md index e83b002ec7..c5603a7388 100644 --- a/docs/basedpython/features/repeated-underscore.md +++ b/docs/basedpython/features/repeated-underscore.md @@ -122,15 +122,17 @@ a base parameter that is positional-only stays so, and a default the base gives a parameter comes with it, as it does for any override (see [inherited default values](inherited-defaults.md)) -a base that repeats `_` itself gives the override its own numbered names, the -first still `_`, and a base that is itself such an override gives the names it -took +a base that repeats `_` itself gives the override its first `_`. the names it +numbers the others with are positional-only, so no call passes one, and the +override numbers its own `_`s there, clear of every name its module spells — +the names its body reads among them. a base that is itself such an override +gives the names it took where the base has no parameter at some `_`'s position, none of the `_`s take a name from it and they are numbered as above. that override is reported as an `invalid-method-override` anyway -a name taken from the base would shadow the same name read from an enclosing +a name the base's author wrote would shadow the same name read from an enclosing scope, so an override whose body reads one is refused as `invalid-repeated-underscore` diff --git a/docs/basedpython/features/type-is.md b/docs/basedpython/features/type-is.md index debbe16344..bf8605c411 100644 --- a/docs/basedpython/features/type-is.md +++ b/docs/basedpython/features/type-is.md @@ -24,6 +24,11 @@ argument has type `T` when it returns `True`, and the checker narrows accordingly at call sites. the name is lost in lowering (`TypeIs` doesn't carry it) but is preserved in the source for readers +a predicate cannot be negated. `-> x is not str` would narrow to everything but +`str` where the call returns `True`, which `TypeIs` has no way to say, so it is +an error. write `-> x is str` and negate the call instead (`if not is_str(x):`), +or [assert](#asserting-a-type) with `asserts x is not str` + ## naming the parameter `TypeIs` always narrows a function's first parameter. the name says which @@ -42,6 +47,38 @@ def f(a: object, b: object): the name is matched against the argument the call passes for that parameter, so a keyword argument narrows the same place +## callable types + +a [callable type](callable.md) returns a predicate the same way, naming +one of its parameters. its return binds tighter than a comparison, as it does +than `|`, so the predicate is parenthesized: + +```by +def pick(check: (first: object, second: object) -> (second is int), a: object, b: object): + if check(a, b): + b # int +``` + +a `def` whose predicate names that parameter is such a callable. a name that +is not one of the callable's parameters is a [place](#narrowing-a-place), as it +is for a `def` + +a caller narrows the argument the callable type names, so a function that tests +a different one is not such a callable, even when the narrowed type is the same: + +```by +def is_int(first: object, second: object) -> first is int: + return isinstance(first, int) + +pick(is_int, 1, "b") # error: narrows `first`, where `second` is expected +``` + +a parameter is matched by position, and by name too where a caller can pass it +by keyword. a `TypeIs` that names no parameter narrows the first one — after +`self` for a method — and a `Callable` narrows the first argument it is passed. +an override is held to the same rule: it narrows what the method it overrides +narrows + ## narrowing a member a guard can name a member of what it narrows, which is how a method vouches diff --git a/docs/basedpython/features/wrapped-results.md b/docs/basedpython/features/wrapped-results.md index 229a3af2c4..82762c03be 100644 --- a/docs/basedpython/features/wrapped-results.md +++ b/docs/basedpython/features/wrapped-results.md @@ -40,8 +40,8 @@ example `T ? E` declares a result with value type `T` and error type `E` — a value that is either an outcome or a failure, carried in the type rather than raised. **not yet implemented** — the `?`-infix result -form does not parse today (only the `T?` optional form does); the example below -is the intended design: +form parses, but is rejected wherever a type is expected; the example below is +the intended design: ```by def g() -> int ? TypeError: @@ -51,6 +51,35 @@ def g() -> int ? TypeError: both forms compose: `T?? E`, `T ? E?`, etc +### `?` outside a type expression + +where a program evaluates `T?` — a type alias, the classes given to +`isinstance` — it is the union `T | None`. over a value that is not a type the +marker is an error, as the `| None` would be where it runs: + +```by +Alias = int? # int | None + +def f(x: object, y: int): + ok = isinstance(x, int?) + flag = x == y? # error: the marker is over the whole comparison +``` + +`?` binds looser than a comparison, so `x == y?` is `(x == y)?`, the optional +of a `bool`. write `x == (y?)` to compare with an optional type. a type test is +the exception: its type takes the marker, so `x is int?` tests for `int | None` +(see [identity swap](identity-swap.md)) + +a wrapped optional — `int??`, or `T?` over a type variable (below) — has no +union to evaluate to. read as a value it is the wrapper class, one class for +every wrapped optional, so an alias of one still means it in an annotation: + +```by +Nested = int?? + +def f(x: Nested): ... # x: int?? +``` + ### `?` over a type variable is the wrapped form a plain union `T | None` would flatten when `T` binds to an optional — calling diff --git a/ty.schema.json b/ty.schema.json index 9c4066f38d..e3e52a2d8a 100644 --- a/ty.schema.json +++ b/ty.schema.json @@ -1823,7 +1823,7 @@ }, "invalid-visibility": { "title": "detects a visibility keyword where it cannot do what it says", - "description": "## What it does\nChecks for a visibility keyword written where it cannot do what it says.\n\n## Why is this bad?\nA visibility keyword renames the member or symbol it is written on, since\nthat is the only enforcement python offers. Some names cannot be renamed\nwithout changing what they mean: a dataclass field's name is its\nconstructor's keyword, an enum member's name is how the enum is looked up,\na `private` abstract method can never be overridden, and a dotted import\nbinds a package no rename can keep. The declaration would read as\nrestricted while doing something else.\n\n## Example\n\n```by\nfrom dataclasses import dataclass\n\n@dataclass\nclass Point:\n private x: int # error: a dataclass field's name is its constructor's keyword\n```", + "description": "## What it does\nChecks for a visibility keyword written where it cannot do what it says.\n\n## Why is this bad?\nA visibility keyword renames the member or symbol it is written on, since\nthat is the only enforcement python offers. Some names cannot be renamed\nwithout changing what they mean: a dataclass field's name is its\nconstructor's keyword, an enum member's name is how the enum is looked up,\na `private` abstract method can never be overridden, and a dotted import\nbinds a package no rename can keep. The declaration would read as\nrestricted while doing something else.\n\nThe rename can also land on a name the class, a base class or a subclass\nalready stores another member under: `protected def m` is stored as `_m`,\nso beside a `def _m` one of the two replaces the other at runtime while\nthe checker still sees both.\n\n## Example\n\n```by\nfrom dataclasses import dataclass\n\n@dataclass\nclass Point:\n private x: int # error: a dataclass field's name is its constructor's keyword\n```", "default": "error", "oneOf": [ { From 888e4d5fdf8f83eca8d4cb804dfd341d3a3aa497 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Wed, 23 Sep 2026 17:11:38 +1000 Subject: [PATCH 15/16] native: keep a bool or int subclass as itself, ask in-place methods, and read repeated `_` names from ty an int register carries a bool or int subclass as the object, asking its own methods for truthiness, unary operators and mixed float arithmetic; unary `+` asks any operand's __pos__; a literal default a place cannot hold as itself goes to the twin; an augmented assignment's slow path asks the in-place method; compiled repeated `_` parameters are named and made positional-only from ty's answer; a float subclass narrowed to its value is documented --- crates/by_build/tests/differential.rs | 597 +++++++++++++++++- crates/by_build/tests/end_to_end.rs | 9 +- crates/by_codegen_c/src/lib.rs | 321 ++++++++-- crates/by_ir/src/builder.rs | 3 +- crates/by_ir/src/ops.rs | 32 + crates/by_ir/src/print.rs | 18 +- crates/by_ir/src/verify.rs | 103 ++- crates/by_irbuild/src/lib.rs | 187 +++++- crates/by_irbuild/src/tests.rs | 49 +- crates/by_opt/src/borrow.rs | 9 +- crates/by_opt/src/coalesce.rs | 5 +- crates/by_opt/src/copy_propagation.rs | 10 +- crates/by_opt/src/dead_registers.rs | 5 +- crates/by_opt/src/field_proofs.rs | 3 +- crates/by_opt/src/fold.rs | 21 +- crates/by_opt/src/guard_loops.rs | 4 +- crates/by_opt/src/infallible.rs | 12 +- crates/by_opt/src/lib.rs | 3 +- crates/by_opt/src/push_lengths.rs | 3 +- crates/by_opt/src/refcount.rs | 4 +- crates/by_opt/src/release_temporaries.rs | 3 +- crates/by_opt/src/runs_python.rs | 25 +- crates/by_opt/src/store_moves.rs | 6 +- crates/by_opt/src/unbox_counters.rs | 13 +- crates/by_opt/src/unswitch.rs | 9 +- crates/by_rt/include/by.h | 221 ++++++- .../development/compilation/plan.md | 2 +- .../compilation/planned-features.md | 13 +- .../development/compilation/runtime.md | 99 ++- 29 files changed, 1597 insertions(+), 192 deletions(-) diff --git a/crates/by_build/tests/differential.rs b/crates/by_build/tests/differential.rs index 6c3a5672b2..ad6b1f613e 100644 --- a/crates/by_build/tests/differential.rs +++ b/crates/by_build/tests/differential.rs @@ -1309,6 +1309,60 @@ def _annotation_report(f): out['annotate'] = attempt(lambda: f.__annotate__ and f.__annotate__.__qualname__) return out +# an `int` subclass with an in-place method for every operator, each answering 1000 more +# than the plain one, so an augmented assignment that asked the plain method shows +class _InPlace(int): + def __repr__(self): + return f'_InPlace({int(self)})' + +for _name in ('add', 'sub', 'mul', 'floordiv', 'mod', 'pow', 'lshift', 'rshift', 'and', 'or', 'xor'): + def _plain(self, other, _name=_name): + return _InPlace(getattr(int, f'__{_name}__')(int(self), int(other))) + def _in_place(self, other, _name=_name): + return _InPlace(getattr(int, f'__{_name}__')(int(self), int(other)) + 1000) + setattr(_InPlace, f'__{_name}__', _plain) + setattr(_InPlace, f'__i{_name}__', _in_place) + +# an `int` subclass whose every method answers something `int`'s own would not, so a +# compiled `int` that let go of the object, or asked `int` instead of it, shows +class _Int(int): + def __repr__(self): + return f'_Int({int(self)})' + def __str__(self): + return f's{int(self)}' + def __format__(self, spec): + return f'f{int(self)}' + def __add__(self, other): + if isinstance(other, float): + return other - 0.5 + return _Int(int(self) + int(other) + 100) + def __radd__(self, other): + return _Int(int(other) + int(self) + 1000) + def __sub__(self, other): + return _Int(int(self) - int(other) - 100) + def __rsub__(self, other): + return _Int(int(other) - int(self) - 1000) + def __eq__(self, other): + return int(self) % 10 == int(other) % 10 + def __ne__(self, other): + return int(self) % 10 != int(other) % 10 + def __lt__(self, other): + return int(self) > int(other) + def __gt__(self, other): + return int(self) < int(other) + def __hash__(self): + return 7 + def __bool__(self): + return False + def __neg__(self): + return _Int(99) + def __pos__(self): + return _Int(77) + def __invert__(self): + return _Int(55) + def __index__(self): + return 0 + # whether what `make` built is still alive once the only references left are the ones # `keep` took from it, handed back beside what `keep` took so a caller can go on to use it def _outlives(make, keep): @@ -3406,6 +3460,313 @@ def both(a: bool, b: bool) -> bool: ); } +#[test] +fn an_int_keeps_the_bool_or_the_subclass_it_was_handed() { + // `bool` is an `int` subclass, and so is a class of the program's own. python hands + // back the very object it was given, so an `int` register has to keep it rather than + // narrow it to its value — through a parameter, a list element, a field, a loop, a + // dict key and a default + agree_python( + "intkeeps", + "\ +class Box: + def __init__(self, v: int) -> None: + self.v = v + +def ident(x: int) -> int: + return x + +def first(xs: list[int]) -> int: + return xs[0] + +def field(b: Box) -> int: + return b.v + +def boxed(x: int) -> int: + return Box(x).v + +def each(xs: list[int]) -> list[int]: + out: list[int] = [] + for x in xs: + out.append(x) + return out + +def keyed(xs: list[int]) -> list[int]: + d: dict[int, int] = {} + for x in xs: + d[x] = 1 + return list(d) + +def fallback(x: int = True) -> int: + return x +", + &[ + "[(m.ident(v), type(m.ident(v)).__name__) for v in (True, False, _Int(5), _Int(2**70), 5)]", + "[m.first([v]) for v in (True, False, _Int(5), _Int(2**70))]", + "[m.field(m.Box(v)) for v in (True, _Int(5))]", + "[m.boxed(v) for v in (True, _Int(5))]", + "m.each([True, _Int(5), 3, False])", + "m.keyed([True, _Int(5), 1, 17])", + "(m.fallback(), m.fallback(_Int(3)))", + ], + ); +} + +#[test] +fn an_int_subclass_answers_every_operation_with_its_own_method() { + // an operation on an `int` is the method its type holds, and a subclass may hold one + // of its own for any of them. `_Int`'s answer differs from `int`'s everywhere, and + // `True` is there beside it because `bool` keeps `int`'s methods for all of these + agree_python( + "intmethods", + "\ +def added(xs: list[int]) -> list[int]: + return [xs[0] + 1, 1 + xs[0], xs[0] - 1, 1 - xs[0], xs[0] + xs[0]] + +def compared(xs: list[int]) -> list[bool]: + return [xs[0] == 5, xs[0] != 5, xs[0] < 3, xs[0] > 3, xs[0] == xs[1]] + +def hashed(xs: list[int]) -> int: + return hash(xs[0]) + +def text(xs: list[int]) -> list[str]: + return [repr(xs[0]), str(xs[0]), f\"{xs[0]}\", f\"{xs[0]:>3}\"] + +def truth(xs: list[int]) -> list[bool]: + seen: list[bool] = [] + if xs[0]: + seen.append(True) + else: + seen.append(False) + seen.append(not xs[0]) + return seen + +def unary(xs: list[int]) -> list[int]: + return [-xs[0], +xs[0], ~xs[0]] + +def mixed(xs: list[int]) -> list[float]: + return [xs[0] + 1.5, 1.5 + xs[0]] + +def total(xs: list[int]) -> int: + t = 0 + for x in xs: + t = t + x + return t +", + &[ + "[m.added([v]) for v in (True, _Int(5), _Int(2**70))]", + "[m.compared([v, w]) for v in (True, _Int(5), _Int(2**70)) for w in (1, _Int(15))]", + "[m.hashed([v]) for v in (True, _Int(5), _Int(2**70))]", + "[m.text([v]) for v in (True, _Int(5))]", + "[m.truth([v]) for v in (True, False, _Int(5), _Int(0), _Int(2**70))]", + "[m.unary([v]) for v in (True, _Int(5), _Int(2**70))]", + "[m.mixed([v]) for v in (True, _Int(5))]", + "[m.total(vs) for vs in ([True, True], [_Int(5), 2], [2, _Int(5)])]", + ], + ); +} + +#[test] +fn an_augmented_assignment_on_an_int_asks_a_subclass_its_in_place_method() { + // python offers the left operand of `x += y` its `__iadd__` before its `__add__`, and + // the same for every other augmented operator. `int` and `bool` have no in-place + // methods, so only a subclass that defines one can tell — `_InPlace` answers 1000 more + // from each of them. the exact `int`s and `True` beside it take the plain operator + agree_python( + "intinplace", + "\ +def each(xs: list[int], y: int) -> list[int]: + out: list[int] = [] + a = xs[0] + a += y + out.append(a) + a = xs[0] + a -= y + out.append(a) + a = xs[0] + a *= y + out.append(a) + a = xs[0] + a //= y + out.append(a) + a = xs[0] + a %= y + out.append(a) + a = xs[0] + a **= 2 + out.append(a) + a = xs[0] + a <<= y + out.append(a) + a = xs[0] + a >>= y + out.append(a) + a = xs[0] + a &= y + out.append(a) + a = xs[0] + a |= y + out.append(a) + a = xs[0] + a ^= y + out.append(a) + return out + +def stepped(xs: list[int]) -> int: + total = xs[0] + for i in range(3): + total += i + return total + +class Tally: + def __init__(self, n: int) -> None: + self.n = n + + def bump(self) -> int: + self.n += 1 + return self.n +", + &[ + "m.each([_InPlace(7)], 2)", + "m.each([_InPlace(7 * 2**70)], 3)", + "[m.each([v], 2) for v in (7, -7, 7 * 2**70, True)]", + "m.each([7], _InPlace(2))", + "[m.stepped([v]) for v in (_InPlace(7), 7, True)]", + "[m.Tally(v).bump() for v in (_InPlace(7), 7, True)]", + ], + ); +} + +#[test] +fn unary_plus_asks_the_operand_its_own_pos_whatever_it_is() { + // `+x` is `type(x).__pos__(x)` for every `x`, not only an `int`. a `Counter` answers + // with a copy that drops each count that is not positive, a `bool` with the `int` it + // stands for, and a class of the program's own with whatever its method returns + agree_python( + "objectpos", + "\ +from collections import Counter + +class Signed: + def __pos__(self) -> str: + return \"positive\" + +def counted(c: Counter[str]) -> Counter[str]: + return +c + +def flag(b: bool) -> int: + return +b + +def signed(s: Signed) -> str: + return +s + +def floating(x: float) -> float: + return +x +", + &[ + "sorted(m.counted(__import__('collections').Counter(a=1, b=-2, c=0)).items())", + "[(m.flag(b), type(m.flag(b)).__name__) for b in (True, False)]", + "m.signed(m.Signed())", + "[repr(m.floating(x)) for x in (1.5, -0.0, float('inf'))]", + ], + ); +} + +#[test] +fn a_literal_default_is_the_object_python_holds_whatever_the_parameter_holds() { + // a default is the object the literal made, whatever the annotation says: `x: float + // = 1` leaves `x` the `int` 1, which a double cannot hold, and `x: int = None` leaves + // it `None`, which a tagged `int` cannot. a call that omits such a parameter is + // handed to the definition that holds the object, and a call that passes one is not + agree_python( + "defaultstands", + "\ +def scaled(x: float = 1) -> float: + return x + +def flagged(x: float = True) -> float: + return x + +def plain(x: float = 1.5) -> float: + return x + +def absent(x: int = None) -> int: + return x + +def text(x: str = None) -> str: + return x + +class Meter: + def reset(self, n: int = None) -> int: + return n +", + &[ + "[(v, type(v).__name__) for v in (m.scaled(), m.scaled(2.5), m.scaled(x=0.5))]", + "[(v, type(v).__name__) for v in (m.flagged(), m.flagged(2.5))]", + "[(v, type(v).__name__) for v in (m.plain(), m.plain(2.5))]", + "(m.absent(), m.absent(3))", + "(m.text(), m.text('a'))", + "(m.Meter().reset(), m.Meter().reset(4))", + ], + ); +} + +#[test] +fn a_counted_range_reads_an_int_subclass_bound_by_its_value() { + // `range` takes each bound through `operator.index`, which copies an `int` subclass + // to the plain `int` of its value and asks it nothing. so the loop counts in plain + // `int`s, and neither `_Int.__lt__` nor `_Int.__add__` is ever reached + agree_python( + "intrange", + "\ +def counted(n: int) -> list[int]: + out: list[int] = [] + for i in range(n): + out.append(i) + return out + +def counted_from(a: int, n: int) -> list[int]: + out: list[int] = [] + for i in range(a, n): + out.append(i) + return out + +def comprehended(a: int, n: int) -> list[int]: + return [i for i in range(a, n)] +", + &[ + "[m.counted(n) for n in (True, _Int(3))]", + "[m.counted_from(a, n) for a in (True, _Int(1)) for n in (4, _Int(4))]", + "[m.comprehended(a, n) for a in (True, _Int(1)) for n in (4, _Int(4))]", + "[type(i).__name__ for i in m.counted_from(_Int(1), _Int(3))]", + ], + ); +} + +#[test] +fn an_unboxed_list_is_indexed_by_a_bool_at_its_value() { + // an index held tagged is an object when it is a `bool` or an `int` subclass, whatever + // its value — so an object index is not, by being one, too big for the buffer + agree_python( + "intbufindex", + "\ +def pick(n: int, k: int) -> float: + out = [i * 1.5 for i in range(n)] + return out[k] + +def store(n: int, k: int) -> float: + out = [i * 1.5 for i in range(n)] + out[k] = 9.5 + return out[1] +", + &[ + "[m.pick(3, k) for k in (True, False, _Int(2), 1)]", + "[m.store(3, k) for k in (True, False, _Int(1))]", + "_capture(m.pick, 3, 2**70)", + ], + ); +} + #[test] fn gradual_values_agree_through_the_object_protocol() { // a gradual parameter now compiles to an `object` register rather than @@ -17097,9 +17458,11 @@ def calls_none() -> int: #[test] fn repeated_underscore_parameters_agree() { // basedpython lets a `def` repeat `_` for parameters it ignores, and python refuses a - // repeated name, so the interpreted definition keeps the first `_` and numbers the rest. - // the compiled one answers to the same names — the forwarder spells them, and a keyword - // call reaches them — where it used to spell `_` twice and fail the module's import + // repeated name, so the interpreted definition keeps the first `_`, numbers the rest and + // makes them positional-only: the numbered name is the lowering's, and no call may spell + // it. the compiled definition answers to the same names and the same `/` — a keyword + // naming a numbered `_` is python's positional-only `TypeError` in both, where the + // compiled one used to bind it let Some(compiled) = agree_in( "underscores", "\ @@ -17108,37 +17471,47 @@ from collections.abc import Iterator def pair(_: int, _: int) -> int: return 7 -def spaced(a: int, _: int, b: int, _: int) -> int: - return a * 10 + b +def labelled(_: int, _: int, label: str) -> str: + return label -def every_kind(_: int, *_: int, _: int, **_: int) -> int: +def every_kind(_: int, _: int, *_: int, **_: int) -> int: return 1 -def counted(_: int, n: int, _: int) -> Iterator[int]: +def counted(_: int, _: int, n: int) -> Iterator[int]: for i in range(n): yield i -def outer(n: int) -> int: +def written_slash(a: int, _: int, b: int, _: int, /) -> int: + return a * 10 + b + +def outer(n: int) -> object: def inner(_: int, _: int) -> int: return n - return inner(1, 2) + return inner -def through_lambda(n: int) -> int: +def through_lambda(n: int) -> object: ignore = lambda _, _: n - return ignore(1, 2) + return ignore class Holder: def method(self, _: int, _: int) -> int: return 5 + +def calls_every_kind() -> int: + return every_kind(1, 2, 3, x=4) ", &[ - "[str(__import__('inspect').signature(f)) for f in (m.pair, m.spaced, m.every_kind, m.counted, m.Holder().method)]", - "(m.pair(1, 2), m.pair(1, _2=2), m.pair(_=1, _2=2))", - "str(_capture_kw(m.pair, (1,), {'_': 2}))", - "(m.spaced(1, 2, 3, _2=4), m.every_kind(1, 2, 3, _2=4, x=5), m.every_kind(1, _2=4))", - "(list(m.counted(1, 3, 2)), list(m.counted(1, 3, _2=2)))", - "(m.outer(1), m.through_lambda(4))", - "(m.Holder().method(5, _2=6), str(_capture(m.Holder().method, 5, 6, 7)))", + "[str(__import__('inspect').signature(f)) for f in (m.pair, m.labelled, m.every_kind, m.counted, m.written_slash, m.Holder().method, m.outer(1), m.through_lambda(1))]", + "(m.pair(1, 2), m.labelled(1, 2, 'a'), m.labelled(1, 2, label='b'), m.every_kind(1, 2, 3, x=4))", + "(list(m.counted(1, 2, 3)), list(m.counted(1, 2, n=2)), m.written_slash(1, 2, 3, 4))", + "(m.outer(3)(1, 2), m.through_lambda(4)(1, 2), m.Holder().method(5, 6), m.calls_every_kind())", + // a keyword naming a numbered `_`, or the first `_`, reaches no parameter + "[str(_capture_kw(f, (1,), {'_2': 2})) for f in (m.pair, m.every_kind, m.outer(1), m.through_lambda(1), m.Holder().method)]", + "[str(_capture_kw(f, (), {'_': 1, '_2': 2})) for f in (m.pair, m.outer(1), m.Holder().method)]", + "str(_capture_kw(m.labelled, (1,), {'_2': 2, 'label': 'c'}))", + "str(_capture_kw(m.counted, (1,), {'_2': 2, 'n': 3}))", + "str(_capture_kw(m.written_slash, (1, 2, 3), {'_2': 4}))", + "str(_capture(m.Holder().method, 5, 6, 7))", ], false, by_irbuild::Language::BasedPython, @@ -17153,9 +17526,191 @@ class Holder: &python, &compiled, "import by_diff_underscores as m\n\ - print([f.__code__.co_filename for f in (m.pair, m.spaced, m.every_kind, m.counted)])\n", + assert m.__file__.endswith('.so'), m.__file__\n\ + print([f.__code__.co_filename for f in (m.pair, m.labelled, m.every_kind, m.counted, m.written_slash)])\n\ + print(type(m.Holder.__dict__['method']).__name__)\n", + ), + "['', '', '', '', '']\n\ + method_descriptor" + ); +} + +#[test] +fn an_override_repeating_underscore_takes_the_base_method_names() { + // an override that repeats `_` takes the names the method it overrides gives those + // positions, so a call written against the base — by keyword too — reaches the override. + // the compiled override is published under those names, where it used to number its + // `_`s and refuse the base's keywords + let Some(compiled) = agree_in( + "underscorebase", + "\ +class A: + def m(self, x: int, y: int) -> int: + return x * 100 + y + + def n(self, _: int, _: int) -> int: + return 1 + +class B(A): + def m(self, _: int, _: int) -> int: + return 7 + + def n(self, _: int, _: int) -> int: + return 2 + +def by_keyword(b: B) -> int: + return b.m(x=3, y=4) + +def through_the_base(a: A) -> int: + return a.m(5, y=6) +", + &[ + "[str(__import__('inspect').signature(f)) for f in (m.B().m, m.B().n)]", + "(m.B().m(3, 4), m.B().m(x=3, y=4), m.B().m(3, y=4), m.by_keyword(m.B()))", + "(m.through_the_base(m.A()), m.through_the_base(m.B()))", + // a base that repeats `_` itself hands on its own numbered names, positional-only + "(m.B().n(1, 2), str(_capture_kw(m.B().n, (1,), {'_2': 2})))", + "str(_capture_kw(m.B().m, (), {'_': 1, '_2': 2}))", + ], + false, + by_irbuild::Language::BasedPython, + ) else { + return; + }; + let Some((python, _)) = environment() else { + return; + }; + assert_eq!( + run( + &python, + &compiled, + "import by_diff_underscorebase as m\n\ + assert m.__file__.endswith('.so'), m.__file__\n\ + print([type(m.B.__dict__[name]).__name__ for name in ('m', 'n')])\n", + ), + "['method_descriptor', 'method_descriptor']" + ); +} + +#[test] +fn a_base_named_override_reading_underscore_declines() { + // an override that took its names from the base binds `_` to the first of them at the + // top of its body, and a read of `_` finds that. the compiled build declines any body + // that reads a repeated `_`, so this one runs from its interpreted definition and the + // two builds answer alike + let Some(compiled) = agree_in( + "underscorebaseread", + "\ +class A: + def m(self, x: int, y: int) -> int: + return x * 100 + y + +class C(A): + def m(self, _: int, _: int) -> int: + return _ + 1 +", + &[ + "str(__import__('inspect').signature(m.C().m))", + "(m.C().m(3, 4), m.C().m(x=3, y=4), m.C().m(3, y=4))", + ], + true, + by_irbuild::Language::BasedPython, + ) else { + return; + }; + let Some((python, _)) = environment() else { + return; + }; + assert_eq!( + run( + &python, + &compiled, + "import by_diff_underscorebaseread as m\n\ + assert m.__file__.endswith('.so'), m.__file__\n\ + print(type(m.C.__dict__['m']).__name__, m.C.m.__code__.co_filename)\n", + ), + "function " + ); +} + +#[test] +fn a_repeated_underscore_shape_the_lowering_refuses_is_not_compiled() { + // a named parameter ahead of a repeated `_` would be dragged positional-only by the `/`, + // and a keyword-only `_` has no name to be reached by, so the transpiler refuses both. + // the compiled build refuses them with it rather than publishing a signature the + // interpreted build does not have + let Some((_, toolchain)) = environment() else { + return; + }; + for (tag, source, refusal) in [ + ( + "underscoredragged", + "def dragged(a: int, _: int, b: int, _: int) -> int:\n return a\n", + "after `a` make it positional-only in `dragged`", + ), + ( + "underscorekwonly", + "def keyword_only(_: int, *, _: int) -> int:\n return 1\n", + "cannot be keyword-only in `keyword_only`", + ), + ] { + let dir = Scratch::new(format!("by_diff_{tag}_c")); + let options = Options { + fallback: Some(Config::default()), + ..Options::default() + }; + let error = build_source(source, tag, &toolchain, &dir, &options) + .expect_err("a refused shape is not compiled"); + assert!( + format!("{error:#}").contains(refusal), + "{tag}: expected the refusal, got {error:#}" + ); + } +} + +#[test] +fn repeated_underscore_in_a_callable_type_agrees() { + // a callable type and an inline protocol's method follow the rules a `def` does: the + // `_`s are numbered and positional-only in the protocol the interpreted build writes, + // and a compiled caller passes them by position + let Some(compiled) = agree_in( + "underscorecallable", + "\ +def apply(cb: (_: int, _: str) -> int) -> int: + return cb(1, 'a') + +def ask(p: protocol(def m(self, _: int, _: str) -> int)) -> int: + return p.m(2, 'b') + +class Impl: + def m(self, _: int, _: str) -> int: + return 3 + +def use() -> int: + return ask(Impl()) +", + &[ + "(m.apply(lambda a, b: a + len(b)), m.use())", + "str(__import__('inspect').signature(m.apply.__annotations__['cb'].__call__))", + "str(__import__('inspect').signature(m.ask.__annotations__['p'].m))", + ], + false, + by_irbuild::Language::BasedPython, + ) else { + return; + }; + let Some((python, _)) = environment() else { + return; + }; + assert_eq!( + run( + &python, + &compiled, + "import by_diff_underscorecallable as m\n\ + assert m.__file__.endswith('.so'), m.__file__\n\ + print([f.__code__.co_filename for f in (m.apply, m.ask, m.use)])\n", ), - "['', '', '', '']" + "['', '', '']" ); } @@ -17179,6 +17734,8 @@ def reads(_: int, _: int) -> list[int]: &[ "(m.defaulted(1, 2), m.reads(1, 2))", "[list(__import__('inspect').signature(f).parameters) for f in (m.defaulted, m.reads)]", + // the numbered name is positional-only there too + "[str(_capture_kw(f, (1,), {'_3': 2})) for f in (m.defaulted, m.reads)]", ], false, by_irbuild::Language::BasedPython, diff --git a/crates/by_build/tests/end_to_end.rs b/crates/by_build/tests/end_to_end.rs index f05db96656..43993639a5 100644 --- a/crates/by_build/tests/end_to_end.rs +++ b/crates/by_build/tests/end_to_end.rs @@ -20,7 +20,7 @@ use by_build::{Options, Toolchain, build_module, build_source}; use by_ir::builder::FunctionBuilder; use by_ir::function::{CallConvention, FallbackCode, ModuleIr, ModuleName}; -use by_ir::ops::{BinOp, CmpOp, Op, Terminator, Value}; +use by_ir::ops::{BinOp, CmpOp, Mutation, Op, Terminator, Value}; use by_ir::rtype::RType; mod common; @@ -116,18 +116,21 @@ fn arith_module() -> ModuleIr { op: BinOp::Add, lhs: Value::Register(a), rhs: Value::Register(b), + mutation: Mutation::Fresh, }); builder.push(Op::IntBinary { dest: scaled, op: BinOp::Mul, lhs: Value::Register(sum), rhs: Value::Register(a), + mutation: Mutation::Fresh, }); builder.push(Op::IntBinary { dest: result, op: BinOp::Sub, lhs: Value::Register(scaled), rhs: Value::Register(b), + mutation: Mutation::Fresh, }); builder.terminate(Terminator::Return(Value::Register(result))); @@ -185,6 +188,7 @@ fn fib_module() -> ModuleIr { op: BinOp::Add, lhs: Value::Register(a), rhs: Value::Register(b), + mutation: Mutation::Fresh, }); builder.assign(a, Value::Register(b)); builder.assign(b, Value::Register(next)); @@ -193,6 +197,7 @@ fn fib_module() -> ModuleIr { op: BinOp::Add, lhs: Value::Register(i), rhs: Value::Int(1), + mutation: Mutation::Fresh, }); builder.terminate(Terminator::Goto(header)); @@ -352,6 +357,7 @@ fn division_floors_like_python_and_raises_on_zero() { op: BinOp::FloorDiv, lhs: Value::Register(a), rhs: Value::Register(b), + mutation: Mutation::Fresh, }); builder.terminate(Terminator::Return(Value::Register(out))); let module = ModuleIr { @@ -468,6 +474,7 @@ fn calls_between_compiled_functions_stay_native() { op: BinOp::Add, lhs: Value::Register(d), rhs: Value::Register(d), + mutation: Mutation::Fresh, }); double.terminate(Terminator::Return(Value::Register(doubled))); diff --git a/crates/by_codegen_c/src/lib.rs b/crates/by_codegen_c/src/lib.rs index d7618751e4..498d3dbee0 100644 --- a/crates/by_codegen_c/src/lib.rs +++ b/crates/by_codegen_c/src/lib.rs @@ -48,6 +48,14 @@ use by_ir::rtype::{Primitive, RType, tuple_mangle}; /// unboxed `None`, a bare byte, and handing that to `By_NewRef` produced a NULL /// the error check read as a failure with no exception behind it fn default_expr(ty: &RType, default: &Value) -> String { + // `x: int = True` holds `True` itself, which a tagged `int` keeps as the object. the + // byte a bool immediate is written as would be read as a tagged word, and 1 is the + // error value + if *ty == RType::INT + && let Value::Bool(value) | Value::Bit(value) = default + { + return format!("By_UnboxInt(Py_{})", if *value { "True" } else { "False" }); + } let expr = value_expr(default); if !matches!( ty, @@ -6693,6 +6701,7 @@ fn unset_test( lhs, rhs, dest: result, + .. }, ) = (require, read, arithmetic) else { @@ -6705,7 +6714,7 @@ fn unset_test( if (*lhs != operand && *rhs != operand) || *result == *dest || function.value_type(&Value::Register(*result)) != Some(RType::INT) - || split_int_binary(*op, lhs, rhs).is_none() + || split_int_binary(*op, lhs, rhs, Mutation::Fresh).is_none() { return None; } @@ -7576,7 +7585,13 @@ fn emit_op(module: &ModuleIr, function: &Function, op: &Op, error_target: ErrorE } out } - Op::IntBinary { dest, op, lhs, rhs } => { + Op::IntBinary { + dest, + op, + lhs, + rhs, + mutation, + } => { // a fixed width is plain machine arithmetic: the C compiler emits one // instruction, and there is no tag to strip if let Some(RType::Primitive(Primitive::Fixed(_))) = operand_type(function, lhs) { @@ -7627,25 +7642,38 @@ fn emit_op(module: &ModuleIr, function: &Function, op: &Op, error_target: ErrorE op: *op, lhs: lhs.clone(), rhs: rhs.clone(), + mutation: *mutation, }, error_target, "", ) { return split; } - let call = match op { - BinOp::Add => "By_IntAdd", - BinOp::Sub => "By_IntSub", - BinOp::Mul => "By_IntMul", - BinOp::FloorDiv => "By_IntFloorDiv", - BinOp::Mod => "By_IntMod", - BinOp::TrueDiv => "By_IntTrueDiv", - BinOp::Pow => "By_IntPow", - BinOp::BitAnd => "By_IntAnd", - BinOp::BitOr => "By_IntOr", - BinOp::BitXor => "By_IntXor", - BinOp::Shl => "By_IntShl", - BinOp::Shr => "By_IntShr", + let call = match (op, mutation) { + (BinOp::Add, Mutation::Fresh) => "By_IntAdd", + (BinOp::Sub, Mutation::Fresh) => "By_IntSub", + (BinOp::Mul, Mutation::Fresh) => "By_IntMul", + (BinOp::FloorDiv, Mutation::Fresh) => "By_IntFloorDiv", + (BinOp::Mod, Mutation::Fresh) => "By_IntMod", + (BinOp::TrueDiv, Mutation::Fresh) => "By_IntTrueDiv", + (BinOp::Pow, Mutation::Fresh) => "By_IntPow", + (BinOp::BitAnd, Mutation::Fresh) => "By_IntAnd", + (BinOp::BitOr, Mutation::Fresh) => "By_IntOr", + (BinOp::BitXor, Mutation::Fresh) => "By_IntXor", + (BinOp::Shl, Mutation::Fresh) => "By_IntShl", + (BinOp::Shr, Mutation::Fresh) => "By_IntShr", + (BinOp::Add, Mutation::InPlace) => "By_IntIAdd", + (BinOp::Sub, Mutation::InPlace) => "By_IntISub", + (BinOp::Mul, Mutation::InPlace) => "By_IntIMul", + (BinOp::FloorDiv, Mutation::InPlace) => "By_IntIFloorDiv", + (BinOp::Mod, Mutation::InPlace) => "By_IntIMod", + (BinOp::TrueDiv, Mutation::InPlace) => "By_IntITrueDiv", + (BinOp::Pow, Mutation::InPlace) => "By_IntIPow", + (BinOp::BitAnd, Mutation::InPlace) => "By_IntIAnd", + (BinOp::BitOr, Mutation::InPlace) => "By_IntIOr", + (BinOp::BitXor, Mutation::InPlace) => "By_IntIXor", + (BinOp::Shl, Mutation::InPlace) => "By_IntIShl", + (BinOp::Shr, Mutation::InPlace) => "By_IntIShr", }; let expr = format!("{call}({}, {})", value_expr(lhs), value_expr(rhs)); assign_checked(module, function, *dest, &expr, error_target) @@ -8092,43 +8120,58 @@ fn emit_op(module: &ModuleIr, function: &Function, op: &Op, error_target: ErrorE assign_checked(module, function, *dest, &expr, error_target) } Op::FloatBinary { dest, op, lhs, rhs } => { - let (lhs, rhs) = (value_expr(lhs), value_expr(rhs)); - // a division fails only on a zero divisor, and every double it can answer is a - // legal one — so the divisor is tested here and the test jumps to the error - // edge, rather than a sentinel answer being told apart from a real one by - // asking the thread for an exception on every division - let divided = match op { - BinOp::FloorDiv => Some(( - "PyNumber_FloorDivide", - format!("By_FloatFloorDivNonzero({lhs}, {rhs})"), - )), - BinOp::Mod => Some(( - "PyNumber_Remainder", - format!("By_FloatModNonzero({lhs}, {rhs})"), - )), - BinOp::TrueDiv => Some(("PyNumber_TrueDivide", format!("({lhs} / {rhs})"))), - _ => None, - }; - if let Some((operation, quotient)) = divided { - return format!( - " if (BY_UNLIKELY({rhs} == 0.0)) {{ By_ZeroDivision({operation}, 1); goto {}; }}\n{}", - error_label(error_target), - assign_owned(module, function, *dest, "ient) - ); - } - let expr = match op { - BinOp::Add => format!("({lhs} + {rhs})"), - BinOp::Sub => format!("({lhs} - {rhs})"), - BinOp::Mul => format!("({lhs} * {rhs})"), - BinOp::Pow => format!("By_FloatPow({lhs}, {rhs})"), - // the bitwise operators have no float form; the frontend routes - // them through the object protocol instead - other => format!("(void)0 /* unreachable: float {} */", other.symbol()), - }; - if op.can_fail() { - assign_checked(module, function, *dest, &expr, error_target) - } else { - assign_owned(module, function, *dest, &expr) + let rhs_c = value_expr(rhs); + match function.value_type(lhs) { + Some(RType::Primitive(Primitive::Fixed(_))) => float_binary( + module, + function, + *dest, + *op, + &format!("(double){}", value_expr(lhs)), + &rhs_c, + error_target, + ), + // an `int` on the left: a short converts, and anything else is asked + Some(RType::Primitive(Primitive::Int)) => { + let tagged = value_expr(lhs); + let short = float_binary( + module, + function, + *dest, + *op, + &format!("(double)By_ShortValue({tagged})"), + &rhs_c, + error_target, + ); + let code = match op { + BinOp::Add => '+', + BinOp::Sub => '-', + BinOp::Mul => '*', + BinOp::TrueDiv => '/', + BinOp::FloorDiv => 'f', + BinOp::Mod => '%', + _ => 'p', + }; + let boxed = assign_checked( + module, + function, + *dest, + &format!("By_IntFloatSlow({tagged}, {rhs_c}, '{code}')"), + error_target, + ); + format!( + " if (BY_LIKELY(By_IsShort({tagged}))) {{\n{short} }} else {{\n{boxed} }}\n" + ) + } + _ => float_binary( + module, + function, + *dest, + *op, + &value_expr(lhs), + &rhs_c, + error_target, + ), } } Op::ObjectBinary { @@ -8195,6 +8238,27 @@ fn emit_op(module: &ModuleIr, function: &Function, op: &Op, error_target: ErrorE assign_checked(module, function, *dest, &expr, error_target) } Op::Truthy { dest, src } => { + // a short is true where it is not zero, and only a pointer — a big `int` or a + // subclass with a `__bool__` of its own — can fail + if function.value_type(src) == Some(RType::INT) { + let value = value_expr(src); + let short = assign_owned( + module, + function, + *dest, + &format!("(char)({value} != By_ShortFrom(0))"), + ); + let boxed = assign_checked( + module, + function, + *dest, + &format!("By_IntTruthySlow({value})"), + error_target, + ); + return format!( + " if (BY_LIKELY(By_IsShort({value}))) {{\n{short} }} else {{\n{boxed} }}\n" + ); + } let expr = format!("By_Truthy({})", value_expr(src)); assign_checked(module, function, *dest, &expr, error_target) } @@ -8285,13 +8349,21 @@ fn emit_op(module: &ModuleIr, function: &Function, op: &Op, error_target: ErrorE } _ => format!("By_IntInvert({})", value_expr(operand)), }, + UnaryOp::Pos => match function.value_type(operand) { + Some(RType::Primitive(Primitive::Object)) => { + format!("By_ObjPos({})", value_expr(operand)) + } + _ => format!("By_IntPos({})", value_expr(operand)), + }, + UnaryOp::Index => format!("By_IntIndex({})", value_expr(operand)), }; - if matches!(op, UnaryOp::Neg | UnaryOp::Invert) - && matches!( - function.value_type(operand), - Some(RType::Primitive(Primitive::Int | Primitive::Object)) - ) - { + if matches!( + op, + UnaryOp::Neg | UnaryOp::Invert | UnaryOp::Pos | UnaryOp::Index + ) && matches!( + function.value_type(operand), + Some(RType::Primitive(Primitive::Int | Primitive::Object)) + ) { assign_checked(module, function, *dest, &expr, error_target) } else { assign_owned(module, function, *dest, &expr) @@ -10429,6 +10501,56 @@ fn defer_tests(function: &Function, receiver: bool) -> Vec { .collect() } +/// a double operation, with each operand already written as a C `double` +/// +/// a division fails only on a zero divisor, and every double it can answer is a legal +/// one — so the divisor is tested here and the test jumps to the error edge, rather than +/// a sentinel answer being told apart from a real one by asking the thread for an +/// exception on every division +fn float_binary( + module: &ModuleIr, + function: &Function, + dest: RegisterId, + op: BinOp, + lhs: &str, + rhs: &str, + error_target: ErrorEdge, +) -> String { + let divided = match op { + BinOp::FloorDiv => Some(( + "PyNumber_FloorDivide", + format!("By_FloatFloorDivNonzero({lhs}, {rhs})"), + )), + BinOp::Mod => Some(( + "PyNumber_Remainder", + format!("By_FloatModNonzero({lhs}, {rhs})"), + )), + BinOp::TrueDiv => Some(("PyNumber_TrueDivide", format!("({lhs} / {rhs})"))), + _ => None, + }; + if let Some((operation, quotient)) = divided { + return format!( + " if (BY_UNLIKELY({rhs} == 0.0)) {{ By_ZeroDivision({operation}, 1); goto {}; }}\n{}", + error_label(error_target), + assign_owned(module, function, dest, "ient) + ); + } + let expr = match op { + BinOp::Add => format!("({lhs} + {rhs})"), + BinOp::Sub => format!("({lhs} - {rhs})"), + BinOp::Mul => format!("({lhs} * {rhs})"), + BinOp::Pow => format!("By_FloatPow({lhs}, {rhs})"), + // the bitwise operators have no float form; the frontend routes + // them through the object protocol instead + other => format!("(void)0 /* unreachable: float {} */", other.symbol()), + }; + if op.can_fail() { + assign_checked(module, function, dest, &expr, error_target) + } else { + assign_owned(module, function, dest, &expr) + } +} + /// the fast path and the slow call of a tagged operation whose fast path cannot fail /// /// a short is an even word, and a sum, difference, product or bitwise combination of two @@ -10443,13 +10565,20 @@ fn split_int_binary_c( error_target: ErrorEdge, unset: &str, ) -> Option { - let Op::IntBinary { dest, op, lhs, rhs } = op else { + let Op::IntBinary { + dest, + op, + lhs, + rhs, + mutation, + } = op + else { return None; }; if function.value_type(&Value::Register(*dest)) != Some(RType::INT) { return None; } - let (fast, slow) = split_int_binary(*op, lhs, rhs)?; + let (fast, slow) = split_int_binary(*op, lhs, rhs, *mutation)?; let target = local(*dest); Some(format!( " {{ ByTagged by_t;\n\ @@ -10701,8 +10830,32 @@ fn unbox_owning_nothing_of_its_source(to: &RType) -> Option<&'static str> { }) } -fn split_int_binary(op: BinOp, lhs: &Value, rhs: &Value) -> Option<(&'static str, String)> { +fn split_int_binary( + op: BinOp, + lhs: &Value, + rhs: &Value, + mutation: Mutation, +) -> Option<(&'static str, String)> { let (l, r) = (value_expr(lhs), value_expr(rhs)); + // the fast path is taken only by two shorts, which are exact `int`s with no in-place + // methods, so an augmented assignment differs from the plain operator only here + if mutation == Mutation::InPlace { + let (fast, operation) = match op { + BinOp::Add => ("By_IntAddShort", "PyNumber_InPlaceAdd"), + BinOp::Sub => ("By_IntSubShort", "PyNumber_InPlaceSubtract"), + BinOp::Mul => ("By_IntMulShort", "PyNumber_InPlaceMultiply"), + BinOp::BitAnd => ("By_IntAndShort", "PyNumber_InPlaceAnd"), + BinOp::BitOr => ("By_IntOrShort", "PyNumber_InPlaceOr"), + BinOp::BitXor => ("By_IntXorShort", "PyNumber_InPlaceXor"), + BinOp::FloorDiv + | BinOp::Mod + | BinOp::TrueDiv + | BinOp::Pow + | BinOp::Shl + | BinOp::Shr => return None, + }; + return Some((fast, format!("By_IntSlowInPlace({l}, {r}, {operation})"))); + } Some(match op { BinOp::Add => ( "By_IntAddShort", @@ -12598,6 +12751,7 @@ mod tests { op: BinOp::Add, lhs: Value::Register(a), rhs: Value::Register(b), + mutation: Mutation::Fresh, }); builder.terminate(Terminator::Return(Value::Register(sum))); builder.finish() @@ -12741,6 +12895,7 @@ mod tests { op: BinOp::Add, lhs: Value::Register(read), rhs: Value::Int(1), + mutation: Mutation::Fresh, }); builder.terminate(Terminator::Return(Value::Register(sum))); let mut function = builder.finish(); @@ -14060,6 +14215,7 @@ mod tests { op: BinOp::Add, lhs: Value::Register(a), rhs: Value::Int(1), + mutation: Mutation::Fresh, }); builder.terminate(Terminator::Return(Value::Register(sum))); let mut module = module_with(builder.finish()); @@ -14106,6 +14262,7 @@ mod tests { op: BinOp::FloorDiv, lhs: Value::Register(n), rhs: Value::Int(2), + mutation: Mutation::Fresh, }); builder.terminate(Terminator::Return(Value::Register(n))); let c = emit_module(&module_with(builder.finish())); @@ -14145,6 +14302,7 @@ mod tests { op: BinOp::Add, lhs: Value::Register(acc), rhs: Value::Register(n), + mutation: Mutation::Fresh, }); builder.terminate(Terminator::Return(Value::Register(acc))); let c = emit_module(&module_with(builder.finish())); @@ -14207,6 +14365,7 @@ mod tests { op, lhs: Value::Register(a), rhs: Value::Register(b), + mutation: Mutation::Fresh, }); builder.terminate(Terminator::Return(Value::Register(out))); let c = emit_module(&module_with(builder.finish())); @@ -14222,6 +14381,45 @@ mod tests { } } + #[test] + fn an_augmented_int_operation_asks_for_the_in_place_method_off_the_short_path() { + // two shorts are exact `int`s, which have no in-place methods, so `x += y` keeps the + // plain operator's fast path unchanged. anything else may be a subclass with an + // `__iadd__` of its own, and the slow path asks the in-place form of the operator. + // an operation with no split form tests the left operand's exactness first + for (op, expected) in [ + ( + BinOp::Add, + "if (BY_UNLIKELY(!By_IntAddShort(r0, r1, &by_t))) {\n\ + \x20 by_t = By_IntSlowInPlace(r0, r1, PyNumber_InPlaceAdd);", + ), + ( + BinOp::BitXor, + "if (BY_UNLIKELY(!By_IntXorShort(r0, r1, &by_t))) {\n\ + \x20 by_t = By_IntSlowInPlace(r0, r1, PyNumber_InPlaceXor);", + ), + (BinOp::FloorDiv, "By_IntIFloorDiv(r0, r1)"), + (BinOp::Mod, "By_IntIMod(r0, r1)"), + (BinOp::Shl, "By_IntIShl(r0, r1)"), + (BinOp::Pow, "By_IntIPow(r0, r1)"), + ] { + let mut builder = FunctionBuilder::new("bump", RType::INT); + let a = builder.param("a", RType::INT); + let b = builder.param("b", RType::INT); + let out = builder.temp(RType::INT); + builder.push(Op::IntBinary { + dest: out, + op, + lhs: Value::Register(a), + rhs: Value::Register(b), + mutation: Mutation::InPlace, + }); + builder.terminate(Terminator::Return(Value::Register(out))); + let c = emit_module(&module_with(builder.finish())); + assert!(c.contains(expected), "{op:?}: {c}"); + } + } + #[test] fn copying_a_register_retains_before_it_releases() { // `a = a` must not release the value it is about to store @@ -15456,6 +15654,7 @@ mod tests { op: BinOp::Add, lhs: Value::Register(a), rhs: Value::Register(b), + mutation: Mutation::Fresh, }); builder.terminate(Terminator::Return(Value::Register(out))); let mut module = ModuleIr::new("app"); diff --git a/crates/by_ir/src/builder.rs b/crates/by_ir/src/builder.rs index 74d42091fc..a1f3a0ad71 100644 --- a/crates/by_ir/src/builder.rs +++ b/crates/by_ir/src/builder.rs @@ -454,7 +454,7 @@ impl FunctionBuilder { #[cfg(test)] mod tests { use super::*; - use crate::ops::BinOp; + use crate::ops::{BinOp, Mutation}; use crate::verify::verify; #[test] @@ -468,6 +468,7 @@ mod tests { op: BinOp::Add, lhs: Value::Register(a), rhs: Value::Register(b), + mutation: Mutation::Fresh, }); builder.terminate(Terminator::Return(Value::Register(sum))); assert_eq!(verify(&builder.finish()), Ok(())); diff --git a/crates/by_ir/src/ops.rs b/crates/by_ir/src/ops.rs index 04f03bed2d..7847f349dc 100644 --- a/crates/by_ir/src/ops.rs +++ b/crates/by_ir/src/ops.rs @@ -74,6 +74,21 @@ impl Value { Self::Bytes(_) => Some(RType::OBJECT), } } + + /// whether a place of `ty` holds this immediate as the very object python holds, so + /// that writing it there — as a default, say — changes nothing a program can see + /// + /// an `object` place holds every immediate, boxed. any other place holds only the + /// immediates of its own representation, and an `int` place a `bool` as well, which + /// it keeps as the object itself. `x: float = 1` does not qualify: the `int` written + /// into a double reads back as `1.0`, where python goes on holding the `int` + pub fn stands_as_itself_in(&self, ty: &RType) -> bool { + match self { + Self::Register(_) => false, + Self::Bool(_) | Self::Bit(_) if *ty == RType::INT => true, + _ => *ty == RType::OBJECT || self.immediate_type().as_ref() == Some(ty), + } + } } /// whether an operation is the augmented form @@ -199,6 +214,13 @@ pub enum UnaryOp { Neg, Not, Invert, + /// `+x`: an `int`'s or an object's `__pos__`, which an exact `int` answers with + /// itself and a `Counter`, say, with a copy that drops what is not positive + Pos, + /// `operator.index(x)` of an `int`: the exact `int` it stands for, which is how + /// `range` reads its bounds. an `int` subclass is copied by value, and no method of + /// its own is asked + Index, } /// which lookup a [`Op::LicenceHolds`] re-asks @@ -280,13 +302,22 @@ pub enum Op { /// `dest = src`, with no representation change Assign { dest: RegisterId, src: Value }, /// arithmetic on tagged integers + /// + /// an `int` register may hold a subclass behind the pointer, and python offers such a + /// left operand of `x += y` its own `__iadd__` first. an exact `int` has no in-place + /// methods, so `mutation` only ever changes what the slow path asks IntBinary { dest: RegisterId, op: BinOp, lhs: Value, rhs: Value, + mutation: Mutation, }, /// arithmetic on unboxed doubles + /// + /// the left operand may be an `int` instead, tagged or a machine integer: python + /// asks an `int` on the left first, so the operation is its method's — the double + /// arithmetic for an exact `int`, and the subclass's own for anything else FloatBinary { dest: RegisterId, op: BinOp, @@ -2528,6 +2559,7 @@ mod tests { op: BinOp::Add, lhs: Value::Register(RegisterId(0)), rhs: Value::Int(1), + mutation: Mutation::Fresh, }; assert_eq!(op.dest(), Some(RegisterId(2))); assert_eq!(op.operands().len(), 2); diff --git a/crates/by_ir/src/print.rs b/crates/by_ir/src/print.rs index c9b099592a..f9be553047 100644 --- a/crates/by_ir/src/print.rs +++ b/crates/by_ir/src/print.rs @@ -343,9 +343,7 @@ fn print_op(function: &Function, op: &Op) -> String { op.symbol(), value(rhs) ), - Op::IntBinary { dest, op, lhs, rhs } - | Op::FloatBinary { dest, op, lhs, rhs } - | Op::FloatObjectBinary { dest, op, lhs, rhs } => { + Op::FloatBinary { dest, op, lhs, rhs } | Op::FloatObjectBinary { dest, op, lhs, rhs } => { format!( "{} = {} {} {}", name(*dest), @@ -354,7 +352,14 @@ fn print_op(function: &Function, op: &Op) -> String { value(rhs) ) } - Op::ObjectBinary { + Op::IntBinary { + dest, + op, + lhs, + rhs, + mutation, + } + | Op::ObjectBinary { dest, op, lhs, @@ -392,6 +397,8 @@ fn print_op(function: &Function, op: &Op) -> String { UnaryOp::Neg => "-", UnaryOp::Not => "not ", UnaryOp::Invert => "~", + UnaryOp::Pos => "+", + UnaryOp::Index => "index ", }; format!("{} = {}{}", name(*dest), symbol, value(operand)) } @@ -1001,7 +1008,7 @@ fn print_terminator(function: &Function, terminator: &Terminator) -> String { mod tests { use super::*; use crate::function::{BasicBlock, CallConvention, Declined, RegisterDecl}; - use crate::ops::{BinOp, BlockId, CmpOp}; + use crate::ops::{BinOp, BlockId, CmpOp, Mutation}; use crate::rtype::RType; fn sample() -> Function { @@ -1022,6 +1029,7 @@ mod tests { op: BinOp::Sub, lhs: Value::Int(0), rhs: Value::Register(RegisterId(0)), + mutation: Mutation::Fresh, }); let identity = BasicBlock::new(Terminator::Return(Value::Register(RegisterId(0)))); diff --git a/crates/by_ir/src/verify.rs b/crates/by_ir/src/verify.rs index 68b65ebe0c..7ff963a97e 100644 --- a/crates/by_ir/src/verify.rs +++ b/crates/by_ir/src/verify.rs @@ -49,6 +49,7 @@ pub fn verify_module(module: &ModuleIr) -> Result<(), Vec> { .flatten() .collect(); errors.extend(check_accessor_coverage(module)); + errors.extend(check_field_defaults(module)); if errors.is_empty() { Ok(()) } else { @@ -56,6 +57,30 @@ pub fn verify_module(module: &ModuleIr) -> Result<(), Vec> { } } +/// a constructor writes a field's default straight into the field, so the field's +/// representation has to hold it as the object python holds — see +/// [`Value::stands_as_itself_in`] +fn check_field_defaults(module: &ModuleIr) -> Vec { + module + .classes + .iter() + .flat_map(|class| { + class.fields.iter().filter_map(move |field| { + let default = field.default.as_ref()?; + (!default.stands_as_itself_in(&field.ty)).then(|| VerifyError { + function: class.name.clone(), + block: None, + message: format!( + "the default {default:?} of field `{}` does not stand as itself in its {} place", + field.name, field.ty + ), + about_the_source: false, + }) + }) + }) + .collect() +} + /// each class's in-module base, which is the whole of the layout chain /// /// an external base has no struct here, so nothing downstream of it is a free pointer @@ -403,11 +428,40 @@ impl Verifier<'_> { self.check_block(id); } + self.check_defaults(); self.check_definite_assignment(); self.check_release_sets(); self.check_borrowed_writes(); } + /// the boundary writes a default straight into its parameter, so the parameter's + /// representation has to hold it as the object python holds — see + /// [`Value::stands_as_itself_in`] + fn check_defaults(&mut self) { + let function = self.function; + for (index, default) in function.defaults.iter().enumerate() { + let Some(default) = default else { + continue; + }; + let Some(param) = function.params().get(index) else { + self.error( + None, + format!("a default for parameter {index}, which does not exist"), + ); + continue; + }; + if !default.stands_as_itself_in(¶m.ty) { + self.error( + None, + format!( + "the default {default:?} of parameter {index} does not stand as itself in its {} place", + param.ty + ), + ); + } + } + } + /// a borrowed register may only be filled by an operation that *lends* /// /// `borrowed` means the frame never took a reference and so never gives one back: @@ -733,7 +787,9 @@ impl Verifier<'_> { self.expect_dest(block, *dest, &expected, "assign"); } } - Op::IntBinary { dest, lhs, rhs, op } => { + Op::IntBinary { + dest, lhs, rhs, op, .. + } => { // a *fixed* width is plain machine arithmetic: no tag, no shift, no // overflow branch. both operands have to agree on it, because the // width is the representation rather than a property of the value @@ -751,7 +807,12 @@ impl Verifier<'_> { self.expect_dest(block, *dest, &result, op.symbol()); } Op::FloatBinary { dest, lhs, rhs, op } => { - self.expect(block, lhs, &RType::FLOAT, op.symbol()); + match self.operand_type(block, lhs) { + Some(ty @ RType::Primitive(Primitive::Int | Primitive::Fixed(_))) => { + self.expect(block, lhs, &ty, op.symbol()); + } + _ => self.expect(block, lhs, &RType::FLOAT, op.symbol()), + } self.expect(block, rhs, &RType::FLOAT, op.symbol()); self.expect_dest(block, *dest, &RType::FLOAT, op.symbol()); } @@ -975,7 +1036,10 @@ impl Verifier<'_> { self.expect_dest(block, *dest, &RType::BIT, op.symbol()); } Op::Truthy { dest, src } => { - self.expect(block, src, &RType::OBJECT, "truthiness"); + // an `int` is asked in its own representation, for its `__bool__` + if self.operand_type(block, src) != Some(RType::INT) { + self.expect(block, src, &RType::OBJECT, "truthiness"); + } self.expect_dest(block, *dest, &RType::BIT, "truthiness"); } Op::IntCompare { dest, lhs, rhs, op } => { @@ -1021,18 +1085,25 @@ impl Verifier<'_> { } self.expect_dest(block, *dest, &operand_ty, "unary `-`"); } - UnaryOp::Invert => { + UnaryOp::Invert | UnaryOp::Pos => { + let symbol = if *op == UnaryOp::Invert { "~" } else { "+" }; if !matches!( operand_ty, RType::Primitive(Primitive::Int | Primitive::Object) ) { self.error( Some(block), - format!("unary `~` expects int or object, found {operand_ty}"), + format!( + "unary `{symbol}` expects int or object, found {operand_ty}" + ), ); return; } - self.expect_dest(block, *dest, &operand_ty, "unary `~`"); + self.expect_dest(block, *dest, &operand_ty, &format!("unary `{symbol}`")); + } + UnaryOp::Index => { + self.expect(block, operand, &RType::INT, "an integer's unary"); + self.expect_dest(block, *dest, &RType::INT, "an integer's unary"); } UnaryOp::Not => { // `bool` and `bit` are both a 0-or-1 byte, so `not` @@ -2218,6 +2289,7 @@ mod tests { op: BinOp::Add, lhs: Value::Register(RegisterId(0)), rhs: Value::Register(RegisterId(1)), + mutation: Mutation::Fresh, }); Function { posonly: 0, @@ -2253,6 +2325,25 @@ mod tests { assert_eq!(verify(&add()), Ok(())); } + #[test] + fn a_default_its_parameter_cannot_hold_as_itself_is_rejected() { + let mut f = add(); + f.defaults = vec![Some(Value::Bool(true)), Some(Value::Int(3))]; + assert_eq!(verify(&f), Ok(())); + f.registers[1].ty = RType::FLOAT; + f.registers[2].ty = RType::FLOAT; + f.blocks[0].ops.clear(); + f.ret = RType::FLOAT; + f.blocks[0].terminator = Terminator::Return(Value::Register(RegisterId(1))); + let errors = verify(&f).unwrap_err(); + assert!( + errors.iter().any(|e| e + .message + .contains("does not stand as itself in its float place")), + "{errors:?}" + ); + } + #[test] fn a_function_with_no_blocks_is_rejected() { let mut f = add(); diff --git a/crates/by_irbuild/src/lib.rs b/crates/by_irbuild/src/lib.rs index a86f47c27b..d4047aa276 100644 --- a/crates/by_irbuild/src/lib.rs +++ b/crates/by_irbuild/src/lib.rs @@ -169,12 +169,11 @@ pub fn build_module( // where the interpreted build checks a value it cannot trust, decided once for the // whole module, so every frame lowered below makes the same checks in the same places let source = ruff_db::source::source_text(db, model.file()); - let soundness = by_transforms::soundness_sites( - model, - by_transforms::WrittenNames::new(source.as_str()), - suite, - soundness, - ); + // the names and the `/` a parameter list that repeats `_` is lowered to are ty's answer, + // the one the interpreted build is transpiled with + let underscores = by_transforms::repeated_underscore_lowerings(model, suite); + let written = by_transforms::WrittenNames::new(source.as_str()).with_lowerings(&underscores); + let soundness = by_transforms::soundness_sites(model, written, suite, soundness); // a call is only lowered natively when the callee is a module-level function // in this same unit, so the set has to be known before any body is lowered @@ -349,6 +348,7 @@ pub fn build_module( db, env, model, + written, method, &layouts, Some(Receiver::Explicit(&receiver)), @@ -406,6 +406,7 @@ pub fn build_module( db, env, model, + written, accessor, &layouts, Some(Receiver::Explicit(&receiver)), @@ -436,7 +437,7 @@ pub fn build_module( .filter_map(|stmt| match stmt { Stmt::FunctionDef(function) => { let mut signature = - signature(db, env, model, function, &layouts, None, &[]).ok()?; + signature(db, env, model, written, function, &layouts, None, &[]).ok()?; resumable_return(function, &mut signature); Some((function.name.to_string(), signature)) } @@ -518,6 +519,7 @@ pub fn build_module( db, env, model, + written, init, &layouts, Some(Receiver::Explicit(&receiver)), @@ -550,7 +552,8 @@ pub fn build_module( _ => None, }) .filter_map(|function| { - let signature = signature(db, env, model, function, &layouts, None, &[]).ok()?; + let signature = + signature(db, env, model, written, function, &layouts, None, &[]).ok()?; Some((generators::direct_name(&function.name), signature)) }), ); @@ -630,6 +633,7 @@ pub fn build_module( accessors: &accessors, language, soundness: &soundness, + written: &written, recheck_licences, bind_functions_early, db, @@ -1321,11 +1325,18 @@ fn lower_generator( .map(|(name, _)| name.clone()) .collect(); // the constructor seeds every one of them, so they are as assigned as a parameter - let source = ruff_db::source::source_text(db, model.file()); - let written = by_transforms::WrittenNames::new(source.as_str()); - let mut assigned = generators::definitely_assigned(function, written); + let mut assigned = generators::definitely_assigned(function, *unit.written); assigned.extend(captured.iter().cloned()); - let mut signed = signature(db, env, model, function, layouts, receiver, &[])?; + let mut signed = signature( + db, + env, + model, + *unit.written, + function, + layouts, + receiver, + &[], + )?; takes_the_first_iterator(captures, &mut signed); let parameters = signed.params; let representation = |name: &str| { @@ -1336,7 +1347,7 @@ fn lower_generator( .map(|(_, rtype)| rtype.clone()) }; let names = { - let mut names = generators::state_names(function, written, &locals); + let mut names = generators::state_names(function, *unit.written, &locals); if captures.is_some_and(|nested| nested.generator_expression.is_some()) { assigned.insert(closures::GENERATOR_ITERATOR.to_string()); names.push(closures::GENERATOR_ITERATOR.to_string()); @@ -1461,6 +1472,7 @@ fn lower_generator( db, env, model, + *unit.written, &entry.def, &layouts_with_state, Some(Receiver::Implicit(&state)), @@ -1611,7 +1623,16 @@ fn lower_generator_constructor( layouts, .. } = unit; - let mut signed = signature(db, env, model, function, layouts, receiver, &[])?; + let mut signed = signature( + db, + env, + model, + *unit.written, + function, + layouts, + receiver, + &[], + )?; takes_the_first_iterator(nested, &mut signed); let Signature { params, @@ -5926,6 +5947,7 @@ fn class_fields( } Some(at) => { let field = &mut fields[at]; + field_default_stands(&name, Some(&default), &field.ty)?; field.default = Some(default); field.optional = false; } @@ -5935,10 +5957,12 @@ fn class_fields( let ty = target.inferred_type(model).ok_or_else(|| { Decline::new("an accessor block's storage has no inferred type") })?; + let ty = map_type_with(db, env, ty, layouts)?; + field_default_stands(&name, Some(&default), &ty)?; fields.push(by_ir::function::FieldDecl { cell: false, name, - ty: map_type_with(db, env, ty, layouts)?, + ty, default: Some(default), optional: false, defaulted_by: None, @@ -6313,10 +6337,12 @@ fn data_fields( .target .inferred_type(model) .ok_or_else(|| Decline::new("a field has no inferred type"))?; + let ty = map_type_with(db, env, ty, layouts)?; + field_default_stands(&name, default.as_ref(), &ty)?; fields.push(by_ir::function::FieldDecl { cell: false, name, - ty: map_type_with(db, env, ty, layouts)?, + ty, default, optional: false, defaulted_by: None, @@ -7406,7 +7432,16 @@ fn lower_function_with_receiver( ret, deferring, computed_defaults, - } = signature(db, env, model, function, layouts, receiver, arrays)?; + } = signature( + db, + env, + model, + *unit.written, + function, + layouts, + receiver, + arrays, + )?; // a boundary that hands the call on takes the twin off the interpreted class, and // for a class method that is already *bound* — to the interpreted class, not to the @@ -7617,6 +7652,7 @@ fn lower_function_with_receiver( db, env, model, + *unit.written, &entry.def, &layouts_with_env, Some(Receiver::Implicit(&receiver)), @@ -8080,6 +8116,9 @@ struct Unit<'a> { /// the runtime soundness checks the interpreted build makes — see /// [`LowerOptions::soundness`] soundness: &'a by_transforms::SoundnessSites, + /// the names the module spells, and how each parameter list in it that repeats `_` is + /// lowered, which is what every parameter is published under + written: &'a by_transforms::WrittenNames<'a>, /// whether each licensed direct call re-asks the lookup it skips — see /// [`LowerOptions::recheck_licences`] recheck_licences: bool, @@ -8619,18 +8658,21 @@ fn supplied_arrays( seen } +#[expect( + clippy::too_many_arguments, + reason = "a signature is read before any `Unit` exists, from the parts one is made of" +)] fn signature( db: &dyn ty_python_semantic::Db, env: &ProgramEnvironment<'_>, model: &SemanticModel<'_>, + written: by_transforms::WrittenNames, function: &ast::StmtFunctionDef, layouts: &Layouts, receiver: Option>, arrays: &[(usize, RType)], ) -> Lowered { let parameters = &function.parameters; - let source = ruff_db::source::source_text(db, model.file()); - let written = by_transforms::WrittenNames::new(source.as_str()); let mut params = Vec::with_capacity(parameters.args.len() + 1); let mut defaults: Vec> = Vec::with_capacity(parameters.args.len() + 1); @@ -8708,7 +8750,6 @@ fn signature( } }, }; - defaults.push(default); let rtype = match (index, receiver) { (0, Some(Receiver::Explicit(receiver))) => receiver.clone(), // the unboxed edition takes the buffer itself. it is never reached from @@ -8768,6 +8809,24 @@ fn signature( } None => rtype, }; + // an immediate the parameter's representation cannot hold as itself is one the + // boundary cannot write either: `x: float = 1` leaves python holding the `int`, + // and a double would read it back as `1.0`. whoever holds a computed default + // holds this one as the object python made — except the environment of a nested + // function, which parks it only to unbox it into that same representation + let default = match default { + Some(value) if !value.stands_as_itself_in(&rtype) => { + if matches!(receiver, Some(Receiver::Implicit(_))) { + return Err(Decline::new(format!( + "the default of `{name}` is a literal its `{rtype}` place cannot hold as itself" + ))); + } + computed_defaults.push(params.len()); + None + } + default => default, + }; + defaults.push(default); params.push((name.to_string(), rtype)); } @@ -8801,7 +8860,7 @@ fn signature( // counted from `params[0]`, which for a synthetic receiver is a slot no // keyword can reach either — a boundary reads this against the parameters it // has, not against the ones that were written - posonly: parameters.posonlyargs.len() + posonly: by_transforms::positional_only_count(parameters, written) + usize::from(matches!(receiver, Some(Receiver::Implicit(_)))), kwonly: parameters.kwonlyargs.len(), ret: return_type(db, env, model, function, layouts)?, @@ -12709,8 +12768,10 @@ impl Lowering<'_, '_> { // each way writes what only the other reads as well, because nothing that checks // the function can see the two never cross self.builder.switch_to(counting); + let start_value = self.range_bound(start_value); self.builder.assign(index, start_value); - self.builder.assign(limit, stop_value.clone()); + let stop_bound = self.range_bound(stop_value.clone()); + self.builder.assign(limit, stop_bound); self.builder.push(Op::Box { dest: iterator, src: Value::None, @@ -12798,6 +12859,7 @@ impl Lowering<'_, '_> { op: BinOp::Add, lhs: Value::Register(index), rhs: Value::Int(step_value), + mutation: Mutation::Fresh, }); self.builder.terminate(Terminator::Goto(header)); @@ -12892,6 +12954,7 @@ impl Lowering<'_, '_> { op: BinOp::Add, lhs: Value::Register(counter), rhs: Value::Fixed(1), + mutation: Mutation::Fresh, }); self.builder.assign(counter, Value::Register(stepped)); self.builder.terminate(Terminator::Goto(header)); @@ -14175,6 +14238,22 @@ impl Lowering<'_, '_> { }) } + /// a bound of a counted `range` as `range` itself reads it: the exact `int` an `int` + /// subclass stands for, so the loop counts in plain `int`s and never asks the + /// subclass's own `__add__` or `__lt__`. an immediate is one already + fn range_bound(&mut self, value: Value) -> Value { + if !matches!(value, Value::Register(_)) { + return value; + } + let dest = self.builder.temp(RType::INT); + self.builder.push(Op::Unary { + dest, + op: UnaryOp::Index, + operand: value, + }); + Value::Register(dest) + } + /// convert `value` from `from` to `to`, or decline if there is no free /// conversion between the two representations fn coerce(&mut self, value: Value, from: &RType, to: &RType) -> Lowered { @@ -15154,14 +15233,10 @@ impl Lowering<'_, '_> { }); Value::Register(dest) } + // not `x != 0`: an `int` subclass answers truthiness with its own `__bool__` RType::Primitive(Primitive::Int) => { let dest = self.builder.temp(RType::BIT); - self.builder.push(Op::IntCompare { - dest, - op: CmpOp::Ne, - lhs: value, - rhs: Value::Int(0), - }); + self.builder.push(Op::Truthy { dest, src: value }); Value::Register(dest) } RType::Primitive(Primitive::Float) => { @@ -15682,7 +15757,27 @@ impl Lowering<'_, '_> { }); Ok((Value::Register(dest), ty)) } - AstUnaryOp::UAdd => Ok((operand, ty)), + // an exact `float` is its own `+`, and a double holds nothing else + AstUnaryOp::UAdd if ty == RType::FLOAT => Ok((operand, ty)), + AstUnaryOp::UAdd => { + if !matches!(ty, RType::Primitive(Primitive::Int | Primitive::Object)) { + let operand = self.widen_to_object(operand, &ty); + let dest = self.builder.temp(RType::OBJECT); + self.builder.push(Op::Unary { + dest, + op: UnaryOp::Pos, + operand, + }); + return Ok((Value::Register(dest), RType::OBJECT)); + } + let dest = self.builder.temp(ty.clone()); + self.builder.push(Op::Unary { + dest, + op: UnaryOp::Pos, + operand, + }); + Ok((Value::Register(dest), ty)) + } // `~`, and the basedpython postfix operators `?` / `!` / `!!` other => Err(Decline::new(format!( "unary `{}` is not lowered yet", @@ -15804,8 +15899,10 @@ impl Lowering<'_, '_> { Ok((Value::Register(dest), result_ty)) } - /// `mutation` only reaches the object protocol: an unboxed pair has no method to - /// offer, so `x += 1` on an `int` register *is* `x = x + 1` + /// `mutation` reaches whatever may hold an object that has in-place methods: the + /// object protocol, a `str` concatenation, and the slow path of an `int` operation, + /// where a subclass stands behind the pointer. a double has no method to offer, so + /// `x += 1.0` on a `float` register *is* `x = x + 1.0` fn emit_binary( &mut self, dest: RegisterId, @@ -15819,7 +15916,13 @@ impl Lowering<'_, '_> { (RType::Primitive(Primitive::Int), RType::Primitive(Primitive::Int)) if !power_may_be_fractional(op, (&lhs, lhs_ty), (&rhs, rhs_ty)) => { - self.builder.push(Op::IntBinary { dest, op, lhs, rhs }); + self.builder.push(Op::IntBinary { + dest, + op, + lhs, + rhs, + mutation, + }); } (RType::Primitive(Primitive::Str), RType::Primitive(Primitive::Str)) if matches!(op, BinOp::Add) => @@ -15845,6 +15948,10 @@ impl Lowering<'_, '_> { // conversion this emits. so the double operation *is* the operation, // not an approximation of one, right down to the `OverflowError` an // integer with no float at all raises + // + // an `int` on the *left* is asked first, though, and an `int` subclass may + // answer with a method of its own. so that side stays tagged, and the + // operation converts it only once it knows it holds an exact `int` (RType::Primitive(Primitive::Float), RType::Primitive(Primitive::Int)) | (RType::Primitive(Primitive::Int), RType::Primitive(Primitive::Float)) if !matches!( @@ -15852,7 +15959,6 @@ impl Lowering<'_, '_> { BinOp::BitAnd | BinOp::BitOr | BinOp::BitXor | BinOp::Shl | BinOp::Shr ) && !power_may_be_complex(op, (&lhs, lhs_ty), (&rhs, rhs_ty)) => { - let lhs = self.widen_to_float(lhs, lhs_ty); let rhs = self.widen_to_float(rhs, rhs_ty); self.builder.push(Op::FloatBinary { dest, op, lhs, rhs }); } @@ -17171,7 +17277,9 @@ impl Lowering<'_, '_> { None => Value::Int(0), Some((value, ty)) => self.coerce(value.clone(), ty, &RType::INT)?, }; + let first = self.range_bound(first); let last = self.coerce(stop.0.clone(), &stop.1, &RType::INT)?; + let last = self.range_bound(last); self.store(counter, first, &RType::INT)?; self.store(bound, last, &RType::INT)?; // each way writes what only the other reads as well, because nothing that checks @@ -17277,6 +17385,7 @@ impl Lowering<'_, '_> { op: BinOp::Add, lhs: Value::Register(counter), rhs: Value::Int(1), + mutation: Mutation::Fresh, }); self.builder.assign(counter, Value::Register(stepped)); self.builder.terminate(Terminator::Goto(header)); @@ -19241,6 +19350,18 @@ fn inherited_default( .ok_or_else(|| Decline::new("a parameter inherits a default that is not an immediate")) } +/// refuse a field whose literal default its representation cannot hold as itself — see +/// [`Value::stands_as_itself_in`]. a constructor writes the default straight into the +/// field, and there is no interpreted definition to hand the object to instead +fn field_default_stands(name: &str, default: Option<&Value>, ty: &RType) -> Lowered<()> { + match default { + Some(value) if !value.stands_as_itself_in(ty) => Err(Decline::new(format!( + "the default of field `{name}` is a literal its `{ty}` place cannot hold as itself" + ))), + _ => Ok(()), + } +} + fn literal_value(expr: &Expr) -> Option { Some(match expr { Expr::NumberLiteral(node) => match &node.value { diff --git a/crates/by_irbuild/src/tests.rs b/crates/by_irbuild/src/tests.rs index 107823767c..ec9f3bfc38 100644 --- a/crates/by_irbuild/src/tests.rs +++ b/crates/by_irbuild/src/tests.rs @@ -157,7 +157,8 @@ def total(n: int) -> int: acc += n return acc "); - assert!(ir.contains("acc = acc + n"), "{ir}"); + // in place: an `int` register may hold a subclass with an `__iadd__` of its own + assert!(ir.contains("acc = acc += n"), "{ir}"); } #[test] @@ -738,14 +739,17 @@ def before(a: str, b: object) -> bool: } #[test] -fn an_int_condition_compares_against_zero() { +fn an_int_condition_asks_the_int_for_its_truth() { + // not `n != 0`: an `int` subclass answers a condition with its `__bool__`, and a + // comparison with its `__ne__` let ir = ir("\ def f(n: int) -> int: if n: return 1 return 0 "); - assert!(ir.contains("n != 0"), "{ir}"); + assert!(ir.contains("truthy n"), "{ir}"); + assert!(!ir.contains("n != 0"), "{ir}"); } #[test] @@ -5938,9 +5942,12 @@ def f(n: int) -> int: !ir.contains("i < n"), "the comparison must not re-read the mutated local: {ir}" ); + // through `operator.index`, which is how `range` reads a bound, into a register of + // its own assert!( - ir.lines() - .any(|line| line.trim_start().starts_with('r') && line.trim_end().ends_with("= n")), + ir.lines().any( + |line| line.trim_start().starts_with('r') && line.trim_end().ends_with("= index n") + ), "the bound is copied to its own register: {ir}" ); } @@ -10144,6 +10151,38 @@ class Cell: }); } +/// a literal written into a place that cannot hold it as itself is not the object python +/// holds: the `int` 4 in a double reads back as `4.0`, and before this was refused it +/// read back as `8.0`, the tagged word taken for a double. a field has no interpreted +/// definition to hand the default to, and a nested function's environment would unbox it +/// into the same double, so both decline +#[test] +fn a_literal_default_its_place_cannot_hold_as_itself_declines() { + let reasons = declines( + "\ +class Cell: + var v: float = 4 + get() = field + set(given): + field = given + +def outer() -> float: + def inner(x: float = 1) -> float: + return x + return inner() +", + ); + for name in ["Cell", "outer"] { + assert!( + reasons + .iter() + .any(|(declined, reason)| declined == name + && reason.contains("cannot hold as itself")), + "{name}: {reasons:?}" + ); + } +} + /// a read-only accessor block declares the storage nothing writes /// /// with no setter there is no assignment to the receiver anywhere in the class, so the diff --git a/crates/by_opt/src/borrow.rs b/crates/by_opt/src/borrow.rs index bc386359ff..38308038a5 100644 --- a/crates/by_opt/src/borrow.rs +++ b/crates/by_opt/src/borrow.rs @@ -646,8 +646,9 @@ fn is_inert(function: &Function, op: &Op) -> bool { // pure double arithmetic touches no reference at all, and neither does asking // whether an instance has a field: a load and a comparison, and a raise at worst, // which leaves before any later use of what is borrowed - Op::FloatBinary { .. } - | Op::FloatCompare { .. } + // an `int` on the left of a double operation is asked for its own method + Op::FloatBinary { lhs, .. } => function.value_type(lhs) != Some(RType::INT), + Op::FloatCompare { .. } | Op::RequireField { .. } | Op::FieldIsSet { .. } | Op::HoldsLayout { .. } => true, @@ -730,6 +731,7 @@ mod tests { op: BinOp::Add, lhs: Value::Register(read), rhs: Value::Register(k), + mutation: Mutation::Fresh, }; let copy = builder.new_block(); builder.push(field.clone()); @@ -795,6 +797,7 @@ mod tests { op: BinOp::Add, lhs: Value::Register(field), rhs: Value::Register(k), + mutation: Mutation::Fresh, }); sum = dest; } @@ -883,6 +886,7 @@ mod tests { op: BinOp::Add, lhs: Value::Register(field), rhs: Value::Register(k), + mutation: Mutation::Fresh, }); builder.terminate(Terminator::Return(Value::Register(sum))); let mut m = module(builder.finish()); @@ -1339,6 +1343,7 @@ mod tests { op: BinOp::Add, lhs: Value::Register(part), rhs: Value::Int(1), + mutation: Mutation::Fresh, }); builder.terminate(Terminator::Return(Value::Register(sum))); let mut m = module(builder.finish()); diff --git a/crates/by_opt/src/coalesce.rs b/crates/by_opt/src/coalesce.rs index 6a47238e9d..02abe02ead 100644 --- a/crates/by_opt/src/coalesce.rs +++ b/crates/by_opt/src/coalesce.rs @@ -113,7 +113,7 @@ fn use_counts(function: &Function) -> Vec { mod tests { use by_ir::builder::FunctionBuilder; use by_ir::function::ModuleIr; - use by_ir::ops::{BinOp, Op, Terminator, Value}; + use by_ir::ops::{BinOp, Mutation, Op, Terminator, Value}; use by_ir::rtype::RType; fn module(function: by_ir::function::Function) -> ModuleIr { @@ -143,6 +143,7 @@ mod tests { op: BinOp::Add, lhs: Value::Register(k), rhs: Value::Int(1), + mutation: Mutation::Fresh, }); builder.assign(k, Value::Register(temp)); builder.terminate(Terminator::Return(Value::Register(k))); @@ -165,6 +166,7 @@ mod tests { op: BinOp::Add, lhs: Value::Register(k), rhs: Value::Int(1), + mutation: Mutation::Fresh, }); builder.assign(k, Value::Register(temp)); builder.terminate(Terminator::Return(Value::Register(temp))); @@ -187,6 +189,7 @@ mod tests { op: BinOp::Add, lhs: Value::Register(k), rhs: Value::Int(1), + mutation: Mutation::Fresh, }); builder.assign(k, Value::Register(named)); builder.terminate(Terminator::Return(Value::Register(k))); diff --git a/crates/by_opt/src/copy_propagation.rs b/crates/by_opt/src/copy_propagation.rs index cb19d36b52..ac1904f701 100644 --- a/crates/by_opt/src/copy_propagation.rs +++ b/crates/by_opt/src/copy_propagation.rs @@ -384,7 +384,7 @@ fn retarget(op: &mut Op, new_dest: RegisterId) { mod tests { use super::*; use by_ir::builder::FunctionBuilder; - use by_ir::ops::{BinOp, Terminator}; + use by_ir::ops::{BinOp, Mutation, Terminator}; use by_ir::print::print_function; use by_ir::rtype::RType; use by_ir::verify::verify; @@ -417,6 +417,7 @@ mod tests { op: BinOp::Add, lhs: Value::Register(a), rhs: Value::Int(1), + mutation: Mutation::Fresh, }); builder.assign(x, Value::Register(temp)); builder.terminate(Terminator::Return(Value::Register(x))); @@ -444,6 +445,7 @@ mod tests { op: BinOp::Add, lhs: Value::Register(a), rhs: Value::Int(1), + mutation: Mutation::Fresh, }); builder.assign(x, Value::Register(temp)); // the second read makes the copy load-bearing @@ -452,6 +454,7 @@ mod tests { op: BinOp::Add, lhs: Value::Register(x), rhs: Value::Register(temp), + mutation: Mutation::Fresh, }); builder.terminate(Terminator::Return(Value::Register(x))); let mut m = module(builder.finish()); @@ -471,6 +474,7 @@ mod tests { op: BinOp::Add, lhs: Value::Register(a), rhs: Value::Int(1), + mutation: Mutation::Fresh, }); builder.assign(x, Value::Register(named)); builder.terminate(Terminator::Return(Value::Register(x))); @@ -507,6 +511,7 @@ mod tests { op: BinOp::Add, lhs: Value::Register(x), rhs: Value::Int(1), + mutation: Mutation::Fresh, }); builder.assign(x, Value::Register(temp)); builder.terminate(Terminator::Return(Value::Register(x))); @@ -528,6 +533,7 @@ mod tests { op: BinOp::Add, lhs: Value::Register(a), rhs: Value::Int(1), + mutation: Mutation::Fresh, }); builder.assign(other, Value::Int(9)); builder.assign(x, Value::Register(temp)); @@ -573,12 +579,14 @@ mod tests { op: BinOp::Add, lhs: Value::Register(a), rhs: Value::Int(1), + mutation: Mutation::Fresh, }); builder.push(Op::IntBinary { dest: t2, op: BinOp::Mul, lhs: Value::Register(t1), rhs: Value::Int(2), + mutation: Mutation::Fresh, }); builder.assign(x, Value::Register(t2)); builder.terminate(Terminator::Return(Value::Register(x))); diff --git a/crates/by_opt/src/dead_registers.rs b/crates/by_opt/src/dead_registers.rs index 953ec1a7b3..037cebd0fe 100644 --- a/crates/by_opt/src/dead_registers.rs +++ b/crates/by_opt/src/dead_registers.rs @@ -110,7 +110,7 @@ fn live_registers(function: &Function) -> HashSet { mod tests { use super::*; use by_ir::builder::FunctionBuilder; - use by_ir::ops::{BinOp, Op, Terminator}; + use by_ir::ops::{BinOp, Mutation, Op, Terminator}; use by_ir::print::print_function; use by_ir::rtype::RType; use by_ir::verify::verify; @@ -143,6 +143,7 @@ mod tests { op: BinOp::Add, lhs: Value::Register(a), rhs: Value::Int(1), + mutation: Mutation::Fresh, }); builder.terminate(Terminator::Return(Value::Register(live))); @@ -164,6 +165,7 @@ mod tests { op: BinOp::Add, lhs: Value::Register(a), rhs: Value::Int(1), + mutation: Mutation::Fresh, }); builder.terminate(Terminator::Return(Value::Register(live))); let mut m = module(builder.finish()); @@ -199,6 +201,7 @@ mod tests { op: BinOp::Add, lhs: Value::Register(a), rhs: Value::Int(1), + mutation: Mutation::Fresh, }); builder.terminate(Terminator::Return(Value::Register(out))); let mut m = module(builder.finish()); diff --git a/crates/by_opt/src/field_proofs.rs b/crates/by_opt/src/field_proofs.rs index fd7dba1168..6e058bdf37 100644 --- a/crates/by_opt/src/field_proofs.rs +++ b/crates/by_opt/src/field_proofs.rs @@ -213,7 +213,7 @@ fn prove(function: &mut Function, layouts: &Layouts) { mod tests { use by_ir::builder::FunctionBuilder; use by_ir::function::{ClassIr, FieldDecl, ModuleIr}; - use by_ir::ops::{BinOp, Op, Terminator, Value}; + use by_ir::ops::{BinOp, Mutation, Op, Terminator, Value}; use by_ir::rtype::RType; fn class(fields: &[(&str, bool)]) -> ClassIr { @@ -299,6 +299,7 @@ mod tests { op: BinOp::Add, lhs: Value::Register(first), rhs: Value::Register(second), + mutation: Mutation::Fresh, }); builder.terminate(Terminator::Return(Value::Register(sum))); let mut module = ModuleIr::new("app"); diff --git a/crates/by_opt/src/fold.rs b/crates/by_opt/src/fold.rs index 9723dc6dc9..a0c9304111 100644 --- a/crates/by_opt/src/fold.rs +++ b/crates/by_opt/src/fold.rs @@ -229,6 +229,7 @@ fn fold_op(op: &Op) -> Option { op: binop, lhs: Value::Int(lhs), rhs: Value::Int(rhs), + .. } => fold_int_binary(*binop, *lhs, *rhs).map(|value| Op::Assign { dest: *dest, src: Value::Int(value), @@ -242,6 +243,14 @@ fn fold_op(op: &Op) -> Option { dest: *dest, src: Value::Bit(compare_ints(*cmp, *lhs, *rhs)), }), + // an immediate is an exact `int`, whose truth is whether it is zero + Op::Truthy { + dest, + src: Value::Int(value), + } => Some(Op::Assign { + dest: *dest, + src: Value::Bit(*value != 0), + }), Op::FloatBinary { dest, op: binop, @@ -286,6 +295,14 @@ fn fold_op(op: &Op) -> Option { dest: *dest, src: Value::Int(!value), }), + Op::Unary { + dest, + op: UnaryOp::Pos | UnaryOp::Index, + operand: Value::Int(value), + } => Some(Op::Assign { + dest: *dest, + src: Value::Int(*value), + }), _ => None, } } @@ -575,7 +592,7 @@ fn compare_ints(op: CmpOp, lhs: i64, rhs: i64) -> bool { mod tests { use super::*; use by_ir::builder::FunctionBuilder; - use by_ir::ops::{BlockId, RegisterId}; + use by_ir::ops::{BlockId, Mutation, RegisterId}; use by_ir::print::print_function; use by_ir::rtype::RType; use by_ir::verify::verify; @@ -606,6 +623,7 @@ mod tests { op: BinOp::Add, lhs: Value::Int(2), rhs: Value::Int(3), + mutation: Mutation::Fresh, }); builder.terminate(Terminator::Return(Value::Register(out))); let mut m = module(builder.finish()); @@ -626,6 +644,7 @@ mod tests { op: BinOp::Mul, lhs: Value::Int(i64::MAX), rhs: Value::Int(2), + mutation: Mutation::Fresh, }); builder.terminate(Terminator::Return(Value::Register(out))); let mut m = module(builder.finish()); diff --git a/crates/by_opt/src/guard_loops.rs b/crates/by_opt/src/guard_loops.rs index 56660ac57b..d05358aeea 100644 --- a/crates/by_opt/src/guard_loops.rs +++ b/crates/by_opt/src/guard_loops.rs @@ -708,7 +708,7 @@ fn reachable(function: &Function, start: BlockId) -> HashSet { mod tests { use by_ir::builder::FunctionBuilder; use by_ir::function::{Function, ModuleIr}; - use by_ir::ops::{BinOp, CmpOp, Op, Terminator, Value}; + use by_ir::ops::{BinOp, CmpOp, Mutation, Op, Terminator, Value}; use by_ir::rtype::{IntWidth, RType}; use by_ir::verify::verify; @@ -815,6 +815,7 @@ mod tests { op: BinOp::Add, lhs: Value::Register(i), rhs: Value::Fixed(1), + mutation: Mutation::Fresh, }); builder.terminate(Terminator::Goto(header)); @@ -1032,6 +1033,7 @@ mod tests { op: BinOp::Add, lhs: Value::Register(a), rhs: Value::Register(b), + mutation: Mutation::Fresh, }); add.terminate(Terminator::Return(Value::Register(sum))); diff --git a/crates/by_opt/src/infallible.rs b/crates/by_opt/src/infallible.rs index 39607a510a..8395f7b89e 100644 --- a/crates/by_opt/src/infallible.rs +++ b/crates/by_opt/src/infallible.rs @@ -300,7 +300,11 @@ fn op_can_fail(module: &ModuleIr, function: &by_ir::function::Function, op: &Op) Op::IsInstance { .. } => true, // …and a soundness check raises when its answer is no Op::CheckSound { .. } => true, - Op::FloatBinary { op, .. } => op.can_fail(), + // an `int` on the left converts only while it is short, and asks cpython + // otherwise + Op::FloatBinary { op, lhs, .. } => { + op.can_fail() || function.value_type(lhs) == Some(RType::INT) + } // the object side may be anything, so any of them may raise Op::FloatObjectBinary { .. } | Op::FloatObjectCompare { .. } => true, @@ -313,7 +317,8 @@ fn op_can_fail(module: &ModuleIr, function: &by_ir::function::Function, op: &Op) function.value_type(operand), Some(RType::Primitive(Primitive::Float)) ), - UnaryOp::Invert => true, + // an exact `int` is its own answer, and a subclass asks python + UnaryOp::Invert | UnaryOp::Pos | UnaryOp::Index => true, }, // a call's failure is decided by the callee, in the fixed point above @@ -338,7 +343,7 @@ fn op_can_fail(module: &ModuleIr, function: &by_ir::function::Function, op: &Op) mod tests { use super::*; use by_ir::builder::FunctionBuilder; - use by_ir::ops::{BinOp, CmpOp, Terminator, Value}; + use by_ir::ops::{BinOp, CmpOp, Mutation, Terminator, Value}; use by_ir::rtype::RType; fn module(functions: Vec) -> ModuleIr { @@ -419,6 +424,7 @@ mod tests { op: BinOp::Add, lhs: Value::Register(a), rhs: Value::Register(b), + mutation: Mutation::Fresh, }); builder.terminate(Terminator::Return(Value::Register(out))); let mut m = module(vec![builder.finish()]); diff --git a/crates/by_opt/src/lib.rs b/crates/by_opt/src/lib.rs index 06fe1ed831..400255655b 100644 --- a/crates/by_opt/src/lib.rs +++ b/crates/by_opt/src/lib.rs @@ -202,7 +202,7 @@ mod tests { use super::*; use by_ir::builder::FunctionBuilder; use by_ir::function::CallConvention; - use by_ir::ops::{BinOp, Op, Terminator, Value}; + use by_ir::ops::{BinOp, Mutation, Op, Terminator, Value}; use by_ir::rtype::RType; #[test] @@ -243,6 +243,7 @@ mod tests { op: BinOp::Add, lhs: Value::Register(index), rhs: Value::Int(1), + mutation: Mutation::Fresh, }); builder.terminate(Terminator::Goto(header)); builder.switch_to(exit); diff --git a/crates/by_opt/src/push_lengths.rs b/crates/by_opt/src/push_lengths.rs index b83c82edcf..b312e4ee7c 100644 --- a/crates/by_opt/src/push_lengths.rs +++ b/crates/by_opt/src/push_lengths.rs @@ -234,7 +234,7 @@ fn hold(function: &mut Function, header: BlockId, body: &[BlockId], array: Regis #[cfg(test)] mod tests { use by_ir::builder::FunctionBuilder; - use by_ir::ops::CmpOp; + use by_ir::ops::{CmpOp, Mutation}; use by_ir::verify::verify; use super::*; @@ -291,6 +291,7 @@ mod tests { op: by_ir::ops::BinOp::Add, lhs: Value::Register(i), rhs: Value::Int(1), + mutation: Mutation::Fresh, }); builder.terminate(Terminator::Goto(header)); builder.switch_to(exit); diff --git a/crates/by_opt/src/refcount.rs b/crates/by_opt/src/refcount.rs index 5a530b0e0b..ba14bd5488 100644 --- a/crates/by_opt/src/refcount.rs +++ b/crates/by_opt/src/refcount.rs @@ -96,7 +96,7 @@ fn release_sets(function: &Function) -> Vec> { mod tests { use super::*; use by_ir::builder::FunctionBuilder; - use by_ir::ops::{BinOp, Op, Terminator, Value}; + use by_ir::ops::{BinOp, Mutation, Op, Terminator, Value}; use by_ir::rtype::RType; use by_ir::verify::verify; @@ -228,6 +228,7 @@ mod tests { op: BinOp::Add, lhs: Value::Register(index), rhs: Value::Int(1), + mutation: Mutation::Fresh, }); builder.terminate(Terminator::Goto(header)); builder.switch_to(exit); @@ -254,6 +255,7 @@ mod tests { op: BinOp::Add, lhs: Value::Register(a), rhs: Value::Int(1), + mutation: Mutation::Fresh, }); builder.terminate(Terminator::Return(Value::Register(sum))); let mut m = module(builder.finish()); diff --git a/crates/by_opt/src/release_temporaries.rs b/crates/by_opt/src/release_temporaries.rs index ef2fce151f..ea04a6b059 100644 --- a/crates/by_opt/src/release_temporaries.rs +++ b/crates/by_opt/src/release_temporaries.rs @@ -438,7 +438,7 @@ fn entered_dead( mod tests { use super::*; use by_ir::builder::FunctionBuilder; - use by_ir::ops::Terminator; + use by_ir::ops::{Mutation, Terminator}; use by_ir::rtype::RType; use by_ir::verify::verify; @@ -690,6 +690,7 @@ mod tests { op: by_ir::ops::BinOp::Add, lhs: Value::Register(made), rhs: Value::Int(1), + mutation: Mutation::Fresh, }); builder.terminate(Terminator::Return(Value::Register(answer))); diff --git a/crates/by_opt/src/runs_python.rs b/crates/by_opt/src/runs_python.rs index 974cb9f8f6..061799c84d 100644 --- a/crates/by_opt/src/runs_python.rs +++ b/crates/by_opt/src/runs_python.rs @@ -699,10 +699,13 @@ impl Held { Some(Kind::Int) } Op::Unary { - op: UnaryOp::Neg | UnaryOp::Invert, + op: UnaryOp::Neg | UnaryOp::Invert | UnaryOp::Pos, operand, .. } if exact(operand) == Some(Kind::Int) && *ty == RType::INT => Some(Kind::Int), + Op::Unary { + op: UnaryOp::Index, .. + } => Some(Kind::Int), // a machine value boxed is built as the builtin, and an exact value boxed is // itself Op::TagShort { .. } => Some(Kind::Int), @@ -785,6 +788,11 @@ fn op_itself_runs_python(function: &Function, op: &Op, held: &Held) -> bool { } let kind = |value: &Value| held.exact_kind(function, value); match op { + // an `int` on the left of a double operation answers with its own method, which + // for an exact `int` is the double arithmetic + Op::FloatBinary { lhs, .. } if function.value_type(lhs) == Some(RType::INT) => { + kind(lhs) != Some(Kind::Int) + } // copies and loads of what the frame already holds, comparisons and arithmetic on // machine values, and questions answered by reading a pointer or a flag Op::Assign { .. } @@ -864,10 +872,16 @@ fn op_itself_runs_python(function: &Function, op: &Op, held: &Held) -> bool { } Op::Unary { op, operand, .. } => match op { UnaryOp::Not => false, - UnaryOp::Neg | UnaryOp::Invert => { + UnaryOp::Neg | UnaryOp::Invert | UnaryOp::Pos => { !matches!(kind(operand), Some(Kind::Int | Kind::Float)) } + // an `int` subclass is copied by value, which asks it nothing + UnaryOp::Index => false, }, + // an exact `int` is true where it is not zero, and a subclass has a `__bool__` + Op::Truthy { src, .. } if function.value_type(src) == Some(RType::INT) => { + kind(src) != Some(Kind::Int) + } // an exact `str`, `bytes` or `list` knows its own size Op::Len { src, .. } => !matches!(kind(src), Some(Kind::Str | Kind::Bytes | Kind::List)), // an element of an exact `list` at an index that is an exact `int`, which raises @@ -994,7 +1008,7 @@ fn op_itself_runs_python(function: &Function, op: &Op, held: &Held) -> bool { mod tests { use by_ir::builder::FunctionBuilder; use by_ir::function::{ClassIr, FieldDecl, Function, ModuleIr}; - use by_ir::ops::{BinOp, BlockId, CmpOp, Op, RegisterId, Terminator, Value}; + use by_ir::ops::{BinOp, BlockId, CmpOp, Mutation, Op, RegisterId, Terminator, Value}; use by_ir::rtype::{IntWidth, RType}; use std::collections::BTreeSet; @@ -1155,6 +1169,7 @@ mod tests { op: BinOp::Add, lhs: Value::Register(a), rhs: Value::Register(b), + mutation: Mutation::Fresh, }); builder.terminate(Terminator::Return(Value::Register(out))); builder.finish() @@ -1231,6 +1246,7 @@ mod tests { op: BinOp::Sub, lhs: Value::Register(n), rhs: Value::Int(1), + mutation: Mutation::Fresh, }); builder.push(Op::FunctionStands { dest: stands, @@ -1283,6 +1299,7 @@ mod tests { op: BinOp::Add, lhs: Value::Register(below), rhs: Value::Int(1), + mutation: Mutation::Fresh, }); builder.terminate(Terminator::Return(Value::Register(sum))); @@ -1437,6 +1454,7 @@ mod tests { op: BinOp::Add, lhs: Value::Register(n), rhs: Value::Int(1), + mutation: Mutation::Fresh, }); } let out = builder.temp(RType::BIT); @@ -1482,6 +1500,7 @@ mod tests { op: BinOp::Add, lhs: Value::Register(n), rhs: Value::Int(1), + mutation: Mutation::Fresh, }); builder.assign(held, Value::Int(1)); builder.terminate(Terminator::Return(Value::Float(0.0))); diff --git a/crates/by_opt/src/store_moves.rs b/crates/by_opt/src/store_moves.rs index c99170579e..e38c1df55b 100644 --- a/crates/by_opt/src/store_moves.rs +++ b/crates/by_opt/src/store_moves.rs @@ -340,7 +340,7 @@ fn may_hand_over(function: &Function, register: RegisterId) -> bool { mod tests { use super::*; use by_ir::builder::FunctionBuilder; - use by_ir::ops::{BinOp, Terminator}; + use by_ir::ops::{BinOp, Mutation, Terminator}; use std::collections::BTreeSet; fn module(function: Function) -> ModuleIr { @@ -394,6 +394,7 @@ mod tests { op: BinOp::Add, lhs: Value::Register(k), rhs: Value::Int(1), + mutation: Mutation::Fresh, }); builder.push(store(receiver, sum)); builder.terminate(Terminator::Return(Value::None)); @@ -415,6 +416,7 @@ mod tests { op: BinOp::Add, lhs: Value::Register(k), rhs: Value::Int(1), + mutation: Mutation::Fresh, }); builder.push(store(receiver, sum)); builder.terminate(Terminator::Return(Value::Register(sum))); @@ -437,6 +439,7 @@ mod tests { op: BinOp::Add, lhs: Value::Register(k), rhs: Value::Int(1), + mutation: Mutation::Fresh, }); builder.push(store(receiver, named)); builder.terminate(Terminator::Return(Value::None)); @@ -557,6 +560,7 @@ mod tests { op, lhs: Value::Register(p), rhs: Value::Int(7), + mutation: Mutation::Fresh, }); } builder.push(Op::TupleBuild { diff --git a/crates/by_opt/src/unbox_counters.rs b/crates/by_opt/src/unbox_counters.rs index 886c289cc9..c65881eb41 100644 --- a/crates/by_opt/src/unbox_counters.rs +++ b/crates/by_opt/src/unbox_counters.rs @@ -344,6 +344,7 @@ pub(crate) fn unbox( op: step, lhs, rhs, + .. } = &op { let resume = resumes[&(*original, index)]; @@ -652,7 +653,9 @@ fn served( | Op::GetItem { index, .. } | Op::ArrayGet { index, .. } | Op::ArraySet { index, .. } => counter(index).into_iter().collect(), - Op::IntToFloat { src, .. } => counter(src).into_iter().collect(), + Op::IntToFloat { src, .. } | Op::FloatBinary { lhs: src, .. } => { + counter(src).into_iter().collect() + } Op::Box { dest, src } if function .register(*dest) @@ -761,7 +764,7 @@ pub(crate) fn narrow_reads( for (op, fixed_lhs) in function.blocks[block.index()].ops.iter_mut().zip(fixed_lhs) { match op { Op::IntCompare { rhs, .. } if fixed_lhs => narrow(rhs), - Op::IntToFloat { src, .. } => narrow(src), + Op::IntToFloat { src, .. } | Op::FloatBinary { lhs: src, .. } => narrow(src), _ => {} } } @@ -992,6 +995,7 @@ fn never_past(function: &Function, counter: RegisterId, side: Side) -> bool { #[cfg(test)] mod tests { use by_ir::builder::FunctionBuilder; + use by_ir::ops::Mutation; use by_ir::verify::verify; use super::*; @@ -1028,6 +1032,7 @@ mod tests { op: BinOp::Add, lhs: Value::Register(counter), rhs: Value::Int(1), + mutation: Mutation::Fresh, }); builder.terminate(Terminator::Goto(header)); builder.switch_to(exit); @@ -1167,12 +1172,14 @@ mod tests { op: BinOp::Add, lhs: Value::Register(n), rhs: Value::Int(1), + mutation: Mutation::Fresh, }); builder.push(Op::IntBinary { dest: product, op: BinOp::Mul, lhs: Value::Register(n), rhs: Value::Int(2), + mutation: Mutation::Fresh, }); let added = match source { "counter" => n, @@ -1184,6 +1191,7 @@ mod tests { op: BinOp::Add, lhs: Value::Register(m), rhs: Value::Register(added), + mutation: Mutation::Fresh, }); builder.terminate(Terminator::Goto(header)); builder.switch_to(exit); @@ -1248,6 +1256,7 @@ mod tests { op: BinOp::Add, lhs: Value::Register(counter), rhs: Value::Int(1), + mutation: Mutation::Fresh, }); builder.terminate(Terminator::Goto(header)); builder.switch_to(exit); diff --git a/crates/by_opt/src/unswitch.rs b/crates/by_opt/src/unswitch.rs index 3d73d44396..f59668a82c 100644 --- a/crates/by_opt/src/unswitch.rs +++ b/crates/by_opt/src/unswitch.rs @@ -288,7 +288,7 @@ fn reachable(function: &Function, start: BlockId, reverse: bool) -> Vec pub(crate) mod tests { use super::*; use by_ir::builder::FunctionBuilder; - use by_ir::ops::{BinOp, CmpOp, RegisterId, Value}; + use by_ir::ops::{BinOp, CmpOp, Mutation, RegisterId, Value}; use by_ir::rtype::RType; use by_ir::verify::verify; @@ -339,6 +339,7 @@ pub(crate) mod tests { op: BinOp::Add, lhs: Value::Register(index), rhs: Value::Int(1), + mutation: Mutation::Fresh, }); builder.assign(index, Value::Register(next)); if body_writes_bound { @@ -347,6 +348,7 @@ pub(crate) mod tests { op: BinOp::Mul, lhs: Value::Register(bound), rhs: Value::Int(1), + mutation: Mutation::Fresh, }); } builder.terminate(Terminator::Goto(header)); @@ -560,6 +562,7 @@ pub(crate) mod tests { op: BinOp::Mul, lhs: Value::Register(rows), rhs: Value::Int(1), + mutation: Mutation::Fresh, }); } builder.terminate(Terminator::Goto(inner)); @@ -593,6 +596,7 @@ pub(crate) mod tests { op: BinOp::Add, lhs: Value::Register(x), rhs: Value::Int(1), + mutation: Mutation::Fresh, }); builder.assign(x, Value::Register(next_x)); builder.terminate(Terminator::Goto(inner)); @@ -603,6 +607,7 @@ pub(crate) mod tests { op: BinOp::Add, lhs: Value::Register(y), rhs: Value::Int(1), + mutation: Mutation::Fresh, }); builder.assign(y, Value::Register(next_y)); builder.terminate(Terminator::Goto(outer)); @@ -707,6 +712,7 @@ pub(crate) mod tests { op: BinOp::Add, lhs: Value::Register(index), rhs: Value::Int(1), + mutation: Mutation::Fresh, }); builder.assign(index, Value::Register(next)); }; @@ -719,6 +725,7 @@ pub(crate) mod tests { op: BinOp::Mul, lhs: Value::Register(index), rhs: Value::Int(2), + mutation: Mutation::Fresh, }); if !stepped_first { step(&mut builder); diff --git a/crates/by_rt/include/by.h b/crates/by_rt/include/by.h index 6acdb23da2..c80764d540 100644 --- a/crates/by_rt/include/by.h +++ b/crates/by_rt/include/by.h @@ -100,6 +100,12 @@ static inline int By_InterpreterMatches(void) { * a ByTagged is a pointer-sized word. an even value is a "short": the integer * shifted left by one. an odd value is a PyLongObject pointer with the low bit * set. python's arbitrary precision is preserved — the tag is only a fast path. + * + * a short is always an exact `int`. a pointer is either an exact `int` too wide to be + * a short, or an instance of an `int` subclass — `True`, or a class of the program's + * own — whatever its value, because that object is what python hands back and what + * python asks for its methods. so a pointer does *not* mean a value outside the short + * range: a fast path may conclude that only from a pointer to an exact `int` */ typedef size_t ByTagged; @@ -263,6 +269,12 @@ BY_HOT PyObject *By_LongOf(ByTagged x) { return (PyObject *)(x & ~BY_INT_TAG); } +/* whether a tagged integer holds exactly an `int` rather than a subclass: a short always + * does, and a pointer is asked its type */ +static inline char By_IsExactInt(ByTagged x) { + return (char)(By_IsShort(x) || PyLong_CheckExact(By_LongOf(x))); +} + /* `By_DecRefTagged` and `By_IncRefTagged` release the sentinel by handing it to * `By_LongOf` and letting the X-forms drop the NULL, rather than comparing against * `BY_INT_ERROR` a second time. that only works while the sentinel carries no object @@ -302,7 +314,7 @@ static inline PyObject *By_BoxIntOwned(ByTagged x) { * * the tag is added and no reference is taken: like a string literal, the module owns * it and a use of it is borrowed. the value lies outside the short range by - * construction, which is the invariant every tagged pointer carries */ + * construction, which is what a pointer to an exact `int` always holds */ BY_HOT ByTagged By_TaggedLiteral(PyObject *o) { return ((ByTagged)(void *)o) | BY_INT_TAG; } @@ -514,16 +526,28 @@ BY_REFUSAL void By_SoundViolationOf(PyObject *o, PyObject *target) { #define BY_DIGITS_PAST_SHORT ((uintptr_t)((sizeof(Py_ssize_t) * 8 - 2) / PyLong_SHIFT + 2)) #endif -/* takes a new reference to `o` when it cannot be represented as a short +/* everything [`By_TaggedFromLong`] is handed that is not an exact `int` * - * anything that is not an `int` raises the `TypeError` an unbox of it raises, and the - * error path never clears: an operator on a big `int` subclass can answer with any - * object at all, and swallowing the failure here once tagged a `str` as an int */ -static inline ByTagged By_TaggedFromLong(PyObject *o) { - if (BY_UNLIKELY(o == NULL || !PyLong_Check(o))) { + * an instance of an `int` subclass is held as itself, with a reference of its own, + * whatever its value. `True` is the one every program meets: an `int` register that + * narrowed it to the short 1 answered `1` where python answers `True`, and a subclass + * lost its methods the same way. anything that is not an `int` at all raises the + * `TypeError` an unbox of it raises, and the error path never clears: an operator on + * an `int` subclass can answer with any object at all, and swallowing the failure here + * once tagged a `str` as an int */ +BY_REFUSAL ByTagged By_TaggedFromOther(PyObject *o) { + if (o == NULL || !PyLong_Check(o)) { By_TypeError("int", o); return BY_INT_ERROR; } + By_IncRef(o); + return ((ByTagged)(void *)o) | BY_INT_TAG; +} + +/* takes a new reference to `o` when it cannot be represented as a short, which an + * exact `int` can whenever its value fits and a subclass never can */ +static inline ByTagged By_TaggedFromLong(PyObject *o) { + if (BY_UNLIKELY(o == NULL || !PyLong_CheckExact(o))) return By_TaggedFromOther(o); #if PY_VERSION_HEX >= 0x030C0000 /* almost every `int` a program unboxes is one that fits a single digit, and * python 3.12 onwards stores such a value in the object's own header rather @@ -645,6 +669,29 @@ BY_COLD ByTagged By_IntSlowBitwise(ByTagged a, ByTagged b, char op) { return tagged; } +/* `a = b` where `a` is not a short: python asks `a` for its in-place method before + * its plain one, and an `int` subclass may define one of its own. `operation` is the + * abstract api's in-place form, which makes both asks, so an exact `int` — which has no + * in-place methods — gets exactly what the plain operator's slow path would give it */ +BY_COLD ByTagged By_IntSlowInPlace(ByTagged a, ByTagged b, binaryfunc operation) { + PyObject *left = By_BoxInt(a); + if (left == NULL) return BY_INT_ERROR; + PyObject *right = By_BoxInt(b); + if (right == NULL) { Py_DECREF(left); return BY_INT_ERROR; } + PyObject *result = operation(left, right); + Py_DECREF(left); + Py_DECREF(right); + if (result == NULL) return BY_INT_ERROR; + ByTagged tagged = By_TaggedFromLong(result); + Py_DECREF(result); + return tagged; +} + +/* `**=` as a two-operand in-place operation, which is the form the slow path calls */ +static inline PyObject *By_NumberInPlacePower(PyObject *a, PyObject *b) { + return PyNumber_InPlacePower(a, b, Py_None); +} + /* the fast paths alone, for a caller that branches on them: each answers 1 with the * result written where both operands are shorts and so is the result, and 0 where the * slow path has to decide. a short is an even word and each of these results is one @@ -800,12 +847,12 @@ BY_HOT ByTagged By_IntMod(ByTagged a, ByTagged b) { return By_IntSlowBinary(a, b, "%"); } -static inline double By_IntTrueDiv(ByTagged a, ByTagged b) { +static inline double By_IntTrueDivWith(ByTagged a, ByTagged b, binaryfunc operation) { PyObject *left = By_BoxInt(a); if (left == NULL) return BY_FLOAT_ERROR; PyObject *right = By_BoxInt(b); if (right == NULL) { Py_DECREF(left); return BY_FLOAT_ERROR; } - PyObject *result = PyNumber_TrueDivide(left, right); + PyObject *result = operation(left, right); Py_DECREF(left); Py_DECREF(right); if (result == NULL) return BY_FLOAT_ERROR; @@ -839,8 +886,54 @@ static inline ByTagged By_IntPow(ByTagged a, ByTagged b) { return By_IntSlowBitwise(a, b, 'p'); } -/* `~a` is `-1 - a`, which the tagged subtraction already handles */ +static inline double By_IntTrueDiv(ByTagged a, ByTagged b) { + return By_IntTrueDivWith(a, b, PyNumber_TrueDivide); +} + +/* the augmented assignments. an exact `int` on the left has no in-place method, so the + * operation is the plain operator's, fast path and all; anything else is asked its own + * in-place method first. `/=` always goes through the protocol, so it asks it directly */ +#define BY_DEFINE_INT_IN_PLACE(name, plain, operation) \ + static inline ByTagged name(ByTagged a, ByTagged b) { \ + if (BY_UNLIKELY(!By_IsExactInt(a))) return By_IntSlowInPlace(a, b, operation); \ + return plain(a, b); \ + } + +BY_DEFINE_INT_IN_PLACE(By_IntIAdd, By_IntAdd, PyNumber_InPlaceAdd) +BY_DEFINE_INT_IN_PLACE(By_IntISub, By_IntSub, PyNumber_InPlaceSubtract) +BY_DEFINE_INT_IN_PLACE(By_IntIMul, By_IntMul, PyNumber_InPlaceMultiply) +BY_DEFINE_INT_IN_PLACE(By_IntIFloorDiv, By_IntFloorDiv, PyNumber_InPlaceFloorDivide) +BY_DEFINE_INT_IN_PLACE(By_IntIMod, By_IntMod, PyNumber_InPlaceRemainder) +BY_DEFINE_INT_IN_PLACE(By_IntIPow, By_IntPow, By_NumberInPlacePower) +BY_DEFINE_INT_IN_PLACE(By_IntIAnd, By_IntAnd, PyNumber_InPlaceAnd) +BY_DEFINE_INT_IN_PLACE(By_IntIOr, By_IntOr, PyNumber_InPlaceOr) +BY_DEFINE_INT_IN_PLACE(By_IntIXor, By_IntXor, PyNumber_InPlaceXor) +BY_DEFINE_INT_IN_PLACE(By_IntIShl, By_IntShl, PyNumber_InPlaceLshift) +BY_DEFINE_INT_IN_PLACE(By_IntIShr, By_IntShr, PyNumber_InPlaceRshift) + +static inline double By_IntITrueDiv(ByTagged a, ByTagged b) { + return By_IntTrueDivWith(a, b, PyNumber_InPlaceTrueDivide); +} + +/* a unary operator on an `int` subclass, which answers through the subclass's own + * method: `-a` is `type(a).__neg__(a)`, which the subtraction the exact forms reduce to + * would have taken to `__rsub__` instead */ +BY_COLD ByTagged By_IntUnarySlow(ByTagged a, PyObject *(*operation)(PyObject *)) { + PyObject *operand = By_BoxInt(a); + PyObject *result; + ByTagged tagged; + if (operand == NULL) return BY_INT_ERROR; + result = operation(operand); + Py_DECREF(operand); + if (result == NULL) return BY_INT_ERROR; + tagged = By_TaggedFromLong(result); + Py_DECREF(result); + return tagged; +} + +/* `~a` is `-1 - a` for an exact `int`, which the tagged subtraction already handles */ static inline ByTagged By_IntInvert(ByTagged a) { + if (BY_UNLIKELY(!By_IsExactInt(a))) return By_IntUnarySlow(a, PyNumber_Invert); return By_IntSub(By_ShortFrom(-1), a); } @@ -849,10 +942,43 @@ static inline ByTagged By_IntNeg(ByTagged a) { Py_ssize_t x = By_ShortValue(a); /* the short range is one wider below zero than above it */ if (x != BY_SHORT_MIN) return By_ShortFrom(-x); + } else if (BY_UNLIKELY(!By_IsExactInt(a))) { + return By_IntUnarySlow(a, PyNumber_Negative); } return By_IntSub(By_ShortFrom(0), a); } +/* `+a`, which is `a` itself for an exact `int` and `int.__pos__`'s exact copy, or a + * method of the subclass's own, for anything else */ +static inline ByTagged By_IntPos(ByTagged a) { + if (BY_UNLIKELY(!By_IsExactInt(a))) return By_IntUnarySlow(a, PyNumber_Positive); + By_IncRefTagged(a); + return a; +} + +/* `operator.index(a)`: the exact `int` a subclass stands for, copied by value, which is + * how `range` reads its bounds. `PyNumber_Index` asks an `int` subclass nothing of its + * own — not even `__index__` — and only copies it */ +static inline ByTagged By_IntIndex(ByTagged a) { + if (BY_UNLIKELY(!By_IsExactInt(a))) return By_IntUnarySlow(a, PyNumber_Index); + By_IncRefTagged(a); + return a; +} + +/* whether an `int` behind a pointer is true: always, for an exact one, and its own + * `__bool__` for a subclass. 2 is an error. a short is tested where it is read */ +BY_COLD char By_IntTruthySlow(ByTagged a) { + PyObject *operand; + int answer; + /* a pointer to an exact `int` holds a value too wide to be a short, so not zero */ + if (By_IsExactInt(a)) return 1; + operand = By_BoxInt(a); + if (operand == NULL) return 2; + answer = PyObject_IsTrue(operand); + Py_DECREF(operand); + return answer < 0 ? 2 : (char)answer; +} + /* ── int comparison ───────────────────────────────────────────────────────── */ BY_COLD char By_IntCompareSlow(ByTagged a, ByTagged b, int op) { @@ -986,6 +1112,47 @@ BY_COLD double By_FloatObjectSlow(double a, PyObject *b, return value; } +/* `a b` for an `int` on the left of a double, where `a` is not a short + * + * python asks the `int` first. an exact one declines a double and the double's reflected + * method converts it, which is what the operation does with a short and what the + * protocol does here with a value too wide for one, `OverflowError` included. a subclass + * may answer with a method of its own, so it is asked. the checker has said the answer + * is a `float`, and an answer that is not one is refused rather than converted */ +BY_COLD double By_IntFloatSlow(ByTagged a, double b, char op) { + PyObject *left = By_BoxInt(a); + PyObject *right; + PyObject *result = NULL; + double value; + if (left == NULL) return BY_FLOAT_ERROR; + right = PyFloat_FromDouble(b); + if (right == NULL) { + Py_DECREF(left); + return BY_FLOAT_ERROR; + } + switch (op) { + case '+': result = PyNumber_Add(left, right); break; + case '-': result = PyNumber_Subtract(left, right); break; + case '*': result = PyNumber_Multiply(left, right); break; + case '/': result = PyNumber_TrueDivide(left, right); break; + case 'f': result = PyNumber_FloorDivide(left, right); break; + case '%': result = PyNumber_Remainder(left, right); break; + case 'p': result = PyNumber_Power(left, right, Py_None); break; + default: PyErr_SetString(PyExc_SystemError, "unknown float operation"); break; + } + Py_DECREF(left); + Py_DECREF(right); + if (result == NULL) return BY_FLOAT_ERROR; + if (!PyFloat_Check(result)) { + By_TypeError("float", result); + Py_DECREF(result); + return BY_FLOAT_ERROR; + } + value = PyFloat_AS_DOUBLE(result); + Py_DECREF(result); + return value; +} + /* `a b` with one side a proven double and the other an object that can only * be an int or a float * @@ -1231,12 +1398,13 @@ static inline PyObject *By_BoxNone(void) { } /* unboxing is a *narrowing*, so it is always checked — this is the - * representation invariant's inserted check, not an assumption */ + * representation invariant's inserted check, not an assumption + * + * the exact-`int` test the conversion opens with is the check: what fails it is tested + * again out of line, and only there is anything refused. asking `PyLong_Check` first as + * well made the narrowing too big for clang to inline into a loop, and `dictget`, which + * narrows two values a trip, retired 5% more instructions for the call */ static inline ByTagged By_UnboxInt(PyObject *o) { - if (o == NULL || !PyLong_Check(o)) { - By_TypeError("int", o); - return BY_INT_ERROR; - } return By_TaggedFromLong(o); } @@ -1424,6 +1592,7 @@ static inline PyObject *By_ObjIShr(PyObject *a, PyObject *b) { static inline PyObject *By_ObjNeg(PyObject *o) { return PyNumber_Negative(o); } static inline PyObject *By_ObjInvert(PyObject *o) { return PyNumber_Invert(o); } +static inline PyObject *By_ObjPos(PyObject *o) { return PyNumber_Positive(o); } /* ── the two questions python's comparison can be answered without asking ────── * @@ -10042,10 +10211,12 @@ static inline Py_ssize_t By_ArrayIndexIn(ByArrayHeader *array, int64_t at, return index; } -/* an index held tagged. a big integer cannot be a valid index into a buffer, but which - * error it raises depends on whether it fits `Py_ssize_t`: python converts the index - * before it compares it with the length */ -BY_COLD Py_ssize_t By_ArrayIndexBig(ByTagged tagged, const char *out_of_range) { +/* an index held tagged as an object: an exact `int` too big to index a buffer, or an + * `int` subclass — `True` among them — which indexes by its value like any other. which + * error a big one raises depends on whether it fits `Py_ssize_t`: python converts the + * index before it compares it with the length */ +BY_COLD Py_ssize_t By_ArrayIndexObject(ByArrayHeader *array, ByTagged tagged, + const char *out_of_range) { PyObject *big = By_LongOf(tagged); Py_ssize_t value = PyLong_AsSsize_t(big); if (value == -1 && PyErr_Occurred()) { @@ -10055,14 +10226,15 @@ BY_COLD Py_ssize_t By_ArrayIndexBig(ByTagged tagged, const char *out_of_range) { Py_TYPE(big)->tp_name); return -1; } - PyErr_SetString(PyExc_IndexError, out_of_range); - return -1; + return By_ArrayIndexIn(array, value, out_of_range); } static inline Py_ssize_t By_ArrayIndexTaggedIn(ByArrayHeader *array, ByTagged tagged, const char *out_of_range) { if (array == NULL) return -1; - if (BY_UNLIKELY(!By_IsShort(tagged))) return By_ArrayIndexBig(tagged, out_of_range); + if (BY_UNLIKELY(!By_IsShort(tagged))) { + return By_ArrayIndexObject(array, tagged, out_of_range); + } return By_ArrayIndexIn(array, By_ShortValue(tagged), out_of_range); } @@ -10954,11 +11126,6 @@ static inline char By_BuiltinStandsOnEntry(ByBuiltinSite *site, PyObject *dict, #endif } -/* whether a tagged integer holds exactly an `int` rather than a subclass: a short always - * does, and a pointer is asked its type */ -static inline char By_IsExactInt(ByTagged x) { - return (char)(By_IsShort(x) || PyLong_CheckExact(By_LongOf(x))); -} /* `str(n)` for a tagged integer, given whatever the name `str` resolved to * diff --git a/docs/basedpython/development/compilation/plan.md b/docs/basedpython/development/compilation/plan.md index 4770c25edd..2173b72bc3 100644 --- a/docs/basedpython/development/compilation/plan.md +++ b/docs/basedpython/development/compilation/plan.md @@ -14,8 +14,8 @@ short for that reason: | monkey-patching a module function | works | works, unless the project binds early | a compiled call asks whether the name still holds the function; see [runtime](runtime.md#a-module-function-rebound-from-outside) | | adding an attribute not in the class body | stored, unless `__slots__` | the same, on 3.13 and above | an instance dict beside the layout; below 3.13 it is refused | | `is` on a fixed-length tuple | identity | unspecified | unboxed tuples have no identity | -| `type(x)` where `x: int` held a `bool` | `bool` | `int` | the tagged form has no room for it | | `is` on a small `int` or interned `str` | identity | unspecified | unboxed and re-boxed values differ | +| a `float` subclass read into a `float` place | keeps its type and its methods | narrowed to its value | a double has nothing to point at; see [runtime](runtime.md#a-float-subclass-read-into-a-place-declared-float) | | a wrong annotation on non-compiled code | `TypeError` (soundness checks) | `TypeError` | same mechanism, already shipped | | `__dict__` on an instance | present unless `__slots__` | present, over the whole state | a real `dict` every write updates; `del` of a *field* raises | | `if __name__ == "__main__"` | n/a | n/a | the entry point is [`main`](../../features/main-function.md) | diff --git a/docs/basedpython/development/compilation/planned-features.md b/docs/basedpython/development/compilation/planned-features.md index f81494894d..3772bcb5c2 100644 --- a/docs/basedpython/development/compilation/planned-features.md +++ b/docs/basedpython/development/compilation/planned-features.md @@ -48,11 +48,12 @@ and `local` together are the complete precondition: `local` proves it does not escape, `final` proves how big it is. neither is sufficient alone, and the existing design only had the first -**a semantic delta gets smaller.** [plan](plan.md#semantic-deltas) records that -an `int`-typed register loses the `True`/`1` distinction, because the tagged -representation has no room for it. `final int` rejects `True` at the checker, so -a `final int` place cannot observe the divergence at all. the delta survives, but -it now has a spelling that opts out of it +**an exactness test goes away.** an `int`-typed register holds a `bool` or an +`int` subclass as the object itself, and an operation on a value behind the pointer +asks whether it is an exact `int` before it takes the value's arithmetic +([runtime](runtime.md#bool-and-int-subclasses)). `final int` rejects `True` at the +checker, so a `final int` place is exact, and once its check is in place (below) +those tests have nothing left to ask ### the trap: exactness is checked, not proven @@ -365,7 +366,7 @@ representation carries `single` unchanged, which is the pleasant part classes; `final` + `local` is the complete `StackAlloc` precondition; `literal` adds a value-keyed monomorphization axis; `single` extends the tagged-union representation to generics -- [plan](plan.md) — the `True`/`1` delta acquires an opt-out; the shared member +- [plan](plan.md) — an exact `int` place drops the subclass tests; the shared member cutoff is a new cross-component invariant to test - nothing in [technology](technology.md) or [runtime](runtime.md) moves. all five modifiers are erase-only, so the transpiled python is unchanged and the diff --git a/docs/basedpython/development/compilation/runtime.md b/docs/basedpython/development/compilation/runtime.md index 4818e60b10..ec89b53764 100644 --- a/docs/basedpython/development/compilation/runtime.md +++ b/docs/basedpython/development/compilation/runtime.md @@ -392,10 +392,58 @@ integer is a plain `int8_t`…`int64_t` with no tag and no overflow branch. this is the representation the numeric loops actually want, and the annotations that select it are ordinary basedpython -one observable consequence, inherited from mypyc: an `int`-typed register loses -the distinction between `True` and `1`, because `bool` is an `int` subclass and -the tagged form has no room for it. covered in -[semantic deltas](plan.md#semantic-deltas) +### `bool` and `int` subclasses + +a short is always an exact `int`. a `bool`, or an instance of any other `int` +subclass, is held behind the pointer as the object itself, whatever its value — so +the pointer means "an exact `int` too wide to be short, or a subclass", never just +"too wide". that is what keeps python's answers: + +```python +class Id(int): + def __repr__(self) -> str: + return f"Id({int(self)})" + +def first(xs: list[int]) -> int: + return xs[0] + +first([True]) # True, not 1 +first([Id(5)]) # Id(5), not 5 +``` + +and every operation on such a value is its type's own method: `+`, `-`, `==`, `<`, +`hash`, `repr`, `str`, a format, `-x`, `+x`, `~x`, truthiness (`__bool__`, never +`!= 0`), and an `int` on the left of a mixed `int`/`float` operation. a short never +reaches any of them, so the fast paths are unchanged. a counted `range` reads its +bounds through `operator.index`, which copies a subclass to the plain `int` of its +value, as `range` itself does + +mypyc narrows every `int` to its value at the boundary and so answers `1` and `5` +above; this compiler used to do the same + +an augmented assignment asks for the in-place method first, as python does. `int` +and `bool` have no in-place methods, so `x += 1` is the plain operator's fast path +while both sides are shorts, and only the slow path offers the left operand its +`__iadd__` — or `__isub__`, `__imul__` and the rest — before its `__add__`: + +```python +class Acc(int): + def __add__(self, other: int) -> "Acc": + return Acc(int(self) + other) + + def __iadd__(self, other: int) -> "Acc": + return Acc(int(self) + 2 * other) + +def bump(xs: list[int]) -> int: + x = xs[0] + x += 1 # Acc.__iadd__, as in python: Acc(3) + return x + +bump([Acc(1)]) +``` + +`//=`, `%=`, `**=`, `<<=` and `>>=` ask whether the left operand is an exact `int` +before they take the plain operator's path, which is one more test on a short ## floats, strings, tuples @@ -1178,6 +1226,49 @@ root(-4.0) # python: (1.2246467991473532e-16+2j); compiled: TypeError an augmented assignment is not one of these: `x **= 0.5` binds the power's result, and the local is chosen to hold whatever that can be +### a `float` subclass read into a place declared `float` + +a place declared `float` holds an unboxed double, and a double has nothing to point at: an +`int` subclass is kept behind the tagged word's pointer (see +[`bool` and `int` subclasses](#bool-and-int-subclasses)), but a `float` has no such arm. so +an instance of a `float` subclass read out of an object into such a place is narrowed to +its value, and it loses its type and every method it overrides: + +```python +class Money(float): + def __add__(self, other: float) -> "Money": + return Money(float(self) + other) + + +def first(xs: list[float]) -> float: + return xs[0] + + +def bump(xs: list[float]) -> float: + return xs[0] + 1.0 + + +first([Money(1.0)]) # python: Money(1.0); compiled: 1.0 +bump([Money(1.0)]) # python: Money(2.0); compiled: 2.0 +``` + +the places that narrow are every read of a `float` out of an object: a list, tuple or dict +element, a loop over one, an unpacking, a comprehension that rebuilds one, a call's result, +an attribute, a field of a class the module lays out, and a method's or a nested function's +parameter. a module-level function's parameter does not: its boundary tests each call and +hands one passing a subclass to the interpreted definition + +`numpy.float64` is a `float` subclass, and the commonest one: its arithmetic is the same +IEEE arithmetic, so a compiled answer has the same value and is a `float` where python's is a +`numpy.float64` + +holding each such read as an object instead was priced with a `.py` module built without +`strict-float`, whose `float` places are `int | float` and held that way, against the same +module with it, in instructions retired: `dot` 5.5x, `prefix` 10x, `objects` 7.7x, `generic` +15x and `comp` 1.17x, where `mandel`, `mandel_inline` and `calls`, which read no float out of +an object, are unchanged. without `strict-float` a place is only held as an object until +it meets a `float` in an arithmetic operation, which unboxes it there + ### what a closure keeps alive python gives each name a closure captures a cell of its own, so a closure keeps alive the From 06c7b16890c4dc56a2dc08ccfaa49890bf50c1b7 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Thu, 24 Sep 2026 04:31:17 +1000 Subject: [PATCH 16/16] native: compile marked methods, follow class rebinding and patching, and slot data classes methods marked override, final or abstractmethod compile, and the report counts what really runs; a method the class body binds again publishes what it bound; a module body that writes over a compiled class's members leaves the module interpreted; one decision says which classes are mutable, including a written __new__; a data class declares its slots, publishes __slots__ and its pickling state, and is compared against the class its decorators produced; __class__ reads the defining class; the header compiles below 3.13 --- crates/by_build/src/annotate.rs | 130 +- crates/by_build/src/lib.rs | 47 +- crates/by_build/tests/differential.rs | 1611 ++++++++++++++++- crates/by_codegen_c/src/lib.rs | 430 +++-- crates/by_ir/src/function.rs | 360 +++- crates/by_irbuild/src/lib.rs | 363 +++- crates/by_irbuild/src/tests.rs | 202 ++- crates/by_opt/src/field_proofs.rs | 5 +- crates/by_opt/src/fold.rs | 20 +- crates/by_opt/src/lib.rs | 5 +- crates/by_opt/src/runs_python.rs | 5 +- crates/by_rt/include/by.h | 478 ++++- .../development/compilation/runtime.md | 133 +- scripts/native-sweeps/installcensus.sh | 51 +- 14 files changed, 3602 insertions(+), 238 deletions(-) diff --git a/crates/by_build/src/annotate.rs b/crates/by_build/src/annotate.rs index dea938c24f..7ffaca937e 100644 --- a/crates/by_build/src/annotate.rs +++ b/crates/by_build/src/annotate.rs @@ -8,6 +8,7 @@ //! the "why not" half is the point. a compiler whose failures are silent is a //! compiler nobody can tune against. +use std::collections::HashSet; use std::fmt::Write; use by_ir::function::{Function, ModuleIr}; @@ -17,17 +18,28 @@ use by_ir::print::print_function; pub(crate) fn report(module: &ModuleIr) -> String { let mut out = format!("# {}\n", module.name.dotted()); - let native: Vec<&Function> = module.all_functions().collect(); + // a function that compiled and is never installed is counted where it runs, and + // that is the interpreted definition + let unreached: HashSet = module + .unreached() + .into_iter() + .map(|unreached| unreached.name) + .collect(); + let native: Vec<&Function> = module + .all_functions() + .filter(|function| !unreached.contains(&function.qualified_name())) + .collect(); + let left = crate::left_interpreted(module); let _ = writeln!( out, "\n{} compiled, {} left interpreted\n", native.len(), - module.declined.len() + left.len() ); - if !module.declined.is_empty() { + if !left.is_empty() { out.push_str("## left to the interpreted definition\n\n"); - for declined in &module.declined { + for declined in &left { let _ = writeln!(out, "- {}: {}", declined.name, declined.reason); } out.push('\n'); @@ -278,6 +290,116 @@ def total(xs: list[float], k: float) -> float: assert!(!text.contains("numeric promotion cost"), "{text}"); } + #[test] + fn a_method_whose_class_installs_the_body_answer_is_counted_as_interpreted() { + // the class holds what the interpreted body's `@mark` handed back, so neither the + // compiled `get` nor the closure only it makes ever runs + let module = lowered( + "\ +def mark(f): + return f + +class Box: + @mark + def get(self, x: int) -> int: + def add(k: int) -> int: + return k + x + + return add(1) + + def plain(self) -> int: + return 1 +", + ); + let text = report(&module); + assert!(text.contains("2 compiled, 2 left interpreted"), "{text}"); + assert!( + text.contains("- Box.get: decorated in the class body"), + "{text}" + ); + assert!( + text.contains("- Box$get$env.add: only `Box.get` reaches this"), + "{text}" + ); + assert!(!text.contains("### Box.get"), "{text}"); + assert!(text.contains("### Box.plain"), "{text}"); + } + + #[test] + fn a_method_the_class_body_binds_again_is_reported_for_that() { + // the class holds what `get = other` left, and the report says so rather than + // blaming a decorator the method does not have + let module = lowered( + "\ +def other(self) -> int: + return 2 + +class Box: + def get(self) -> int: + return 1 + + get = other +", + ); + let text = report(&module); + assert!(text.contains("1 compiled, 1 left interpreted"), "{text}"); + assert!( + text.contains( + "- Box.get: its name is bound again in the class body, so the class holds what that binding left and this one is never installed" + ), + "{text}" + ); + } + + #[test] + fn a_property_on_a_class_its_metaclass_builds_is_counted_as_interpreted() { + // `Number` writes a metaclass, so `Real`'s type spec is refused and `ABCMeta` is + // called instead — with the `property` the interpreted body built, which is what + // the class keeps. `Plain` stands on nothing of the kind, and its own is compiled + let mut module = by_irbuild::module_from_source( + "\ +from abc import ABCMeta + + +class Number(metaclass=ABCMeta): + pass + + +class Real(Number): + @property + def real(self) -> int: + return 1 + + def plain(self) -> int: + return 2 + + +class Root: + pass + + +class Plain(Root): + @property + def real(self) -> int: + return 1 +", + "app", + by_irbuild::Language::Python, + ); + by_opt::optimize(&mut module).expect("the pipeline verifies"); + let text = report(&module); + assert!( + text.contains( + "- Real.real$get: a half of a property on a class built by calling its metaclass" + ), + "{text}" + ); + assert!(!text.contains("### Real.real$get"), "{text}"); + assert!(text.contains("### Real.plain"), "{text}"); + assert!(text.contains("### Plain.real$get"), "{text}"); + assert!(!text.contains("- Plain.real$get"), "{text}"); + } + #[test] fn a_module_with_nothing_compiled_still_reports() { let mut module = ModuleIr::new("app"); diff --git a/crates/by_build/src/lib.rs b/crates/by_build/src/lib.rs index 530978b4b6..9be77f29be 100644 --- a/crates/by_build/src/lib.rs +++ b/crates/by_build/src/lib.rs @@ -54,7 +54,7 @@ pub fn build_source( artifact.annotation = write_annotation(&module, out_dir, options)?; Ok(Built { artifact, - declined: module.declined, + declined: left_interpreted(&module), }) } @@ -76,7 +76,7 @@ pub fn build_lowered( artifact.annotation = write_annotation(&module, out_dir, options)?; Ok(Built { artifact, - declined: module.declined, + declined: left_interpreted(&module), }) } @@ -138,7 +138,7 @@ fn emit_verified(module: &ModuleIr, out_dir: &Path, options: &Options) -> Result extension: None, annotation: write_annotation(module, out_dir, options)?, }, - declined: module.declined.clone(), + declined: left_interpreted(module), }) } @@ -288,6 +288,29 @@ fn write_annotation( Ok(Some(path)) } +/// every function an import of `module` runs from its interpreted definition: each one +/// that declined, and each one that compiled and is never installed — see +/// [`ModuleIr::unreached`](by_ir::function::ModuleIr::unreached) +pub(crate) fn left_interpreted( + module: &by_ir::function::ModuleIr, +) -> Vec { + module + .declined + .iter() + .cloned() + .chain( + module + .unreached() + .into_iter() + .map(|unreached| by_ir::function::Declined { + name: unreached.name, + reason: unreached.reason, + range: unreached.range, + }), + ) + .collect() +} + fn render_declines<'a>(declines: impl Iterator) -> String { declines .map(|declined| format!(" {}: {}", declined.name, declined.reason)) @@ -337,12 +360,15 @@ fn finish( module.verify_install = options.verify_install; module.follow_recursion_limit = options.follow_recursion_limit; - if options.require_native && !module.declined.is_empty() { - bail!( - "`require-native` is on and {} function(s) were left interpreted:\n{}", - module.declined.len(), - render_declines(module.declined.iter()) - ); + if options.require_native { + let left = left_interpreted(&module); + if !left.is_empty() { + bail!( + "`require-native` is on and {} function(s) were left interpreted:\n{}", + left.len(), + render_declines(left.iter()) + ); + } } if options.no_any && !module.gradual.is_empty() { let places = module @@ -391,6 +417,9 @@ fn finish( // too, over the twin's — once for each definition rather than once for the name let twin = by_irbuild::without_init_decorators(&twin, &module) .map_err(|error| anyhow::anyhow!("could not prepare the interpreted fallback: {error}"))?; + // and a class init reads against what its statement bound has that statement record it + let twin = by_irbuild::with_bound_records(&twin, &module) + .map_err(|error| anyhow::anyhow!("could not prepare the interpreted fallback: {error}"))?; // a nested function's annotations are the twin's, evaluated by python, so they are // written from the twin's text by_irbuild::annotations::write_factories(&mut module, &twin); diff --git a/crates/by_build/tests/differential.rs b/crates/by_build/tests/differential.rs index ad6b1f613e..d74e3174ce 100644 --- a/crates/by_build/tests/differential.rs +++ b/crates/by_build/tests/differential.rs @@ -1391,6 +1391,16 @@ fn agree_with_declines(tag: &str, source: &str, calls: &[&str]) { agree_inner(tag, source, calls, true); } +/// what a build left to the interpreted definition, by name: each function that declined, +/// and each that compiled and is never installed +fn left_interpreted(built: &by_build::Built) -> Vec<&str> { + built + .declined + .iter() + .map(|declined| declined.name.as_str()) + .collect() +} + /// as [`agree`], but the source is ordinary python /// /// the interpreted leg is then the source *itself* rather than a transpilation of @@ -7736,7 +7746,7 @@ class C: /// twice #[test] fn a_method_decorator_runs_once() { - agree_python( + agree_python_with_declines( "methoddecoratoronce", MARKED_METHODS, &[ @@ -7780,7 +7790,8 @@ fn a_decorated_method_is_the_interpreted_one_and_its_siblings_are_not() { return; } }; - assert!(built.declined.is_empty(), "declined: {:?}", built.declined); + // the two decorated methods compiled, and neither is installed + assert_eq!(left_interpreted(&built), ["C.g", "C.doubled"]); let out = run( &python, &dir, @@ -7795,6 +7806,510 @@ fn a_decorated_method_is_the_interpreted_one_and_its_siblings_are_not() { assert_eq!(out, "function function method_descriptor"); } +/// the source the marking-decorator tests below compile +/// +/// `typing.override`, `typing.final` and `abc.abstractmethod` hand back the function they +/// were given with one attribute written onto it, which is all a class body's decorated +/// function has that a fresh `def` lacks. `Cube` overrides an override and reaches both +/// bodies above it through `super()`; `total` dispatches over all three through a +/// base-typed element, which is the call a compiled override is worth having for +const MARKING_METHODS: &str = "\ +import typing +from abc import ABC, abstractmethod +from typing import final, override + + +class Shape: + def area(self) -> int: + return 1 + + def describe(self) -> str: + return 'shape' + + +class Square(Shape): + def __init__(self, side: int) -> None: + self.side = side + + @override + def area(self) -> int: + return self.side * self.side + + @typing.override + def describe(self) -> str: + return 'square of ' + str(self.side) + ' on a ' + super().describe() + + +class Cube(Square): + @override + @final + def area(self) -> int: + return super().area() * self.side + + +class Base(ABC): + @abstractmethod + def size(self) -> int: + raise NotImplementedError + + +class Sized(Base): + @override + def size(self) -> int: + return 3 + + +def total(shapes: list[Shape]) -> int: + running = 0 + for shape in shapes: + running = running + shape.area() + return running +"; + +/// a method whose decorators only mark it answers as python's does, marks included +#[test] +fn a_method_whose_decorators_only_mark_it_agrees() { + agree_python( + "markingdeco", + MARKING_METHODS, + &[ + "m.total([m.Shape(), m.Square(3), m.Cube(2)])", + "(m.Square(3).describe(), m.Cube(2).describe())", + // the mark is on the class's entry, on what binding it hands back, and absent + // where no decorator wrote one + "[getattr(c.__dict__.get(n), '__override__', None) for c in (m.Shape, m.Square, m.Cube) for n in ('area', 'describe')]", + "(m.Square(1).area.__override__, m.Cube.area.__final__, getattr(m.Square.area, '__final__', None))", + "(m.Base.size.__isabstractmethod__, sorted(m.Base.__abstractmethods__), sorted(m.Sized.__abstractmethods__))", + "type(_capture(m.Base)).__name__", + "m.Sized().size()", + // a subclass written in the interpreter overrides again, and the dispatch + // over the base-typed element reaches its body rather than the compiled one + "m.total([type('Tile', (m.Square,), {'area': lambda self: 99})(1), m.Cube(1)])", + "type('Tile', (m.Cube,), {})(2).area()", + ], + ); +} + +/// …and the method that answers is the compiled one +/// +/// the test above agrees whichever leg answered, and a marked method used to be the +/// interpreted definition: the class took the function its body decorated rather than +/// calling the decorator a second time. `type` is what tells the two apart +#[test] +fn a_method_whose_decorators_only_mark_it_is_the_compiled_one() { + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = Scratch::new("by_diff_markingdeco_t"); + let built = match build_source( + MARKING_METHODS, + "by_diff_markingdeco_t", + &toolchain, + &dir, + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + assert!(built.declined.is_empty(), "declined: {:?}", built.declined); + let out = run( + &python, + &dir, + "import by_diff_markingdeco_t as m\n\ + print(m.__file__.endswith('.so'))\n\ + print(type(m.Shape.area).__name__, type(m.Square.area).__name__,\n\ + \x20 type(m.Square.describe).__name__, type(m.Cube.area).__name__,\n\ + \x20 type(m.Base.size).__name__, type(m.Sized.size).__name__)\n", + ); + assert_eq!( + out, + "True\n\ + method_descriptor method_descriptor method_descriptor method_descriptor \ + method_descriptor method_descriptor" + ); +} + +/// a marked method whose name the class body binds again keeps the body's answer +/// +/// python's class holds whatever the last binding left, and here that is `other`, not the +/// function the marked `def` made. copying marks onto the compiled `m` and publishing it +/// would answer with the body python threw away +#[test] +fn a_marked_method_the_class_body_rebinds_answers_with_the_rebinding() { + agree_python_with_declines( + "markedrebound", + "\ +from typing import final + + +def other(self) -> int: + return 2 + + +class A: + @final + def m(self) -> int: + return 1 + + m = other + + +def call(a: A) -> int: + return a.m() +", + &["(m.call(m.A()), m.A().m(), m.A.m is m.other)"], + ); +} + +/// a method whose name the class body binds again, with no decorator at all +/// +/// python's class holds what the last binding left: another function, a lambda, a sibling +/// method, a value that is not callable, the `staticmethod`, `classmethod` or `property` +/// older code wraps its own `def` in by name, or — where the binding stands *above* the +/// `def` — the `def`'s own function. the class publishes that, and a call through a typed receiver +/// asks the class rather than calling the compiled `def` it holds nothing of +const REBOUND_METHODS: &str = "\ +def other(self) -> int: + return 2 + + +class Func: + def m(self) -> int: + return 1 + + m = other + + def kept(self) -> int: + return 8 + + +class Lam: + def m(self) -> int: + return 1 + + m = lambda self: 3 + + +class Sibling: + def m(self) -> int: + return 1 + + def n(self) -> int: + return 4 + + m = n + + +class Value: + def m(self) -> int: + return 1 + + m = 5 + + +class Above: + m = other + + def m(self) -> int: + return 6 + + +class Walrus: + def m(self) -> int: + return 1 + + x = (m := other) + + +class Private: + def __m(self) -> int: + return 1 + + __m = other + + def m(self) -> int: + return self.__m() * 10 + + +class Based(Func): + def m(self) -> int: + return 1 + + m = lambda self: 7 + + +class OldStyle: + def m(x) -> int: + return 1 if x is None else 2 + + m = staticmethod(m) + + def c(cls) -> str: + return cls.__name__ + + c = classmethod(c) + + def p(self) -> int: + return 9 + + p = property(p) + + +def through_func(a: Func) -> int: + return a.m() + a.kept() + + +def through_lam(a: Lam) -> int: + return a.m() + + +def through_sibling(a: Sibling) -> int: + return a.m() + + +def through_above(a: Above) -> int: + return a.m() + + +def through_walrus(a: Walrus) -> int: + return a.m() + + +def through_private(a: Private) -> int: + return a.m() + + +def through_based(a: Based) -> int: + return a.m() + + +def value_of(a: Value) -> object: + return a.m + + +def through_old_style(a: OldStyle) -> object: + return (a.m(None), a.c(), a.p) +"; + +#[test] +fn a_method_the_class_body_rebinds_answers_with_the_rebinding() { + agree_python_with_declines( + "reboundmethods", + REBOUND_METHODS, + &[ + "(m.through_func(m.Func()), m.through_lam(m.Lam()), m.through_sibling(m.Sibling()))", + "(m.through_above(m.Above()), m.through_walrus(m.Walrus()), m.through_private(m.Private()))", + "(m.through_based(m.Based()), m.value_of(m.Value()), m.Walrus.x is m.other)", + "(m.Func.m is m.other, m.Sibling.m is m.Sibling.n, m.Private._Private__m is m.other)", + "str(_capture(lambda: m.Value().m()))", + "(m.Lam.m.__name__, m.Above.m.__name__, m.Above.m.__qualname__)", + "(m.through_old_style(m.OldStyle()), m.OldStyle.m(None), m.OldStyle.c())", + ], + ); +} + +/// …and the compiled class is what answers, with only the rebound `def`s left out +/// +/// the test above agrees whichever leg answered, and a class that fell back to its +/// interpreted definition would agree too. `type` is what tells the two apart: the sibling +/// the class copied under the rebound name is the compiled method, and so is every method +/// nothing rebinds +#[test] +fn a_method_the_class_body_rebinds_is_the_only_one_left_out() { + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = Scratch::new("by_diff_reboundmethods_t"); + let built = match build_source( + REBOUND_METHODS, + "by_diff_reboundmethods_t", + &toolchain, + &dir, + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + assert_eq!( + left_interpreted(&built), + [ + "Func.m", + "Lam.m", + "Sibling.m", + "Value.m", + "Above.m", + "Walrus.m", + "Private._Private__m", + "Based.m", + "OldStyle.m", + "OldStyle.c", + "OldStyle.p" + ] + ); + let out = run( + &python, + &dir, + "import by_diff_reboundmethods_t as m\n\ + print(m.__file__.endswith('.so'))\n\ + print(type(m.Func.kept).__name__, type(m.Sibling.m).__name__,\n\ + \x20 type(m.Sibling.n).__name__, type(m.Private.m).__name__)\n\ + print(m.through_sibling(m.Sibling()), m.through_above(m.Above()))\n", + ); + assert_eq!( + out, + "True\n\ + method_descriptor method_descriptor method_descriptor method_descriptor\n\ + 4 6" + ); +} + +/// a marked dunder, which a type slot answers as well as its name +/// +/// the name is where the mark has to be, and the class holds a slot wrapper there for an +/// undecorated dunder, which takes no attributes. so the marked one stands under its name +/// as the compiled method with the mark copied on, while the slot keeps calling the same +/// compiled body +const MARKED_DUNDERS: &str = "\ +from typing import override + + +class Money: + amount: int + + def __init__(self, amount: int) -> None: + self.amount = amount + + @override + def __eq__(self, other: object) -> bool: + return isinstance(other, Money) and self.amount == other.amount + + @override + def __hash__(self) -> int: + return self.amount + + @override + def __repr__(self) -> str: + return 'Money(' + str(self.amount) + ')' + + +class Cents(Money): + @override + def __init__(self, amount: int) -> None: + super().__init__(amount * 100) + + @override + def __repr__(self) -> str: + return 'Cents:' + super().__repr__() + + +def same(a: Money, b: Money) -> bool: + return a == b +"; + +#[test] +fn a_marked_dunder_agrees() { + agree_python( + "markeddunder", + MARKED_DUNDERS, + &[ + "(m.Money(3) == m.Money(3), m.Money(3) != m.Money(4), m.Money(3).__eq__(m.Money(3)))", + "(hash(m.Money(3)), repr(m.Money(3)), str(m.Cents(2)), m.Cents(1) == m.Money(100))", + "m.same(m.Money(2), m.Cents(0))", + "[getattr(m.Money.__dict__[n], '__override__', None) for n in ('__eq__', '__hash__', '__repr__')]", + "(m.Cents.__init__.__override__, m.Money.__hash__ is m.Money.__dict__['__hash__'])", + ], + ); +} + +/// …and the name answers with the compiled method +#[test] +fn a_marked_dunder_is_the_compiled_method() { + let Some((_dir, answers)) = compiled_answers( + "markeddunder_t", + MARKED_DUNDERS, + Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + &[ + "[type(m.Money.__dict__[n]).__name__ for n in ('__eq__', '__hash__', '__repr__')] + [type(m.Cents.__dict__['__init__']).__name__]", + ], + ) else { + return; + }; + assert_eq!( + answers, + ["['method_descriptor', 'method_descriptor', 'method_descriptor', 'method_descriptor']"] + ); +} + +/// the `override` and `final` modifiers, which the transpiler writes out as the `typing` +/// decorators under an import of its own +const MARKING_MODIFIERS: &str = "\ +class Shape: + def area(self) -> int: + return 1 + +class Square(Shape): + side: int + + def __init__(self, side: int): + self.side = side + + override def area(self) -> int: + return self.side * self.side + +class Cube(Square): + override final def area(self) -> int: + return super().area() * self.side + +def total(shapes: list[Shape]) -> int: + running = 0 + for shape in shapes: + running = running + shape.area() + return running +"; + +#[test] +fn an_override_modifier_agrees() { + agree( + "overridemodifier", + MARKING_MODIFIERS, + &[ + "m.total([m.Shape(), m.Square(3), m.Cube(2)])", + "(m.Square.area.__override__, m.Cube.area.__override__, m.Cube.area.__final__)", + ], + ); +} + +/// …and the method it marks is the compiled one +#[test] +fn an_override_modifier_is_the_compiled_method() { + let Some((_dir, answers)) = compiled_answers( + "overridemodifier_t", + MARKING_MODIFIERS, + Options::default(), + &["(type(m.Square.area).__name__, type(m.Cube.area).__name__)"], + ) else { + return; + }; + assert_eq!(answers, ["('method_descriptor', 'method_descriptor')"]); +} + /// the source both class-decorator tests below compile /// /// `mark` hands back what it was given and records only the name, so the binding is right @@ -8047,11 +8562,11 @@ def probe() -> int: /// the differential legs agree whichever one answered, so a decorator test passes with /// the codegen path switched off. this is where it is pinned. /// -/// a *decorated method* is deliberately not one of the things `type` can pin any more: it -/// is the interpreted definition on purpose, because a decorator is handed whatever the -/// class body gave it and applying it again to the native method would run it twice. so -/// the class is pinned by its undecorated sibling, which is still the compiled type's own -/// `method_descriptor`, and the decorator by the effect it had +/// `abc.abstractmethod` only writes `__isabstractmethod__` onto what it is handed, so the +/// method it marks is the compiled one with that attribute copied on, and `type` pins it +/// like its undecorated sibling. a decorator that does anything else leaves the +/// interpreted definition standing on purpose — see +/// `a_decorated_method_is_the_interpreted_one_and_its_siblings_are_not` #[test] fn a_path_decorated_definition_is_the_compiled_one() { let Some((python, toolchain)) = environment() else { @@ -8125,7 +8640,7 @@ class Held: assert_eq!( out, "_lru_cache_wrapper native\n\ - function method_descriptor method_descriptor\n\ + method_descriptor method_descriptor method_descriptor\n\ 8 True seen" ); } @@ -9987,9 +10502,10 @@ fn a_native_class_has_a_fixed_layout() { }; let dir = Scratch::new("by_diff_layout"); let source = "\ -data class Point: - x: int - y: int +class Point: + def __init__(self, x: int, y: int) -> None: + self.x = x + self.y = y "; if build_source( source, @@ -13006,6 +13522,313 @@ def referenced(t: Tight) -> bool: ); } +/// a `data class` is `@dataclass(slots=True)`, so the emitted class is slotted exactly as +/// the twin is: no dict unless a base gave one, no weak-reference list unless a base gave +/// one, and a `__slots__` naming the fields it adds +const SLOTTED_DATA_CLASSES: &str = "\ +data class Point: + x: int + y: int + + +data class Point3(Point): + z: int + + +frozen data class Fixed: + n: int + + +class Plain: + pass + + +data class OnPlain(Plain): + q: int + + +class Loose(Point): + def __init__(self, x: int, y: int) -> None: + super().__init__(x, y) + self.w = x + y + + +class Bare(Point): + pass + + +data class Defaulted: + x: int + y: int = 3 +"; + +#[test] +fn a_data_class_is_slotted_as_its_twin_is() { + // the exception's type rather than its text: the emitted type's name is qualified by + // its module, and python words both refusals with it + let dir = agree_in( + "dcslots", + SLOTTED_DATA_CLASSES, + &[ + "[type(_capture(setattr, o, 'extra', 5)).__name__ \ + for o in (m.Point(1, 2), m.Point3(1, 2, 3), m.Fixed(1))]", + "[c.__dict__.get('__slots__') for c in m.Point3.__mro__]", + "(m.Fixed.__slots__, m.OnPlain.__slots__)", + "[hasattr(o, '__dict__') \ + for o in (m.Point(1, 2), m.Fixed(1), m.OnPlain(4), m.Loose(1, 2))]", + "[type(_outcome(vars, o)).__name__ for o in (m.Point(1, 2), m.Fixed(1))]", + "[type(_outcome(__import__('weakref').ref, o)).__name__ \ + for o in (m.Point(1, 2), m.Fixed(1), m.OnPlain(4), m.Loose(1, 2))]", + // a slot is not in the dict a base gave the instance, and `copyreg` hands it + // over beside that dict + "(lambda o: (setattr(o, 'extra', 5), vars(o), o.__getstate__())[1:])(m.OnPlain(4))", + "(lambda o: (setattr(o, 'extra', 5), vars(o), o.__getstate__())[1:])(m.Loose(1, 2))", + // python caches what `copyreg` reads on a class only once one is copied + "['__slotnames__' in c.__dict__ for c in (m.Point, m.Point3, m.OnPlain, m.Bare)]", + // `Point` keeps the words `Loose` keeps its dict and weak references in, which + // python's own `__getstate__` would take for state it cannot see + "[o.__getstate__() for o in (m.Point(1, 2), m.Point3(1, 2, 3), m.Fixed(1), \ + m.Defaulted(1), m.Bare(1, 2))]", + "[__import__('pickle').loads(__import__('pickle').dumps(o)) == o \ + for o in (m.Point(1, 2), m.Point3(1, 2, 3), m.Fixed(1), m.OnPlain(4), \ + m.Defaulted(1, 5), m.Bare(1, 2))]", + "[__import__('copy').copy(o) == o and __import__('copy').deepcopy(o) == o \ + for o in (m.Point(1, 2), m.Point3(1, 2, 3), m.Fixed(1), m.OnPlain(4), \ + m.Defaulted(1, 5), m.Bare(1, 2))]", + "(lambda o: (setattr(o, 'extra', 5), o.__getstate__(), \ + vars(__import__('pickle').loads(__import__('pickle').dumps(o))))[1:])(m.Bare(1, 2))", + "(lambda o: (setattr(o, 'extra', 5), \ + vars(__import__('pickle').loads(__import__('pickle').dumps(o))), \ + vars(__import__('copy').copy(o)))[1:])(m.Loose(1, 2))", + // a frozen data class restores a copy through `object.__setattr__`, onto an + // instance `__new__` made with nothing written + "(lambda f: (object.__setattr__(f, 'n', 7), f)[1])(m.Fixed.__new__(m.Fixed))", + ], + false, + by_irbuild::Language::BasedPython, + ); + let (Some(dir), Some((python, _))) = (dir, environment()) else { + return; + }; + // `wrapper_descriptor` says the emitted types answered rather than the twins. a write + // over a frozen field already written is the one refusal python does not make — see + // `runtime.md` + let out = run( + &python, + &dir, + "import by_diff_dcslots as m\n\ + print(*(type(c.__dict__[n]).__name__ for c, n in ((m.Point, '__repr__'),\n\ + \x20 (m.Point3, '__repr__'), (m.Fixed, '__repr__'), (m.OnPlain, '__repr__'),\n\ + \x20 (m.Loose, '__init__'))))\n\ + f = m.Fixed(1)\n\ + try:\n\ + \x20 object.__setattr__(f, 'n', 2)\n\ + except AttributeError as e:\n\ + \x20 print(e, f.n)\n", + ); + assert_eq!( + out, + "wrapper_descriptor wrapper_descriptor wrapper_descriptor wrapper_descriptor \ + wrapper_descriptor\n\ + attribute 'n' of 'by_diff_dcslots.Fixed' objects is not writable 1" + ); +} + +#[test] +fn a_decorated_data_class_is_slotted_as_its_twin_is() { + // `@dataclass(slots=True)` is the twin's innermost decorator, so what the one written + // above it is handed has no dict in python either + agree( + "dcslotdeco", + "\ +def mark(cls: type) -> type: + return cls + + +@mark +data class Marked: + x: int +", + &[ + "type(_capture(setattr, m.Marked(1), 'extra', 5)).__name__", + "(m.Marked.__slots__, hasattr(m.Marked(1), '__dict__'))", + ], + ); +} + +#[test] +fn a_slotted_class_copies_whatever_its_layout_keeps_for_a_subclass() { + // a base keeps the words a subclass keeps its dict and weak references in, and python's + // own `__getstate__` takes an object bigger than its slots account for as C state it + // cannot see, and refuses to copy it. `collections._Link` is this shape. a class that + // reduces itself never reaches that check, and keeps `object`'s `__getstate__` + agree_python( + "slotcopy", + "\ +class Link: + __slots__ = ('prev', 'key') + + +class Tagged(Link): + pass + + +class Reduced: + __slots__ = ('v',) + + def __init__(self, v: int) -> None: + self.v = v + + def __reduce__(self): + return (Reduced, (self.v,)) + + +class Wider(Reduced): + pass +", + &[ + "(lambda l: (setattr(l, 'key', 3), __import__('copy').copy(l).key, \ + __import__('pickle').loads(__import__('pickle').dumps(l)).key, \ + l.__getstate__()))(m.Link())", + "(__import__('copy').copy(m.Reduced(4)).v, '__getstate__' in m.Reduced.__dict__)", + ], + ); +} + +#[test] +fn a_slot_over_a_base_that_keeps_a_dict_stays_out_of_it() { + // python keeps a declared slot in the slot even where a base gave the instance a dict, + // so `vars()` does not name it and `copyreg` hands it over beside the dict. an + // attribute the declaration left out goes into that dict like any other + agree_python( + "slotvars", + "\ +class Plain: + pass + + +class Mixed(Plain): + __slots__ = ('a',) + + def __init__(self, a: int) -> None: + self.a = a + self.b = a + 1 +", + &[ + "(lambda o: (setattr(o, 'extra', 5), vars(o), o.__getstate__())[1:])(m.Mixed(1))", + "(lambda o: (o.a, o.b, vars(o)))(__import__('pickle').loads(\ + __import__('pickle').dumps(m.Mixed(1))))", + "(lambda o: (o.a, o.b, vars(o)))(__import__('copy').copy(m.Mixed(1)))", + ], + ); +} + +#[test] +fn reading_a_class_s_annotations_leaves_the_module_compiled() { + // from python 3.14 a class's annotations are worked out when they are first read, and + // the read writes its answer back onto the class: the mapping, and `None` under + // `__annotate_func__` where the class has no function computing one. `@dataclass` + // reads a base's that way, so a `data class` over a class of the module used to leave + // the whole module interpreted, as if its body had rewritten a dunder + let dir = agree_in( + "annotread", + "\ +class Base: + def describe(self) -> str: + return 'base' + + +data class Child(Base): + x: int + + override def describe(self) -> str: + return 'child' + + +class Bare: + def describe(self) -> str: + return 'bare' + + +seen = getattr(Bare, '__annotations__') +", + &[ + "(m.Child(1).describe(), repr(m.Child(1)), m.Base().describe())", + "(m.seen, m.Bare.__annotations__, m.Base.__annotations__, m.Child.__annotations__)", + ], + false, + by_irbuild::Language::BasedPython, + ); + let (Some(dir), Some((python, _))) = (dir, environment()) else { + return; + }; + let out = run( + &python, + &dir, + "import by_diff_annotread as m\n\ + print(*(type(c.__dict__['describe']).__name__ for c in (m.Base, m.Child, m.Bare)))\n", + ); + assert_eq!(out, "method_descriptor method_descriptor method_descriptor"); +} + +#[test] +fn a_module_body_rewriting_a_class_s_annotations_leaves_the_module_interpreted() { + // what a read writes is told apart from what the body wrote by what the `class` + // statement left: a class with no annotations had nothing under `__annotate_func__`, + // and a read leaves `None` there. a function written there instead, or `None` over + // the one an annotated class was built with, is the body's own write — and the + // compiled type would go on answering the annotations the statement wrote + for (tag, write, read) in [ + ( + "annotwritefn", + "setattr(Bare, '__annotate__', lambda format: {'q': int})", + "m.Bare.__annotations__", + ), + ( + "annotwritenone", + "setattr(Tagged, '__annotate__', None)", + "m.Tagged.__annotations__", + ), + ] { + let source = format!( + "\ +class Bare: + def describe(self) -> str: + return 'bare' + + +class Tagged: + x: int + + def describe(self) -> str: + return 'tagged' + + +{write} +" + ); + let dir = agree_in( + tag, + &source, + &[read, "(m.Bare().describe(), m.Tagged().describe())"], + false, + by_irbuild::Language::BasedPython, + ); + let (Some(dir), Some((python, _))) = (dir, environment()) else { + return; + }; + let out = run( + &python, + &dir, + &format!( + "import by_diff_{tag} as m\n\ + print(*(type(c.__dict__['describe']).__name__ for c in (m.Bare, m.Tagged)))\n" + ), + ); + assert_eq!(out, "function function", "{tag}"); + } +} + #[test] fn an_inherited_default_binds_the_same_value_in_both_halves() { // basedpython carries a method's defaults to its overrides, so `Loud.say` may be @@ -18301,6 +19124,61 @@ class Box: ); } +#[test] +fn a_property_a_metaclass_built_class_keeps_is_the_interpreted_one_the_report_names() { + // `Real` stands on a class of this module that writes `metaclass=ABCMeta`, so its type + // spec is refused and the metaclass is handed the `property` the interpreted body + // built — which is what the class keeps. `--annotate` counts that getter as left to + // the interpreted definition, and this is the runtime half of that claim: a report + // counting it compiled was counting code no import runs. `Plain` stands on nothing of + // the kind and keeps its compiled getter + let Some(compiled) = agree_in( + "metaprop", + "\ +from abc import ABCMeta + + +class Number(metaclass=ABCMeta): + pass + + +class Complex(Number): + pass + + +class Real(Complex): + @property + def real(self) -> int: + return 1 + + +class Root: + pass + + +class Plain(Root): + @property + def real(self) -> int: + return 2 +", + &["m.Real().real", "m.Plain().real"], + true, + by_irbuild::Language::Python, + ) else { + return; + }; + let Some((python, _)) = environment() else { + return; + }; + let out = run( + &python, + &compiled, + "import by_diff_metaprop as m\n\ + print(type(m.Real.__dict__['real'].fget).__name__, type(m.Plain.__dict__['real'].fget).__name__)\n", + ); + assert_eq!(out, "function method_descriptor"); +} + /// a `@property` over an abstract one is not still abstract on the emitted class /// /// a class whose base carries `abc.ABCMeta` is built by *calling* that metaclass, because @@ -19317,7 +20195,7 @@ data class Point: /// spec out and the same value goes into the namespace the metaclass is handed instead #[test] fn a_mutating_method_decorator_agrees() { - agree_python( + agree_python_with_declines( "mutatingdeco", "\ from abc import ABC, abstractmethod @@ -22064,7 +22942,7 @@ fn a_class_keyed_over_a_base_this_module_emits_is_the_compiled_type() { /// the value the interpreted body already produced #[test] fn a_decorated_method_on_a_class_built_through_its_metaclass_agrees() { - agree_python( + agree_python_with_declines( "metadeco", "\ from abc import ABCMeta, abstractmethod @@ -22143,7 +23021,9 @@ class Filled(Marked): /// class that fell back to its interpreted definition answers every one of those calls /// identically. the undecorated sibling is what says which leg ran — a compiled type /// holds a `method_descriptor` where the interpreted class holds a plain function — and -/// the carried methods stand beside it as what the body gave them +/// the carried method stands beside it as what the body gave it. the `abstractmethod` one +/// is carried into the namespace the metaclass reads and then replaced by the compiled +/// method with its mark copied on, so `ABCMeta` counts it and the compiled body answers #[test] fn a_class_carrying_a_decorated_method_is_the_compiled_one() { let Some((python, toolchain)) = environment() else { @@ -22191,7 +23071,7 @@ class Marked(metaclass=ABCMeta): return; } }; - assert!(built.declined.is_empty(), "declined: {:?}", built.declined); + assert_eq!(left_interpreted(&built), ["Marked.counted"]); let out = run( &python, &dir, @@ -22205,7 +23085,7 @@ class Marked(metaclass=ABCMeta): out, // the plain method is the compiled entry, so this class is not the twin "method_descriptor classmethod\n\ - function function\n\ + method_descriptor function\n\ ['area'] ABCMeta" ); } @@ -25299,6 +26179,445 @@ def extra(t: Trimmed) -> str: ); } +/// agree on `calls` over `source` with a class `Kept` added below it, which nothing writes +/// to, and hand back what `Kept.m` is on the compiled leg: `function` where init left the +/// module as its interpreted definition built it, `method_descriptor` where the compiled +/// class stands +fn kept_after(tag: &str, source: &str, calls: &[&str]) -> Option { + let source = format!("{source}\n\nclass Kept:\n def m(self) -> int:\n return 0\n"); + let compiled = agree_python_built(tag, &source, calls)?; + let (python, _) = environment()?; + Some(run( + &python, + &compiled, + &format!("import by_diff_{tag} as m\nprint(type(m.Kept.m).__name__)\n"), + )) +} + +#[test] +fn a_member_the_module_body_rewrites_on_a_class_nothing_can_rebind_leaves_the_module_interpreted() { + // a class nothing extends or decorates is an immutable type, so a call, a property read + // or a field access on it goes straight to the compiled body with no lookup. the module + // body runs before that type exists, though, and what it writes over a member lands on + // the twin and is carried into the type's dict — where none of those accesses looks. a + // typed `call_p` answered 1 where python answers 2, with nothing to report it, and a + // compiled module function as the new value failed the import instead + // + // the compiled code reading the class is fixed by then, so init leaves the whole module + // as its interpreted definition built it. each shape is its own module, so each is seen + // to be enough on its own + let method = "\ +class P: + def m(self) -> int: + return 1 + + def n(self) -> int: + return 5 + + +def call_p(p: P) -> int: + return p.m() +"; + let shapes = [ + ("outgrowattr", "P.m = lambda self: 2"), + ("outgrowsetattr", "setattr(P, 'm', lambda self: 2)"), + ( + "outgrowcompiled", + "def two(self: object) -> int:\n return 2\n\n\nP.m = two", + ), + ( + "outgrowhelper", + "def install() -> None:\n P.m = lambda self: 2\n\n\ninstall()", + ), + ("outgrowdel", "del P.m"), + ]; + for (tag, write) in shapes { + let Some(kept) = kept_after( + tag, + &format!("{method}\n\n{write}\n"), + &[ + "str(_capture(m.call_p, m.P()))", + "str(_capture(lambda: m.P().m()))", + "'m' in vars(m.P)", + ], + ) else { + return; + }; + assert_eq!(kept, "function", "{tag}"); + } + + // the same for a class whose instances keep no dict, which reaches the method without + // even the question about the instance + let Some(kept) = kept_after( + "outgrowslotted", + "\ +class P: + __slots__ = ('x',) + + def __init__(self) -> None: + self.x = 0 + + def m(self) -> int: + return 1 + + +P.m = lambda self: 2 + + +def call_p(p: P) -> int: + return p.m() +", + &["m.call_p(m.P())", "m.P().m()"], + ) else { + return; + }; + assert_eq!(kept, "function"); + + // a property half and a field are reached the same way + let Some(kept) = kept_after( + "outgrowprop", + "\ +class P: + def __init__(self) -> None: + self._v = 1 + + @property + def v(self) -> int: + return self._v + + +P.v = property(lambda self: 2) + + +def read_v(p: P) -> int: + return p.v +", + &["m.read_v(m.P())", "m.P().v"], + ) else { + return; + }; + assert_eq!(kept, "function"); + let Some(kept) = kept_after( + "outgrowfield", + "\ +class P: + def __init__(self) -> None: + self.x = 1 + + +P.x = 5 + + +def read_x(p: P) -> int: + return p.x +", + &["m.read_x(m.P())", "m.P.x"], + ) else { + return; + }; + assert_eq!(kept, "function"); +} + +#[test] +fn a_method_the_module_body_rewrites_on_a_data_class_leaves_the_module_interpreted() { + // a `data class`'s twin is the class `@dataclass(slots=True)` made again out of what the + // statement built, so the twin is not the statement's class and the rewrite was neither + // carried nor noticed: `P(1).m()` answered 1 where python answers 2, typed call or not. + // the decorator copies every method across as the same object, which is what still says + // the module body wrote over one + let source = "\ +data class P: + x: int + + def m(self) -> int: + return 1 + + +P.m = lambda self: 2 + + +def call_p(p: P) -> int: + return p.m() + + +class Kept: + def m(self) -> int: + return 0 +"; + let Some(compiled) = agree_in( + "outgrowdata", + source, + &["m.call_p(m.P(1))", "m.P(1).m()", "m.P(1)"], + false, + by_irbuild::Language::BasedPython, + ) else { + return; + }; + let Some((python, _)) = environment() else { + return; + }; + let out = run( + &python, + &compiled, + "import by_diff_outgrowdata as m\nprint(type(m.Kept.m).__name__)\n", + ); + assert_eq!(out, "function"); + + // and a data class nothing wrote to stays compiled + let Some(compiled) = agree_in( + "outgrowdatakept", + "\ +data class P: + x: int + + def m(self) -> int: + return 1 + + +def call_p(p: P) -> int: + return p.m() +", + &["m.call_p(m.P(1))", "m.P(1)"], + false, + by_irbuild::Language::BasedPython, + ) else { + return; + }; + let out = run( + &python, + &compiled, + "import by_diff_outgrowdatakept as m\nprint(type(m.P.m).__name__)\n", + ); + assert_eq!(out, "method_descriptor"); +} + +#[test] +fn a_dunder_or_a_field_the_module_body_writes_on_a_data_class_leaves_the_module_interpreted() { + // the decorator writes every dunder it generates onto the class it makes, and a slot + // for every field, so the body the statement left cannot say whether one of those was + // the decorator's or the module body's. the class the statement bound, after the + // decorator, can. each write is a module of its own, since any one of them leaves the + // whole module interpreted and would hide the rest + let cases: [(&str, &str, &str); 3] = [ + // `repr` answered `Point(x=1)` where python answers `point` + ( + "outgrowdunder", + "\ +data class Point: + x: int + + +setattr(Point, '__repr__', lambda self: 'point') +", + "repr(m.Point(1))", + ), + // `==` answered False where python answers True + ( + "outgrowhelper", + "\ +def install(cls: type) -> None: + setattr(cls, '__eq__', lambda self, other: True) + + +frozen data class Pair: + a: int + + +install(Pair) +", + "m.Pair(1) == m.Pair(2)", + ), + // a field read at its offset answered 1 where python cannot build the instance + ( + "outgrowfield", + "\ +data class Held: + v: int + + +setattr(Held, 'v', property(lambda self: 42)) + + +def read_v(h: Held) -> int: + return h.v +", + "str(_capture(lambda: m.read_v(m.Held(1))))", + ), + ]; + let Some((python, _)) = environment() else { + return; + }; + for (tag, source, call) in cases { + let source = + format!("{source}\n\nclass Kept:\n def m(self) -> int:\n return 0\n"); + let Some(compiled) = agree_in( + tag, + &source, + &[call], + false, + by_irbuild::Language::BasedPython, + ) else { + return; + }; + let out = run( + &python, + &compiled, + &format!("import by_diff_{tag} as m\nprint(type(m.Kept.m).__name__)\n"), + ); + assert_eq!(out, "function", "{tag}"); + } + + // and the dunders the decorator wrote itself, on a frozen data class and on one standing + // on another, leave the module compiled + let Some(compiled) = agree_in( + "outgrowdunderkept", + "\ +data class Base: + x: int + + +data class Point(Base): + y: int + + +frozen data class Pair: + a: int +", + &[ + "repr(m.Point(1, 2))", + "m.Pair(1) == m.Pair(1)", + "hash(m.Pair(1)) == hash(m.Pair(1))", + ], + false, + by_irbuild::Language::BasedPython, + ) else { + return; + }; + let out = run( + &python, + &compiled, + "import by_diff_outgrowdunderkept as m\n\ + print(type(m.Point.__dict__['__repr__']).__name__, type(m.Pair.__dict__['__eq__']).__name__)\n", + ); + assert_eq!(out, "wrapper_descriptor wrapper_descriptor"); +} + +#[test] +fn a_dunder_the_module_body_writes_without_naming_it_leaves_the_module_interpreted() { + // a dunder is what a type slot answers, and the emitted type's slot is its compiled body + // — so a dunder the module body writes onto the twin is never carried. `C.__repr__ = f` + // is declined where it is written, but `setattr` and a helper the body calls are not + // shapes the source says anything about: `repr` answered 'compiled' where python answers + // 'rewritten', and `len` raised where python answers 4. a class another extends has the + // same slot, so it is the same answer + let Some(kept) = kept_after( + "outgrowdunderset", + "\ +class D: + def __repr__(self) -> str: + return 'compiled' + + +setattr(D, '__repr__', lambda self: 'rewritten') + + +def show(d: D) -> str: + return repr(d) +", + &["repr(m.D())", "m.show(m.D())"], + ) else { + return; + }; + assert_eq!(kept, "function"); + let Some(kept) = kept_after( + "outgrowdunderhelper", + "\ +class D: + def m(self) -> int: + return 1 + + +def give() -> None: + D.__len__ = lambda self: 4 + + +give() +", + &["str(_capture(len, m.D()))"], + ) else { + return; + }; + assert_eq!(kept, "function"); + let Some(kept) = kept_after( + "outgrowdunderbase", + "\ +class Base: + def __repr__(self) -> str: + return 'compiled' + + +class Derived(Base): + pass + + +setattr(Base, '__repr__', lambda self: 'rewritten') +", + &["repr(m.Base())", "repr(m.Derived())"], + ) else { + return; + }; + assert_eq!(kept, "function"); +} + +#[test] +fn a_write_the_compiled_module_answers_for_leaves_it_compiled() { + // what `By_CarryRewrittenMembers` carries is enough wherever every access to the member + // asks the class: a method of a class another extends is called behind a licence armed + // after the carry, a name the class statement never wrote is reached by lookup, and so + // is a class constant. none of those leaves the module interpreted + let Some(kept) = kept_after( + "outgrowkept", + "\ +class Base: + def m(self) -> int: + return 1 + + +class Derived(Base): + pass + + +class Plain: + LIMIT = 3 + + def m(self) -> int: + return 1 + + +Base.m = lambda self: 2 +Plain.extra = lambda self: 9 +Plain.LIMIT = 4 + + +def call_base(b: Base) -> int: + return b.m() + + +def limit(p: Plain) -> int: + return p.LIMIT +", + &[ + "m.call_base(m.Base())", + "m.call_base(m.Derived())", + "m.Plain().extra()", + "m.limit(m.Plain())", + ], + ) else { + return; + }; + assert_eq!(kept, "method_descriptor"); +} + #[test] fn a_class_keeps_what_a_factory_installed_on_it_after_the_class_statement() { // `multiprocessing.managers` is the case this is drawn from. it defines `SyncManager` @@ -26989,6 +28308,112 @@ class Dec(Base): ); } +/// every shape of function python gives the `__class__` cell to: a method, a static and +/// a class method, a generator, and a nested function, a lambda and a comprehension +/// inside a method — each written in the body of `A`, so each reads `A` whatever class +/// the receiver is. a method that binds `__class__` itself reads its own local, and a +/// function outside any class body has no cell at all +const CLASS_CELL_READS: &str = "\ +class A: + def me(self) -> object: + return __class__ + + def name(self) -> str: + return __class__.__name__ + + def nested(self) -> object: + def inner() -> object: + return __class__ + return inner() + + def lam(self) -> object: + return (lambda: __class__)() + + def comp(self) -> list[object]: + return [__class__ for _ in range(2)] + + @staticmethod + def st() -> object: + return __class__ + + @classmethod + def cm(cls) -> object: + return __class__ + + def local(self) -> object: + __class__ = 5 + return __class__ + + def gen(self): + yield __class__ + + +class B(A): + def mine(self) -> object: + return __class__ + + +def outside() -> object: + try: + return __class__ + except NameError as e: + return str(e) +"; + +#[test] +fn a_method_reads_the_class_it_is_written_in_as_its_class_cell() { + let dir = agree_python_built( + "classcell", + CLASS_CELL_READS, + &[ + "(m.A().me() is m.A, m.A().name(), m.B().me() is m.A, m.B().mine() is m.B)", + "(m.B().nested() is m.A, m.B().lam() is m.A, [c is m.A for c in m.B().comp()])", + "(m.A.st() is m.A, m.B.cm() is m.A, next(m.B().gen()) is m.A, m.B().local())", + "m.outside()", + ], + ); + let (Some(dir), Some((python, _))) = (dir, environment()) else { + return; + }; + let out = run( + &python, + &dir, + "import by_diff_classcell as m\n\ + print(*(type(m.A.__dict__[n]).__name__ for n in ('me', 'nested', 'lam', 'comp', 'gen')),\n\ + \x20 type(m.A.__dict__['st'].__func__).__name__, type(m.A.__dict__['cm'].__func__).__name__)\n", + ); + assert_eq!(out, "method_descriptor ".repeat(7).trim_end()); +} + +#[test] +fn a_data_class_method_reads_the_emitted_type_as_its_class_cell() { + // `@dataclass(slots=True)` makes the class again and moves each method's cell onto + // the class it made, which is the one the name holds — and here the emitted type + let dir = agree_in( + "classcelldc", + "\ +data class Point: + x: int + + def owner(self) -> object: + return __class__ +", + &["m.Point(1).owner() is m.Point"], + false, + by_irbuild::Language::BasedPython, + ); + let (Some(dir), Some((python, _))) = (dir, environment()) else { + return; + }; + let out = run( + &python, + &dir, + "import by_diff_classcelldc as m\n\ + print(type(m.Point.__dict__['owner']).__name__)\n", + ); + assert_eq!(out, "method_descriptor"); +} + #[test] fn a_zero_argument_super_declines_where_slot_zero_is_not_the_receiver() { // the lowering is only sound where slot zero holds what python would read out of @@ -34775,6 +36200,66 @@ data class OnPlain(Plain): ); } +#[test] +fn a_zero_argument_super_in_a_data_class_reaches_the_base() { + // below 3.13 `dataclass(slots=True)` makes the class again and leaves each method's + // `__class__` cell on the class it replaced, so the twin is slotted another way there. + // a compiled method names its class outright, and `tag` is left to the twin: its cell + // has to end up on the emitted type, or `super()` refuses the instance. run this on + // 3.11 and 3.12 as well as the versions ci pins + let Some(compiled) = agree_built( + "dcsuper", + "\ +def keep(f: object) -> object: + return f + + +data class Child: + x: int + + def describe(self) -> str: + return 'child ' + super().__format__('') + + @property + def label(self) -> str: + return 'label ' + super().__format__('') + + @property + @keep + def tag(self) -> str: + return 'tag ' + super().__format__('') + + +frozen data class Point: + x: int + + def describe(self) -> str: + return 'point ' + super().__format__('') +", + &[ + "m.Child(1).describe()", + "m.Child(1).label", + "m.Child(1).tag", + "m.Point(2).describe()", + ], + true, + by_irbuild::Language::BasedPython, + Config::default(), + ) else { + return; + }; + let Some((python, _)) = environment() else { + return; + }; + let out = run( + &python, + &compiled, + "import by_diff_dcsuper as m\n\ + print(type(m.Child.__dict__['describe']).__name__, type(m.Point.__dict__['describe']).__name__)\n", + ); + assert_eq!(out, "method_descriptor method_descriptor"); +} + #[test] fn a_frozen_field_is_read_once_across_a_call() { // the optimization is invisible from outside, which is the point — this pins @@ -42573,6 +44058,56 @@ def through(): ); } +#[test] +fn a_class_writing_new_is_reached_through_what_a_subclass_or_a_rebinding_left() { + // a written `__new__` is published by assigning it onto the finished type, so the type + // is one python can subclass and write to. compiled code reaching `m` or `v` on such a + // class has to see an interpreted subclass's override and a method rebound after import, + // as a class with a base does. the last call rebinds `m`, so it stays last + let Some(compiled) = agree_python_built( + "newsubclassed", + "\ +class N: + def __new__(cls) -> 'N': + return object.__new__(cls) + + def m(self) -> int: + return 1 + + @property + def v(self) -> int: + return 10 + + +def call_m(n: N) -> int: + return n.m() + + +def read_v(n: N) -> int: + return n.v +", + &[ + "m.call_m(m.N())", + "m.call_m(type('S', (m.N,), {'m': lambda self: 2})())", + "m.read_v(type('S', (m.N,), {'v': property(lambda self: 20)})())", + "(setattr(m.N, 'm', lambda self: 3), m.call_m(m.N()))[1]", + ], + ) else { + return; + }; + // the interpreted leg holds functions, so the answers above only say something where + // the compiled leg holds its own method + let Some((python, _)) = environment() else { + return; + }; + let out = run( + &python, + &compiled, + "import by_diff_newsubclassed as m\nprint(type(m.N.__dict__['m']).__name__)\n", + ); + assert_eq!(out, "method_descriptor"); +} + #[test] fn a_new_that_answers_another_class_is_declined() { // python lets `__new__` answer with anything and runs `__init__` only where the answer @@ -43833,6 +45368,50 @@ class Mixin(Root, Extra): ); } +#[test] +fn a_property_the_metaclass_kept_is_counted_interpreted() { + // a class on a base from outside whose metaclass is not `type` installs, but it is built + // by calling that metaclass, which is handed the interpreted `property` and keeps it. the + // report counts `OnAbc.v`'s getter compiled all the same: only the import knows what + // `abc.ABC` meant. so each published property writes a census row of its own, and the + // getter it names is the one that runs + let source = "\ +import abc + + +class OnAbc(abc.ABC): + @property + def v(self) -> int: + return 1 + + +class Plain: + @property + def p(self) -> int: + return 2 +"; + let Some((python, dir, report)) = built_python("installprop", source) else { + return; + }; + assert!(report.contains("property v: get"), "{report}"); + assert!(report.contains("property p: get"), "{report}"); + + let out = install_census( + &python, + &dir, + "by_diff_installprop", + "print(type(m.OnAbc.__dict__['v'].fget).__name__, type(m.Plain.__dict__['p'].fget).__name__)\n", + ); + assert_eq!( + out, + "function method_descriptor\n\ + by_diff_installprop\tOnAbc.v\tproperty-interpreted\n\ + by_diff_installprop\tPlain.p\tproperty-compiled\n\ + by_diff_installprop\tOnAbc\tinstalled\n\ + by_diff_installprop\tPlain\tinstalled" + ); +} + #[test] fn an_installed_class_keeps_the_abstract_base_registry_its_body_built() { // `register` records its argument inside the class's *own* `_abc_impl`, and the diff --git a/crates/by_codegen_c/src/lib.rs b/crates/by_codegen_c/src/lib.rs index 498d3dbee0..063a1f56bf 100644 --- a/crates/by_codegen_c/src/lib.rs +++ b/crates/by_codegen_c/src/lib.rs @@ -314,7 +314,7 @@ pub fn emit_module(module: &ModuleIr) -> String { // every class a name reaches is a heap type built from a spec at module init — // so its name is a pointer rather than a static struct. only a generator's state // and a closure's environment stay static - let heap = heap_type(module, class); + let heap = heap_type(class); let type_name = class.type_name(module.name.dotted()); let _ = writeln!( out, @@ -1038,10 +1038,10 @@ fn absent_values_owner<'a>(module: &'a ModuleIr, class: &'a ClassIr) -> Option<& /// reads a C function there as a base that owns the allocation, and refuses /// `object.__new__(cls)` on the class outright. see `By_PublishAllocatingNew` fn publishes_absent_values_new(module: &ModuleIr, class: &ClassIr) -> bool { - heap_type(module, class) + heap_type(class) && !external_storage(module, class) && !inherits_layout(module, class) - && initializes(module, class) + && initializes(class) && !constructs_through_a_written_new(module, class) && absent_values_owner(module, class).is_some() } @@ -1055,7 +1055,7 @@ fn publishes_absent_values_new(module: &ModuleIr, class: &ClassIr) -> bool { /// absent values publishes one: a class that inherits that layout reaches its base's through /// the mro, which is the allocator it needs fn publishes_init_subclass(module: &ModuleIr, class: &ClassIr) -> bool { - heap_type(module, class) && writes_absent_values(module, class) + heap_type(class) && writes_absent_values(module, class) } /// the `__init_subclass__` [`publishes_init_subclass`] describes, and its method definition @@ -1079,8 +1079,8 @@ fn emit_init_subclass(module: &ModuleIr, class: &ClassIr) -> String { /// cannot name that function, and one left without a `tp_new` on `object` is made /// uninstantiable by `PyType_Ready`, so it is written in first. nothing subclasses a static /// struct, so no allocator has to be handed on — see [`publishes_absent_values_new`] -fn static_type_ready(module: &ModuleIr, class: &ClassIr, type_name: &str) -> String { - let new = if made_only_by_its_frame(module, class) { +fn static_type_ready(class: &ClassIr, type_name: &str) -> String { + let new = if made_only_by_its_frame(class) { String::new() } else { format!(" {type_name}.tp_new = PyBaseObject_Type.tp_new;\n") @@ -1216,7 +1216,7 @@ const RECYCLED_INSTANCES: usize = 16; /// every object. those are inside the reset, and zeroing them would cut a live list /// in two, so such a build recycles nothing — see [`emit_instance_recycling`] /// -/// and the block has to be the right *size*. [`mutable_type`] is false only for a +/// and the block has to be the right *size*. [`ClassIr::mutable`] is false only for a /// sealed leaf: no base, nothing derived from it, no decorator and no written /// `__new__`. that is the shape whose `tp_alloc` and `tp_free` nothing can inherit, /// so every block that reaches the array was allocated by this class's own @@ -1227,7 +1227,7 @@ const RECYCLED_INSTANCES: usize = 16; /// plain member inside the struct — and a weakref list, which a spec of ours never /// asks for fn recycles_instances(module: &ModuleIr, class: &ClassIr) -> bool { - !mutable_type(module, class) + !class.mutable && !finalizes(module, class) && class .resume @@ -1457,7 +1457,7 @@ fn emit_class_type(module: &ModuleIr, class: &ClassIr) -> String { // a class with nothing to initialize publishes no `__init__` at all, so that // `object.__init__` is what a construction reaches — exactly as in the source - if !initializes(module, class) || made_only_by_its_frame(module, class) { + if !initializes(class) || made_only_by_its_frame(class) { out.push_str(&emit_class_members(module, class)); return out; } @@ -1867,7 +1867,7 @@ fn missing_attribute(module: &ModuleIr, class: &ClassIr, object: &str, field: &s format!( "By_FieldMissing({object}, {}, {});", c_string(field), - i32::from(class.declares_slots && !instance_dict(module, class)) + i32::from(class.slots.is_some() && !instance_dict(module, class)) ) } @@ -1948,7 +1948,9 @@ fn delete_field( // before the release, for the reason the order above is what it is: a `__del__` // the release runs can read this instance, and it must not find the mapping // still naming an attribute the object has already given up - unpublish = if reserves_dict_word(module, owner) { + unpublish = if reserves_dict_word(module, owner) + && !held_in_a_slot(module, owner, &field.name) + { format!( "\x20 if (BY_UNLIKELY(By_HasPublishedDict({storage}->{BY_DICT_MEMBER})))\n\ \x20 By_UnpublishedField({storage}->{BY_DICT_MEMBER}, {});\n", @@ -2022,6 +2024,59 @@ fn dataclass_slots(class: &ClassIr, type_name: &str) -> Vec<(&'static str, Strin slots } +/// the setter a field of an *immutable* class publishes, which writes only a field the +/// instance does not have yet +/// +/// python's `frozen` refuses a write in `__setattr__`, and leaves the slot itself writable +/// through `object.__setattr__` — which is how `@dataclass(frozen=True, slots=True)`'s own +/// `__setstate__` restores a copied or unpickled instance, onto one `__new__` made with +/// every field absent. a write over a field already written is refused instead, because +/// compiled code reads a frozen field once and keeps the answer across any call +fn emit_initializing_setter( + module: &ModuleIr, + class: &ClassIr, + type_name: &str, + field: &by_ir::function::FieldDecl, +) -> String { + let owner = field_owner(module, class, &field.name); + let Some(absent) = field_absent(field, "self") else { + return format!( + "static int {type_name}_set_{name}(PyObject *selfobj, PyObject *by_value, void *closure) {{\n\ + \x20 (void)by_value; (void)closure;\n\ + \x20 return By_FieldNotWritable(selfobj, {quoted});\n}}\n", + name = field.name, + quoted = c_string(&field.name), + ); + }; + format!( + "static int {type_name}_set_{name}(PyObject *selfobj, PyObject *by_value, void *closure) {{\n\ + \x20 (void)closure;\n\ + \x20 {bind}\n\ + \x20 if (by_value == NULL || !({absent})) return By_FieldNotWritable(selfobj, {quoted});\n\ + \x20 {ty} by_v = {unbox};\n\ + \x20 if ({failed}) return -1;\n\ + \x20 self->{member} = by_v;\n\ + {present}\ + {publish}\ + \x20 return 0;\n}}\n", + name = field.name, + quoted = c_string(&field.name), + bind = bind_self(module, owner, "selfobj"), + ty = ctype(module, &field.ty), + unbox = unbox_checked(module, &field.ty, "by_value"), + failed = error_check(&field.ty, "by_v"), + member = field.member(), + present = mark_present(field, "self"), + publish = publish_field( + module, + owner, + "self", + field, + &format!("self->{}", field.member()) + ), + ) +} + /// the slot id a designated initializer names as a field fn slot_id(field: &str) -> String { format!("Py_{field}") @@ -2123,6 +2178,7 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { // wrong representation for field in published_fields(class) { if !class.writable() { + out.push_str(&emit_initializing_setter(module, class, &type_name, field)); continue; } let _ = writeln!( @@ -2197,7 +2253,7 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { out, "static const By_DictField {type_name}_dictfields[] = {{" ); - for field in &class.fields { + for field in dict_fields(module, class) { let present = if field_default(module, class, field).is_some() { format!("{type_name}_has_{}", field.name) } else { @@ -2205,7 +2261,7 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { }; // the same setter the field's descriptor publishes, and the one condition it // is emitted under - let set = if class.writable() { + let set = if class.exported { format!("{type_name}_set_{}", field.name) } else { "NULL".to_string() @@ -2226,11 +2282,26 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { // instance names the extra attributes and none of the class's own — so `copy` and // `pickle` would be handed half an object's state, quietly if publishes_a_state_method(module, class) { + let slots = slot_names(module, class); + let held = if slots.is_empty() { + "NULL".to_string() + } else { + let _ = writeln!( + out, + "static const char *const {type_name}_slotnames[] = {{{}, NULL}};", + slots + .iter() + .map(|name| c_string(name)) + .collect::>() + .join(", ") + ); + format!("{type_name}_slotnames") + }; let _ = write!( out, "static PyObject *{type_name}_getstate(PyObject *selfobj, PyObject *by_unused) {{\n\ \x20 (void)by_unused;\n\ - \x20 return By_InstanceState(selfobj, {type_name}_dictfields);\n}}\n" + \x20 return By_InstanceState(selfobj, {type_name}_dictfields, {held});\n}}\n" ); } } @@ -2246,10 +2317,10 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { if field_default(module, class, field).is_some() { continue; } - let setter = if !class.writable() { - "NULL".to_string() - } else { + let setter = if class.exported { format!("{type_name}_set_{}", field.name) + } else { + "NULL".to_string() }; let _ = writeln!( out, @@ -2263,7 +2334,7 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { // over both halves instead. handing back the dict there would be an *empty* mapping // where the interpreted class gives a full one: quiet, and wrong if instance_dict(module, class) { - if class.fields.is_empty() { + if !publishes_a_dict_view(module, class) { out.push_str( " {\"__dict__\", PyObject_GenericGetDict, PyObject_GenericSetDict, NULL, NULL},\n", ); @@ -3010,8 +3081,8 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { let dotted = module.name.dotted(); // a static struct is what a class no name reaches gets: a generator's state or a // closure's environment, neither of which anything can ask about - if !heap_type(module, class) { - let (refused, construction) = if made_only_by_its_frame(module, class) { + if !heap_type(class) { + let (refused, construction) = if made_only_by_its_frame(class) { (" | Py_TPFLAGS_DISALLOW_INSTANTIATION", String::new()) } else { ( @@ -3061,7 +3132,7 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { // emitted class that cannot be subclassed is a difference from the interpreted // one for no gain. a sealed one has given up neither, and both of those rest on the // same thing: that nothing can rebind a method and nothing can override one - let basetype = if mutable_type(module, class) { + let basetype = if class.mutable { " | Py_TPFLAGS_BASETYPE" } else { " | Py_TPFLAGS_IMMUTABLETYPE" @@ -3084,7 +3155,7 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { // a class that publishes no `__init__` supplies no slot for it either, and the // base's is what a construction then reaches — which is the source's own answer. // it is the same question whichever layout the class has, so it is asked once - let init = if initializes(module, class) { + let init = if initializes(class) { format!("\x20 {{Py_tp_init, (void *){type_name}_init}},\n") } else { String::new() @@ -3215,12 +3286,16 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { /// whether this class or an in-module base of it carries a class decorator /// /// the base chain, because an instance discipline is not a per-class answer: a subclass -/// allocates and frees instances of a shape its base decided +/// allocates and frees instances of a shape its base decided. +/// +/// a `data class` rung does not count. `@dataclass(slots=True)` is the twin's innermost +/// decorator, so every decorator written above it is handed a class whose instances have +/// no dict in python either fn decorated_chain(module: &ModuleIr, class: &ClassIr) -> bool { let mut current = class; // bounded by the class count, for the reason `inherits_layout` gives for _ in 0..=module.classes.len() { - if !current.decorators.is_empty() { + if !current.decorators.is_empty() && !current.dataclass { return true; } match current @@ -3271,7 +3346,7 @@ fn decorated_chain(module: &ModuleIr, class: &ClassIr) -> bool { /// machine are types nothing can name, so no attribute can be written on one fn instance_dict(module: &ModuleIr, class: &ClassIr) -> bool { class.exported - && heap_type(module, class) + && heap_type(class) && !inherits_layout(module, class) && (decorated_chain(module, class) || !slots_declared_throughout(module, class)) } @@ -3283,7 +3358,7 @@ fn instance_dict(module: &ModuleIr, class: &ClassIr) -> bool { fn publishes_offsets(module: &ModuleIr, class: &ClassIr) -> bool { instance_dict(module, class) || (class.exported - && heap_type(module, class) + && heap_type(class) && !inherits_layout(module, class) && module.keeps_weak_references(class)) } @@ -3291,10 +3366,60 @@ fn publishes_offsets(module: &ModuleIr, class: &ClassIr) -> bool { /// whether this class answers `__dict__` with a view over its layout as well as its dict /// /// a class keeping a dict and no fields has nothing outside that dict, so python's own -/// answer — the dict — is the whole of the mapping. every other class keeps its attributes -/// in two places at once, and only a view over both is the mapping python promises +/// answer — the dict — is the whole of the mapping, and so has one whose every field is +/// held in a slot, which python's mapping names none of. every other class keeps its +/// attributes in two places at once, and only a view over both is the mapping python +/// promises fn publishes_a_dict_view(module: &ModuleIr, class: &ClassIr) -> bool { - instance_dict(module, class) && !class.fields.is_empty() + instance_dict(module, class) && dict_fields(module, class).next().is_some() +} + +/// the fields an instance's `__dict__` names: every one but those held in a slot +fn dict_fields<'a>( + module: &'a ModuleIr, + class: &'a ClassIr, +) -> impl Iterator { + class + .fields + .iter() + .filter(|field| !held_in_a_slot(module, class, &field.name)) +} + +/// what an instance of `class` holds in a slot, in the order `copyreg._slotnames` lists +/// it: the class's own declaration first and each base's after it +fn slot_names<'a>(module: &'a ModuleIr, class: &'a ClassIr) -> Vec<&'a str> { + let mut names = Vec::new(); + let mut current = class; + // bounded by the class count, for the reason [`inherits_layout`] gives + for _ in 0..=module.classes.len() { + names.extend( + current + .slots + .iter() + .flatten() + .map(String::as_str) + .filter(|name| !matches!(*name, "__dict__" | "__weakref__")), + ); + match current + .base + .as_ref() + .and_then(ClassBase::in_module) + .and_then(|base| class_named(module, base)) + { + Some(next) => current = next, + None => break, + } + } + names +} + +/// whether `class` or an in-module base of it declares `name` in its `__slots__` +/// +/// python keeps such an attribute in the slot, never in a dict a base gave the instance, +/// so `vars()` does not name it and `copyreg` hands it over beside the dict rather than +/// in it +fn held_in_a_slot(module: &ModuleIr, class: &ClassIr, name: &str) -> bool { + slot_names(module, class).contains(&name) } /// whether this class answers `__getstate__` over the whole of an instance's state @@ -3337,7 +3462,7 @@ fn publish_field( field: &by_ir::function::FieldDecl, held: &str, ) -> String { - if !reserves_dict_word(module, owner) { + if !reserves_dict_word(module, owner) || held_in_a_slot(module, owner, &field.name) { return String::new(); } format!( @@ -3432,7 +3557,7 @@ fn slots_declared_throughout(module: &ModuleIr, class: &ClassIr) -> bool { // bounded the way [`inherits_layout`] is: a chain that visits a class twice is a // cycle, and one here would hang the compiler rather than fail it for _ in 0..=module.classes.len() { - if !current.declares_slots { + if current.slots.is_none() { return false; } let Some(name) = current.base.as_ref().and_then(ClassBase::in_module) else { @@ -3519,8 +3644,8 @@ fn finalizes(module: &ModuleIr, class: &ClassIr) -> bool { /// hand a static type `object`'s. nothing is built on such a class — being a base is /// what makes one mutable — so no other mro can reach an `__init__` of its, and a /// sealed type answers the question the same way its static edition did -fn initializes(module: &ModuleIr, class: &ClassIr) -> bool { - !mutable_type(module, class) || !class.inherited_init +fn initializes(class: &ClassIr) -> bool { + !class.mutable || !class.inherited_init } /// whether an instance of this class may only come from the compiled frame that owns it @@ -3539,8 +3664,8 @@ fn initializes(module: &ModuleIr, class: &ClassIr) -> bool { /// /// so the type refuses instantiation the way python's own `generator` does, and has no /// `__init__` for a second call to reach -fn made_only_by_its_frame(module: &ModuleIr, class: &ClassIr) -> bool { - !class.exported && !heap_type(module, class) +fn made_only_by_its_frame(class: &ClassIr) -> bool { + !class.exported && !heap_type(class) } /// whether this class keeps its fields *after* a base's instance rather than inside one @@ -4364,35 +4489,6 @@ fn stands_on_an_emitted_base(module: &ModuleIr, class: &ClassIr) -> bool { }) } -/// whether any class in the module is built on this one -/// -/// not only the ones that extend its *layout*: a class standing beside names from -/// outside still has to be a type python can derive from, and only a heap type is -fn is_base(module: &ModuleIr, class: &ClassIr) -> bool { - module.classes.iter().any(|candidate| { - candidate - .base - .as_ref() - .is_some_and(|base| base.plain_names().any(|name| name == class.name)) - }) -} - -/// whether a class needs a *mutable* heap type rather than a sealed one -/// -/// a sealed type is immutable and cannot be subclassed, which is exactly what -/// licenses the direct method call. a decorator that touches the class, and a -/// subclass, each need the opposite — so those classes pay for it. -/// -/// a written `__new__` is the third: it is published by *assigning* it onto the finished -/// type, which is the only way to reach the slot fixup that a class statement runs and a -/// type spec does not — see [`publishes_new`]. an immutable type refuses that assignment -fn mutable_type(module: &ModuleIr, class: &ClassIr) -> bool { - !class.decorators.is_empty() - || is_base(module, class) - || class.base.is_some() - || publishes_new(class).is_some() -} - /// the written `__new__` this class publishes onto its finished type, where it wrote one /// /// `tp_new` is deliberately **not** filled from the spec. a C function there is one @@ -4458,8 +4554,8 @@ fn constructs_through_a_written_new(module: &ModuleIr, class: &ClassIr) -> bool /// so as a heap type it would answer its own descriptor where the class should answer /// the module's name. staying static is what it already was, refusal on /// `__annotations__` included -fn heap_type(module: &ModuleIr, class: &ClassIr) -> bool { - mutable_type(module, class) +fn heap_type(class: &ClassIr) -> bool { + class.mutable || (class.exported && !class .fields @@ -11463,12 +11559,14 @@ fn emit_install_verification(module: &ModuleIr) -> String { for (slot, class) in published_classes(module).iter().enumerate() { let type_name = class.type_name(module.name.dotted()); let bases = emitted_base_slots(module, class); - // a decorated method's entry is whatever its decorator returned, which is not - // the descriptor the method table put there and is not this check's business + // a decorated method's entry is whatever its decorator returned, and a rebound + // one's whatever the body bound last, neither of which is the descriptor the + // method table put there nor this check's business. one that only marks is the + // compiled method, and is held to that like any other let settled: Vec<&str> = class .methods .iter() - .filter(|method| !method.decorators.is_empty()) + .filter(|method| class.installs_the_body_answer(method)) .map(|method| method.name.as_str()) .collect(); let mut block = String::new(); @@ -11539,14 +11637,41 @@ fn emit_install_verification(module: &ModuleIr) -> String { /// whether init has an exit that leaves the *whole* module interpreted /// -/// only a class built ahead of everything else can refuse in a way init cannot go on -/// from — see the layout guard in [`emit_module_init`] — so a module with none of those -/// installs its classes or fails the import, and never quietly stands down entire +/// two things can refuse in a way init cannot go on from — see the layout guard in +/// [`emit_module_init`]: a class built ahead of everything else, and a class the module body +/// changed after its statement in a way the type replacing it cannot answer for. a module +/// with neither installs its classes or fails the import, and never quietly stands down +/// entire fn stands_the_module_down(module: &ModuleIr) -> bool { module .classes .iter() - .any(|class| built_ahead(module, class)) + .any(|class| built_ahead(module, class) || class.checked_after_the_body()) +} + +/// what compiled code reaches on `class` with no lookup at all, by the name the class +/// publishes each under — empty where every access is checked +/// +/// a method whose entry is the body's own answer is left out: a call to one looks it up +/// already, so a rewrite of it is carried and obeyed. a `data class` is read against the +/// class its statement bound rather than the body, so the fields and dunders `@dataclass` +/// replaced are the ones that class holds — see `By_BodyOutgrewClass` +fn reached_without_lookup(class: &ClassIr) -> Vec<&str> { + if !class.unchecked_licences { + return Vec::new(); + } + class + .table_methods() + .filter(|method| !class.installs_the_body_answer(method)) + .map(|method| method.name.as_str()) + .chain( + class + .properties + .iter() + .map(|property| property.name.as_str()), + ) + .chain(class.fields.iter().map(|field| field.name.as_str())) + .collect() } fn emit_module_init(module: &ModuleIr) -> String { @@ -11676,7 +11801,39 @@ fn emit_module_init(module: &ModuleIr) -> String { .iter() .any(|class| built_ahead(module, class)) { - conditions.push("!BY_HAS_TYPE_DATA"); + conditions.push("!BY_HAS_TYPE_DATA".to_string()); + } + // the module body has run, and every write it made to a class after the statement is + // on the twin. one the emitted type cannot answer for leaves the module interpreted, + // which is the one refusal left once the compiled code reading the class is fixed — + // see `By_BodyOutgrewClass` + let mut outgrown_members = String::new(); + for (at, class) in module + .classes + .iter() + .filter(|class| class.checked_after_the_body()) + .enumerate() + { + let members = reached_without_lookup(class); + let members = if members.is_empty() { + "NULL".to_string() + } else { + let listed = members + .iter() + .map(|member| c_string(member)) + .collect::>() + .join(", "); + let _ = writeln!( + outgrown_members, + " static const char *const by_unlooked_{at}[] = {{{listed}, NULL}};" + ); + format!("by_unlooked_{at}") + }; + conditions.push(format!( + "By_BodyOutgrewClass(by_bodies, by_statements, by_bound, dict, {}, {members}, {})", + c_string(&class.name), + i32::from(class.records_what_it_bound()) + )); } // whether the fallback source has to be run with its class bodies captured. the // capture costs a dict copy per class the body writes, so a module with nothing to @@ -11686,7 +11843,7 @@ fn emit_module_init(module: &ModuleIr) -> String { // named by the one list, so this asks that rather than repeating it let captures_bodies = module.classes.iter().any(|class| class.exported); let release_bodies = if captures_bodies { - " Py_XDECREF(by_bodies);\n Py_XDECREF(by_statements);\n" + " Py_XDECREF(by_bodies);\n Py_XDECREF(by_statements);\n Py_XDECREF(by_bound);\n" } else { "" }; @@ -11712,8 +11869,8 @@ fn emit_module_init(module: &ModuleIr) -> String { // definitions, which is where python would have run them let _ = write!( layout_guard, - " if ({}) {{\n{record_interpreted}{}{drop_installer}{release_bodies} return 0;\n }}\n", - conditions.join(" || "), + "{outgrown_members} if ({}) {{\n{record_interpreted}{}{drop_installer}{release_bodies} return 0;\n }}\n", + conditions.join("\n || "), twin_decorators(module) ); } @@ -11917,6 +12074,19 @@ fn emit_module_init(module: &ModuleIr) -> String { class.type_name(module.name.dotted()) ); } + // after the carry, which is where a frozen data class takes the `__getstate__` its + // decorator wrote, and after `__slots__` is in every type's dict, which is what + // `copyreg` reads. a class with a dict view answers its own state already + for class in &twins { + if slot_names(module, class).is_empty() || publishes_a_dict_view(module, class) { + continue; + } + let _ = writeln!( + adopt_init, + " if (By_PublishSlottedState({}_OBJ) < 0) return -1;", + class.type_name(module.name.dotted()) + ); + } // and now that every type holds everything its body gave it, the methods those // values captured. this waits for the adoption because a table a factory installed // after the `class` statement is one of the values it has to reach @@ -12036,8 +12206,8 @@ fn emit_module_init(module: &ModuleIr) -> String { let ready = if built_ahead(module, class) { // already built, by the one construction open to it String::new() - } else if !heap_type(module, class) { - static_type_ready(module, class, &type_name) + } else if !heap_type(class) { + static_type_ready(class, &type_name) } else if let Some(base) = class .base @@ -12156,7 +12326,7 @@ fn emit_module_init(module: &ModuleIr) -> String { // a spec cannot say a slot is empty, so the ones python empties are taken off the // finished type before anything inherits them let emptied = emptied_sequence_slots(class); - if !emptied.is_empty() && heap_type(module, class) { + if !emptied.is_empty() && heap_type(class) { let mut clears = String::new(); for member in emptied { let _ = write!(clears, " by_sequence->{member} = NULL;"); @@ -12213,7 +12383,8 @@ fn emit_module_init(module: &ModuleIr) -> String { .join(", "); let _ = writeln!( installed, - " if (By_PublishProperty({type_name}_OBJ, dict, {}, {}, {defs}) < 0) return -1;", + " if (By_PublishProperty({}, {type_name}_OBJ, dict, {}, {}, {defs}) < 0) return -1;", + c_string(module.name.dotted()), c_string(&class.name), c_string(&property.name) ); @@ -12267,7 +12438,7 @@ fn emit_module_init(module: &ModuleIr) -> String { let Some((cell, holds)) = field_default(module, class, field) else { continue; }; - let setter = if class.writable() { + let setter = if class.exported { format!("{type_name}_set_{}", field.name) } else { "NULL".to_string() @@ -12314,6 +12485,20 @@ fn emit_module_init(module: &ModuleIr) -> String { // decorator's answer from — and never ran the decorators either, so // applying them there is the only application rather than a second one let body = slot.map_or_else(|| "NULL".to_string(), |slot| format!("by_body[{slot}]")); + // decorators that only mark what they were handed leave nothing in the body's + // answer but its `__dict__`, so the compiled method is published with that + // copied onto it rather than the body's function in its place + if let Some(entry) = class.marked_entry(method) { + let _ = writeln!( + installed, + " {{ static const char *const by_decorators[] = {{{names}}};\n\ + \x20 if (By_MarkedMethod({body}, (PyTypeObject *){type_name}_OBJ, dict, {}, &{type_name}_methods[{}], by_decorators, {}, &by_twins) < 0) return -1; }}", + c_string(&class.name), + synthetic_table_entries(class, &type_name).len() + entry, + method.decorators.len() + ); + continue; + } let _ = writeln!( installed, " {{ static const char *const by_decorators[] = {{{names}}};\n\ @@ -12432,8 +12617,9 @@ fn emit_module_init(module: &ModuleIr) -> String { let run_body = if captures_bodies { "\x20 PyObject *by_bodies = NULL;\n\ \x20 PyObject *by_statements = NULL;\n\ + \x20 PyObject *by_bound = NULL;\n\ \x20 if (by_fallback_source[0] != '\\0') {\n\ - \x20 by_bodies = By_RunModuleBody(&by_fallback, dict, &by_statements);\n\ + \x20 by_bodies = By_RunModuleBody(&by_fallback, dict, &by_statements, &by_bound);\n\ \x20 if (by_bodies == NULL) return -1;\n\ \x20 }\n" .to_string() @@ -12839,14 +13025,15 @@ mod tests { name: "Cell".to_string(), immutable: true, environment: false, + mutable: false, + unchecked_licences: false, exported: true, base: None, inherited_init: false, fields_are_parameters: true, dataclass: false, generic: false, - declares_slots: false, - slots_weak_references: false, + slots: None, constants: Vec::new(), properties: Vec::new(), slot_aliases: Vec::new(), @@ -13107,14 +13294,15 @@ mod tests { name: "Wrapped".to_string(), immutable: false, environment: false, + mutable: true, + unchecked_licences: false, exported: true, base: Some(ClassBase::External(vec!["Exception".to_string()])), inherited_init: false, fields_are_parameters: true, dataclass: false, generic: false, - declares_slots: false, - slots_weak_references: false, + slots: None, constants: Vec::new(), properties: Vec::new(), slot_aliases: Vec::new(), @@ -13203,6 +13391,7 @@ mod tests { let mut owned = appending_class(); owned.name = "Owned".to_string(); owned.base = None; + owned.mutable = false; module.classes.push(owned); let c = emit_module(&module); @@ -13383,7 +13572,8 @@ mod tests { let mut wakeup = appending_class(); wakeup.name = "Wakeup".to_string(); wakeup.base = None; - wakeup.declares_slots = true; + wakeup.mutable = false; + declare_slots(&mut wakeup); wakeup.methods.push(writes_an_unheld_attribute("Wakeup")); module.classes.push(wakeup); let c = emit_module(&module); @@ -13403,7 +13593,8 @@ mod tests { let mut wakeup = appending_class(); wakeup.name = "Wakeup".to_string(); wakeup.base = None; - wakeup.declares_slots = true; + wakeup.mutable = false; + declare_slots(&mut wakeup); wakeup.fields.push(by_ir::function::FieldDecl { cell: false, name: "spare".to_string(), @@ -13434,6 +13625,7 @@ mod tests { let mut open = appending_class(); open.name = "Open".to_string(); open.base = None; + open.mutable = false; module.classes.push(open); let c = emit_module(&module); @@ -13466,7 +13658,7 @@ mod tests { let mut tight = appending_class(); tight.name = "Tight".to_string(); tight.base = None; - tight.declares_slots = true; + declare_slots(&mut tight); module.classes.push(tight); let mut loose = appending_class(); loose.name = "Loose".to_string(); @@ -13508,12 +13700,12 @@ mod tests { let mut tight = appending_class(); tight.name = "Tight".to_string(); tight.base = None; - tight.declares_slots = true; + declare_slots(&mut tight); module.classes.push(tight); let mut tighter = appending_class(); tighter.name = "Tighter".to_string(); tighter.base = Some(ClassBase::InModule("Tight".to_string())); - tighter.declares_slots = true; + declare_slots(&mut tighter); module.classes.push(tighter); let c = emit_module(&module); @@ -13533,6 +13725,7 @@ mod tests { let mut wakeup = appending_class(); wakeup.name = "Wakeup".to_string(); wakeup.base = None; + wakeup.mutable = false; wakeup.methods.push(writes_an_unheld_attribute("Wakeup")); module.classes.push(wakeup); let c = emit_module(&module); @@ -13977,23 +14170,35 @@ mod tests { /// installed nothing fn refuses_whole_module(type_name: &str) -> String { format!( - "if ({type_name} == NULL) {{\n by_record_interpreted();\n Py_XDECREF(by_bodies);\n Py_XDECREF(by_statements);\n return 0;\n }}" + "if ({type_name} == NULL) {{\n by_record_interpreted();\n Py_XDECREF(by_bodies);\n Py_XDECREF(by_statements);\n Py_XDECREF(by_bound);\n return 0;\n }}" ) } + /// a `__slots__` on `class` naming exactly the fields it has + fn declare_slots(class: &mut ClassIr) { + class.slots = Some( + class + .fields + .iter() + .map(|field| field.name.clone()) + .collect(), + ); + } + fn appending_class() -> ClassIr { ClassIr { name: "Wrapped".to_string(), immutable: false, environment: false, + mutable: true, + unchecked_licences: false, exported: true, base: Some(ClassBase::External(vec!["Exception".to_string()])), inherited_init: false, fields_are_parameters: true, dataclass: false, generic: false, - declares_slots: false, - slots_weak_references: false, + slots: None, constants: Vec::new(), properties: Vec::new(), slot_aliases: Vec::new(), @@ -14673,14 +14878,15 @@ mod tests { name: "Point".to_string(), immutable: false, environment: false, + mutable: false, + unchecked_licences: false, exported: true, base: None, inherited_init: false, fields_are_parameters: true, dataclass: false, generic: false, - declares_slots: false, - slots_weak_references: false, + slots: None, constants: Vec::new(), properties: Vec::new(), slot_aliases: Vec::new(), @@ -15166,6 +15372,7 @@ mod tests { let mut class = appending_class(); class.name = "Point".to_string(); class.base = None; + class.mutable = false; module.classes.push(class); let c = emit_module(&module); @@ -15211,6 +15418,7 @@ mod tests { let mut class = appending_class(); class.name = "Point".to_string(); class.base = None; + class.mutable = false; module.classes.push(class); for function in &module.functions { assert_eq!(verify(function), Ok(()), "{}", function.name); @@ -15708,14 +15916,15 @@ mod tests { name: "Holder".to_string(), immutable: false, environment: false, + mutable: false, + unchecked_licences: false, exported: true, base: None, inherited_init: false, fields_are_parameters: true, dataclass: false, generic: false, - declares_slots: false, - slots_weak_references: false, + slots: None, constants: Vec::new(), properties: Vec::new(), slot_aliases: Vec::new(), @@ -15801,14 +16010,15 @@ mod tests { name: "outer$env".to_string(), immutable: false, environment: false, + mutable: false, + unchecked_licences: false, exported: false, base: None, inherited_init: false, fields_are_parameters: true, dataclass: false, generic: false, - declares_slots: false, - slots_weak_references: false, + slots: None, constants: Vec::new(), properties: Vec::new(), slot_aliases: Vec::new(), @@ -15879,14 +16089,15 @@ mod tests { name: "Var".to_string(), immutable: false, environment: false, + mutable: false, + unchecked_licences: false, exported: true, base: None, inherited_init: false, fields_are_parameters: true, dataclass: false, generic: false, - declares_slots: false, - slots_weak_references: false, + slots: None, constants: Vec::new(), properties: Vec::new(), slot_aliases: Vec::new(), @@ -16016,6 +16227,7 @@ mod tests { let mut class = appending_class(); class.name = "Env".to_string(); class.base = None; + class.mutable = false; class.environment = true; class.methods = vec![nested]; @@ -16113,10 +16325,13 @@ mod tests { #[test] fn a_subclassable_class_keeps_no_instance_memory() { let mut module = module_with(add()); + let mut base = sealed_class(); + base.mutable = true; let mut derived = sealed_class(); derived.name = "Shifted".to_string(); derived.base = Some(ClassBase::InModule("Point".to_string())); - module.classes = vec![sealed_class(), derived]; + derived.mutable = true; + module.classes = vec![base, derived]; let c = emit_module(&module); assert!( @@ -16151,7 +16366,7 @@ mod tests { fn an_uncollected_class_recycles_through_the_plain_allocator() { let mut module = module_with(add()); let mut class = sealed_class(); - class.declares_slots = true; + declare_slots(&mut class); module.classes = vec![class]; let c = emit_module(&module); @@ -16237,14 +16452,15 @@ mod tests { name: "Point".to_string(), immutable: false, environment: false, + mutable: false, + unchecked_licences: false, exported: true, base: None, inherited_init: false, fields_are_parameters: true, dataclass: false, generic: false, - declares_slots: false, - slots_weak_references: false, + slots: None, constants: Vec::new(), properties: Vec::new(), slot_aliases: Vec::new(), diff --git a/crates/by_ir/src/function.rs b/crates/by_ir/src/function.rs index b4459b01ce..3e3c9f88ea 100644 --- a/crates/by_ir/src/function.rs +++ b/crates/by_ir/src/function.rs @@ -4,6 +4,7 @@ //! declared once with a type, which is what makes the representation invariant //! checkable: every write to a register must produce that register's type. +use std::collections::HashMap; use std::path::PathBuf; use crate::ops::{BlockId, Op, RegisterId, Terminator, Value}; @@ -168,6 +169,14 @@ pub enum Decorator { Path { root: String, attributes: Vec, + /// whether the decorator is known to hand back the very function it was given, + /// having done nothing but write attributes onto it — `typing.override`, + /// `typing.final`, `abc.abstractmethod` + /// + /// what such a decorator did is entirely in the function's `__dict__`, so a class + /// can publish its compiled method with those attributes copied across instead of + /// the function its body decorated — see [`ClassIr::installs_the_body_answer`] + marks_only: bool, }, } @@ -178,13 +187,39 @@ impl Decorator { Self::Path { root: name.into(), attributes: Vec::new(), + marks_only: false, + } + } + + /// the same decorator, known to only mark what it was handed — see + /// [`Self::marks_only`] + #[must_use] + pub fn marking(self) -> Self { + match self { + Self::Path { + root, attributes, .. + } => Self::Path { + root, + attributes, + marks_only: true, + }, + } + } + + /// whether this decorator hands back what it was given, having only written + /// attributes onto it + pub fn marks_only(&self) -> bool { + match self { + Self::Path { marks_only, .. } => *marks_only, } } /// the bare name this is, where it is one pub fn as_name(&self) -> Option<&str> { match self { - Self::Path { root, attributes } if attributes.is_empty() => Some(root), + Self::Path { + root, attributes, .. + } if attributes.is_empty() => Some(root), Self::Path { .. } => None, } } @@ -200,8 +235,12 @@ impl Decorator { /// identifier holds no `.`, so the join is unambiguous to split again pub fn dotted(&self) -> String { match self { - Self::Path { root, attributes } if attributes.is_empty() => root.clone(), - Self::Path { root, attributes } => format!("{root}.{}", attributes.join(".")), + Self::Path { + root, attributes, .. + } if attributes.is_empty() => root.clone(), + Self::Path { + root, attributes, .. + } => format!("{root}.{}", attributes.join(".")), } } } @@ -826,6 +865,24 @@ pub struct ClassIr { /// their captures from, which a nested function reading its own name holds a cycle /// through, so the collector has to be able to walk it pub environment: bool, + /// whether the class is emitted as a *mutable* heap type python can subclass and write + /// to, rather than a sealed one + /// + /// one decision with two readers, taken once when the module's classes are first + /// seen: the emitter sets `Py_TPFLAGS_BASETYPE` on exactly these, and lowering calls a + /// method, runs a property half or reads a field with no lookup on exactly the others. + /// a class the emitter made subclassable while lowering still called it directly + /// ignored every override an interpreted subclass wrote + pub mutable: bool, + /// whether compiled code reaches this class's members under licences nothing checks at + /// runtime: a method called, a property half run and a field read at its offset, each + /// with no lookup at all + /// + /// that rests on the class being one nothing can subclass or write to once import is + /// over. what the module body writes over a member *before* then lands on the twin and + /// is carried to the type's dict, where none of those accesses looks — so module init + /// leaves the whole module interpreted instead. see `By_BodyOutgrewClass` + pub unchecked_licences: bool, /// whether the module's python namespace exposes this class. /// /// a closure environment is a real type with a real layout, but it is an @@ -854,18 +911,26 @@ pub struct ClassIr { /// `@dataclass(frozen=True, slots=True)` where [`immutable`](Self::immutable) is set, /// and those two are the whole option matrix: a written `@dataclass(...)` declines /// because a decorator call cannot be moved to module init. so what the twin's class - /// grows is settled, and the emitted type has to grow the same + /// grows is settled, and the emitted type has to grow the same. + /// + /// below 3.13 a body naming `super` or `__class__` has its slots added by the + /// transpiler's `_by_dataclass_slots` rather than by `slots=True`, which there leaves + /// each method's `__class__` cell on the class it replaced. the class it makes is the + /// same shape, and a half left to the twin then reaches the emitted type through the + /// cell module init settles pub dataclass: bool, - /// whether the class body declares `__slots__`. - /// - /// python reads that as "this instance's attributes are exactly these names", and - /// gives such an instance no `__dict__` — so a class that declares it is asking for - /// the layout an emitted class has anyway, and one that does not is asking for a - /// name it never mentioned to still have somewhere to go - pub declares_slots: bool, - /// whether that `__slots__` names `__weakref__`, which asks for the weak-reference list - /// a class without `__slots__` is given anyway - pub slots_weak_references: bool, + /// the names the class's `__slots__` declares, where it declares one: in its body, or + /// — for a `data class` — through the `@dataclass(slots=True)` its twin carries, which + /// declares the fields the class adds and no `__weakref__`. + /// + /// python reads a declaration as "this instance's attributes are exactly these names", + /// and gives such an instance no `__dict__` — so a class that declares one is asking + /// for the layout an emitted class has anyway, and one that does not is asking for a + /// name it never mentioned to still have somewhere to go. a name declared here is held + /// in a slot rather than in a dict any base gives the instance, so `vars()` leaves it + /// out. `__weakref__` among them asks for the weak-reference list a class without + /// `__slots__` is given anyway + pub slots: Option>, /// whether the class declares type parameters. /// /// they are erased in the *layout* — every `T` field is an object, whatever @@ -960,6 +1025,27 @@ impl ClassIr { !self.immutable && self.exported } + /// whether module init checks this class against what the module body did to it after + /// its `class` statement, and leaves the whole module interpreted where it changed in + /// a way the emitted type cannot answer for — see `By_BodyOutgrewClass` + /// + /// a decorated class is left out: its decorator is applied to the emitted type again, + /// which rewrites the same names on it as it rewrote on the twin + pub fn checked_after_the_body(&self) -> bool { + self.exported && self.decorators.is_empty() + } + + /// whether the twin's `class` statement records the class it bound, after its + /// decorators, for that check to read the class against + /// + /// a `data class` is the one class whose twin is not what its statement built: the + /// `@dataclass` the transpiler wrote makes it again and writes every dunder it + /// generates, so the body the statement left says nothing about which of those the + /// module body wrote over afterwards + pub fn records_what_it_bound(&self) -> bool { + self.dataclass && self.checked_after_the_body() + } + /// the methods python reaches by name, through the method table /// /// a property's accessors are reached through the `property` object instead. a table @@ -987,6 +1073,52 @@ impl ClassIr { }) } + /// whether module init publishes the function the class body decorated under + /// `method`'s name, rather than `method` itself + /// + /// a method's decorators run inside the class body, which the interpreted definition + /// ran before anything of this module existed. applying them again to the compiled + /// method would call each a second time, so init takes the body's answer instead — + /// see `By_DecoratedMethod` — and the compiled body is emitted and never installed. + /// only a class some `class` statement wrote has a body to take it from, and that is + /// an exported one + /// + /// a method whose every decorator [only marks](Decorator::marks_only) what it was + /// handed is the exception: those attributes are all the body's answer adds to the + /// function, and init copies them onto the compiled method and installs that — see + /// `By_MarkedMethod` + /// + /// a method whose name the class body binds again takes the body's answer whatever its + /// decorators are, marks or none: python's class holds what the last binding left, and + /// init copies that across as a class-level constant — see [`Self::rebinds`] + pub fn installs_the_body_answer(&self, method: &Function) -> bool { + self.exported + && (self.rebinds(method) + || (!method.decorators.is_empty() && self.marked_entry(method).is_none())) + } + + /// whether the class body binds `method`'s name somewhere besides its `def` + /// + /// `m = other` under `def m`, `x = (m := other)`, or `m = other` above it: the name is + /// then a class-level constant as well as a method, and the constant is what the body + /// left under it + pub fn rebinds(&self, method: &Function) -> bool { + self.constants.contains(&method.name) + } + + /// where `method` stands among [`Self::table_methods`], when it carries decorators and + /// every one of them only marks what it was handed + /// + /// the table entry is what init publishes under the name with the marks copied on, so + /// a method with no entry of its own keeps taking the body's answer + pub fn marked_entry(&self, method: &Function) -> Option { + if method.decorators.is_empty() || !method.decorators.iter().all(Decorator::marks_only) { + return None; + } + self.table_methods() + .position(|entry| entry.name == method.name) + } + /// whether `method` is the one this class's resumable frame steps through pub fn resumes_through(&self, method: &Function) -> bool { self.resume @@ -1167,6 +1299,15 @@ pub struct PromotedPlace { pub missed: RType, } +/// a function the compiler lowered and emitted, and that no import of the module runs +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Unreached { + /// the qualified name, as a decline names its function + pub name: String, + pub reason: String, + pub range: Option<(u32, u32)>, +} + /// a function the compiler could not lower natively #[derive(Debug, Clone, PartialEq, Eq)] pub struct Declined { @@ -1319,7 +1460,7 @@ impl ModuleIr { /// `__slots__` without asking for one, and an emitted class keeps a weak-reference list /// on the same terms — so `weakref.ref`, `WeakSet`, `WeakKeyDictionary`, `finalize` and /// every other weak container reach one of its instances as they would the interpreted - /// one's. a `data class`'s twin is `@dataclass(slots=True)`, which asks for none. + /// one's. a `data class` declares slots without one, as `@dataclass(slots=True)` does. /// /// the layout has to be this module's all the way down to `object`: a class standing on /// a base from outside takes that base's instance whole, and has no word of its own to @@ -1333,10 +1474,10 @@ impl ModuleIr { // bounded by the class count: a base chain cannot visit one twice without being a // cycle, and a cycle here would otherwise hang rather than answer for _ in 0..=self.classes.len() { - if current.dataclass { - return false; - } - unslotted |= !current.declares_slots || current.slots_weak_references; + unslotted |= current + .slots + .as_ref() + .is_none_or(|names| names.iter().any(|name| name == "__weakref__")); match ¤t.base { None => return unslotted, Some(ClassBase::External(_)) => return false, @@ -1382,6 +1523,187 @@ impl ModuleIr { .chain(self.classes.iter().flat_map(|class| class.methods.iter())) } + /// whether module init builds `class` by calling its metaclass, whatever its bases turn + /// out to resolve to + /// + /// a class with a base is built by whichever construction the running interpreter + /// allows, and a type spec is refused wherever a base's metaclass is not `type`. three + /// shapes settle that before anything runs: + /// + /// - a class keyword, which a spec has nowhere to put + /// - a class of this module's standing beside a base from outside, a shape a spec + /// cannot work out + /// - a class of this module's among its bases that writes a `metaclass` keyword, or + /// stands on one that does: the type it is built as is an instance of that metaclass + /// + /// a base from outside with a metaclass of its own — `abc.ABC` — sends a class the same + /// way, and only the interpreter knows what such a name meant, so a class on one is not + /// counted here. and a keyword is read as the metaclass it names without asking which: + /// a class on one written `metaclass=type` is counted though a spec builds it + pub fn built_through_its_metaclass(&self, class: &ClassIr) -> bool { + let Some(base) = &class.base else { + return false; + }; + !class.keywords.is_empty() + || (base.external().is_some() && base.plain_names().any(|name| self.emits(name))) + || base + .plain_names() + .any(|name| self.writes_a_metaclass(name, &mut Vec::new())) + } + + /// whether this module emits an exported class called `name` + fn emits(&self, name: &str) -> bool { + self.classes + .iter() + .any(|class| class.exported && class.name == name) + } + + /// whether the exported class called `name`, or one of this module's classes it stands + /// on, writes a `metaclass` keyword. `seen` is the walk so far, which a base cannot + /// name twice in a module python could run + fn writes_a_metaclass<'a>(&'a self, name: &'a str, seen: &mut Vec<&'a str>) -> bool { + if seen.contains(&name) { + return false; + } + seen.push(name); + let Some(class) = self + .classes + .iter() + .find(|class| class.exported && class.name == name) + else { + return false; + }; + class + .keywords + .iter() + .any(|keyword| keyword.name == "metaclass") + || class.base.as_ref().is_some_and(|base| { + base.plain_names() + .any(|base| self.writes_a_metaclass(base, seen)) + }) + } + + /// every compiled function no import of this module runs, in module order + /// + /// a method whose class installs the body's decorated function in its place — see + /// [`ClassIr::installs_the_body_answer`] — is one. so is everything only such a method + /// makes: a closure's body and a generator's frame are methods of a class nothing can + /// name, reached solely through the frame that builds that class's instance, and a + /// frame that never runs builds none. + /// + /// a property half of a class [built through its + /// metaclass](Self::built_through_its_metaclass) is one too. such a class is handed the + /// `property` the interpreted body built, which is the object a `class` statement + /// would have left under the name, and that is what the class keeps + /// + /// the answer is the functions those methods reach and nothing else reaches. a + /// function this does not look at — one of a hidden class no frame is seen building — + /// is left counted as it was, so what moves is exactly what the installed body answer + /// takes out of reach + pub fn unreached(&self) -> Vec { + let hidden: HashMap<&str, &ClassIr> = self + .classes + .iter() + .filter(|class| !class.exported) + .map(|class| (class.name.as_str(), class)) + .collect(); + let by_name: HashMap = self + .all_functions() + .map(|function| (function.qualified_name(), function)) + .collect(); + // what a function can run next: a method it calls directly, and every method of a + // hidden class it names, since naming one is how a frame builds or reads it + let reached_from = |function: &Function| -> Vec { + let mut next = Vec::new(); + for op in function.blocks.iter().flat_map(|block| &block.ops) { + if let Op::CallNative { owner, callee, .. } = op { + next.push(qualify(owner.as_deref(), callee)); + } + for class in op.named_classes() { + if let Some(class) = hidden.get(class) { + next.extend(class.methods.iter().map(Function::qualified_name)); + } + } + } + next + }; + // every function reachable from `roots`, each with the root it was first reached + // from + let walk = |roots: Vec<(String, String)>| -> HashMap { + let mut seen: HashMap = HashMap::new(); + let mut pending = roots; + while let Some((name, root)) = pending.pop() { + if seen.contains_key(&name) { + continue; + } + let Some(function) = by_name.get(&name) else { + continue; + }; + pending.extend( + reached_from(function) + .into_iter() + .map(|next| (next, root.clone())), + ); + seen.insert(name, root); + } + seen + }; + let (mut rebound, mut carried) = (Vec::new(), Vec::new()); + let (mut shadowed, mut installed) = (Vec::new(), Vec::new()); + for function in &self.functions { + installed.push(function.qualified_name()); + } + for class in self.classes.iter().filter(|class| class.exported) { + let through_metaclass = self.built_through_its_metaclass(class); + for method in &class.methods { + let name = method.qualified_name(); + if class.rebinds(method) { + rebound.push(name.clone()); + } + let a_carried_half = through_metaclass + && class + .properties + .iter() + .any(|property| property.holds(&method.name)); + if a_carried_half { + carried.push(name.clone()); + } + if a_carried_half || class.installs_the_body_answer(method) { + shadowed.push((name.clone(), name)); + } else { + installed.push(name); + } + } + } + let live = walk( + installed + .into_iter() + .map(|name| (name, String::new())) + .collect(), + ); + let dead = walk(shadowed); + self.all_functions() + .filter_map(|function| { + let name = function.qualified_name(); + let root = dead.get(&name).filter(|_| !live.contains_key(&name))?; + let reason = if *root == name && carried.contains(&name) { + "a half of a property on a class built by calling its metaclass, which is handed the `property` the interpreted definition built and keeps it".to_string() + } else if *root == name && rebound.contains(&name) { + "its name is bound again in the class body, so the class holds what that binding left and this one is never installed".to_string() + } else if *root == name { + "decorated in the class body, so the class holds the function the interpreted definition decorated and this one is never installed".to_string() + } else { + format!("only `{root}` reaches this, and `{root}` is never installed") + }; + Some(Unreached { + name, + reason, + range: function.range, + }) + }) + .collect() + } + pub fn all_functions_mut(&mut self) -> impl Iterator { self.functions.iter_mut().chain( self.classes diff --git a/crates/by_irbuild/src/lib.rs b/crates/by_irbuild/src/lib.rs index d4047aa276..f1de48a4ea 100644 --- a/crates/by_irbuild/src/lib.rs +++ b/crates/by_irbuild/src/lib.rs @@ -293,11 +293,20 @@ pub fn build_module( // the signature of every method of every emitted class. a direct call needs // the callee's representations *before* the callee is lowered, because a // method may call a sibling — or itself - // a class emitted as a mutable *heap* type — one with a decorator, one another - // extends, or one that extends another — gives up the direct method call: python - // can rebind a method on it, or override it in a subclass, and a direct call - // would see neither. a plain class is a static type that can be neither modified - // nor subclassed, which is what licenses the direct call + // a class emitted as a mutable *heap* type gives up the direct method call: python + // can rebind a method on it, or override it in a subclass, and a direct call would + // see neither. a plain class is a static type that can be neither modified nor + // subclassed, which is what licenses the direct call. this set is the whole + // decision — the emitter reads it back off `ClassIr::mutable` rather than working + // it out again — so every reason the emitter needs a mutable type is here: + // + // - a decorator, which may write onto the class it is handed, and a name a frame + // declares `global`, which may be bound to something else — `decorated` holds both + // - a base, and being one: a subclass may override a method, and a direct call on a + // receiver typed as the base would not see it + // - a written `__new__`, which is published by *assigning* it onto the finished + // type, the only way to reach the slot fixup a class statement runs and a type + // spec does not. an immutable type refuses that assignment let declared: Vec<&ast::StmtClassDef> = suite .iter() .filter_map(|stmt| match stmt { @@ -307,7 +316,7 @@ pub fn build_module( .collect(); let mut mutable: HashSet<&str> = HashSet::new(); for class in &declared { - if decorated.contains(class.name.as_str()) { + if decorated.contains(class.name.as_str()) || writes_new(class) { mutable.insert(class.name.as_str()); } if let Some(base) = base_class(db, env, model, suite, class, &layouts) @@ -343,7 +352,17 @@ pub fn build_module( .body .iter() .filter_map(|member| match member { - Stmt::FunctionDef(method) if method.decorator_list.is_empty() => { + // a method whose decorators only mark it is published compiled, marks + // copied on, so a call may reach it as directly as an undecorated one. + // one whose name the body binds again is not what the class holds, for + // either kind, and a call has to ask the class + Stmt::FunctionDef(method) + if bound_only_by(&class.body, method) + && method + .decorator_list + .iter() + .all(|decorator| marks_only(db, model, decorator)) => + { let mut signature = signature( db, env, @@ -569,7 +588,7 @@ pub fn build_module( let slotted: HashSet = declared .iter() - .filter(|class| declared_slots(class).is_some()) + .filter(|class| declares_slots(class)) .map(|class| class.name.to_string()) .collect(); @@ -594,7 +613,7 @@ pub fn build_module( .iter() .filter_map(|statement| match statement { Stmt::ClassDef(class) - if layouts.contains_key(class.name.as_str()) && declared_slots(class).is_none() => + if layouts.contains_key(class.name.as_str()) && !declares_slots(class) => { Some((class.name.to_string(), class_method_names(class))) } @@ -605,20 +624,10 @@ pub fn build_module( // read off the source rather than off the lowered methods: a `__new__` python never // reaches the method table for — one carrying a decorator, one whose class declined — // is still the constructor a `C(...)` in this module has to run - let constructs: HashSet = suite + let constructs: HashSet = declared .iter() - .filter_map(|stmt| { - match stmt { - Stmt::ClassDef(class) => class - .body - .iter() - .any(|member| { - matches!(member, Stmt::FunctionDef(method) if method.name.as_str() == "__new__") - }) - .then(|| class.name.to_string()), - _ => None, - } - }) + .filter(|class| writes_new(class)) + .map(|class| class.name.to_string()) .collect(); let no_directs: HashSet = HashSet::new(); @@ -1564,8 +1573,7 @@ fn lower_generator( constructor, vec![by_ir::function::ClassIr { exported: false, - declares_slots: false, - slots_weak_references: false, + slots: None, name: class, // the machine is the same; only the surface differs. a coroutine answers // `__await__` and is deliberately *not* iterable @@ -1593,6 +1601,8 @@ fn lower_generator( dataclass: false, immutable: false, environment: false, + mutable: false, + unchecked_licences: false, keywords: Vec::new(), }] .into_iter() @@ -2881,6 +2891,7 @@ fn lower_class<'a>( let mut constants = Vec::new(); let mut slot_aliases = Vec::new(); let mut published = Vec::new(); + let mut rebound_dunders = Vec::new(); for group in &properties { let mut published_group = by_ir::function::PropertyIr { name: mangled(Some(&class.name), group.name), @@ -2973,7 +2984,23 @@ fn lower_class<'a>( ))); } defined_once(&class.body, method)?; - let (method, produced) = lower_method(unit, method, &class.name)?; + let rebound = !bound_only_by(&class.body, method); + if rebound && method.name.starts_with("__") && method.name.ends_with("__") { + rebound_dunders.push(method.name.as_str()); + } + let (mut method, produced) = lower_method(unit, method, &class.name)?; + if rebound { + for decorator in &mut method.decorators { + let Decorator::Path { marks_only, .. } = decorator; + *marks_only = false; + } + // the class holds what the last binding left, and that is taken off + // the body as a constant is — which also covers a binding above the + // `def`, and a `:=`, neither of which is otherwise one + if !constants.contains(&method.name) { + constants.push(method.name.clone()); + } + } if method.name == "__new__" { new_answers_its_own_class(&method, &class.name)?; } @@ -3069,6 +3096,19 @@ fn lower_class<'a>( _ => return Err(Decline::new("only fields and methods are lowered yet")), } } + // a `:=` binds a class-level name as an assignment does, and the value is taken off + // the body the same way + for name in walrus_bindings(&class.body) { + if name.starts_with("__") && name.ends_with("__") { + return Err(Decline::new(format!( + "`{name}` is bound by `:=` in the class body, and a dunder is settled before one runs" + ))); + } + let name = mangled(Some(&class.name), name); + if !constants.contains(&name) { + constants.push(name); + } + } // the two questions a metaclass construction raises are asked while the layouts // settle, in `metaclass_carries_the_body`, so that a class they turn down leaves the @@ -3107,6 +3147,13 @@ fn lower_class<'a>( clash.name ))); } + // and whatever else binds a dunder's name again, since its type slot is filled from + // the `def` whatever the body left under the name + if let Some(name) = rebound_dunders.first() { + return Err(Decline::new(format!( + "`{name}` is bound again by the class body, and a dunder is settled from its `def`" + ))); + } // a property is written into the type's dict under its name, and a field's descriptor // is already sitting under its own name there — the same collision a class-level // constant has with one, and the same answer @@ -3138,6 +3185,7 @@ fn lower_class<'a>( "a property half named `{clash}` reaches the same symbol as a method of this class" ))); } + let slots = class_slots(class, &fields, inherited)?; Ok(( by_ir::function::ClassIr { resume: None, @@ -3145,6 +3193,8 @@ fn lower_class<'a>( name: class.name.to_string(), immutable, environment: false, + mutable: unit.mutable.contains(class.name.as_str()), + unchecked_licences: unit.sealed.contains(class.name.as_str()), base, // neither written nor generated: a `data class` always gets one, a written // `__init__` is a method of its own, and an accessor block's storage is @@ -3159,8 +3209,7 @@ fn lower_class<'a>( fields, decorators: class_decorators, generic: class.type_params.is_some(), - declares_slots: declared_slots(class).is_some(), - slots_weak_references: slots_ask_for_weak_references(class)?, + slots, constants, slot_aliases, methods: lowered, @@ -3198,6 +3247,13 @@ fn getattr_hook_stands_alone(base: Option<&ClassBase>) -> Lowered<()> { Ok(()) } +/// whether the class body writes a `def __new__` +fn writes_new(class: &ast::StmtClassDef) -> bool { + class.body.iter().any( + |member| matches!(member, Stmt::FunctionDef(method) if method.name.as_str() == "__new__"), + ) +} + /// whether the emitter can publish this class's written `__new__` /// /// the method itself needs nothing special: it is bound onto the finished type as the @@ -3842,12 +3898,34 @@ fn declared_slots(class: &ast::StmtClassDef) -> Option<&Expr> { }) } -/// whether a class's `__slots__` names `__weakref__` -fn slots_ask_for_weak_references(class: &ast::StmtClassDef) -> Lowered { - let Some(value) = declared_slots(class) else { - return Ok(false); - }; - Ok(slot_names(value)?.contains(&"__weakref__")) +/// the names a class's `__slots__` declares — see `ClassIr::slots` +/// +/// a `data class` declares one without writing it: `@dataclass(slots=True)` names each +/// field the class adds past those its base already holds +fn class_slots( + class: &ast::StmtClassDef, + fields: &[by_ir::function::FieldDecl], + inherited: &[by_ir::function::FieldDecl], +) -> Lowered>> { + if let Some(value) = declared_slots(class) { + // mangled, for the reason `slot_fields` gives + return Ok(Some( + slot_names(value)? + .into_iter() + .map(|name| mangled(Some(&class.name), name)) + .collect(), + )); + } + if !carries_the_data_class_marker(class) { + return Ok(None); + } + Ok(Some( + fields + .iter() + .filter(|field| !inherited.iter().any(|held| held.name == field.name)) + .map(|field| field.name.clone()) + .collect(), + )) } /// the attribute names a `__slots__` value declares @@ -4025,6 +4103,7 @@ fn decorator_path(expression: &Expr) -> Lowered { return Ok(Decorator::Path { root: name.id.to_string(), attributes, + marks_only: false, }); } Expr::Attribute(attribute) => { @@ -4294,6 +4373,55 @@ pub fn without_init_decorators(source: &str, module: &ModuleIr) -> Result Result { + let wanted: HashSet<&str> = module + .classes + .iter() + .filter(|class| class.records_what_it_bound()) + .map(|class| class.name.as_str()) + .collect(); + if wanted.is_empty() { + return Ok(source.to_string()); + } + let parsed = ruff_python_parser::parse_module(source) + .map_err(|error| format!("the interpreted fallback does not parse: {error}"))?; + let mut wrapped: Vec<(usize, usize)> = Vec::new(); + let mut found: HashSet<&str> = HashSet::new(); + for statement in parsed.suite() { + if let Stmt::ClassDef(class) = statement + && let Some(name) = wanted.get(class.name.as_str()) + && let Some(outermost) = class.decorator_list.first() + { + let range = outermost.expression.range(); + wrapped.push((range.start().to_usize(), range.end().to_usize())); + found.insert(name); + } + } + if let Some(missing) = wanted.iter().find(|name| !found.contains(*name)) { + return Err(format!( + "the interpreted fallback writes no decorated `class {missing}` to record" + )); + } + let mut out = source.to_string(); + wrapped.sort_unstable(); + for (start, end) in wrapped.into_iter().rev() { + out.insert(end, ')'); + out.insert_str(start, "__build_class__("); + } + Ok(out) +} + /// the attributes each emitted class publishes as a `property` /// /// every one is a *data* descriptor, so a write on an instance reaches it rather than @@ -4836,6 +4964,23 @@ fn defined_once(scope: &[Stmt], function: &ast::StmtFunctionDef) -> Lowered<()> Ok(()) } +/// whether `function` is the only statement of its class body that binds or unbinds its +/// name +/// +/// a method whose decorators only mark it is published as the compiled method with the +/// marks copied off whatever the body left under its name, which is the `def`'s own +/// function only where nothing below rebound the name. `m = other` under a marked `def m` +/// leaves `other` there, and publishing the compiled `m` in its place would answer with the +/// body python discarded +fn bound_only_by(body: &[Stmt], function: &ast::StmtFunctionDef) -> bool { + let name = function.name.as_str(); + body.iter().all(|statement| match statement { + Stmt::FunctionDef(other) if std::ptr::eq(other, function) => true, + Stmt::Delete(node) => !node.targets.iter().any(|target| binds_name(target, name)), + _ => !class_body_binds(std::slice::from_ref(statement), name), + }) +} + /// whether a class body binds this name /// /// a decorator is resolved out of the *module* namespace at module init, and a class @@ -4843,6 +4988,9 @@ fn defined_once(scope: &[Stmt], function: &ast::StmtFunctionDef) -> Lowered<()> /// two statements up, which is nowhere init can look — so the method keeps its /// interpreted definition, and with it the whole class fn class_body_binds(body: &[Stmt], name: &str) -> bool { + if walrus_bindings(body).contains(&name) { + return true; + } body.iter().any(|statement| match statement { Stmt::FunctionDef(node) => node.name.as_str() == name, Stmt::ClassDef(node) => node.name.as_str() == name, @@ -4872,6 +5020,47 @@ fn class_body_binds(body: &[Stmt], name: &str) -> bool { }) } +/// the names a `:=` binds in the class namespace, anywhere in the class body +/// +/// a walrus binds in the scope it runs in, and every expression a class body evaluates +/// runs in the class namespace: a value, a method's default or decorator, an annotation +/// python 3.13 evaluates, a subscript in a target, a block's condition. a method's own +/// body is a scope of its own and is not looked into +fn walrus_bindings<'a>(body: &'a [Stmt]) -> Vec<&'a str> { + let mut names = Vec::new(); + let mut look = |expr: &'a Expr| { + closures::visit_expressions(expr, &mut |child| { + if let Expr::Named(node) = child + && let Expr::Name(name) = node.target.as_ref() + { + names.push(name.id.as_str()); + } + }); + }; + for statement in walk(body) { + for expr in closures::statement_expressions(statement) { + look(expr); + } + match statement { + Stmt::Assign(node) => node.targets.iter().for_each(&mut look), + Stmt::AnnAssign(node) => { + look(&node.target); + look(&node.annotation); + } + Stmt::For(node) => look(&node.target), + Stmt::FunctionDef(node) => { + node.parameters + .iter() + .filter_map(ast::AnyParameterRef::annotation) + .for_each(&mut look); + node.returns.iter().for_each(|returns| look(returns)); + } + _ => {} + } + } + names +} + /// the names a block nested in a class body could bind into the class namespace /// /// this is deliberately an over-approximation, and the direction it errs in is the whole @@ -5198,6 +5387,48 @@ fn class_modifier( } } +/// the standard-library decorators that hand back the very function they were given, +/// having done nothing to it but write one attribute, each paired with the module it is +/// written in +/// +/// `typing.override` and `typing.final` set `__override__` and `__final__`, swallowing +/// the error an object that takes no attributes raises; `abc.abstractmethod` sets +/// `__isabstractmethod__`. the `typing_extensions` spellings are the `typing` functions +/// themselves on every version this compiler targets, and the same body where they are +/// not. so everything one of these did to a function is in that function's `__dict__`, +/// and a class can publish its compiled method with the dict copied across — see +/// [`Decorator::marks_only`] +/// +/// the module is half of the entry for the reason [`FRAME_WALKERS`] gives: `final` and +/// `override` are names a module may bind to something of its own +const MARKING_DECORATORS: &[(&str, &str)] = &[ + ("typing", "override"), + ("typing_extensions", "override"), + ("typing", "final"), + ("typing_extensions", "final"), + ("abc", "abstractmethod"), +]; + +/// whether `decorator` resolves to one of [`MARKING_DECORATORS`] +/// +/// resolved rather than read: `@override` is only the stdlib's where the name is bound to +/// it, and `@typing.override` names it under a spelling of its own. a *modifier* is a +/// language keyword the transpiler writes out as one of these under its own import, and is +/// asked the same question of what it resolves to +fn marks_only( + db: &dyn ty_python_semantic::Db, + model: &SemanticModel<'_>, + decorator: &ast::Decorator, +) -> bool { + let env = &model.program_environment(); + let Some(resolved) = decorator.expression.inferred_type(model) else { + return false; + }; + MARKING_DECORATORS.iter().any(|(module, name)| { + ty_python_semantic::basedpython_module_symbol(db, env, module, name) == Some(resolved) + }) +} + /// the same for a function or a method /// /// the modifier mapping is the transpiler's: each of these becomes the named python @@ -6241,14 +6472,25 @@ fn metaclass_carries_the_body( fn is_a_data_class(suite: &[Stmt], name: &str) -> bool { suite.iter().any(|statement| { matches!(statement, Stmt::ClassDef(candidate) - if candidate.name.as_str() == name - && candidate.decorator_list.iter().any(|decorator| { - matches!(&decorator.expression, Expr::Name(marker) - if matches!(marker.id.as_str(), "data_class" | "frozen_data_class")) - })) + if candidate.name.as_str() == name && carries_the_data_class_marker(candidate)) + }) +} + +/// whether `class` carries the `data class` marker +fn carries_the_data_class_marker(class: &ast::StmtClassDef) -> bool { + class.decorator_list.iter().any(|decorator| { + matches!(&decorator.expression, Expr::Name(marker) + if matches!(marker.id.as_str(), "data_class" | "frozen_data_class")) }) } +/// whether `class` declares `__slots__` — see `ClassIr::slots` +/// +/// a `data class` does so without writing one: its twin is `@dataclass(slots=True)` +fn declares_slots(class: &ast::StmtClassDef) -> bool { + declared_slots(class).is_some() || carries_the_data_class_marker(class) +} + /// an annotation in a `data class` body that declares something other than a field /// /// `@dataclass` reads three annotations as instructions rather than as storage, and each @@ -7366,11 +7608,18 @@ fn lower_function_with_receiver( let mut decorators = Vec::with_capacity(function.decorator_list.len()); let mut wrote_a_decorator = false; for decorator in function.decorator_list.iter().filter(|_| !nested) { + let marking = |name: Decorator| { + if marks_only(db, model, decorator) { + name.marking() + } else { + name + } + }; match function_modifier(db, model, decorator)? { - Modifier::Apply(name) => decorators.push(name), + Modifier::Apply(name) => decorators.push(marking(name)), Modifier::Written(name) => { wrote_a_decorator = true; - decorators.push(name); + decorators.push(marking(name)); } Modifier::Erased | Modifier::DataClass => {} } @@ -8031,8 +8280,7 @@ fn lower_function_with_receiver( resume: None, keywords: Vec::new(), exported: false, - declares_slots: false, - slots_weak_references: false, + slots: None, decorators: Vec::new(), constants: Vec::new(), slot_aliases: Vec::new(), @@ -8047,14 +8295,15 @@ fn lower_function_with_receiver( dataclass: false, immutable: false, environment: true, + mutable: false, + unchecked_licences: false, }) .collect(); all.push(by_ir::function::ClassIr { resume: None, keywords: Vec::new(), exported: false, - declares_slots: false, - slots_weak_references: false, + slots: None, decorators: Vec::new(), constants: Vec::new(), slot_aliases: Vec::new(), @@ -8069,6 +8318,8 @@ fn lower_function_with_receiver( dataclass: false, immutable: false, environment: true, + mutable: false, + unchecked_licences: false, }); all.extend(inner_environments); all @@ -8162,8 +8413,8 @@ struct Unit<'a> { /// one is exactly that layout. it is what licenses a class pattern to read fields — /// see [`class_pattern_reads`] sealed: &'a HashSet, - /// the classes whose body declares `__slots__`, and whose instances therefore have - /// no dict to hold a value shadowing a method — see + /// the classes that declare `__slots__`, a `data class` among them, and whose + /// instances therefore have no dict to hold a value shadowing a method — see /// [`Lowering::keeps_instance_dict`] slotted: &'a HashSet, /// the classes whose body writes a `__new__`, which is what a construction of one @@ -15582,6 +15833,11 @@ impl Lowering<'_, '_> { Expr::Name(node) => { let name = node.id.as_str(); match self.place(name) { + None if name == "__class__" + && let Some(owner) = self.owner.clone() => + { + self.class_cell(owner) + } Some(Place::Global { .. }) | None => { // a name this frame does not have is a global, resolved the way // `LOAD_GLOBAL` resolves it — and a name it *declares* `global` @@ -17832,6 +18088,21 @@ impl Lowering<'_, '_> { self.narrow_call_result(dest, &Expr::Attribute(node.clone())) } + /// `__class__` read in a frame written inside the body of `owner`, which python + /// answers out of a cell holding the class the `def` is written in — the class the + /// `class` statement made, whatever the receiver's own class is and however the + /// function is bound. a nested function and a comprehension close over the same cell + fn class_cell(&mut self, owner: String) -> Lowered<(Value, RType)> { + if !self.layouts.contains_key(&owner) { + return Err(Decline::new(format!( + "`__class__` names `{owner}`, which this module does not emit a type for" + ))); + } + let dest = self.builder.temp(RType::OBJECT); + self.builder.push(Op::LoadClass { dest, class: owner }); + Ok((Value::Register(dest), RType::OBJECT)) + } + /// `super()`, as the `super(__class__, )` python's compiler writes /// /// the class is named by identity rather than through the namespace: a class diff --git a/crates/by_irbuild/src/tests.rs b/crates/by_irbuild/src/tests.rs index ec9f3bfc38..5c5da52c50 100644 --- a/crates/by_irbuild/src/tests.rs +++ b/crates/by_irbuild/src/tests.rs @@ -3067,6 +3067,109 @@ class Marked(metaclass=ABCMeta): ); } +/// each method of one emitted class, with whether each decorator it keeps only marks +/// what it was handed +fn method_marks(source: &str, class: &str) -> Vec<(String, Vec)> { + with_source(source, |db, env, model, suite| { + let module = + crate::build_module(db, env, model, suite, "app", crate::Language::BasedPython); + assert!(module.declined.is_empty(), "{:?}", module.declined); + module + .classes + .iter() + .find(|candidate| candidate.name == class) + .unwrap_or_else(|| panic!("{class} is emitted")) + .methods + .iter() + .map(|method| { + ( + method.name.clone(), + method + .decorators + .iter() + .map(by_ir::function::Decorator::marks_only) + .collect(), + ) + }) + .collect() + }) +} + +#[test] +fn a_decorator_only_marks_where_it_resolves_to_a_marking_one() { + // written with an `@` under its own name, under a spelling of its own, and as a + // language modifier, which the transpiler writes out as the `typing` one + assert_eq!( + method_marks( + "\ +import abc +import typing +import typing_extensions +from typing import final, override + + +class Base: + def area(self) -> int: + return 0 + + def name(self) -> str: + return \"\" + + def perimeter(self) -> int: + return 0 + + +class Square(Base): + @override + def area(self) -> int: + return 1 + + @typing.override + @final + def name(self) -> str: + return \"square\" + + @abc.abstractmethod + def edges(self) -> int: + return 4 + + override def perimeter(self) -> int: + return 4 + + @typing_extensions.final + def sides(self) -> int: + return 4 +", + "Square" + ), + vec![ + ("area".to_string(), vec![true]), + ("name".to_string(), vec![true, true]), + ("edges".to_string(), vec![true]), + ("perimeter".to_string(), vec![true]), + ("sides".to_string(), vec![true]), + ] + ); + // a module's own `override` is whatever it wrote, and a decorator that happens to + // return its argument is not known to by being read + assert_eq!( + method_marks( + "\ +def override(f): + return f + + +class Box: + @override + def size(self) -> int: + return 1 +", + "Box" + ), + vec![("size".to_string(), vec![false])] + ); +} + #[test] fn a_class_level_constant_beside_a_base_of_ours_keeps_that_base_emitted() { // the shape the stdlib is made of: no keyword at all, a base this module emits @@ -5449,6 +5552,7 @@ def f(n: int) -> int: [by_ir::function::Decorator::Path { root: "functools".to_string(), attributes: vec!["cache".to_string()], + marks_only: false, }] ); }); @@ -6513,9 +6617,10 @@ fn a_method_call_on_an_emitted_class_is_direct() { // and there is no other reason for one to appear with_source( "\ -data class Point: - x: int - y: int +class Point: + def __init__(self, x: int, y: int) -> None: + self.x = x + self.y = y def total(self) -> int: return self.x + self.y @@ -6589,6 +6694,39 @@ def use(c: Cell) -> int: ); } +#[test] +fn a_data_class_reaches_its_body_with_nothing_asked() { + // a `data class` is `@dataclass(slots=True)`, which declares its fields as its + // `__slots__` — so it has no dict for a shadowing value either + with_source( + "\ +data class Cell: + n: int + + def doubled(self) -> int: + return self.n * 2 + +def use(c: Cell) -> int: + return c.doubled() +", + |db, env, model, suite| { + let module = + crate::build_module(db, env, model, suite, "app", crate::Language::BasedPython); + assert!(module.declined.is_empty(), "{:?}", module.declined); + let function = &module.functions[0]; + let text = print_function(function); + assert!(text.contains("call Cell.doubled"), "{text}"); + assert!( + !has_op(function, |op| matches!( + op, + Op::DictShadows { .. } | Op::CallMethod { .. } + )), + "{text}" + ); + }, + ); +} + #[test] fn a_direct_method_call_keeps_the_unboxed_representations() { with_source( @@ -8643,6 +8781,44 @@ class Table: ); } +#[test] +fn a_dunder_the_class_body_binds_again_declines() { + // a dunder is settled from its `def`: a type slot is filled from it, and a name python + // looks up on the type is published beside it. an assignment to a dunder with a slot + // declines as a slot alias clashing with the `def`, which `__format__` has none of, + // and a `:=` makes no assignment at all + let reasons = declines( + "\ +def other(self) -> int: + return 2 + + +class Formatted: + def __format__(self, spec: str) -> str: + return \"one\" + + __format__ = other + + +class Walrused: + def __len__(self) -> int: + return 1 + + x = (__len__ := other) +", + ); + assert!( + reasons.iter().any(|(name, reason)| name == "Formatted" + && reason == "`__format__` is bound again by the class body, and a dunder is settled from its `def`"), + "{reasons:?}" + ); + assert!( + reasons.iter().any(|(name, reason)| name == "Walrused" + && reason == "`__len__` is bound by `:=` in the class body, and a dunder is settled before one runs"), + "{reasons:?}" + ); +} + #[test] fn a_conditional_dunder_declines() { let reasons = declines( @@ -8906,6 +9082,26 @@ data class B(A): assert!(ir.contains("callobj r1()"), "{ir}"); } +#[test] +fn a_class_cell_read_is_the_class_the_body_was_written_in() { + // `__class__` is the cell `super()` reads its pivot from, so it is the same `class B` + // rather than a global named `__class__`, which is what it used to be read as — and + // raised `NameError` at every call + let ir = method_ir( + &format!( + "{A_BASE} +data class B(A): + def owner(self) -> object: + return __class__ +" + ), + "B", + "owner", + ); + assert!(ir.contains("= class B"), "{ir}"); + assert!(!ir.contains("__class__"), "{ir}"); +} + #[test] fn a_rebound_super_is_an_ordinary_call() { // the zero-argument form is only sugar when `super` *is* the builtin. a module that diff --git a/crates/by_opt/src/field_proofs.rs b/crates/by_opt/src/field_proofs.rs index 6e058bdf37..d888b10461 100644 --- a/crates/by_opt/src/field_proofs.rs +++ b/crates/by_opt/src/field_proofs.rs @@ -221,14 +221,15 @@ mod tests { name: "State".to_string(), immutable: false, environment: false, + mutable: false, + unchecked_licences: false, exported: true, base: None, inherited_init: false, fields_are_parameters: false, dataclass: false, generic: false, - declares_slots: false, - slots_weak_references: false, + slots: None, constants: Vec::new(), properties: Vec::new(), slot_aliases: Vec::new(), diff --git a/crates/by_opt/src/fold.rs b/crates/by_opt/src/fold.rs index a0c9304111..c13e2c20c0 100644 --- a/crates/by_opt/src/fold.rs +++ b/crates/by_opt/src/fold.rs @@ -1021,6 +1021,8 @@ mod tests { name: "Vec2".to_string(), immutable: true, environment: false, + mutable: false, + unchecked_licences: false, resume: None, keywords: Vec::new(), exported: true, @@ -1041,8 +1043,7 @@ mod tests { properties: Vec::new(), slot_aliases: Vec::new(), generic: false, - declares_slots: false, - slots_weak_references: false, + slots: None, methods: Vec::new(), }); run(&mut m); @@ -1091,6 +1092,8 @@ mod tests { name: "Loose".to_string(), immutable: false, environment: false, + mutable: false, + unchecked_licences: false, resume: None, keywords: Vec::new(), exported: true, @@ -1111,8 +1114,7 @@ mod tests { properties: Vec::new(), slot_aliases: Vec::new(), generic: false, - declares_slots: false, - slots_weak_references: false, + slots: None, methods: Vec::new(), }); run(&mut m); @@ -1152,6 +1154,8 @@ mod tests { name: "Vec2".to_string(), immutable: true, environment: false, + mutable: false, + unchecked_licences: false, resume: None, keywords: Vec::new(), exported: true, @@ -1172,8 +1176,7 @@ mod tests { properties: Vec::new(), slot_aliases: Vec::new(), generic: false, - declares_slots: false, - slots_weak_references: false, + slots: None, methods: Vec::new(), }); run(&mut m); @@ -1215,6 +1218,8 @@ mod tests { name: "Vec2".to_string(), immutable: true, environment: false, + mutable: false, + unchecked_licences: false, resume: None, keywords: Vec::new(), exported: true, @@ -1235,8 +1240,7 @@ mod tests { properties: Vec::new(), slot_aliases: Vec::new(), generic: false, - declares_slots: false, - slots_weak_references: false, + slots: None, methods: Vec::new(), }); run(&mut m); diff --git a/crates/by_opt/src/lib.rs b/crates/by_opt/src/lib.rs index 400255655b..5ae3fd3295 100644 --- a/crates/by_opt/src/lib.rs +++ b/crates/by_opt/src/lib.rs @@ -386,14 +386,15 @@ mod tests { properties: Vec::new(), slot_aliases: Vec::new(), generic: false, - declares_slots: false, - slots_weak_references: false, + slots: None, base: None, inherited_init: false, fields_are_parameters: true, dataclass: false, immutable: false, environment: false, + mutable: false, + unchecked_licences: false, }); assert_eq!(optimize(&mut module), Ok(())); diff --git a/crates/by_opt/src/runs_python.rs b/crates/by_opt/src/runs_python.rs index 061799c84d..d238c2bc10 100644 --- a/crates/by_opt/src/runs_python.rs +++ b/crates/by_opt/src/runs_python.rs @@ -1085,14 +1085,15 @@ mod tests { name: name.to_string(), immutable: false, environment: false, + mutable: false, + unchecked_licences: false, exported: true, base: None, inherited_init: false, fields_are_parameters: false, dataclass: false, generic: false, - declares_slots: false, - slots_weak_references: false, + slots: None, constants: Vec::new(), properties: Vec::new(), slot_aliases: Vec::new(), diff --git a/crates/by_rt/include/by.h b/crates/by_rt/include/by.h index c80764d540..d8c4f174fa 100644 --- a/crates/by_rt/include/by.h +++ b/crates/by_rt/include/by.h @@ -50,6 +50,12 @@ #error "a basedpython extension needs python 3.11 or later" #endif +/* 3.13 made the fastcall-with-keywords function type public; below it the same type is + * only spelled with a leading underscore */ +#if PY_VERSION_HEX < 0x030D0000 +typedef _PyCFunctionFastWithKeywords PyCFunctionFastWithKeywords; +#endif + /* the major and minor at the front of a cpython version string * * `Py_GetVersion` answers the whole banner — `"3.14.0a1 (main, ...) [Clang ...]"` — and @@ -1772,6 +1778,14 @@ BY_COLD void By_FieldMissing(PyObject *o, const char *name, int qualified) { } } +/* raise the `AttributeError` a descriptor with no setter raises, for a field that + * cannot be written */ +BY_COLD int By_FieldNotWritable(PyObject *o, const char *name) { + PyErr_Format(PyExc_AttributeError, "attribute '%s' of '%s' objects is not writable", name, + Py_TYPE(o)->tp_name); + return -1; +} + /* python truthiness, which can raise from a user `__bool__` or `__len__` */ static inline char By_Truthy(PyObject *o) { int result = PyObject_IsTrue(o); @@ -6961,6 +6975,13 @@ static inline int By_CarryDataclassMembers(PyObject *twin, PyObject *type) { /* `@dataclass` writes the class's signature here where the body wrote no * docstring, and leaves the docstring alone where it did */ "__doc__", + /* the field names `slots=True` declares, which `copyreg` reads to learn what an + * instance with no dict holds — and so what `copy` and `pickle` hand over */ + "__slots__", + /* what `slots=True` adds to a frozen class, which pickles through them: python's + * own state protocol would set each field back through the `__setattr__` that + * refuses every write */ + "__getstate__", "__setstate__", NULL}; const char *const *name; PyObject *source, *target; @@ -7408,6 +7429,17 @@ static void By_RecordInstall(const char *module, const char *name, const char *v fclose(out); } +/* one census row for a published property, named `Class.name` so it can never be taken for + * a class's own row. nothing is formatted unless the census was asked for */ +static void By_RecordProperty(const char *module, const char *owner, const char *name, + const char *verdict) { + char qualified[512]; + const char *path = getenv(BY_INSTALL_CENSUS_ENV); + if (path == NULL || path[0] == '\0') return; + PyOS_snprintf(qualified, sizeof qualified, "%s.%s", owner, name); + By_RecordInstall(module, qualified, verdict); +} + /* whether `wanted` is among the type's own bases, by identity */ static int By_HasBase(PyObject *type, PyObject *wanted) { PyObject *chain = ((PyTypeObject *)type)->tp_bases; @@ -7567,6 +7599,51 @@ static int By_RecordClassBody(PyObject *state, PyObject *name, PyObject *body, P return PyDict_SetItem(statements, name, cls); } +/* the class a `class` statement bound, after every one of its decorators, recorded with + * what its dict held at that moment + * + * `pair` is `(state, decorator)`, where `decorator` is the outermost one the statement + * wrote. its answer is what the name is bound to, so the dict read here is exactly what the + * decorators left and nothing the module body did afterwards — see `By_BodyOutgrewClass` */ +static PyObject *By_RecordBound(PyObject *pair, PyObject *written) { + PyObject *state = PyTuple_GET_ITEM(pair, 0); + PyObject *decorator = PyTuple_GET_ITEM(pair, 1); + PyObject *bound = PyDict_GetItemString(state, "bound"); + PyObject *cls, *name, *snapshot, *record; + int failed; + if (bound == NULL) { + PyErr_SetString(PyExc_RuntimeError, "module body capture lost its record"); + return NULL; + } + cls = PyObject_CallOneArg(decorator, written); + if (cls == NULL || !PyType_Check(cls) || ((PyTypeObject *)cls)->tp_dict == NULL) return cls; + name = PyObject_GetAttrString(cls, "__name__"); + snapshot = name == NULL ? NULL : PyDict_Copy(((PyTypeObject *)cls)->tp_dict); + record = snapshot == NULL ? NULL : PyTuple_Pack(2, cls, snapshot); + /* a record that cannot be kept is raised out of the statement, as a body that cannot + * be is: init would otherwise leave the module interpreted without saying why */ + failed = record == NULL || PyDict_SetItem(bound, name, record) < 0; + Py_XDECREF(record); + Py_XDECREF(snapshot); + Py_XDECREF(name); + if (failed) { + Py_DECREF(cls); + return NULL; + } + return cls; +} + +/* the decorator `__build_class__(decorator)` hands back in place of `decorator`: the same + * call, recorded by `By_RecordBound` */ +static PyObject *By_BoundRecorder(PyObject *state, PyObject *decorator) { + static PyMethodDef record = {"__build_class__", (PyCFunction)By_RecordBound, METH_O, NULL}; + PyObject *pair = PyTuple_Pack(2, state, decorator), *recorder; + if (pair == NULL) return NULL; + recorder = PyCFunction_New(&record, pair); + Py_DECREF(pair); + return recorder; +} + /* `__build_class__`, recording what each module-level `class` statement wrote * * `state` holds `delegate` (the `__build_class__` this one displaced), `bodies` (the @@ -7591,6 +7668,14 @@ static PyObject *By_CaptureClassBody(PyObject *state, PyObject *args, PyObject * PyErr_SetString(PyExc_RuntimeError, "module body capture lost its `__build_class__`"); return NULL; } + /* `@__build_class__(decorator)` is what the twin writes around the outermost decorator + * of a class init checks against what its statement bound. python's own refuses a + * single argument, so the form means nothing else — and the frame asking is what says + * whether it was written in this module or reached this hook from another run's */ + if (PyTuple_GET_SIZE(args) == 1 && (kwds == NULL || PyDict_GET_SIZE(kwds) == 0) + && PyEval_GetGlobals() == held) { + return By_BoundRecorder(state, PyTuple_GET_ITEM(args, 0)); + } cls = PyObject_Call(delegate, args, kwds); if (cls == NULL || PyTuple_GET_SIZE(args) < 2 || !PyType_Check(cls)) return cls; if (((PyTypeObject *)cls)->tp_dict == NULL) return cls; @@ -7813,20 +7898,26 @@ static inline PyObject *By_ExecModuleBody(const By_Fallback *fallback, PyObject * hands back `{name: body}` for the classes the body wrote at module level, as a new * reference, or NULL with an exception set where the body raised. `*statements` is given * `{name: class}` alongside it, the class each of those statements built before any - * decorator was handed it, as a new reference or NULL */ + * decorator was handed it, and `*bound` is given `{name: (class, dict)}` for each statement + * the twin wrote `@__build_class__(...)` over — see `By_RecordBound` — each as a new + * reference or NULL */ static inline PyObject *By_RunModuleBody(const By_Fallback *fallback, PyObject *dict, - PyObject **statements) { + PyObject **statements, PyObject **bound) { static PyMethodDef capture = {"__build_class__", (PyCFunction)(void (*)(void))By_CaptureClassBody, METH_VARARGS | METH_KEYWORDS, NULL}; PyObject *bodies, *stood, *mapping, *displaced, *state, *wrapper, *result; int failed; *statements = NULL; + *bound = NULL; bodies = PyDict_New(); if (bodies == NULL) return NULL; *statements = PyDict_New(); - if (*statements == NULL) { + *bound = PyDict_New(); + if (*statements == NULL || *bound == NULL) { Py_DECREF(bodies); + Py_CLEAR(*statements); + Py_CLEAR(*bound); return NULL; } /* an emitted module's dict has no `__builtins__` of its own, and python would then @@ -7842,6 +7933,7 @@ static inline PyObject *By_RunModuleBody(const By_Fallback *fallback, PyObject * Py_XDECREF(stood); Py_DECREF(bodies); Py_CLEAR(*statements); + Py_CLEAR(*bound); if (!PyErr_Occurred()) { PyErr_SetString(PyExc_RuntimeError, "no builtins `__build_class__` to run the module body against"); @@ -7853,6 +7945,7 @@ static inline PyObject *By_RunModuleBody(const By_Fallback *fallback, PyObject * failed = state == NULL || PyDict_SetItemString(state, "delegate", displaced) < 0 || PyDict_SetItemString(state, "bodies", bodies) < 0 || PyDict_SetItemString(state, "statements", *statements) < 0 + || PyDict_SetItemString(state, "bound", *bound) < 0 || PyDict_SetItemString(state, "globals", dict) < 0; wrapper = failed ? NULL : PyCFunction_New(&capture, state); failed = failed || wrapper == NULL @@ -7879,6 +7972,7 @@ static inline PyObject *By_RunModuleBody(const By_Fallback *fallback, PyObject * if (failed || result == NULL) { Py_DECREF(bodies); Py_CLEAR(*statements); + Py_CLEAR(*bound); return NULL; } return bodies; @@ -7890,6 +7984,109 @@ static inline PyObject *By_ClassBody(PyObject *bodies, const char *name) { return PyDict_GetItemString(bodies, name); } +/* whether `a` and `b` hold different objects under `key`, absence included */ +static inline int By_HoldsApart(PyObject *a, PyObject *b, PyObject *key) { + return PyDict_GetItem(a, key) != PyDict_GetItem(b, key); +} + +/* whether reading the class wrote `key` into `now`, where the `class` statement left + * `body` without it + * + * 3.14 works a class's annotations out on demand, and two reads write the answer back: + * `__annotations__` keeps the mapping under `BY_ANNOTATIONS`, which is carried, and + * `__annotate__` — which the first also asks — keeps `None` under `__annotate_func__` + * where the class has no function computing them. a class with one holds it from the + * statement on, so anything else under that key, or its going, is a write the body made */ +static inline int By_ReadWrote(PyObject *now, PyObject *body, PyObject *key) { +#if PY_VERSION_HEX >= 0x030E0000 + return PyUnicode_CompareWithASCIIString(key, "__annotate_func__") == 0 + && PyDict_GetItem(body, key) == NULL && PyDict_GetItem(now, key) == Py_None; +#else + (void)now; + (void)body; + (void)key; + return 0; +#endif +} + +/* whether `key` is a dunder the class as it stands, `now`, and the dict its statement + * left, `body`, hold different objects under — the annotations aside, which are carried + * (see `By_CarryAnnotations`), and so is what a read writes (see `By_ReadWrote`) */ +static inline int By_DunderApart(PyObject *now, PyObject *body, PyObject *key) { + return PyUnicode_CheckExact(key) && By_IsDunder(key) + && PyUnicode_CompareWithASCIIString(key, BY_ANNOTATIONS) != 0 + && !By_ReadWrote(now, body, key) && By_HoldsApart(now, body, key); +} + +/* whether the module body changed a class after its `class` statement in a way the type + * replacing it cannot answer for, so that init has to leave the module as its interpreted + * definition built it. both kinds of change are read off the twin against the dict the + * statement left — see `By_CaptureClassBody`: + * + * - a dunder added, rewritten or deleted, on any class. `By_AdoptTwinAttributes` says why + * none is carried: a dunder is what a type slot answers, and the slot would go on + * answering the compiled body. the frontend declines a class the module body writes a + * dunder onto by name, `C.__len__ = f`, and this is the same refusal for the shapes no + * reading of the source can see — `setattr(C, name, f)`, or a helper the body calls. the + * annotations are carried, so they are left out + * - anything written over one of `members`, which names what compiled code reaches on the + * class with no lookup at all: a method called, a property half run, a field read at its + * offset. that is sound only because nothing can write to the class once import is over, + * and a write the body made before then is carried into the type's dict, where none of + * those accesses looks. NULL for a class whose every access is checked, which copes with + * a rewrite as it copes with one made after import + * + * a class whose name no longer holds what its statement built is left alone, as + * `By_CarryRewrittenMembers` leaves it. `rebuilt` says the twin is a `data class`, which + * `@dataclass(slots=True)` made again out of what the statement built and wrote every + * dunder it generates onto — so what that class is read against is not the body but what + * the statement *bound*, after its decorators, which `bound` recorded (see + * `By_RecordBound`). a dunder the module body wrote afterwards is then told apart from the + * decorator's own exactly, whatever the decorator wrote. a data class with no record says + * nothing about what its decorator left, and is answered the way that cannot be wrong. + * the keys are exact strings because nothing that can run python may be asked while a dict + * is being walked */ +static inline int By_BodyOutgrewClass(PyObject *bodies, PyObject *statements, PyObject *bound, + PyObject *dict, const char *name, + const char *const *members, int rebuilt) { + PyObject *body = By_ClassBody(bodies, name); + PyObject *statement = By_ClassBody(statements, name); + PyObject *twin = PyDict_GetItemString(dict, name); + PyObject *now, *key, *value, *member; + Py_ssize_t position = 0, at; + int apart; + if (rebuilt) { + PyObject *record = By_ClassBody(bound, name); + if (record == NULL || !PyTuple_Check(record) || PyTuple_GET_SIZE(record) != 2) return 1; + statement = PyTuple_GET_ITEM(record, 0); + body = PyTuple_GET_ITEM(record, 1); + } + if (body == NULL || twin == NULL || twin != statement) return 0; + if (!PyDict_Check(body) || !PyType_Check(twin)) return 0; + now = ((PyTypeObject *)twin)->tp_dict; + if (now == NULL) return 0; + /* both dicts, so that a dunder only one of them holds is seen whichever it is */ + while (PyDict_Next(now, &position, &key, &value)) { + if (By_DunderApart(now, body, key)) return 1; + } + position = 0; + while (PyDict_Next(body, &position, &key, &value)) { + if (By_DunderApart(now, body, key)) return 1; + } + for (at = 0; members != NULL && members[at] != NULL; at++) { + member = PyUnicode_FromString(members[at]); + if (member == NULL) { + /* a question that cannot be asked is answered the way that cannot be wrong */ + PyErr_Clear(); + return 1; + } + apart = By_HoldsApart(now, body, member); + Py_DECREF(member); + if (apart) return 1; + } + return 0; +} + /* the answer a class pattern gives when the attribute it named is simply absent * * a missing attribute is *no match*, not an error — `case Point(z=1):` against a @@ -9091,20 +9288,211 @@ static void By_ReleaseInstanceDict(PyObject **slot) { Py_DECREF(dict); } +/* `object.__getstate__` as python's own answers it for an instance holding slots, less the + * one refusal it makes about the layout + * + * python pickles such an instance as a pair: its dict, or `None` where that is empty, and + * the slots it has written, which `copyreg._slotnames` names. before that it checks that the + * instance is no bigger than `object`, a dict, a weak-reference list and one word a slot — + * and refuses the object otherwise, taking the rest for C state it cannot see. an emitted + * instance can be bigger without holding any: a base keeps the word a subclass keeps its + * dict in whether or not it has one itself, and a field that may be absent keeps a byte. + * see `By_PublishSlottedState` for where this stands in for python's */ +static PyObject *By_SlottedGetState(PyObject *self, PyObject *unused) { + PyTypeObject *type = Py_TYPE(self); + PyObject *state = Py_None, *copyreg, *names, *held, *pair; + Py_ssize_t index; + (void)unused; + if (type->tp_dictoffset > 0) { + PyObject *dict = *(PyObject **)((char *)self + type->tp_dictoffset); + if (dict != NULL && PyDict_Check(dict) && PyDict_GET_SIZE(dict) > 0) state = dict; + } + Py_INCREF(state); + copyreg = PyImport_ImportModule("copyreg"); + names = copyreg == NULL ? NULL + : PyObject_CallMethod(copyreg, "_slotnames", "O", (PyObject *)type); + Py_XDECREF(copyreg); + if (names == NULL || !PyList_Check(names)) { + if (names != NULL) PyErr_SetString(PyExc_TypeError, "copyreg._slotnames didn't return a list"); + Py_XDECREF(names); + Py_DECREF(state); + return NULL; + } + held = PyDict_New(); + if (held == NULL) goto failed; + for (index = 0; index < PyList_GET_SIZE(names); index++) { + PyObject *name = PyList_GET_ITEM(names, index); + PyObject *value = PyObject_GetAttr(self, name); + if (value == NULL) { + if (!PyErr_ExceptionMatches(PyExc_AttributeError)) goto failed; + PyErr_Clear(); + continue; + } + if (PyDict_SetItem(held, name, value) < 0) { + Py_DECREF(value); + goto failed; + } + Py_DECREF(value); + } + Py_DECREF(names); + if (PyDict_GET_SIZE(held) == 0) { + Py_DECREF(held); + return state; + } + pair = PyTuple_Pack(2, state, held); + Py_DECREF(state); + Py_DECREF(held); + return pair; +failed: + Py_DECREF(names); + Py_XDECREF(held); + Py_DECREF(state); + return NULL; +} + +static PyMethodDef By_SlottedGetStateDef = {"__getstate__", By_SlottedGetState, METH_NOARGS, + NULL}; + +/* how many names `copyreg._slotnames` would list for `type`, or -1 with an exception set + * + * counted the way it walks them — every class in the mro whose own dict holds `__slots__`, + * a string standing for one name, `__dict__` and `__weakref__` left out — rather than by + * calling it: it caches its answer in the class's dict and needs `copyreg` imported, and + * python does neither to a class until one of its instances is pickled */ +static inline Py_ssize_t By_SlotNameCount(PyTypeObject *type) { + PyObject *mro = type->tp_mro, *cached; + Py_ssize_t index, count = 0; + cached = PyDict_GetItemString(type->tp_dict, "__slotnames__"); + if (cached != NULL && PyList_Check(cached)) return PyList_GET_SIZE(cached); + if (mro == NULL || !PyTuple_Check(mro)) return 0; + for (index = 0; index < PyTuple_GET_SIZE(mro); index++) { + PyObject *base = PyTuple_GET_ITEM(mro, index), *slots, *iterator, *name; + if (!PyType_Check(base) || ((PyTypeObject *)base)->tp_dict == NULL) continue; + slots = PyDict_GetItemString(((PyTypeObject *)base)->tp_dict, "__slots__"); + if (slots == NULL) continue; + if (PyUnicode_Check(slots)) { + count++; + continue; + } + iterator = PyObject_GetIter(slots); + if (iterator == NULL) return -1; + while ((name = PyIter_Next(iterator)) != NULL) { + if (!PyUnicode_Check(name) + || (PyUnicode_CompareWithASCIIString(name, "__dict__") != 0 + && PyUnicode_CompareWithASCIIString(name, "__weakref__") != 0)) { + count++; + } + Py_DECREF(name); + } + Py_DECREF(iterator); + if (PyErr_Occurred()) return -1; + } + return count; +} + +/* whether `type` answers `name` with `object`'s own, or -1 with an exception set */ +static inline int By_AnswersAsObject(PyObject *type, const char *name) { + PyObject *standing = PyObject_GetAttrString(type, name); + PyObject *object_s = standing == NULL + ? NULL + : PyObject_GetAttrString((PyObject *)&PyBaseObject_Type, name); + int same = standing != NULL && standing == object_s; + Py_XDECREF(standing); + Py_XDECREF(object_s); + if (PyErr_Occurred()) return -1; + return same; +} + +/* give `type` [`By_SlottedGetState`] where python's own `__getstate__` would refuse it + * + * only there. python makes the check on one path alone — `object.__reduce_ex__` asking for + * the state of an object that is not a `list` or a `dict` and hands `__new__` nothing — so a + * class that reduces itself, a container, and one with `__getnewargs__` are never refused, + * and a direct call never is. where the layout is the size python expects, its own answer + * is already the right one, and a class answering `__getstate__` with anything but + * `object`'s — the source's own, or the one `@dataclass(frozen=True, slots=True)` writes — + * keeps what it has */ +static inline int By_PublishSlottedState(PyObject *type) { + PyTypeObject *owner = (PyTypeObject *)type; + static const char *const asked[] = {"__getstate__", "__reduce_ex__", "__reduce__", NULL}; + const char *const *name; + PyObject *method; + Py_ssize_t expected = PyBaseObject_Type.tp_basicsize, names; + if (PyType_IsSubtype(owner, &PyList_Type) || PyType_IsSubtype(owner, &PyDict_Type)) return 0; + for (name = asked; *name != NULL; name++) { + int same = By_AnswersAsObject(type, *name); + if (same <= 0) return same; + } + if (PyObject_HasAttrString(type, "__getnewargs_ex__") + || PyObject_HasAttrString(type, "__getnewargs__")) { + return 0; + } + names = By_SlotNameCount(owner); + if (names < 0) return -1; + expected += (Py_ssize_t)sizeof(PyObject *) * names; + if (owner->tp_dictoffset != 0) expected += (Py_ssize_t)sizeof(PyObject *); + if (owner->tp_weaklistoffset > 0) expected += (Py_ssize_t)sizeof(PyObject *); + if (owner->tp_basicsize <= expected) return 0; + method = PyDescr_NewMethod(owner, &By_SlottedGetStateDef); + if (method == NULL) return -1; + if (PyDict_SetItemString(owner->tp_dict, "__getstate__", method) < 0) { + Py_DECREF(method); + return -1; + } + Py_DECREF(method); + PyType_Modified(owner); + return 0; +} + /* `object.__getstate__`, over the whole of an instance's state * * python's own reads the dict word straight out of the instance, which on an emitted one * is the half holding the *extra* attributes — so `copy` and `pickle` are handed a state * naming none of the class's own fields, and handed it quietly. `None` is python's own - * answer where there is nothing in it */ -static inline PyObject *By_InstanceState(PyObject *owner, const By_DictField *fields) { + * answer where there is nothing in it. + * + * `slots` names what the class holds in a slot, as `copyreg._slotnames` lists it, and + * NULL where it holds nothing there. python hands those over beside the mapping, as a + * pair, wherever the instance has one written; one it has not is left out */ +static inline PyObject *By_InstanceState(PyObject *owner, const By_DictField *fields, + const char *const *slots) { PyObject *state = By_InstanceMapping(owner, fields); + PyObject *held, *pair; if (state == NULL) return NULL; if (PyDict_Size(state) == 0) { + Py_SETREF(state, By_NewRef(Py_None)); + } + if (slots == NULL) return state; + held = PyDict_New(); + if (held == NULL) { Py_DECREF(state); - Py_RETURN_NONE; + return NULL; + } + for (; *slots != NULL; slots++) { + PyObject *value = PyObject_GetAttrString(owner, *slots); + if (value == NULL) { + if (!PyErr_ExceptionMatches(PyExc_AttributeError)) goto failed; + PyErr_Clear(); + continue; + } + if (PyDict_SetItemString(held, *slots, value) < 0) { + Py_DECREF(value); + goto failed; + } + Py_DECREF(value); } - return state; + if (PyDict_Size(held) == 0) { + Py_DECREF(held); + return state; + } + pair = PyTuple_Pack(2, state, held); + Py_DECREF(state); + Py_DECREF(held); + return pair; +failed: + Py_DECREF(state); + Py_DECREF(held); + return NULL; } /* `obj.__dict__ = mapping`, which python takes as replacing the whole of an object's @@ -9377,17 +9765,23 @@ static PyObject *By_InitSubclass(PyObject *owner, allocfunc alloc, PyObject *con * * so a group on a class python's own metaclass machinery built keeps running interpreted. * that is the same answer it had before any of this, and it is reached without the class - * having to decline */ -static inline int By_PublishProperty(PyObject *type, PyObject *module_dict, const char *owner, - const char *name, PyMethodDef *get, PyMethodDef *set, - PyMethodDef *del) { + * having to decline. + * + * which of the two it was is only known here, at import: whether a base's metaclass is + * `type` is a question about what the base's name meant. so each property writes a census + * row of its own — `property-compiled` or `property-interpreted` under `Class.name`, see + * `By_RecordProperty` — and the report's count of compiled halves is held to that */ +static inline int By_PublishProperty(const char *module, PyObject *type, PyObject *module_dict, + const char *owner, const char *name, PyMethodDef *get, + PyMethodDef *set, PyMethodDef *del) { PyMethodDef *defs[3]; PyObject *halves[3]; PyObject *published, *named, *dict; int at, stored; - if (type == PyDict_GetItemString(module_dict, owner)) return 0; - if (PyType_Check(type) && ((PyTypeObject *)type)->tp_dict != NULL - && PyDict_GetItemString(((PyTypeObject *)type)->tp_dict, name) != NULL) { + if (type == PyDict_GetItemString(module_dict, owner) + || (PyType_Check(type) && ((PyTypeObject *)type)->tp_dict != NULL + && PyDict_GetItemString(((PyTypeObject *)type)->tp_dict, name) != NULL)) { + By_RecordProperty(module, owner, name, "property-interpreted"); return 0; } defs[0] = get; @@ -9424,6 +9818,7 @@ static inline int By_PublishProperty(PyObject *type, PyObject *module_dict, cons stored = PyDict_SetItemString(dict, name, published); Py_DECREF(published); PyType_Modified((PyTypeObject *)type); + if (stored == 0) By_RecordProperty(module, owner, name, "property-compiled"); return stored; } @@ -10711,6 +11106,61 @@ static inline int By_DecoratedMethod(PyObject *body, PyTypeObject *type, PyObjec return By_ApplyMethodDecorators(type, dict, owner, name, decorators, count); } +/* a method every decorator of which only marks what it was handed, published compiled + * + * `typing.override`, `typing.final` and `abc.abstractmethod` hand back the very function + * they were given, having written one attribute onto it. so the body's answer is the + * function its `def` made, and all it holds beyond what a fresh `def` holds is its + * `__dict__`. the compiled method takes a copy of that dict and stands under the name, + * which is what `By_DecoratedMethod` could not do: no decorator runs a second time, and + * the method that answers is the compiled one. + * + * the entry the type holds is the method table's own `def` where the type came from its + * spec, and the body's function where it was built by handing its metaclass a namespace — + * the body's function is carried into that namespace so a metaclass reads the marks the + * way it reads them off a `class` statement's, `abc.ABCMeta` collecting + * `__abstractmethods__` above all. either way what is published is `def`. + * + * a body holding something other than a function under the name did not get it from one + * of these, and its answer is taken as `By_DecoratedMethod` takes it. where there is no + * body answer at all the decorators never ran, and are applied: that is their only + * application */ +static inline int By_MarkedMethod(PyObject *body, PyTypeObject *type, PyObject *dict, + const char *owner, PyMethodDef *def, + const char *const *decorators, Py_ssize_t count, + const By_Twins *twins) { + PyObject *answer, *found, *descriptor, *method, *marks, *own; + int failed; + if ((PyObject *)type == PyDict_GetItemString(dict, owner)) return 0; + answer = body == NULL ? NULL : PyDict_GetItemString(body, def->ml_name); /* borrowed */ + if (answer == NULL) { + return By_ApplyMethodDecorators(type, dict, owner, def->ml_name, decorators, count); + } + if (!PyFunction_Check(answer)) { + return By_DecoratedMethod(body, type, dict, owner, def->ml_name, decorators, count, twins); + } + found = PyDict_GetItemString(type->tp_dict, def->ml_name); /* borrowed */ + descriptor = By_MethodDescriptorOf(found); + if (descriptor != NULL && ((PyMethodDescrObject *)descriptor)->d_method == def) { + method = By_Method(found); + } else { + descriptor = By_MethodDescriptor(type, def); + method = By_Method(descriptor); + Py_XDECREF(descriptor); + } + if (method == NULL) return -1; + marks = PyObject_GetAttrString(answer, "__dict__"); + own = marks == NULL ? NULL : PyObject_GenericGetDict(method, NULL); + failed = own == NULL || PyDict_Update(own, marks) < 0 + || PyDict_SetItemString(type->tp_dict, def->ml_name, method) < 0; + Py_XDECREF(own); + Py_XDECREF(marks); + Py_DECREF(method); + if (failed) return -1; + PyType_Modified(type); + return 0; +} + /* apply the decorator `decorator` names to `dict[name]`, in place. this is what * lets a decorated function still be compiled: the native one goes into the * namespace, then the decorator wraps it, exactly as the `def` statement would diff --git a/docs/basedpython/development/compilation/runtime.md b/docs/basedpython/development/compilation/runtime.md index ec89b53764..0ca975de0e 100644 --- a/docs/basedpython/development/compilation/runtime.md +++ b/docs/basedpython/development/compilation/runtime.md @@ -72,6 +72,13 @@ would be the *opposite* divergence: accepting what the interpreted twin refuses. python asks the whole chain rather than one class, so a `__slots__` over a base that declares none still has the dict that base gave the instance. +a `data class` declares its slots without writing them: its twin is +`@dataclass(slots=True)`, which names each field the class adds and asks for neither a +dict nor a weak-reference list. so the emitted class takes neither unless a base gives +one, publishes the same `__slots__`, and refuses `p.extra = 5` and `vars(p)` as python +does. a slot stays out of the dict a base gave the instance, as python keeps it, so +`vars()` does not name it and `copy` and `pickle` are handed it beside the dict. + the dict is a **managed** one, which python keeps in the pre-header — so the struct, its base's prefix and every offset a compiled function reads are untouched. what it costs is allocation: the pre-header and the two words a @@ -262,6 +269,38 @@ which is where a compiled class's speed lives. `type(C.g)` then answers `function`, which is also what python answers — so the change removed a second divergence rather than adding one +the exception is a decorator known to do nothing but mark what it is handed: +`typing.override`, `typing.final` and `abc.abstractmethod`, their +`typing_extensions` spellings, and the `override`, `final` and `abstract` +modifiers the transpiler writes out as them. each hands back the very function +it was given with one attribute written onto it, so the body's answer differs +from a fresh `def` only in its `__dict__`. the compiled method takes a copy of +that dict and is installed in its place, so an idiomatic override is compiled, +answers `__override__` as python's does, and is reached by the same direct +dispatch an undecorated method is. which decorator it is comes from what the +decorator expression resolves to, never from how it is spelled: a module's own +`def override(f)` is an arbitrary decorator like any other. on a class built by +calling its metaclass, the body's marked function is still what goes into the +namespace — so `ABCMeta` counts an `@abstractmethod` exactly as it does off a +`class` statement — and the compiled method replaces it on the finished type + +a method whose name the class body binds a second time takes the body's answer +too, with or without decorators. `m = other` below `def m`, `x = (m := other)` +anywhere in the body, and `m = other` *above* `def m` all leave python's class +holding what the last binding left — `other` for the first two, the `def`'s own +function for the third — and that object is copied across the way a class-level +constant is, so the class answers with it and a call through a typed receiver +looks the name up instead of calling the compiled `def`. `m = n`, naming a +sibling method, therefore publishes the compiled `n` under both names. a dunder +the body binds again declines instead, because its type slot is filled from the +`def` whatever the dict ends up holding + +the report counts such a method where it runs. its body is still lowered and +emitted, and it used to be counted as compiled with it; `--annotate`, the +decline count `by compile` prints and `require-native` now all list it as left +to the interpreted definition, along with the closures and generator frames only +it makes, since a frame that never runs builds none of them + ### boxed classes and interpreted fallbacks a construct with no native lowering is not a compile error. the module emits its @@ -658,6 +697,13 @@ environment variable, and a build that compares those rows against the report is holding the report to what ran. `scripts/native-sweeps/installcensus.sh` does exactly that over the corpus. +a property is recorded on its own, as `Class.name` with `property-compiled` or +`property-interpreted`. a class on a base from outside whose metaclass is not +`type` — `class X(abc.ABC)` — installs, but it is built by calling that metaclass, +which is handed the interpreted `property` and keeps it, so the halves the report +counts compiled never run. what the base's name meant is only known at import, +which is why the report cannot say so and the census does. + the check runs once per module at import, never per call, and `by compile --no-verify-install` leaves it out. @@ -1096,10 +1142,8 @@ def patch() -> None: Sealed.late = lambda self: 3 -patch() # python: None; compiled: TypeError: cannot set 'late' attribute of immutable type - - -# from another module +# from another module, once this one is imported +mod.patch() # python: None; compiled: TypeError: cannot set 'late' attribute of immutable type mod.Sealed.other = 5 # python: stored; compiled: TypeError del mod.Sealed.read # python: deleted; compiled: TypeError ``` @@ -1107,6 +1151,44 @@ del mod.Sealed.read # python: deleted; compiled: TypeError a class the module extends, or one the source decorates, is a mutable type and takes the write as python's does +what the module body writes *over* something the compiled class already answers is not +carried, though. a call, a property read or a field access on such a class goes straight +to the compiled body with no lookup — nothing can change what the class answers once init +is over — so a value the body wrote over a method, a property or a field, or a `del` of +one, would sit in the type's dict where none of those accesses looks. the compiled code +reading the class is fixed by the time the body has run, so the whole module is left as +its interpreted definition built it instead: + +```python +class Sealed: + def read(self) -> int: + return 1 + + +def call(s: Sealed) -> int: + return s.read() + + +Sealed.read = lambda self: 2 # the module runs interpreted: call(Sealed()) is 2 in both +``` + +a dunder is the same on every class, extended or not: it is what a type slot answers, and +the compiled type's slot is its compiled body. `C.__len__ = f` written in the module body +declines `C` where it is written, and `setattr(C, "__len__", f)`, or a helper the body +calls, is seen once the body has run and leaves the whole module interpreted. a write the +compiled class does answer for — a name the `class` statement never wrote, a class-level +constant, or a method of a class the module extends — is carried, and the module stays +compiled. `type(mod.Other.method).__name__` for a class nothing wrote to tells the two +apart: `function` where the module was left interpreted, `method_descriptor` where it was +not + +a `data class` is read the same way, against the class its statement bound rather than the +body the statement wrote: `@dataclass` makes the class again, writing every dunder it +generates and a slot for every field, so what counts as written afterwards is what differs +from the class the decorator handed back. `setattr(Point, "__repr__", f)` after a +`data class Point`, or a write over one of its fields, leaves the module interpreted, and +what the decorator wrote itself does not + ### an interpreted subclass whose `__init_subclass__` does not chain up a compiled class keeps an `int` field unboxed, and a field no `__init__` has set holds a @@ -1314,6 +1396,49 @@ alive() is not None # python: False; compiled: True, for as long as `kept` live a cell for each captured name would close this, and that is a different shape for every closure a compiled frame makes +### a frozen field written through `object.__setattr__` + +python's `frozen data class` refuses a write in `__setattr__` and leaves each field +writable underneath it, and compiled code reads a frozen field once and keeps the answer +across any call. so a field is written through `object.__setattr__` only while the +instance does not have it yet, which is how `copy` and `pickle` restore one onto an +instance `__new__` made. a second write raises where python stores it: + +```python +frozen data class Fixed: + n: int + + +f = Fixed(1) +object.__setattr__(f, "n", 2) # python: stored; compiled: AttributeError: attribute 'n' of 'mod.Fixed' objects is not writable +object.__delattr__(f, "n") # python: deleted; compiled: the same AttributeError +``` + +### the state of an instance that holds slots + +python pickles an instance holding slots through `object.__getstate__`, which first checks +that the instance is no bigger than a dict, a weak-reference list and one word a slot, and +refuses it otherwise. an emitted instance can be bigger without holding more: a base keeps +the words a subclass keeps its dict and weak references in whether or not it uses them, +and a field that may be absent keeps a byte. such a class — where it neither reduces itself +nor hands `__new__` arguments, which is the one path python checks — publishes a +`__getstate__` of its own that answers exactly what python's does, so `copy` and `pickle` +agree. what can tell the two apart is the class's own dict, and pickling protocols 0 and 1, +which python refuses for any class with `__slots__` and no `__getstate__` of its own: + +```python +data class Point: + x: int + + +class Loose(Point): + pass + + +"__getstate__" in Point.__dict__ # python: False; compiled: True +pickle.dumps(Point(1), protocol=1) # python: TypeError; compiled: pickled +``` + ### a nested function's annotations a compiled nested function answers `__annotations__`, `__type_params__` and, from 3.14, diff --git a/scripts/native-sweeps/installcensus.sh b/scripts/native-sweeps/installcensus.sh index 4cf2e88412..cb087ef9cb 100755 --- a/scripts/native-sweeps/installcensus.sh +++ b/scripts/native-sweeps/installcensus.sh @@ -18,6 +18,14 @@ # exist to leave a class interpreted where installing it would be wrong — but it is # exactly the population every coverage figure has been silently including # +# a property is asked the same question on its own. a class on a base from outside whose +# metaclass is not `type` — `class X(abc.ABC)` — installs, but is built by calling that +# metaclass, which is handed the interpreted `property` and keeps it: the halves the report +# counts compiled never run. only the import knows what the base's name meant, so each +# published property writes a row of its own, and `property-stood-down` names one whose +# compiled halves the report counts and the import did not publish. `halves:` in the +# summary is the report's compiled halves against the ones that ran +# # usage: installcensus.sh SP BY PY OUT [MODULE...] SP="$1"; BY="$2"; PY="$3"; OUT="$4"; shift 4 # shellcheck source=scripts/native-sweeps/sweeplib.sh @@ -49,6 +57,29 @@ for b in $(sweep_modules "$LIB" "$@"); do reported="$SWEEP_ROOT/reported" awk '/^## class / && $0 !~ /not published/ { print $3 }' "$report" | LC_ALL=C sort -u > "$reported" + # every property the report names on a published class, as `Class.name` and the number + # of its halves the report counts compiled. a half the report already lists as left to + # the interpreted definition — `- Class.name$get: ...` — is not one of them + props="$SWEEP_ROOT/props" + awk ' + /^## left to the interpreted definition/ { left = 1; next } + /^## / { left = 0 } + left && /^- / { name = $2; sub(/:$/, "", name); interpreted[name] = 1; next } + /^## class / { owner = ($0 ~ /not published/) ? "" : $3; next } + owner != "" && /^property / { + name = $2; sub(/:$/, "", name) + rest = $0; sub(/^property [^:]*: /, "", rest) + count = split(rest, halves, /, /) + for (at = 1; at <= count; at++) wanted[owner "." name] = wanted[owner "." name] " " halves[at] + } + END { + for (property in wanted) { + compiled = 0 + count = split(wanted[property], halves, " ") + for (at = 1; at <= count; at++) if (!((property "$" halves[at]) in interpreted)) compiled++ + if (compiled > 0) print property "\t" compiled + } + }' "$report" | LC_ALL=C sort -u > "$props" sweep_place "$d" census="$SWEEP_ROOT/census" @@ -71,7 +102,8 @@ for b in $(sweep_modules "$LIB" "$@"); do # handed the interpreted definition back to stand as the class — but it is kept apart # because it is the one the report has no idea about awk -F'\t' '$3 == "installed" { print $2 }' "$census" | LC_ALL=C sort -u > "$installed" - awk -F'\t' '$3 != "installed" { print $2 }' "$census" | LC_ALL=C sort -u > "$stood" + awk -F'\t' '$3 != "installed" && $3 !~ /^property-/ { print $2 }' "$census" | + LC_ALL=C sort -u > "$stood" while read -r name; do [ -n "$name" ] && printf '%s\ttwin\t%s\n' "$b" "$name" >> "$OUT" done < <(awk -F'\t' '$3 == "twin" { print $2 }' "$census" | LC_ALL=C sort -u) @@ -85,8 +117,23 @@ for b in $(sweep_modules "$LIB" "$@"); do while read -r name; do [ -n "$name" ] && printf '%s\tUNREPORTED\t%s\n' "$b" "$name" >> "$OUT" done < <(LC_ALL=C comm -13 "$reported" "$installed") + # a property the report counts compiled halves of, and the import published nothing + # compiled for — whether it kept the interpreted `property` or never reached it because + # its class stood down + ran="$SWEEP_ROOT/ran" + awk -F'\t' '$3 == "property-compiled" { print $2 }' "$census" | LC_ALL=C sort -u > "$ran" + while IFS=$'\t' read -r property count; do + [ -n "$property" ] || continue + if ! grep -qxF "$property" "$ran"; then + printf '%s\tproperty-stood-down\t%s\t%s\n' "$b" "$property" "$count" >> "$OUT" + fi + done < "$props" + printf '%s\thalves\t%s\t%s\n' "$b" \ + "$(awk -F'\t' '{ n += $2 } END { print n + 0 }' "$props")" \ + "$(awk -F'\t' 'NR == FNR { ran[$1] = 1; next } ($1 in ran) { n += $2 } END { print n + 0 }' "$ran" "$props")" \ + >> "$OUT" printf '%s\tcounts\t%s\t%s\t%s\n' "$b" \ "$(grep -c . "$reported")" "$(grep -c . "$installed")" "$(grep -c . "$stood")" >> "$OUT" done sweep_end || exit 1 -echo "walked: $(cat "$OUT.walked") reported: $(awk -F'\t' '$2=="counts"{n+=$3} END{print n+0}' "$OUT") installed: $(awk -F'\t' '$2=="counts"{n+=$4} END{print n+0}' "$OUT") stood-down: $(grep -c $'\tstood-down' "$OUT") of-those-twins: $(grep -c $'\ttwin\t' "$OUT") unreported: $(grep -c $'\tUNREPORTED' "$OUT") import-failed: $(grep -c $'\timport-failed' "$OUT") no-artifact: $(grep -c $'\tno-artifact' "$OUT") died: $(grep -c $'\tDIED' "$OUT") timed-out: $(grep -c $'\ttimed-out' "$OUT")" +echo "walked: $(cat "$OUT.walked") reported: $(awk -F'\t' '$2=="counts"{n+=$3} END{print n+0}' "$OUT") installed: $(awk -F'\t' '$2=="counts"{n+=$4} END{print n+0}' "$OUT") stood-down: $(grep -c $'\tstood-down' "$OUT") halves: $(awk -F'\t' '$2=="halves"{r+=$3; n+=$4} END{print n+0 " of " r+0 " ran"}' "$OUT") property-stood-down: $(grep -c $'\tproperty-stood-down' "$OUT") of-those-twins: $(grep -c $'\ttwin\t' "$OUT") unreported: $(grep -c $'\tUNREPORTED' "$OUT") import-failed: $(grep -c $'\timport-failed' "$OUT") no-artifact: $(grep -c $'\tno-artifact' "$OUT") died: $(grep -c $'\tDIED' "$OUT") timed-out: $(grep -c $'\ttimed-out' "$OUT")"