Implement experimental PEP 671 late-bound argument defaults

Author: yotatameguchi-boop

Doc/howto/index.rst

@@ -32,6 +32,7 @@ Python Library Reference.
    instrumentation.rst
    perf_profiling.rst
    annotations.rst
+   late-bound-defaults.rst
    isolating-extensions.rst
    timerfd.rst
    mro.rst

Doc/howto/late-bound-defaults.rst

@@ -0,0 +1,79 @@
+.. _late-bound-defaults-howto:
+
+Experimental late-bound argument defaults
+========================================
+
+.. warning::
+
+   This branch implements draft :pep:`671` for experimentation. The PEP has
+   not been accepted, and this syntax is not available in released Python.
+
+A default written with ``=`` is evaluated when a function is defined. A default
+written with ``=>`` is evaluated in the function's local scope when the argument
+is omitted::
+
+    def add_item(item, target=>[]):
+        target.append(item)
+        return target
+
+    assert add_item(1) == [1]
+    assert add_item(2) == [2]
+
+An explicitly supplied argument, including ``None`` or ``Ellipsis``, suppresses
+evaluation of its default. Late-bound defaults are evaluated from left to right
+and can refer to earlier arguments::
+
+    def window(items, start=0, stop=>len(items)):
+        return items[start:stop]
+
+Positional-only and keyword-only parameters, annotated parameters, methods,
+generic functions, and lambdas support the same syntax. All supplied arguments
+and early-bound defaults are bound before late-bound defaults are evaluated.
+An unbound reference raises :exc:`UnboundLocalError`::
+
+    def invalid(value=>value):
+        return value
+
+    # invalid() raises UnboundLocalError; invalid(3) returns 3.
+
+Coroutine and generator prologues run when execution starts, as does their
+function body. An argument error is reported before any default is evaluated.
+
+Introspection
+-------------
+
+``function.__defaults__`` and ``function.__kwdefaults__`` contain ``Ellipsis``
+placeholders for late-bound parameters. ``function.__defaults_extra__`` holds a
+parallel tuple of normalized expression strings (or ``None`` for early-bound
+entries), and ``function.__kwdefaults_extra__`` holds expression strings keyed
+by keyword-only parameter name. Both metadata attributes are ``None`` when no
+corresponding late-bound defaults exist. These attributes can be replaced or
+deleted using the same tuple/dict restrictions as the ordinary defaults.
+
+``inspect.signature()`` displays the ``=>`` expression. Its parameters expose
+the source string through ``Parameter.late_default``. ``getfullargspec()`` keeps
+its existing seven-field result; its defaults contain the same placeholders as
+the function attributes. Signature binding does not execute expressions.
+
+``ast.arg.late_default`` is an optional string containing the normalized source
+expression. The actual expression remains in ``arguments.defaults`` or
+``arguments.kw_defaults``. This preserves the existing defaults fields and
+allows :func:`ast.unparse` to reconstruct the new syntax.
+
+Implementation notes
+--------------------
+
+The call binder leaves omitted late-bound arguments unbound. Function prologues
+use ``QUERY_FAST`` and ``QUERY_DEREF`` to determine which expressions to evaluate.
+``QUERY_DEREF`` supports parameters captured by closures. Expression descriptions
+are captured before AST constant folding. Code objects using late-bound defaults
+carry the experimental ``CO_LATE_DEFAULTS`` flag.
+
+Existing call specializations are disabled for these functions because they
+assume all arguments are bound before entering a frame. The compiler also treats
+all parameters as potentially unbound during local-variable analysis. More
+precise analysis and specialized call paths are future performance work.
+
+Two metadata pointers are added to each function object. This prototype changes
+the internal function layout and bytecode magic number; extensions must be
+rebuilt for it. It does not add public C API entry points.

Grammar/python.gram

@@ -369,13 +369,17 @@ param_no_default_star_annotation[arg_ty]:
     | a=param_star_annotation ',' tc=TYPE_COMMENT? { _PyPegen_add_type_comment_to_arg(p, a, tc) }
     | a=param_star_annotation tc=TYPE_COMMENT? &')' { _PyPegen_add_type_comment_to_arg(p, a, tc) }
 param_with_default[NameDefaultPair*]:
+    | a=param '=' '>' c=expression ',' tc=TYPE_COMMENT? { _PyPegen_late_default_pair(p, a, c, tc) }
+    | a=param '=' '>' c=expression tc=TYPE_COMMENT? &')' { _PyPegen_late_default_pair(p, a, c, tc) }
     | a=param c=default ',' tc=TYPE_COMMENT? { _PyPegen_name_default_pair(p, a, c, tc) }
     | a=param c=default tc=TYPE_COMMENT? &')' { _PyPegen_name_default_pair(p, a, c, tc) }
 param_maybe_default[NameDefaultPair*]:
+    | a=param '=' '>' c=expression ',' tc=TYPE_COMMENT? { _PyPegen_late_default_pair(p, a, c, tc) }
+    | a=param '=' '>' c=expression tc=TYPE_COMMENT? &')' { _PyPegen_late_default_pair(p, a, c, tc) }
     | a=param c=default? ',' tc=TYPE_COMMENT? { _PyPegen_name_default_pair(p, a, c, tc) }
     | a=param c=default? tc=TYPE_COMMENT? &')' { _PyPegen_name_default_pair(p, a, c, tc) }
-param[arg_ty]: a=NAME b=annotation? { _PyAST_arg(a->v.Name.id, b, NULL, EXTRA) }
-param_star_annotation[arg_ty]: a=NAME b=star_annotation { _PyAST_arg(a->v.Name.id, b, NULL, EXTRA) }
+param[arg_ty]: a=NAME b=annotation? { _PyAST_arg(a->v.Name.id, b, NULL, NULL, EXTRA) }
+param_star_annotation[arg_ty]: a=NAME b=star_annotation { _PyAST_arg(a->v.Name.id, b, NULL, NULL, EXTRA) }
 annotation[expr_ty]: ':' a=expression { a }
 star_annotation[expr_ty]: ':' a=star_expression { a }
 default[expr_ty]: '=' a=expression { a } | invalid_default
@@ -941,12 +945,16 @@ lambda_param_no_default[arg_ty]:
     | a=lambda_param ',' { a }
     | a=lambda_param &':' { a }
 lambda_param_with_default[NameDefaultPair*]:
+    | a=lambda_param '=' '>' c=expression ',' { _PyPegen_late_default_pair(p, a, c, NULL) }
+    | a=lambda_param '=' '>' c=expression &':' { _PyPegen_late_default_pair(p, a, c, NULL) }
     | a=lambda_param c=default ',' { _PyPegen_name_default_pair(p, a, c, NULL) }
     | a=lambda_param c=default &':' { _PyPegen_name_default_pair(p, a, c, NULL) }
 lambda_param_maybe_default[NameDefaultPair*]:
+    | a=lambda_param '=' '>' c=expression ',' { _PyPegen_late_default_pair(p, a, c, NULL) }
+    | a=lambda_param '=' '>' c=expression &':' { _PyPegen_late_default_pair(p, a, c, NULL) }
     | a=lambda_param c=default? ',' { _PyPegen_name_default_pair(p, a, c, NULL) }
     | a=lambda_param c=default? &':' { _PyPegen_name_default_pair(p, a, c, NULL) }
-lambda_param[arg_ty]: a=NAME { _PyAST_arg(a->v.Name.id, NULL, NULL, EXTRA) }
+lambda_param[arg_ty]: a=NAME { _PyAST_arg(a->v.Name.id, NULL, NULL, NULL, EXTRA) }
 
 # LITERALS
 # ========

Include/cpython/code.h

@@ -131,6 +131,8 @@ struct PyCodeObject _PyCode_DEF(1);
 #define CO_COROUTINE            0x0080
 #define CO_ITERABLE_COROUTINE   0x0100
 #define CO_ASYNC_GENERATOR      0x0200
+/* Experimental PEP 671 call-time default evaluation. */
+#define CO_LATE_DEFAULTS        0x0400
 
 /* bpo-39562: These constant values are changed in Python 3.9
    to prevent collision with compiler flags. CO_FUTURE_ and PyCF_

Include/cpython/funcobject.h

@@ -36,6 +36,8 @@ typedef struct {
 typedef struct {
     PyObject_HEAD
     _Py_COMMON_FIELDS(func_)
+    PyObject *func_defaults_extra;  /* NULL or tuple of expression strings */
+    PyObject *func_kwdefaults_extra; /* NULL or dict of expression strings */
     PyObject *func_doc;         /* The __doc__ attribute, can be anything */
     PyObject *func_dict;        /* The __dict__ attribute, a dict or NULL */
     PyObject *func_weakreflist; /* List of weak references */

Include/internal/pycore_ast.h

@@ -43,13 +43,15 @@ struct _Py_global_strings {
         STRUCT_FOR_STR(dbl_open_br, "{{")
         STRUCT_FOR_STR(dbl_percent, "%%")
         STRUCT_FOR_STR(defaults, ".defaults")
+        STRUCT_FOR_STR(defaults_extra, ".defaults_extra")
         STRUCT_FOR_STR(dot_locals, ".<locals>")
         STRUCT_FOR_STR(empty, "")
         STRUCT_FOR_STR(format, ".format")
         STRUCT_FOR_STR(gc, "<GC>")
         STRUCT_FOR_STR(generic_base, ".generic_base")
         STRUCT_FOR_STR(json_decoder, "json.decoder")
         STRUCT_FOR_STR(kwdefaults, ".kwdefaults")
+        STRUCT_FOR_STR(kwdefaults_extra, ".kwdefaults_extra")
         STRUCT_FOR_STR(list_err, "list index out of range")
         STRUCT_FOR_STR(native, "<native>")
         STRUCT_FOR_STR(str_replace_inf, "1e309")

Include/internal/pycore_ast_state.h

@@ -313,7 +313,8 @@ Known values:
 
 */
 
-#define PYC_MAGIC_NUMBER 3706
+/* 3707: experimental PEP 671 query opcodes and function metadata. */
+#define PYC_MAGIC_NUMBER 3707
 /* This is equivalent to converting PYC_MAGIC_NUMBER to 2 bytes
    (little-endian) and then appending b'\r\n'. */
 #define PYC_MAGIC_NUMBER_TOKEN \