turbo: espressif boards loader enable native .mpy

main.c

@@ -249,6 +249,7 @@ static void stop_mp(void) {
     qstr_reset();
 
     gc_deinit();
+    port_gc_deinit();
     port_free(_heap);
     _heap = NULL;
 

ports/espressif/Makefile

@@ -872,7 +872,12 @@ ifneq ($(CIRCUITPY_BLEIO_NATIVE),0)
 	BLE_SDKCONFIG := ;esp-idf-config/sdkconfig-ble.defaults
 endif
 
-SDKCONFIGS := esp-idf-config/sdkconfig.defaults;$(DEBUG_SDKCONFIG);$(FLASH_SIZE_SDKCONFIG);$(FLASH_MODE_SDKCONFIG);$(FLASH_SPEED_SDKCONFIG);$(PSRAM_SDKCONFIG);$(PSRAM_SIZE_SDKCONFIG);$(PSRAM_MODE_SDKCONFIG);$(PSRAM_SPEED_SDKCONFIG);$(BLE_SDKCONFIG);$(TARGET_SDKCONFIG);boards/$(BOARD)/sdkconfig
+# Loaded native code executes from RAM, which memory protection forbids.
+ifneq (,$(filter 1,$(CIRCUITPY_ENABLE_MPY_NATIVE) $(CIRCUITPY_LOAD_NATIVE)))
+	NATIVE_SDKCONFIG := ;esp-idf-config/sdkconfig-native.defaults
+endif
+
+SDKCONFIGS := esp-idf-config/sdkconfig.defaults;$(DEBUG_SDKCONFIG);$(FLASH_SIZE_SDKCONFIG);$(FLASH_MODE_SDKCONFIG);$(FLASH_SPEED_SDKCONFIG);$(PSRAM_SDKCONFIG);$(PSRAM_SIZE_SDKCONFIG);$(PSRAM_MODE_SDKCONFIG);$(PSRAM_SPEED_SDKCONFIG);$(BLE_SDKCONFIG)$(NATIVE_SDKCONFIG);$(TARGET_SDKCONFIG);boards/$(BOARD)/sdkconfig
 
 # create the config headers
 .PHONY: do-sdkconfig
@@ -884,6 +889,8 @@ $(BUILD)/esp-idf/config/sdkconfig.h: boards/$(BOARD)/sdkconfig boards/$(BOARD)/m
 	$(Q)$(PYTHON) tools/check-sdkconfig.py \
 		CIRCUITPY_DUALBANK=$(CIRCUITPY_DUALBANK) \
 		CIRCUITPY_STORAGE_EXTEND=$(CIRCUITPY_STORAGE_EXTEND) \
+		CIRCUITPY_ENABLE_MPY_NATIVE=$(CIRCUITPY_ENABLE_MPY_NATIVE) \
+		CIRCUITPY_LOAD_NATIVE=$(CIRCUITPY_LOAD_NATIVE) \
 		$@
 
 # build a lib

ports/espressif/esp-idf-config/sdkconfig-native.defaults

@@ -0,0 +1,15 @@
+#
+# Espressif IoT Development Framework Configuration
+#
+#
+# Component config
+#
+#
+# ESP System Settings
+#
+# CONFIG_ESP_SYSTEM_MEMPROT is not set
+# end of ESP System Settings
+
+# end of Component config
+
+# end of Espressif IoT Development Framework Configuration

ports/espressif/mpconfigport.h

@@ -74,3 +74,9 @@ extern portMUX_TYPE background_task_mutex;
 #ifndef CIRCUITPY_ESP32P4_SWAP_LSFS
 #define CIRCUITPY_ESP32P4_SWAP_LSFS (0)
 #endif
+
+#if MICROPY_PERSISTENT_CODE_LOAD_NATIVE
+// Loaded native code is copied out of the GC heap into executable RAM.
+void *esp_native_code_commit(void *buf, size_t len, void *reloc);
+#define MP_PLAT_COMMIT_EXEC(buf, len, reloc) esp_native_code_commit(buf, len, reloc)
+#endif

ports/espressif/mpconfigport.mk

@@ -395,6 +395,9 @@ CIRCUITPY_AUDIOIO = 0
 endif
 #### end chip-specific choices ########################################
 
+# By default, load native .mpy files. Boards without the flash space turn it off.
+CIRCUITPY_LOAD_NATIVE ?= 1
+
 # By default, enable dualbank, and it'll be disabled for small flash sizes
 CIRCUITPY_DUALBANK ?= 1
 

ports/espressif/supervisor/port.c

@@ -15,6 +15,9 @@
 #include "supervisor/shared/serial.h"
 #include "py/mpprint.h"
 #include "py/runtime.h"
+#if MICROPY_PERSISTENT_CODE_LOAD_NATIVE
+#include "py/persistentcode.h"
+#endif
 
 #include "esp_mac.h"
 #include "freertos/FreeRTOS.h"
@@ -87,6 +90,10 @@
 #include "esp_rom_efuse.h"
 #include "esp_timer.h"
 
+#if MICROPY_PERSISTENT_CODE_LOAD_NATIVE && defined(CONFIG_IDF_TARGET_ESP32S2)
+#include "esp_memory_utils.h"
+#endif
+
 #ifdef CONFIG_IDF_TARGET_ESP32
 #include "hal/efuse_hal.h"
 #include "esp32/rom/efuse.h"
@@ -347,6 +354,56 @@ size_t port_heap_get_largest_free_size(void) {
     return free_size;
 }
 
+#if MICROPY_PERSISTENT_CODE_LOAD_NATIVE
+// Loaded native code, kept outside the GC heap until port_gc_deinit().
+typedef struct _native_code_node_t {
+    struct _native_code_node_t *next;
+    uint32_t data[];
+} native_code_node_t;
+
+static native_code_node_t *native_code_head = NULL;
+
+void port_gc_deinit(void) {
+    while (native_code_head != NULL) {
+        native_code_node_t *next = native_code_head->next;
+        heap_caps_free(native_code_head);
+        native_code_head = next;
+    }
+}
+
+// Copy `len` bytes of machine code from `buf` into executable memory and return
+// the executable address, applying the relocations in `reloc` (if any) against
+// that address first. Raises MemoryError when no executable memory is available.
+void *esp_native_code_commit(void *buf, size_t len, void *reloc) {
+    len = (len + 3) & ~3;
+    size_t len_node = sizeof(native_code_node_t) + len;
+    native_code_node_t *node = heap_caps_malloc(len_node, MALLOC_CAP_EXEC);
+    #if defined(CONFIG_IDF_TARGET_ESP32S2)
+    // The S2 can hand out MALLOC_CAP_EXEC memory that the CPU cannot fetch from.
+    if (node != NULL && !esp_ptr_executable(node)) {
+        heap_caps_free(node);
+        node = NULL;
+    }
+    #endif
+    if (node == NULL) {
+        m_malloc_fail(len_node);
+    }
+    node->next = native_code_head;
+    native_code_head = node;
+    void *p = node->data;
+    if (reloc) {
+        mp_native_relocate(reloc, buf, (uintptr_t)p);
+    }
+    // Word copy: Xtensa executable RAM is not byte-addressable.
+    const uint32_t *src = buf;
+    uint32_t *dst = p;
+    for (size_t i = 0; i < len / 4; i++) {
+        dst[i] = src[i];
+    }
+    return p;
+}
+#endif
+
 void reset_port_early(void) {
     // esp-camera adds an I2C device on the ESP I2C bus, and keeps it there. This
     // is unlike busio.I2C, which adds and removes the device on each operation.

ports/espressif/tools/check-sdkconfig.py

@@ -34,6 +34,17 @@ def validate(sdk_config, circuitpy_config):
                         f"{var} is incompatible with {partition_table=} (no ota_1 partition)"
                     )
 
+    # Native machine code executes from RAM; the PMS/PMP memory protection
+    # forbids that and makes MALLOC_CAP_EXEC allocations fail.
+    if (
+        circuitpy_config.get("CIRCUITPY_ENABLE_MPY_NATIVE")
+        or circuitpy_config.get("CIRCUITPY_LOAD_NATIVE")
+    ) and sdk_config.get("CONFIG_ESP_SYSTEM_MEMPROT"):
+        raise SystemExit(
+            "CIRCUITPY_ENABLE_MPY_NATIVE=1 / CIRCUITPY_LOAD_NATIVE=1 require CONFIG_ESP_SYSTEM_MEMPROT=n "
+            "(see esp-idf-config/sdkconfig-native.defaults)"
+        )
+
     # Add more checks here for other things we want to verify.
     return
 

ports/espressif/tools/update_sdkconfig.py

@@ -164,6 +164,8 @@ def update(debug, board, update_all):  # noqa: C901 too complex
     psram_size = "0"
     uf2_bootloader = None
     ble_enabled = None
+    load_native = True  # Matches the mpconfigport.mk default.
+    mpy_native = False
     for line in board_make.read_text().split("\n"):
         if "=" not in line or line.startswith("#"):
             continue
@@ -192,6 +194,10 @@ def update(debug, board, update_all):  # noqa: C901 too complex
             uf2_bootloader = not (value == "0")
         elif key == "CIRCUITPY_BLEIO_NATIVE":
             ble_enabled = not (value == "0")
+        elif key == "CIRCUITPY_LOAD_NATIVE":
+            load_native = not (value == "0")
+        elif key == "CIRCUITPY_ENABLE_MPY_NATIVE":
+            mpy_native = value == "1"
 
     os.environ["IDF_TARGET"] = target
     os.environ["COMPONENT_KCONFIGS_PROJBUILD_SOURCE_FILE"] = (
@@ -239,6 +245,9 @@ def update(debug, board, update_all):  # noqa: C901 too complex
     if ble_enabled:
         ble_config = pathlib.Path("esp-idf-config/sdkconfig-ble.defaults")
         sdkconfigs.append(ble_config)
+    if load_native or mpy_native:
+        native_config = pathlib.Path("esp-idf-config/sdkconfig-native.defaults")
+        sdkconfigs.append(native_config)
     board_config = pathlib.Path(f"boards/{board}/sdkconfig")
     # Don't include the board file in cp defaults. The board may have custom
     # overrides.

py/mpconfig.h

@@ -556,7 +556,12 @@ typedef uint64_t mp_uint_t;
 // Some architectures cannot read byte-wise from executable memory.  In this case
 // the prelude for a native function (which usually sits after the machine code)
 // must be separated and placed somewhere where it can be read byte-wise.
+// CIRCUITPY-CHANGE: also when only loading native code on windowed Xtensa.
+#if defined(__XTENSA_WINDOWED_ABI__)
+#define MICROPY_EMIT_NATIVE_PRELUDE_SEPARATE_FROM_MACHINE_CODE (MICROPY_EMIT_XTENSAWIN || MICROPY_PERSISTENT_CODE_LOAD_NATIVE)
+#else
 #define MICROPY_EMIT_NATIVE_PRELUDE_SEPARATE_FROM_MACHINE_CODE (MICROPY_EMIT_XTENSAWIN)
+#endif
 
 // Convenience definition for whether any inline assembler emitter is enabled
 #define MICROPY_EMIT_INLINE_ASM (MICROPY_EMIT_INLINE_THUMB || MICROPY_EMIT_INLINE_XTENSA || MICROPY_EMIT_INLINE_RV32)

supervisor/port.h

@@ -115,6 +115,11 @@ void port_boot_info(void);
 // A default weak implementation is provided that does nothing.
 void port_gc_collect(void);
 
+// Called once the VM heap is torn down and its finalisers have run. Ports free
+// memory here that heap objects may point into.
+// A default weak implementation is provided that does nothing.
+void port_gc_deinit(void);
+
 // Most ports that implement CIRCUITPY_BOOT_BUTTON use a generic version of
 // this function to sense the button. Ports that need to can override this
 // function to provide their own implementation.

supervisor/shared/port.c

@@ -119,6 +119,10 @@ MP_WEAK bool port_boot_button_pressed(void) {
 MP_WEAK void port_gc_collect(void) {
 }
 
+// Ports may provide an implementation of this function if it is needed
+MP_WEAK void port_gc_deinit(void) {
+}
+
 // Allocates an object in the port heap, not the VM heap, and also sets type, for mp_obj_malloc{,_var} macros.
 MP_NOINLINE void *mp_obj_port_malloc_helper(size_t num_bytes, const mp_obj_type_t *type) {
     mp_obj_base_t *base = (mp_obj_base_t *)port_malloc_zero(num_bytes, false);